@bryan-thompson/inspector-assessment-client 1.25.4 → 1.25.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-DE62cdTZ.js → OAuthCallback-Dl4GYls3.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-CWjFdCIE.js → OAuthDebugCallback-BdJ38Z-r.js} +1 -1
- package/dist/assets/{index-Df9Sx1jt.css → index-cHhcEXbr.css} +4 -0
- package/dist/assets/{index-PCQVSwHa.js → index-pfUiTdQb.js} +4 -4
- package/dist/index.html +2 -2
- package/lib/lib/assessment/coreTypes.d.ts +65 -0
- package/lib/lib/assessment/coreTypes.d.ts.map +1 -1
- package/lib/lib/assessment/extendedTypes.d.ts +127 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +45 -0
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +3 -7
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +13 -2
- package/lib/services/assessment/TestDataGenerator.d.ts +9 -1
- package/lib/services/assessment/TestDataGenerator.d.ts.map +1 -1
- package/lib/services/assessment/TestDataGenerator.js +32 -6
- package/lib/services/assessment/TestScenarioEngine.d.ts +9 -1
- package/lib/services/assessment/TestScenarioEngine.d.ts.map +1 -1
- package/lib/services/assessment/TestScenarioEngine.js +17 -14
- package/lib/services/assessment/config/annotationPatterns.d.ts +3 -1
- package/lib/services/assessment/config/annotationPatterns.d.ts.map +1 -1
- package/lib/services/assessment/config/annotationPatterns.js +5 -2
- package/lib/services/assessment/config/architecturePatterns.d.ts +101 -0
- package/lib/services/assessment/config/architecturePatterns.d.ts.map +1 -0
- package/lib/services/assessment/config/architecturePatterns.js +248 -0
- package/lib/services/assessment/config/performanceConfig.d.ts +122 -0
- package/lib/services/assessment/config/performanceConfig.d.ts.map +1 -0
- package/lib/services/assessment/config/performanceConfig.js +154 -0
- package/lib/services/assessment/config/sanitizationPatterns.d.ts +63 -0
- package/lib/services/assessment/config/sanitizationPatterns.d.ts.map +1 -0
- package/lib/services/assessment/config/sanitizationPatterns.js +223 -0
- package/lib/services/assessment/lib/claudeCodeBridge.d.ts +3 -1
- package/lib/services/assessment/lib/claudeCodeBridge.d.ts.map +1 -1
- package/lib/services/assessment/lib/claudeCodeBridge.js +5 -3
- package/lib/services/assessment/lib/concurrencyLimit.d.ts +6 -2
- package/lib/services/assessment/lib/concurrencyLimit.d.ts.map +1 -1
- package/lib/services/assessment/lib/concurrencyLimit.js +13 -6
- package/lib/services/assessment/lib/errors.d.ts +90 -0
- package/lib/services/assessment/lib/errors.d.ts.map +1 -0
- package/lib/services/assessment/lib/errors.js +136 -0
- package/lib/services/assessment/lib/timeoutUtils.d.ts +69 -0
- package/lib/services/assessment/lib/timeoutUtils.d.ts.map +1 -0
- package/lib/services/assessment/lib/timeoutUtils.js +103 -0
- package/lib/services/assessment/modules/BaseAssessor.d.ts +43 -8
- package/lib/services/assessment/modules/BaseAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/BaseAssessor.js +103 -34
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts +38 -1
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.js +185 -19
- package/lib/services/assessment/modules/DocumentationAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/DocumentationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/DocumentationAssessor.js +11 -0
- package/lib/services/assessment/modules/ErrorHandlingAssessor.js +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.js +6 -3
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts +3 -0
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.js +14 -2
- package/lib/services/assessment/modules/ManifestValidationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ManifestValidationAssessor.js +7 -2
- package/lib/services/assessment/modules/PromptAssessor.d.ts +1 -0
- package/lib/services/assessment/modules/PromptAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/PromptAssessor.js +26 -16
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.js +6 -2
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.js +15 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +8 -2
- package/lib/services/assessment/modules/SecurityAssessor.d.ts +3 -171
- package/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/SecurityAssessor.js +25 -1480
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +27 -28
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +340 -863
- package/lib/services/assessment/modules/UsabilityAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/UsabilityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/UsabilityAssessor.js +11 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.d.ts +57 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.js +176 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.d.ts +67 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.js +239 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.d.ts +46 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.js +394 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.d.ts +64 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.js +304 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts +43 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.js +276 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.d.ts +122 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.js +388 -0
- package/lib/services/assessment/modules/annotations/index.d.ts +13 -0
- package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/index.js +15 -0
- package/lib/services/assessment/modules/index.d.ts +10 -0
- package/lib/services/assessment/modules/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/index.js +13 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.d.ts +125 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.js +345 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts +33 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.js +128 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts +67 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +372 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts +178 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +1207 -0
- package/lib/services/assessment/modules/securityTests/index.d.ts +8 -0
- package/lib/services/assessment/modules/securityTests/index.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/index.js +7 -0
- package/lib/services/assessment/tool-classifier-patterns.d.ts +1 -0
- package/lib/services/assessment/tool-classifier-patterns.d.ts.map +1 -1
- package/lib/services/assessment/tool-classifier-patterns.js +17 -0
- package/package.json +1 -1
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
* Simple concurrency limiter for parallel async operations
|
|
3
3
|
* Provides the same interface as p-limit but is CJS-compatible
|
|
4
4
|
*/
|
|
5
|
+
import { DEFAULT_PERFORMANCE_CONFIG } from "../config/performanceConfig.js";
|
|
5
6
|
/**
|
|
6
7
|
* Warning threshold for queue depth monitoring.
|
|
7
8
|
* If queue exceeds this size, a warning is emitted to help diagnose
|
|
@@ -12,13 +13,16 @@
|
|
|
12
13
|
*
|
|
13
14
|
* Threshold of 10,000 provides ~146% headroom to accommodate larger
|
|
14
15
|
* tool sets while catching true runaway scenarios.
|
|
16
|
+
*
|
|
17
|
+
* @see PerformanceConfig.queueWarningThreshold (Issue #37)
|
|
15
18
|
*/
|
|
16
|
-
export const QUEUE_WARNING_THRESHOLD =
|
|
19
|
+
export const QUEUE_WARNING_THRESHOLD = DEFAULT_PERFORMANCE_CONFIG.queueWarningThreshold;
|
|
17
20
|
/**
|
|
18
21
|
* Creates a concurrency limiter that allows only a specified number
|
|
19
22
|
* of async operations to run simultaneously
|
|
20
23
|
*
|
|
21
24
|
* @param concurrency - Maximum number of concurrent operations
|
|
25
|
+
* @param logger - Optional logger instance for queue depth warnings
|
|
22
26
|
* @returns A function that wraps async operations with the concurrency limit
|
|
23
27
|
*
|
|
24
28
|
* @example
|
|
@@ -27,7 +31,7 @@ export const QUEUE_WARNING_THRESHOLD = 10000;
|
|
|
27
31
|
* items.map(item => limit(() => processItem(item)))
|
|
28
32
|
* );
|
|
29
33
|
*/
|
|
30
|
-
export function createConcurrencyLimit(concurrency) {
|
|
34
|
+
export function createConcurrencyLimit(concurrency, logger) {
|
|
31
35
|
if (concurrency < 1) {
|
|
32
36
|
throw new Error("Concurrency must be at least 1");
|
|
33
37
|
}
|
|
@@ -62,10 +66,13 @@ export function createConcurrencyLimit(concurrency) {
|
|
|
62
66
|
// Only warn once per limiter instance to avoid log spam
|
|
63
67
|
if (queue.length > QUEUE_WARNING_THRESHOLD && !hasWarned) {
|
|
64
68
|
hasWarned = true;
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
+
logger?.warn("Queue depth exceeded threshold", {
|
|
70
|
+
queueDepth: queue.length,
|
|
71
|
+
threshold: QUEUE_WARNING_THRESHOLD,
|
|
72
|
+
activeCount,
|
|
73
|
+
concurrency,
|
|
74
|
+
message: "This may indicate a slow/stalled server",
|
|
75
|
+
});
|
|
69
76
|
}
|
|
70
77
|
next();
|
|
71
78
|
});
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessment Error Types
|
|
3
|
+
*
|
|
4
|
+
* Provides standardized error handling across all assessment modules.
|
|
5
|
+
* See docs/ERROR_HANDLING_CONVENTIONS.md for usage guidelines.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Error categories for classification and debugging
|
|
9
|
+
*/
|
|
10
|
+
export declare enum ErrorCategory {
|
|
11
|
+
/** Network connectivity issues (ECONNREFUSED, DNS failures, etc.) */
|
|
12
|
+
CONNECTION = "CONNECTION",
|
|
13
|
+
/** MCP protocol violations or unexpected responses */
|
|
14
|
+
PROTOCOL = "PROTOCOL",
|
|
15
|
+
/** Input validation failures (invalid parameters, missing fields) */
|
|
16
|
+
VALIDATION = "VALIDATION",
|
|
17
|
+
/** Operation exceeded time limit */
|
|
18
|
+
TIMEOUT = "TIMEOUT",
|
|
19
|
+
/** JSON or data parsing failures */
|
|
20
|
+
PARSE = "PARSE",
|
|
21
|
+
/** Unclassified errors */
|
|
22
|
+
UNKNOWN = "UNKNOWN"
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Custom error class for assessment operations
|
|
26
|
+
*
|
|
27
|
+
* @example
|
|
28
|
+
* throw new AssessmentError(
|
|
29
|
+
* 'Failed to connect to MCP server',
|
|
30
|
+
* ErrorCategory.CONNECTION,
|
|
31
|
+
* false, // not recoverable
|
|
32
|
+
* { url: 'http://localhost:3000', attempt: 3 }
|
|
33
|
+
* );
|
|
34
|
+
*/
|
|
35
|
+
export declare class AssessmentError extends Error {
|
|
36
|
+
readonly code: ErrorCategory;
|
|
37
|
+
readonly recoverable: boolean;
|
|
38
|
+
readonly context?: Record<string, unknown>;
|
|
39
|
+
constructor(message: string, code: ErrorCategory, recoverable?: boolean, context?: Record<string, unknown>);
|
|
40
|
+
/**
|
|
41
|
+
* Create a structured object for serialization
|
|
42
|
+
*/
|
|
43
|
+
toJSON(): ErrorInfo;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Structured error information for result objects
|
|
47
|
+
*/
|
|
48
|
+
export interface ErrorInfo {
|
|
49
|
+
/** Human-readable error message */
|
|
50
|
+
message: string;
|
|
51
|
+
/** Error category for classification */
|
|
52
|
+
code: ErrorCategory;
|
|
53
|
+
/** Whether the operation can be retried */
|
|
54
|
+
recoverable: boolean;
|
|
55
|
+
/** Stack trace (optional, for debugging) */
|
|
56
|
+
stack?: string;
|
|
57
|
+
/** Additional context about the error */
|
|
58
|
+
context?: Record<string, unknown>;
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Interface for result objects that may contain errors
|
|
62
|
+
*
|
|
63
|
+
* @example
|
|
64
|
+
* interface ToolTestResult extends ErrorResult {
|
|
65
|
+
* toolName: string;
|
|
66
|
+
* passed: boolean;
|
|
67
|
+
* }
|
|
68
|
+
*/
|
|
69
|
+
export interface ErrorResult {
|
|
70
|
+
error?: ErrorInfo;
|
|
71
|
+
}
|
|
72
|
+
/**
|
|
73
|
+
* Type guard to check if a value is an AssessmentError
|
|
74
|
+
*/
|
|
75
|
+
export declare function isAssessmentError(error: unknown): error is AssessmentError;
|
|
76
|
+
/**
|
|
77
|
+
* Categorize an error based on its message content
|
|
78
|
+
*
|
|
79
|
+
* @param error - The error to categorize
|
|
80
|
+
* @returns The appropriate ErrorCategory
|
|
81
|
+
*/
|
|
82
|
+
export declare function categorizeError(error: unknown): ErrorCategory;
|
|
83
|
+
/**
|
|
84
|
+
* Extract error message from various error types
|
|
85
|
+
*
|
|
86
|
+
* @param error - The error to extract message from
|
|
87
|
+
* @returns A string error message
|
|
88
|
+
*/
|
|
89
|
+
export declare function extractErrorMessage(error: unknown): string;
|
|
90
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/lib/errors.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH;;GAEG;AACH,oBAAY,aAAa;IACvB,qEAAqE;IACrE,UAAU,eAAe;IACzB,sDAAsD;IACtD,QAAQ,aAAa;IACrB,qEAAqE;IACrE,UAAU,eAAe;IACzB,oCAAoC;IACpC,OAAO,YAAY;IACnB,oCAAoC;IACpC,KAAK,UAAU;IACf,0BAA0B;IAC1B,OAAO,YAAY;CACpB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,eAAgB,SAAQ,KAAK;aAGtB,IAAI,EAAE,aAAa;aACnB,WAAW,EAAE,OAAO;aACpB,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;gBAHjD,OAAO,EAAE,MAAM,EACC,IAAI,EAAE,aAAa,EACnB,WAAW,GAAE,OAAc,EAC3B,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC;IAUnD;;OAEG;IACH,MAAM,IAAI,SAAS;CASpB;AAED;;GAEG;AACH,MAAM,WAAW,SAAS;IACxB,mCAAmC;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,wCAAwC;IACxC,IAAI,EAAE,aAAa,CAAC;IACpB,2CAA2C;IAC3C,WAAW,EAAE,OAAO,CAAC;IACrB,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACnC;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW;IAC1B,KAAK,CAAC,EAAE,SAAS,CAAC;CACnB;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,eAAe,CAE1E;AAED;;;;;GAKG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,aAAa,CAsC7D;AAED;;;;;GAKG;AACH,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAwB1D"}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Assessment Error Types
|
|
3
|
+
*
|
|
4
|
+
* Provides standardized error handling across all assessment modules.
|
|
5
|
+
* See docs/ERROR_HANDLING_CONVENTIONS.md for usage guidelines.
|
|
6
|
+
*/
|
|
7
|
+
/**
|
|
8
|
+
* Error categories for classification and debugging
|
|
9
|
+
*/
|
|
10
|
+
export var ErrorCategory;
|
|
11
|
+
(function (ErrorCategory) {
|
|
12
|
+
/** Network connectivity issues (ECONNREFUSED, DNS failures, etc.) */
|
|
13
|
+
ErrorCategory["CONNECTION"] = "CONNECTION";
|
|
14
|
+
/** MCP protocol violations or unexpected responses */
|
|
15
|
+
ErrorCategory["PROTOCOL"] = "PROTOCOL";
|
|
16
|
+
/** Input validation failures (invalid parameters, missing fields) */
|
|
17
|
+
ErrorCategory["VALIDATION"] = "VALIDATION";
|
|
18
|
+
/** Operation exceeded time limit */
|
|
19
|
+
ErrorCategory["TIMEOUT"] = "TIMEOUT";
|
|
20
|
+
/** JSON or data parsing failures */
|
|
21
|
+
ErrorCategory["PARSE"] = "PARSE";
|
|
22
|
+
/** Unclassified errors */
|
|
23
|
+
ErrorCategory["UNKNOWN"] = "UNKNOWN";
|
|
24
|
+
})(ErrorCategory || (ErrorCategory = {}));
|
|
25
|
+
/**
|
|
26
|
+
* Custom error class for assessment operations
|
|
27
|
+
*
|
|
28
|
+
* @example
|
|
29
|
+
* throw new AssessmentError(
|
|
30
|
+
* 'Failed to connect to MCP server',
|
|
31
|
+
* ErrorCategory.CONNECTION,
|
|
32
|
+
* false, // not recoverable
|
|
33
|
+
* { url: 'http://localhost:3000', attempt: 3 }
|
|
34
|
+
* );
|
|
35
|
+
*/
|
|
36
|
+
export class AssessmentError extends Error {
|
|
37
|
+
code;
|
|
38
|
+
recoverable;
|
|
39
|
+
context;
|
|
40
|
+
constructor(message, code, recoverable = true, context) {
|
|
41
|
+
super(message);
|
|
42
|
+
this.code = code;
|
|
43
|
+
this.recoverable = recoverable;
|
|
44
|
+
this.context = context;
|
|
45
|
+
this.name = "AssessmentError";
|
|
46
|
+
// Maintains proper stack trace in V8 environments
|
|
47
|
+
if (Error.captureStackTrace) {
|
|
48
|
+
Error.captureStackTrace(this, AssessmentError);
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Create a structured object for serialization
|
|
53
|
+
*/
|
|
54
|
+
toJSON() {
|
|
55
|
+
return {
|
|
56
|
+
message: this.message,
|
|
57
|
+
code: this.code,
|
|
58
|
+
recoverable: this.recoverable,
|
|
59
|
+
stack: this.stack,
|
|
60
|
+
context: this.context,
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Type guard to check if a value is an AssessmentError
|
|
66
|
+
*/
|
|
67
|
+
export function isAssessmentError(error) {
|
|
68
|
+
return error instanceof AssessmentError;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Categorize an error based on its message content
|
|
72
|
+
*
|
|
73
|
+
* @param error - The error to categorize
|
|
74
|
+
* @returns The appropriate ErrorCategory
|
|
75
|
+
*/
|
|
76
|
+
export function categorizeError(error) {
|
|
77
|
+
const message = extractErrorMessage(error).toLowerCase();
|
|
78
|
+
if (message.includes("timeout") || message.includes("timed out")) {
|
|
79
|
+
return ErrorCategory.TIMEOUT;
|
|
80
|
+
}
|
|
81
|
+
if (message.includes("connection") ||
|
|
82
|
+
message.includes("econnrefused") ||
|
|
83
|
+
message.includes("enotfound") ||
|
|
84
|
+
message.includes("network")) {
|
|
85
|
+
return ErrorCategory.CONNECTION;
|
|
86
|
+
}
|
|
87
|
+
if (message.includes("parse") ||
|
|
88
|
+
message.includes("json") ||
|
|
89
|
+
message.includes("syntax")) {
|
|
90
|
+
return ErrorCategory.PARSE;
|
|
91
|
+
}
|
|
92
|
+
if (message.includes("protocol") ||
|
|
93
|
+
message.includes("mcp") ||
|
|
94
|
+
message.includes("invalid response")) {
|
|
95
|
+
return ErrorCategory.PROTOCOL;
|
|
96
|
+
}
|
|
97
|
+
if (message.includes("invalid") ||
|
|
98
|
+
message.includes("required") ||
|
|
99
|
+
message.includes("missing") ||
|
|
100
|
+
message.includes("validation")) {
|
|
101
|
+
return ErrorCategory.VALIDATION;
|
|
102
|
+
}
|
|
103
|
+
return ErrorCategory.UNKNOWN;
|
|
104
|
+
}
|
|
105
|
+
/**
|
|
106
|
+
* Extract error message from various error types
|
|
107
|
+
*
|
|
108
|
+
* @param error - The error to extract message from
|
|
109
|
+
* @returns A string error message
|
|
110
|
+
*/
|
|
111
|
+
export function extractErrorMessage(error) {
|
|
112
|
+
if (typeof error === "string") {
|
|
113
|
+
return error;
|
|
114
|
+
}
|
|
115
|
+
if (error instanceof Error) {
|
|
116
|
+
return error.message;
|
|
117
|
+
}
|
|
118
|
+
if (error && typeof error === "object") {
|
|
119
|
+
const err = error;
|
|
120
|
+
if (typeof err.message === "string") {
|
|
121
|
+
return err.message;
|
|
122
|
+
}
|
|
123
|
+
if (typeof err.error === "string") {
|
|
124
|
+
return err.error;
|
|
125
|
+
}
|
|
126
|
+
if (err.error && typeof err.error === "object") {
|
|
127
|
+
return extractErrorMessage(err.error);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
try {
|
|
131
|
+
return JSON.stringify(error);
|
|
132
|
+
}
|
|
133
|
+
catch {
|
|
134
|
+
return String(error);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Timeout Utilities for Assessment Operations
|
|
3
|
+
*
|
|
4
|
+
* Provides AbortController-based timeout handling with proper cleanup.
|
|
5
|
+
* Fixes GitHub issue #38: Promise.race patterns that leave timers hanging.
|
|
6
|
+
*
|
|
7
|
+
* Key improvements over basic Promise.race:
|
|
8
|
+
* 1. Clears setTimeout when operation completes (prevents timer leaks)
|
|
9
|
+
* 2. Uses AbortController for signal-based cancellation
|
|
10
|
+
* 3. Provides consistent error messages
|
|
11
|
+
*/
|
|
12
|
+
export interface TimeoutOptions {
|
|
13
|
+
/** Timeout in milliseconds */
|
|
14
|
+
timeoutMs: number;
|
|
15
|
+
/** Optional custom error message for timeout */
|
|
16
|
+
errorMessage?: string;
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Execute a promise with timeout and proper cleanup.
|
|
20
|
+
*
|
|
21
|
+
* This function wraps any promise with a timeout, ensuring the timer
|
|
22
|
+
* is always cleaned up whether the operation succeeds, fails, or times out.
|
|
23
|
+
*
|
|
24
|
+
* @param promise - The promise to execute
|
|
25
|
+
* @param options - Timeout configuration
|
|
26
|
+
* @returns The result of the promise
|
|
27
|
+
* @throws Error if operation times out
|
|
28
|
+
*
|
|
29
|
+
* @example Basic usage
|
|
30
|
+
* ```typescript
|
|
31
|
+
* const result = await executeWithTimeout(
|
|
32
|
+
* callTool(tool.name, params),
|
|
33
|
+
* { timeoutMs: 5000 }
|
|
34
|
+
* );
|
|
35
|
+
* ```
|
|
36
|
+
*
|
|
37
|
+
* @example With custom error message
|
|
38
|
+
* ```typescript
|
|
39
|
+
* const result = await executeWithTimeout(
|
|
40
|
+
* fetchData(),
|
|
41
|
+
* { timeoutMs: 10000, errorMessage: "Data fetch timed out" }
|
|
42
|
+
* );
|
|
43
|
+
* ```
|
|
44
|
+
*/
|
|
45
|
+
export declare function executeWithTimeout<T>(promise: Promise<T>, options: TimeoutOptions): Promise<T>;
|
|
46
|
+
/**
|
|
47
|
+
* Execute an async function with timeout and AbortSignal support.
|
|
48
|
+
*
|
|
49
|
+
* This variant accepts a function that receives an AbortSignal, enabling
|
|
50
|
+
* cancellation of fetch requests and other abortable operations.
|
|
51
|
+
*
|
|
52
|
+
* @param fn - Function that accepts an AbortSignal and returns a Promise
|
|
53
|
+
* @param options - Timeout configuration
|
|
54
|
+
* @returns The result of the function
|
|
55
|
+
* @throws Error if operation times out
|
|
56
|
+
*
|
|
57
|
+
* @example With fetch
|
|
58
|
+
* ```typescript
|
|
59
|
+
* const data = await executeWithTimeoutAndSignal(
|
|
60
|
+
* async (signal) => {
|
|
61
|
+
* const response = await fetch(url, { signal });
|
|
62
|
+
* return response.json();
|
|
63
|
+
* },
|
|
64
|
+
* { timeoutMs: 5000 }
|
|
65
|
+
* );
|
|
66
|
+
* ```
|
|
67
|
+
*/
|
|
68
|
+
export declare function executeWithTimeoutAndSignal<T>(fn: (signal: AbortSignal) => Promise<T>, options: TimeoutOptions): Promise<T>;
|
|
69
|
+
//# sourceMappingURL=timeoutUtils.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"timeoutUtils.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/lib/timeoutUtils.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,cAAc;IAC7B,8BAA8B;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,gDAAgD;IAChD,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,wBAAsB,kBAAkB,CAAC,CAAC,EACxC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EACnB,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,CAAC,CAAC,CA2BZ;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,wBAAsB,2BAA2B,CAAC,CAAC,EACjD,EAAE,EAAE,CAAC,MAAM,EAAE,WAAW,KAAK,OAAO,CAAC,CAAC,CAAC,EACvC,OAAO,EAAE,cAAc,GACtB,OAAO,CAAC,CAAC,CAAC,CAqBZ"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Timeout Utilities for Assessment Operations
|
|
3
|
+
*
|
|
4
|
+
* Provides AbortController-based timeout handling with proper cleanup.
|
|
5
|
+
* Fixes GitHub issue #38: Promise.race patterns that leave timers hanging.
|
|
6
|
+
*
|
|
7
|
+
* Key improvements over basic Promise.race:
|
|
8
|
+
* 1. Clears setTimeout when operation completes (prevents timer leaks)
|
|
9
|
+
* 2. Uses AbortController for signal-based cancellation
|
|
10
|
+
* 3. Provides consistent error messages
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Execute a promise with timeout and proper cleanup.
|
|
14
|
+
*
|
|
15
|
+
* This function wraps any promise with a timeout, ensuring the timer
|
|
16
|
+
* is always cleaned up whether the operation succeeds, fails, or times out.
|
|
17
|
+
*
|
|
18
|
+
* @param promise - The promise to execute
|
|
19
|
+
* @param options - Timeout configuration
|
|
20
|
+
* @returns The result of the promise
|
|
21
|
+
* @throws Error if operation times out
|
|
22
|
+
*
|
|
23
|
+
* @example Basic usage
|
|
24
|
+
* ```typescript
|
|
25
|
+
* const result = await executeWithTimeout(
|
|
26
|
+
* callTool(tool.name, params),
|
|
27
|
+
* { timeoutMs: 5000 }
|
|
28
|
+
* );
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* @example With custom error message
|
|
32
|
+
* ```typescript
|
|
33
|
+
* const result = await executeWithTimeout(
|
|
34
|
+
* fetchData(),
|
|
35
|
+
* { timeoutMs: 10000, errorMessage: "Data fetch timed out" }
|
|
36
|
+
* );
|
|
37
|
+
* ```
|
|
38
|
+
*/
|
|
39
|
+
export async function executeWithTimeout(promise, options) {
|
|
40
|
+
const { timeoutMs, errorMessage } = options;
|
|
41
|
+
// Create abort controller for timeout signaling
|
|
42
|
+
const controller = new AbortController();
|
|
43
|
+
// Set up timeout that triggers abort
|
|
44
|
+
const timeoutId = setTimeout(() => {
|
|
45
|
+
controller.abort();
|
|
46
|
+
}, timeoutMs);
|
|
47
|
+
// Create timeout promise that rejects when abort signal fires
|
|
48
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
49
|
+
controller.signal.addEventListener("abort", () => {
|
|
50
|
+
reject(new Error(errorMessage ?? `Operation timed out after ${timeoutMs}ms`));
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
try {
|
|
54
|
+
// Race the operation against timeout
|
|
55
|
+
return await Promise.race([promise, timeoutPromise]);
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
// CRITICAL: Always clean up timer to prevent leaks
|
|
59
|
+
clearTimeout(timeoutId);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Execute an async function with timeout and AbortSignal support.
|
|
64
|
+
*
|
|
65
|
+
* This variant accepts a function that receives an AbortSignal, enabling
|
|
66
|
+
* cancellation of fetch requests and other abortable operations.
|
|
67
|
+
*
|
|
68
|
+
* @param fn - Function that accepts an AbortSignal and returns a Promise
|
|
69
|
+
* @param options - Timeout configuration
|
|
70
|
+
* @returns The result of the function
|
|
71
|
+
* @throws Error if operation times out
|
|
72
|
+
*
|
|
73
|
+
* @example With fetch
|
|
74
|
+
* ```typescript
|
|
75
|
+
* const data = await executeWithTimeoutAndSignal(
|
|
76
|
+
* async (signal) => {
|
|
77
|
+
* const response = await fetch(url, { signal });
|
|
78
|
+
* return response.json();
|
|
79
|
+
* },
|
|
80
|
+
* { timeoutMs: 5000 }
|
|
81
|
+
* );
|
|
82
|
+
* ```
|
|
83
|
+
*/
|
|
84
|
+
export async function executeWithTimeoutAndSignal(fn, options) {
|
|
85
|
+
const { timeoutMs, errorMessage } = options;
|
|
86
|
+
const controller = new AbortController();
|
|
87
|
+
const timeoutId = setTimeout(() => {
|
|
88
|
+
controller.abort();
|
|
89
|
+
}, timeoutMs);
|
|
90
|
+
try {
|
|
91
|
+
return await fn(controller.signal);
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
// Convert AbortError to timeout error for consistency
|
|
95
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
96
|
+
throw new Error(errorMessage ?? `Operation timed out after ${timeoutMs}ms`);
|
|
97
|
+
}
|
|
98
|
+
throw error;
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
clearTimeout(timeoutId);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -5,10 +5,12 @@
|
|
|
5
5
|
import { AssessmentConfiguration, AssessmentStatus } from "../../../lib/assessmentTypes.js";
|
|
6
6
|
import { AssessmentContext } from "../AssessmentOrchestrator.js";
|
|
7
7
|
import { Logger } from "../lib/logger.js";
|
|
8
|
+
import { ErrorResult } from "../lib/errors.js";
|
|
8
9
|
export declare abstract class BaseAssessor<T = unknown> {
|
|
9
10
|
protected config: AssessmentConfiguration;
|
|
10
11
|
protected logger: Logger;
|
|
11
12
|
protected testCount: number;
|
|
13
|
+
private deprecationWarningsEmitted;
|
|
12
14
|
constructor(config: AssessmentConfiguration);
|
|
13
15
|
/**
|
|
14
16
|
* Abstract method that each assessor must implement
|
|
@@ -20,14 +22,39 @@ export declare abstract class BaseAssessor<T = unknown> {
|
|
|
20
22
|
protected determineStatus(passed: number, total: number, threshold?: number): AssessmentStatus;
|
|
21
23
|
/**
|
|
22
24
|
* Log assessment progress
|
|
23
|
-
* @deprecated Use this.logger.info() directly for structured logging with context
|
|
25
|
+
* @deprecated Use this.logger.info() directly for structured logging with context. Will be removed in v2.0.0.
|
|
24
26
|
*/
|
|
25
27
|
protected log(message: string): void;
|
|
26
28
|
/**
|
|
27
|
-
* Log error
|
|
28
|
-
* @deprecated Use this.logger.error() directly for structured logging with context
|
|
29
|
+
* Log error with optional context
|
|
30
|
+
* @deprecated Use this.logger.error() directly for structured logging with context. Will be removed in v2.0.0.
|
|
31
|
+
*
|
|
32
|
+
* @param message - Description of what operation failed
|
|
33
|
+
* @param error - The error that occurred (optional)
|
|
29
34
|
*/
|
|
30
35
|
protected logError(message: string, error?: unknown): void;
|
|
36
|
+
/**
|
|
37
|
+
* Handle an error with logging and structured result
|
|
38
|
+
*
|
|
39
|
+
* Use this method in catch blocks to ensure consistent error handling:
|
|
40
|
+
* 1. Logs the error with context
|
|
41
|
+
* 2. Categorizes the error automatically
|
|
42
|
+
* 3. Returns a structured result with error info
|
|
43
|
+
*
|
|
44
|
+
* @param error - The caught error
|
|
45
|
+
* @param context - Description of what operation failed
|
|
46
|
+
* @param defaults - Default values to merge into result
|
|
47
|
+
* @returns A result object with error information
|
|
48
|
+
*
|
|
49
|
+
* @example
|
|
50
|
+
* try {
|
|
51
|
+
* const result = await this.callTool(tool);
|
|
52
|
+
* return { passed: true, result };
|
|
53
|
+
* } catch (error) {
|
|
54
|
+
* return this.handleError(error, `Failed to call tool ${tool.name}`, { passed: false });
|
|
55
|
+
* }
|
|
56
|
+
*/
|
|
57
|
+
protected handleError<T extends ErrorResult>(error: unknown, context: string, defaults?: Partial<T>): T;
|
|
31
58
|
/**
|
|
32
59
|
* Get test count for this assessor
|
|
33
60
|
*/
|
|
@@ -45,17 +72,25 @@ export declare abstract class BaseAssessor<T = unknown> {
|
|
|
45
72
|
*/
|
|
46
73
|
protected sleep(ms: number): Promise<void>;
|
|
47
74
|
/**
|
|
48
|
-
* Execute with timeout
|
|
75
|
+
* Execute with timeout and proper cleanup.
|
|
76
|
+
*
|
|
77
|
+
* Uses AbortController-based timeout handling that clears the timer
|
|
78
|
+
* when the operation completes, preventing timer leaks.
|
|
79
|
+
*
|
|
80
|
+
* @param promise - The promise to execute
|
|
81
|
+
* @param timeoutMs - Timeout in milliseconds (defaults to config.testTimeout)
|
|
82
|
+
* @returns The result of the promise
|
|
83
|
+
* @throws Error if operation times out
|
|
49
84
|
*/
|
|
50
85
|
protected executeWithTimeout<T>(promise: Promise<T>, timeoutMs?: number): Promise<T>;
|
|
51
86
|
/**
|
|
52
87
|
* Safe JSON parse with error handling
|
|
53
88
|
*/
|
|
54
|
-
protected safeJsonParse(text: string):
|
|
89
|
+
protected safeJsonParse(text: string): unknown;
|
|
55
90
|
/**
|
|
56
91
|
* Extract error message from various error types
|
|
57
92
|
*/
|
|
58
|
-
protected extractErrorMessage(error:
|
|
93
|
+
protected extractErrorMessage(error: unknown): string;
|
|
59
94
|
/**
|
|
60
95
|
* Check if a response indicates an error
|
|
61
96
|
* Handles various MCP response formats
|
|
@@ -63,11 +98,11 @@ export declare abstract class BaseAssessor<T = unknown> {
|
|
|
63
98
|
* @param response - The response to check
|
|
64
99
|
* @param strictMode - If true, only check explicit error indicators (default: false)
|
|
65
100
|
*/
|
|
66
|
-
protected isErrorResponse(response:
|
|
101
|
+
protected isErrorResponse(response: unknown, strictMode?: boolean): boolean;
|
|
67
102
|
/**
|
|
68
103
|
* Extract error information from a response
|
|
69
104
|
*/
|
|
70
|
-
protected extractErrorInfo(response:
|
|
105
|
+
protected extractErrorInfo(response: unknown): {
|
|
71
106
|
code?: string | number;
|
|
72
107
|
message?: string;
|
|
73
108
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BaseAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/BaseAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EACvB,gBAAgB,
|
|
1
|
+
{"version":3,"file":"BaseAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/BaseAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EACvB,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAwC,MAAM,eAAe,CAAC;AAC7E,OAAO,EAGL,WAAW,EAEZ,MAAM,eAAe,CAAC;AAGvB,8BAAsB,YAAY,CAAC,CAAC,GAAG,OAAO;IAC5C,SAAS,CAAC,MAAM,EAAE,uBAAuB,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,SAAS,EAAE,MAAM,CAAK;IAGhC,OAAO,CAAC,0BAA0B,CAGhC;gBAEU,MAAM,EAAE,uBAAuB;IAS3C;;OAEG;IACH,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,CAAC,CAAC;IAEvD;;OAEG;IACH,SAAS,CAAC,eAAe,CACvB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,SAAS,GAAE,MAAY,GACtB,gBAAgB;IAUnB;;;OAGG;IACH,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAWpC;;;;;;OAMG;IACH,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,GAAG,IAAI;IAW1D;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,SAAS,CAAC,WAAW,CAAC,CAAC,SAAS,WAAW,EACzC,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,MAAM,EACf,QAAQ,GAAE,OAAO,CAAC,CAAC,CAAM,GACxB,CAAC;IAsBJ;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACH,cAAc,IAAI,IAAI;IAItB;;OAEG;IACH,SAAS,CAAC,gBAAgB,CACxB,OAAO,EAAE,MAAM,uBAAuB,CAAC,sBAAsB,CAAC,GAC7D,OAAO;IAIV;;OAEG;cACa,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhD;;;;;;;;;;OAUG;cACa,kBAAkB,CAAC,CAAC,EAClC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EACnB,SAAS,GAAE,MAAgC,GAC1C,OAAO,CAAC,CAAC,CAAC;IAOb;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAS9C;;OAEG;IACH,SAAS,CAAC,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM;IAgBrD;;;;;;OAMG;IACH,SAAS,CAAC,eAAe,CACvB,QAAQ,EAAE,OAAO,EACjB,UAAU,GAAE,OAAe,GAC1B,OAAO;IA+CV;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,GAAG;QAC7C,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB;CA2BF"}
|