@bryan-thompson/inspector-assessment-client 1.25.4 → 1.25.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-DE62cdTZ.js → OAuthCallback-D6y8tFfF.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-CWjFdCIE.js → OAuthDebugCallback-DHegnqTa.js} +1 -1
- package/dist/assets/{index-PCQVSwHa.js → index-Cu02Ah3g.js} +4 -4
- package/dist/assets/{index-Df9Sx1jt.css → index-cHhcEXbr.css} +4 -0
- package/dist/index.html +2 -2
- package/lib/lib/assessment/coreTypes.d.ts +65 -0
- package/lib/lib/assessment/coreTypes.d.ts.map +1 -1
- package/lib/lib/assessment/extendedTypes.d.ts +127 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +45 -0
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/lib/moduleScoring.d.ts +2 -2
- package/lib/lib/moduleScoring.d.ts.map +1 -1
- package/lib/lib/moduleScoring.js +3 -2
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +3 -7
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +13 -2
- package/lib/services/assessment/TestDataGenerator.d.ts +9 -1
- package/lib/services/assessment/TestDataGenerator.d.ts.map +1 -1
- package/lib/services/assessment/TestDataGenerator.js +32 -6
- package/lib/services/assessment/TestScenarioEngine.d.ts +9 -1
- package/lib/services/assessment/TestScenarioEngine.d.ts.map +1 -1
- package/lib/services/assessment/TestScenarioEngine.js +17 -14
- package/lib/services/assessment/config/annotationPatterns.d.ts +3 -1
- package/lib/services/assessment/config/annotationPatterns.d.ts.map +1 -1
- package/lib/services/assessment/config/annotationPatterns.js +5 -2
- package/lib/services/assessment/config/architecturePatterns.d.ts +101 -0
- package/lib/services/assessment/config/architecturePatterns.d.ts.map +1 -0
- package/lib/services/assessment/config/architecturePatterns.js +248 -0
- package/lib/services/assessment/config/performanceConfig.d.ts +122 -0
- package/lib/services/assessment/config/performanceConfig.d.ts.map +1 -0
- package/lib/services/assessment/config/performanceConfig.js +154 -0
- package/lib/services/assessment/config/sanitizationPatterns.d.ts +63 -0
- package/lib/services/assessment/config/sanitizationPatterns.d.ts.map +1 -0
- package/lib/services/assessment/config/sanitizationPatterns.js +223 -0
- package/lib/services/assessment/lib/claudeCodeBridge.d.ts +40 -3
- package/lib/services/assessment/lib/claudeCodeBridge.d.ts.map +1 -1
- package/lib/services/assessment/lib/claudeCodeBridge.js +149 -8
- package/lib/services/assessment/lib/concurrencyLimit.d.ts +6 -2
- package/lib/services/assessment/lib/concurrencyLimit.d.ts.map +1 -1
- package/lib/services/assessment/lib/concurrencyLimit.js +13 -6
- package/lib/services/assessment/lib/errors.d.ts +90 -0
- package/lib/services/assessment/lib/errors.d.ts.map +1 -0
- package/lib/services/assessment/lib/errors.js +136 -0
- package/lib/services/assessment/lib/timeoutUtils.d.ts +69 -0
- package/lib/services/assessment/lib/timeoutUtils.d.ts.map +1 -0
- package/lib/services/assessment/lib/timeoutUtils.js +103 -0
- package/lib/services/assessment/modules/BaseAssessor.d.ts +43 -8
- package/lib/services/assessment/modules/BaseAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/BaseAssessor.js +103 -34
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts +38 -1
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.js +185 -19
- package/lib/services/assessment/modules/DocumentationAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/DocumentationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/DocumentationAssessor.js +11 -0
- package/lib/services/assessment/modules/ErrorHandlingAssessor.js +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.js +6 -3
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts +3 -0
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.js +14 -2
- package/lib/services/assessment/modules/ManifestValidationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ManifestValidationAssessor.js +7 -2
- package/lib/services/assessment/modules/PromptAssessor.d.ts +1 -0
- package/lib/services/assessment/modules/PromptAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/PromptAssessor.js +26 -16
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.js +6 -2
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.js +15 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +8 -2
- package/lib/services/assessment/modules/SecurityAssessor.d.ts +3 -171
- package/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/SecurityAssessor.js +25 -1480
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +27 -28
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +340 -863
- package/lib/services/assessment/modules/UsabilityAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/UsabilityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/UsabilityAssessor.js +11 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.d.ts +57 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.js +176 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.d.ts +67 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.js +239 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.d.ts +46 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.js +394 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.d.ts +64 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.js +304 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts +43 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.js +276 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.d.ts +122 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.js +388 -0
- package/lib/services/assessment/modules/annotations/index.d.ts +13 -0
- package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/index.js +15 -0
- package/lib/services/assessment/modules/index.d.ts +10 -0
- package/lib/services/assessment/modules/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/index.js +13 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.d.ts +125 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.js +345 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts +33 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.js +128 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts +67 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +372 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts +178 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +1207 -0
- package/lib/services/assessment/modules/securityTests/index.d.ts +8 -0
- package/lib/services/assessment/modules/securityTests/index.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/index.js +7 -0
- package/lib/services/assessment/tool-classifier-patterns.d.ts +1 -0
- package/lib/services/assessment/tool-classifier-patterns.d.ts.map +1 -1
- package/lib/services/assessment/tool-classifier-patterns.js +17 -0
- package/package.json +1 -1
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Timeout Utilities for Assessment Operations
|
|
3
|
+
*
|
|
4
|
+
* Provides AbortController-based timeout handling with proper cleanup.
|
|
5
|
+
* Fixes GitHub issue #38: Promise.race patterns that leave timers hanging.
|
|
6
|
+
*
|
|
7
|
+
* Key improvements over basic Promise.race:
|
|
8
|
+
* 1. Clears setTimeout when operation completes (prevents timer leaks)
|
|
9
|
+
* 2. Uses AbortController for signal-based cancellation
|
|
10
|
+
* 3. Provides consistent error messages
|
|
11
|
+
*/
|
|
12
|
+
/**
|
|
13
|
+
* Execute a promise with timeout and proper cleanup.
|
|
14
|
+
*
|
|
15
|
+
* This function wraps any promise with a timeout, ensuring the timer
|
|
16
|
+
* is always cleaned up whether the operation succeeds, fails, or times out.
|
|
17
|
+
*
|
|
18
|
+
* @param promise - The promise to execute
|
|
19
|
+
* @param options - Timeout configuration
|
|
20
|
+
* @returns The result of the promise
|
|
21
|
+
* @throws Error if operation times out
|
|
22
|
+
*
|
|
23
|
+
* @example Basic usage
|
|
24
|
+
* ```typescript
|
|
25
|
+
* const result = await executeWithTimeout(
|
|
26
|
+
* callTool(tool.name, params),
|
|
27
|
+
* { timeoutMs: 5000 }
|
|
28
|
+
* );
|
|
29
|
+
* ```
|
|
30
|
+
*
|
|
31
|
+
* @example With custom error message
|
|
32
|
+
* ```typescript
|
|
33
|
+
* const result = await executeWithTimeout(
|
|
34
|
+
* fetchData(),
|
|
35
|
+
* { timeoutMs: 10000, errorMessage: "Data fetch timed out" }
|
|
36
|
+
* );
|
|
37
|
+
* ```
|
|
38
|
+
*/
|
|
39
|
+
export async function executeWithTimeout(promise, options) {
|
|
40
|
+
const { timeoutMs, errorMessage } = options;
|
|
41
|
+
// Create abort controller for timeout signaling
|
|
42
|
+
const controller = new AbortController();
|
|
43
|
+
// Set up timeout that triggers abort
|
|
44
|
+
const timeoutId = setTimeout(() => {
|
|
45
|
+
controller.abort();
|
|
46
|
+
}, timeoutMs);
|
|
47
|
+
// Create timeout promise that rejects when abort signal fires
|
|
48
|
+
const timeoutPromise = new Promise((_, reject) => {
|
|
49
|
+
controller.signal.addEventListener("abort", () => {
|
|
50
|
+
reject(new Error(errorMessage ?? `Operation timed out after ${timeoutMs}ms`));
|
|
51
|
+
});
|
|
52
|
+
});
|
|
53
|
+
try {
|
|
54
|
+
// Race the operation against timeout
|
|
55
|
+
return await Promise.race([promise, timeoutPromise]);
|
|
56
|
+
}
|
|
57
|
+
finally {
|
|
58
|
+
// CRITICAL: Always clean up timer to prevent leaks
|
|
59
|
+
clearTimeout(timeoutId);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
/**
|
|
63
|
+
* Execute an async function with timeout and AbortSignal support.
|
|
64
|
+
*
|
|
65
|
+
* This variant accepts a function that receives an AbortSignal, enabling
|
|
66
|
+
* cancellation of fetch requests and other abortable operations.
|
|
67
|
+
*
|
|
68
|
+
* @param fn - Function that accepts an AbortSignal and returns a Promise
|
|
69
|
+
* @param options - Timeout configuration
|
|
70
|
+
* @returns The result of the function
|
|
71
|
+
* @throws Error if operation times out
|
|
72
|
+
*
|
|
73
|
+
* @example With fetch
|
|
74
|
+
* ```typescript
|
|
75
|
+
* const data = await executeWithTimeoutAndSignal(
|
|
76
|
+
* async (signal) => {
|
|
77
|
+
* const response = await fetch(url, { signal });
|
|
78
|
+
* return response.json();
|
|
79
|
+
* },
|
|
80
|
+
* { timeoutMs: 5000 }
|
|
81
|
+
* );
|
|
82
|
+
* ```
|
|
83
|
+
*/
|
|
84
|
+
export async function executeWithTimeoutAndSignal(fn, options) {
|
|
85
|
+
const { timeoutMs, errorMessage } = options;
|
|
86
|
+
const controller = new AbortController();
|
|
87
|
+
const timeoutId = setTimeout(() => {
|
|
88
|
+
controller.abort();
|
|
89
|
+
}, timeoutMs);
|
|
90
|
+
try {
|
|
91
|
+
return await fn(controller.signal);
|
|
92
|
+
}
|
|
93
|
+
catch (error) {
|
|
94
|
+
// Convert AbortError to timeout error for consistency
|
|
95
|
+
if (error instanceof Error && error.name === "AbortError") {
|
|
96
|
+
throw new Error(errorMessage ?? `Operation timed out after ${timeoutMs}ms`);
|
|
97
|
+
}
|
|
98
|
+
throw error;
|
|
99
|
+
}
|
|
100
|
+
finally {
|
|
101
|
+
clearTimeout(timeoutId);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
@@ -5,10 +5,12 @@
|
|
|
5
5
|
import { AssessmentConfiguration, AssessmentStatus } from "../../../lib/assessmentTypes.js";
|
|
6
6
|
import { AssessmentContext } from "../AssessmentOrchestrator.js";
|
|
7
7
|
import { Logger } from "../lib/logger.js";
|
|
8
|
+
import { ErrorResult } from "../lib/errors.js";
|
|
8
9
|
export declare abstract class BaseAssessor<T = unknown> {
|
|
9
10
|
protected config: AssessmentConfiguration;
|
|
10
11
|
protected logger: Logger;
|
|
11
12
|
protected testCount: number;
|
|
13
|
+
private deprecationWarningsEmitted;
|
|
12
14
|
constructor(config: AssessmentConfiguration);
|
|
13
15
|
/**
|
|
14
16
|
* Abstract method that each assessor must implement
|
|
@@ -20,14 +22,39 @@ export declare abstract class BaseAssessor<T = unknown> {
|
|
|
20
22
|
protected determineStatus(passed: number, total: number, threshold?: number): AssessmentStatus;
|
|
21
23
|
/**
|
|
22
24
|
* Log assessment progress
|
|
23
|
-
* @deprecated Use this.logger.info() directly for structured logging with context
|
|
25
|
+
* @deprecated Use this.logger.info() directly for structured logging with context. Will be removed in v2.0.0.
|
|
24
26
|
*/
|
|
25
27
|
protected log(message: string): void;
|
|
26
28
|
/**
|
|
27
|
-
* Log error
|
|
28
|
-
* @deprecated Use this.logger.error() directly for structured logging with context
|
|
29
|
+
* Log error with optional context
|
|
30
|
+
* @deprecated Use this.logger.error() directly for structured logging with context. Will be removed in v2.0.0.
|
|
31
|
+
*
|
|
32
|
+
* @param message - Description of what operation failed
|
|
33
|
+
* @param error - The error that occurred (optional)
|
|
29
34
|
*/
|
|
30
35
|
protected logError(message: string, error?: unknown): void;
|
|
36
|
+
/**
|
|
37
|
+
* Handle an error with logging and structured result
|
|
38
|
+
*
|
|
39
|
+
* Use this method in catch blocks to ensure consistent error handling:
|
|
40
|
+
* 1. Logs the error with context
|
|
41
|
+
* 2. Categorizes the error automatically
|
|
42
|
+
* 3. Returns a structured result with error info
|
|
43
|
+
*
|
|
44
|
+
* @param error - The caught error
|
|
45
|
+
* @param context - Description of what operation failed
|
|
46
|
+
* @param defaults - Default values to merge into result
|
|
47
|
+
* @returns A result object with error information
|
|
48
|
+
*
|
|
49
|
+
* @example
|
|
50
|
+
* try {
|
|
51
|
+
* const result = await this.callTool(tool);
|
|
52
|
+
* return { passed: true, result };
|
|
53
|
+
* } catch (error) {
|
|
54
|
+
* return this.handleError(error, `Failed to call tool ${tool.name}`, { passed: false });
|
|
55
|
+
* }
|
|
56
|
+
*/
|
|
57
|
+
protected handleError<T extends ErrorResult>(error: unknown, context: string, defaults?: Partial<T>): T;
|
|
31
58
|
/**
|
|
32
59
|
* Get test count for this assessor
|
|
33
60
|
*/
|
|
@@ -45,17 +72,25 @@ export declare abstract class BaseAssessor<T = unknown> {
|
|
|
45
72
|
*/
|
|
46
73
|
protected sleep(ms: number): Promise<void>;
|
|
47
74
|
/**
|
|
48
|
-
* Execute with timeout
|
|
75
|
+
* Execute with timeout and proper cleanup.
|
|
76
|
+
*
|
|
77
|
+
* Uses AbortController-based timeout handling that clears the timer
|
|
78
|
+
* when the operation completes, preventing timer leaks.
|
|
79
|
+
*
|
|
80
|
+
* @param promise - The promise to execute
|
|
81
|
+
* @param timeoutMs - Timeout in milliseconds (defaults to config.testTimeout)
|
|
82
|
+
* @returns The result of the promise
|
|
83
|
+
* @throws Error if operation times out
|
|
49
84
|
*/
|
|
50
85
|
protected executeWithTimeout<T>(promise: Promise<T>, timeoutMs?: number): Promise<T>;
|
|
51
86
|
/**
|
|
52
87
|
* Safe JSON parse with error handling
|
|
53
88
|
*/
|
|
54
|
-
protected safeJsonParse(text: string):
|
|
89
|
+
protected safeJsonParse(text: string): unknown;
|
|
55
90
|
/**
|
|
56
91
|
* Extract error message from various error types
|
|
57
92
|
*/
|
|
58
|
-
protected extractErrorMessage(error:
|
|
93
|
+
protected extractErrorMessage(error: unknown): string;
|
|
59
94
|
/**
|
|
60
95
|
* Check if a response indicates an error
|
|
61
96
|
* Handles various MCP response formats
|
|
@@ -63,11 +98,11 @@ export declare abstract class BaseAssessor<T = unknown> {
|
|
|
63
98
|
* @param response - The response to check
|
|
64
99
|
* @param strictMode - If true, only check explicit error indicators (default: false)
|
|
65
100
|
*/
|
|
66
|
-
protected isErrorResponse(response:
|
|
101
|
+
protected isErrorResponse(response: unknown, strictMode?: boolean): boolean;
|
|
67
102
|
/**
|
|
68
103
|
* Extract error information from a response
|
|
69
104
|
*/
|
|
70
|
-
protected extractErrorInfo(response:
|
|
105
|
+
protected extractErrorInfo(response: unknown): {
|
|
71
106
|
code?: string | number;
|
|
72
107
|
message?: string;
|
|
73
108
|
};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"BaseAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/BaseAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EACvB,gBAAgB,
|
|
1
|
+
{"version":3,"file":"BaseAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/BaseAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EACvB,gBAAgB,EAEjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EAAE,MAAM,EAAwC,MAAM,eAAe,CAAC;AAC7E,OAAO,EAGL,WAAW,EAEZ,MAAM,eAAe,CAAC;AAGvB,8BAAsB,YAAY,CAAC,CAAC,GAAG,OAAO;IAC5C,SAAS,CAAC,MAAM,EAAE,uBAAuB,CAAC;IAC1C,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC;IACzB,SAAS,CAAC,SAAS,EAAE,MAAM,CAAK;IAGhC,OAAO,CAAC,0BAA0B,CAGhC;gBAEU,MAAM,EAAE,uBAAuB;IAS3C;;OAEG;IACH,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,CAAC,CAAC;IAEvD;;OAEG;IACH,SAAS,CAAC,eAAe,CACvB,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,SAAS,GAAE,MAAY,GACtB,gBAAgB;IAUnB;;;OAGG;IACH,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAWpC;;;;;;OAMG;IACH,SAAS,CAAC,QAAQ,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,GAAG,IAAI;IAW1D;;;;;;;;;;;;;;;;;;;;OAoBG;IACH,SAAS,CAAC,WAAW,CAAC,CAAC,SAAS,WAAW,EACzC,KAAK,EAAE,OAAO,EACd,OAAO,EAAE,MAAM,EACf,QAAQ,GAAE,OAAO,CAAC,CAAC,CAAM,GACxB,CAAC;IAsBJ;;OAEG;IACH,YAAY,IAAI,MAAM;IAItB;;OAEG;IACH,cAAc,IAAI,IAAI;IAItB;;OAEG;IACH,SAAS,CAAC,gBAAgB,CACxB,OAAO,EAAE,MAAM,uBAAuB,CAAC,sBAAsB,CAAC,GAC7D,OAAO;IAIV;;OAEG;cACa,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIhD;;;;;;;;;;OAUG;cACa,kBAAkB,CAAC,CAAC,EAClC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,EACnB,SAAS,GAAE,MAAgC,GAC1C,OAAO,CAAC,CAAC,CAAC;IAOb;;OAEG;IACH,SAAS,CAAC,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO;IAS9C;;OAEG;IACH,SAAS,CAAC,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM;IAgBrD;;;;;;OAMG;IACH,SAAS,CAAC,eAAe,CACvB,QAAQ,EAAE,OAAO,EACjB,UAAU,GAAE,OAAe,GAC1B,OAAO;IA+CV;;OAEG;IACH,SAAS,CAAC,gBAAgB,CAAC,QAAQ,EAAE,OAAO,GAAG;QAC7C,IAAI,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC;QACvB,OAAO,CAAC,EAAE,MAAM,CAAC;KAClB;CA2BF"}
|
|
@@ -3,10 +3,17 @@
|
|
|
3
3
|
* Provides common functionality for all assessment modules
|
|
4
4
|
*/
|
|
5
5
|
import { createLogger, DEFAULT_LOGGING_CONFIG } from "../lib/logger.js";
|
|
6
|
+
import { ErrorCategory, categorizeError, } from "../lib/errors.js";
|
|
7
|
+
import { executeWithTimeout as executeWithTimeoutUtil } from "../lib/timeoutUtils.js";
|
|
6
8
|
export class BaseAssessor {
|
|
7
9
|
config;
|
|
8
10
|
logger;
|
|
9
11
|
testCount = 0;
|
|
12
|
+
// Track deprecation warnings to emit only once per instance
|
|
13
|
+
deprecationWarningsEmitted = {
|
|
14
|
+
log: false,
|
|
15
|
+
logError: false,
|
|
16
|
+
};
|
|
10
17
|
constructor(config) {
|
|
11
18
|
this.config = config;
|
|
12
19
|
// Create logger from config, using class name as prefix
|
|
@@ -27,18 +34,70 @@ export class BaseAssessor {
|
|
|
27
34
|
}
|
|
28
35
|
/**
|
|
29
36
|
* Log assessment progress
|
|
30
|
-
* @deprecated Use this.logger.info() directly for structured logging with context
|
|
37
|
+
* @deprecated Use this.logger.info() directly for structured logging with context. Will be removed in v2.0.0.
|
|
31
38
|
*/
|
|
32
39
|
log(message) {
|
|
40
|
+
if (!this.deprecationWarningsEmitted.log) {
|
|
41
|
+
this.logger.warn("BaseAssessor.log() is deprecated. Use this.logger.info() instead. " +
|
|
42
|
+
"This method will be removed in v2.0.0.");
|
|
43
|
+
this.deprecationWarningsEmitted.log = true;
|
|
44
|
+
}
|
|
33
45
|
this.logger.info(message);
|
|
34
46
|
}
|
|
35
47
|
/**
|
|
36
|
-
* Log error
|
|
37
|
-
* @deprecated Use this.logger.error() directly for structured logging with context
|
|
48
|
+
* Log error with optional context
|
|
49
|
+
* @deprecated Use this.logger.error() directly for structured logging with context. Will be removed in v2.0.0.
|
|
50
|
+
*
|
|
51
|
+
* @param message - Description of what operation failed
|
|
52
|
+
* @param error - The error that occurred (optional)
|
|
38
53
|
*/
|
|
39
54
|
logError(message, error) {
|
|
55
|
+
if (!this.deprecationWarningsEmitted.logError) {
|
|
56
|
+
this.logger.warn("BaseAssessor.logError() is deprecated. Use this.logger.error() instead. " +
|
|
57
|
+
"This method will be removed in v2.0.0.");
|
|
58
|
+
this.deprecationWarningsEmitted.logError = true;
|
|
59
|
+
}
|
|
40
60
|
this.logger.error(message, error ? { error: String(error) } : undefined);
|
|
41
61
|
}
|
|
62
|
+
/**
|
|
63
|
+
* Handle an error with logging and structured result
|
|
64
|
+
*
|
|
65
|
+
* Use this method in catch blocks to ensure consistent error handling:
|
|
66
|
+
* 1. Logs the error with context
|
|
67
|
+
* 2. Categorizes the error automatically
|
|
68
|
+
* 3. Returns a structured result with error info
|
|
69
|
+
*
|
|
70
|
+
* @param error - The caught error
|
|
71
|
+
* @param context - Description of what operation failed
|
|
72
|
+
* @param defaults - Default values to merge into result
|
|
73
|
+
* @returns A result object with error information
|
|
74
|
+
*
|
|
75
|
+
* @example
|
|
76
|
+
* try {
|
|
77
|
+
* const result = await this.callTool(tool);
|
|
78
|
+
* return { passed: true, result };
|
|
79
|
+
* } catch (error) {
|
|
80
|
+
* return this.handleError(error, `Failed to call tool ${tool.name}`, { passed: false });
|
|
81
|
+
* }
|
|
82
|
+
*/
|
|
83
|
+
handleError(error, context, defaults = {}) {
|
|
84
|
+
const category = categorizeError(error);
|
|
85
|
+
const message = this.extractErrorMessage(error);
|
|
86
|
+
this.logger.error(context, {
|
|
87
|
+
error: message,
|
|
88
|
+
category,
|
|
89
|
+
});
|
|
90
|
+
const errorInfo = {
|
|
91
|
+
message,
|
|
92
|
+
code: category,
|
|
93
|
+
recoverable: category !== ErrorCategory.CONNECTION,
|
|
94
|
+
stack: error instanceof Error ? error.stack : undefined,
|
|
95
|
+
};
|
|
96
|
+
return {
|
|
97
|
+
...defaults,
|
|
98
|
+
error: errorInfo,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
42
101
|
/**
|
|
43
102
|
* Get test count for this assessor
|
|
44
103
|
*/
|
|
@@ -64,13 +123,21 @@ export class BaseAssessor {
|
|
|
64
123
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
65
124
|
}
|
|
66
125
|
/**
|
|
67
|
-
* Execute with timeout
|
|
126
|
+
* Execute with timeout and proper cleanup.
|
|
127
|
+
*
|
|
128
|
+
* Uses AbortController-based timeout handling that clears the timer
|
|
129
|
+
* when the operation completes, preventing timer leaks.
|
|
130
|
+
*
|
|
131
|
+
* @param promise - The promise to execute
|
|
132
|
+
* @param timeoutMs - Timeout in milliseconds (defaults to config.testTimeout)
|
|
133
|
+
* @returns The result of the promise
|
|
134
|
+
* @throws Error if operation times out
|
|
68
135
|
*/
|
|
69
136
|
async executeWithTimeout(promise, timeoutMs = this.config.testTimeout) {
|
|
70
|
-
|
|
71
|
-
|
|
137
|
+
return executeWithTimeoutUtil(promise, {
|
|
138
|
+
timeoutMs,
|
|
139
|
+
errorMessage: `Operation timed out after ${timeoutMs}ms`,
|
|
72
140
|
});
|
|
73
|
-
return Promise.race([promise, timeoutPromise]);
|
|
74
141
|
}
|
|
75
142
|
/**
|
|
76
143
|
* Safe JSON parse with error handling
|
|
@@ -90,17 +157,19 @@ export class BaseAssessor {
|
|
|
90
157
|
extractErrorMessage(error) {
|
|
91
158
|
if (typeof error === "string")
|
|
92
159
|
return error;
|
|
93
|
-
if (error
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
.join(" ");
|
|
160
|
+
if (error && typeof error === "object") {
|
|
161
|
+
const err = error;
|
|
162
|
+
if (err.message && typeof err.message === "string")
|
|
163
|
+
return err.message;
|
|
164
|
+
if (err.error)
|
|
165
|
+
return this.extractErrorMessage(err.error);
|
|
166
|
+
if (err.content) {
|
|
167
|
+
if (Array.isArray(err.content)) {
|
|
168
|
+
return err.content.map((c) => c.text || "").join(" ");
|
|
169
|
+
}
|
|
170
|
+
if (typeof err.content === "string")
|
|
171
|
+
return err.content;
|
|
102
172
|
}
|
|
103
|
-
return error.content;
|
|
104
173
|
}
|
|
105
174
|
return JSON.stringify(error);
|
|
106
175
|
}
|
|
@@ -112,10 +181,11 @@ export class BaseAssessor {
|
|
|
112
181
|
* @param strictMode - If true, only check explicit error indicators (default: false)
|
|
113
182
|
*/
|
|
114
183
|
isErrorResponse(response, strictMode = false) {
|
|
115
|
-
if (!response)
|
|
184
|
+
if (!response || typeof response !== "object")
|
|
116
185
|
return false;
|
|
186
|
+
const resp = response;
|
|
117
187
|
// Check explicit error flag first (always check these)
|
|
118
|
-
if (
|
|
188
|
+
if (resp.isError === true || resp.error !== undefined) {
|
|
119
189
|
return true;
|
|
120
190
|
}
|
|
121
191
|
// In strict mode, only rely on explicit error indicators
|
|
@@ -125,9 +195,9 @@ export class BaseAssessor {
|
|
|
125
195
|
}
|
|
126
196
|
// In non-strict mode, also check content for error patterns
|
|
127
197
|
// Used by ErrorHandlingAssessor where we're deliberately triggering errors
|
|
128
|
-
if (
|
|
129
|
-
if (typeof
|
|
130
|
-
const lower =
|
|
198
|
+
if (resp.content) {
|
|
199
|
+
if (typeof resp.content === "string") {
|
|
200
|
+
const lower = resp.content.toLowerCase();
|
|
131
201
|
// Only flag if error appears at start or with strong indicators
|
|
132
202
|
return (lower.startsWith("error:") ||
|
|
133
203
|
lower.startsWith("error ") ||
|
|
@@ -135,9 +205,9 @@ export class BaseAssessor {
|
|
|
135
205
|
lower.includes("failed to") ||
|
|
136
206
|
lower.includes("exception:"));
|
|
137
207
|
}
|
|
138
|
-
else if (Array.isArray(
|
|
208
|
+
else if (Array.isArray(resp.content)) {
|
|
139
209
|
// Check if any text content starts with error indicators
|
|
140
|
-
return
|
|
210
|
+
return resp.content.some((c) => {
|
|
141
211
|
if (c.type !== "text" || !c.text)
|
|
142
212
|
return false;
|
|
143
213
|
const lower = c.text.toLowerCase();
|
|
@@ -155,23 +225,22 @@ export class BaseAssessor {
|
|
|
155
225
|
* Extract error information from a response
|
|
156
226
|
*/
|
|
157
227
|
extractErrorInfo(response) {
|
|
158
|
-
if (!response)
|
|
228
|
+
if (!response || typeof response !== "object")
|
|
159
229
|
return {};
|
|
230
|
+
const resp = response;
|
|
160
231
|
// Extract text from content array if present
|
|
161
232
|
let contentText;
|
|
162
|
-
if (Array.isArray(
|
|
163
|
-
const textContent =
|
|
233
|
+
if (Array.isArray(resp.content)) {
|
|
234
|
+
const textContent = resp.content.find((c) => c.type === "text");
|
|
164
235
|
contentText = textContent?.text;
|
|
165
236
|
}
|
|
166
|
-
else if (typeof
|
|
167
|
-
contentText =
|
|
237
|
+
else if (typeof resp.content === "string") {
|
|
238
|
+
contentText = resp.content;
|
|
168
239
|
}
|
|
240
|
+
const error = resp.error;
|
|
169
241
|
return {
|
|
170
|
-
code:
|
|
171
|
-
message:
|
|
172
|
-
response.message ||
|
|
173
|
-
response.error?.message ||
|
|
174
|
-
contentText,
|
|
242
|
+
code: (resp.errorCode ?? resp.code ?? error?.code),
|
|
243
|
+
message: (resp.errorMessage ?? resp.message ?? error?.message) || contentText,
|
|
175
244
|
};
|
|
176
245
|
}
|
|
177
246
|
}
|
|
@@ -51,7 +51,44 @@ export declare class DeveloperExperienceAssessor extends BaseAssessor<DeveloperE
|
|
|
51
51
|
private extractSection;
|
|
52
52
|
private extractSectionHeadings;
|
|
53
53
|
private classifyCodeExample;
|
|
54
|
-
|
|
54
|
+
/**
|
|
55
|
+
* Assess documentation quality using Issue #55 point-based scoring
|
|
56
|
+
* Max 100 points: README (30), Install (20), Config (20), Examples (20), License (10)
|
|
57
|
+
*/
|
|
58
|
+
private assessDocumentationQuality;
|
|
59
|
+
/**
|
|
60
|
+
* Determine README quality tier based on size
|
|
61
|
+
* - minimal: <5KB
|
|
62
|
+
* - adequate: 5KB-15KB
|
|
63
|
+
* - comprehensive: >15KB
|
|
64
|
+
*/
|
|
65
|
+
private determineReadmeQuality;
|
|
66
|
+
/**
|
|
67
|
+
* Calculate point-based quality score per Issue #55
|
|
68
|
+
* Max 100 points:
|
|
69
|
+
* - README exists: +10
|
|
70
|
+
* - README >5KB: +10 (adequate)
|
|
71
|
+
* - README >15KB: +10 more (comprehensive = +20 total)
|
|
72
|
+
* - Installation section: +20
|
|
73
|
+
* - Configuration section: +20
|
|
74
|
+
* - Examples present: +20
|
|
75
|
+
* - License file: +10
|
|
76
|
+
*/
|
|
77
|
+
private calculateQualityScore;
|
|
78
|
+
/**
|
|
79
|
+
* Check for configuration/environment section
|
|
80
|
+
* Looks for: configuration, config, environment, env vars, .env
|
|
81
|
+
*/
|
|
82
|
+
private checkConfigurationSection;
|
|
83
|
+
/**
|
|
84
|
+
* Detect license presence from context
|
|
85
|
+
* Checks sourceCodeFiles for LICENSE/LICENSE.md or README for license section
|
|
86
|
+
*/
|
|
87
|
+
private detectLicense;
|
|
88
|
+
/**
|
|
89
|
+
* Detect license type (MIT, Apache-2.0, GPL, BSD, etc.)
|
|
90
|
+
*/
|
|
91
|
+
private detectLicenseType;
|
|
55
92
|
private analyzeUsability;
|
|
56
93
|
private analyzeNamingConvention;
|
|
57
94
|
private analyzeParameterClarity;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DeveloperExperienceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/DeveloperExperienceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAEhB,gBAAgB,
|
|
1
|
+
{"version":3,"file":"DeveloperExperienceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/DeveloperExperienceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EACL,oBAAoB,EACpB,gBAAgB,EAEhB,gBAAgB,EAIjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D;;GAEG;AACH,MAAM,WAAW,6BAA6B;IAC5C,yCAAyC;IACzC,aAAa,EAAE,oBAAoB,CAAC;IACpC,qCAAqC;IACrC,SAAS,EAAE,gBAAgB,CAAC;IAC5B,gDAAgD;IAChD,MAAM,EAAE,gBAAgB,CAAC;IACzB,iCAAiC;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,sCAAsC;IACtC,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,iDAAiD;IACjD,MAAM,EAAE;QACN,aAAa,EAAE,MAAM,CAAC;QACtB,SAAS,EAAE,MAAM,CAAC;QAClB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAED,qBAAa,2BAA4B,SAAQ,YAAY,CAAC,6BAA6B,CAAC;IACpF,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,6BAA6B,CAAC;IAsEzC,OAAO,CAAC,oBAAoB;IAmH5B,OAAO,CAAC,yBAAyB;IA4CjC,OAAO,CAAC,wBAAwB;IAiBhC,OAAO,CAAC,sBAAsB;IA8B9B,OAAO,CAAC,aAAa;IAKrB,OAAO,CAAC,mBAAmB;IAoB3B,OAAO,CAAC,mBAAmB;IAwC3B,OAAO,CAAC,wBAAwB;IAahC,OAAO,CAAC,eAAe;IAYvB,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,sBAAsB;IAY9B,OAAO,CAAC,mBAAmB;IA6B3B;;;OAGG;IACH,OAAO,CAAC,0BAA0B;IAkBlC;;;;;OAKG;IACH,OAAO,CAAC,sBAAsB;IAW9B;;;;;;;;;;OAUG;IACH,OAAO,CAAC,qBAAqB;IAgC7B;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAgBjC;;;OAGG;IACH,OAAO,CAAC,aAAa;IAoBrB;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAsEzB,OAAO,CAAC,gBAAgB;IAcxB,OAAO,CAAC,uBAAuB;IAoC/B,OAAO,CAAC,uBAAuB;IAsC/B,OAAO,CAAC,iBAAiB;IAczB,OAAO,CAAC,kBAAkB;IAsC1B,OAAO,CAAC,iBAAiB;IAiCzB,OAAO,CAAC,aAAa;IAQrB,OAAO,CAAC,uBAAuB;IAgB/B,OAAO,CAAC,sBAAsB;IAM9B,OAAO,CAAC,mBAAmB;IAyC3B,OAAO,CAAC,uBAAuB;CAmDhC"}
|