@bryan-thompson/inspector-assessment-client 1.25.4 → 1.25.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/assets/{OAuthCallback-DE62cdTZ.js → OAuthCallback-D6y8tFfF.js} +1 -1
- package/dist/assets/{OAuthDebugCallback-CWjFdCIE.js → OAuthDebugCallback-DHegnqTa.js} +1 -1
- package/dist/assets/{index-PCQVSwHa.js → index-Cu02Ah3g.js} +4 -4
- package/dist/assets/{index-Df9Sx1jt.css → index-cHhcEXbr.css} +4 -0
- package/dist/index.html +2 -2
- package/lib/lib/assessment/coreTypes.d.ts +65 -0
- package/lib/lib/assessment/coreTypes.d.ts.map +1 -1
- package/lib/lib/assessment/extendedTypes.d.ts +127 -0
- package/lib/lib/assessment/extendedTypes.d.ts.map +1 -1
- package/lib/lib/assessment/resultTypes.d.ts +45 -0
- package/lib/lib/assessment/resultTypes.d.ts.map +1 -1
- package/lib/lib/moduleScoring.d.ts +2 -2
- package/lib/lib/moduleScoring.d.ts.map +1 -1
- package/lib/lib/moduleScoring.js +3 -2
- package/lib/services/assessment/AssessmentOrchestrator.d.ts +3 -7
- package/lib/services/assessment/AssessmentOrchestrator.d.ts.map +1 -1
- package/lib/services/assessment/AssessmentOrchestrator.js +13 -2
- package/lib/services/assessment/TestDataGenerator.d.ts +9 -1
- package/lib/services/assessment/TestDataGenerator.d.ts.map +1 -1
- package/lib/services/assessment/TestDataGenerator.js +32 -6
- package/lib/services/assessment/TestScenarioEngine.d.ts +9 -1
- package/lib/services/assessment/TestScenarioEngine.d.ts.map +1 -1
- package/lib/services/assessment/TestScenarioEngine.js +17 -14
- package/lib/services/assessment/config/annotationPatterns.d.ts +3 -1
- package/lib/services/assessment/config/annotationPatterns.d.ts.map +1 -1
- package/lib/services/assessment/config/annotationPatterns.js +5 -2
- package/lib/services/assessment/config/architecturePatterns.d.ts +101 -0
- package/lib/services/assessment/config/architecturePatterns.d.ts.map +1 -0
- package/lib/services/assessment/config/architecturePatterns.js +248 -0
- package/lib/services/assessment/config/performanceConfig.d.ts +122 -0
- package/lib/services/assessment/config/performanceConfig.d.ts.map +1 -0
- package/lib/services/assessment/config/performanceConfig.js +154 -0
- package/lib/services/assessment/config/sanitizationPatterns.d.ts +63 -0
- package/lib/services/assessment/config/sanitizationPatterns.d.ts.map +1 -0
- package/lib/services/assessment/config/sanitizationPatterns.js +223 -0
- package/lib/services/assessment/lib/claudeCodeBridge.d.ts +40 -3
- package/lib/services/assessment/lib/claudeCodeBridge.d.ts.map +1 -1
- package/lib/services/assessment/lib/claudeCodeBridge.js +149 -8
- package/lib/services/assessment/lib/concurrencyLimit.d.ts +6 -2
- package/lib/services/assessment/lib/concurrencyLimit.d.ts.map +1 -1
- package/lib/services/assessment/lib/concurrencyLimit.js +13 -6
- package/lib/services/assessment/lib/errors.d.ts +90 -0
- package/lib/services/assessment/lib/errors.d.ts.map +1 -0
- package/lib/services/assessment/lib/errors.js +136 -0
- package/lib/services/assessment/lib/timeoutUtils.d.ts +69 -0
- package/lib/services/assessment/lib/timeoutUtils.d.ts.map +1 -0
- package/lib/services/assessment/lib/timeoutUtils.js +103 -0
- package/lib/services/assessment/modules/BaseAssessor.d.ts +43 -8
- package/lib/services/assessment/modules/BaseAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/BaseAssessor.js +103 -34
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts +38 -1
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/DeveloperExperienceAssessor.js +185 -19
- package/lib/services/assessment/modules/DocumentationAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/DocumentationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/DocumentationAssessor.js +11 -0
- package/lib/services/assessment/modules/ErrorHandlingAssessor.js +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/FunctionalityAssessor.js +6 -3
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts +3 -0
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/MCPSpecComplianceAssessor.js +14 -2
- package/lib/services/assessment/modules/ManifestValidationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ManifestValidationAssessor.js +7 -2
- package/lib/services/assessment/modules/PromptAssessor.d.ts +1 -0
- package/lib/services/assessment/modules/PromptAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/PromptAssessor.js +26 -16
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ProtocolComplianceAssessor.js +6 -2
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ProtocolConformanceAssessor.js +15 -0
- package/lib/services/assessment/modules/ResourceAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ResourceAssessor.js +8 -2
- package/lib/services/assessment/modules/SecurityAssessor.d.ts +3 -171
- package/lib/services/assessment/modules/SecurityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/SecurityAssessor.js +25 -1480
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts +27 -28
- package/lib/services/assessment/modules/ToolAnnotationAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/ToolAnnotationAssessor.js +340 -863
- package/lib/services/assessment/modules/UsabilityAssessor.d.ts +5 -0
- package/lib/services/assessment/modules/UsabilityAssessor.d.ts.map +1 -1
- package/lib/services/assessment/modules/UsabilityAssessor.js +11 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.d.ts +57 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/AnnotationDeceptionDetector.js +176 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.d.ts +67 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/ArchitectureDetector.js +239 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.d.ts +46 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/BehaviorInference.js +394 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.d.ts +64 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/DescriptionAnalyzer.js +304 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts +43 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/DescriptionPoisoningDetector.js +276 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.d.ts +122 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/SchemaAnalyzer.js +388 -0
- package/lib/services/assessment/modules/annotations/index.d.ts +13 -0
- package/lib/services/assessment/modules/annotations/index.d.ts.map +1 -0
- package/lib/services/assessment/modules/annotations/index.js +15 -0
- package/lib/services/assessment/modules/index.d.ts +10 -0
- package/lib/services/assessment/modules/index.d.ts.map +1 -1
- package/lib/services/assessment/modules/index.js +13 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.d.ts +125 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SanitizationDetector.js +345 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts +33 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadGenerator.js +128 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts +67 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityPayloadTester.js +372 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts +178 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/SecurityResponseAnalyzer.js +1207 -0
- package/lib/services/assessment/modules/securityTests/index.d.ts +8 -0
- package/lib/services/assessment/modules/securityTests/index.d.ts.map +1 -0
- package/lib/services/assessment/modules/securityTests/index.js +7 -0
- package/lib/services/assessment/tool-classifier-patterns.d.ts +1 -0
- package/lib/services/assessment/tool-classifier-patterns.d.ts.map +1 -1
- package/lib/services/assessment/tool-classifier-patterns.js +17 -0
- package/package.json +1 -1
|
@@ -16,20 +16,28 @@ import { BaseAssessor } from "./BaseAssessor.js";
|
|
|
16
16
|
export class DeveloperExperienceAssessor extends BaseAssessor {
|
|
17
17
|
async assess(context) {
|
|
18
18
|
this.log("Starting developer experience assessment");
|
|
19
|
+
const readmeContent = context.readmeContent || "";
|
|
19
20
|
// Assess documentation
|
|
20
|
-
const documentationMetrics = this.analyzeDocumentation(
|
|
21
|
-
|
|
21
|
+
const documentationMetrics = this.analyzeDocumentation(readmeContent, context.tools, "verbose");
|
|
22
|
+
// Issue #55: Add quality scoring
|
|
23
|
+
const { checks: qualityChecks, score: qualityScore } = this.assessDocumentationQuality(readmeContent, context);
|
|
24
|
+
// Add quality data to metrics
|
|
25
|
+
documentationMetrics.qualityChecks = qualityChecks;
|
|
26
|
+
documentationMetrics.qualityScore = qualityScore;
|
|
27
|
+
documentationMetrics.readmeSizeBytes = Buffer.byteLength(readmeContent, "utf8");
|
|
28
|
+
// Use quality score for documentation scoring (Issue #55)
|
|
29
|
+
const documentationScore = qualityScore.total;
|
|
22
30
|
// Assess usability
|
|
23
31
|
const usabilityMetrics = this.analyzeUsability(context.tools);
|
|
24
32
|
const usabilityScore = this.calculateUsabilityScore(usabilityMetrics);
|
|
25
|
-
// Calculate overall score (weighted average)
|
|
33
|
+
// Calculate overall score (weighted average: 60% docs, 40% usability)
|
|
26
34
|
const overallScore = Math.round(documentationScore * 0.6 + usabilityScore * 0.4);
|
|
27
|
-
// Determine status
|
|
35
|
+
// Determine status using Issue #55 thresholds
|
|
28
36
|
const status = this.determineOverallStatus(overallScore);
|
|
29
37
|
// Generate explanation and recommendations
|
|
30
38
|
const explanation = this.generateExplanation(documentationMetrics, usabilityMetrics, context.tools);
|
|
31
39
|
const recommendations = this.generateRecommendations(documentationMetrics, usabilityMetrics);
|
|
32
|
-
this.testCount =
|
|
40
|
+
this.testCount = 15; // Documentation (5) + Quality (6) + Usability (4) checks
|
|
33
41
|
return {
|
|
34
42
|
documentation: documentationMetrics,
|
|
35
43
|
usability: usabilityMetrics,
|
|
@@ -321,20 +329,178 @@ export class DeveloperExperienceAssessor extends BaseAssessor {
|
|
|
321
329
|
}
|
|
322
330
|
return "functional";
|
|
323
331
|
}
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
332
|
+
// ============================================================================
|
|
333
|
+
// Issue #55: Documentation Quality Scoring
|
|
334
|
+
// ============================================================================
|
|
335
|
+
/**
|
|
336
|
+
* Assess documentation quality using Issue #55 point-based scoring
|
|
337
|
+
* Max 100 points: README (30), Install (20), Config (20), Examples (20), License (10)
|
|
338
|
+
*/
|
|
339
|
+
assessDocumentationQuality(content, context) {
|
|
340
|
+
const checks = {
|
|
341
|
+
hasReadme: content.length > 0,
|
|
342
|
+
readmeQuality: this.determineReadmeQuality(content),
|
|
343
|
+
hasInstallation: this.checkInstallInstructions(content),
|
|
344
|
+
hasConfiguration: this.checkConfigurationSection(content),
|
|
345
|
+
hasExamples: this.checkUsageGuide(content),
|
|
346
|
+
hasLicense: this.detectLicense(context),
|
|
347
|
+
licenseType: this.detectLicenseType(context),
|
|
348
|
+
};
|
|
349
|
+
const score = this.calculateQualityScore(checks, content);
|
|
350
|
+
return { checks, score };
|
|
351
|
+
}
|
|
352
|
+
/**
|
|
353
|
+
* Determine README quality tier based on size
|
|
354
|
+
* - minimal: <5KB
|
|
355
|
+
* - adequate: 5KB-15KB
|
|
356
|
+
* - comprehensive: >15KB
|
|
357
|
+
*/
|
|
358
|
+
determineReadmeQuality(content) {
|
|
359
|
+
const sizeBytes = Buffer.byteLength(content, "utf8");
|
|
360
|
+
const sizeKB = sizeBytes / 1024;
|
|
361
|
+
if (sizeKB > 15)
|
|
362
|
+
return "comprehensive";
|
|
363
|
+
if (sizeKB > 5)
|
|
364
|
+
return "adequate";
|
|
365
|
+
return "minimal";
|
|
366
|
+
}
|
|
367
|
+
/**
|
|
368
|
+
* Calculate point-based quality score per Issue #55
|
|
369
|
+
* Max 100 points:
|
|
370
|
+
* - README exists: +10
|
|
371
|
+
* - README >5KB: +10 (adequate)
|
|
372
|
+
* - README >15KB: +10 more (comprehensive = +20 total)
|
|
373
|
+
* - Installation section: +20
|
|
374
|
+
* - Configuration section: +20
|
|
375
|
+
* - Examples present: +20
|
|
376
|
+
* - License file: +10
|
|
377
|
+
*/
|
|
378
|
+
calculateQualityScore(checks, content) {
|
|
379
|
+
const sizeBytes = Buffer.byteLength(content, "utf8");
|
|
380
|
+
const sizeKB = sizeBytes / 1024;
|
|
381
|
+
// Calculate README size bonus
|
|
382
|
+
let readmeComprehensive = 0;
|
|
383
|
+
if (checks.hasReadme) {
|
|
384
|
+
if (sizeKB > 15) {
|
|
385
|
+
readmeComprehensive = 20; // comprehensive: +10 + +10
|
|
386
|
+
}
|
|
387
|
+
else if (sizeKB > 5) {
|
|
388
|
+
readmeComprehensive = 10; // adequate: +10
|
|
389
|
+
}
|
|
390
|
+
}
|
|
391
|
+
const breakdown = {
|
|
392
|
+
readmeExists: checks.hasReadme ? 10 : 0,
|
|
393
|
+
readmeComprehensive,
|
|
394
|
+
installation: checks.hasInstallation ? 20 : 0,
|
|
395
|
+
configuration: checks.hasConfiguration ? 20 : 0,
|
|
396
|
+
examples: checks.hasExamples ? 20 : 0,
|
|
397
|
+
license: checks.hasLicense ? 10 : 0,
|
|
398
|
+
};
|
|
399
|
+
return {
|
|
400
|
+
total: Object.values(breakdown).reduce((sum, v) => sum + v, 0),
|
|
401
|
+
breakdown,
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
/**
|
|
405
|
+
* Check for configuration/environment section
|
|
406
|
+
* Looks for: configuration, config, environment, env vars, .env
|
|
407
|
+
*/
|
|
408
|
+
checkConfigurationSection(content) {
|
|
409
|
+
const configKeywords = [
|
|
410
|
+
"configuration",
|
|
411
|
+
"config",
|
|
412
|
+
"environment variable",
|
|
413
|
+
"env var",
|
|
414
|
+
".env",
|
|
415
|
+
"api key",
|
|
416
|
+
"api_key",
|
|
417
|
+
"apikey",
|
|
418
|
+
"setup",
|
|
419
|
+
];
|
|
420
|
+
const contentLower = content.toLowerCase();
|
|
421
|
+
return configKeywords.some((keyword) => contentLower.includes(keyword));
|
|
422
|
+
}
|
|
423
|
+
/**
|
|
424
|
+
* Detect license presence from context
|
|
425
|
+
* Checks sourceCodeFiles for LICENSE/LICENSE.md or README for license section
|
|
426
|
+
*/
|
|
427
|
+
detectLicense(context) {
|
|
428
|
+
// Check source code files if available
|
|
429
|
+
if (context.sourceCodeFiles) {
|
|
430
|
+
const licenseFiles = [
|
|
431
|
+
"LICENSE",
|
|
432
|
+
"LICENSE.md",
|
|
433
|
+
"LICENSE.txt",
|
|
434
|
+
"LICENCE",
|
|
435
|
+
"LICENCE.md",
|
|
436
|
+
];
|
|
437
|
+
for (const file of licenseFiles) {
|
|
438
|
+
if (context.sourceCodeFiles.has(file))
|
|
439
|
+
return true;
|
|
440
|
+
}
|
|
441
|
+
}
|
|
442
|
+
// Fallback: check README for license section
|
|
443
|
+
const content = context.readmeContent || "";
|
|
444
|
+
return /^#+\s*licen[sc]e/im.test(content);
|
|
445
|
+
}
|
|
446
|
+
/**
|
|
447
|
+
* Detect license type (MIT, Apache-2.0, GPL, BSD, etc.)
|
|
448
|
+
*/
|
|
449
|
+
detectLicenseType(context) {
|
|
450
|
+
if (!context.sourceCodeFiles)
|
|
451
|
+
return undefined;
|
|
452
|
+
// Try common license file names
|
|
453
|
+
const licenseFiles = [
|
|
454
|
+
"LICENSE",
|
|
455
|
+
"LICENSE.md",
|
|
456
|
+
"LICENSE.txt",
|
|
457
|
+
"LICENCE",
|
|
458
|
+
"LICENCE.md",
|
|
459
|
+
];
|
|
460
|
+
let licenseContent;
|
|
461
|
+
for (const file of licenseFiles) {
|
|
462
|
+
if (context.sourceCodeFiles.has(file)) {
|
|
463
|
+
licenseContent = context.sourceCodeFiles.get(file);
|
|
464
|
+
break;
|
|
465
|
+
}
|
|
466
|
+
}
|
|
467
|
+
if (!licenseContent)
|
|
468
|
+
return undefined;
|
|
469
|
+
// Simple license detection patterns
|
|
470
|
+
if (licenseContent.includes("MIT License") ||
|
|
471
|
+
licenseContent.includes("Permission is hereby granted, free of charge")) {
|
|
472
|
+
return "MIT";
|
|
473
|
+
}
|
|
474
|
+
if (licenseContent.includes("Apache License") &&
|
|
475
|
+
licenseContent.includes("2.0")) {
|
|
476
|
+
return "Apache-2.0";
|
|
477
|
+
}
|
|
478
|
+
if (licenseContent.includes("GNU GENERAL PUBLIC LICENSE")) {
|
|
479
|
+
if (licenseContent.includes("Version 3"))
|
|
480
|
+
return "GPL-3.0";
|
|
481
|
+
if (licenseContent.includes("Version 2"))
|
|
482
|
+
return "GPL-2.0";
|
|
483
|
+
return "GPL";
|
|
484
|
+
}
|
|
485
|
+
if (licenseContent.includes("BSD")) {
|
|
486
|
+
if (licenseContent.includes("3-Clause") || licenseContent.includes("New"))
|
|
487
|
+
return "BSD-3-Clause";
|
|
488
|
+
if (licenseContent.includes("2-Clause") ||
|
|
489
|
+
licenseContent.includes("Simplified"))
|
|
490
|
+
return "BSD-2-Clause";
|
|
491
|
+
return "BSD";
|
|
492
|
+
}
|
|
493
|
+
if (licenseContent.includes("ISC License")) {
|
|
494
|
+
return "ISC";
|
|
495
|
+
}
|
|
496
|
+
if (licenseContent.includes("Mozilla Public License")) {
|
|
497
|
+
return "MPL-2.0";
|
|
498
|
+
}
|
|
499
|
+
if (licenseContent.includes("UNLICENSE") ||
|
|
500
|
+
licenseContent.includes("unlicense")) {
|
|
501
|
+
return "Unlicense";
|
|
502
|
+
}
|
|
503
|
+
return "Unknown";
|
|
338
504
|
}
|
|
339
505
|
// ============================================================================
|
|
340
506
|
// Usability Analysis (from UsabilityAssessor)
|
|
@@ -3,9 +3,14 @@
|
|
|
3
3
|
* Evaluates documentation quality and completeness
|
|
4
4
|
*/
|
|
5
5
|
import { DocumentationAssessment } from "../../../lib/assessmentTypes.js";
|
|
6
|
+
import { AssessmentConfiguration } from "../../../lib/assessment/configTypes.js";
|
|
6
7
|
import { BaseAssessor } from "./BaseAssessor.js";
|
|
7
8
|
import { AssessmentContext } from "../AssessmentOrchestrator.js";
|
|
9
|
+
/**
|
|
10
|
+
* @deprecated Use DeveloperExperienceAssessor instead. Will be removed in v2.0.0.
|
|
11
|
+
*/
|
|
8
12
|
export declare class DocumentationAssessor extends BaseAssessor {
|
|
13
|
+
constructor(config: AssessmentConfiguration);
|
|
9
14
|
assess(context: AssessmentContext): Promise<DocumentationAssessment>;
|
|
10
15
|
private analyzeDocumentation;
|
|
11
16
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"DocumentationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/DocumentationAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAKxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,qBAAa,qBAAsB,SAAQ,YAAY;
|
|
1
|
+
{"version":3,"file":"DocumentationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/DocumentationAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAKxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D;;GAEG;AACH,qBAAa,qBAAsB,SAAQ,YAAY;gBACzC,MAAM,EAAE,uBAAuB;IAYrC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;IAyC1E,OAAO,CAAC,oBAAoB;IAuJ5B;;;OAGG;IACH,OAAO,CAAC,yBAAyB;IAuEjC;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAiBhC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAmC9B;;OAEG;IACH,OAAO,CAAC,aAAa;IAKrB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAqB3B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA4C3B,OAAO,CAAC,wBAAwB;IAchC,OAAO,CAAC,eAAe;IAavB,OAAO,CAAC,iBAAiB;IAezB,OAAO,CAAC,cAAc;IAUtB;;;OAGG;IACH,OAAO,CAAC,sBAAsB;IAY9B;;;OAGG;IACH,OAAO,CAAC,mBAAmB;IA4B3B,OAAO,CAAC,4BAA4B;IAmBpC,OAAO,CAAC,mBAAmB;IAyB3B,OAAO,CAAC,uBAAuB;CA+BhC"}
|
|
@@ -3,7 +3,18 @@
|
|
|
3
3
|
* Evaluates documentation quality and completeness
|
|
4
4
|
*/
|
|
5
5
|
import { BaseAssessor } from "./BaseAssessor.js";
|
|
6
|
+
/**
|
|
7
|
+
* @deprecated Use DeveloperExperienceAssessor instead. Will be removed in v2.0.0.
|
|
8
|
+
*/
|
|
6
9
|
export class DocumentationAssessor extends BaseAssessor {
|
|
10
|
+
constructor(config) {
|
|
11
|
+
super(config);
|
|
12
|
+
this.logger.warn("DocumentationAssessor is deprecated. Use DeveloperExperienceAssessor instead. " +
|
|
13
|
+
"This module will be removed in v2.0.0.", {
|
|
14
|
+
module: "DocumentationAssessor",
|
|
15
|
+
replacement: "DeveloperExperienceAssessor",
|
|
16
|
+
});
|
|
17
|
+
}
|
|
7
18
|
async assess(context) {
|
|
8
19
|
this.log("Starting documentation assessment");
|
|
9
20
|
const readmeContent = context.readmeContent || "";
|
|
@@ -13,7 +13,7 @@ export class ErrorHandlingAssessor extends BaseAssessor {
|
|
|
13
13
|
const toolsToTest = this.selectToolsForTesting(context.tools);
|
|
14
14
|
// Parallel tool testing with concurrency limit
|
|
15
15
|
const concurrency = this.config.maxParallelTests ?? 5;
|
|
16
|
-
const limit = createConcurrencyLimit(concurrency);
|
|
16
|
+
const limit = createConcurrencyLimit(concurrency, this.logger);
|
|
17
17
|
this.log(`Testing ${toolsToTest.length} tools for error handling with concurrency limit of ${concurrency}`);
|
|
18
18
|
const allToolTests = await Promise.all(toolsToTest.map((tool) => limit(async () => {
|
|
19
19
|
const toolTests = await this.testToolErrorHandling(tool, context.callTool);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"FunctionalityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/FunctionalityAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAGxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"FunctionalityAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/FunctionalityAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,uBAAuB,EAGxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAU9D,qBAAa,qBAAsB,SAAQ,YAAY;IACrD,OAAO,CAAC,cAAc,CAAwB;IAE9C;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAoCvB,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,uBAAuB,CAAC;YAmI5D,QAAQ;IAiGtB,OAAO,CAAC,qBAAqB;IAmE7B,OAAO,CAAC,kBAAkB;IA4G1B;;;OAGG;IACH,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,uBAAuB,CAe7C;IAEF;;;OAGG;IACH,OAAO,CAAC,mCAAmC;IAsF3C;;OAEG;IACH,OAAO,CAAC,iBAAiB;IAWlB,iBAAiB,CAAC,MAAM,EAAE,GAAG,GAAG,OAAO;IAI9C,OAAO,CAAC,mBAAmB;CA+B5B"}
|
|
@@ -9,6 +9,7 @@ import { ToolClassifier, ToolCategory } from "../ToolClassifier.js";
|
|
|
9
9
|
import { TestDataGenerator } from "../TestDataGenerator.js";
|
|
10
10
|
import { cleanParams } from "../../../utils/paramUtils.js";
|
|
11
11
|
import { resolveRef, normalizeUnionType } from "../../../utils/schemaUtils.js";
|
|
12
|
+
import { DEFAULT_PERFORMANCE_CONFIG } from "../config/performanceConfig.js";
|
|
12
13
|
export class FunctionalityAssessor extends BaseAssessor {
|
|
13
14
|
toolClassifier = new ToolClassifier();
|
|
14
15
|
/**
|
|
@@ -46,14 +47,15 @@ export class FunctionalityAssessor extends BaseAssessor {
|
|
|
46
47
|
const toolsToTest = this.selectToolsForTesting(context.tools);
|
|
47
48
|
// Parallel tool testing with concurrency limit
|
|
48
49
|
const concurrency = this.config.maxParallelTests ?? 5;
|
|
49
|
-
const limit = createConcurrencyLimit(concurrency);
|
|
50
|
+
const limit = createConcurrencyLimit(concurrency, this.logger);
|
|
50
51
|
// Progress tracking for batched events
|
|
52
|
+
// Uses centralized PerformanceConfig values (Issue #37)
|
|
51
53
|
const totalEstimate = toolsToTest.length;
|
|
52
54
|
let completedTests = 0;
|
|
53
55
|
let lastBatchTime = Date.now();
|
|
54
56
|
const startTime = Date.now();
|
|
55
|
-
const BATCH_INTERVAL =
|
|
56
|
-
const BATCH_SIZE =
|
|
57
|
+
const BATCH_INTERVAL = DEFAULT_PERFORMANCE_CONFIG.batchFlushIntervalMs;
|
|
58
|
+
const BATCH_SIZE = DEFAULT_PERFORMANCE_CONFIG.functionalityBatchSize;
|
|
57
59
|
let batchCount = 0;
|
|
58
60
|
const emitProgressBatch = () => {
|
|
59
61
|
if (context.onProgress) {
|
|
@@ -195,6 +197,7 @@ export class FunctionalityAssessor extends BaseAssessor {
|
|
|
195
197
|
};
|
|
196
198
|
}
|
|
197
199
|
catch (error) {
|
|
200
|
+
this.logError(`Tool execution failed: ${tool.name}`, error);
|
|
198
201
|
return {
|
|
199
202
|
toolName: tool.name,
|
|
200
203
|
tested: true,
|
|
@@ -5,6 +5,9 @@
|
|
|
5
5
|
import { MCPSpecComplianceAssessment, AssessmentConfiguration } from "../../../lib/assessmentTypes.js";
|
|
6
6
|
import { BaseAssessor } from "./BaseAssessor.js";
|
|
7
7
|
import { AssessmentContext } from "../AssessmentOrchestrator.js";
|
|
8
|
+
/**
|
|
9
|
+
* @deprecated Use ProtocolComplianceAssessor instead. Will be removed in v2.0.0.
|
|
10
|
+
*/
|
|
8
11
|
export declare class MCPSpecComplianceAssessor extends BaseAssessor {
|
|
9
12
|
private ajv;
|
|
10
13
|
constructor(config: AssessmentConfiguration);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"MCPSpecComplianceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/MCPSpecComplianceAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,2BAA2B,EAM3B,uBAAuB,EAGxB,MAAM,uBAAuB,CAAC;AAO/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D,qBAAa,yBAA0B,SAAQ,YAAY;IACzD,OAAO,CAAC,GAAG,CAAc;gBAEb,MAAM,EAAE,uBAAuB;
|
|
1
|
+
{"version":3,"file":"MCPSpecComplianceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/MCPSpecComplianceAssessor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EACL,2BAA2B,EAM3B,uBAAuB,EAGxB,MAAM,uBAAuB,CAAC;AAO/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAE9D;;GAEG;AACH,qBAAa,yBAA0B,SAAQ,YAAY;IACzD,OAAO,CAAC,GAAG,CAAc;gBAEb,MAAM,EAAE,uBAAuB;IAa3C;;;OAGG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,2BAA2B,CAAC;IAmHvC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAwB9B;;;OAGG;YACW,sBAAsB;IA6BpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAyB/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA0C7B;;;OAGG;YACW,mBAAmB;IAsCjC;;OAEG;IACH,OAAO,CAAC,4BAA4B;IAiBpC;;;OAGG;IACH,OAAO,CAAC,2BAA2B;IA0FnC;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAyFjC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAsB/B;;OAEG;IACH,OAAO,CAAC,sBAAsB;IA4B9B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA2C7B;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAoF5B;;OAEG;IACH,OAAO,CAAC,yBAAyB;IAyBjC;;OAEG;IACH,OAAO,CAAC,6BAA6B;CA0DtC"}
|
|
@@ -4,10 +4,18 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import Ajv from "ajv";
|
|
6
6
|
import { BaseAssessor } from "./BaseAssessor.js";
|
|
7
|
+
/**
|
|
8
|
+
* @deprecated Use ProtocolComplianceAssessor instead. Will be removed in v2.0.0.
|
|
9
|
+
*/
|
|
7
10
|
export class MCPSpecComplianceAssessor extends BaseAssessor {
|
|
8
11
|
ajv;
|
|
9
12
|
constructor(config) {
|
|
10
13
|
super(config);
|
|
14
|
+
this.logger.warn("MCPSpecComplianceAssessor is deprecated. Use ProtocolComplianceAssessor instead. " +
|
|
15
|
+
"This module will be removed in v2.0.0.", {
|
|
16
|
+
module: "MCPSpecComplianceAssessor",
|
|
17
|
+
replacement: "ProtocolComplianceAssessor",
|
|
18
|
+
});
|
|
11
19
|
this.ajv = new Ajv({ allErrors: true });
|
|
12
20
|
}
|
|
13
21
|
/**
|
|
@@ -198,7 +206,9 @@ export class MCPSpecComplianceAssessor extends BaseAssessor {
|
|
|
198
206
|
hasErrors = true;
|
|
199
207
|
const errorMsg = `${tool.name}: ${JSON.stringify(this.ajv.errors)}`;
|
|
200
208
|
errors.push(errorMsg);
|
|
201
|
-
|
|
209
|
+
this.logger.warn(`Invalid schema for tool ${tool.name}`, {
|
|
210
|
+
errors: this.ajv.errors,
|
|
211
|
+
});
|
|
202
212
|
}
|
|
203
213
|
}
|
|
204
214
|
}
|
|
@@ -210,7 +220,9 @@ export class MCPSpecComplianceAssessor extends BaseAssessor {
|
|
|
210
220
|
};
|
|
211
221
|
}
|
|
212
222
|
catch (error) {
|
|
213
|
-
|
|
223
|
+
this.logger.error("Schema compliance check failed", {
|
|
224
|
+
error: String(error),
|
|
225
|
+
});
|
|
214
226
|
return {
|
|
215
227
|
passed: false,
|
|
216
228
|
confidence: "low",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ManifestValidationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ManifestValidationAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,4BAA4B,EAK7B,MAAM,uBAAuB,CAAC;AAM/B,qBAAa,0BAA2B,SAAQ,YAAY;IAC1D;;OAEG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,4BAA4B,CAAC;IA6JxC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAyB9B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAmB/B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAgC/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAiC7B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAiChC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA+CzB;;OAEG;IACH,OAAO,CAAC,YAAY;IAqCpB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA+B1B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA8B7B;;OAEG;YACW,yBAAyB;
|
|
1
|
+
{"version":3,"file":"ManifestValidationAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ManifestValidationAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,KAAK,EACV,4BAA4B,EAK7B,MAAM,uBAAuB,CAAC;AAM/B,qBAAa,0BAA2B,SAAQ,YAAY;IAC1D;;OAEG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,4BAA4B,CAAC;IA6JxC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAyB9B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAmB/B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAgC/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAiC7B;;OAEG;IACH,OAAO,CAAC,wBAAwB;IAiChC;;OAEG;IACH,OAAO,CAAC,iBAAiB;IA+CzB;;OAEG;IACH,OAAO,CAAC,YAAY;IAqCpB;;OAEG;IACH,OAAO,CAAC,kBAAkB;IA+B1B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IA8B7B;;OAEG;YACW,yBAAyB;IAoFvC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAsB/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA0C3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CA+ChC"}
|
|
@@ -412,7 +412,8 @@ export class ManifestValidationAssessor extends BaseAssessor {
|
|
|
412
412
|
try {
|
|
413
413
|
new URL(url);
|
|
414
414
|
}
|
|
415
|
-
catch {
|
|
415
|
+
catch (error) {
|
|
416
|
+
this.logError(`Invalid privacy policy URL format: ${url}`, error);
|
|
416
417
|
results.push({
|
|
417
418
|
url,
|
|
418
419
|
accessible: false,
|
|
@@ -437,8 +438,11 @@ export class ManifestValidationAssessor extends BaseAssessor {
|
|
|
437
438
|
contentType: response.headers.get("content-type") || undefined,
|
|
438
439
|
});
|
|
439
440
|
}
|
|
440
|
-
catch {
|
|
441
|
+
catch (headError) {
|
|
441
442
|
// Try GET request as fallback (some servers reject HEAD)
|
|
443
|
+
this.logger.debug(`HEAD request failed for ${url}, trying GET`, {
|
|
444
|
+
error: headError instanceof Error ? headError.message : String(headError),
|
|
445
|
+
});
|
|
442
446
|
try {
|
|
443
447
|
const controller = new AbortController();
|
|
444
448
|
const timeoutId = setTimeout(() => controller.abort(), 5000);
|
|
@@ -456,6 +460,7 @@ export class ManifestValidationAssessor extends BaseAssessor {
|
|
|
456
460
|
});
|
|
457
461
|
}
|
|
458
462
|
catch (fetchError) {
|
|
463
|
+
this.logError(`Failed to fetch privacy policy URL: ${url}`, fetchError);
|
|
459
464
|
results.push({
|
|
460
465
|
url,
|
|
461
466
|
accessible: false,
|
|
@@ -32,6 +32,7 @@ export declare class PromptAssessor extends BaseAssessor {
|
|
|
32
32
|
private analyzePromptTemplate;
|
|
33
33
|
/**
|
|
34
34
|
* Analyze dynamic content characteristics for enrichment (Issue #9)
|
|
35
|
+
* Enhanced with SanitizationDetector for library-aware detection (Issue #56)
|
|
35
36
|
*/
|
|
36
37
|
private analyzeDynamicContent;
|
|
37
38
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"PromptAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/PromptAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,gBAAgB,EAGjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAa,MAAM,2BAA2B,CAAC;
|
|
1
|
+
{"version":3,"file":"PromptAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/PromptAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,gBAAgB,EAGjB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAa,MAAM,2BAA2B,CAAC;AA8DzE,qBAAa,cAAe,SAAQ,YAAY;IACxC,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,gBAAgB,CAAC;IAqDnE,OAAO,CAAC,uBAAuB;YAajB,UAAU;IAsFxB,OAAO,CAAC,oBAAoB;IAK5B,OAAO,CAAC,kBAAkB;IAa1B,OAAO,CAAC,qBAAqB;YAuBf,mBAAmB;IAwCjC,OAAO,CAAC,6BAA6B;YAqBvB,mBAAmB;IAmDjC,OAAO,CAAC,qBAAqB;YAsCf,sBAAsB;IAqCpC,OAAO,CAAC,qBAAqB;IAe7B,OAAO,CAAC,mBAAmB;IAmC3B,OAAO,CAAC,uBAAuB;IAqC/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAkC7B;;;OAGG;IACH,OAAO,CAAC,qBAAqB;CAsD9B"}
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* - Required vs optional argument handling
|
|
10
10
|
*/
|
|
11
11
|
import { BaseAssessor } from "./BaseAssessor.js";
|
|
12
|
+
import { SanitizationDetector } from "./securityTests/SanitizationDetector.js";
|
|
12
13
|
// AUP violation patterns in prompt descriptions/content
|
|
13
14
|
const AUP_VIOLATION_PATTERNS = [
|
|
14
15
|
// Harmful content generation
|
|
@@ -215,6 +216,7 @@ export class PromptAssessor extends BaseAssessor {
|
|
|
215
216
|
return { success: true, unsafeContent, executionTime };
|
|
216
217
|
}
|
|
217
218
|
catch (error) {
|
|
219
|
+
this.logError(`Prompt execution failed: ${prompt.name}`, error);
|
|
218
220
|
return {
|
|
219
221
|
success: false,
|
|
220
222
|
unsafeContent: false,
|
|
@@ -265,8 +267,11 @@ export class PromptAssessor extends BaseAssessor {
|
|
|
265
267
|
}
|
|
266
268
|
return { vulnerable: false };
|
|
267
269
|
}
|
|
268
|
-
catch {
|
|
270
|
+
catch (error) {
|
|
269
271
|
// Error handling payload is good - not vulnerable
|
|
272
|
+
this.logger.debug(`Injection payload rejected for ${prompt.name} (good)`, {
|
|
273
|
+
error: error instanceof Error ? error.message : String(error),
|
|
274
|
+
});
|
|
270
275
|
return { vulnerable: false };
|
|
271
276
|
}
|
|
272
277
|
}
|
|
@@ -311,8 +316,11 @@ export class PromptAssessor extends BaseAssessor {
|
|
|
311
316
|
// If we got here without error, validation failed
|
|
312
317
|
return false;
|
|
313
318
|
}
|
|
314
|
-
catch {
|
|
319
|
+
catch (error) {
|
|
315
320
|
// Expected - missing required arg should throw
|
|
321
|
+
this.logger.debug(`Missing arg ${arg.name} correctly rejected for ${prompt.name}`, {
|
|
322
|
+
error: error instanceof Error ? error.message : String(error),
|
|
323
|
+
});
|
|
316
324
|
continue;
|
|
317
325
|
}
|
|
318
326
|
}
|
|
@@ -399,6 +407,7 @@ export class PromptAssessor extends BaseAssessor {
|
|
|
399
407
|
}
|
|
400
408
|
/**
|
|
401
409
|
* Analyze dynamic content characteristics for enrichment (Issue #9)
|
|
410
|
+
* Enhanced with SanitizationDetector for library-aware detection (Issue #56)
|
|
402
411
|
*/
|
|
403
412
|
analyzeDynamicContent(prompt) {
|
|
404
413
|
const description = prompt.description || "";
|
|
@@ -409,25 +418,26 @@ export class PromptAssessor extends BaseAssessor {
|
|
|
409
418
|
/\$\{.*\}/i.test(fullText) ||
|
|
410
419
|
/\{[a-zA-Z_][a-zA-Z0-9_]*\}/i.test(fullText) ||
|
|
411
420
|
(prompt.arguments?.length || 0) > 0;
|
|
412
|
-
//
|
|
413
|
-
const
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
if (/validat/i.test(fullText))
|
|
421
|
-
escapingApplied.push("validation");
|
|
422
|
-
if (/filter/i.test(fullText))
|
|
423
|
-
escapingApplied.push("filtering");
|
|
421
|
+
// Issue #56: Use SanitizationDetector for library-aware detection
|
|
422
|
+
const sanitizationDetector = new SanitizationDetector();
|
|
423
|
+
const sanitizationResult = sanitizationDetector.detectFromText(fullText);
|
|
424
|
+
// Combine library detection with generic patterns for escapingApplied
|
|
425
|
+
const escapingApplied = [
|
|
426
|
+
...sanitizationResult.libraries,
|
|
427
|
+
...sanitizationResult.genericPatterns,
|
|
428
|
+
];
|
|
424
429
|
// Infer injection safety from multiple signals
|
|
425
430
|
const hasTypeChecks = prompt.arguments?.some((a) => a.description?.toLowerCase().includes("type") ||
|
|
426
431
|
a.description?.toLowerCase().includes("must be"));
|
|
427
432
|
const hasLengthLimits = prompt.arguments?.some((a) => a.description?.toLowerCase().includes("max") ||
|
|
428
433
|
a.description?.toLowerCase().includes("limit"));
|
|
429
|
-
//
|
|
430
|
-
|
|
434
|
+
// Issue #56: Enhanced injection safety determination
|
|
435
|
+
// Now considers specific libraries (stronger signal) in addition to generic patterns
|
|
436
|
+
const injectionSafe = sanitizationResult.libraries.length > 0 || // Specific library = strong signal
|
|
437
|
+
sanitizationResult.genericPatterns.length >= 2 || // Multiple generic patterns
|
|
438
|
+
hasTypeChecks ||
|
|
439
|
+
hasLengthLimits ||
|
|
440
|
+
false;
|
|
431
441
|
return {
|
|
432
442
|
hasInterpolation,
|
|
433
443
|
injectionSafe,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ProtocolComplianceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ProtocolComplianceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EACL,2BAA2B,EAM3B,uBAAuB,EAGxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAOpE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAmB9D;;;GAGG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC/E,2EAA2E;IAC3E,iBAAiB,CAAC,EAAE;QAClB,mBAAmB,EAAE,aAAa,CAAC;QACnC,kBAAkB,EAAE,aAAa,CAAC;QAClC,uBAAuB,EAAE,aAAa,CAAC;KACxC,CAAC;CACH;AAED,qBAAa,0BAA2B,SAAQ,YAAY,CAAC,4BAA4B,CAAC;IACxF,OAAO,CAAC,GAAG,CAAc;gBAEb,MAAM,EAAE,uBAAuB;IAK3C;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;;OAGG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,4BAA4B,CAAC;IAmIxC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAqB9B;;OAEG;YACW,sBAAsB;IAuBpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAsB/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;
|
|
1
|
+
{"version":3,"file":"ProtocolComplianceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ProtocolComplianceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAEH,OAAO,EACL,2BAA2B,EAM3B,uBAAuB,EAGxB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,gCAAgC,CAAC;AAOpE,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAmB9D;;;GAGG;AACH,MAAM,WAAW,4BAA6B,SAAQ,2BAA2B;IAC/E,2EAA2E;IAC3E,iBAAiB,CAAC,EAAE;QAClB,mBAAmB,EAAE,aAAa,CAAC;QACnC,kBAAkB,EAAE,aAAa,CAAC;QAClC,uBAAuB,EAAE,aAAa,CAAC;KACxC,CAAC;CACH;AAED,qBAAa,0BAA2B,SAAQ,YAAY,CAAC,4BAA4B,CAAC;IACxF,OAAO,CAAC,GAAG,CAAc;gBAEb,MAAM,EAAE,uBAAuB;IAK3C;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;;OAGG;IACG,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,4BAA4B,CAAC;IAmIxC;;OAEG;IACH,OAAO,CAAC,sBAAsB;IAqB9B;;OAEG;YACW,sBAAsB;IAuBpC;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAsB/B;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAwC7B;;OAEG;YACW,mBAAmB;IAiCjC;;OAEG;IACH,OAAO,CAAC,4BAA4B;IAYpC;;OAEG;IACH,OAAO,CAAC,2BAA2B;IAkEnC;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAS7B;;OAEG;YACW,wBAAwB;IA4GtC;;OAEG;YACW,uBAAuB;IA2FrC;;OAEG;YACW,4BAA4B;IAoD1C,OAAO,CAAC,yBAAyB;IAkEjC,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,sBAAsB;IA0B9B,OAAO,CAAC,qBAAqB;IAgC7B,OAAO,CAAC,oBAAoB;IA8E5B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAoC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CAqEhC"}
|
|
@@ -229,7 +229,9 @@ export class ProtocolComplianceAssessor extends BaseAssessor {
|
|
|
229
229
|
hasErrors = true;
|
|
230
230
|
const errorMsg = `${tool.name}: ${JSON.stringify(this.ajv.errors)}`;
|
|
231
231
|
errors.push(errorMsg);
|
|
232
|
-
|
|
232
|
+
this.logger.warn(`Invalid schema for tool ${tool.name}`, {
|
|
233
|
+
errors: this.ajv.errors,
|
|
234
|
+
});
|
|
233
235
|
}
|
|
234
236
|
}
|
|
235
237
|
}
|
|
@@ -240,7 +242,9 @@ export class ProtocolComplianceAssessor extends BaseAssessor {
|
|
|
240
242
|
};
|
|
241
243
|
}
|
|
242
244
|
catch (error) {
|
|
243
|
-
|
|
245
|
+
this.logger.error("Schema compliance check failed", {
|
|
246
|
+
error: String(error),
|
|
247
|
+
});
|
|
244
248
|
return {
|
|
245
249
|
passed: false,
|
|
246
250
|
confidence: "low",
|
|
@@ -11,10 +11,15 @@
|
|
|
11
11
|
*
|
|
12
12
|
* @module assessment/modules/ProtocolConformanceAssessor
|
|
13
13
|
*/
|
|
14
|
+
import { AssessmentConfiguration } from "../../../lib/assessment/configTypes.js";
|
|
14
15
|
import type { ProtocolConformanceAssessment } from "../../../lib/assessment/extendedTypes.js";
|
|
15
16
|
import { BaseAssessor } from "./BaseAssessor.js";
|
|
16
17
|
import { AssessmentContext } from "../AssessmentOrchestrator.js";
|
|
18
|
+
/**
|
|
19
|
+
* @deprecated Use ProtocolComplianceAssessor instead. Will be removed in v2.0.0.
|
|
20
|
+
*/
|
|
17
21
|
export declare class ProtocolConformanceAssessor extends BaseAssessor<ProtocolConformanceAssessment> {
|
|
22
|
+
constructor(config: AssessmentConfiguration);
|
|
18
23
|
/**
|
|
19
24
|
* Select representative tools for testing (first, middle, last for diversity)
|
|
20
25
|
*/
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ProtocolConformanceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ProtocolConformanceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EACV,6BAA6B,EAE9B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAmB9D,qBAAa,2BAA4B,SAAQ,YAAY,CAAC,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"ProtocolConformanceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ProtocolConformanceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,KAAK,EACV,6BAA6B,EAE9B,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAmB9D;;GAEG;AACH,qBAAa,2BAA4B,SAAQ,YAAY,CAAC,6BAA6B,CAAC;gBAC9E,MAAM,EAAE,uBAAuB;IAY3C;;OAEG;IACH,OAAO,CAAC,qBAAqB;IAS7B;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,cAAc;IAItB;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAI3B;;OAEG;IACH,OAAO,CAAC,eAAe;IAIjB,MAAM,CACV,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,6BAA6B,CAAC;IAqCzC;;;;;;;;;OASG;YACW,wBAAwB;IA0HtC;;;;;OAKG;YACW,uBAAuB;IAmGrC;;;;;;;;OAQG;YACW,4BAA4B;IAkD1C;;OAEG;IACH,OAAO,CAAC,yBAAyB;IA6BjC;;OAEG;IACH,OAAO,CAAC,mBAAmB;IAmC3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;CA6ChC"}
|
|
@@ -20,7 +20,18 @@ const VALID_CONTENT_TYPES = [
|
|
|
20
20
|
"resource",
|
|
21
21
|
"resource_link",
|
|
22
22
|
];
|
|
23
|
+
/**
|
|
24
|
+
* @deprecated Use ProtocolComplianceAssessor instead. Will be removed in v2.0.0.
|
|
25
|
+
*/
|
|
23
26
|
export class ProtocolConformanceAssessor extends BaseAssessor {
|
|
27
|
+
constructor(config) {
|
|
28
|
+
super(config);
|
|
29
|
+
this.logger.warn("ProtocolConformanceAssessor is deprecated. Use ProtocolComplianceAssessor instead. " +
|
|
30
|
+
"This module will be removed in v2.0.0.", {
|
|
31
|
+
module: "ProtocolConformanceAssessor",
|
|
32
|
+
replacement: "ProtocolComplianceAssessor",
|
|
33
|
+
});
|
|
34
|
+
}
|
|
24
35
|
/**
|
|
25
36
|
* Select representative tools for testing (first, middle, last for diversity)
|
|
26
37
|
*/
|
|
@@ -147,6 +158,9 @@ export class ProtocolConformanceAssessor extends BaseAssessor {
|
|
|
147
158
|
}
|
|
148
159
|
catch (error) {
|
|
149
160
|
// Tool threw exception instead of returning error response
|
|
161
|
+
this.logger.debug(`Tool ${testTool.name} threw exception instead of error response`, {
|
|
162
|
+
error: error instanceof Error ? error.message : String(error),
|
|
163
|
+
});
|
|
150
164
|
results.push({
|
|
151
165
|
toolName: testTool.name,
|
|
152
166
|
passed: false,
|
|
@@ -254,6 +268,7 @@ export class ProtocolConformanceAssessor extends BaseAssessor {
|
|
|
254
268
|
};
|
|
255
269
|
}
|
|
256
270
|
catch (error) {
|
|
271
|
+
this.logError("Content type validation failed", error);
|
|
257
272
|
return {
|
|
258
273
|
passed: false,
|
|
259
274
|
confidence: "medium",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ResourceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ResourceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,kBAAkB,EAGnB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAoN9D,qBAAa,gBAAiB,SAAQ,YAAY;IAC1C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAgFrE,OAAO,CAAC,yBAAyB;YAiBnB,YAAY;IAoG1B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAY/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;YAsBjB,oBAAoB;
|
|
1
|
+
{"version":3,"file":"ResourceAssessor.d.ts","sourceRoot":"","sources":["../../../../src/services/assessment/modules/ResourceAssessor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EACL,kBAAkB,EAGnB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAoN9D,qBAAa,gBAAiB,SAAQ,YAAY;IAC1C,MAAM,CAAC,OAAO,EAAE,iBAAiB,GAAG,OAAO,CAAC,kBAAkB,CAAC;IAgFrE,OAAO,CAAC,yBAAyB;YAiBnB,YAAY;IAoG1B;;OAEG;IACH,OAAO,CAAC,uBAAuB;IAY/B;;OAEG;IACH,OAAO,CAAC,mBAAmB;IA4B3B;;OAEG;IACH,OAAO,CAAC,uBAAuB;YAsBjB,oBAAoB;IAkGlC,OAAO,CAAC,UAAU;IAsBlB,OAAO,CAAC,kBAAkB;IAM1B,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,wBAAwB;IAIhC;;;OAGG;IACH,OAAO,CAAC,qBAAqB;IAc7B,OAAO,CAAC,yBAAyB;IAYjC,OAAO,CAAC,uBAAuB;IAqB/B,OAAO,CAAC,mBAAmB;IAoC3B,OAAO,CAAC,uBAAuB;CA+DhC"}
|