framework-mcp 1.5.3 β 2.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/commands/speckit.analyze.md +184 -0
- package/.claude/commands/speckit.checklist.md +294 -0
- package/.claude/commands/speckit.clarify.md +181 -0
- package/.claude/commands/speckit.constitution.md +82 -0
- package/.claude/commands/speckit.implement.md +135 -0
- package/.claude/commands/speckit.plan.md +89 -0
- package/.claude/commands/speckit.specify.md +258 -0
- package/.claude/commands/speckit.tasks.md +137 -0
- package/.claude/commands/speckit.taskstoissues.md +30 -0
- package/.do/app.yaml +1 -1
- package/.github/dependabot.yml +15 -0
- package/.github/workflows/ci.yml +31 -2
- package/.specify/memory/constitution.md +50 -0
- package/.specify/scripts/bash/check-prerequisites.sh +166 -0
- package/.specify/scripts/bash/common.sh +156 -0
- package/.specify/scripts/bash/create-new-feature.sh +297 -0
- package/.specify/scripts/bash/setup-plan.sh +61 -0
- package/.specify/scripts/bash/update-agent-context.sh +799 -0
- package/.specify/templates/agent-file-template.md +28 -0
- package/.specify/templates/checklist-template.md +40 -0
- package/.specify/templates/plan-template.md +104 -0
- package/.specify/templates/spec-template.md +115 -0
- package/.specify/templates/tasks-template.md +251 -0
- package/README.md +84 -435
- package/dist/core/safeguard-manager.d.ts.map +1 -1
- package/dist/core/safeguard-manager.js +7105 -2543
- package/dist/core/safeguard-manager.js.map +1 -1
- package/dist/interfaces/http/http-server.d.ts +2 -0
- package/dist/interfaces/http/http-server.d.ts.map +1 -1
- package/dist/interfaces/http/http-server.js +95 -3
- package/dist/interfaces/http/http-server.js.map +1 -1
- package/dist/interfaces/mcp/mcp-server.js +3 -3
- package/dist/shared/types.d.ts +85 -2
- package/dist/shared/types.d.ts.map +1 -1
- package/dist/shared/types.js +132 -1
- package/dist/shared/types.js.map +1 -1
- package/package.json +7 -4
- package/scripts/standardize-prompts.js +325 -0
- package/scripts/validate-capability-prompts.js +110 -0
- package/src/core/safeguard-manager.ts +12387 -2630
- package/src/interfaces/http/http-server.ts +109 -4
- package/src/interfaces/mcp/mcp-server.ts +3 -3
- package/src/shared/types.ts +268 -2
- package/swagger.json +56 -36
- package/CAPABILITY_TRANSFORMATION_SUMMARY.md +0 -158
- package/CIS_CONTROLS_IMPLEMENTATION_PLAN.md +0 -220
- package/COPILOT_INTEGRATION.md +0 -322
- package/DAYS_5_6_COMPLETION_SUMMARY.md +0 -178
- package/DEPLOYMENT_GUIDE.md +0 -334
- package/DEPLOYMENT_SUMMARY_v1.1.3.md +0 -211
- package/MCP_INTEGRATION_GUIDE.md +0 -285
- package/MIGRATION_GUIDE_v1.4.0.md +0 -190
- package/RELEASE_NOTES_v1.1.3.md +0 -321
- package/RELEASE_NOTES_v1.2.0.md +0 -396
- package/RELEASE_NOTES_v1.3.7.md +0 -275
- package/RELEASE_NOTES_v1.4.0.md +0 -178
- package/SAFEGUARDS_VERIFICATION_LOG.md +0 -157
- package/coordinator-bot-compact.txt +0 -33
- package/coordinator-bot-system-prompt.md +0 -192
- package/docs/installation.md +0 -71
- package/scripts/analyze-data-structure.cjs +0 -74
- package/scripts/clean-safeguards-data.cjs +0 -60
- package/scripts/comprehensive-validation.cjs +0 -176
- package/scripts/count-safeguards.cjs +0 -89
- package/scripts/format-safeguards-proper.cjs +0 -44
- package/scripts/validate-formatted-data.cjs +0 -88
- package/test_capability_integration.js +0 -73
- package/test_comprehensive_scenarios.js +0 -192
- package/test_enhanced_detection.js +0 -74
- package/test_integrated_validation.js +0 -112
- package/test_language_update.js +0 -101
- package/test_live_validation.json +0 -12
- package/test_performance_monitoring.js +0 -124
- package/test_runner_automated.js +0 -156
- package/test_suite_comprehensive.js +0 -218
|
@@ -1,192 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Comprehensive test of integrated system with domain validation scenarios
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
console.log("π§ͺ COMPREHENSIVE SYSTEM TEST: Capability Analysis + Domain Validation\n");
|
|
8
|
-
|
|
9
|
-
const comprehensiveTestScenarios = [
|
|
10
|
-
// PASS scenarios - Domain alignment
|
|
11
|
-
{
|
|
12
|
-
category: "β
SHOULD PASS",
|
|
13
|
-
name: "Asset Management Tool β Asset Inventory Safeguard",
|
|
14
|
-
vendor: "AssetMax Pro",
|
|
15
|
-
safeguard: "1.1",
|
|
16
|
-
claimedCapability: "full",
|
|
17
|
-
supportingText: "Our comprehensive asset management platform provides automated discovery of all enterprise devices, maintains detailed hardware and software inventories, tracks ownership and location data, provides real-time asset status monitoring, and includes documented inventory procedures with bi-annual review capabilities.",
|
|
18
|
-
expectedOutcome: {
|
|
19
|
-
toolType: "inventory",
|
|
20
|
-
domainMatch: true,
|
|
21
|
-
validationStatus: "SUPPORTED",
|
|
22
|
-
capabilityAdjusted: false,
|
|
23
|
-
confidence: "High (85-95%)"
|
|
24
|
-
}
|
|
25
|
-
},
|
|
26
|
-
{
|
|
27
|
-
category: "β
SHOULD PASS",
|
|
28
|
-
name: "Identity Management Tool β Account Inventory Safeguard",
|
|
29
|
-
vendor: "IdentityMax",
|
|
30
|
-
safeguard: "5.1",
|
|
31
|
-
claimedCapability: "partial",
|
|
32
|
-
supportingText: "Our identity management system maintains comprehensive user account inventories including privileged accounts, tracks account lifecycle events, provides basic reporting on account status and access patterns with active directory integration and SSO capabilities.",
|
|
33
|
-
expectedOutcome: {
|
|
34
|
-
toolType: "identity_management",
|
|
35
|
-
domainMatch: true,
|
|
36
|
-
validationStatus: "SUPPORTED",
|
|
37
|
-
capabilityAdjusted: false,
|
|
38
|
-
confidence: "High (80-90%)"
|
|
39
|
-
}
|
|
40
|
-
},
|
|
41
|
-
{
|
|
42
|
-
category: "β
SHOULD PASS",
|
|
43
|
-
name: "Vulnerability Management Tool β Vulnerability Process Safeguard",
|
|
44
|
-
vendor: "VulnManager Pro",
|
|
45
|
-
safeguard: "7.1",
|
|
46
|
-
claimedCapability: "full",
|
|
47
|
-
supportingText: "Our vulnerability management solution performs comprehensive security scanning, patch management automation, vulnerability assessment reporting, and penetration testing capabilities with detailed remediation guidance and compliance tracking.",
|
|
48
|
-
expectedOutcome: {
|
|
49
|
-
toolType: "vulnerability_management",
|
|
50
|
-
domainMatch: true,
|
|
51
|
-
validationStatus: "SUPPORTED",
|
|
52
|
-
capabilityAdjusted: false,
|
|
53
|
-
confidence: "High (85-95%)"
|
|
54
|
-
}
|
|
55
|
-
},
|
|
56
|
-
|
|
57
|
-
// DOWNGRADE scenarios - Domain mismatch
|
|
58
|
-
{
|
|
59
|
-
category: "β¬οΈ SHOULD DOWNGRADE",
|
|
60
|
-
name: "Threat Intelligence β Asset Inventory (Wrong Domain)",
|
|
61
|
-
vendor: "ThreatIntel Pro",
|
|
62
|
-
safeguard: "1.1",
|
|
63
|
-
claimedCapability: "full",
|
|
64
|
-
supportingText: "Our threat intelligence service provides comprehensive network scanning, identifies potential security risks across enterprise infrastructure, maintains detailed device databases, and offers complete visibility into network-connected assets with advanced threat correlation capabilities.",
|
|
65
|
-
expectedOutcome: {
|
|
66
|
-
toolType: "threat_intelligence",
|
|
67
|
-
domainMatch: false,
|
|
68
|
-
validationStatus: "QUESTIONABLE",
|
|
69
|
-
capabilityAdjusted: true,
|
|
70
|
-
originalClaim: "full",
|
|
71
|
-
adjustedCapability: "facilitates",
|
|
72
|
-
confidence: "Medium (45-65%)"
|
|
73
|
-
}
|
|
74
|
-
},
|
|
75
|
-
{
|
|
76
|
-
category: "β¬οΈ SHOULD DOWNGRADE",
|
|
77
|
-
name: "Asset Discovery β Account Inventory (Wrong Domain)",
|
|
78
|
-
vendor: "AssetDiscovery Corp",
|
|
79
|
-
safeguard: "5.1",
|
|
80
|
-
claimedCapability: "full",
|
|
81
|
-
supportingText: "Our network discovery tool scans all enterprise systems, identifies user accounts across multiple platforms, maintains detailed account databases, and provides comprehensive visibility into identity infrastructure components.",
|
|
82
|
-
expectedOutcome: {
|
|
83
|
-
toolType: "inventory",
|
|
84
|
-
domainMatch: false,
|
|
85
|
-
validationStatus: "QUESTIONABLE",
|
|
86
|
-
capabilityAdjusted: true,
|
|
87
|
-
originalClaim: "full",
|
|
88
|
-
adjustedCapability: "facilitates",
|
|
89
|
-
confidence: "Medium (40-60%)"
|
|
90
|
-
}
|
|
91
|
-
},
|
|
92
|
-
{
|
|
93
|
-
category: "β¬οΈ SHOULD DOWNGRADE",
|
|
94
|
-
name: "SIEM β Asset Inventory (Wrong Domain)",
|
|
95
|
-
vendor: "SecurityAnalytics Elite",
|
|
96
|
-
safeguard: "1.1",
|
|
97
|
-
claimedCapability: "partial",
|
|
98
|
-
supportingText: "Our SIEM platform provides security analytics, log management, event correlation, and comprehensive asset monitoring with detailed inventory tracking and security orchestration capabilities.",
|
|
99
|
-
expectedOutcome: {
|
|
100
|
-
toolType: "security_analytics",
|
|
101
|
-
domainMatch: false,
|
|
102
|
-
validationStatus: "QUESTIONABLE",
|
|
103
|
-
capabilityAdjusted: true,
|
|
104
|
-
originalClaim: "partial",
|
|
105
|
-
adjustedCapability: "facilitates",
|
|
106
|
-
confidence: "Medium (45-65%)"
|
|
107
|
-
}
|
|
108
|
-
},
|
|
109
|
-
|
|
110
|
-
// NO DOWNGRADE scenarios - Non-implementation capabilities
|
|
111
|
-
{
|
|
112
|
-
category: "β
NO DOWNGRADE",
|
|
113
|
-
name: "Vulnerability Scanner β Asset Inventory (FACILITATES OK)",
|
|
114
|
-
vendor: "VulnScanner Elite",
|
|
115
|
-
safeguard: "1.1",
|
|
116
|
-
claimedCapability: "facilitates",
|
|
117
|
-
supportingText: "Our vulnerability scanning platform enhances existing asset management by providing additional context on device types, operating systems, and software versions discovered during security assessments, helping organizations improve their asset inventory accuracy.",
|
|
118
|
-
expectedOutcome: {
|
|
119
|
-
toolType: "vulnerability_management",
|
|
120
|
-
domainMatch: true,
|
|
121
|
-
validationStatus: "SUPPORTED",
|
|
122
|
-
capabilityAdjusted: false,
|
|
123
|
-
confidence: "High (75-85%)"
|
|
124
|
-
}
|
|
125
|
-
},
|
|
126
|
-
{
|
|
127
|
-
category: "β
NO DOWNGRADE",
|
|
128
|
-
name: "GRC Platform β Asset Inventory (GOVERNANCE OK)",
|
|
129
|
-
vendor: "ComplianceMax",
|
|
130
|
-
safeguard: "1.1",
|
|
131
|
-
claimedCapability: "governance",
|
|
132
|
-
supportingText: "Our GRC platform provides comprehensive policy management, compliance tracking, audit management, and governance workflows for asset inventory processes including documentation, approval workflows, and regulatory compliance reporting.",
|
|
133
|
-
expectedOutcome: {
|
|
134
|
-
toolType: "governance",
|
|
135
|
-
domainMatch: true,
|
|
136
|
-
validationStatus: "SUPPORTED",
|
|
137
|
-
capabilityAdjusted: false,
|
|
138
|
-
confidence: "High (80-90%)"
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
];
|
|
142
|
-
|
|
143
|
-
console.log("π COMPREHENSIVE TEST SCENARIOS");
|
|
144
|
-
console.log("=" .repeat(90));
|
|
145
|
-
|
|
146
|
-
let passCount = 0;
|
|
147
|
-
let downgradeCount = 0;
|
|
148
|
-
let noDowngradeCount = 0;
|
|
149
|
-
|
|
150
|
-
comprehensiveTestScenarios.forEach((scenario, i) => {
|
|
151
|
-
console.log(`\n${i + 1}. ${scenario.name}`);
|
|
152
|
-
console.log(` Category: ${scenario.category}`);
|
|
153
|
-
console.log(` Vendor: ${scenario.vendor}`);
|
|
154
|
-
console.log(` Safeguard: ${scenario.safeguard}`);
|
|
155
|
-
console.log(` Claimed: ${scenario.claimedCapability.toUpperCase()}`);
|
|
156
|
-
console.log(` Expected Tool Type: ${scenario.expectedOutcome.toolType}`);
|
|
157
|
-
console.log(` Expected Domain Match: ${scenario.expectedOutcome.domainMatch}`);
|
|
158
|
-
console.log(` Expected Status: ${scenario.expectedOutcome.validationStatus}`);
|
|
159
|
-
|
|
160
|
-
if (scenario.expectedOutcome.capabilityAdjusted) {
|
|
161
|
-
console.log(` Expected Adjustment: ${scenario.expectedOutcome.originalClaim?.toUpperCase()} β ${scenario.expectedOutcome.adjustedCapability?.toUpperCase()}`);
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
console.log(` Expected Confidence: ${scenario.expectedOutcome.confidence}`);
|
|
165
|
-
console.log(` Text: "${scenario.supportingText.substring(0, 100)}..."`);
|
|
166
|
-
|
|
167
|
-
// Count categories
|
|
168
|
-
if (scenario.category.includes("SHOULD PASS")) passCount++;
|
|
169
|
-
if (scenario.category.includes("SHOULD DOWNGRADE")) downgradeCount++;
|
|
170
|
-
if (scenario.category.includes("NO DOWNGRADE")) noDowngradeCount++;
|
|
171
|
-
|
|
172
|
-
console.log("-".repeat(70));
|
|
173
|
-
});
|
|
174
|
-
|
|
175
|
-
console.log("\nπ TEST SCENARIO SUMMARY:");
|
|
176
|
-
console.log(`β
Should Pass (Domain Aligned): ${passCount} scenarios`);
|
|
177
|
-
console.log(`β¬οΈ Should Downgrade (Domain Mismatch): ${downgradeCount} scenarios`);
|
|
178
|
-
console.log(`π« No Downgrade (Non-Implementation): ${noDowngradeCount} scenarios`);
|
|
179
|
-
console.log(`π Total Scenarios: ${comprehensiveTestScenarios.length}`);
|
|
180
|
-
|
|
181
|
-
console.log("\nπ― VALIDATION LOGIC COVERAGE:");
|
|
182
|
-
console.log("β
Domain alignment validation (inventory β 1.1, identity β 5.1, vuln β 7.1)");
|
|
183
|
-
console.log("β
Domain mismatch detection and auto-downgrade to FACILITATES");
|
|
184
|
-
console.log("β
Non-implementation capability preservation (FACILITATES, GOVERNANCE, VALIDATES)");
|
|
185
|
-
console.log("β
Tool type detection accuracy with safeguard context");
|
|
186
|
-
console.log("β
Confidence scoring based on evidence quality and domain alignment");
|
|
187
|
-
|
|
188
|
-
console.log("\nπ SYSTEM INTEGRATION SUCCESS:");
|
|
189
|
-
console.log("π§ Capability-focused analysis correctly categorizes tool roles");
|
|
190
|
-
console.log("π‘οΈ Domain validation prevents inappropriate FULL/PARTIAL claims");
|
|
191
|
-
console.log("π Enhanced tool detection with context-aware scoring");
|
|
192
|
-
console.log("β¨ Paradigm shift from compliance scoring to capability assessment COMPLETE!");
|
|
@@ -1,74 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Test the enhanced tool type detection with safeguard-specific context weighting
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
console.log("π Testing Enhanced Tool Type Detection with Safeguard Context\n");
|
|
8
|
-
|
|
9
|
-
const testCases = [
|
|
10
|
-
{
|
|
11
|
-
name: "Asset Management Tool (1.1 context)",
|
|
12
|
-
text: "Our comprehensive asset management platform provides automated discovery of all enterprise devices, maintains detailed hardware and software inventories, tracks ownership and location data, provides real-time asset status monitoring, and includes documented inventory procedures with bi-annual review capabilities.",
|
|
13
|
-
safeguard: "1.1",
|
|
14
|
-
expectedToolType: "inventory",
|
|
15
|
-
expectedScore: "High (primary keywords + context bonus)"
|
|
16
|
-
},
|
|
17
|
-
{
|
|
18
|
-
name: "Threat Intelligence Tool (1.1 context)",
|
|
19
|
-
text: "Our threat intelligence service provides comprehensive network scanning, identifies potential security risks across enterprise infrastructure, maintains detailed device databases, and offers complete visibility into network-connected assets with advanced threat correlation capabilities.",
|
|
20
|
-
safeguard: "1.1",
|
|
21
|
-
expectedToolType: "threat_intelligence",
|
|
22
|
-
expectedScore: "High (primary keywords)"
|
|
23
|
-
},
|
|
24
|
-
{
|
|
25
|
-
name: "Identity Management Tool (5.1 context)",
|
|
26
|
-
text: "Our identity management system maintains comprehensive user account inventories including privileged accounts, tracks account lifecycle events, and provides basic reporting on account status and access patterns with active directory integration and SSO capabilities.",
|
|
27
|
-
safeguard: "5.1",
|
|
28
|
-
expectedToolType: "identity_management",
|
|
29
|
-
expectedScore: "High (primary keywords + context bonus)"
|
|
30
|
-
},
|
|
31
|
-
{
|
|
32
|
-
name: "Mixed Keywords - Should prioritize by score",
|
|
33
|
-
text: "Our security platform includes basic inventory discovery capabilities and provides threat intelligence feeds with vulnerability scanning features for comprehensive asset visibility.",
|
|
34
|
-
safeguard: "1.1",
|
|
35
|
-
expectedToolType: "Depends on keyword weights",
|
|
36
|
-
expectedScore: "Variable based on keyword frequency"
|
|
37
|
-
},
|
|
38
|
-
{
|
|
39
|
-
name: "Vulnerability Scanner (7.1 context)",
|
|
40
|
-
text: "Our vulnerability management solution performs comprehensive security scanning, patch management automation, vulnerability assessment reporting, and penetration testing capabilities with detailed remediation guidance.",
|
|
41
|
-
safeguard: "7.1",
|
|
42
|
-
expectedToolType: "vulnerability_management",
|
|
43
|
-
expectedScore: "High (primary keywords + context bonus)"
|
|
44
|
-
}
|
|
45
|
-
];
|
|
46
|
-
|
|
47
|
-
console.log("π ENHANCED TOOL TYPE DETECTION SCENARIOS:\n");
|
|
48
|
-
console.log("=" .repeat(80));
|
|
49
|
-
|
|
50
|
-
testCases.forEach((testCase, i) => {
|
|
51
|
-
console.log(`\n${i + 1}. ${testCase.name}`);
|
|
52
|
-
console.log(` Safeguard Context: ${testCase.safeguard}`);
|
|
53
|
-
console.log(` Expected Tool Type: ${testCase.expectedToolType}`);
|
|
54
|
-
console.log(` Expected Score: ${testCase.expectedScore}`);
|
|
55
|
-
console.log(` Text: "${testCase.text.substring(0, 120)}..."`);
|
|
56
|
-
console.log("-".repeat(60));
|
|
57
|
-
});
|
|
58
|
-
|
|
59
|
-
console.log("\nπ― ENHANCED DETECTION FEATURES:");
|
|
60
|
-
console.log("β
Weighted keyword scoring (Primary: 3 points, Secondary: 1 point)");
|
|
61
|
-
console.log("β
Safeguard-specific context bonuses (+1 for domain alignment)");
|
|
62
|
-
console.log("β
Minimum threshold (β₯2 points) to avoid false positives");
|
|
63
|
-
console.log("β
Comprehensive keyword coverage for all major tool categories");
|
|
64
|
-
console.log("β
Context-aware scoring based on safeguard domain requirements");
|
|
65
|
-
|
|
66
|
-
console.log("\nπ§ CONTEXT BONUSES APPLIED:");
|
|
67
|
-
console.log("β’ Safeguards 1.1-1.2 (Asset Inventory) β inventory tools get +1 bonus");
|
|
68
|
-
console.log("β’ Safeguards 5.1-5.3 (Account Inventory) β identity_management tools get +1 bonus");
|
|
69
|
-
console.log("β’ Safeguards 6.1-6.3 (Authentication) β identity_management tools get +1 bonus");
|
|
70
|
-
console.log("β’ Safeguards 7.1-7.7 (Vulnerability Management) β vulnerability_management tools get +1 bonus");
|
|
71
|
-
|
|
72
|
-
console.log("\nβ
Enhanced tool type detection implementation complete");
|
|
73
|
-
console.log("π More accurate tool categorization for capability analysis");
|
|
74
|
-
console.log("π― Better alignment with safeguard domain requirements");
|
|
@@ -1,112 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Test the integrated capability-focused analysis with domain validation
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
import { readFileSync } from 'fs';
|
|
8
|
-
import { fileURLToPath } from 'url';
|
|
9
|
-
import { dirname, join } from 'path';
|
|
10
|
-
|
|
11
|
-
const __filename = fileURLToPath(import.meta.url);
|
|
12
|
-
const __dirname = dirname(__filename);
|
|
13
|
-
|
|
14
|
-
// Import the server functionality
|
|
15
|
-
const serverPath = join(__dirname, 'dist', 'index.js');
|
|
16
|
-
let server;
|
|
17
|
-
|
|
18
|
-
try {
|
|
19
|
-
server = await import(serverPath);
|
|
20
|
-
} catch (error) {
|
|
21
|
-
console.error('Failed to import server:', error.message);
|
|
22
|
-
process.exit(1);
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
// Test cases for domain validation scenarios
|
|
26
|
-
const testCases = [
|
|
27
|
-
{
|
|
28
|
-
name: "Inventory Tool - FULL Capability (Should PASS)",
|
|
29
|
-
vendor: "AssetTracker Pro",
|
|
30
|
-
safeguard: "1.1",
|
|
31
|
-
claimedCapability: "full",
|
|
32
|
-
supportingText: "Our comprehensive asset management platform provides automated discovery of all enterprise devices, maintains detailed hardware and software inventories, tracks ownership and location data, provides real-time asset status monitoring, and includes documented inventory procedures with bi-annual review capabilities."
|
|
33
|
-
},
|
|
34
|
-
{
|
|
35
|
-
name: "Threat Intelligence Tool - FULL Capability (Should DOWNGRADE to FACILITATES)",
|
|
36
|
-
vendor: "ThreatIntel Pro",
|
|
37
|
-
safeguard: "1.1",
|
|
38
|
-
claimedCapability: "full",
|
|
39
|
-
supportingText: "Our threat intelligence service provides comprehensive network scanning, identifies potential security risks across enterprise infrastructure, maintains detailed device databases, and offers complete visibility into network-connected assets with advanced threat correlation capabilities."
|
|
40
|
-
},
|
|
41
|
-
{
|
|
42
|
-
name: "Identity Tool - PARTIAL Capability (Should PASS for 5.1)",
|
|
43
|
-
vendor: "IdentityMax",
|
|
44
|
-
safeguard: "5.1",
|
|
45
|
-
claimedCapability: "partial",
|
|
46
|
-
supportingText: "Our identity management system maintains comprehensive user account inventories including privileged accounts, tracks account lifecycle events, and provides basic reporting on account status and access patterns."
|
|
47
|
-
},
|
|
48
|
-
{
|
|
49
|
-
name: "Asset Tool on Identity Safeguard (Should DOWNGRADE)",
|
|
50
|
-
vendor: "AssetDiscovery Corp",
|
|
51
|
-
safeguard: "5.1",
|
|
52
|
-
claimedCapability: "full",
|
|
53
|
-
supportingText: "Our network discovery tool scans all enterprise systems, identifies user accounts across multiple platforms, maintains detailed account databases, and provides comprehensive visibility into identity infrastructure components."
|
|
54
|
-
},
|
|
55
|
-
{
|
|
56
|
-
name: "Vulnerability Tool - FACILITATES Capability (Should PASS)",
|
|
57
|
-
vendor: "VulnScanner Elite",
|
|
58
|
-
safeguard: "1.1",
|
|
59
|
-
claimedCapability: "facilitates",
|
|
60
|
-
supportingText: "Our vulnerability scanning platform enhances existing asset management by providing additional context on device types, operating systems, and software versions discovered during security assessments, helping organizations improve their asset inventory accuracy."
|
|
61
|
-
}
|
|
62
|
-
];
|
|
63
|
-
|
|
64
|
-
console.log("π§ͺ Testing Integrated Capability-Focused Analysis with Domain Validation\n");
|
|
65
|
-
console.log("=" .repeat(80));
|
|
66
|
-
|
|
67
|
-
for (let i = 0; i < testCases.length; i++) {
|
|
68
|
-
const testCase = testCases[i];
|
|
69
|
-
console.log(`\nπ TEST ${i + 1}: ${testCase.name}`);
|
|
70
|
-
console.log("-".repeat(60));
|
|
71
|
-
|
|
72
|
-
try {
|
|
73
|
-
// Create the request parameters
|
|
74
|
-
const request = {
|
|
75
|
-
name: "validate_vendor_mapping",
|
|
76
|
-
arguments: {
|
|
77
|
-
vendor_name: testCase.vendor,
|
|
78
|
-
safeguard_id: testCase.safeguard,
|
|
79
|
-
claimed_capability: testCase.claimedCapability,
|
|
80
|
-
supporting_text: testCase.supportingText
|
|
81
|
-
}
|
|
82
|
-
};
|
|
83
|
-
|
|
84
|
-
console.log(`Vendor: ${testCase.vendor}`);
|
|
85
|
-
console.log(`Safeguard: ${testCase.safeguard}`);
|
|
86
|
-
console.log(`Claimed: ${testCase.claimedCapability.toUpperCase()}`);
|
|
87
|
-
console.log(`Text: "${testCase.supportingText.substring(0, 100)}..."`);
|
|
88
|
-
|
|
89
|
-
// This would normally be called through MCP, but we'll simulate the call
|
|
90
|
-
console.log("\nπ Processing...");
|
|
91
|
-
|
|
92
|
-
// For now, log what we would expect to see
|
|
93
|
-
console.log("\nπ EXPECTED BEHAVIOR:");
|
|
94
|
-
if (testCase.name.includes("Should PASS")) {
|
|
95
|
-
console.log("β
VALIDATION STATUS: SUPPORTED");
|
|
96
|
-
console.log("β
NO CAPABILITY DOWNGRADE");
|
|
97
|
-
} else if (testCase.name.includes("DOWNGRADE")) {
|
|
98
|
-
console.log("β οΈ VALIDATION STATUS: QUESTIONABLE");
|
|
99
|
-
console.log("β¬οΈ CAPABILITY DOWNGRADED");
|
|
100
|
-
console.log("π§ REASON: Domain mismatch detected");
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
} catch (error) {
|
|
104
|
-
console.log(`β TEST FAILED: ${error.message}`);
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
console.log("\n" + "=".repeat(80));
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
console.log("\nβ
Integration test scenarios defined");
|
|
111
|
-
console.log("πββοΈ Run 'node dist/index.js' to start server for live testing");
|
|
112
|
-
console.log("π Use Claude Code to execute: validate_vendor_mapping with above test cases");
|
package/test_language_update.js
DELETED
|
@@ -1,101 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Test the updated capability-focused language in user-facing messages
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
console.log("π CAPABILITY-FOCUSED LANGUAGE UPDATE VALIDATION\n");
|
|
8
|
-
|
|
9
|
-
const languageUpdates = [
|
|
10
|
-
{
|
|
11
|
-
category: "π§ Tool Descriptions",
|
|
12
|
-
updates: [
|
|
13
|
-
"analyze_vendor_response: 'Analyze a vendor response to determine their tool capability role'",
|
|
14
|
-
"validate_coverage_claim: 'Validate a vendor's implementation capability claim (FULL/PARTIAL)'",
|
|
15
|
-
"validate_vendor_mapping: 'Validate whether a vendor's claimed capability role is supported by evidence'"
|
|
16
|
-
]
|
|
17
|
-
},
|
|
18
|
-
{
|
|
19
|
-
category: "π Parameter Descriptions",
|
|
20
|
-
updates: [
|
|
21
|
-
"response_text: 'Vendor response text describing their tool capabilities for the safeguard'",
|
|
22
|
-
"claimed_capability: 'Vendor's claimed capability role: full (complete implementation), partial (limited implementation), facilitates (enables/enhances), governance (policies/processes), validates (evidence/reporting)'",
|
|
23
|
-
"supporting_text: 'Vendor's supporting evidence explaining how their tool fulfills the claimed capability role'"
|
|
24
|
-
]
|
|
25
|
-
},
|
|
26
|
-
{
|
|
27
|
-
category: "π·οΈ Domain Validation Messages",
|
|
28
|
-
updates: [
|
|
29
|
-
"Required tool types for FULL/PARTIAL implementation capability (not coverage)",
|
|
30
|
-
"Domain mismatch reasoning uses 'implementation capability' instead of 'coverage'",
|
|
31
|
-
"Auto-downgrade messages focus on capability roles rather than compliance percentages"
|
|
32
|
-
]
|
|
33
|
-
},
|
|
34
|
-
{
|
|
35
|
-
category: "π Validation Feedback",
|
|
36
|
-
updates: [
|
|
37
|
-
"FULL/PARTIAL implementation capability claims (not coverage claims)",
|
|
38
|
-
"Capability role validation instead of compliance scoring",
|
|
39
|
-
"Evidence alignment with claimed capability role",
|
|
40
|
-
"Strengths/gaps identified in capability evidence quality"
|
|
41
|
-
]
|
|
42
|
-
},
|
|
43
|
-
{
|
|
44
|
-
category: "π― Core Terminology Changes",
|
|
45
|
-
changes: [
|
|
46
|
-
"β 'Coverage claim' β β
'Implementation capability claim'",
|
|
47
|
-
"β 'Compliance validation' β β
'Validation reporting'",
|
|
48
|
-
"β 'Element coverage scoring' β β
'Capability role assessment'",
|
|
49
|
-
"β 'Vendor coverage' β β
'Vendor capabilities'",
|
|
50
|
-
"β 'Capability mapping' β β
'Capability role'"
|
|
51
|
-
]
|
|
52
|
-
}
|
|
53
|
-
];
|
|
54
|
-
|
|
55
|
-
console.log("π LANGUAGE TRANSFORMATION SUMMARY");
|
|
56
|
-
console.log("=" .repeat(70));
|
|
57
|
-
|
|
58
|
-
languageUpdates.forEach((update, i) => {
|
|
59
|
-
console.log(`\n${i + 1}. ${update.category}`);
|
|
60
|
-
console.log("-".repeat(50));
|
|
61
|
-
|
|
62
|
-
if (update.updates) {
|
|
63
|
-
update.updates.forEach(item => {
|
|
64
|
-
console.log(` β’ ${item}`);
|
|
65
|
-
});
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
if (update.changes) {
|
|
69
|
-
update.changes.forEach(change => {
|
|
70
|
-
console.log(` ${change}`);
|
|
71
|
-
});
|
|
72
|
-
}
|
|
73
|
-
});
|
|
74
|
-
|
|
75
|
-
console.log("\nπ― CAPABILITY ROLE TAXONOMY (User-Facing):");
|
|
76
|
-
console.log("βββββββββββββββββββ¬ββββββββββββββββββββββββββββββββββββββββββββββ");
|
|
77
|
-
console.log("β Capability Role β Description β");
|
|
78
|
-
console.log("βββββββββββββββββββΌββββββββββββββββββββββββββββββββββββββββββββββ€");
|
|
79
|
-
console.log("β FULL β Complete implementation of safeguard β");
|
|
80
|
-
console.log("β PARTIAL β Limited scope implementation of safeguard β");
|
|
81
|
-
console.log("β FACILITATES β Enables/enhances others' implementation β");
|
|
82
|
-
console.log("β GOVERNANCE β Provides policies/processes/oversight β");
|
|
83
|
-
console.log("β VALIDATES β Provides evidence/audit/reporting β");
|
|
84
|
-
console.log("βββββββββββββββββββ΄ββββββββββββββββββββββββββββββββββββββββββββββ");
|
|
85
|
-
|
|
86
|
-
console.log("\nβ
PARADIGM SHIFT COMPLETION:");
|
|
87
|
-
console.log("π FROM: 'How much compliance coverage does this vendor provide?'");
|
|
88
|
-
console.log("π― TO: 'What capability role does this vendor tool play?'");
|
|
89
|
-
|
|
90
|
-
console.log("\nπ MESSAGING IMPACT:");
|
|
91
|
-
console.log("β’ Tool descriptions clearly explain capability role determination");
|
|
92
|
-
console.log("β’ Parameter descriptions emphasize evidence-based validation");
|
|
93
|
-
console.log("β’ Domain validation messages focus on appropriate tool types");
|
|
94
|
-
console.log("β’ Validation feedback emphasizes capability role alignment");
|
|
95
|
-
console.log("β’ All user-facing text uses capability-focused terminology");
|
|
96
|
-
|
|
97
|
-
console.log("\nπ CAPABILITY-FOCUSED LANGUAGE IMPLEMENTATION COMPLETE!");
|
|
98
|
-
console.log("π All user-facing messages now reflect the tool's true purpose:")
|
|
99
|
-
console.log(" π§ Categorizing vendor tools by their actual capability roles")
|
|
100
|
-
console.log(" π‘οΈ Preventing inappropriate implementation claims")
|
|
101
|
-
console.log(" π Enabling realistic capability planning and vendor selection");
|
|
@@ -1,12 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"method": "tools/call",
|
|
3
|
-
"params": {
|
|
4
|
-
"name": "validate_vendor_mapping",
|
|
5
|
-
"arguments": {
|
|
6
|
-
"vendor_name": "ThreatIntel Pro",
|
|
7
|
-
"safeguard_id": "1.1",
|
|
8
|
-
"claimed_capability": "full",
|
|
9
|
-
"supporting_text": "Our threat intelligence service provides comprehensive network scanning, identifies potential security risks across enterprise infrastructure, maintains detailed device databases, and offers complete visibility into network-connected assets with advanced threat correlation capabilities."
|
|
10
|
-
}
|
|
11
|
-
}
|
|
12
|
-
}
|
|
@@ -1,124 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
|
|
3
|
-
/**
|
|
4
|
-
* Test the performance monitoring and optimization features
|
|
5
|
-
*/
|
|
6
|
-
|
|
7
|
-
console.log("β‘ PERFORMANCE OPTIMIZATION & ERROR HANDLING VALIDATION\n");
|
|
8
|
-
|
|
9
|
-
const performanceFeatures = [
|
|
10
|
-
{
|
|
11
|
-
category: "π Performance Monitoring",
|
|
12
|
-
features: [
|
|
13
|
-
"Tool execution time tracking with rolling averages (last 100 measurements)",
|
|
14
|
-
"Request counting and error rate monitoring",
|
|
15
|
-
"Automatic performance stats logging every 5 minutes in production",
|
|
16
|
-
"Memory-efficient metrics storage with automatic cleanup",
|
|
17
|
-
"Per-tool performance breakdown for optimization insights"
|
|
18
|
-
]
|
|
19
|
-
},
|
|
20
|
-
{
|
|
21
|
-
category: "πΎ Intelligent Caching",
|
|
22
|
-
features: [
|
|
23
|
-
"Safeguard details caching with 5-minute TTL for frequently accessed data",
|
|
24
|
-
"Safeguard list caching with 10-minute TTL for complete lists",
|
|
25
|
-
"Automatic cache cleanup every 10 minutes to prevent memory leaks",
|
|
26
|
-
"Cache hit optimization for repeated requests",
|
|
27
|
-
"Memory-efficient cache keys with timestamp-based invalidation"
|
|
28
|
-
]
|
|
29
|
-
},
|
|
30
|
-
{
|
|
31
|
-
category: "π‘οΈ Enhanced Error Handling",
|
|
32
|
-
features: [
|
|
33
|
-
"Production-friendly error messages with actionable guidance",
|
|
34
|
-
"Comprehensive input validation for all tool parameters",
|
|
35
|
-
"Detailed error logging with tool context and execution time",
|
|
36
|
-
"Graceful degradation for invalid inputs with helpful suggestions",
|
|
37
|
-
"Error categorization with appropriate HTTP-style status feedback"
|
|
38
|
-
]
|
|
39
|
-
},
|
|
40
|
-
{
|
|
41
|
-
category: "π Input Validation & Security",
|
|
42
|
-
features: [
|
|
43
|
-
"Safeguard ID format validation (X.Y pattern matching)",
|
|
44
|
-
"Text input length limits (10,000 characters max, 10 characters min)",
|
|
45
|
-
"Capability value validation against allowed enum values",
|
|
46
|
-
"Vendor name requirement validation",
|
|
47
|
-
"XSS prevention through input sanitization and validation"
|
|
48
|
-
]
|
|
49
|
-
},
|
|
50
|
-
{
|
|
51
|
-
category: "π Production Monitoring",
|
|
52
|
-
features: [
|
|
53
|
-
"Uptime tracking from server start",
|
|
54
|
-
"Total request count across all tools",
|
|
55
|
-
"Error count and error rate monitoring",
|
|
56
|
-
"Per-tool average execution time reporting",
|
|
57
|
-
"Automated performance logging in production environments"
|
|
58
|
-
]
|
|
59
|
-
}
|
|
60
|
-
];
|
|
61
|
-
|
|
62
|
-
console.log("ποΈ PRODUCTION-READY ENHANCEMENTS");
|
|
63
|
-
console.log("=" .repeat(70));
|
|
64
|
-
|
|
65
|
-
performanceFeatures.forEach((category, i) => {
|
|
66
|
-
console.log(`\n${i + 1}. ${category.category}`);
|
|
67
|
-
console.log("-".repeat(50));
|
|
68
|
-
|
|
69
|
-
category.features.forEach((feature, j) => {
|
|
70
|
-
console.log(` ${j + 1}. ${feature}`);
|
|
71
|
-
});
|
|
72
|
-
});
|
|
73
|
-
|
|
74
|
-
console.log("\nβ‘ PERFORMANCE OPTIMIZATION METRICS:");
|
|
75
|
-
console.log("βββββββββββββββββββββββ¬βββββββββββββββββββββββββββββββββββββ");
|
|
76
|
-
console.log("β Optimization β Expected Impact β");
|
|
77
|
-
console.log("βββββββββββββββββββββββΌβββββββββββββββββββββββββββββββββββββ€");
|
|
78
|
-
console.log("β Safeguard Caching β 95%+ faster repeated requests β");
|
|
79
|
-
console.log("β List Caching β 90%+ faster safeguard browsing β");
|
|
80
|
-
console.log("β Input Validation β Early error detection & preventionβ");
|
|
81
|
-
console.log("β Memory Management β Stable long-running performance β");
|
|
82
|
-
console.log("β Error Categorizationβ Improved user experience β");
|
|
83
|
-
console.log("βββββββββββββββββββββββ΄βββββββββββββββββββββββββββββββββββββ");
|
|
84
|
-
|
|
85
|
-
console.log("\nπ‘οΈ ERROR HANDLING IMPROVEMENTS:");
|
|
86
|
-
console.log("β’ β 'Safeguard X.Y not found' β 'Invalid safeguard ID. Use list_available_safeguards'");
|
|
87
|
-
console.log("β’ β 'Unknown tool' β 'Tool not available. Available tools: analyze_vendor_response, ...'");
|
|
88
|
-
console.log("β’ β Generic errors β Specific, actionable error messages with context");
|
|
89
|
-
console.log("β’ β Stack traces β Production-friendly user guidance");
|
|
90
|
-
|
|
91
|
-
console.log("\nπ MONITORING & OBSERVABILITY:");
|
|
92
|
-
console.log("β’ Real-time performance metrics collection");
|
|
93
|
-
console.log("β’ Automatic performance stats logging (production mode)");
|
|
94
|
-
console.log("β’ Memory leak prevention through cache cleanup");
|
|
95
|
-
console.log("β’ Error rate monitoring and alerting capabilities");
|
|
96
|
-
console.log("β’ Tool-specific performance profiling");
|
|
97
|
-
|
|
98
|
-
console.log("\nπ SECURITY & VALIDATION ENHANCEMENTS:");
|
|
99
|
-
console.log("β’ Input sanitization prevents XSS and injection attacks");
|
|
100
|
-
console.log("β’ Length limits prevent DoS through large payloads");
|
|
101
|
-
console.log("β’ Format validation prevents malformed data processing");
|
|
102
|
-
console.log("β’ Enum validation ensures only valid capability values");
|
|
103
|
-
console.log("β’ Graceful error handling prevents information disclosure");
|
|
104
|
-
|
|
105
|
-
console.log("\nπ PRODUCTION DEPLOYMENT READINESS:");
|
|
106
|
-
console.log("β
Performance monitoring and optimization");
|
|
107
|
-
console.log("β
Memory leak prevention and cache management");
|
|
108
|
-
console.log("β
Comprehensive input validation and security");
|
|
109
|
-
console.log("β
Production-friendly error handling and logging");
|
|
110
|
-
console.log("β
Automated monitoring and observability");
|
|
111
|
-
|
|
112
|
-
console.log("\nπ― DEPLOYMENT RECOMMENDATIONS:");
|
|
113
|
-
console.log("β’ Set NODE_ENV=production for automated performance logging");
|
|
114
|
-
console.log("β’ Monitor performance stats logs for optimization opportunities");
|
|
115
|
-
console.log("β’ Configure log aggregation for error tracking and alerting");
|
|
116
|
-
console.log("β’ Set up health checks using list_available_safeguards endpoint");
|
|
117
|
-
console.log("β’ Monitor memory usage for cache efficiency validation");
|
|
118
|
-
|
|
119
|
-
console.log("\n⨠PERFORMANCE OPTIMIZATION & ERROR HANDLING COMPLETE!");
|
|
120
|
-
console.log("π Framework MCP is now production-ready with:");
|
|
121
|
-
console.log(" β‘ Intelligent caching for 90%+ performance improvement");
|
|
122
|
-
console.log(" π‘οΈ Comprehensive security and input validation");
|
|
123
|
-
console.log(" π Real-time monitoring and observability");
|
|
124
|
-
console.log(" π― Production-friendly error handling and user guidance");
|