agent-security-lens 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +10 -0
- package/.mcp/server.json +42 -0
- package/CHANGELOG.md +17 -0
- package/LICENSE +17 -0
- package/PRIVACY.md +37 -0
- package/README.md +150 -0
- package/RELEASE-MANIFEST.json +449 -0
- package/SECURITY.md +24 -0
- package/apps/mcp-server/agent-security-lens-mcp.mjs +441 -0
- package/bin/agent-security-lens.mjs +117 -0
- package/data/ecosystems/agent-candidates.json +230 -0
- package/data/intelligence/components.json +22989 -0
- package/data/intelligence/security-evaluation-standard.json +221 -0
- package/data/recommendations/core/recommendations.json +256 -0
- package/data/trust/signal-taxonomy.json +107 -0
- package/docs/asl-agent-component-safety-standard-v0.2.md +56 -0
- package/examples/dot-hermes/.hermes/config.json +17 -0
- package/examples/dot-openclaw/.openclaw/openclaw.json +17 -0
- package/examples/hermes-like/.env.example +2 -0
- package/examples/hermes-like/config.json +37 -0
- package/examples/hermes-like/optional-mcps/github-tools.json +8 -0
- package/examples/hermes-like/skills/openclaw-imports/browser-skill/SKILL.md +8 -0
- package/examples/openclaw-like/.env.example +2 -0
- package/examples/openclaw-like/AGENTS.md +7 -0
- package/examples/openclaw-like/openclaw.json +28 -0
- package/examples/openclaw-like/workspace/skills/browser-control/SKILL.md +8 -0
- package/llms.txt +25 -0
- package/package.json +50 -0
- package/profiles/generic-agent/profile.json +19 -0
- package/profiles/hermes-like/profile.json +23 -0
- package/profiles/mcp-server/profile.json +18 -0
- package/profiles/openclaw-like/profile.json +22 -0
- package/profiles/skill-runtime/profile.json +19 -0
- package/rule-packs/core/rules.json +82 -0
- package/rule-packs/hermes/rules.json +44 -0
- package/rule-packs/mcp/rules.json +65 -0
- package/rule-packs/openclaw/rules.json +46 -0
- package/rule-packs/skills/rules.json +45 -0
- package/schemas/agent-install-decision.schema.json +432 -0
- package/schemas/agent-usage-event.schema.json +45 -0
- package/schemas/assessment-result.schema.json +361 -0
- package/schemas/comparison-result.schema.json +113 -0
- package/schemas/component-alternative-graph.schema.json +187 -0
- package/schemas/component-intelligence.schema.json +93 -0
- package/schemas/decision-feedback.schema.json +49 -0
- package/schemas/ecosystem-candidate-registry.schema.json +98 -0
- package/schemas/profile.schema.json +65 -0
- package/schemas/recommendation-pack.schema.json +114 -0
- package/schemas/rule-pack.schema.json +113 -0
- package/schemas/trust-signal-taxonomy.schema.json +68 -0
- package/scripts/verify-examples.mjs +121 -0
- package/scripts/verify-mcp-server.mjs +278 -0
- package/scripts/verify-registry.mjs +264 -0
- package/server.json +42 -0
- package/src/assessment/assess.mjs +108 -0
- package/src/assessment/discover-targets.mjs +127 -0
- package/src/assessment/risk-domains.mjs +83 -0
- package/src/assessment/summarize.mjs +57 -0
- package/src/core/files.mjs +74 -0
- package/src/intelligence/cloud-client.mjs +260 -0
- package/src/intelligence/component-intelligence.mjs +358 -0
- package/src/intelligence/decision-engine.mjs +772 -0
- package/src/intelligence/finding-context.mjs +180 -0
- package/src/intelligence/safety-score-v0.2.mjs +294 -0
- package/src/observations/json-observations.mjs +211 -0
- package/src/observations/observation-rules.mjs +157 -0
- package/src/profiles/load-profiles.mjs +130 -0
- package/src/recommendations/component-alternative-graph.mjs +94 -0
- package/src/recommendations/load-recommendations.mjs +17 -0
- package/src/recommendations/match-recommendations.mjs +79 -0
- package/src/report/comparison-console.mjs +71 -0
- package/src/report/console.mjs +103 -0
- package/src/report/markdown.mjs +145 -0
- package/src/results/compare-results.mjs +106 -0
- package/src/results/save-result.mjs +29 -0
- package/src/rules/load-rules.mjs +22 -0
- package/src/rules/match-rules.mjs +99 -0
- package/src/rules/supersedes.mjs +39 -0
- package/src/store/assessment-store.mjs +78 -0
- package/src/trust/derive-trust-signals.mjs +73 -0
- package/src/trust/load-trust-signals.mjs +17 -0
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { assess } from "../src/assessment/assess.mjs";
|
|
4
|
+
import { compareAssessmentResults } from "../src/results/compare-results.mjs";
|
|
5
|
+
|
|
6
|
+
const EXAMPLES = [
|
|
7
|
+
{
|
|
8
|
+
name: "openclaw-like",
|
|
9
|
+
target: "./examples/openclaw-like",
|
|
10
|
+
profile: "openclaw-like",
|
|
11
|
+
expectedProfile: "openclaw-like",
|
|
12
|
+
minFindings: 8
|
|
13
|
+
},
|
|
14
|
+
{
|
|
15
|
+
name: "hermes-like",
|
|
16
|
+
target: "./examples/hermes-like",
|
|
17
|
+
profile: "hermes-like",
|
|
18
|
+
expectedProfile: "hermes-like",
|
|
19
|
+
minFindings: 8
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
name: "dot-openclaw autodetect",
|
|
23
|
+
target: "./examples/dot-openclaw",
|
|
24
|
+
expectedProfile: "openclaw-like",
|
|
25
|
+
minFindings: 3
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
name: "dot-hermes autodetect",
|
|
29
|
+
target: "./examples/dot-hermes",
|
|
30
|
+
expectedProfile: "hermes-like",
|
|
31
|
+
minFindings: 3
|
|
32
|
+
}
|
|
33
|
+
];
|
|
34
|
+
|
|
35
|
+
let failed = false;
|
|
36
|
+
const results = new Map();
|
|
37
|
+
|
|
38
|
+
for (const example of EXAMPLES) {
|
|
39
|
+
const result = await assess({
|
|
40
|
+
targetPath: example.target,
|
|
41
|
+
requestedProfile: example.profile
|
|
42
|
+
});
|
|
43
|
+
results.set(example.name, result);
|
|
44
|
+
|
|
45
|
+
const findingCount = result.findings.length;
|
|
46
|
+
const profileIds = result.profiles.map((profile) => profile.id);
|
|
47
|
+
const hasRecommendations = result.findings.every((finding) => {
|
|
48
|
+
return finding.recommended_actions.length && finding.recommended_alternatives.length;
|
|
49
|
+
});
|
|
50
|
+
const hasLineage =
|
|
51
|
+
result.assessment.id &&
|
|
52
|
+
result.lineage.profile_selection.selected_profile &&
|
|
53
|
+
result.lineage.algorithms.length &&
|
|
54
|
+
result.rule_packs.length &&
|
|
55
|
+
result.recommendation_packs.length &&
|
|
56
|
+
result.trust_signal_taxonomies.length;
|
|
57
|
+
const hasStructuredRecommendations = result.findings.every((finding) => {
|
|
58
|
+
return finding.recommendations?.length && finding.recommendations[0].one_step_commands?.length;
|
|
59
|
+
});
|
|
60
|
+
const hasTrustSignals =
|
|
61
|
+
result.trust_signals.length &&
|
|
62
|
+
result.summary.trust_signal_summary?.total === result.trust_signals.length;
|
|
63
|
+
|
|
64
|
+
console.log(
|
|
65
|
+
`${example.name}: ${findingCount} findings, trust score ${result.summary.trust_score}, profiles ${profileIds.join(", ")}`
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
if (!profileIds.includes(example.expectedProfile)) {
|
|
69
|
+
console.error(`Expected ${example.name} to include profile ${example.expectedProfile}.`);
|
|
70
|
+
failed = true;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if (findingCount < example.minFindings) {
|
|
74
|
+
console.error(`Expected at least ${example.minFindings} findings for ${example.name}.`);
|
|
75
|
+
failed = true;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if (!hasRecommendations) {
|
|
79
|
+
console.error(`Expected every finding to include actions and alternatives for ${example.name}.`);
|
|
80
|
+
failed = true;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
if (!hasLineage) {
|
|
84
|
+
console.error(`Expected assessment lineage and rule pack metadata for ${example.name}.`);
|
|
85
|
+
failed = true;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (!hasStructuredRecommendations) {
|
|
89
|
+
console.error(`Expected every finding to include structured recommendations for ${example.name}.`);
|
|
90
|
+
failed = true;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
if (!hasTrustSignals) {
|
|
94
|
+
console.error(`Expected trust signals and trust signal summary for ${example.name}.`);
|
|
95
|
+
failed = true;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
const comparison = compareAssessmentResults(
|
|
100
|
+
results.get("openclaw-like"),
|
|
101
|
+
results.get("dot-openclaw autodetect")
|
|
102
|
+
);
|
|
103
|
+
|
|
104
|
+
if (!Number.isInteger(comparison.score.delta)) {
|
|
105
|
+
console.error("Expected comparison score delta to be an integer.");
|
|
106
|
+
failed = true;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
if (
|
|
110
|
+
comparison.finding_counts.added +
|
|
111
|
+
comparison.finding_counts.resolved +
|
|
112
|
+
comparison.finding_counts.persistent ===
|
|
113
|
+
0
|
|
114
|
+
) {
|
|
115
|
+
console.error("Expected comparison to include finding delta data.");
|
|
116
|
+
failed = true;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
if (failed) {
|
|
120
|
+
process.exitCode = 1;
|
|
121
|
+
}
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { spawn } from "node:child_process";
|
|
4
|
+
import { existsSync } from "node:fs";
|
|
5
|
+
import { join } from "node:path";
|
|
6
|
+
|
|
7
|
+
const child = spawn(process.execPath, ["./apps/mcp-server/agent-security-lens-mcp.mjs"], {
|
|
8
|
+
cwd: process.cwd(),
|
|
9
|
+
env: { ...process.env, ASL_MODE: "local" },
|
|
10
|
+
stdio: ["pipe", "pipe", "pipe"]
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
let output = "";
|
|
14
|
+
let errorOutput = "";
|
|
15
|
+
child.stdout.on("data", (chunk) => {
|
|
16
|
+
output += chunk.toString("utf8");
|
|
17
|
+
});
|
|
18
|
+
child.stderr.on("data", (chunk) => {
|
|
19
|
+
errorOutput += chunk.toString("utf8");
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
function send(message) {
|
|
23
|
+
child.stdin.write(`${JSON.stringify(message)}\n`);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
send({
|
|
27
|
+
jsonrpc: "2.0",
|
|
28
|
+
id: 1,
|
|
29
|
+
method: "initialize",
|
|
30
|
+
params: { protocolVersion: "2024-11-05", capabilities: {}, clientInfo: { name: "smoke", version: "0.1.0" } }
|
|
31
|
+
});
|
|
32
|
+
send({ jsonrpc: "2.0", id: 2, method: "tools/list", params: {} });
|
|
33
|
+
send({
|
|
34
|
+
jsonrpc: "2.0",
|
|
35
|
+
id: 4,
|
|
36
|
+
method: "tools/call",
|
|
37
|
+
params: {
|
|
38
|
+
name: "get_install_policy",
|
|
39
|
+
arguments: {}
|
|
40
|
+
}
|
|
41
|
+
});
|
|
42
|
+
send({
|
|
43
|
+
jsonrpc: "2.0",
|
|
44
|
+
id: 5,
|
|
45
|
+
method: "tools/call",
|
|
46
|
+
params: {
|
|
47
|
+
name: "get_intelligence_status",
|
|
48
|
+
arguments: {}
|
|
49
|
+
}
|
|
50
|
+
});
|
|
51
|
+
send({
|
|
52
|
+
jsonrpc: "2.0",
|
|
53
|
+
id: 3,
|
|
54
|
+
method: "tools/call",
|
|
55
|
+
params: {
|
|
56
|
+
name: "review_before_install",
|
|
57
|
+
arguments: {
|
|
58
|
+
component_name: "filesystem",
|
|
59
|
+
component_type: "mcp",
|
|
60
|
+
install_command: "npx -y @modelcontextprotocol/server-filesystem ."
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
});
|
|
64
|
+
send({
|
|
65
|
+
jsonrpc: "2.0",
|
|
66
|
+
id: 6,
|
|
67
|
+
method: "tools/call",
|
|
68
|
+
params: {
|
|
69
|
+
name: "review_before_install",
|
|
70
|
+
arguments: {
|
|
71
|
+
component_name: "mcp-chrome",
|
|
72
|
+
component_type: "mcp",
|
|
73
|
+
source_url: "https://github.com/hangwin/mcp-chrome",
|
|
74
|
+
planned_use: "Browser automation for autonomous web tasks.",
|
|
75
|
+
requested_permissions: ["browser-access", "network-access"],
|
|
76
|
+
submit_if_unknown: true
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
});
|
|
80
|
+
send({
|
|
81
|
+
jsonrpc: "2.0",
|
|
82
|
+
id: 7,
|
|
83
|
+
method: "tools/call",
|
|
84
|
+
params: {
|
|
85
|
+
name: "report_install_outcome",
|
|
86
|
+
arguments: {
|
|
87
|
+
component_name: "filesystem",
|
|
88
|
+
component_type: "mcp",
|
|
89
|
+
decision: "allow_with_restrictions",
|
|
90
|
+
outcome: "restriction_applied",
|
|
91
|
+
restriction_applied: true
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
});
|
|
95
|
+
send({
|
|
96
|
+
jsonrpc: "2.0",
|
|
97
|
+
id: 8,
|
|
98
|
+
method: "tools/call",
|
|
99
|
+
params: {
|
|
100
|
+
name: "submit_decision_feedback",
|
|
101
|
+
arguments: {
|
|
102
|
+
component_name: "filesystem",
|
|
103
|
+
component_type: "mcp",
|
|
104
|
+
decision: "allow_with_restrictions",
|
|
105
|
+
feedback_type: "helpful",
|
|
106
|
+
rating: 5
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
const responseDeadline = Date.now() + 5000;
|
|
112
|
+
while (Date.now() < responseDeadline) {
|
|
113
|
+
const responseIds = new Set(
|
|
114
|
+
output
|
|
115
|
+
.split(/\r?\n/)
|
|
116
|
+
.filter(Boolean)
|
|
117
|
+
.map((line) => {
|
|
118
|
+
try {
|
|
119
|
+
return JSON.parse(line).id;
|
|
120
|
+
} catch {
|
|
121
|
+
return null;
|
|
122
|
+
}
|
|
123
|
+
})
|
|
124
|
+
);
|
|
125
|
+
if ([1, 2, 3, 4, 5, 6, 7, 8].every((id) => responseIds.has(id))) break;
|
|
126
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
127
|
+
}
|
|
128
|
+
child.kill();
|
|
129
|
+
|
|
130
|
+
const lines = output
|
|
131
|
+
.split(/\r?\n/)
|
|
132
|
+
.filter(Boolean)
|
|
133
|
+
.map((line) => JSON.parse(line));
|
|
134
|
+
|
|
135
|
+
const toolList = lines.find((line) => line.id === 2);
|
|
136
|
+
const review = lines.find((line) => line.id === 3);
|
|
137
|
+
if (!toolList?.result?.tools?.some((tool) => tool.name === "review_before_install")) {
|
|
138
|
+
console.error("MCP smoke failed: review_before_install not listed");
|
|
139
|
+
console.error(output || errorOutput);
|
|
140
|
+
process.exit(1);
|
|
141
|
+
}
|
|
142
|
+
if (!toolList?.result?.tools?.some((tool) => tool.name === "get_install_policy")) {
|
|
143
|
+
console.error("MCP smoke failed: get_install_policy not listed");
|
|
144
|
+
console.error(output || errorOutput);
|
|
145
|
+
process.exit(1);
|
|
146
|
+
}
|
|
147
|
+
if (!toolList?.result?.tools?.some((tool) => tool.name === "get_intelligence_status")) {
|
|
148
|
+
console.error("MCP smoke failed: get_intelligence_status not listed");
|
|
149
|
+
console.error(output || errorOutput);
|
|
150
|
+
process.exit(1);
|
|
151
|
+
}
|
|
152
|
+
if (!toolList?.result?.tools?.some((tool) => tool.name === "report_install_outcome")) {
|
|
153
|
+
console.error("MCP smoke failed: report_install_outcome not listed");
|
|
154
|
+
console.error(output || errorOutput);
|
|
155
|
+
process.exit(1);
|
|
156
|
+
}
|
|
157
|
+
if (!toolList?.result?.tools?.some((tool) => tool.name === "submit_decision_feedback")) {
|
|
158
|
+
console.error("MCP smoke failed: submit_decision_feedback not listed");
|
|
159
|
+
console.error(output || errorOutput);
|
|
160
|
+
process.exit(1);
|
|
161
|
+
}
|
|
162
|
+
if (!toolList?.result?.tools?.some((tool) => tool.name === "get_research_status")) {
|
|
163
|
+
console.error("MCP smoke failed: get_research_status not listed");
|
|
164
|
+
console.error(output || errorOutput);
|
|
165
|
+
process.exit(1);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
const reviewText = review?.result?.content?.[0]?.text || "";
|
|
169
|
+
if (!reviewText.includes("ask_user") || !reviewText.includes("filesystem-write")) {
|
|
170
|
+
console.error("MCP smoke failed: review result did not include expected decision and risk");
|
|
171
|
+
console.error(output || errorOutput);
|
|
172
|
+
process.exit(1);
|
|
173
|
+
}
|
|
174
|
+
const reviewJson = JSON.parse(reviewText);
|
|
175
|
+
if (
|
|
176
|
+
reviewJson.agent_decision_contract?.contract_version !== "asl-agent-decision-contract@0.2.0" ||
|
|
177
|
+
reviewJson.agent_decision_contract?.automatic_install_allowed !== false ||
|
|
178
|
+
reviewJson.component?.curated_baseline !== true ||
|
|
179
|
+
reviewJson.component?.reviewed !== false
|
|
180
|
+
) {
|
|
181
|
+
console.error("MCP smoke failed: review result did not include expected agent decision contract");
|
|
182
|
+
console.error(output || errorOutput);
|
|
183
|
+
process.exit(1);
|
|
184
|
+
}
|
|
185
|
+
if (!reviewJson.one_step_action || reviewJson.one_step_action.action_type !== "ask_user_before_install") {
|
|
186
|
+
console.error("MCP smoke failed: one_step_action was not machine executable");
|
|
187
|
+
console.error(output || errorOutput);
|
|
188
|
+
process.exit(1);
|
|
189
|
+
}
|
|
190
|
+
if (!reviewJson.agent_actions?.some((action) => action.id === "report-install-outcome" && action.tool === "report_install_outcome")) {
|
|
191
|
+
console.error("MCP smoke failed: agent action lifecycle did not require outcome reporting");
|
|
192
|
+
console.error(output || errorOutput);
|
|
193
|
+
process.exit(1);
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
const policy = lines.find((line) => line.id === 4);
|
|
197
|
+
const policyText = policy?.result?.content?.[0]?.text || "";
|
|
198
|
+
if (!policyText.includes("review_before_install") || !policyText.includes("agent_decision_contract")) {
|
|
199
|
+
console.error("MCP smoke failed: install policy did not include expected agent behavior");
|
|
200
|
+
console.error(output || errorOutput);
|
|
201
|
+
process.exit(1);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const status = lines.find((line) => line.id === 5);
|
|
205
|
+
const statusText = status?.result?.content?.[0]?.text || "";
|
|
206
|
+
if (!statusText.includes('"mode": "local"') || !statusText.includes("offline fallback")) {
|
|
207
|
+
console.error("MCP smoke failed: intelligence status did not include expected local mode");
|
|
208
|
+
console.error(output || errorOutput);
|
|
209
|
+
process.exit(1);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
const candidateReview = lines.find((line) => line.id === 6);
|
|
213
|
+
const candidateText = candidateReview?.result?.content?.[0]?.text || "";
|
|
214
|
+
const candidateJson = JSON.parse(candidateText);
|
|
215
|
+
const candidateCatalogAvailable = existsSync(
|
|
216
|
+
join(process.cwd(), "data", "intelligence", "candidates", "2026-06-07-component-catalog.json")
|
|
217
|
+
);
|
|
218
|
+
if (candidateCatalogAvailable) {
|
|
219
|
+
if (
|
|
220
|
+
candidateJson.component?.intelligence_state !== "monitored" ||
|
|
221
|
+
candidateJson.intelligence_coverage?.source !== "monitored_catalog" ||
|
|
222
|
+
candidateJson.decision !== "ask_user" ||
|
|
223
|
+
candidateJson.agent_decision_contract?.research_status_required_before_retry !== true ||
|
|
224
|
+
candidateJson.recommended_alternatives?.length !== 0 ||
|
|
225
|
+
candidateJson.alternative_coverage?.status !== "not_applicable"
|
|
226
|
+
) {
|
|
227
|
+
console.error("MCP smoke failed: candidate intelligence path did not return expected contract");
|
|
228
|
+
console.error(output || errorOutput);
|
|
229
|
+
process.exit(1);
|
|
230
|
+
}
|
|
231
|
+
if (
|
|
232
|
+
!candidateJson.catalog_research_submission?.queued ||
|
|
233
|
+
candidateJson.catalog_research_submission?.research_task?.next_agent_action?.automatic_install_retry_allowed !== false
|
|
234
|
+
) {
|
|
235
|
+
console.error("MCP smoke failed: cataloged candidate was not queued for ASL research");
|
|
236
|
+
console.error(output || errorOutput);
|
|
237
|
+
process.exit(1);
|
|
238
|
+
}
|
|
239
|
+
} else {
|
|
240
|
+
if (
|
|
241
|
+
candidateJson.component?.intelligence_state !== "unknown" ||
|
|
242
|
+
candidateJson.intelligence_coverage?.source !== "submitted_metadata_inference" ||
|
|
243
|
+
candidateJson.agent_decision_contract?.blocks_install !== true ||
|
|
244
|
+
candidateJson.agent_decision_contract?.research_status_required_before_retry !== true ||
|
|
245
|
+
candidateJson.recommended_alternatives?.length !== 0 ||
|
|
246
|
+
candidateJson.alternative_coverage?.status !== "not_applicable"
|
|
247
|
+
) {
|
|
248
|
+
console.error("MCP smoke failed: public fallback unknown-component path did not return expected contract");
|
|
249
|
+
console.error(output || errorOutput);
|
|
250
|
+
process.exit(1);
|
|
251
|
+
}
|
|
252
|
+
if (
|
|
253
|
+
!candidateJson.unknown_component?.submission?.queued ||
|
|
254
|
+
candidateJson.unknown_component?.submission?.research_task?.next_agent_action?.automatic_install_retry_allowed !== false
|
|
255
|
+
) {
|
|
256
|
+
console.error("MCP smoke failed: unknown public component was not queued for ASL research");
|
|
257
|
+
console.error(output || errorOutput);
|
|
258
|
+
process.exit(1);
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const outcome = lines.find((line) => line.id === 7);
|
|
263
|
+
const outcomeText = outcome?.result?.content?.[0]?.text || "";
|
|
264
|
+
if (!outcomeText.includes('"source": "local_fallback"') && !outcomeText.includes('"source": "asl_cloud"')) {
|
|
265
|
+
console.error("MCP smoke failed: install outcome was not recorded");
|
|
266
|
+
console.error(output || errorOutput);
|
|
267
|
+
process.exit(1);
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
const feedback = lines.find((line) => line.id === 8);
|
|
271
|
+
const feedbackText = feedback?.result?.content?.[0]?.text || "";
|
|
272
|
+
if (!feedbackText.includes('"source": "local_fallback"') && !feedbackText.includes('"source": "asl_cloud"')) {
|
|
273
|
+
console.error("MCP smoke failed: decision feedback was not recorded");
|
|
274
|
+
console.error(output || errorOutput);
|
|
275
|
+
process.exit(1);
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
console.log("mcp server: tools/list and review_before_install checked");
|
|
@@ -0,0 +1,264 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
import { readdir, readFile } from "node:fs/promises";
|
|
4
|
+
import { dirname, join } from "node:path";
|
|
5
|
+
import { fileURLToPath } from "node:url";
|
|
6
|
+
|
|
7
|
+
const ROOT = dirname(dirname(fileURLToPath(import.meta.url)));
|
|
8
|
+
const VALID_SEVERITIES = new Set(["critical", "high", "medium", "low", "info"]);
|
|
9
|
+
const VALID_CATEGORIES = new Set([
|
|
10
|
+
"execution-risk",
|
|
11
|
+
"remote-access-risk",
|
|
12
|
+
"data-exposure-risk",
|
|
13
|
+
"supply-chain-risk",
|
|
14
|
+
"persistence-automation-risk"
|
|
15
|
+
]);
|
|
16
|
+
|
|
17
|
+
async function readJson(path) {
|
|
18
|
+
return JSON.parse(await readFile(path, "utf8"));
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
async function listJsonFiles(dir) {
|
|
22
|
+
const entries = await readdir(dir, { withFileTypes: true });
|
|
23
|
+
const files = [];
|
|
24
|
+
for (const entry of entries) {
|
|
25
|
+
const path = join(dir, entry.name);
|
|
26
|
+
if (entry.isDirectory()) {
|
|
27
|
+
files.push(...(await listJsonFiles(path)));
|
|
28
|
+
} else if (entry.name.endsWith(".json")) {
|
|
29
|
+
files.push(path);
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
return files;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function requireString(item, field, label, failures) {
|
|
36
|
+
if (!item[field] || typeof item[field] !== "string") {
|
|
37
|
+
failures.push(`${label} missing string field: ${field}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function requireArray(item, field, label, failures) {
|
|
42
|
+
if (!Array.isArray(item[field])) {
|
|
43
|
+
failures.push(`${label} missing array field: ${field}`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const profileFiles = await listJsonFiles(join(ROOT, "profiles"));
|
|
48
|
+
const rulePackFiles = await listJsonFiles(join(ROOT, "rule-packs"));
|
|
49
|
+
const recommendationPackFiles = await listJsonFiles(join(ROOT, "data", "recommendations"));
|
|
50
|
+
const ecosystemFiles = await listJsonFiles(join(ROOT, "data", "ecosystems"));
|
|
51
|
+
const trustFiles = await listJsonFiles(join(ROOT, "data", "trust"));
|
|
52
|
+
const profiles = [];
|
|
53
|
+
const rulePacks = [];
|
|
54
|
+
const recommendationPacks = [];
|
|
55
|
+
const ecosystemRegistries = [];
|
|
56
|
+
const trustRegistries = [];
|
|
57
|
+
const failures = [];
|
|
58
|
+
|
|
59
|
+
for (const file of profileFiles) {
|
|
60
|
+
profiles.push({ file, data: await readJson(file) });
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
for (const file of rulePackFiles) {
|
|
64
|
+
rulePacks.push({ file, data: await readJson(file) });
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
for (const file of recommendationPackFiles) {
|
|
68
|
+
recommendationPacks.push({ file, data: await readJson(file) });
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
for (const file of ecosystemFiles) {
|
|
72
|
+
ecosystemRegistries.push({ file, data: await readJson(file) });
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
for (const file of trustFiles) {
|
|
76
|
+
trustRegistries.push({ file, data: await readJson(file) });
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const profileIds = new Map();
|
|
80
|
+
const rulePackIds = new Map();
|
|
81
|
+
|
|
82
|
+
for (const { file, data } of profiles) {
|
|
83
|
+
const label = `profile ${data.id || file}`;
|
|
84
|
+
requireString(data, "id", label, failures);
|
|
85
|
+
requireString(data, "version", label, failures);
|
|
86
|
+
requireString(data, "status", label, failures);
|
|
87
|
+
requireArray(data, "rule_packs", label, failures);
|
|
88
|
+
requireArray(data, "known_limitations", label, failures);
|
|
89
|
+
|
|
90
|
+
if (profileIds.has(data.id)) failures.push(`duplicate profile id: ${data.id}`);
|
|
91
|
+
profileIds.set(data.id, data);
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
for (const { file, data } of rulePacks) {
|
|
95
|
+
const label = `rule pack ${data.id || file}`;
|
|
96
|
+
requireString(data, "id", label, failures);
|
|
97
|
+
requireString(data, "version", label, failures);
|
|
98
|
+
requireArray(data, "rules", label, failures);
|
|
99
|
+
|
|
100
|
+
if (rulePackIds.has(data.id)) failures.push(`duplicate rule pack id: ${data.id}`);
|
|
101
|
+
rulePackIds.set(data.id, data);
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
for (const { data } of profiles) {
|
|
105
|
+
for (const parentId of data.extends || []) {
|
|
106
|
+
if (!profileIds.has(parentId)) failures.push(`profile ${data.id} extends unknown profile ${parentId}`);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
for (const packId of data.rule_packs || []) {
|
|
110
|
+
if (!rulePackIds.has(packId)) failures.push(`profile ${data.id} references unknown rule pack ${packId}`);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
const ruleIds = new Set();
|
|
115
|
+
for (const { data: pack } of rulePacks) {
|
|
116
|
+
for (const rule of pack.rules || []) {
|
|
117
|
+
const label = `rule ${pack.id}/${rule.id || "unknown"}`;
|
|
118
|
+
requireString(rule, "id", label, failures);
|
|
119
|
+
requireString(rule, "title", label, failures);
|
|
120
|
+
requireString(rule, "category", label, failures);
|
|
121
|
+
requireString(rule, "severity", label, failures);
|
|
122
|
+
requireArray(rule, "permissions", label, failures);
|
|
123
|
+
requireArray(rule, "patterns", label, failures);
|
|
124
|
+
requireArray(rule, "recommended_actions", label, failures);
|
|
125
|
+
requireArray(rule, "recommended_alternatives", label, failures);
|
|
126
|
+
requireString(rule, "migration_instruction", label, failures);
|
|
127
|
+
|
|
128
|
+
if (ruleIds.has(rule.id)) failures.push(`duplicate rule id: ${rule.id}`);
|
|
129
|
+
ruleIds.add(rule.id);
|
|
130
|
+
if (!VALID_SEVERITIES.has(rule.severity)) failures.push(`${label} has invalid severity: ${rule.severity}`);
|
|
131
|
+
if (!VALID_CATEGORIES.has(rule.category)) failures.push(`${label} has invalid category: ${rule.category}`);
|
|
132
|
+
if (typeof rule.confidence !== "number" || rule.confidence < 0 || rule.confidence > 1) {
|
|
133
|
+
failures.push(`${label} confidence must be between 0 and 1`);
|
|
134
|
+
}
|
|
135
|
+
if (!rule.recommended_actions?.length) failures.push(`${label} must include recommended actions`);
|
|
136
|
+
if (!rule.recommended_alternatives?.length) failures.push(`${label} must include recommended alternatives`);
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
const recommendationIds = new Set();
|
|
141
|
+
for (const { data: pack } of recommendationPacks) {
|
|
142
|
+
const label = `recommendation pack ${pack.id}`;
|
|
143
|
+
requireString(pack, "id", label, failures);
|
|
144
|
+
requireString(pack, "version", label, failures);
|
|
145
|
+
requireString(pack, "status", label, failures);
|
|
146
|
+
requireArray(pack, "recommendations", label, failures);
|
|
147
|
+
|
|
148
|
+
for (const recommendation of pack.recommendations || []) {
|
|
149
|
+
const itemLabel = `recommendation ${pack.id}/${recommendation.id || "unknown"}`;
|
|
150
|
+
requireString(recommendation, "id", itemLabel, failures);
|
|
151
|
+
requireString(recommendation, "title", itemLabel, failures);
|
|
152
|
+
requireString(recommendation, "type", itemLabel, failures);
|
|
153
|
+
requireString(recommendation, "status", itemLabel, failures);
|
|
154
|
+
requireString(recommendation, "source", itemLabel, failures);
|
|
155
|
+
requireArray(recommendation, "recommended_actions", itemLabel, failures);
|
|
156
|
+
requireArray(recommendation, "recommended_alternatives", itemLabel, failures);
|
|
157
|
+
requireString(recommendation, "agent_instruction", itemLabel, failures);
|
|
158
|
+
requireString(recommendation, "rollback_note", itemLabel, failures);
|
|
159
|
+
|
|
160
|
+
if (recommendationIds.has(recommendation.id)) {
|
|
161
|
+
failures.push(`duplicate recommendation id: ${recommendation.id}`);
|
|
162
|
+
}
|
|
163
|
+
recommendationIds.add(recommendation.id);
|
|
164
|
+
|
|
165
|
+
if (typeof recommendation.confidence !== "number" || recommendation.confidence < 0 || recommendation.confidence > 1) {
|
|
166
|
+
failures.push(`${itemLabel} confidence must be between 0 and 1`);
|
|
167
|
+
}
|
|
168
|
+
if (!Number.isInteger(recommendation.rank)) {
|
|
169
|
+
failures.push(`${itemLabel} rank must be an integer`);
|
|
170
|
+
}
|
|
171
|
+
if (!recommendation.applies_to || typeof recommendation.applies_to !== "object") {
|
|
172
|
+
failures.push(`${itemLabel} missing applies_to object`);
|
|
173
|
+
}
|
|
174
|
+
if (!recommendation.recommended_actions?.length) {
|
|
175
|
+
failures.push(`${itemLabel} must include recommended actions`);
|
|
176
|
+
}
|
|
177
|
+
if (!recommendation.recommended_alternatives?.length) {
|
|
178
|
+
failures.push(`${itemLabel} must include recommended alternatives`);
|
|
179
|
+
}
|
|
180
|
+
if (!recommendation.one_step_commands?.length) {
|
|
181
|
+
failures.push(`${itemLabel} must include at least one one-step command or instruction`);
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const ecosystemCandidateIds = new Set();
|
|
187
|
+
for (const { data: registry } of ecosystemRegistries) {
|
|
188
|
+
const label = `ecosystem registry ${registry.id}`;
|
|
189
|
+
requireString(registry, "id", label, failures);
|
|
190
|
+
requireString(registry, "version", label, failures);
|
|
191
|
+
requireString(registry, "status", label, failures);
|
|
192
|
+
requireArray(registry, "candidates", label, failures);
|
|
193
|
+
|
|
194
|
+
for (const candidate of registry.candidates || []) {
|
|
195
|
+
const itemLabel = `ecosystem candidate ${registry.id}/${candidate.id || "unknown"}`;
|
|
196
|
+
requireString(candidate, "id", itemLabel, failures);
|
|
197
|
+
requireString(candidate, "name", itemLabel, failures);
|
|
198
|
+
requireString(candidate, "entity_type", itemLabel, failures);
|
|
199
|
+
requireString(candidate, "lifecycle_status", itemLabel, failures);
|
|
200
|
+
requireString(candidate, "claim_status", itemLabel, failures);
|
|
201
|
+
requireArray(candidate, "regions", itemLabel, failures);
|
|
202
|
+
requireArray(candidate, "why_candidate", itemLabel, failures);
|
|
203
|
+
requireArray(candidate, "known_or_expected_artifacts", itemLabel, failures);
|
|
204
|
+
requireArray(candidate, "data_needs", itemLabel, failures);
|
|
205
|
+
requireArray(candidate, "profile_impacts", itemLabel, failures);
|
|
206
|
+
|
|
207
|
+
if (ecosystemCandidateIds.has(candidate.id)) {
|
|
208
|
+
failures.push(`duplicate ecosystem candidate id: ${candidate.id}`);
|
|
209
|
+
}
|
|
210
|
+
ecosystemCandidateIds.add(candidate.id);
|
|
211
|
+
if (!Number.isInteger(candidate.priority)) {
|
|
212
|
+
failures.push(`${itemLabel} priority must be an integer`);
|
|
213
|
+
}
|
|
214
|
+
if (!candidate.why_candidate?.length) {
|
|
215
|
+
failures.push(`${itemLabel} must explain why it is tracked`);
|
|
216
|
+
}
|
|
217
|
+
if (!candidate.data_needs?.length) {
|
|
218
|
+
failures.push(`${itemLabel} must list data needs`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
const trustSignalIds = new Set();
|
|
224
|
+
for (const { data: registry } of trustRegistries) {
|
|
225
|
+
const label = `trust registry ${registry.id}`;
|
|
226
|
+
requireString(registry, "id", label, failures);
|
|
227
|
+
requireString(registry, "version", label, failures);
|
|
228
|
+
requireString(registry, "status", label, failures);
|
|
229
|
+
requireArray(registry, "signals", label, failures);
|
|
230
|
+
|
|
231
|
+
for (const signal of registry.signals || []) {
|
|
232
|
+
const itemLabel = `trust signal ${registry.id}/${signal.id || "unknown"}`;
|
|
233
|
+
requireString(signal, "id", itemLabel, failures);
|
|
234
|
+
requireString(signal, "title", itemLabel, failures);
|
|
235
|
+
requireString(signal, "direction", itemLabel, failures);
|
|
236
|
+
requireString(signal, "source_type", itemLabel, failures);
|
|
237
|
+
requireArray(signal, "applies_to", itemLabel, failures);
|
|
238
|
+
requireArray(signal, "evidence_required", itemLabel, failures);
|
|
239
|
+
requireString(signal, "description", itemLabel, failures);
|
|
240
|
+
|
|
241
|
+
if (trustSignalIds.has(signal.id)) {
|
|
242
|
+
failures.push(`duplicate trust signal id: ${signal.id}`);
|
|
243
|
+
}
|
|
244
|
+
trustSignalIds.add(signal.id);
|
|
245
|
+
if (!Number.isInteger(signal.weight) || signal.weight < -100 || signal.weight > 100) {
|
|
246
|
+
failures.push(`${itemLabel} weight must be an integer between -100 and 100`);
|
|
247
|
+
}
|
|
248
|
+
if (!["positive", "negative", "neutral"].includes(signal.direction)) {
|
|
249
|
+
failures.push(`${itemLabel} has invalid direction: ${signal.direction}`);
|
|
250
|
+
}
|
|
251
|
+
if (!signal.evidence_required?.length) {
|
|
252
|
+
failures.push(`${itemLabel} must define required evidence`);
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
console.log(
|
|
258
|
+
`registry: ${profiles.length} profiles, ${rulePacks.length} rule packs, ${ruleIds.size} rules, ${recommendationIds.size} recommendations, ${ecosystemCandidateIds.size} ecosystem candidates, ${trustSignalIds.size} trust signals checked`
|
|
259
|
+
);
|
|
260
|
+
|
|
261
|
+
if (failures.length) {
|
|
262
|
+
for (const failure of failures) console.error(`- ${failure}`);
|
|
263
|
+
process.exitCode = 1;
|
|
264
|
+
}
|