@agentspec/cli 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +2092 -0
- package/dist/cli.js.map +1 -0
- package/package.json +47 -0
package/dist/cli.js
ADDED
|
@@ -0,0 +1,2092 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
// src/cli.ts
|
|
4
|
+
import { Command } from "commander";
|
|
5
|
+
|
|
6
|
+
// src/commands/validate.ts
|
|
7
|
+
import chalk2 from "chalk";
|
|
8
|
+
import { loadManifest } from "@agentspec/sdk";
|
|
9
|
+
|
|
10
|
+
// src/utils/output.ts
|
|
11
|
+
import chalk from "chalk";
|
|
12
|
+
var symbols = {
|
|
13
|
+
pass: chalk.green("\u2713"),
|
|
14
|
+
fail: chalk.red("\u2717"),
|
|
15
|
+
warn: chalk.yellow("\u26A0"),
|
|
16
|
+
skip: chalk.gray("\u2013"),
|
|
17
|
+
info: chalk.blue("\u2139"),
|
|
18
|
+
arrow: chalk.gray("\u2192")
|
|
19
|
+
};
|
|
20
|
+
function formatSeverity(severity) {
|
|
21
|
+
switch (severity) {
|
|
22
|
+
case "critical":
|
|
23
|
+
return chalk.bgRed.white(" CRITICAL ");
|
|
24
|
+
case "error":
|
|
25
|
+
return chalk.red("[error]");
|
|
26
|
+
case "high":
|
|
27
|
+
return chalk.red("[high]");
|
|
28
|
+
case "medium":
|
|
29
|
+
return chalk.yellow("[medium]");
|
|
30
|
+
case "warning":
|
|
31
|
+
return chalk.yellow("[warning]");
|
|
32
|
+
case "low":
|
|
33
|
+
return chalk.blue("[low]");
|
|
34
|
+
case "info":
|
|
35
|
+
return chalk.gray("[info]");
|
|
36
|
+
default:
|
|
37
|
+
return chalk.gray(`[${severity}]`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
function formatGrade(grade) {
|
|
41
|
+
switch (grade) {
|
|
42
|
+
case "A":
|
|
43
|
+
return chalk.bgGreen.black(` ${grade} `);
|
|
44
|
+
case "B":
|
|
45
|
+
return chalk.bgGreenBright.black(` ${grade} `);
|
|
46
|
+
case "C":
|
|
47
|
+
return chalk.bgYellow.black(` ${grade} `);
|
|
48
|
+
case "D":
|
|
49
|
+
return chalk.bgYellowBright.black(` ${grade} `);
|
|
50
|
+
case "F":
|
|
51
|
+
return chalk.bgRed.white(` ${grade} `);
|
|
52
|
+
default:
|
|
53
|
+
return chalk.gray(grade);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
function formatHealthStatus(status) {
|
|
57
|
+
switch (status) {
|
|
58
|
+
case "healthy":
|
|
59
|
+
return chalk.green("\u25CF healthy");
|
|
60
|
+
case "degraded":
|
|
61
|
+
return chalk.yellow("\u25D0 degraded");
|
|
62
|
+
case "unhealthy":
|
|
63
|
+
return chalk.red("\u25CB unhealthy");
|
|
64
|
+
default:
|
|
65
|
+
return status;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
function printHeader(title) {
|
|
69
|
+
console.log();
|
|
70
|
+
console.log(chalk.bold.cyan(` ${title}`));
|
|
71
|
+
console.log(chalk.gray(" " + "\u2500".repeat(title.length + 2)));
|
|
72
|
+
}
|
|
73
|
+
function printError(msg) {
|
|
74
|
+
console.error(chalk.red(`
|
|
75
|
+
\u2717 ${msg}
|
|
76
|
+
`));
|
|
77
|
+
}
|
|
78
|
+
function printSuccess(msg) {
|
|
79
|
+
console.log(chalk.green(`
|
|
80
|
+
\u2713 ${msg}
|
|
81
|
+
`));
|
|
82
|
+
}
|
|
83
|
+
function scoreColor(score) {
|
|
84
|
+
if (score >= 75) return chalk.green;
|
|
85
|
+
if (score >= 50) return chalk.yellow;
|
|
86
|
+
return chalk.red;
|
|
87
|
+
}
|
|
88
|
+
function formatCiGate(result) {
|
|
89
|
+
if (result === "PASS") return chalk.green("PASS");
|
|
90
|
+
if (result === "FAIL") return chalk.red("FAIL");
|
|
91
|
+
return chalk.gray("N/A");
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// src/commands/validate.ts
|
|
95
|
+
function registerValidateCommand(program2) {
|
|
96
|
+
program2.command("validate <file>").description("Validate an agent.yaml manifest against the schema (no I/O)").option("--json", "Output as JSON").action(async (file, opts) => {
|
|
97
|
+
try {
|
|
98
|
+
const { manifest } = loadManifest(file, { resolve: false });
|
|
99
|
+
if (opts.json) {
|
|
100
|
+
console.log(
|
|
101
|
+
JSON.stringify(
|
|
102
|
+
{
|
|
103
|
+
valid: true,
|
|
104
|
+
agentName: manifest.metadata.name,
|
|
105
|
+
version: manifest.metadata.version,
|
|
106
|
+
apiVersion: manifest.apiVersion
|
|
107
|
+
},
|
|
108
|
+
null,
|
|
109
|
+
2
|
|
110
|
+
)
|
|
111
|
+
);
|
|
112
|
+
return;
|
|
113
|
+
}
|
|
114
|
+
printHeader("AgentSpec Validate");
|
|
115
|
+
printSuccess(
|
|
116
|
+
`Manifest valid \u2014 ${chalk2.cyan(manifest.metadata.name)} v${manifest.metadata.version} (agentspec.io/v1)`
|
|
117
|
+
);
|
|
118
|
+
console.log(chalk2.gray(` Provider : ${manifest.spec.model.provider}/${manifest.spec.model.id}`));
|
|
119
|
+
console.log(chalk2.gray(` Tools : ${manifest.spec.tools?.length ?? 0}`));
|
|
120
|
+
console.log(chalk2.gray(` MCP : ${manifest.spec.mcp?.servers?.length ?? 0} servers`));
|
|
121
|
+
console.log(chalk2.gray(` Memory : ${manifest.spec.memory ? "configured" : "none"}`));
|
|
122
|
+
console.log();
|
|
123
|
+
} catch (err) {
|
|
124
|
+
if (opts.json) {
|
|
125
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
126
|
+
console.log(JSON.stringify({ valid: false, error: message }, null, 2));
|
|
127
|
+
process.exit(1);
|
|
128
|
+
}
|
|
129
|
+
printHeader("AgentSpec Validate");
|
|
130
|
+
if (err instanceof Error && err.constructor.name === "ZodError") {
|
|
131
|
+
printError("Manifest validation failed:");
|
|
132
|
+
const raw = JSON.parse(
|
|
133
|
+
err.message.startsWith("[") ? err.message : JSON.stringify([{ message: err.message }])
|
|
134
|
+
);
|
|
135
|
+
for (const issue of raw) {
|
|
136
|
+
const path = issue.path?.join(".") || "root";
|
|
137
|
+
console.error(chalk2.red(` \u2022 ${path}: ${issue.message}`));
|
|
138
|
+
}
|
|
139
|
+
} else {
|
|
140
|
+
printError(String(err));
|
|
141
|
+
}
|
|
142
|
+
console.log();
|
|
143
|
+
process.exit(1);
|
|
144
|
+
}
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// src/commands/health.ts
|
|
149
|
+
import chalk3 from "chalk";
|
|
150
|
+
import { loadManifest as loadManifest2, runHealthCheck } from "@agentspec/sdk";
|
|
151
|
+
function registerHealthCommand(program2) {
|
|
152
|
+
program2.command("health <file>").description("Run runtime health checks against all declared dependencies").option("--json", "Output as JSON").option("--format <fmt>", "Output format: table|json", "table").option("--fail-on <level>", "Exit 1 on: error|warning|info", "error").option("--no-model", "Skip model API reachability checks").option("--no-mcp", "Skip MCP server checks").option("--no-memory", "Skip memory backend checks").action(
|
|
153
|
+
async (file, opts) => {
|
|
154
|
+
let manifest;
|
|
155
|
+
try {
|
|
156
|
+
manifest = loadManifest2(file, { resolve: false });
|
|
157
|
+
} catch (err) {
|
|
158
|
+
printError(`Cannot load manifest: ${String(err)}`);
|
|
159
|
+
process.exit(1);
|
|
160
|
+
}
|
|
161
|
+
const report = await runHealthCheck(manifest.manifest, {
|
|
162
|
+
checkModel: opts.model !== false,
|
|
163
|
+
checkMcp: opts.mcp !== false,
|
|
164
|
+
checkMemory: opts.memory !== false,
|
|
165
|
+
baseDir: manifest.baseDir,
|
|
166
|
+
rawManifest: manifest.manifest
|
|
167
|
+
// raw unresolved for ref collection
|
|
168
|
+
});
|
|
169
|
+
if (opts.json || opts.format === "json") {
|
|
170
|
+
console.log(JSON.stringify(report, null, 2));
|
|
171
|
+
process.exit(shouldFail(report.checks, opts.failOn ?? "error") ? 1 : 0);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
printHeader(`AgentSpec Health \u2014 ${manifest.manifest.metadata.name}`);
|
|
175
|
+
console.log(` Status: ${formatHealthStatus(report.status)}`);
|
|
176
|
+
console.log(
|
|
177
|
+
chalk3.gray(
|
|
178
|
+
` Passed: ${chalk3.green(report.summary.passed)} Failed: ${chalk3.red(report.summary.failed)} Skipped: ${chalk3.gray(report.summary.skipped)}`
|
|
179
|
+
)
|
|
180
|
+
);
|
|
181
|
+
console.log();
|
|
182
|
+
const grouped = groupByCategory(report.checks);
|
|
183
|
+
for (const [category, checks] of Object.entries(grouped)) {
|
|
184
|
+
console.log(chalk3.bold(` ${category.toUpperCase()}`));
|
|
185
|
+
for (const check of checks) {
|
|
186
|
+
const icon = check.status === "pass" ? symbols.pass : check.status === "fail" ? symbols.fail : check.status === "warn" ? symbols.warn : symbols.skip;
|
|
187
|
+
const latency = check.latencyMs ? chalk3.gray(` (${check.latencyMs}ms)`) : "";
|
|
188
|
+
console.log(` ${icon} ${chalk3.white(check.id)}${latency}`);
|
|
189
|
+
if (check.message) {
|
|
190
|
+
console.log(` ${chalk3.gray(check.message)}`);
|
|
191
|
+
}
|
|
192
|
+
if (check.remediation && check.status === "fail") {
|
|
193
|
+
console.log(` ${chalk3.cyan("\u2192")} ${chalk3.cyan(check.remediation)}`);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
console.log();
|
|
197
|
+
}
|
|
198
|
+
const failed = shouldFail(report.checks, opts.failOn ?? "error");
|
|
199
|
+
if (failed) process.exit(1);
|
|
200
|
+
}
|
|
201
|
+
);
|
|
202
|
+
}
|
|
203
|
+
function groupByCategory(checks) {
|
|
204
|
+
const groups = {};
|
|
205
|
+
for (const check of checks) {
|
|
206
|
+
if (!groups[check.category]) groups[check.category] = [];
|
|
207
|
+
groups[check.category].push(check);
|
|
208
|
+
}
|
|
209
|
+
return groups;
|
|
210
|
+
}
|
|
211
|
+
function shouldFail(checks, failOn) {
|
|
212
|
+
const failedChecks = checks.filter((c) => c.status === "fail");
|
|
213
|
+
if (failOn === "info") return failedChecks.length > 0;
|
|
214
|
+
if (failOn === "warning") {
|
|
215
|
+
return failedChecks.some((c) => c.severity === "error" || c.severity === "warning");
|
|
216
|
+
}
|
|
217
|
+
return failedChecks.some((c) => c.severity === "error");
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// src/commands/audit.ts
|
|
221
|
+
import { writeFileSync } from "fs";
|
|
222
|
+
import chalk4 from "chalk";
|
|
223
|
+
import { loadManifest as loadManifest3, runAudit, isProofRecord } from "@agentspec/sdk";
|
|
224
|
+
function registerAuditCommand(program2) {
|
|
225
|
+
program2.command("audit <file>").description("Run compliance audit against configured packs").option("--pack <pack>", "Run a specific compliance pack only").option("--url <url>", "Sidecar URL to fetch proof records (e.g. http://localhost:4001)").option("--json", "Output as JSON").option("--output <file>", "Write report to file").option("--fail-below <score>", "Exit 1 if score is below this threshold", "0").action(
|
|
226
|
+
async (file, opts) => {
|
|
227
|
+
let parsed;
|
|
228
|
+
try {
|
|
229
|
+
parsed = loadManifest3(file, { resolve: false });
|
|
230
|
+
} catch (err) {
|
|
231
|
+
printError(`Cannot load manifest: ${String(err)}`);
|
|
232
|
+
process.exit(1);
|
|
233
|
+
}
|
|
234
|
+
const { manifest } = parsed;
|
|
235
|
+
const packs = opts.pack ? [opts.pack] : void 0;
|
|
236
|
+
const proofRecords = opts.url ? await fetchProofRecords(opts.url) : void 0;
|
|
237
|
+
const report = runAudit(manifest, { packs, proofRecords });
|
|
238
|
+
const outputJson = JSON.stringify(report, null, 2);
|
|
239
|
+
if (opts.output) {
|
|
240
|
+
writeFileSync(opts.output, outputJson, "utf-8");
|
|
241
|
+
console.log(chalk4.green(` \u2713 Report written to ${opts.output}`));
|
|
242
|
+
}
|
|
243
|
+
if (opts.json) {
|
|
244
|
+
console.log(outputJson);
|
|
245
|
+
const threshold2 = parseInt(opts.failBelow ?? "0", 10);
|
|
246
|
+
if (report.overallScore < threshold2) process.exit(1);
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
printHeader(`AgentSpec Audit \u2014 ${manifest.metadata.name}`);
|
|
250
|
+
printScoreSummary(report, opts.url);
|
|
251
|
+
console.log(
|
|
252
|
+
` Rules : ${chalk4.green(report.passedRules)} passed / ${chalk4.red(report.totalRules - report.passedRules)} failed / ${report.totalRules} total`
|
|
253
|
+
);
|
|
254
|
+
console.log();
|
|
255
|
+
console.log(chalk4.bold(" Category Scores"));
|
|
256
|
+
for (const [cat, score] of Object.entries(report.categoryScores)) {
|
|
257
|
+
const bar = progressBar(score);
|
|
258
|
+
const color = scoreColor(score);
|
|
259
|
+
console.log(` ${chalk4.gray(cat.padEnd(24))} ${color(String(score).padStart(3))}% ${bar}`);
|
|
260
|
+
}
|
|
261
|
+
console.log();
|
|
262
|
+
if (report.provedScore !== void 0) {
|
|
263
|
+
renderDualModeViolations(report.violations, report, opts.url);
|
|
264
|
+
} else {
|
|
265
|
+
renderViolations(report.violations);
|
|
266
|
+
}
|
|
267
|
+
if (report.suppressions.length > 0) {
|
|
268
|
+
console.log();
|
|
269
|
+
console.log(chalk4.bold(` Suppressions (${report.suppressions.length})`));
|
|
270
|
+
for (const s of report.suppressions) {
|
|
271
|
+
console.log(` ${chalk4.gray("\u2013")} ${s.ruleId}: ${s.reason}`);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
if (report.evidenceBreakdown) {
|
|
275
|
+
formatEvidenceBreakdown(report.evidenceBreakdown, opts.url);
|
|
276
|
+
}
|
|
277
|
+
console.log();
|
|
278
|
+
const threshold = parseInt(opts.failBelow ?? "0", 10);
|
|
279
|
+
if (report.overallScore < threshold) {
|
|
280
|
+
console.error(chalk4.red(` \u2717 Score ${report.overallScore} is below threshold ${threshold}`));
|
|
281
|
+
process.exit(1);
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
);
|
|
285
|
+
}
|
|
286
|
+
async function fetchProofRecords(url) {
|
|
287
|
+
try {
|
|
288
|
+
new URL(url);
|
|
289
|
+
} catch {
|
|
290
|
+
printError(`Invalid URL: ${url}`);
|
|
291
|
+
process.exit(1);
|
|
292
|
+
}
|
|
293
|
+
try {
|
|
294
|
+
const proofUrl = `${url.replace(/\/$/, "")}/proof`;
|
|
295
|
+
const res = await fetch(proofUrl, {
|
|
296
|
+
headers: { Accept: "application/json" },
|
|
297
|
+
signal: AbortSignal.timeout(5e3)
|
|
298
|
+
});
|
|
299
|
+
if (res.ok) {
|
|
300
|
+
const data = await res.json();
|
|
301
|
+
if (!Array.isArray(data)) {
|
|
302
|
+
process.stderr.write(
|
|
303
|
+
chalk4.yellow(` \u26A0 Unexpected proof response from ${proofUrl} \u2014 expected array, got ${typeof data}
|
|
304
|
+
`)
|
|
305
|
+
);
|
|
306
|
+
} else {
|
|
307
|
+
return data.filter(isProofRecord);
|
|
308
|
+
}
|
|
309
|
+
} else {
|
|
310
|
+
process.stderr.write(
|
|
311
|
+
chalk4.yellow(` \u26A0 Could not fetch proof records from ${proofUrl} (${res.status}) \u2014 showing declared score only
|
|
312
|
+
`)
|
|
313
|
+
);
|
|
314
|
+
}
|
|
315
|
+
} catch {
|
|
316
|
+
process.stderr.write(
|
|
317
|
+
chalk4.yellow(` \u26A0 Could not reach sidecar at ${url} \u2014 showing declared score only
|
|
318
|
+
`)
|
|
319
|
+
);
|
|
320
|
+
}
|
|
321
|
+
return void 0;
|
|
322
|
+
}
|
|
323
|
+
function printScoreSummary(report, sidecarUrl) {
|
|
324
|
+
if (report.provedScore !== void 0) {
|
|
325
|
+
console.log(
|
|
326
|
+
` Declared score : ${formatGrade(report.grade)} ${chalk4.bold(String(report.overallScore))}/100 \u2014 what your spec says`
|
|
327
|
+
);
|
|
328
|
+
console.log(
|
|
329
|
+
` Proved score : ${formatGrade(report.provedGrade)} ${chalk4.bold(String(report.provedScore))}/100 \u2014 what has been verified`
|
|
330
|
+
);
|
|
331
|
+
console.log(
|
|
332
|
+
` Pending proof : ${chalk4.yellow(String(report.pendingProofCount))} rules \u2014 run external tools and POST to ${sidecarUrl}/proof/rule/:ruleId`
|
|
333
|
+
);
|
|
334
|
+
} else {
|
|
335
|
+
console.log(
|
|
336
|
+
` Score : ${formatGrade(report.grade)} ${chalk4.bold(String(report.overallScore))}/100`
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
}
|
|
340
|
+
function formatEvidenceBreakdown(breakdown, sidecarUrl) {
|
|
341
|
+
void sidecarUrl;
|
|
342
|
+
console.log();
|
|
343
|
+
console.log(chalk4.bold(" Evidence Breakdown"));
|
|
344
|
+
const { declarative, probed, behavioral, external } = breakdown;
|
|
345
|
+
const dLabel = declarative.total > 0 ? `${declarative.passed}/${declarative.total}` : "N/A";
|
|
346
|
+
const pLabel = probed.total > 0 ? `${probed.passed}/${probed.total}` : "N/A (run `agentspec health <file>` for live checks)";
|
|
347
|
+
const bLabel = behavioral.total > 0 ? `${behavioral.passed}/${behavioral.total}` : "N/A (no runtime events \u2014 deploy with sdk-langgraph + EventPush)";
|
|
348
|
+
const xLabel = external.total > 0 ? `${external.passed}/${external.total}` : "N/A";
|
|
349
|
+
console.log(` ${chalk4.gray("[D]")} Declarative ${chalk4.cyan(dLabel)} (manifest declarations)`);
|
|
350
|
+
console.log(` ${chalk4.gray("[P]")} Probed ${chalk4.cyan(pLabel)}`);
|
|
351
|
+
console.log(` ${chalk4.gray("[B]")} Behavioral ${chalk4.cyan(bLabel)}`);
|
|
352
|
+
console.log(` ${chalk4.gray("[X]")} External ${chalk4.cyan(xLabel)} (k6, Presidio, Promptfoo, LiteLLM)`);
|
|
353
|
+
}
|
|
354
|
+
function progressBar(score) {
|
|
355
|
+
const width = 20;
|
|
356
|
+
const filled = Math.round(score / 100 * width);
|
|
357
|
+
const bar = "\u2588".repeat(filled) + "\u2591".repeat(width - filled);
|
|
358
|
+
return scoreColor(score)(bar);
|
|
359
|
+
}
|
|
360
|
+
function evidenceBadge(level) {
|
|
361
|
+
switch (level) {
|
|
362
|
+
case "declarative":
|
|
363
|
+
return chalk4.gray("[D]");
|
|
364
|
+
case "probed":
|
|
365
|
+
return chalk4.blue("[P]");
|
|
366
|
+
case "behavioral":
|
|
367
|
+
return chalk4.magenta("[B]");
|
|
368
|
+
case "external":
|
|
369
|
+
return chalk4.yellow("[X]");
|
|
370
|
+
default:
|
|
371
|
+
return chalk4.gray("[D]");
|
|
372
|
+
}
|
|
373
|
+
}
|
|
374
|
+
function renderViolationDetail(v, badge, sidecarUrl) {
|
|
375
|
+
console.log(`
|
|
376
|
+
${formatSeverity(v.severity)} ${badge} ${chalk4.bold(v.ruleId)} \u2014 ${chalk4.white(v.title)}`);
|
|
377
|
+
console.log(` ${chalk4.gray(v.message)}`);
|
|
378
|
+
if (v.path) console.log(` Path: ${chalk4.cyan(v.path)}`);
|
|
379
|
+
if (v.recommendation) console.log(` ${chalk4.cyan("\u2192")} ${v.recommendation}`);
|
|
380
|
+
if (v.proofTool) {
|
|
381
|
+
if (sidecarUrl) {
|
|
382
|
+
const endpoint = `POST ${sidecarUrl.replace(/\/$/, "")}/proof/rule/${v.ruleId}`;
|
|
383
|
+
console.log(` ${chalk4.yellow("\u2192 Prove:")} ${v.proofTool} \u2192 ${chalk4.cyan(endpoint)}`);
|
|
384
|
+
} else {
|
|
385
|
+
console.log(` ${chalk4.yellow("\u2192 Prove:")} ${v.proofTool}`);
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
if (v.proofToolUrl) console.log(` ${chalk4.blue(v.proofToolUrl)}`);
|
|
389
|
+
if (v.references?.length) {
|
|
390
|
+
for (const ref of v.references) console.log(` ${chalk4.blue(ref)}`);
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
function renderViolations(violations) {
|
|
394
|
+
if (violations.length === 0) {
|
|
395
|
+
console.log(chalk4.green(" \u2713 No violations found"));
|
|
396
|
+
return;
|
|
397
|
+
}
|
|
398
|
+
console.log(chalk4.bold(` Violations (${violations.length})`));
|
|
399
|
+
for (const v of violations) renderViolationDetail(v, evidenceBadge(v.evidenceLevel));
|
|
400
|
+
}
|
|
401
|
+
function renderDualModeViolations(violations, report, sidecarUrl) {
|
|
402
|
+
if (violations.length === 0) {
|
|
403
|
+
console.log(chalk4.green(" \u2713 No violations found"));
|
|
404
|
+
return;
|
|
405
|
+
}
|
|
406
|
+
const needsProof = violations.filter((v) => v.evidenceLevel === "external");
|
|
407
|
+
const notDeclared = violations.filter((v) => v.evidenceLevel !== "external");
|
|
408
|
+
if (notDeclared.length > 0) {
|
|
409
|
+
console.log(chalk4.bold(` NOT DECLARED (${notDeclared.length})`));
|
|
410
|
+
for (const v of notDeclared) renderViolationDetail(v, evidenceBadge(v.evidenceLevel));
|
|
411
|
+
}
|
|
412
|
+
if (needsProof.length > 0) {
|
|
413
|
+
console.log();
|
|
414
|
+
console.log(chalk4.bold(` DECLARED, AWAITING PROOF (${needsProof.length})`));
|
|
415
|
+
for (const v of needsProof) renderViolationDetail(v, chalk4.yellow("[D\u2192X]"), sidecarUrl);
|
|
416
|
+
}
|
|
417
|
+
if (report.pendingProofCount && report.pendingProofCount > 0) {
|
|
418
|
+
console.log();
|
|
419
|
+
console.log(chalk4.yellow(` \u26A0 ${report.pendingProofCount} rules pass declaratively but await external proof`));
|
|
420
|
+
console.log(chalk4.gray(` Run the external tools and POST results to ${sidecarUrl ?? "<sidecar-url>"}/proof/rule/:ruleId`));
|
|
421
|
+
console.log(chalk4.gray(" See: https://agentspec.io/docs/guides/proof-integration"));
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
// src/commands/init.ts
|
|
426
|
+
import { writeFileSync as writeFileSync2, existsSync } from "fs";
|
|
427
|
+
import { resolve, join } from "path";
|
|
428
|
+
import chalk5 from "chalk";
|
|
429
|
+
import * as p from "@clack/prompts";
|
|
430
|
+
function registerInitCommand(program2) {
|
|
431
|
+
program2.command("init [dir]").description("Interactive wizard to create a new agent.yaml manifest").option("--yes", "Skip prompts, create a minimal manifest").action(async (dir = ".", opts) => {
|
|
432
|
+
const outDir = resolve(dir);
|
|
433
|
+
const outFile = join(outDir, "agent.yaml");
|
|
434
|
+
printHeader("AgentSpec Init");
|
|
435
|
+
if (existsSync(outFile) && !opts.yes) {
|
|
436
|
+
const overwrite = await p.confirm({
|
|
437
|
+
message: `agent.yaml already exists at ${outFile}. Overwrite?`,
|
|
438
|
+
initialValue: false
|
|
439
|
+
});
|
|
440
|
+
if (!overwrite || p.isCancel(overwrite)) {
|
|
441
|
+
p.cancel("Init cancelled.");
|
|
442
|
+
return;
|
|
443
|
+
}
|
|
444
|
+
}
|
|
445
|
+
let name = "my-agent";
|
|
446
|
+
let description = "An AI agent";
|
|
447
|
+
let version = "0.1.0";
|
|
448
|
+
let provider = "openai";
|
|
449
|
+
let modelId = "gpt-4o-mini";
|
|
450
|
+
let includeMemory = false;
|
|
451
|
+
let includeGuardrails = true;
|
|
452
|
+
let includeEval = false;
|
|
453
|
+
if (!opts.yes) {
|
|
454
|
+
p.intro(chalk5.cyan("Creating your agent.yaml"));
|
|
455
|
+
const answers = await p.group(
|
|
456
|
+
{
|
|
457
|
+
name: () => p.text({
|
|
458
|
+
message: "Agent name (slug)",
|
|
459
|
+
placeholder: "my-agent",
|
|
460
|
+
validate: (v) => /^[a-z0-9-]+$/.test(v) ? void 0 : "Must be lowercase slug (a-z, 0-9, -)"
|
|
461
|
+
}),
|
|
462
|
+
description: () => p.text({
|
|
463
|
+
message: "Description",
|
|
464
|
+
placeholder: "An AI agent that..."
|
|
465
|
+
}),
|
|
466
|
+
version: () => p.text({
|
|
467
|
+
message: "Version",
|
|
468
|
+
placeholder: "0.1.0",
|
|
469
|
+
initialValue: "0.1.0"
|
|
470
|
+
}),
|
|
471
|
+
provider: () => p.select({
|
|
472
|
+
message: "Model provider",
|
|
473
|
+
options: [
|
|
474
|
+
{ value: "openai", label: "OpenAI" },
|
|
475
|
+
{ value: "anthropic", label: "Anthropic" },
|
|
476
|
+
{ value: "groq", label: "Groq" },
|
|
477
|
+
{ value: "google", label: "Google" },
|
|
478
|
+
{ value: "mistral", label: "Mistral" },
|
|
479
|
+
{ value: "azure", label: "Azure OpenAI" }
|
|
480
|
+
]
|
|
481
|
+
}),
|
|
482
|
+
modelId: () => p.text({
|
|
483
|
+
message: "Model ID",
|
|
484
|
+
placeholder: "gpt-4o-mini"
|
|
485
|
+
}),
|
|
486
|
+
includeMemory: () => p.confirm({
|
|
487
|
+
message: "Include memory configuration?",
|
|
488
|
+
initialValue: false
|
|
489
|
+
}),
|
|
490
|
+
includeGuardrails: () => p.confirm({
|
|
491
|
+
message: "Include guardrails?",
|
|
492
|
+
initialValue: true
|
|
493
|
+
}),
|
|
494
|
+
includeEval: () => p.confirm({
|
|
495
|
+
message: "Include evaluation configuration?",
|
|
496
|
+
initialValue: false
|
|
497
|
+
})
|
|
498
|
+
},
|
|
499
|
+
{
|
|
500
|
+
onCancel: () => {
|
|
501
|
+
p.cancel("Init cancelled.");
|
|
502
|
+
process.exit(0);
|
|
503
|
+
}
|
|
504
|
+
}
|
|
505
|
+
);
|
|
506
|
+
name = String(answers.name || "my-agent");
|
|
507
|
+
description = String(answers.description || "An AI agent");
|
|
508
|
+
version = String(answers.version || "0.1.0");
|
|
509
|
+
provider = String(answers.provider || "openai");
|
|
510
|
+
modelId = String(answers.modelId || "gpt-4o-mini");
|
|
511
|
+
includeMemory = Boolean(answers.includeMemory);
|
|
512
|
+
includeGuardrails = Boolean(answers.includeGuardrails);
|
|
513
|
+
includeEval = Boolean(answers.includeEval);
|
|
514
|
+
}
|
|
515
|
+
const apiKeyEnv = `${provider.toUpperCase()}_API_KEY`;
|
|
516
|
+
const yaml3 = generateManifest({
|
|
517
|
+
name,
|
|
518
|
+
description,
|
|
519
|
+
version,
|
|
520
|
+
provider,
|
|
521
|
+
modelId,
|
|
522
|
+
apiKeyEnv,
|
|
523
|
+
includeMemory,
|
|
524
|
+
includeGuardrails,
|
|
525
|
+
includeEval
|
|
526
|
+
});
|
|
527
|
+
writeFileSync2(outFile, yaml3, "utf-8");
|
|
528
|
+
if (!opts.yes) p.outro(chalk5.green(`\u2713 Created ${outFile}`));
|
|
529
|
+
else console.log(chalk5.green(`
|
|
530
|
+
\u2713 Created ${outFile}
|
|
531
|
+
`));
|
|
532
|
+
console.log(chalk5.gray(" Next steps:"));
|
|
533
|
+
console.log(chalk5.gray(` 1. Edit ${outFile} to customize your agent`));
|
|
534
|
+
console.log(chalk5.gray(` 2. Run: npx agentspec validate agent.yaml`));
|
|
535
|
+
console.log(chalk5.gray(` 3. Run: npx agentspec health agent.yaml`));
|
|
536
|
+
console.log(chalk5.gray(` 4. Run: npx agentspec audit agent.yaml`));
|
|
537
|
+
console.log();
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
function generateManifest(opts) {
|
|
541
|
+
const sections = [
|
|
542
|
+
`apiVersion: agentspec.io/v1
|
|
543
|
+
kind: AgentSpec
|
|
544
|
+
|
|
545
|
+
metadata:
|
|
546
|
+
name: ${opts.name}
|
|
547
|
+
version: ${opts.version}
|
|
548
|
+
description: "${opts.description}"
|
|
549
|
+
tags: []
|
|
550
|
+
author: ""
|
|
551
|
+
license: MIT
|
|
552
|
+
|
|
553
|
+
spec:
|
|
554
|
+
# \u2500\u2500 MODEL \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
555
|
+
model:
|
|
556
|
+
provider: ${opts.provider}
|
|
557
|
+
id: ${opts.modelId}
|
|
558
|
+
apiKey: $env:${opts.apiKeyEnv}
|
|
559
|
+
parameters:
|
|
560
|
+
temperature: 0.7
|
|
561
|
+
maxTokens: 2000
|
|
562
|
+
# Uncomment to add fallback:
|
|
563
|
+
# fallback:
|
|
564
|
+
# provider: openai
|
|
565
|
+
# id: gpt-4o-mini
|
|
566
|
+
# apiKey: $env:OPENAI_API_KEY
|
|
567
|
+
# triggerOn: [rate_limit, timeout, error_5xx]
|
|
568
|
+
# maxRetries: 2
|
|
569
|
+
|
|
570
|
+
# \u2500\u2500 PROMPTS \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
571
|
+
prompts:
|
|
572
|
+
system: $file:prompts/system.md
|
|
573
|
+
fallback: "I'm experiencing difficulties. Please try again."
|
|
574
|
+
hotReload: false
|
|
575
|
+
|
|
576
|
+
# \u2500\u2500 TOOLS (optional) \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
577
|
+
# tools:
|
|
578
|
+
# - name: my-tool
|
|
579
|
+
# type: function
|
|
580
|
+
# description: "Description of what this tool does"
|
|
581
|
+
# module: $file:tools/my_tool.py
|
|
582
|
+
# function: my_tool_function
|
|
583
|
+
# annotations:
|
|
584
|
+
# readOnlyHint: true
|
|
585
|
+
# destructiveHint: false`
|
|
586
|
+
];
|
|
587
|
+
if (opts.includeMemory) {
|
|
588
|
+
sections.push(`
|
|
589
|
+
# \u2500\u2500 MEMORY \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
590
|
+
memory:
|
|
591
|
+
shortTerm:
|
|
592
|
+
backend: in-memory
|
|
593
|
+
maxTurns: 20
|
|
594
|
+
maxTokens: 8000
|
|
595
|
+
hygiene:
|
|
596
|
+
piiScrubFields: []
|
|
597
|
+
auditLog: false`);
|
|
598
|
+
}
|
|
599
|
+
if (opts.includeGuardrails) {
|
|
600
|
+
sections.push(`
|
|
601
|
+
# \u2500\u2500 GUARDRAILS \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
602
|
+
guardrails:
|
|
603
|
+
input:
|
|
604
|
+
- type: prompt-injection
|
|
605
|
+
action: reject
|
|
606
|
+
sensitivity: high
|
|
607
|
+
output:
|
|
608
|
+
- type: toxicity-filter
|
|
609
|
+
threshold: 0.7
|
|
610
|
+
action: reject`);
|
|
611
|
+
}
|
|
612
|
+
if (opts.includeEval) {
|
|
613
|
+
sections.push(`
|
|
614
|
+
# \u2500\u2500 EVALUATION \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
615
|
+
evaluation:
|
|
616
|
+
framework: deepeval
|
|
617
|
+
datasets:
|
|
618
|
+
- name: qa-test
|
|
619
|
+
path: $file:eval/datasets/qa.jsonl
|
|
620
|
+
metrics:
|
|
621
|
+
- faithfulness
|
|
622
|
+
- hallucination
|
|
623
|
+
thresholds:
|
|
624
|
+
hallucination: 0.05
|
|
625
|
+
ciGate: false`);
|
|
626
|
+
}
|
|
627
|
+
sections.push(`
|
|
628
|
+
# \u2500\u2500 COMPLIANCE \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
629
|
+
compliance:
|
|
630
|
+
packs:
|
|
631
|
+
- owasp-llm-top10
|
|
632
|
+
- model-resilience
|
|
633
|
+
- memory-hygiene
|
|
634
|
+
|
|
635
|
+
# \u2500\u2500 RUNTIME REQUIREMENTS \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500
|
|
636
|
+
requires:
|
|
637
|
+
envVars:
|
|
638
|
+
- ${opts.apiKeyEnv}
|
|
639
|
+
`);
|
|
640
|
+
return sections.join("\n");
|
|
641
|
+
}
|
|
642
|
+
|
|
643
|
+
// src/commands/generate.ts
|
|
644
|
+
import { existsSync as existsSync2, mkdirSync, readFileSync, writeFileSync as writeFileSync3 } from "fs";
|
|
645
|
+
import { basename, dirname, join as join2, resolve as resolve2, sep } from "path";
|
|
646
|
+
import chalk6 from "chalk";
|
|
647
|
+
import { spinner } from "@clack/prompts";
|
|
648
|
+
import { loadManifest as loadManifest4 } from "@agentspec/sdk";
|
|
649
|
+
import { generateWithClaude, listFrameworks } from "@agentspec/adapter-claude";
|
|
650
|
+
|
|
651
|
+
// src/deploy/k8s.ts
|
|
652
|
+
var SIDECAR_IMAGE = "ghcr.io/agentspec/sidecar:latest";
|
|
653
|
+
var DEFAULT_AGENT_PORT = 8e3;
|
|
654
|
+
var PROXY_PORT = 4e3;
|
|
655
|
+
var CONTROL_PORT = 4001;
|
|
656
|
+
var K8S_NAME_RE = /^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$|^[a-z0-9]$/;
|
|
657
|
+
function escapeYamlDoubleQuotedValue(value) {
|
|
658
|
+
return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
|
|
659
|
+
}
|
|
660
|
+
function generateK8sManifests(manifest) {
|
|
661
|
+
const name = manifest.metadata.name;
|
|
662
|
+
if (!K8S_NAME_RE.test(name)) {
|
|
663
|
+
throw new Error(
|
|
664
|
+
`Invalid agent name "${name}" for Kubernetes \u2014 must be a lowercase DNS label (a-z, 0-9, hyphens, 1-63 chars)`
|
|
665
|
+
);
|
|
666
|
+
}
|
|
667
|
+
const port = manifest.spec.api?.port ?? DEFAULT_AGENT_PORT;
|
|
668
|
+
const envRefs = collectEnvRefs(manifest);
|
|
669
|
+
return {
|
|
670
|
+
"k8s/deployment.yaml": generateDeployment(name, port, envRefs),
|
|
671
|
+
"k8s/service.yaml": generateService(name, port),
|
|
672
|
+
"k8s/configmap.yaml": generateConfigMap(name, manifest),
|
|
673
|
+
"k8s/secret.yaml.example": generateSecretExample(name, envRefs)
|
|
674
|
+
};
|
|
675
|
+
}
|
|
676
|
+
function collectEnvRefs(manifest) {
|
|
677
|
+
const raw = JSON.stringify(manifest);
|
|
678
|
+
const matches = [...raw.matchAll(/\$env:([A-Za-z0-9_]+)/g)];
|
|
679
|
+
const names = matches.map((m) => m[1]);
|
|
680
|
+
return [...new Set(names)].sort();
|
|
681
|
+
}
|
|
682
|
+
function generateDeployment(name, port, envRefs) {
|
|
683
|
+
const secretEnvBlock = envRefs.length > 0 ? envRefs.map(
|
|
684
|
+
(v) => ` - name: ${v}
|
|
685
|
+
valueFrom:
|
|
686
|
+
secretKeyRef:
|
|
687
|
+
name: ${name}-secrets
|
|
688
|
+
key: ${v}`
|
|
689
|
+
).join("\n") : " []";
|
|
690
|
+
return `apiVersion: apps/v1
|
|
691
|
+
kind: Deployment
|
|
692
|
+
metadata:
|
|
693
|
+
name: ${name}
|
|
694
|
+
labels:
|
|
695
|
+
app: ${name}
|
|
696
|
+
spec:
|
|
697
|
+
replicas: 1
|
|
698
|
+
selector:
|
|
699
|
+
matchLabels:
|
|
700
|
+
app: ${name}
|
|
701
|
+
template:
|
|
702
|
+
metadata:
|
|
703
|
+
labels:
|
|
704
|
+
app: ${name}
|
|
705
|
+
spec:
|
|
706
|
+
containers:
|
|
707
|
+
- name: ${name}
|
|
708
|
+
image: ${name}:latest
|
|
709
|
+
ports:
|
|
710
|
+
- containerPort: ${port}
|
|
711
|
+
envFrom:
|
|
712
|
+
- configMapRef:
|
|
713
|
+
name: ${name}-config
|
|
714
|
+
env:
|
|
715
|
+
${secretEnvBlock}
|
|
716
|
+
- name: agentspec-sidecar
|
|
717
|
+
image: ${SIDECAR_IMAGE}
|
|
718
|
+
ports:
|
|
719
|
+
- containerPort: ${PROXY_PORT}
|
|
720
|
+
name: proxy
|
|
721
|
+
- containerPort: ${CONTROL_PORT}
|
|
722
|
+
name: control
|
|
723
|
+
env:
|
|
724
|
+
- name: UPSTREAM_URL
|
|
725
|
+
value: "http://localhost:${port}"
|
|
726
|
+
- name: MANIFEST_PATH
|
|
727
|
+
value: /manifest/agent.yaml
|
|
728
|
+
`;
|
|
729
|
+
}
|
|
730
|
+
function generateService(name, port) {
|
|
731
|
+
return `apiVersion: v1
|
|
732
|
+
kind: Service
|
|
733
|
+
metadata:
|
|
734
|
+
name: ${name}
|
|
735
|
+
labels:
|
|
736
|
+
app: ${name}
|
|
737
|
+
spec:
|
|
738
|
+
type: ClusterIP
|
|
739
|
+
selector:
|
|
740
|
+
app: ${name}
|
|
741
|
+
ports:
|
|
742
|
+
- name: http
|
|
743
|
+
port: ${port}
|
|
744
|
+
targetPort: ${port}
|
|
745
|
+
- name: proxy
|
|
746
|
+
port: ${PROXY_PORT}
|
|
747
|
+
targetPort: ${PROXY_PORT}
|
|
748
|
+
- name: control
|
|
749
|
+
port: ${CONTROL_PORT}
|
|
750
|
+
targetPort: ${CONTROL_PORT}
|
|
751
|
+
`;
|
|
752
|
+
}
|
|
753
|
+
function generateConfigMap(name, manifest) {
|
|
754
|
+
const data = [
|
|
755
|
+
` AGENT_NAME: "${escapeYamlDoubleQuotedValue(name)}"`,
|
|
756
|
+
` MODEL_PROVIDER: "${escapeYamlDoubleQuotedValue(manifest.spec.model.provider)}"`,
|
|
757
|
+
` MODEL_ID: "${escapeYamlDoubleQuotedValue(manifest.spec.model.id)}"`
|
|
758
|
+
].join("\n");
|
|
759
|
+
return `apiVersion: v1
|
|
760
|
+
kind: ConfigMap
|
|
761
|
+
metadata:
|
|
762
|
+
name: ${name}-config
|
|
763
|
+
labels:
|
|
764
|
+
app: ${name}
|
|
765
|
+
data:
|
|
766
|
+
${data}
|
|
767
|
+
`;
|
|
768
|
+
}
|
|
769
|
+
function generateSecretExample(name, envRefs) {
|
|
770
|
+
const dataLines = envRefs.length > 0 ? envRefs.map((v) => ` ${v}: <base64-encoded-value> # echo -n "<value>" | base64`).join("\n") : " # No $env: references found in manifest";
|
|
771
|
+
return `# Example Kubernetes Secret \u2014 fill in real values and apply with:
|
|
772
|
+
# kubectl apply -f k8s/secret.yaml
|
|
773
|
+
# DO NOT commit real values. This file is a template only.
|
|
774
|
+
apiVersion: v1
|
|
775
|
+
kind: Secret
|
|
776
|
+
metadata:
|
|
777
|
+
name: ${name}-secrets
|
|
778
|
+
labels:
|
|
779
|
+
app: ${name}
|
|
780
|
+
type: Opaque
|
|
781
|
+
data:
|
|
782
|
+
${dataLines}
|
|
783
|
+
`;
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
// src/commands/generate.ts
|
|
787
|
+
var DEPLOY_TARGETS = ["k8s", "helm"];
|
|
788
|
+
function writeGeneratedFiles(files, outDir) {
|
|
789
|
+
const resolvedOutDir = resolve2(outDir);
|
|
790
|
+
mkdirSync(resolvedOutDir, { recursive: true });
|
|
791
|
+
const safeOutDir = resolvedOutDir.endsWith(sep) ? resolvedOutDir : resolvedOutDir + sep;
|
|
792
|
+
for (const [filename, content] of Object.entries(files)) {
|
|
793
|
+
const outPath = resolve2(resolvedOutDir, filename);
|
|
794
|
+
if (!outPath.startsWith(safeOutDir) && outPath !== resolvedOutDir) {
|
|
795
|
+
throw new Error(`Rejected generated filename "${filename}" \u2014 path traversal detected`);
|
|
796
|
+
}
|
|
797
|
+
mkdirSync(dirname(outPath), { recursive: true });
|
|
798
|
+
writeFileSync3(outPath, content, "utf-8");
|
|
799
|
+
console.log(` ${chalk6.green("\u2713")} ${filename}`);
|
|
800
|
+
}
|
|
801
|
+
}
|
|
802
|
+
function copyManifestToOutput(srcFile, outDir, alreadyGenerated) {
|
|
803
|
+
const manifestSrc = resolve2(srcFile);
|
|
804
|
+
const resolvedOutDir = resolve2(outDir);
|
|
805
|
+
if (!existsSync2(manifestSrc) || alreadyGenerated[basename(manifestSrc)] !== void 0) {
|
|
806
|
+
return;
|
|
807
|
+
}
|
|
808
|
+
try {
|
|
809
|
+
let content = readFileSync(manifestSrc, "utf-8");
|
|
810
|
+
content = content.replace(/\$secret:[^\s\n'"]+/g, "<redacted>");
|
|
811
|
+
writeFileSync3(join2(resolvedOutDir, basename(manifestSrc)), content, "utf-8");
|
|
812
|
+
console.log(` ${chalk6.green("\u2713")} ${basename(manifestSrc)} (manifest copy)`);
|
|
813
|
+
} catch {
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
function validateFramework(framework) {
|
|
817
|
+
let available;
|
|
818
|
+
try {
|
|
819
|
+
available = listFrameworks();
|
|
820
|
+
} catch {
|
|
821
|
+
printError(
|
|
822
|
+
"Failed to load available frameworks. Is @agentspec/adapter-claude installed correctly?"
|
|
823
|
+
);
|
|
824
|
+
process.exit(1);
|
|
825
|
+
}
|
|
826
|
+
if (!available.includes(framework)) {
|
|
827
|
+
printError(`Unknown framework "${framework}". Available: ${available.join(", ")}`);
|
|
828
|
+
process.exit(1);
|
|
829
|
+
}
|
|
830
|
+
}
|
|
831
|
+
function handleK8sGeneration(manifest, outDir, outputOpt) {
|
|
832
|
+
printHeader("AgentSpec Deploy \u2014 Kubernetes");
|
|
833
|
+
console.log();
|
|
834
|
+
try {
|
|
835
|
+
writeGeneratedFiles(generateK8sManifests(manifest), outDir);
|
|
836
|
+
} catch (err) {
|
|
837
|
+
printError(String(err));
|
|
838
|
+
process.exit(1);
|
|
839
|
+
}
|
|
840
|
+
printSuccess(`Written to ${outputOpt}`);
|
|
841
|
+
}
|
|
842
|
+
async function handleLLMGeneration(manifest, framework, manifestDir, spin, displayModel) {
|
|
843
|
+
try {
|
|
844
|
+
return await generateWithClaude(manifest, {
|
|
845
|
+
framework,
|
|
846
|
+
manifestDir,
|
|
847
|
+
onProgress: ({ outputChars }) => {
|
|
848
|
+
const kb = (outputChars / 1024).toFixed(1);
|
|
849
|
+
spin.message(`Generating with ${displayModel} \xB7 ${kb}k chars`);
|
|
850
|
+
}
|
|
851
|
+
});
|
|
852
|
+
} catch (err) {
|
|
853
|
+
spin.stop("Generation failed");
|
|
854
|
+
printError(`Generation failed: ${String(err)}`);
|
|
855
|
+
process.exit(1);
|
|
856
|
+
}
|
|
857
|
+
}
|
|
858
|
+
function writePushModeEnv(outDir) {
|
|
859
|
+
const envContent = [
|
|
860
|
+
"AGENTSPEC_URL=https://control-plane.agentspec.io",
|
|
861
|
+
"AGENTSPEC_KEY=<paste key from: agentspec register>",
|
|
862
|
+
""
|
|
863
|
+
].join("\n");
|
|
864
|
+
writeFileSync3(join2(outDir, ".env.agentspec"), envContent, "utf-8");
|
|
865
|
+
console.log(
|
|
866
|
+
` ${chalk6.green("\u2713")} .env.agentspec (push mode \u2014 paste your key from agentspec register)`
|
|
867
|
+
);
|
|
868
|
+
}
|
|
869
|
+
function printDryRunOutput(files) {
|
|
870
|
+
console.log();
|
|
871
|
+
console.log(chalk6.bold(" Files to be generated:"));
|
|
872
|
+
for (const [filename, content] of Object.entries(files)) {
|
|
873
|
+
console.log();
|
|
874
|
+
console.log(chalk6.cyan(` \u2500\u2500 ${filename} \u2500\u2500`));
|
|
875
|
+
console.log(content.split("\n").map((l) => ` ${l}`).join("\n"));
|
|
876
|
+
}
|
|
877
|
+
console.log();
|
|
878
|
+
}
|
|
879
|
+
function printPostGeneration(generated, outputDir) {
|
|
880
|
+
console.log();
|
|
881
|
+
if (generated.installCommands.length > 0) {
|
|
882
|
+
console.log(chalk6.bold(" Install commands:"));
|
|
883
|
+
for (const cmd of generated.installCommands) {
|
|
884
|
+
console.log(` ${chalk6.gray("$")} ${chalk6.cyan(cmd)}`);
|
|
885
|
+
}
|
|
886
|
+
console.log();
|
|
887
|
+
}
|
|
888
|
+
if (generated.envVars.length > 0) {
|
|
889
|
+
console.log(chalk6.bold(" Required env vars:"));
|
|
890
|
+
for (const v of generated.envVars) {
|
|
891
|
+
console.log(` ${chalk6.gray(v)}`);
|
|
892
|
+
}
|
|
893
|
+
console.log();
|
|
894
|
+
}
|
|
895
|
+
printSuccess(`Written to ${outputDir}`);
|
|
896
|
+
}
|
|
897
|
+
async function runDeployTarget(target, manifest, outDir) {
|
|
898
|
+
if (target === "k8s") {
|
|
899
|
+
console.log();
|
|
900
|
+
console.log(chalk6.bold(" Kubernetes manifests:"));
|
|
901
|
+
const deployFiles = generateK8sManifests(manifest);
|
|
902
|
+
writeGeneratedFiles(deployFiles, outDir);
|
|
903
|
+
return;
|
|
904
|
+
}
|
|
905
|
+
if (target === "helm") {
|
|
906
|
+
console.log();
|
|
907
|
+
console.log(chalk6.bold(" Helm chart (Claude-generated):"));
|
|
908
|
+
const helmGenerated = await generateWithClaude(manifest, { framework: "helm" });
|
|
909
|
+
writeGeneratedFiles(helmGenerated.files, outDir);
|
|
910
|
+
}
|
|
911
|
+
}
|
|
912
|
+
function registerGenerateCommand(program2) {
|
|
913
|
+
program2.command("generate <file>").description("Generate framework-specific agent code from a manifest").requiredOption("--framework <fw>", "Target framework (langgraph, crewai, mastra)").option("--output <dir>", "Output directory", "./generated").option("--dry-run", "Print generated files without writing them").option(
|
|
914
|
+
"--deploy <target>",
|
|
915
|
+
`Also generate deployment manifests: ${DEPLOY_TARGETS.join(", ")}`
|
|
916
|
+
).option("--push", "Write .env.agentspec with push mode env var placeholders").action(
|
|
917
|
+
async (file, opts) => {
|
|
918
|
+
validateFramework(opts.framework);
|
|
919
|
+
if (opts.deploy && !DEPLOY_TARGETS.includes(opts.deploy)) {
|
|
920
|
+
printError(
|
|
921
|
+
`Unknown deploy target "${opts.deploy}". Available: ${DEPLOY_TARGETS.join(", ")}`
|
|
922
|
+
);
|
|
923
|
+
process.exit(1);
|
|
924
|
+
}
|
|
925
|
+
let parsed;
|
|
926
|
+
try {
|
|
927
|
+
parsed = loadManifest4(file, { resolve: false });
|
|
928
|
+
} catch (err) {
|
|
929
|
+
printError(`Cannot load manifest: ${String(err)}`);
|
|
930
|
+
process.exit(1);
|
|
931
|
+
}
|
|
932
|
+
if (opts.deploy === "k8s") {
|
|
933
|
+
handleK8sGeneration(parsed.manifest, resolve2(opts.output), opts.output);
|
|
934
|
+
return;
|
|
935
|
+
}
|
|
936
|
+
if (!process.env["ANTHROPIC_API_KEY"]) {
|
|
937
|
+
printError(
|
|
938
|
+
"ANTHROPIC_API_KEY is not set. AgentSpec generates code using Claude.\n Get a key at https://console.anthropic.com and add it to your environment."
|
|
939
|
+
);
|
|
940
|
+
process.exit(1);
|
|
941
|
+
}
|
|
942
|
+
printHeader(`AgentSpec Generate \u2014 ${opts.framework}`);
|
|
943
|
+
const displayModel = process.env["ANTHROPIC_MODEL"] ?? "claude-opus-4-6";
|
|
944
|
+
const spin = spinner();
|
|
945
|
+
spin.start(`Generating with ${displayModel}`);
|
|
946
|
+
const manifestDir = dirname(resolve2(file));
|
|
947
|
+
const generated = await handleLLMGeneration(
|
|
948
|
+
parsed.manifest,
|
|
949
|
+
opts.framework,
|
|
950
|
+
manifestDir,
|
|
951
|
+
spin,
|
|
952
|
+
displayModel
|
|
953
|
+
);
|
|
954
|
+
const totalKb = (Object.values(generated.files).reduce((n, c) => n + c.length, 0) / 1024).toFixed(1);
|
|
955
|
+
spin.stop(`Generated ${Object.keys(generated.files).length} files \xB7 ${totalKb}k chars`);
|
|
956
|
+
if (opts.dryRun) {
|
|
957
|
+
printDryRunOutput(generated.files);
|
|
958
|
+
return;
|
|
959
|
+
}
|
|
960
|
+
const outDir = resolve2(opts.output);
|
|
961
|
+
console.log();
|
|
962
|
+
try {
|
|
963
|
+
writeGeneratedFiles(generated.files, outDir);
|
|
964
|
+
} catch (err) {
|
|
965
|
+
printError(String(err));
|
|
966
|
+
process.exit(1);
|
|
967
|
+
}
|
|
968
|
+
copyManifestToOutput(file, outDir, generated.files);
|
|
969
|
+
if (opts.push) {
|
|
970
|
+
writePushModeEnv(outDir);
|
|
971
|
+
}
|
|
972
|
+
if (opts.deploy === "helm") {
|
|
973
|
+
await runDeployTarget("helm", parsed.manifest, outDir);
|
|
974
|
+
}
|
|
975
|
+
printPostGeneration(generated, opts.output);
|
|
976
|
+
}
|
|
977
|
+
);
|
|
978
|
+
}
|
|
979
|
+
|
|
980
|
+
// src/commands/export.ts
|
|
981
|
+
import { loadManifest as loadManifest5 } from "@agentspec/sdk";
|
|
982
|
+
function registerExportCommand(program2) {
|
|
983
|
+
program2.command("export <file>").description("Export manifest to other formats (agentcard, agents-md-block)").requiredOption("--format <fmt>", "Export format: agentcard|agents-md-block").action(async (file, opts) => {
|
|
984
|
+
let parsed;
|
|
985
|
+
try {
|
|
986
|
+
parsed = loadManifest5(file, { resolve: false });
|
|
987
|
+
} catch (err) {
|
|
988
|
+
printError(`Cannot load manifest: ${String(err)}`);
|
|
989
|
+
process.exit(1);
|
|
990
|
+
}
|
|
991
|
+
const { manifest } = parsed;
|
|
992
|
+
switch (opts.format) {
|
|
993
|
+
case "agentcard": {
|
|
994
|
+
const card = toAgentCard(manifest);
|
|
995
|
+
console.log(JSON.stringify(card, null, 2));
|
|
996
|
+
break;
|
|
997
|
+
}
|
|
998
|
+
case "agents-md-block": {
|
|
999
|
+
const block = toAgentsMdBlock(manifest);
|
|
1000
|
+
console.log(block);
|
|
1001
|
+
break;
|
|
1002
|
+
}
|
|
1003
|
+
default:
|
|
1004
|
+
printError(`Unknown format: ${opts.format}. Use: agentcard|agents-md-block`);
|
|
1005
|
+
process.exit(1);
|
|
1006
|
+
}
|
|
1007
|
+
});
|
|
1008
|
+
}
|
|
1009
|
+
function toAgentCard(manifest) {
|
|
1010
|
+
return {
|
|
1011
|
+
name: manifest.metadata.name,
|
|
1012
|
+
description: manifest.metadata.description,
|
|
1013
|
+
version: manifest.metadata.version,
|
|
1014
|
+
url: manifest.spec.api ? `http://localhost:${manifest.spec.api.port ?? 8e3}${manifest.spec.api.pathPrefix ?? "/api/v1"}` : void 0,
|
|
1015
|
+
capabilities: {
|
|
1016
|
+
streaming: manifest.spec.api?.streaming ?? false,
|
|
1017
|
+
pushNotifications: false,
|
|
1018
|
+
stateTransitionHistory: !!manifest.spec.memory?.longTerm
|
|
1019
|
+
},
|
|
1020
|
+
defaultInputModes: ["text/plain"],
|
|
1021
|
+
defaultOutputModes: ["text/plain"],
|
|
1022
|
+
skills: manifest.spec.skills?.map((s) => ({
|
|
1023
|
+
id: s.id,
|
|
1024
|
+
name: s.id,
|
|
1025
|
+
description: "",
|
|
1026
|
+
tags: manifest.metadata.tags ?? [],
|
|
1027
|
+
examples: []
|
|
1028
|
+
})) ?? [],
|
|
1029
|
+
provider: {
|
|
1030
|
+
organization: manifest.metadata.author ?? "Unknown"
|
|
1031
|
+
},
|
|
1032
|
+
agentSpec: {
|
|
1033
|
+
apiVersion: manifest.apiVersion,
|
|
1034
|
+
manifestRef: "agent.yaml"
|
|
1035
|
+
}
|
|
1036
|
+
};
|
|
1037
|
+
}
|
|
1038
|
+
function toAgentsMdBlock(manifest) {
|
|
1039
|
+
return `## Agent Manifest
|
|
1040
|
+
|
|
1041
|
+
This project uses [AgentSpec](https://agentspec.io) for agent configuration.
|
|
1042
|
+
|
|
1043
|
+
| Field | Value |
|
|
1044
|
+
|-------|-------|
|
|
1045
|
+
| **Name** | ${manifest.metadata.name} |
|
|
1046
|
+
| **Version** | ${manifest.metadata.version} |
|
|
1047
|
+
| **Model** | ${manifest.spec.model.provider}/${manifest.spec.model.id} |
|
|
1048
|
+
| **Manifest** | [agent.yaml](./agent.yaml) |
|
|
1049
|
+
|
|
1050
|
+
See [agent.yaml](./agent.yaml) for the full manifest.
|
|
1051
|
+
|
|
1052
|
+
\`\`\`bash
|
|
1053
|
+
# Check runtime dependencies
|
|
1054
|
+
npx agentspec health agent.yaml
|
|
1055
|
+
|
|
1056
|
+
# Run compliance audit
|
|
1057
|
+
npx agentspec audit agent.yaml
|
|
1058
|
+
|
|
1059
|
+
# Generate framework code
|
|
1060
|
+
npx agentspec generate agent.yaml --framework langgraph --output ./generated/
|
|
1061
|
+
\`\`\`
|
|
1062
|
+
`;
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
// src/commands/migrate.ts
|
|
1066
|
+
import { readFileSync as readFileSync2, writeFileSync as writeFileSync4 } from "fs";
|
|
1067
|
+
import { resolve as resolve3 } from "path";
|
|
1068
|
+
import chalk7 from "chalk";
|
|
1069
|
+
import yaml from "js-yaml";
|
|
1070
|
+
import {
|
|
1071
|
+
migrateManifest,
|
|
1072
|
+
detectVersion,
|
|
1073
|
+
isLatestVersion,
|
|
1074
|
+
LATEST_API_VERSION
|
|
1075
|
+
} from "@agentspec/sdk";
|
|
1076
|
+
function registerMigrateCommand(program2) {
|
|
1077
|
+
program2.command("migrate <file>").description("Migrate an agent.yaml manifest to the latest schema version").option("--dry-run", "Show what would change without writing files").option("-o, --output <file>", "Write migrated manifest to a different file").action(async (file, opts) => {
|
|
1078
|
+
printHeader("AgentSpec Migrate");
|
|
1079
|
+
const absPath = resolve3(file);
|
|
1080
|
+
let raw;
|
|
1081
|
+
try {
|
|
1082
|
+
raw = readFileSync2(absPath, "utf-8");
|
|
1083
|
+
} catch (err) {
|
|
1084
|
+
printError(`Cannot read manifest: ${absPath}
|
|
1085
|
+
${String(err)}`);
|
|
1086
|
+
process.exit(1);
|
|
1087
|
+
}
|
|
1088
|
+
let parsed;
|
|
1089
|
+
try {
|
|
1090
|
+
parsed = yaml.load(raw);
|
|
1091
|
+
} catch (err) {
|
|
1092
|
+
printError(`Invalid YAML: ${String(err)}`);
|
|
1093
|
+
process.exit(1);
|
|
1094
|
+
}
|
|
1095
|
+
const fromVersion = detectVersion(parsed);
|
|
1096
|
+
console.log(chalk7.gray(` File : ${absPath}`));
|
|
1097
|
+
console.log(chalk7.gray(` Version : ${fromVersion}`));
|
|
1098
|
+
console.log();
|
|
1099
|
+
if (isLatestVersion(parsed)) {
|
|
1100
|
+
printSuccess(`Already at latest version (${LATEST_API_VERSION}) \u2014 no migration needed.`);
|
|
1101
|
+
return;
|
|
1102
|
+
}
|
|
1103
|
+
const { result, migrationsApplied } = migrateManifest(parsed);
|
|
1104
|
+
if (migrationsApplied.length === 0) {
|
|
1105
|
+
console.log(chalk7.yellow(` \u26A0 No migration path found from ${fromVersion} to ${LATEST_API_VERSION}.`));
|
|
1106
|
+
console.log(chalk7.gray(" Manual update may be required."));
|
|
1107
|
+
console.log();
|
|
1108
|
+
process.exit(1);
|
|
1109
|
+
}
|
|
1110
|
+
console.log(chalk7.cyan(" Migrations applied:"));
|
|
1111
|
+
for (const m of migrationsApplied) {
|
|
1112
|
+
console.log(chalk7.gray(` \u2022 ${m}`));
|
|
1113
|
+
}
|
|
1114
|
+
console.log();
|
|
1115
|
+
const migratedYaml = yaml.dump(result, { lineWidth: 120 });
|
|
1116
|
+
if (opts.dryRun) {
|
|
1117
|
+
console.log(chalk7.cyan(" --- Migrated manifest (dry-run) ---"));
|
|
1118
|
+
console.log(chalk7.gray(migratedYaml));
|
|
1119
|
+
console.log(chalk7.yellow(" Dry run \u2014 no files written."));
|
|
1120
|
+
console.log();
|
|
1121
|
+
return;
|
|
1122
|
+
}
|
|
1123
|
+
const outputPath = opts.output ? resolve3(opts.output) : absPath;
|
|
1124
|
+
try {
|
|
1125
|
+
writeFileSync4(outputPath, migratedYaml, "utf-8");
|
|
1126
|
+
} catch (err) {
|
|
1127
|
+
printError(`Cannot write output: ${outputPath}
|
|
1128
|
+
${String(err)}`);
|
|
1129
|
+
process.exit(1);
|
|
1130
|
+
}
|
|
1131
|
+
printSuccess(`Migrated to ${LATEST_API_VERSION} \u2014 written to ${outputPath}`);
|
|
1132
|
+
});
|
|
1133
|
+
}
|
|
1134
|
+
|
|
1135
|
+
// src/commands/scan.ts
|
|
1136
|
+
import {
|
|
1137
|
+
existsSync as existsSync3,
|
|
1138
|
+
lstatSync,
|
|
1139
|
+
readdirSync,
|
|
1140
|
+
readFileSync as readFileSync3,
|
|
1141
|
+
realpathSync,
|
|
1142
|
+
writeFileSync as writeFileSync5
|
|
1143
|
+
} from "fs";
|
|
1144
|
+
import { extname, join as join3, resolve as resolve4 } from "path";
|
|
1145
|
+
import { spinner as spinner2 } from "@clack/prompts";
|
|
1146
|
+
import { generateWithClaude as generateWithClaude2 } from "@agentspec/adapter-claude";
|
|
1147
|
+
var MAX_FILES = 50;
|
|
1148
|
+
var MAX_BYTES = 200 * 1024;
|
|
1149
|
+
var SOURCE_EXTENSIONS = /* @__PURE__ */ new Set([".py", ".ts", ".js", ".mjs", ".cjs"]);
|
|
1150
|
+
var SKIP_DIRS = /* @__PURE__ */ new Set([
|
|
1151
|
+
"node_modules",
|
|
1152
|
+
".git",
|
|
1153
|
+
"__pycache__",
|
|
1154
|
+
".venv",
|
|
1155
|
+
"venv",
|
|
1156
|
+
".env",
|
|
1157
|
+
"dist",
|
|
1158
|
+
"build",
|
|
1159
|
+
".next",
|
|
1160
|
+
".nuxt",
|
|
1161
|
+
"coverage",
|
|
1162
|
+
".tox",
|
|
1163
|
+
".mypy_cache",
|
|
1164
|
+
".pytest_cache",
|
|
1165
|
+
".ruff_cache",
|
|
1166
|
+
"target",
|
|
1167
|
+
"out",
|
|
1168
|
+
".turbo"
|
|
1169
|
+
]);
|
|
1170
|
+
function collectSourceFiles(srcDir, maxFiles = MAX_FILES, maxBytes = MAX_BYTES) {
|
|
1171
|
+
let resolvedBase;
|
|
1172
|
+
try {
|
|
1173
|
+
resolvedBase = realpathSync(resolve4(srcDir));
|
|
1174
|
+
} catch {
|
|
1175
|
+
resolvedBase = resolve4(srcDir);
|
|
1176
|
+
}
|
|
1177
|
+
const results = [];
|
|
1178
|
+
let totalBytes = 0;
|
|
1179
|
+
function walk(dir) {
|
|
1180
|
+
if (results.length >= maxFiles) return;
|
|
1181
|
+
if (totalBytes >= maxBytes) return;
|
|
1182
|
+
let entries;
|
|
1183
|
+
try {
|
|
1184
|
+
entries = readdirSync(dir).sort();
|
|
1185
|
+
} catch {
|
|
1186
|
+
return;
|
|
1187
|
+
}
|
|
1188
|
+
for (const entry of entries) {
|
|
1189
|
+
if (results.length >= maxFiles) break;
|
|
1190
|
+
if (totalBytes >= maxBytes) break;
|
|
1191
|
+
if (entry.startsWith(".") || SKIP_DIRS.has(entry)) continue;
|
|
1192
|
+
const fullPath = join3(dir, entry);
|
|
1193
|
+
let stat;
|
|
1194
|
+
try {
|
|
1195
|
+
stat = lstatSync(fullPath);
|
|
1196
|
+
} catch {
|
|
1197
|
+
continue;
|
|
1198
|
+
}
|
|
1199
|
+
if (stat.isSymbolicLink()) continue;
|
|
1200
|
+
if (stat.isDirectory()) {
|
|
1201
|
+
let resolvedDir;
|
|
1202
|
+
try {
|
|
1203
|
+
resolvedDir = realpathSync(fullPath);
|
|
1204
|
+
} catch {
|
|
1205
|
+
continue;
|
|
1206
|
+
}
|
|
1207
|
+
if (!resolvedDir.startsWith(resolvedBase + "/") && resolvedDir !== resolvedBase) continue;
|
|
1208
|
+
walk(fullPath);
|
|
1209
|
+
} else if (stat.isFile() && SOURCE_EXTENSIONS.has(extname(entry))) {
|
|
1210
|
+
let realPath;
|
|
1211
|
+
try {
|
|
1212
|
+
realPath = realpathSync(fullPath);
|
|
1213
|
+
} catch {
|
|
1214
|
+
continue;
|
|
1215
|
+
}
|
|
1216
|
+
if (!realPath.startsWith(resolvedBase + "/") && realPath !== resolvedBase) continue;
|
|
1217
|
+
let content;
|
|
1218
|
+
try {
|
|
1219
|
+
content = readFileSync3(fullPath, "utf-8");
|
|
1220
|
+
} catch {
|
|
1221
|
+
continue;
|
|
1222
|
+
}
|
|
1223
|
+
const remaining = maxBytes - totalBytes;
|
|
1224
|
+
if (content.length > remaining) {
|
|
1225
|
+
content = content.slice(0, remaining);
|
|
1226
|
+
}
|
|
1227
|
+
totalBytes += content.length;
|
|
1228
|
+
results.push({ path: fullPath, content });
|
|
1229
|
+
}
|
|
1230
|
+
}
|
|
1231
|
+
}
|
|
1232
|
+
walk(resolvedBase);
|
|
1233
|
+
return results;
|
|
1234
|
+
}
|
|
1235
|
+
function resolveOutputPath(opts) {
|
|
1236
|
+
if (opts.out) return resolve4(opts.out);
|
|
1237
|
+
const defaultPath = join3(resolve4(opts.srcDir), "agent.yaml");
|
|
1238
|
+
if (opts.update || opts.dryRun) return defaultPath;
|
|
1239
|
+
if (existsSync3(defaultPath)) return join3(resolve4(opts.srcDir), "agent.yaml.new");
|
|
1240
|
+
return defaultPath;
|
|
1241
|
+
}
|
|
1242
|
+
function collectAndValidateSourceFiles(srcDir) {
|
|
1243
|
+
const files = collectSourceFiles(srcDir);
|
|
1244
|
+
if (files.length === 0) {
|
|
1245
|
+
console.warn(`No source files found in ${srcDir}`);
|
|
1246
|
+
}
|
|
1247
|
+
const rawCount = countSourceFiles(srcDir);
|
|
1248
|
+
if (rawCount > MAX_FILES) {
|
|
1249
|
+
console.warn(
|
|
1250
|
+
`Found ${rawCount} source files \u2014 truncating to ${MAX_FILES} files cap. Use a narrower --dir path to scan specific modules.`
|
|
1251
|
+
);
|
|
1252
|
+
}
|
|
1253
|
+
return files;
|
|
1254
|
+
}
|
|
1255
|
+
function validateScanResponse(rawResult) {
|
|
1256
|
+
if (!rawResult || typeof rawResult !== "object" || !("files" in rawResult) || typeof rawResult.files !== "object" || rawResult.files === null) {
|
|
1257
|
+
console.error('Claude returned an unexpected response format (missing "files" object).');
|
|
1258
|
+
process.exit(1);
|
|
1259
|
+
}
|
|
1260
|
+
const agentYaml = rawResult.files["agent.yaml"];
|
|
1261
|
+
if (!agentYaml) {
|
|
1262
|
+
console.error("Claude did not return an agent.yaml in the output.");
|
|
1263
|
+
process.exit(1);
|
|
1264
|
+
}
|
|
1265
|
+
return agentYaml;
|
|
1266
|
+
}
|
|
1267
|
+
function registerScanCommand(program2) {
|
|
1268
|
+
program2.command("scan").description("Scan source code and generate an agent.yaml manifest (Claude-powered)").requiredOption("-d, --dir <src>", "Source directory to scan").option("--out <path>", "Explicit output path").option("--update", "Overwrite existing agent.yaml in place").option("--dry-run", "Print generated YAML to stdout without writing").action(async (opts) => {
|
|
1269
|
+
if (!process.env["ANTHROPIC_API_KEY"]) {
|
|
1270
|
+
console.error(
|
|
1271
|
+
"ANTHROPIC_API_KEY is not set. agentspec scan uses Claude to analyse source code.\nGet a key at https://console.anthropic.com"
|
|
1272
|
+
);
|
|
1273
|
+
process.exit(1);
|
|
1274
|
+
}
|
|
1275
|
+
const srcDir = resolve4(opts.dir);
|
|
1276
|
+
const sourceFiles = collectAndValidateSourceFiles(srcDir);
|
|
1277
|
+
const s = spinner2();
|
|
1278
|
+
s.start("Analysing source code with Claude\u2026");
|
|
1279
|
+
let rawResult;
|
|
1280
|
+
try {
|
|
1281
|
+
rawResult = await generateWithClaude2(
|
|
1282
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
1283
|
+
{},
|
|
1284
|
+
// empty manifest — the scan skill generates one from source
|
|
1285
|
+
{
|
|
1286
|
+
framework: "scan",
|
|
1287
|
+
contextFiles: sourceFiles.map((f) => f.path),
|
|
1288
|
+
manifestDir: srcDir
|
|
1289
|
+
}
|
|
1290
|
+
);
|
|
1291
|
+
} catch (err) {
|
|
1292
|
+
s.stop("Failed");
|
|
1293
|
+
console.error(`Scan failed: ${err.message}`);
|
|
1294
|
+
process.exit(1);
|
|
1295
|
+
}
|
|
1296
|
+
s.stop("Analysis complete");
|
|
1297
|
+
const agentYaml = validateScanResponse(rawResult);
|
|
1298
|
+
if (opts.dryRun) {
|
|
1299
|
+
console.log(agentYaml);
|
|
1300
|
+
return;
|
|
1301
|
+
}
|
|
1302
|
+
const outPath = resolveOutputPath({
|
|
1303
|
+
srcDir,
|
|
1304
|
+
out: opts.out,
|
|
1305
|
+
update: opts.update,
|
|
1306
|
+
dryRun: opts.dryRun
|
|
1307
|
+
});
|
|
1308
|
+
writeFileSync5(outPath, agentYaml, "utf-8");
|
|
1309
|
+
console.log(`\u2713 Written: ${outPath}`);
|
|
1310
|
+
});
|
|
1311
|
+
}
|
|
1312
|
+
function countSourceFiles(srcDir) {
|
|
1313
|
+
let resolvedBase;
|
|
1314
|
+
try {
|
|
1315
|
+
resolvedBase = realpathSync(resolve4(srcDir));
|
|
1316
|
+
} catch {
|
|
1317
|
+
resolvedBase = resolve4(srcDir);
|
|
1318
|
+
}
|
|
1319
|
+
let count = 0;
|
|
1320
|
+
function walk(dir) {
|
|
1321
|
+
let entries;
|
|
1322
|
+
try {
|
|
1323
|
+
entries = readdirSync(dir);
|
|
1324
|
+
} catch {
|
|
1325
|
+
return;
|
|
1326
|
+
}
|
|
1327
|
+
for (const entry of entries) {
|
|
1328
|
+
if (entry.startsWith(".") || SKIP_DIRS.has(entry)) continue;
|
|
1329
|
+
const fullPath = join3(dir, entry);
|
|
1330
|
+
let stat;
|
|
1331
|
+
try {
|
|
1332
|
+
stat = lstatSync(fullPath);
|
|
1333
|
+
} catch {
|
|
1334
|
+
continue;
|
|
1335
|
+
}
|
|
1336
|
+
if (stat.isSymbolicLink()) continue;
|
|
1337
|
+
if (stat.isDirectory()) {
|
|
1338
|
+
let resolvedDir;
|
|
1339
|
+
try {
|
|
1340
|
+
resolvedDir = realpathSync(fullPath);
|
|
1341
|
+
} catch {
|
|
1342
|
+
continue;
|
|
1343
|
+
}
|
|
1344
|
+
if (!resolvedDir.startsWith(resolvedBase + "/") && resolvedDir !== resolvedBase) continue;
|
|
1345
|
+
walk(fullPath);
|
|
1346
|
+
} else if (stat.isFile() && SOURCE_EXTENSIONS.has(extname(entry))) {
|
|
1347
|
+
count++;
|
|
1348
|
+
}
|
|
1349
|
+
}
|
|
1350
|
+
}
|
|
1351
|
+
walk(resolvedBase);
|
|
1352
|
+
return count;
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
// src/commands/diff.ts
|
|
1356
|
+
import { readFileSync as readFileSync4 } from "fs";
|
|
1357
|
+
import yaml2 from "js-yaml";
|
|
1358
|
+
|
|
1359
|
+
// src/utils/diff-score.ts
|
|
1360
|
+
import { scoreToGrade } from "@agentspec/sdk";
|
|
1361
|
+
var DIFF_SCORE_TABLE = {
|
|
1362
|
+
// ── Guardrails ─────────────────────────────────────────────────────────────
|
|
1363
|
+
"spec.guardrails.content_filter": {
|
|
1364
|
+
severity: "HIGH",
|
|
1365
|
+
scoreImpact: -15,
|
|
1366
|
+
description: "Content filtering removed \u2014 user input reaches model unfiltered"
|
|
1367
|
+
},
|
|
1368
|
+
"spec.guardrails.rate_limit": {
|
|
1369
|
+
severity: "HIGH",
|
|
1370
|
+
scoreImpact: -10,
|
|
1371
|
+
description: "Rate limiting removed \u2014 DoS risk increased"
|
|
1372
|
+
},
|
|
1373
|
+
"spec.guardrails.output_validator": {
|
|
1374
|
+
severity: "HIGH",
|
|
1375
|
+
scoreImpact: -10,
|
|
1376
|
+
description: "Output validation removed \u2014 model responses are unchecked"
|
|
1377
|
+
},
|
|
1378
|
+
"spec.guardrails.pii_scrubber": {
|
|
1379
|
+
severity: "HIGH",
|
|
1380
|
+
scoreImpact: -12,
|
|
1381
|
+
description: "PII scrubbing removed \u2014 sensitive data may leak in outputs"
|
|
1382
|
+
},
|
|
1383
|
+
// ── Model ──────────────────────────────────────────────────────────────────
|
|
1384
|
+
"spec.model.apiKey": {
|
|
1385
|
+
severity: "HIGH",
|
|
1386
|
+
scoreImpact: -12,
|
|
1387
|
+
description: "Model API key reference removed \u2014 agent will fail to authenticate"
|
|
1388
|
+
},
|
|
1389
|
+
"spec.model.name": {
|
|
1390
|
+
severity: "MEDIUM",
|
|
1391
|
+
scoreImpact: -5,
|
|
1392
|
+
description: "Model changed \u2014 re-evaluate compliance for new model"
|
|
1393
|
+
},
|
|
1394
|
+
"spec.model.provider": {
|
|
1395
|
+
severity: "MEDIUM",
|
|
1396
|
+
scoreImpact: -8,
|
|
1397
|
+
description: "Model provider changed \u2014 verify guardrails apply to new provider"
|
|
1398
|
+
},
|
|
1399
|
+
"spec.model.fallback": {
|
|
1400
|
+
severity: "MEDIUM",
|
|
1401
|
+
scoreImpact: -5,
|
|
1402
|
+
description: "Fallback model removed \u2014 no resilience on primary model failure"
|
|
1403
|
+
},
|
|
1404
|
+
// ── Evaluation ─────────────────────────────────────────────────────────────
|
|
1405
|
+
"spec.eval.hooks": {
|
|
1406
|
+
severity: "MEDIUM",
|
|
1407
|
+
scoreImpact: -8,
|
|
1408
|
+
description: "Eval hooks removed \u2014 regression detection disabled"
|
|
1409
|
+
},
|
|
1410
|
+
"spec.eval.datasets": {
|
|
1411
|
+
severity: "MEDIUM",
|
|
1412
|
+
scoreImpact: -5,
|
|
1413
|
+
description: "Eval datasets removed \u2014 coverage reduced"
|
|
1414
|
+
},
|
|
1415
|
+
// ── Memory ─────────────────────────────────────────────────────────────────
|
|
1416
|
+
"spec.memory.pii_scrub": {
|
|
1417
|
+
severity: "HIGH",
|
|
1418
|
+
scoreImpact: -10,
|
|
1419
|
+
description: "Memory PII scrubbing removed \u2014 stored data may contain PII"
|
|
1420
|
+
},
|
|
1421
|
+
"spec.memory.ttl": {
|
|
1422
|
+
severity: "LOW",
|
|
1423
|
+
scoreImpact: -3,
|
|
1424
|
+
description: "Memory TTL removed \u2014 data retained indefinitely"
|
|
1425
|
+
},
|
|
1426
|
+
// ── Tools (additions) ──────────────────────────────────────────────────────
|
|
1427
|
+
"spec.tools[+]": {
|
|
1428
|
+
severity: "LOW",
|
|
1429
|
+
scoreImpact: 0,
|
|
1430
|
+
description: "New tool added \u2014 verify it does not expose sensitive data"
|
|
1431
|
+
},
|
|
1432
|
+
// ── Observability ──────────────────────────────────────────────────────────
|
|
1433
|
+
"spec.observability.metrics": {
|
|
1434
|
+
severity: "LOW",
|
|
1435
|
+
scoreImpact: -3,
|
|
1436
|
+
description: "Metrics collection removed \u2014 visibility reduced"
|
|
1437
|
+
},
|
|
1438
|
+
"spec.observability.tracing": {
|
|
1439
|
+
severity: "LOW",
|
|
1440
|
+
scoreImpact: -3,
|
|
1441
|
+
description: "Tracing removed \u2014 debugging capability reduced"
|
|
1442
|
+
}
|
|
1443
|
+
};
|
|
1444
|
+
|
|
1445
|
+
// src/commands/diff.ts
|
|
1446
|
+
var DANGEROUS_KEYS = /* @__PURE__ */ new Set(["__proto__", "constructor", "prototype"]);
|
|
1447
|
+
function flattenObject(obj, prefix = "") {
|
|
1448
|
+
if (obj === null || typeof obj !== "object") {
|
|
1449
|
+
return { [prefix]: obj };
|
|
1450
|
+
}
|
|
1451
|
+
const result = {};
|
|
1452
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
1453
|
+
if (DANGEROUS_KEYS.has(key)) continue;
|
|
1454
|
+
const fullKey = prefix ? `${prefix}.${key}` : key;
|
|
1455
|
+
if (value !== null && typeof value === "object" && !Array.isArray(value)) {
|
|
1456
|
+
Object.assign(result, flattenObject(value, fullKey));
|
|
1457
|
+
} else if (Array.isArray(value)) {
|
|
1458
|
+
if (value.length === 0) {
|
|
1459
|
+
result[fullKey] = value;
|
|
1460
|
+
} else {
|
|
1461
|
+
value.forEach((item, idx) => {
|
|
1462
|
+
const itemKey = `${fullKey}.${idx}`;
|
|
1463
|
+
if (item !== null && typeof item === "object") {
|
|
1464
|
+
Object.assign(result, flattenObject(item, itemKey));
|
|
1465
|
+
} else {
|
|
1466
|
+
result[itemKey] = item;
|
|
1467
|
+
}
|
|
1468
|
+
});
|
|
1469
|
+
}
|
|
1470
|
+
} else {
|
|
1471
|
+
result[fullKey] = value;
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
return result;
|
|
1475
|
+
}
|
|
1476
|
+
function lookupScore(property, changeType) {
|
|
1477
|
+
if (DIFF_SCORE_TABLE[property]) return DIFF_SCORE_TABLE[property];
|
|
1478
|
+
const parts = property.split(".");
|
|
1479
|
+
for (let i = parts.length - 1; i > 0; i--) {
|
|
1480
|
+
const candidate = parts.slice(0, i).join(".");
|
|
1481
|
+
if (DIFF_SCORE_TABLE[candidate]) return DIFF_SCORE_TABLE[candidate];
|
|
1482
|
+
}
|
|
1483
|
+
if (changeType === "added") {
|
|
1484
|
+
return { severity: "LOW", scoreImpact: 0, description: `New field added: ${property}` };
|
|
1485
|
+
}
|
|
1486
|
+
return {
|
|
1487
|
+
severity: "LOW",
|
|
1488
|
+
scoreImpact: -2,
|
|
1489
|
+
description: `Field ${changeType}: ${property}`
|
|
1490
|
+
};
|
|
1491
|
+
}
|
|
1492
|
+
function computeDiff(from, to) {
|
|
1493
|
+
const flatFrom = flattenObject(from);
|
|
1494
|
+
const flatTo = flattenObject(to);
|
|
1495
|
+
const changes = [];
|
|
1496
|
+
const allKeys = /* @__PURE__ */ new Set([...Object.keys(flatFrom), ...Object.keys(flatTo)]);
|
|
1497
|
+
for (const key of allKeys) {
|
|
1498
|
+
const inFrom = Object.prototype.hasOwnProperty.call(flatFrom, key);
|
|
1499
|
+
const inTo = Object.prototype.hasOwnProperty.call(flatTo, key);
|
|
1500
|
+
if (inFrom && !inTo) {
|
|
1501
|
+
const entry = lookupScore(key, "removed");
|
|
1502
|
+
changes.push({
|
|
1503
|
+
type: "removed",
|
|
1504
|
+
property: key,
|
|
1505
|
+
severity: entry.severity,
|
|
1506
|
+
scoreImpact: entry.scoreImpact,
|
|
1507
|
+
description: entry.description,
|
|
1508
|
+
from: flatFrom[key]
|
|
1509
|
+
});
|
|
1510
|
+
} else if (!inFrom && inTo) {
|
|
1511
|
+
const isTool = key.startsWith("spec.tools.");
|
|
1512
|
+
const entry = isTool ? DIFF_SCORE_TABLE["spec.tools[+]"] : lookupScore(key, "added");
|
|
1513
|
+
changes.push({
|
|
1514
|
+
type: "added",
|
|
1515
|
+
property: key,
|
|
1516
|
+
severity: entry.severity,
|
|
1517
|
+
scoreImpact: entry.scoreImpact,
|
|
1518
|
+
description: entry.description,
|
|
1519
|
+
to: flatTo[key]
|
|
1520
|
+
});
|
|
1521
|
+
} else if (JSON.stringify(flatFrom[key]) !== JSON.stringify(flatTo[key])) {
|
|
1522
|
+
const entry = lookupScore(key, "changed");
|
|
1523
|
+
changes.push({
|
|
1524
|
+
type: "changed",
|
|
1525
|
+
property: key,
|
|
1526
|
+
severity: entry.severity,
|
|
1527
|
+
scoreImpact: entry.scoreImpact,
|
|
1528
|
+
description: entry.description,
|
|
1529
|
+
from: flatFrom[key],
|
|
1530
|
+
to: flatTo[key]
|
|
1531
|
+
});
|
|
1532
|
+
}
|
|
1533
|
+
}
|
|
1534
|
+
return changes;
|
|
1535
|
+
}
|
|
1536
|
+
var SEVERITY_COLOR = {
|
|
1537
|
+
HIGH: "\x1B[31m",
|
|
1538
|
+
// red
|
|
1539
|
+
MEDIUM: "\x1B[33m",
|
|
1540
|
+
// yellow
|
|
1541
|
+
LOW: "\x1B[36m",
|
|
1542
|
+
// cyan
|
|
1543
|
+
INFO: "\x1B[37m"
|
|
1544
|
+
// white
|
|
1545
|
+
};
|
|
1546
|
+
var RESET = "\x1B[0m";
|
|
1547
|
+
function printHumanReport(result) {
|
|
1548
|
+
const bar = "\u2550".repeat(54);
|
|
1549
|
+
console.log(`
|
|
1550
|
+
agentspec diff \u2014 compliance drift analysis`);
|
|
1551
|
+
console.log(bar);
|
|
1552
|
+
console.log(` Comparing: ${result.from} \u2192 ${result.to}`);
|
|
1553
|
+
if (result.changes.length === 0) {
|
|
1554
|
+
console.log(`
|
|
1555
|
+
No compliance drift detected.
|
|
1556
|
+
`);
|
|
1557
|
+
console.log(bar);
|
|
1558
|
+
return;
|
|
1559
|
+
}
|
|
1560
|
+
console.log();
|
|
1561
|
+
for (const c of result.changes) {
|
|
1562
|
+
const typeLabel = c.type.toUpperCase().padEnd(8);
|
|
1563
|
+
const impactStr = c.scoreImpact === 0 ? "+0" : String(c.scoreImpact);
|
|
1564
|
+
const color = SEVERITY_COLOR[c.severity] ?? "";
|
|
1565
|
+
console.log(
|
|
1566
|
+
` ${typeLabel} ${c.property.padEnd(38)} [${impactStr} score] ${color}${c.severity}${RESET}`
|
|
1567
|
+
);
|
|
1568
|
+
console.log(` ${c.description}`);
|
|
1569
|
+
console.log();
|
|
1570
|
+
}
|
|
1571
|
+
const netStr = result.netScoreChange >= 0 ? `+${result.netScoreChange}` : String(result.netScoreChange);
|
|
1572
|
+
console.log(
|
|
1573
|
+
` Net score change: ${netStr} (${result.scoreFrom} \u2192 ${result.scoreTo}, ${result.gradeFrom} \u2192 ${result.gradeTo})`
|
|
1574
|
+
);
|
|
1575
|
+
if (result.netScoreChange < 0) {
|
|
1576
|
+
const highChanges = result.changes.filter((c) => c.severity === "HIGH");
|
|
1577
|
+
if (highChanges.length > 0) {
|
|
1578
|
+
console.log(`
|
|
1579
|
+
Recommendation: restore ${highChanges[0].property} before deploying`);
|
|
1580
|
+
}
|
|
1581
|
+
}
|
|
1582
|
+
console.log(bar);
|
|
1583
|
+
}
|
|
1584
|
+
function printJsonReport(result) {
|
|
1585
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1586
|
+
}
|
|
1587
|
+
function registerDiffCommand(program2) {
|
|
1588
|
+
program2.command("diff <from> <to>").description("Detect compliance drift between two agent.yaml manifests").option("--json", "Output machine-readable JSON").option("--exit-code", "Exit with code 1 if drift is detected").action(async (fromPath, toPath, opts) => {
|
|
1589
|
+
let fromRaw;
|
|
1590
|
+
let toRaw;
|
|
1591
|
+
try {
|
|
1592
|
+
fromRaw = yaml2.load(readFileSync4(fromPath, "utf-8"), { schema: yaml2.JSON_SCHEMA });
|
|
1593
|
+
toRaw = yaml2.load(readFileSync4(toPath, "utf-8"), { schema: yaml2.JSON_SCHEMA });
|
|
1594
|
+
} catch (err) {
|
|
1595
|
+
console.error(`Error loading manifests: ${err.message}`);
|
|
1596
|
+
process.exit(1);
|
|
1597
|
+
}
|
|
1598
|
+
const changes = computeDiff(fromRaw, toRaw);
|
|
1599
|
+
const netScoreChange = changes.reduce((sum, c) => sum + c.scoreImpact, 0);
|
|
1600
|
+
const scoreFrom = 100;
|
|
1601
|
+
const scoreTo = Math.max(0, scoreFrom + netScoreChange);
|
|
1602
|
+
const result = {
|
|
1603
|
+
from: fromPath,
|
|
1604
|
+
to: toPath,
|
|
1605
|
+
scoreFrom,
|
|
1606
|
+
scoreTo,
|
|
1607
|
+
gradeFrom: scoreToGrade(scoreFrom),
|
|
1608
|
+
gradeTo: scoreToGrade(scoreTo),
|
|
1609
|
+
netScoreChange,
|
|
1610
|
+
changes
|
|
1611
|
+
};
|
|
1612
|
+
if (opts.json) {
|
|
1613
|
+
printJsonReport(result);
|
|
1614
|
+
} else {
|
|
1615
|
+
printHumanReport(result);
|
|
1616
|
+
}
|
|
1617
|
+
if (opts.exitCode && changes.length > 0) {
|
|
1618
|
+
process.exit(1);
|
|
1619
|
+
}
|
|
1620
|
+
});
|
|
1621
|
+
}
|
|
1622
|
+
|
|
1623
|
+
// src/commands/generate-policy.ts
|
|
1624
|
+
import { mkdirSync as mkdirSync2, writeFileSync as writeFileSync6 } from "fs";
|
|
1625
|
+
import { join as join4, resolve as resolve5 } from "path";
|
|
1626
|
+
import { loadManifest as loadManifest6 } from "@agentspec/sdk";
|
|
1627
|
+
function sanitizePackageName(name) {
|
|
1628
|
+
return name.replace(/-/g, "_");
|
|
1629
|
+
}
|
|
1630
|
+
function generateRegoPolicy(manifest) {
|
|
1631
|
+
const agentName = manifest.metadata.name;
|
|
1632
|
+
const packageName = sanitizePackageName(agentName);
|
|
1633
|
+
const spec = manifest.spec;
|
|
1634
|
+
const lines = [];
|
|
1635
|
+
lines.push(`# Generated by agentspec generate-policy`);
|
|
1636
|
+
lines.push(`# Agent: ${agentName}`);
|
|
1637
|
+
lines.push(`# Generated: ${(/* @__PURE__ */ new Date()).toISOString()}`);
|
|
1638
|
+
lines.push(`#`);
|
|
1639
|
+
lines.push(`# OPA evaluates this policy on every LLM request forwarded by the`);
|
|
1640
|
+
lines.push(`# framework sub-SDK (sdk-langgraph, sdk-crewai, \u2026).`);
|
|
1641
|
+
lines.push(`# Only deny rules relevant to this agent's manifest declarations are included.`);
|
|
1642
|
+
lines.push(`#`);
|
|
1643
|
+
lines.push(`# OPA query: POST /v1/data/agentspec/agent/${packageName}/deny`);
|
|
1644
|
+
lines.push(``);
|
|
1645
|
+
lines.push(`package agentspec.agent.${packageName}`);
|
|
1646
|
+
lines.push(``);
|
|
1647
|
+
lines.push(`import rego.v1`);
|
|
1648
|
+
lines.push(``);
|
|
1649
|
+
lines.push(`default allow := false`);
|
|
1650
|
+
lines.push(``);
|
|
1651
|
+
lines.push(`allow if {`);
|
|
1652
|
+
lines.push(` count(deny) == 0`);
|
|
1653
|
+
lines.push(`}`);
|
|
1654
|
+
lines.push(``);
|
|
1655
|
+
const inputGuardrails = spec.guardrails?.input ?? [];
|
|
1656
|
+
if (inputGuardrails.length > 0) {
|
|
1657
|
+
lines.push(`# \u2500\u2500 Input guardrail enforcement \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500`);
|
|
1658
|
+
lines.push(`# Each declared input guardrail MUST appear in input.guardrails_invoked.`);
|
|
1659
|
+
lines.push(`# The framework sub-SDK is responsible for populating that list.`);
|
|
1660
|
+
lines.push(``);
|
|
1661
|
+
for (const g of inputGuardrails) {
|
|
1662
|
+
const ruleId = g.type.replace(/-/g, "_") + "_not_invoked";
|
|
1663
|
+
lines.push(`deny contains "${ruleId}" if {`);
|
|
1664
|
+
lines.push(` input.request_type == "llm_call"`);
|
|
1665
|
+
lines.push(` not "${g.type}" in input.guardrails_invoked`);
|
|
1666
|
+
lines.push(`}`);
|
|
1667
|
+
lines.push(``);
|
|
1668
|
+
}
|
|
1669
|
+
}
|
|
1670
|
+
const outputGuardrails = spec.guardrails?.output ?? [];
|
|
1671
|
+
if (outputGuardrails.length > 0) {
|
|
1672
|
+
lines.push(`# \u2500\u2500 Output guardrail enforcement \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500`);
|
|
1673
|
+
lines.push(``);
|
|
1674
|
+
for (const g of outputGuardrails) {
|
|
1675
|
+
if (g.type === "toxicity-filter") {
|
|
1676
|
+
lines.push(`deny contains "toxicity_threshold_exceeded" if {`);
|
|
1677
|
+
lines.push(` input.request_type == "llm_call"`);
|
|
1678
|
+
lines.push(` input.toxicity_score >= data.toxicityThreshold`);
|
|
1679
|
+
lines.push(`}`);
|
|
1680
|
+
lines.push(``);
|
|
1681
|
+
} else if (g.type === "hallucination-detector") {
|
|
1682
|
+
lines.push(`deny contains "hallucination_threshold_exceeded" if {`);
|
|
1683
|
+
lines.push(` input.request_type == "llm_call"`);
|
|
1684
|
+
lines.push(` input.hallucination_score >= data.hallucinationThreshold`);
|
|
1685
|
+
lines.push(`}`);
|
|
1686
|
+
lines.push(``);
|
|
1687
|
+
}
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
if (spec.model.costControls?.maxMonthlyUSD !== void 0) {
|
|
1691
|
+
lines.push(`# \u2500\u2500 Cost control enforcement \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500`);
|
|
1692
|
+
lines.push(`# Monthly cost projected from daily spend \xD7 30.`);
|
|
1693
|
+
lines.push(``);
|
|
1694
|
+
lines.push(`deny contains "monthly_cost_limit_exceeded" if {`);
|
|
1695
|
+
lines.push(` input.request_type == "llm_call"`);
|
|
1696
|
+
lines.push(` input.cost_today_usd * 30 > data.maxMonthlyUSD`);
|
|
1697
|
+
lines.push(`}`);
|
|
1698
|
+
lines.push(``);
|
|
1699
|
+
}
|
|
1700
|
+
const ttlSeconds = spec.memory?.shortTerm?.ttlSeconds;
|
|
1701
|
+
if (ttlSeconds !== void 0) {
|
|
1702
|
+
lines.push(`# \u2500\u2500 Memory TTL enforcement \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500`);
|
|
1703
|
+
lines.push(`# Memory writes must use the declared TTL exactly.`);
|
|
1704
|
+
lines.push(``);
|
|
1705
|
+
lines.push(`deny contains "memory_ttl_mismatch" if {`);
|
|
1706
|
+
lines.push(` input.memory_write.ttl_seconds != data.shortTermTtlSeconds`);
|
|
1707
|
+
lines.push(`}`);
|
|
1708
|
+
lines.push(``);
|
|
1709
|
+
}
|
|
1710
|
+
const destructiveTools = (spec.tools ?? []).filter((t) => t.annotations?.destructiveHint === true).map((t) => t.name);
|
|
1711
|
+
if (destructiveTools.length > 0) {
|
|
1712
|
+
lines.push(`# \u2500\u2500 Destructive tool confirmation \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500`);
|
|
1713
|
+
lines.push(`# Tools annotated destructiveHint=true require explicit user confirmation.`);
|
|
1714
|
+
lines.push(``);
|
|
1715
|
+
lines.push(`deny contains "destructive_tool_without_confirmation" if {`);
|
|
1716
|
+
lines.push(` some tool in input.tools_called`);
|
|
1717
|
+
lines.push(` tool in data.destructiveTools`);
|
|
1718
|
+
lines.push(` not input.user_confirmed`);
|
|
1719
|
+
lines.push(`}`);
|
|
1720
|
+
lines.push(``);
|
|
1721
|
+
}
|
|
1722
|
+
return lines.join("\n");
|
|
1723
|
+
}
|
|
1724
|
+
function generateDataJson(manifest) {
|
|
1725
|
+
const spec = manifest.spec;
|
|
1726
|
+
const data = {
|
|
1727
|
+
destructiveTools: []
|
|
1728
|
+
};
|
|
1729
|
+
const toxicityGuardrail = (spec.guardrails?.output ?? []).find(
|
|
1730
|
+
(g) => g.type === "toxicity-filter"
|
|
1731
|
+
);
|
|
1732
|
+
if (toxicityGuardrail && "threshold" in toxicityGuardrail) {
|
|
1733
|
+
data.toxicityThreshold = toxicityGuardrail.threshold;
|
|
1734
|
+
}
|
|
1735
|
+
const hallucinationGuardrail = (spec.guardrails?.output ?? []).find(
|
|
1736
|
+
(g) => g.type === "hallucination-detector"
|
|
1737
|
+
);
|
|
1738
|
+
if (hallucinationGuardrail && "threshold" in hallucinationGuardrail) {
|
|
1739
|
+
data.hallucinationThreshold = hallucinationGuardrail.threshold;
|
|
1740
|
+
}
|
|
1741
|
+
if (spec.memory?.shortTerm?.ttlSeconds !== void 0) {
|
|
1742
|
+
data.shortTermTtlSeconds = spec.memory.shortTerm.ttlSeconds;
|
|
1743
|
+
}
|
|
1744
|
+
if (spec.model.costControls?.maxMonthlyUSD !== void 0) {
|
|
1745
|
+
data.maxMonthlyUSD = spec.model.costControls.maxMonthlyUSD;
|
|
1746
|
+
}
|
|
1747
|
+
if (spec.model.costControls?.alertAtUSD !== void 0) {
|
|
1748
|
+
data.alertAtUSD = spec.model.costControls.alertAtUSD;
|
|
1749
|
+
}
|
|
1750
|
+
data.destructiveTools = (spec.tools ?? []).filter((t) => t.annotations?.destructiveHint === true).map((t) => t.name);
|
|
1751
|
+
if (spec.memory?.hygiene?.piiScrubFields?.length) {
|
|
1752
|
+
data.piiScrubFields = [...spec.memory.hygiene.piiScrubFields];
|
|
1753
|
+
}
|
|
1754
|
+
const topicFilter = (spec.guardrails?.input ?? []).find((g) => g.type === "topic-filter");
|
|
1755
|
+
if (topicFilter && "blockedTopics" in topicFilter) {
|
|
1756
|
+
data.blockedTopics = [...topicFilter.blockedTopics];
|
|
1757
|
+
}
|
|
1758
|
+
return data;
|
|
1759
|
+
}
|
|
1760
|
+
function registerGeneratePolicyCommand(program2) {
|
|
1761
|
+
program2.command("generate-policy <file>").description(
|
|
1762
|
+
"Generate OPA Rego policy bundle from agent.yaml declarations"
|
|
1763
|
+
).option("--out <dir>", "Output directory for generated policy files").option("--json", "Output generation summary as JSON").action(
|
|
1764
|
+
async (file, opts) => {
|
|
1765
|
+
let manifest;
|
|
1766
|
+
try {
|
|
1767
|
+
const result2 = loadManifest6(file, { resolve: false });
|
|
1768
|
+
manifest = result2.manifest;
|
|
1769
|
+
} catch (err) {
|
|
1770
|
+
printError(`Cannot load manifest: ${String(err)}`);
|
|
1771
|
+
process.exit(1);
|
|
1772
|
+
}
|
|
1773
|
+
const agentName = manifest.metadata.name;
|
|
1774
|
+
const outputDir = resolve5(opts.out ?? join4("policies", agentName));
|
|
1775
|
+
const rego = generateRegoPolicy(manifest);
|
|
1776
|
+
const dataJson = generateDataJson(manifest);
|
|
1777
|
+
const rulesGenerated = (rego.match(/^deny contains/gm) ?? []).length;
|
|
1778
|
+
mkdirSync2(outputDir, { recursive: true });
|
|
1779
|
+
const regoPath = join4(outputDir, "policy.rego");
|
|
1780
|
+
const dataPath = join4(outputDir, "data.json");
|
|
1781
|
+
writeFileSync6(regoPath, rego, "utf-8");
|
|
1782
|
+
writeFileSync6(dataPath, JSON.stringify(dataJson, null, 2), "utf-8");
|
|
1783
|
+
const result = {
|
|
1784
|
+
agentName,
|
|
1785
|
+
outputDir,
|
|
1786
|
+
files: ["policy.rego", "data.json"],
|
|
1787
|
+
rulesGenerated
|
|
1788
|
+
};
|
|
1789
|
+
if (opts.json) {
|
|
1790
|
+
console.log(JSON.stringify(result, null, 2));
|
|
1791
|
+
return;
|
|
1792
|
+
}
|
|
1793
|
+
printHeader(`AgentSpec Generate Policy \u2014 ${agentName}`);
|
|
1794
|
+
console.log(` Output : ${outputDir}`);
|
|
1795
|
+
console.log(` Files : policy.rego, data.json`);
|
|
1796
|
+
console.log(` Rules : ${rulesGenerated} deny rules generated`);
|
|
1797
|
+
console.log();
|
|
1798
|
+
console.log(` To serve with OPA:`);
|
|
1799
|
+
console.log(` opa run --server --bundle ${outputDir}`);
|
|
1800
|
+
console.log();
|
|
1801
|
+
}
|
|
1802
|
+
);
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
// src/commands/evaluate.ts
|
|
1806
|
+
import { readFileSync as readFileSync5, existsSync as existsSync4 } from "fs";
|
|
1807
|
+
import { resolve as resolve6, dirname as dirname2 } from "path";
|
|
1808
|
+
import chalk8 from "chalk";
|
|
1809
|
+
import { loadManifest as loadManifest7 } from "@agentspec/sdk";
|
|
1810
|
+
function loadDataset(datasetPath) {
|
|
1811
|
+
if (!existsSync4(datasetPath)) {
|
|
1812
|
+
throw new Error(`Dataset file not found: ${datasetPath}`);
|
|
1813
|
+
}
|
|
1814
|
+
const content = readFileSync5(datasetPath, "utf-8");
|
|
1815
|
+
const samples = [];
|
|
1816
|
+
for (const line of content.split("\n")) {
|
|
1817
|
+
const trimmed = line.trim();
|
|
1818
|
+
if (!trimmed) continue;
|
|
1819
|
+
try {
|
|
1820
|
+
const parsed = JSON.parse(trimmed);
|
|
1821
|
+
if (typeof parsed.input === "string" && typeof parsed.expected === "string") {
|
|
1822
|
+
samples.push({
|
|
1823
|
+
input: parsed.input,
|
|
1824
|
+
expected: parsed.expected,
|
|
1825
|
+
context: Array.isArray(parsed.context) ? parsed.context : void 0,
|
|
1826
|
+
reference_contexts: Array.isArray(parsed.reference_contexts) ? parsed.reference_contexts : void 0,
|
|
1827
|
+
tags: Array.isArray(parsed.tags) ? parsed.tags : void 0,
|
|
1828
|
+
metadata: parsed.metadata !== null && typeof parsed.metadata === "object" && !Array.isArray(parsed.metadata) ? parsed.metadata : void 0
|
|
1829
|
+
});
|
|
1830
|
+
}
|
|
1831
|
+
} catch {
|
|
1832
|
+
}
|
|
1833
|
+
}
|
|
1834
|
+
return samples;
|
|
1835
|
+
}
|
|
1836
|
+
async function sendToAgent(baseUrl, chatPath, input, timeoutMs) {
|
|
1837
|
+
const url = `${baseUrl.replace(/\/$/, "")}${chatPath}`;
|
|
1838
|
+
const res = await fetch(url, {
|
|
1839
|
+
method: "POST",
|
|
1840
|
+
headers: { "Content-Type": "application/json" },
|
|
1841
|
+
body: JSON.stringify({ message: input }),
|
|
1842
|
+
signal: AbortSignal.timeout(timeoutMs)
|
|
1843
|
+
});
|
|
1844
|
+
if (!res.ok) {
|
|
1845
|
+
throw new Error(`Agent returned HTTP ${res.status}`);
|
|
1846
|
+
}
|
|
1847
|
+
const body = await res.json();
|
|
1848
|
+
const text2 = ["response", "message", "content"].map((k) => body[k]).find((v) => typeof v === "string") ?? JSON.stringify(body);
|
|
1849
|
+
return text2;
|
|
1850
|
+
}
|
|
1851
|
+
function scoreStringMatch(expected, actual) {
|
|
1852
|
+
return actual.toLowerCase().includes(expected.toLowerCase());
|
|
1853
|
+
}
|
|
1854
|
+
function buildReport(results, datasetName, agentUrl, manifest) {
|
|
1855
|
+
const total = results.length;
|
|
1856
|
+
const passed = results.filter((r) => r.pass).length;
|
|
1857
|
+
const passRate = total === 0 ? 1 : passed / total;
|
|
1858
|
+
const threshold = manifest.spec.evaluation?.thresholds?.["pass_rate"];
|
|
1859
|
+
const ciGate = manifest.spec.evaluation?.ciGate ?? false;
|
|
1860
|
+
let ciGateResult = "N/A";
|
|
1861
|
+
if (ciGate && threshold !== void 0) {
|
|
1862
|
+
ciGateResult = passRate >= threshold ? "PASS" : "FAIL";
|
|
1863
|
+
}
|
|
1864
|
+
return {
|
|
1865
|
+
dataset: datasetName,
|
|
1866
|
+
agentUrl,
|
|
1867
|
+
totalSamples: total,
|
|
1868
|
+
metrics: {
|
|
1869
|
+
pass_rate: passRate
|
|
1870
|
+
},
|
|
1871
|
+
threshold,
|
|
1872
|
+
ciGateResult,
|
|
1873
|
+
samples: results
|
|
1874
|
+
};
|
|
1875
|
+
}
|
|
1876
|
+
function printTable(report) {
|
|
1877
|
+
printHeader(`AgentSpec Evaluate \u2014 ${report.dataset}`);
|
|
1878
|
+
console.log(
|
|
1879
|
+
` Evaluating: ${chalk8.bold(report.dataset)} ${chalk8.gray(String(report.totalSamples) + " samples")} agent: ${chalk8.cyan(report.agentUrl)}`
|
|
1880
|
+
);
|
|
1881
|
+
console.log(chalk8.gray(" " + "\u2500".repeat(65)));
|
|
1882
|
+
console.log();
|
|
1883
|
+
for (const s of report.samples) {
|
|
1884
|
+
const icon = s.pass ? chalk8.green(" \u2713") : chalk8.red(" \u2717");
|
|
1885
|
+
const expected = chalk8.gray(`"${s.expected}"`);
|
|
1886
|
+
if (s.error) {
|
|
1887
|
+
console.log(`${icon} ${String(s.index + 1).padStart(2)} ${chalk8.gray(s.input.slice(0, 50))} \u2192 ${chalk8.red("error: " + s.error)}`);
|
|
1888
|
+
} else if (s.pass) {
|
|
1889
|
+
console.log(`${icon} ${String(s.index + 1).padStart(2)} ${chalk8.gray(s.input.slice(0, 50))} \u2192 found ${expected} ${chalk8.gray("[" + s.latencyMs + "ms]")}`);
|
|
1890
|
+
} else {
|
|
1891
|
+
console.log(`${icon} ${String(s.index + 1).padStart(2)} ${chalk8.gray(s.input.slice(0, 50))} \u2192 expected ${expected} not found ${chalk8.gray("[" + s.latencyMs + "ms]")}`);
|
|
1892
|
+
}
|
|
1893
|
+
}
|
|
1894
|
+
console.log();
|
|
1895
|
+
console.log(chalk8.bold(" Results"));
|
|
1896
|
+
const { pass_rate } = report.metrics;
|
|
1897
|
+
const pct = Math.round(pass_rate * 100);
|
|
1898
|
+
const pctColor = scoreColor(pct);
|
|
1899
|
+
const threshStr = report.threshold !== void 0 ? `(threshold: ${Math.round(report.threshold * 100)}%)` : "(no threshold declared)";
|
|
1900
|
+
console.log(
|
|
1901
|
+
` pass_rate ${pctColor(pct + "%")} ${chalk8.gray(threshStr)} ${formatCiGate(report.ciGateResult)}`
|
|
1902
|
+
);
|
|
1903
|
+
console.log();
|
|
1904
|
+
console.log(` ciGate: ${formatCiGate(report.ciGateResult)}`);
|
|
1905
|
+
console.log(` Exit code: ${report.ciGateResult === "FAIL" ? chalk8.red("1") : chalk8.green("0")}`);
|
|
1906
|
+
console.log();
|
|
1907
|
+
}
|
|
1908
|
+
function printJSON(report) {
|
|
1909
|
+
console.log(JSON.stringify(report, null, 2));
|
|
1910
|
+
}
|
|
1911
|
+
function resolveChatEndpoint(manifest) {
|
|
1912
|
+
return manifest.spec.api?.chatEndpoint?.path ?? "/v1/chat";
|
|
1913
|
+
}
|
|
1914
|
+
async function runInference(samples, agentUrl, chatPath, timeoutMs) {
|
|
1915
|
+
const results = [];
|
|
1916
|
+
for (let i = 0; i < samples.length; i++) {
|
|
1917
|
+
const sample = samples[i];
|
|
1918
|
+
const start = Date.now();
|
|
1919
|
+
try {
|
|
1920
|
+
const actual = await sendToAgent(agentUrl, chatPath, sample.input, timeoutMs);
|
|
1921
|
+
results.push({
|
|
1922
|
+
index: i,
|
|
1923
|
+
input: sample.input,
|
|
1924
|
+
expected: sample.expected,
|
|
1925
|
+
actual,
|
|
1926
|
+
pass: scoreStringMatch(sample.expected, actual),
|
|
1927
|
+
latencyMs: Date.now() - start
|
|
1928
|
+
});
|
|
1929
|
+
} catch (err) {
|
|
1930
|
+
results.push({
|
|
1931
|
+
index: i,
|
|
1932
|
+
input: sample.input,
|
|
1933
|
+
expected: sample.expected,
|
|
1934
|
+
actual: null,
|
|
1935
|
+
pass: false,
|
|
1936
|
+
latencyMs: Date.now() - start,
|
|
1937
|
+
error: String(err)
|
|
1938
|
+
});
|
|
1939
|
+
}
|
|
1940
|
+
}
|
|
1941
|
+
return results;
|
|
1942
|
+
}
|
|
1943
|
+
function determineCiGateExit(report) {
|
|
1944
|
+
if (report.ciGateResult === "FAIL") process.exit(1);
|
|
1945
|
+
}
|
|
1946
|
+
function registerEvaluateCommand(program2) {
|
|
1947
|
+
program2.command("evaluate <file>").description("Run declared evaluation dataset against a live agent and score outputs").requiredOption("--url <url>", "Agent base URL (e.g. http://localhost:4000)").requiredOption("--dataset <name>", "Dataset name from spec.evaluation.datasets[]").option("--sample-size <n>", "Run only N random samples (default: all)").option("--tag <tag>", "Filter samples by tag").option("--timeout <ms>", "Per-request timeout in ms", "10000").option("--json", "Output JSON instead of table").action(
|
|
1948
|
+
async (file, opts) => {
|
|
1949
|
+
let parsed;
|
|
1950
|
+
try {
|
|
1951
|
+
parsed = loadManifest7(file, { resolve: false });
|
|
1952
|
+
} catch (err) {
|
|
1953
|
+
printError(`Cannot load manifest: ${String(err)}`);
|
|
1954
|
+
process.exit(1);
|
|
1955
|
+
}
|
|
1956
|
+
const { manifest } = parsed;
|
|
1957
|
+
const manifestDir = dirname2(resolve6(file));
|
|
1958
|
+
const timeoutMs = parseInt(opts.timeout, 10) || 1e4;
|
|
1959
|
+
const datasets = manifest.spec.evaluation?.datasets ?? [];
|
|
1960
|
+
const datasetEntry = datasets.find((d) => d.name === opts.dataset);
|
|
1961
|
+
if (!datasetEntry) {
|
|
1962
|
+
printError(
|
|
1963
|
+
`Dataset "${opts.dataset}" not found in spec.evaluation.datasets. Available: ${datasets.map((d) => d.name).join(", ") || "(none)"}`
|
|
1964
|
+
);
|
|
1965
|
+
process.exit(1);
|
|
1966
|
+
}
|
|
1967
|
+
const rawPath = datasetEntry.path;
|
|
1968
|
+
const relPath = rawPath.startsWith("$file:") ? rawPath.slice(6) : rawPath;
|
|
1969
|
+
const absPath = resolve6(manifestDir, relPath);
|
|
1970
|
+
let samples;
|
|
1971
|
+
try {
|
|
1972
|
+
samples = loadDataset(absPath);
|
|
1973
|
+
} catch (err) {
|
|
1974
|
+
printError(`Cannot load dataset: ${String(err)}`);
|
|
1975
|
+
process.exit(1);
|
|
1976
|
+
}
|
|
1977
|
+
if (opts.tag) {
|
|
1978
|
+
samples = samples.filter((s) => s.tags?.includes(opts.tag));
|
|
1979
|
+
}
|
|
1980
|
+
if (opts.sampleSize) {
|
|
1981
|
+
const n = parseInt(opts.sampleSize, 10);
|
|
1982
|
+
if (n > 0 && n < samples.length) {
|
|
1983
|
+
const shuffled = [...samples].sort(() => Math.random() - 0.5);
|
|
1984
|
+
samples = shuffled.slice(0, n);
|
|
1985
|
+
}
|
|
1986
|
+
}
|
|
1987
|
+
const chatPath = resolveChatEndpoint(manifest);
|
|
1988
|
+
const results = await runInference(samples, opts.url, chatPath, timeoutMs);
|
|
1989
|
+
const report = buildReport(results, opts.dataset, opts.url, manifest);
|
|
1990
|
+
if (opts.json) {
|
|
1991
|
+
printJSON(report);
|
|
1992
|
+
} else {
|
|
1993
|
+
printTable(report);
|
|
1994
|
+
}
|
|
1995
|
+
determineCiGateExit(report);
|
|
1996
|
+
}
|
|
1997
|
+
);
|
|
1998
|
+
}
|
|
1999
|
+
|
|
2000
|
+
// src/commands/probe.ts
|
|
2001
|
+
import { spawnSync } from "child_process";
|
|
2002
|
+
import { existsSync as existsSync5 } from "fs";
|
|
2003
|
+
import chalk9 from "chalk";
|
|
2004
|
+
function registerProbeCommand(program2) {
|
|
2005
|
+
const probe = program2.command("probe").description("Run external probes to generate compliance proof records");
|
|
2006
|
+
probe.command("pii <file>").description("Scan agent outputs or logs for unredacted PII (proves SEC-LLM-06, MEM-01, OBS-03)").option("--log-file <path>", "Log file to scan line-by-line (proves OBS-03)").option("--text <text>", "Text to scan for PII; repeat for multiple (proves SEC-LLM-06 + MEM-01)", collect, []).option("--sidecar-url <url>", "Sidecar URL to submit proof on pass (e.g. http://localhost:4001)").option("--submit", "Auto-submit proof to sidecar when scan passes (requires --sidecar-url)").option("--threshold <float>", "Minimum Presidio confidence score to report as PII (default: 0.7)").option("--json", "Output ProbeScanResult as JSON").action(
|
|
2007
|
+
(file, opts) => {
|
|
2008
|
+
if (opts.logFile && opts.text && opts.text.length > 0) {
|
|
2009
|
+
console.error(chalk9.red(" \u2717 --log-file and --text are mutually exclusive"));
|
|
2010
|
+
process.exit(1);
|
|
2011
|
+
}
|
|
2012
|
+
if (!opts.logFile && (!opts.text || opts.text.length === 0)) {
|
|
2013
|
+
console.error(chalk9.red(" \u2717 Provide --log-file <path> or --text <text> (at least one required)"));
|
|
2014
|
+
process.exit(1);
|
|
2015
|
+
}
|
|
2016
|
+
if (opts.submit && !opts.sidecarUrl) {
|
|
2017
|
+
console.error(chalk9.red(" \u2717 --submit requires --sidecar-url"));
|
|
2018
|
+
process.exit(1);
|
|
2019
|
+
}
|
|
2020
|
+
if (opts.threshold !== void 0) {
|
|
2021
|
+
const t = parseFloat(opts.threshold);
|
|
2022
|
+
if (isNaN(t) || t < 0 || t > 1) {
|
|
2023
|
+
console.error(chalk9.red(" \u2717 --threshold must be a number between 0.0 and 1.0"));
|
|
2024
|
+
process.exit(1);
|
|
2025
|
+
}
|
|
2026
|
+
}
|
|
2027
|
+
if (opts.logFile && !existsSync5(opts.logFile)) {
|
|
2028
|
+
console.error(chalk9.red(` \u2717 Log file not found: ${opts.logFile}`));
|
|
2029
|
+
process.exit(1);
|
|
2030
|
+
}
|
|
2031
|
+
const pyArgs = ["-m", "agentspec.presidio_probe", "--manifest", file];
|
|
2032
|
+
if (opts.logFile) {
|
|
2033
|
+
pyArgs.push("--log-file", opts.logFile);
|
|
2034
|
+
}
|
|
2035
|
+
for (const t of opts.text ?? []) {
|
|
2036
|
+
pyArgs.push("--text", t);
|
|
2037
|
+
}
|
|
2038
|
+
if (opts.sidecarUrl) {
|
|
2039
|
+
pyArgs.push("--sidecar-url", opts.sidecarUrl);
|
|
2040
|
+
}
|
|
2041
|
+
if (opts.submit) {
|
|
2042
|
+
pyArgs.push("--submit");
|
|
2043
|
+
}
|
|
2044
|
+
if (opts.threshold) {
|
|
2045
|
+
pyArgs.push("--threshold", opts.threshold);
|
|
2046
|
+
}
|
|
2047
|
+
if (opts.json) {
|
|
2048
|
+
pyArgs.push("--json");
|
|
2049
|
+
}
|
|
2050
|
+
const python = resolvePython();
|
|
2051
|
+
if (!python) {
|
|
2052
|
+
console.error(chalk9.red(" \u2717 Python 3 not found. Install Python 3.10+ to use agentspec probe."));
|
|
2053
|
+
process.exit(1);
|
|
2054
|
+
}
|
|
2055
|
+
const result = spawnSync(python, pyArgs, { stdio: "inherit" });
|
|
2056
|
+
if (result.error) {
|
|
2057
|
+
console.error(chalk9.red(` \u2717 Failed to run Presidio probe: ${result.error.message}`));
|
|
2058
|
+
console.error(chalk9.gray(` Ensure the agentspec Python SDK is installed: pip install agentspec`));
|
|
2059
|
+
process.exit(1);
|
|
2060
|
+
}
|
|
2061
|
+
process.exit(result.status ?? 1);
|
|
2062
|
+
}
|
|
2063
|
+
);
|
|
2064
|
+
}
|
|
2065
|
+
function collect(val, prev) {
|
|
2066
|
+
return [...prev, val];
|
|
2067
|
+
}
|
|
2068
|
+
function resolvePython() {
|
|
2069
|
+
for (const bin of ["python3", "python"]) {
|
|
2070
|
+
const probe = spawnSync(bin, ["--version"], { stdio: "pipe" });
|
|
2071
|
+
if (probe.status === 0) return bin;
|
|
2072
|
+
}
|
|
2073
|
+
return null;
|
|
2074
|
+
}
|
|
2075
|
+
|
|
2076
|
+
// src/cli.ts
|
|
2077
|
+
var program = new Command();
|
|
2078
|
+
program.name("agentspec").description("Universal Agent Manifest System \u2014 validate, health-check, audit, and generate agents").version("0.1.0");
|
|
2079
|
+
registerValidateCommand(program);
|
|
2080
|
+
registerHealthCommand(program);
|
|
2081
|
+
registerAuditCommand(program);
|
|
2082
|
+
registerInitCommand(program);
|
|
2083
|
+
registerGenerateCommand(program);
|
|
2084
|
+
registerExportCommand(program);
|
|
2085
|
+
registerMigrateCommand(program);
|
|
2086
|
+
registerScanCommand(program);
|
|
2087
|
+
registerDiffCommand(program);
|
|
2088
|
+
registerGeneratePolicyCommand(program);
|
|
2089
|
+
registerEvaluateCommand(program);
|
|
2090
|
+
registerProbeCommand(program);
|
|
2091
|
+
program.parse(process.argv);
|
|
2092
|
+
//# sourceMappingURL=cli.js.map
|