agentcert 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -7
- package/dist/cli.js +140 -2
- package/dist/evidence-signing.js +85 -0
- package/dist/index.js +2 -0
- package/dist/normalizers.js +25 -0
- package/dist/release-gate.js +457 -0
- package/dist/schema-validator.js +14 -2
- package/package.json +6 -3
package/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# AgentCert CLI
|
|
2
2
|
|
|
3
|
-
Unified evidence, corpus, monitor, and lab CLI for AgentCert.
|
|
3
|
+
Unified release assurance, evidence, corpus, monitor, and lab CLI for AgentCert.
|
|
4
4
|
|
|
5
|
-
AgentCert
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
AgentCert checks what an agent may do, whether it passed pre-release evidence,
|
|
6
|
+
whether a high-risk runtime action may proceed, and who can verify the observed
|
|
7
|
+
outcome. It writes portable reports and accumulates a local failure corpus.
|
|
8
8
|
|
|
9
9
|
## 5-minute local path
|
|
10
10
|
|
|
@@ -42,16 +42,23 @@ npx agentcert corpus export-reviewed --corpus .agentcert/corpus/corpus.jsonl --o
|
|
|
42
42
|
npx agentcert corpus classifier-eval --corpus .agentcert/corpus/corpus.jsonl --out .agentcert/latest/failure-classifier-evaluation.json
|
|
43
43
|
npx agentcert validate .agentcert/latest/agentcert-evidence.json
|
|
44
44
|
npx agentcert validate .agentcert/latest/agentcert-evidence.json --check-artifacts
|
|
45
|
+
npx agentcert release-gate --config agentcert.config.json --strict
|
|
46
|
+
npx agentcert release-gate --evidence .agentcert/latest/agentcert-evidence.json --baseline .agentcert/baselines/main.json
|
|
45
47
|
npx agentcert schema validate --schema evidence-bundle --file .agentcert/latest/agentcert-evidence.json
|
|
46
48
|
npx agentcert schema validate --schema classifier-eval --file examples/agentcert/classifier-eval.example.json
|
|
47
49
|
```
|
|
48
50
|
|
|
49
|
-
|
|
51
|
+
The release gate writes fixed JSON, HTML, Markdown, JUnit, and badge outputs,
|
|
52
|
+
records SHA-256 artifact provenance, and supports optional Ed25519 signatures:
|
|
50
53
|
|
|
51
|
-
```
|
|
52
|
-
|
|
54
|
+
```bash
|
|
55
|
+
npx agentcert evidence keygen --private-key .agentcert/keys/evidence-private.pem --public-key .agentcert/keys/evidence-public.pem
|
|
56
|
+
npx agentcert evidence sign .agentcert/latest/agentcert-evidence.json --private-key .agentcert/keys/evidence-private.pem
|
|
53
57
|
```
|
|
54
58
|
|
|
59
|
+
Control semantics and attestation format:
|
|
60
|
+
[release gate checklist](https://github.com/Kakarottoooo/agentcert/blob/main/docs/release-gate-checklist.md).
|
|
61
|
+
|
|
55
62
|
CI users can run Tripwire and AgentCert together with
|
|
56
63
|
`Kakarottoooo/agentcert/actions/tripwire@v0`.
|
|
57
64
|
|
package/dist/cli.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
|
-
import { dirname, resolve } from "node:path";
|
|
3
|
+
import { dirname, join, resolve } from "node:path";
|
|
4
4
|
import { validateEvidenceArtifacts } from "./artifact-validation.js";
|
|
5
5
|
import { renderAgentCertBadge } from "./badge.js";
|
|
6
6
|
import { buildEvidenceBundle } from "./bundle.js";
|
|
@@ -10,9 +10,11 @@ import { applyFailureReviews, appendFailureReview, createFailureReview, findFail
|
|
|
10
10
|
import { buildMonitorSnapshot, writeMonitorSnapshot } from "./monitor.js";
|
|
11
11
|
import { normalizeMcpBenchResult, normalizeOnegentAuditPacket, normalizeTripwireResult } from "./normalizers.js";
|
|
12
12
|
import { renderHtmlReport, renderMarkdownReport } from "./report.js";
|
|
13
|
+
import { generateEvidenceSigningKeyPair, signEvidenceFile, verifyEvidenceFile, } from "./evidence-signing.js";
|
|
14
|
+
import { buildReleaseGateReport, renderReleaseGateSummary, writeReleaseGateArtifacts, } from "./release-gate.js";
|
|
13
15
|
import { serveAgentCertMonitor } from "./local-server.js";
|
|
14
16
|
import { parseSchemaId, validateAgentCertSchema } from "./schema-validator.js";
|
|
15
|
-
import { loadRunProfile, profileFromArtifactFlags, renderRunSummary, runAgentCertProfile, } from "./runner.js";
|
|
17
|
+
import { applyRunOverrides, loadRunProfile, profileFromArtifactFlags, renderRunSummary, runAgentCertProfile, } from "./runner.js";
|
|
16
18
|
import { buildRobustnessLabSnapshot, readRobustnessLabConfig, renderRobustnessLabSummary, writeRobustnessLabSnapshot } from "./lab.js";
|
|
17
19
|
process.on("uncaughtException", reportFatalError);
|
|
18
20
|
process.on("unhandledRejection", reportFatalError);
|
|
@@ -52,6 +54,9 @@ if (command === "init") {
|
|
|
52
54
|
},
|
|
53
55
|
gate: {
|
|
54
56
|
failOnVerdict: true,
|
|
57
|
+
strict: false,
|
|
58
|
+
outDir: ".agentcert/latest",
|
|
59
|
+
maxScoreDrop: 0,
|
|
55
60
|
},
|
|
56
61
|
manifest: {
|
|
57
62
|
out: ".agentcert/latest/agentcert-run-manifest.json",
|
|
@@ -266,6 +271,106 @@ else if (command === "run") {
|
|
|
266
271
|
process.stdout.write(renderRunSummary(outcome));
|
|
267
272
|
process.exitCode = outcome.exitCode;
|
|
268
273
|
}
|
|
274
|
+
else if (command === "release-gate") {
|
|
275
|
+
const overrides = readRunOverrides();
|
|
276
|
+
const profileName = readFlag("--profile");
|
|
277
|
+
const configPath = readFlag("--config");
|
|
278
|
+
const evidenceInput = readFlag("--evidence");
|
|
279
|
+
let profile;
|
|
280
|
+
let bundle;
|
|
281
|
+
let evidenceBundlePath;
|
|
282
|
+
let sourceArtifacts;
|
|
283
|
+
if (evidenceInput) {
|
|
284
|
+
profile = configPath ? applyRunOverrides(await loadRunProfile(profileName, configPath), overrides) : undefined;
|
|
285
|
+
bundle = (await readJson(evidenceInput));
|
|
286
|
+
evidenceBundlePath = resolve(evidenceInput);
|
|
287
|
+
sourceArtifacts = bundle.artifacts;
|
|
288
|
+
}
|
|
289
|
+
else {
|
|
290
|
+
const hasArtifactFlags = Boolean(overrides.mcpbench || overrides.tripwire || overrides.onegent);
|
|
291
|
+
const loadedProfile = hasArtifactFlags && !profileName && !configPath
|
|
292
|
+
? profileFromArtifactFlags(overrides)
|
|
293
|
+
: await loadRunProfile(profileName, configPath);
|
|
294
|
+
profile = applyRunOverrides(loadedProfile, overrides);
|
|
295
|
+
const outcome = await runAgentCertProfile(profile, {
|
|
296
|
+
runCommands: !readBoolFlag("--skip-commands"),
|
|
297
|
+
});
|
|
298
|
+
bundle = outcome.bundle;
|
|
299
|
+
evidenceBundlePath = resolve(outcome.manifest.outputs.evidenceBundle ?? join(profile.outputDir, "agentcert-evidence.json"));
|
|
300
|
+
sourceArtifacts = profile.artifacts;
|
|
301
|
+
if (!outcome.manifest.outputs.evidenceBundle) {
|
|
302
|
+
await mkdir(dirname(evidenceBundlePath), { recursive: true });
|
|
303
|
+
await writeFile(evidenceBundlePath, `${JSON.stringify(outcome.bundle, null, 2)}\n`);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
const outDir = resolve(readFlag("--out") ?? profile?.run?.gate?.outDir ?? profile?.outputDir ?? dirname(evidenceBundlePath));
|
|
307
|
+
await mkdir(outDir, { recursive: true });
|
|
308
|
+
const baselinePath = readFlag("--baseline") ?? profile?.run?.gate?.baseline;
|
|
309
|
+
const baseline = baselinePath ? (await readJson(baselinePath)) : undefined;
|
|
310
|
+
const report = await buildReleaseGateReport(bundle, {
|
|
311
|
+
strict: readBoolFlag("--strict") || profile?.run?.gate?.strict,
|
|
312
|
+
requireBaseline: readBoolFlag("--require-baseline") || profile?.run?.gate?.requireBaseline,
|
|
313
|
+
maxScoreDrop: parseOptionalNonNegativeNumber(readFlag("--max-score-drop"), "--max-score-drop") ?? profile?.run?.gate?.maxScoreDrop,
|
|
314
|
+
baseline,
|
|
315
|
+
attestations: profile?.run?.gate?.controls,
|
|
316
|
+
sourceArtifacts,
|
|
317
|
+
evidenceBundlePath,
|
|
318
|
+
});
|
|
319
|
+
const paths = await writeReleaseGateArtifacts(outDir, report);
|
|
320
|
+
const saveBaselinePath = readFlag("--save-baseline");
|
|
321
|
+
if (saveBaselinePath) {
|
|
322
|
+
if (report.verdict.passed) {
|
|
323
|
+
await mkdir(dirname(resolve(saveBaselinePath)), { recursive: true });
|
|
324
|
+
await writeFile(resolve(saveBaselinePath), `${JSON.stringify(bundle, null, 2)}\n`);
|
|
325
|
+
process.stdout.write(`Saved passing baseline ${resolve(saveBaselinePath)}\n`);
|
|
326
|
+
}
|
|
327
|
+
else {
|
|
328
|
+
process.stdout.write(`Baseline not saved because the release gate failed: ${resolve(saveBaselinePath)}\n`);
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
const privateKey = readFlag("--sign-private-key") ?? profile?.run?.gate?.signing?.privateKey;
|
|
332
|
+
if (privateKey) {
|
|
333
|
+
await signEvidenceFile(evidenceBundlePath, privateKey);
|
|
334
|
+
await signEvidenceFile(paths.json, privateKey);
|
|
335
|
+
process.stdout.write(`Signed ${evidenceBundlePath}\nSigned ${paths.json}\n`);
|
|
336
|
+
}
|
|
337
|
+
process.stdout.write(renderReleaseGateSummary(report));
|
|
338
|
+
process.stdout.write(`Release gate JSON: ${paths.json}\nRelease gate HTML: ${paths.html}\nRelease gate JUnit: ${paths.junit}\n`);
|
|
339
|
+
process.exitCode = report.verdict.passed ? 0 : 1;
|
|
340
|
+
}
|
|
341
|
+
else if (command === "evidence") {
|
|
342
|
+
const action = process.argv[3] ?? "help";
|
|
343
|
+
if (action === "keygen") {
|
|
344
|
+
const privateKeyPath = readFlag("--private-key") ?? ".agentcert/keys/evidence-private.pem";
|
|
345
|
+
const publicKeyPath = readFlag("--public-key") ?? ".agentcert/keys/evidence-public.pem";
|
|
346
|
+
await generateEvidenceSigningKeyPair(privateKeyPath, publicKeyPath);
|
|
347
|
+
process.stdout.write(`Wrote private key ${resolve(privateKeyPath)}\nWrote public key ${resolve(publicKeyPath)}\n`);
|
|
348
|
+
}
|
|
349
|
+
else if (action === "sign") {
|
|
350
|
+
const file = readFlag("--file") ?? readPositional(4);
|
|
351
|
+
if (!file)
|
|
352
|
+
throw new Error("Missing evidence file. Usage: agentcert evidence sign <file> --private-key <path>.");
|
|
353
|
+
const privateKeyPath = requiredFlag("--private-key");
|
|
354
|
+
const signaturePath = readFlag("--out") ?? `${file}.sig.json`;
|
|
355
|
+
const signature = await signEvidenceFile(file, privateKeyPath, signaturePath);
|
|
356
|
+
process.stdout.write(`Signed ${resolve(file)}\nSignature: ${resolve(signaturePath)}\nKey id: ${signature.keyId}\n`);
|
|
357
|
+
}
|
|
358
|
+
else if (action === "verify") {
|
|
359
|
+
const file = readFlag("--file") ?? readPositional(4);
|
|
360
|
+
if (!file)
|
|
361
|
+
throw new Error("Missing evidence file. Usage: agentcert evidence verify <file> --signature <path> --public-key <path>.");
|
|
362
|
+
const result = await verifyEvidenceFile(file, requiredFlag("--signature"), requiredFlag("--public-key"));
|
|
363
|
+
process.stdout.write(`${result.valid ? "Valid" : "Invalid"} evidence signature: ${resolve(file)}\nKey id: ${result.keyId}\nSHA-256: ${result.artifactSha256}\n`);
|
|
364
|
+
process.exitCode = result.valid ? 0 : 1;
|
|
365
|
+
}
|
|
366
|
+
else {
|
|
367
|
+
process.stdout.write(`Usage:
|
|
368
|
+
agentcert evidence keygen --private-key .agentcert/keys/evidence-private.pem --public-key .agentcert/keys/evidence-public.pem
|
|
369
|
+
agentcert evidence sign .agentcert/latest/agentcert-evidence.json --private-key .agentcert/keys/evidence-private.pem
|
|
370
|
+
agentcert evidence verify .agentcert/latest/agentcert-evidence.json --signature .agentcert/latest/agentcert-evidence.json.sig.json --public-key .agentcert/keys/evidence-public.pem
|
|
371
|
+
`);
|
|
372
|
+
}
|
|
373
|
+
}
|
|
269
374
|
else if (command === "lab") {
|
|
270
375
|
const action = process.argv[3] ?? "help";
|
|
271
376
|
if (action === "build") {
|
|
@@ -318,6 +423,8 @@ else if (command === "schema") {
|
|
|
318
423
|
agentcert schema validate --schema monitor-snapshot --file .agentcert/latest/monitor.json
|
|
319
424
|
agentcert schema validate --schema corpus-record --file examples/agentcert/corpus-record.example.json
|
|
320
425
|
agentcert schema validate --schema classifier-eval --file examples/agentcert/classifier-eval.example.json
|
|
426
|
+
agentcert schema validate --schema release-gate --file .agentcert/latest/agentcert-release-gate.json
|
|
427
|
+
agentcert schema validate --schema evidence-signature --file .agentcert/latest/agentcert-evidence.json.sig.json
|
|
321
428
|
`);
|
|
322
429
|
}
|
|
323
430
|
}
|
|
@@ -365,6 +472,13 @@ else {
|
|
|
365
472
|
agentcert monitor build --store sqlite --sqlite .agentcert/corpus/agentcert.sqlite --out .agentcert/latest/monitor.json --subject my-agent
|
|
366
473
|
agentcert run --profile public-demo
|
|
367
474
|
agentcert run --mcpbench .mcpbench/latest/results.json --tripwire .tripwire/latest/tripwire-result.json --onegent .onegent/procurement/audit-packet.json --out .agentcert/latest --corpus .agentcert/corpus/corpus.jsonl --monitor-out .agentcert/latest/monitor.json --reviewed-dataset-out .agentcert/latest/reviewed-failure-dataset.jsonl
|
|
475
|
+
agentcert release-gate --config agentcert.config.json --strict
|
|
476
|
+
agentcert release-gate --evidence .agentcert/latest/agentcert-evidence.json --baseline .agentcert/baselines/main.json
|
|
477
|
+
agentcert release-gate --evidence .agentcert/latest/agentcert-evidence.json --save-baseline .agentcert/baselines/main.json
|
|
478
|
+
agentcert release-gate --tripwire .tripwire/latest/tripwire-result.json --baseline .agentcert/baselines/main.json
|
|
479
|
+
agentcert evidence keygen --private-key .agentcert/keys/evidence-private.pem --public-key .agentcert/keys/evidence-public.pem
|
|
480
|
+
agentcert evidence sign .agentcert/latest/agentcert-evidence.json --private-key .agentcert/keys/evidence-private.pem
|
|
481
|
+
agentcert evidence verify .agentcert/latest/agentcert-evidence.json --signature .agentcert/latest/agentcert-evidence.json.sig.json --public-key .agentcert/keys/evidence-public.pem
|
|
368
482
|
agentcert lab build --config examples/real-agents/robustness-lab/lab.config.json --out public-demo/real-agent-robustness/evidence/lab-snapshot.json
|
|
369
483
|
agentcert serve --corpus .agentcert/corpus/corpus.jsonl --static public-demo/agentcert-monitor --artifact-root public-demo/browser-agent-robustness/evidence/tripwire-public-demo
|
|
370
484
|
agentcert validate .agentcert/latest/agentcert-evidence.json
|
|
@@ -418,6 +532,19 @@ function readFirstPositionalAfterCommand() {
|
|
|
418
532
|
}
|
|
419
533
|
return undefined;
|
|
420
534
|
}
|
|
535
|
+
function readPositional(startIndex) {
|
|
536
|
+
for (let index = startIndex; index < process.argv.length; index += 1) {
|
|
537
|
+
const value = process.argv[index];
|
|
538
|
+
if (!value)
|
|
539
|
+
continue;
|
|
540
|
+
if (value.startsWith("--")) {
|
|
541
|
+
index += 1;
|
|
542
|
+
continue;
|
|
543
|
+
}
|
|
544
|
+
return value;
|
|
545
|
+
}
|
|
546
|
+
return undefined;
|
|
547
|
+
}
|
|
421
548
|
function readBoolFlag(name) {
|
|
422
549
|
return process.argv.includes(name);
|
|
423
550
|
}
|
|
@@ -493,6 +620,15 @@ function parseOptionalNonNegativeInteger(input, flagName) {
|
|
|
493
620
|
}
|
|
494
621
|
return value;
|
|
495
622
|
}
|
|
623
|
+
function parseOptionalNonNegativeNumber(input, flagName) {
|
|
624
|
+
if (input === undefined)
|
|
625
|
+
return undefined;
|
|
626
|
+
const value = Number(input);
|
|
627
|
+
if (!Number.isFinite(value) || value < 0) {
|
|
628
|
+
throw new Error(`${flagName} must be a non-negative number.`);
|
|
629
|
+
}
|
|
630
|
+
return value;
|
|
631
|
+
}
|
|
496
632
|
function readReviewsPath() {
|
|
497
633
|
return readFlag("--reviews") ?? process.env.AGENTCERT_FAILURE_REVIEWS;
|
|
498
634
|
}
|
|
@@ -636,6 +772,8 @@ jobs:
|
|
|
636
772
|
subject: ${JSON.stringify(subject)}
|
|
637
773
|
agentcert-out: .agentcert/latest
|
|
638
774
|
fail-on-verdict: "true"
|
|
775
|
+
release-gate: "true"
|
|
776
|
+
strict-release-gate: "false"
|
|
639
777
|
# publish-pages: "true"
|
|
640
778
|
`;
|
|
641
779
|
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { createHash, createPrivateKey, createPublicKey, generateKeyPairSync, sign, verify } from "node:crypto";
|
|
2
|
+
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
|
+
import { dirname, resolve } from "node:path";
|
|
4
|
+
export const AGENTCERT_SIGNATURE_SCHEMA_VERSION = "agentcert.evidence_signature.v0.1";
|
|
5
|
+
export async function generateEvidenceSigningKeyPair(privateKeyPath, publicKeyPath) {
|
|
6
|
+
const { privateKey, publicKey } = generateKeyPairSync("ed25519", {
|
|
7
|
+
privateKeyEncoding: { type: "pkcs8", format: "pem" },
|
|
8
|
+
publicKeyEncoding: { type: "spki", format: "pem" },
|
|
9
|
+
});
|
|
10
|
+
await Promise.all([
|
|
11
|
+
writePrivateFile(privateKeyPath, privateKey),
|
|
12
|
+
writePublicFile(publicKeyPath, publicKey),
|
|
13
|
+
]);
|
|
14
|
+
}
|
|
15
|
+
export async function signEvidenceFile(artifactPath, privateKeyPath, signaturePath = `${artifactPath}.sig.json`) {
|
|
16
|
+
const [artifact, privatePem] = await Promise.all([readFile(resolve(artifactPath)), readFile(resolve(privateKeyPath), "utf8")]);
|
|
17
|
+
const privateKey = createPrivateKey(privatePem);
|
|
18
|
+
const publicKey = createPublicKey(privateKey);
|
|
19
|
+
const envelope = {
|
|
20
|
+
schemaVersion: AGENTCERT_SIGNATURE_SCHEMA_VERSION,
|
|
21
|
+
kind: "agentcert.evidence_signature",
|
|
22
|
+
algorithm: "Ed25519",
|
|
23
|
+
keyId: keyId(publicKey),
|
|
24
|
+
signedAt: new Date().toISOString(),
|
|
25
|
+
artifactPath: artifactPath.replace(/\\/g, "/"),
|
|
26
|
+
artifactSha256: sha256(artifact),
|
|
27
|
+
signature: sign(null, artifact, privateKey).toString("base64"),
|
|
28
|
+
};
|
|
29
|
+
const resolvedSignaturePath = resolve(signaturePath);
|
|
30
|
+
await mkdir(dirname(resolvedSignaturePath), { recursive: true });
|
|
31
|
+
await writeFile(resolvedSignaturePath, `${JSON.stringify(envelope, null, 2)}\n`);
|
|
32
|
+
return envelope;
|
|
33
|
+
}
|
|
34
|
+
export async function verifyEvidenceFile(artifactPath, signaturePath, publicKeyPath) {
|
|
35
|
+
const [artifact, signatureRaw, publicPem] = await Promise.all([
|
|
36
|
+
readFile(resolve(artifactPath)),
|
|
37
|
+
readFile(resolve(signaturePath), "utf8"),
|
|
38
|
+
readFile(resolve(publicKeyPath), "utf8"),
|
|
39
|
+
]);
|
|
40
|
+
const envelope = JSON.parse(signatureRaw);
|
|
41
|
+
const errors = validateEvidenceSignature(envelope);
|
|
42
|
+
if (errors.length > 0)
|
|
43
|
+
throw new Error(`Invalid evidence signature: ${errors.join(" ")}`);
|
|
44
|
+
const publicKey = createPublicKey(publicPem);
|
|
45
|
+
const artifactSha256 = sha256(artifact);
|
|
46
|
+
const actualKeyId = keyId(publicKey);
|
|
47
|
+
const digestMatches = envelope.artifactSha256 === artifactSha256;
|
|
48
|
+
const keyIdMatches = envelope.keyId === actualKeyId;
|
|
49
|
+
const signatureMatches = verify(null, artifact, publicKey, Buffer.from(envelope.signature, "base64"));
|
|
50
|
+
return { valid: digestMatches && signatureMatches && keyIdMatches, digestMatches, signatureMatches, keyIdMatches, artifactSha256, keyId: actualKeyId };
|
|
51
|
+
}
|
|
52
|
+
export function validateEvidenceSignature(input) {
|
|
53
|
+
if (!input || typeof input !== "object" || Array.isArray(input))
|
|
54
|
+
return ["$ must be an object."];
|
|
55
|
+
const value = input;
|
|
56
|
+
const errors = [];
|
|
57
|
+
if (value.schemaVersion !== AGENTCERT_SIGNATURE_SCHEMA_VERSION)
|
|
58
|
+
errors.push(`schemaVersion must be ${JSON.stringify(AGENTCERT_SIGNATURE_SCHEMA_VERSION)}.`);
|
|
59
|
+
if (value.kind !== "agentcert.evidence_signature")
|
|
60
|
+
errors.push('kind must be "agentcert.evidence_signature".');
|
|
61
|
+
if (value.algorithm !== "Ed25519")
|
|
62
|
+
errors.push('algorithm must be "Ed25519".');
|
|
63
|
+
for (const field of ["keyId", "signedAt", "artifactPath", "artifactSha256", "signature"]) {
|
|
64
|
+
if (typeof value[field] !== "string" || value[field].length === 0)
|
|
65
|
+
errors.push(`${field} must be a non-empty string.`);
|
|
66
|
+
}
|
|
67
|
+
return errors;
|
|
68
|
+
}
|
|
69
|
+
function keyId(publicKey) {
|
|
70
|
+
const der = publicKey.export({ type: "spki", format: "der" });
|
|
71
|
+
return `sha256:${sha256(der).slice(0, 32)}`;
|
|
72
|
+
}
|
|
73
|
+
function sha256(input) {
|
|
74
|
+
return createHash("sha256").update(input).digest("hex");
|
|
75
|
+
}
|
|
76
|
+
async function writePrivateFile(path, content) {
|
|
77
|
+
const resolved = resolve(path);
|
|
78
|
+
await mkdir(dirname(resolved), { recursive: true });
|
|
79
|
+
await writeFile(resolved, content, { mode: 0o600, flag: "wx" });
|
|
80
|
+
}
|
|
81
|
+
async function writePublicFile(path, content) {
|
|
82
|
+
const resolved = resolve(path);
|
|
83
|
+
await mkdir(dirname(resolved), { recursive: true });
|
|
84
|
+
await writeFile(resolved, content, { flag: "wx" });
|
|
85
|
+
}
|
package/dist/index.js
CHANGED
|
@@ -2,9 +2,11 @@ export * from "./bundle.js";
|
|
|
2
2
|
export * from "./corpus.js";
|
|
3
3
|
export * from "./corpus-store.js";
|
|
4
4
|
export * from "./failure-review.js";
|
|
5
|
+
export * from "./evidence-signing.js";
|
|
5
6
|
export * from "./local-server.js";
|
|
6
7
|
export * from "./monitor.js";
|
|
7
8
|
export * from "./normalizers.js";
|
|
8
9
|
export * from "./report.js";
|
|
10
|
+
export * from "./release-gate.js";
|
|
9
11
|
export * from "./schema-validator.js";
|
|
10
12
|
export * from "./types.js";
|
package/dist/normalizers.js
CHANGED
|
@@ -94,6 +94,8 @@ export function normalizeOnegentAuditPacket(input, artifactPath) {
|
|
|
94
94
|
const action = asRecord(value.actionIntent);
|
|
95
95
|
const risk = asRecord(value.riskAssessment);
|
|
96
96
|
const approval = asRecord(value.approvalRequest);
|
|
97
|
+
const principal = asRecord(action.principal);
|
|
98
|
+
const authorization = asRecord(value.authorizationDecision);
|
|
97
99
|
const verification = asRecord(value.verificationResult);
|
|
98
100
|
const auditEvents = Array.isArray(value.auditEvents) ? value.auditEvents : [];
|
|
99
101
|
const verificationSuccess = verification.success === true;
|
|
@@ -101,6 +103,18 @@ export function normalizeOnegentAuditPacket(input, artifactPath) {
|
|
|
101
103
|
const passed = verificationSuccess && approved;
|
|
102
104
|
const riskLevel = stringValue(risk.riskLevel) ?? "UNKNOWN";
|
|
103
105
|
const evidence = [
|
|
106
|
+
{
|
|
107
|
+
id: "onegent_principal",
|
|
108
|
+
kind: "runtime_identity",
|
|
109
|
+
severity: "info",
|
|
110
|
+
message: `Runtime action principal: ${stringValue(principal.id) ?? stringValue(action.sourceAgentName) ?? "UNKNOWN"}.`,
|
|
111
|
+
source: "onegent-runtime",
|
|
112
|
+
artifactPath,
|
|
113
|
+
metadata: {
|
|
114
|
+
principal,
|
|
115
|
+
requestedPermissions: action.requestedPermissions,
|
|
116
|
+
},
|
|
117
|
+
},
|
|
104
118
|
{
|
|
105
119
|
id: "onegent_risk_assessment",
|
|
106
120
|
kind: "runtime_risk_assessment",
|
|
@@ -111,6 +125,17 @@ export function normalizeOnegentAuditPacket(input, artifactPath) {
|
|
|
111
125
|
metadata: risk,
|
|
112
126
|
},
|
|
113
127
|
];
|
|
128
|
+
if (authorization.decision) {
|
|
129
|
+
evidence.push({
|
|
130
|
+
id: "onegent_authorization",
|
|
131
|
+
kind: "authorization_decision",
|
|
132
|
+
severity: authorization.decision === "ALLOW" ? "info" : "critical",
|
|
133
|
+
message: `Runtime authorization decision: ${String(authorization.decision)}.`,
|
|
134
|
+
source: "onegent-runtime",
|
|
135
|
+
artifactPath,
|
|
136
|
+
metadata: authorization,
|
|
137
|
+
});
|
|
138
|
+
}
|
|
114
139
|
if (approval.status) {
|
|
115
140
|
evidence.push({
|
|
116
141
|
id: "onegent_approval",
|
|
@@ -0,0 +1,457 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { access, mkdir, readFile, stat, writeFile } from "node:fs/promises";
|
|
3
|
+
import { resolve } from "node:path";
|
|
4
|
+
export const AGENTCERT_RELEASE_GATE_SCHEMA_VERSION = "agentcert.release_gate.v0.1";
|
|
5
|
+
export const RELEASE_GATE_CONTROL_IDS = [
|
|
6
|
+
"permission-boundary",
|
|
7
|
+
"data-boundary",
|
|
8
|
+
"tool-contract",
|
|
9
|
+
"state-verification",
|
|
10
|
+
"human-handoff",
|
|
11
|
+
"rate-loop-cost-limits",
|
|
12
|
+
"idempotency-retry-safety",
|
|
13
|
+
"observability-auditability",
|
|
14
|
+
"rollback-kill-switch",
|
|
15
|
+
"supply-chain-dependency-boundary",
|
|
16
|
+
];
|
|
17
|
+
const CONTROL_DEFINITIONS = [
|
|
18
|
+
{
|
|
19
|
+
id: "permission-boundary",
|
|
20
|
+
name: "Identity, authorization, and least privilege",
|
|
21
|
+
mode: "evidence-required",
|
|
22
|
+
evaluate: evaluatePermissionBoundary,
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
id: "data-boundary",
|
|
26
|
+
name: "Sensitive data boundary",
|
|
27
|
+
mode: "evidence-required",
|
|
28
|
+
evaluate: evaluateDataBoundary,
|
|
29
|
+
},
|
|
30
|
+
{
|
|
31
|
+
id: "tool-contract",
|
|
32
|
+
name: "Tool and MCP contract",
|
|
33
|
+
mode: "automated",
|
|
34
|
+
evaluate: (bundle) => evaluateProduct(bundle, "mcpbench", "MCPBench tool contract evidence is not present."),
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
id: "state-verification",
|
|
38
|
+
name: "Observed state verification",
|
|
39
|
+
mode: "automated",
|
|
40
|
+
evaluate: evaluateStateVerification,
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
id: "human-handoff",
|
|
44
|
+
name: "High-risk action approval and handoff",
|
|
45
|
+
mode: "evidence-required",
|
|
46
|
+
evaluate: evaluateHumanHandoff,
|
|
47
|
+
},
|
|
48
|
+
{
|
|
49
|
+
id: "rate-loop-cost-limits",
|
|
50
|
+
name: "Rate, loop, timeout, and cost limits",
|
|
51
|
+
mode: "evidence-required",
|
|
52
|
+
evaluate: () => unresolved("needs-evidence", "Provide configured step, timeout, retry, and budget limits."),
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
id: "idempotency-retry-safety",
|
|
56
|
+
name: "Idempotency and retry safety",
|
|
57
|
+
mode: "manual",
|
|
58
|
+
evaluate: () => unresolved("manual-review", "A named owner must review duplicate execution and partial failure behavior."),
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
id: "observability-auditability",
|
|
62
|
+
name: "Observability, incident trace, and accountability",
|
|
63
|
+
mode: "automated",
|
|
64
|
+
evaluate: evaluateObservability,
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
id: "rollback-kill-switch",
|
|
68
|
+
name: "Rollback and kill switch",
|
|
69
|
+
mode: "manual",
|
|
70
|
+
evaluate: () => unresolved("manual-review", "A named owner must provide a rollback or kill-switch runbook."),
|
|
71
|
+
},
|
|
72
|
+
{
|
|
73
|
+
id: "supply-chain-dependency-boundary",
|
|
74
|
+
name: "Supply-chain and dependency boundary",
|
|
75
|
+
mode: "manual",
|
|
76
|
+
evaluate: () => unresolved("manual-review", "A named owner must review models, packages, MCP servers, and adapters."),
|
|
77
|
+
},
|
|
78
|
+
];
|
|
79
|
+
export async function buildReleaseGateReport(bundle, options = {}) {
|
|
80
|
+
const strict = options.strict ?? false;
|
|
81
|
+
const controls = CONTROL_DEFINITIONS.map((definition) => {
|
|
82
|
+
const evaluated = definition.evaluate(bundle);
|
|
83
|
+
return applyAttestation({ id: definition.id, name: definition.name, mode: definition.mode, ...evaluated }, options.attestations?.[definition.id]);
|
|
84
|
+
});
|
|
85
|
+
const regression = compareWithBaseline(bundle, options.baseline, {
|
|
86
|
+
requireBaseline: options.requireBaseline ?? false,
|
|
87
|
+
maxScoreDrop: options.maxScoreDrop ?? 0,
|
|
88
|
+
});
|
|
89
|
+
const cwd = resolve(options.cwd ?? process.cwd());
|
|
90
|
+
const sourceArtifacts = await digestArtifacts(options.sourceArtifacts ?? {}, cwd);
|
|
91
|
+
const evidenceBundle = options.evidenceBundlePath
|
|
92
|
+
? await digestArtifact("agentcert-evidence", options.evidenceBundlePath, cwd)
|
|
93
|
+
: undefined;
|
|
94
|
+
const blockers = collectBlockers(bundle, controls, regression, strict, options.requireBaseline ?? false, sourceArtifacts, evidenceBundle);
|
|
95
|
+
return {
|
|
96
|
+
schemaVersion: AGENTCERT_RELEASE_GATE_SCHEMA_VERSION,
|
|
97
|
+
kind: "agentcert.release_gate",
|
|
98
|
+
runId: `release_gate_${createHash("sha256").update(`${bundle.runId}:${JSON.stringify(controls)}`).digest("hex").slice(0, 12)}`,
|
|
99
|
+
generatedAt: new Date().toISOString(),
|
|
100
|
+
subject: bundle.subject,
|
|
101
|
+
strict,
|
|
102
|
+
verdict: {
|
|
103
|
+
passed: blockers.length === 0,
|
|
104
|
+
blockers,
|
|
105
|
+
passedControls: controls.filter((control) => control.status === "pass").length,
|
|
106
|
+
failedControls: controls.filter((control) => control.status === "fail").length,
|
|
107
|
+
needsEvidenceControls: controls.filter((control) => control.status === "needs-evidence").length,
|
|
108
|
+
manualReviewControls: controls.filter((control) => control.status === "manual-review").length,
|
|
109
|
+
},
|
|
110
|
+
bundleVerdict: bundle.verdict,
|
|
111
|
+
controls,
|
|
112
|
+
regression,
|
|
113
|
+
provenance: { evidenceBundle, sourceArtifacts },
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
export async function writeReleaseGateArtifacts(outDir, report) {
|
|
117
|
+
const resolvedOutDir = resolve(outDir);
|
|
118
|
+
await mkdir(resolvedOutDir, { recursive: true });
|
|
119
|
+
const paths = {
|
|
120
|
+
json: resolve(resolvedOutDir, "agentcert-release-gate.json"),
|
|
121
|
+
markdown: resolve(resolvedOutDir, "agentcert-release-gate.md"),
|
|
122
|
+
html: resolve(resolvedOutDir, "agentcert-release-gate.html"),
|
|
123
|
+
junit: resolve(resolvedOutDir, "agentcert-release-gate-junit.xml"),
|
|
124
|
+
badge: resolve(resolvedOutDir, "release-gate-badge.svg"),
|
|
125
|
+
};
|
|
126
|
+
await Promise.all([
|
|
127
|
+
writeFile(paths.json, `${JSON.stringify(report, null, 2)}\n`),
|
|
128
|
+
writeFile(paths.markdown, renderReleaseGateMarkdown(report)),
|
|
129
|
+
writeFile(paths.html, renderReleaseGateHtml(report)),
|
|
130
|
+
writeFile(paths.junit, renderReleaseGateJunit(report)),
|
|
131
|
+
writeFile(paths.badge, renderReleaseGateBadge(report)),
|
|
132
|
+
]);
|
|
133
|
+
return paths;
|
|
134
|
+
}
|
|
135
|
+
export function renderReleaseGateSummary(report) {
|
|
136
|
+
const lines = [
|
|
137
|
+
"# AgentCert Release Gate",
|
|
138
|
+
"",
|
|
139
|
+
`Subject: ${report.subject.name}`,
|
|
140
|
+
`Verdict: ${report.verdict.passed ? "PASS" : "FAIL"}`,
|
|
141
|
+
`Controls: ${report.verdict.passedControls}/10 passed, ${report.verdict.failedControls} failed, ${report.verdict.needsEvidenceControls} need evidence, ${report.verdict.manualReviewControls} need manual review`,
|
|
142
|
+
`Regression: ${report.regression.status}`,
|
|
143
|
+
];
|
|
144
|
+
if (report.verdict.blockers.length > 0) {
|
|
145
|
+
lines.push("", "Blockers:", ...report.verdict.blockers.map((blocker) => `- ${blocker}`));
|
|
146
|
+
}
|
|
147
|
+
return `${lines.join("\n")}\n`;
|
|
148
|
+
}
|
|
149
|
+
export function validateReleaseGateReport(input) {
|
|
150
|
+
const errors = [];
|
|
151
|
+
if (!input || typeof input !== "object" || Array.isArray(input))
|
|
152
|
+
return ["$ must be an object."];
|
|
153
|
+
const value = input;
|
|
154
|
+
if (value.schemaVersion !== AGENTCERT_RELEASE_GATE_SCHEMA_VERSION) {
|
|
155
|
+
errors.push(`schemaVersion must be ${JSON.stringify(AGENTCERT_RELEASE_GATE_SCHEMA_VERSION)}.`);
|
|
156
|
+
}
|
|
157
|
+
if (value.kind !== "agentcert.release_gate")
|
|
158
|
+
errors.push('kind must be "agentcert.release_gate".');
|
|
159
|
+
if (typeof value.runId !== "string" || value.runId.length === 0)
|
|
160
|
+
errors.push("runId must be a non-empty string.");
|
|
161
|
+
if (!Array.isArray(value.controls) || value.controls.length !== RELEASE_GATE_CONTROL_IDS.length) {
|
|
162
|
+
errors.push(`controls must contain ${RELEASE_GATE_CONTROL_IDS.length} entries.`);
|
|
163
|
+
}
|
|
164
|
+
else {
|
|
165
|
+
const seen = new Set();
|
|
166
|
+
for (const [index, controlInput] of value.controls.entries()) {
|
|
167
|
+
const control = record(controlInput);
|
|
168
|
+
if (typeof control.id !== "string" || !RELEASE_GATE_CONTROL_IDS.includes(control.id)) {
|
|
169
|
+
errors.push(`controls[${index}].id is not a supported release control.`);
|
|
170
|
+
}
|
|
171
|
+
else {
|
|
172
|
+
seen.add(control.id);
|
|
173
|
+
}
|
|
174
|
+
if (!["automated", "evidence-required", "manual"].includes(control.mode)) {
|
|
175
|
+
errors.push(`controls[${index}].mode is invalid.`);
|
|
176
|
+
}
|
|
177
|
+
if (!["pass", "fail", "needs-evidence", "manual-review"].includes(control.status)) {
|
|
178
|
+
errors.push(`controls[${index}].status is invalid.`);
|
|
179
|
+
}
|
|
180
|
+
if (!Array.isArray(control.evidence))
|
|
181
|
+
errors.push(`controls[${index}].evidence must be an array.`);
|
|
182
|
+
}
|
|
183
|
+
if (seen.size !== RELEASE_GATE_CONTROL_IDS.length)
|
|
184
|
+
errors.push("controls must contain every release control exactly once.");
|
|
185
|
+
}
|
|
186
|
+
if (!value.verdict || typeof value.verdict !== "object" || Array.isArray(value.verdict)) {
|
|
187
|
+
errors.push("verdict must be an object.");
|
|
188
|
+
}
|
|
189
|
+
else if (typeof record(value.verdict).passed !== "boolean") {
|
|
190
|
+
errors.push("verdict.passed must be a boolean.");
|
|
191
|
+
}
|
|
192
|
+
if (!value.regression || typeof value.regression !== "object" || Array.isArray(value.regression))
|
|
193
|
+
errors.push("regression must be an object.");
|
|
194
|
+
if (!value.provenance || typeof value.provenance !== "object" || Array.isArray(value.provenance))
|
|
195
|
+
errors.push("provenance must be an object.");
|
|
196
|
+
return errors;
|
|
197
|
+
}
|
|
198
|
+
function evaluatePermissionBoundary(bundle) {
|
|
199
|
+
const decision = bundle.evidence.find((item) => item.kind === "authorization_decision");
|
|
200
|
+
if (!decision) {
|
|
201
|
+
return unresolved("needs-evidence", "No runtime authorization decision proves the principal and granted permissions.");
|
|
202
|
+
}
|
|
203
|
+
const metadata = record(decision.metadata);
|
|
204
|
+
const allowed = metadata.decision === "ALLOW";
|
|
205
|
+
return {
|
|
206
|
+
status: allowed ? "pass" : "fail",
|
|
207
|
+
summary: allowed ? "Runtime authorization allowed the recorded principal within granted permissions." : "Runtime authorization denied the action.",
|
|
208
|
+
evidence: evidenceReferences([decision]),
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
function evaluateDataBoundary(bundle) {
|
|
212
|
+
const exposures = bundle.evidence.filter((item) => item.kind === "sensitive_data_exposure" ||
|
|
213
|
+
item.kind === "secret_exposure" ||
|
|
214
|
+
/sensitive|secret|credential|pii/i.test(item.kind));
|
|
215
|
+
if (exposures.some((item) => item.severity === "critical" || item.severity === "high")) {
|
|
216
|
+
return { status: "fail", summary: "Sensitive-data evidence contains a blocking finding.", evidence: evidenceReferences(exposures) };
|
|
217
|
+
}
|
|
218
|
+
return unresolved("needs-evidence", "No blocking leak was observed, but an explicit data-boundary test or attestation is required.");
|
|
219
|
+
}
|
|
220
|
+
function evaluateProduct(bundle, product, missingSummary) {
|
|
221
|
+
const result = bundle.results.find((item) => item.product === product);
|
|
222
|
+
if (!result)
|
|
223
|
+
return unresolved("needs-evidence", missingSummary);
|
|
224
|
+
return {
|
|
225
|
+
status: result.passed ? "pass" : "fail",
|
|
226
|
+
summary: result.summary ?? `${product} ${result.passed ? "passed" : "failed"}.`,
|
|
227
|
+
evidence: resultEvidence(result),
|
|
228
|
+
};
|
|
229
|
+
}
|
|
230
|
+
function evaluateStateVerification(bundle) {
|
|
231
|
+
const results = bundle.results.filter((result) => result.product === "tripwire-ci" || result.product === "onegent-runtime");
|
|
232
|
+
if (results.length === 0) {
|
|
233
|
+
return unresolved("needs-evidence", "No Tripwire or Onegent result proves expected state against observed state.");
|
|
234
|
+
}
|
|
235
|
+
const failed = results.filter((result) => !result.passed);
|
|
236
|
+
return {
|
|
237
|
+
status: failed.length === 0 ? "pass" : "fail",
|
|
238
|
+
summary: failed.length === 0 ? "Configured browser/runtime state verification passed." : `${failed.map((item) => item.product).join(", ")} state verification failed.`,
|
|
239
|
+
evidence: results.flatMap(resultEvidence),
|
|
240
|
+
};
|
|
241
|
+
}
|
|
242
|
+
function evaluateHumanHandoff(bundle) {
|
|
243
|
+
const onegent = bundle.results.find((result) => result.product === "onegent-runtime");
|
|
244
|
+
if (!onegent)
|
|
245
|
+
return unresolved("needs-evidence", "No runtime approval or human-handoff record is present.");
|
|
246
|
+
const approval = onegent.evidence.find((item) => item.kind === "approval_record");
|
|
247
|
+
const risk = onegent.evidence.find((item) => item.kind === "runtime_risk_assessment");
|
|
248
|
+
const requiresApproval = record(risk?.metadata).requiresHumanApproval;
|
|
249
|
+
if (!approval && requiresApproval === false) {
|
|
250
|
+
return { status: "pass", summary: "Recorded risk assessment did not require human approval.", evidence: evidenceReferences(risk ? [risk] : []) };
|
|
251
|
+
}
|
|
252
|
+
if (!approval)
|
|
253
|
+
return unresolved("needs-evidence", "The runtime result does not contain an approval decision.");
|
|
254
|
+
const status = record(approval.metadata).status;
|
|
255
|
+
return {
|
|
256
|
+
status: status === "APPROVED" ? "pass" : "fail",
|
|
257
|
+
summary: status === "APPROVED" ? "The high-risk action has a recorded human approval." : `Approval status ${String(status ?? "UNKNOWN")} does not allow execution.`,
|
|
258
|
+
evidence: evidenceReferences([approval]),
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
function evaluateObservability(bundle) {
|
|
262
|
+
const missingArtifacts = bundle.results.filter((result) => Object.values(result.artifacts).filter(Boolean).length === 0);
|
|
263
|
+
if (!bundle.runId || !bundle.generatedAt || bundle.results.length === 0 || missingArtifacts.length > 0) {
|
|
264
|
+
return { status: "fail", summary: "The bundle is missing traceable run metadata or product artifacts.", evidence: [] };
|
|
265
|
+
}
|
|
266
|
+
return {
|
|
267
|
+
status: "pass",
|
|
268
|
+
summary: "Run identity, timestamps, normalized results, findings, and artifact pointers are present.",
|
|
269
|
+
evidence: bundle.results.flatMap(resultEvidence),
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
function applyAttestation(control, attestation) {
|
|
273
|
+
if (!attestation)
|
|
274
|
+
return control;
|
|
275
|
+
const attestationEvidence = Array.isArray(attestation.evidence) ? attestation.evidence.filter(Boolean) : [];
|
|
276
|
+
const complete = Boolean(attestation.owner && attestation.reviewedAt && attestationEvidence.length > 0);
|
|
277
|
+
if (!complete) {
|
|
278
|
+
return {
|
|
279
|
+
...control,
|
|
280
|
+
status: control.status === "fail" ? "fail" : control.mode === "manual" ? "manual-review" : "needs-evidence",
|
|
281
|
+
summary: `${control.summary} The configured attestation is incomplete; owner, reviewedAt, and evidence are required.`,
|
|
282
|
+
};
|
|
283
|
+
}
|
|
284
|
+
if (control.status === "fail") {
|
|
285
|
+
return { ...control, evidence: [...control.evidence, ...attestationEvidence], owner: attestation.owner, reviewedAt: attestation.reviewedAt };
|
|
286
|
+
}
|
|
287
|
+
if (control.mode === "automated" && control.status !== "pass") {
|
|
288
|
+
return {
|
|
289
|
+
...control,
|
|
290
|
+
summary: `${control.summary} An attestation cannot replace missing automated evidence.`,
|
|
291
|
+
evidence: [...control.evidence, ...attestationEvidence],
|
|
292
|
+
owner: attestation.owner,
|
|
293
|
+
reviewedAt: attestation.reviewedAt,
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
return {
|
|
297
|
+
...control,
|
|
298
|
+
status: attestation.status,
|
|
299
|
+
summary: attestation.note ?? `Control attested ${attestation.status} by ${attestation.owner}.`,
|
|
300
|
+
evidence: [...control.evidence, ...attestationEvidence],
|
|
301
|
+
owner: attestation.owner,
|
|
302
|
+
reviewedAt: attestation.reviewedAt,
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
function compareWithBaseline(current, baseline, options) {
|
|
306
|
+
if (!baseline) {
|
|
307
|
+
return {
|
|
308
|
+
status: options.requireBaseline ? "fail" : "not-configured",
|
|
309
|
+
currentRunId: current.runId,
|
|
310
|
+
maxScoreDrop: options.maxScoreDrop,
|
|
311
|
+
regressions: options.requireBaseline ? ["A baseline evidence bundle is required but was not provided."] : [],
|
|
312
|
+
};
|
|
313
|
+
}
|
|
314
|
+
const regressions = [];
|
|
315
|
+
const scoreDelta = current.verdict.score - baseline.verdict.score;
|
|
316
|
+
if (current.subject.name !== baseline.subject.name) {
|
|
317
|
+
regressions.push(`Baseline subject ${baseline.subject.name} does not match current subject ${current.subject.name}.`);
|
|
318
|
+
}
|
|
319
|
+
if (baseline.verdict.passed && !current.verdict.passed)
|
|
320
|
+
regressions.push("Overall verdict changed from pass to fail.");
|
|
321
|
+
if (scoreDelta < -options.maxScoreDrop) {
|
|
322
|
+
regressions.push(`Overall score dropped by ${Math.abs(scoreDelta)} points; allowed drop is ${options.maxScoreDrop}.`);
|
|
323
|
+
}
|
|
324
|
+
for (const baselineResult of baseline.results) {
|
|
325
|
+
const currentResult = current.results.find((item) => item.product === baselineResult.product);
|
|
326
|
+
if (!currentResult) {
|
|
327
|
+
regressions.push(`${baselineResult.product} evidence is missing from the current run.`);
|
|
328
|
+
}
|
|
329
|
+
else if (baselineResult.passed && !currentResult.passed) {
|
|
330
|
+
regressions.push(`${baselineResult.product} changed from pass to fail.`);
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
return {
|
|
334
|
+
status: regressions.length === 0 ? "pass" : "fail",
|
|
335
|
+
baselineRunId: baseline.runId,
|
|
336
|
+
currentRunId: current.runId,
|
|
337
|
+
scoreDelta,
|
|
338
|
+
maxScoreDrop: options.maxScoreDrop,
|
|
339
|
+
regressions,
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
function collectBlockers(bundle, controls, regression, strict, requireBaseline, sourceArtifacts, evidenceBundle) {
|
|
343
|
+
const blockers = [];
|
|
344
|
+
if (!bundle.verdict.passed)
|
|
345
|
+
blockers.push("The unified AgentCert evidence bundle verdict failed.");
|
|
346
|
+
for (const control of controls) {
|
|
347
|
+
if (control.status === "fail")
|
|
348
|
+
blockers.push(`${control.name}: ${control.summary}`);
|
|
349
|
+
if (strict && control.status === "needs-evidence")
|
|
350
|
+
blockers.push(`${control.name}: evidence is required in strict mode.`);
|
|
351
|
+
if (strict && control.status === "manual-review")
|
|
352
|
+
blockers.push(`${control.name}: manual review is required in strict mode.`);
|
|
353
|
+
}
|
|
354
|
+
if (regression.status === "fail" && (regression.baselineRunId || requireBaseline))
|
|
355
|
+
blockers.push(...regression.regressions);
|
|
356
|
+
for (const artifact of [...sourceArtifacts, ...(evidenceBundle ? [evidenceBundle] : [])]) {
|
|
357
|
+
if (artifact.status === "missing")
|
|
358
|
+
blockers.push(`Referenced artifact is missing: ${artifact.path}`);
|
|
359
|
+
}
|
|
360
|
+
return [...new Set(blockers)];
|
|
361
|
+
}
|
|
362
|
+
async function digestArtifacts(input, cwd) {
|
|
363
|
+
const entries = Object.entries(input).filter((entry) => Boolean(entry[1]));
|
|
364
|
+
return Promise.all(entries.map(([name, path]) => digestArtifact(name, path, cwd)));
|
|
365
|
+
}
|
|
366
|
+
async function digestArtifact(name, path, cwd) {
|
|
367
|
+
if (/^[a-z][a-z0-9+.-]*:\/\//i.test(path))
|
|
368
|
+
return { name, path, status: "remote" };
|
|
369
|
+
const fullPath = resolve(cwd, path);
|
|
370
|
+
try {
|
|
371
|
+
await access(fullPath);
|
|
372
|
+
if ((await stat(fullPath)).isDirectory()) {
|
|
373
|
+
return { name, path: path.replace(/\\/g, "/"), status: "verified" };
|
|
374
|
+
}
|
|
375
|
+
const bytes = await readFile(fullPath);
|
|
376
|
+
return { name, path: path.replace(/\\/g, "/"), sha256: createHash("sha256").update(bytes).digest("hex"), status: "verified" };
|
|
377
|
+
}
|
|
378
|
+
catch {
|
|
379
|
+
return { name, path: path.replace(/\\/g, "/"), status: "missing" };
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
function resultEvidence(result) {
|
|
383
|
+
const evidence = evidenceReferences(result.evidence);
|
|
384
|
+
const artifacts = Object.values(result.artifacts).filter(Boolean);
|
|
385
|
+
return [...new Set([...evidence, ...artifacts])];
|
|
386
|
+
}
|
|
387
|
+
function evidenceReferences(evidence) {
|
|
388
|
+
return evidence.map((item) => item.artifactPath ?? `${item.source ?? "agentcert"}:${item.id}`);
|
|
389
|
+
}
|
|
390
|
+
function unresolved(status, summary) {
|
|
391
|
+
return { status, summary, evidence: [] };
|
|
392
|
+
}
|
|
393
|
+
function record(value) {
|
|
394
|
+
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
395
|
+
}
|
|
396
|
+
function renderReleaseGateMarkdown(report) {
|
|
397
|
+
const lines = [
|
|
398
|
+
"# AgentCert Release Gate",
|
|
399
|
+
"",
|
|
400
|
+
`Subject: ${report.subject.name}`,
|
|
401
|
+
`Generated: ${report.generatedAt}`,
|
|
402
|
+
`Verdict: ${report.verdict.passed ? "PASS" : "FAIL"}`,
|
|
403
|
+
`Mode: ${report.strict ? "strict" : "advisory"}`,
|
|
404
|
+
"",
|
|
405
|
+
"## Controls",
|
|
406
|
+
"",
|
|
407
|
+
"| Control | Mode | Status | Summary |",
|
|
408
|
+
"|---|---|---|---|",
|
|
409
|
+
...report.controls.map((control) => `| ${control.name} | ${control.mode} | ${control.status} | ${control.summary.replaceAll("|", "\\|")} |`),
|
|
410
|
+
"",
|
|
411
|
+
"## Regression",
|
|
412
|
+
"",
|
|
413
|
+
`Status: ${report.regression.status}`,
|
|
414
|
+
...(report.regression.regressions.length > 0 ? report.regression.regressions.map((item) => `- ${item}`) : ["No regression detected or no baseline configured."]),
|
|
415
|
+
"",
|
|
416
|
+
"## Blockers",
|
|
417
|
+
"",
|
|
418
|
+
...(report.verdict.blockers.length > 0 ? report.verdict.blockers.map((item) => `- ${item}`) : ["No release blockers."]),
|
|
419
|
+
"",
|
|
420
|
+
"## Provenance",
|
|
421
|
+
"",
|
|
422
|
+
...[...(report.provenance.evidenceBundle ? [report.provenance.evidenceBundle] : []), ...report.provenance.sourceArtifacts].map((artifact) => `- ${artifact.name}: \`${artifact.path}\` (${artifact.status}${artifact.sha256 ? `, sha256 ${artifact.sha256}` : ""})`),
|
|
423
|
+
];
|
|
424
|
+
return `${lines.join("\n")}\n`;
|
|
425
|
+
}
|
|
426
|
+
function renderReleaseGateHtml(report) {
|
|
427
|
+
const rows = report.controls.map((control) => `<tr><td>${escapeHtml(control.name)}</td><td>${escapeHtml(control.mode)}</td><td><strong class="${control.status}">${escapeHtml(control.status)}</strong></td><td>${escapeHtml(control.summary)}</td></tr>`).join("");
|
|
428
|
+
const blockers = report.verdict.blockers.length > 0 ? report.verdict.blockers.map((item) => `<li>${escapeHtml(item)}</li>`).join("") : "<li>No release blockers.</li>";
|
|
429
|
+
return `<!doctype html><html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>AgentCert Release Gate</title><style>:root{--ink:#132033;--muted:#617083;--line:#d7dee8;--bg:#f5f7fa;--pass:#087f5b;--fail:#c92a2a;--warn:#a45a00}body{margin:0;background:var(--bg);color:var(--ink);font:15px/1.5 system-ui,sans-serif}main{width:min(1100px,calc(100% - 32px));margin:auto;padding:36px 0}h1{font-size:42px;margin:0}.verdict{font-size:28px;color:${report.verdict.passed ? "var(--pass)" : "var(--fail)"}}section{background:#fff;border:1px solid var(--line);border-radius:8px;padding:20px;margin-top:16px;overflow:auto}table{width:100%;border-collapse:collapse;min-width:760px}th,td{text-align:left;padding:10px;border-bottom:1px solid var(--line);vertical-align:top}.pass{color:var(--pass)}.fail{color:var(--fail)}.needs-evidence,.manual-review{color:var(--warn)}p{color:var(--muted)}</style></head><body><main><p>AgentCert Assurance Control Plane</p><h1>${escapeHtml(report.subject.name)}</h1><strong class="verdict">${report.verdict.passed ? "RELEASE READY" : "RELEASE BLOCKED"}</strong><p>${escapeHtml(report.generatedAt)} · ${report.strict ? "strict" : "advisory"} mode · baseline ${escapeHtml(report.regression.status)}</p><section><h2>Ten release controls</h2><table><thead><tr><th>Control</th><th>Mode</th><th>Status</th><th>Decision</th></tr></thead><tbody>${rows}</tbody></table></section><section><h2>Blockers</h2><ul>${blockers}</ul></section></main></body></html>\n`;
|
|
430
|
+
}
|
|
431
|
+
function renderReleaseGateJunit(report) {
|
|
432
|
+
const failures = report.controls.filter((control) => control.status === "fail" || (report.strict && control.status !== "pass")).length + (report.regression.status === "fail" ? 1 : 0);
|
|
433
|
+
const skipped = (report.strict ? 0 : report.controls.filter((control) => control.status === "needs-evidence" || control.status === "manual-review").length) + (report.regression.status === "not-configured" ? 1 : 0);
|
|
434
|
+
const tests = report.controls.length + 1;
|
|
435
|
+
const cases = report.controls.map((control) => {
|
|
436
|
+
const body = control.status === "fail" || (report.strict && control.status !== "pass")
|
|
437
|
+
? `<failure message="${escapeXml(control.summary)}"/>`
|
|
438
|
+
: control.status === "needs-evidence" || control.status === "manual-review"
|
|
439
|
+
? `<skipped message="${escapeXml(control.summary)}"/>`
|
|
440
|
+
: "";
|
|
441
|
+
return `<testcase classname="agentcert.release-gate" name="${escapeXml(control.id)}">${body}</testcase>`;
|
|
442
|
+
});
|
|
443
|
+
const regressionBody = report.regression.status === "fail" ? `<failure message="${escapeXml(report.regression.regressions.join(" "))}"/>` : report.regression.status === "not-configured" ? '<skipped message="No baseline configured."/>' : "";
|
|
444
|
+
cases.push(`<testcase classname="agentcert.release-gate" name="continuous-regression">${regressionBody}</testcase>`);
|
|
445
|
+
return `<?xml version="1.0" encoding="UTF-8"?>\n<testsuite name="AgentCert release gate" tests="${tests}" failures="${failures}" skipped="${skipped}">\n ${cases.join("\n ")}\n</testsuite>\n`;
|
|
446
|
+
}
|
|
447
|
+
function renderReleaseGateBadge(report) {
|
|
448
|
+
const status = report.verdict.passed ? "ready" : "blocked";
|
|
449
|
+
const color = report.verdict.passed ? "#087f5b" : "#c92a2a";
|
|
450
|
+
return `<svg xmlns="http://www.w3.org/2000/svg" width="188" height="20" role="img" aria-label="agentcert release: ${status}"><title>agentcert release: ${status}</title><rect width="112" height="20" rx="3" fill="#132033"/><rect x="109" width="79" height="20" rx="3" fill="${color}"/><g fill="#fff" text-anchor="middle" font-family="Verdana,sans-serif" font-size="11"><text x="56" y="14">agentcert release</text><text x="149" y="14">${status}</text></g></svg>\n`;
|
|
451
|
+
}
|
|
452
|
+
function escapeHtml(value) {
|
|
453
|
+
return String(value).replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """);
|
|
454
|
+
}
|
|
455
|
+
function escapeXml(value) {
|
|
456
|
+
return escapeHtml(value).replaceAll("'", "'");
|
|
457
|
+
}
|
package/dist/schema-validator.js
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
import { AGENTCERT_EVIDENCE_SCHEMA_SEMVER, AGENTCERT_EVIDENCE_SCHEMA_VERSION } from "./types.js";
|
|
2
|
+
import { validateEvidenceSignature } from "./evidence-signing.js";
|
|
3
|
+
import { validateReleaseGateReport } from "./release-gate.js";
|
|
2
4
|
export function parseSchemaId(input) {
|
|
3
5
|
const value = input ?? "evidence-bundle";
|
|
4
6
|
if (value === "evidence-bundle" ||
|
|
@@ -7,13 +9,23 @@ export function parseSchemaId(input) {
|
|
|
7
9
|
value === "failure-review" ||
|
|
8
10
|
value === "classifier-eval" ||
|
|
9
11
|
value === "monitor-snapshot" ||
|
|
10
|
-
value === "robustness-lab"
|
|
12
|
+
value === "robustness-lab" ||
|
|
13
|
+
value === "release-gate" ||
|
|
14
|
+
value === "evidence-signature") {
|
|
11
15
|
return value;
|
|
12
16
|
}
|
|
13
|
-
throw new Error(`Unsupported schema "${value}". Use evidence-bundle, result, corpus-record, failure-review, classifier-eval, monitor-snapshot, or
|
|
17
|
+
throw new Error(`Unsupported schema "${value}". Use evidence-bundle, result, corpus-record, failure-review, classifier-eval, monitor-snapshot, robustness-lab, release-gate, or evidence-signature.`);
|
|
14
18
|
}
|
|
15
19
|
export function validateAgentCertSchema(schema, input) {
|
|
16
20
|
const errors = [];
|
|
21
|
+
if (schema === "release-gate") {
|
|
22
|
+
errors.push(...validateReleaseGateReport(input));
|
|
23
|
+
return { schema, valid: errors.length === 0, errors };
|
|
24
|
+
}
|
|
25
|
+
if (schema === "evidence-signature") {
|
|
26
|
+
errors.push(...validateEvidenceSignature(input));
|
|
27
|
+
return { schema, valid: errors.length === 0, errors };
|
|
28
|
+
}
|
|
17
29
|
const value = object(input, "$", errors);
|
|
18
30
|
if (value) {
|
|
19
31
|
if (schema === "evidence-bundle")
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agentcert",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Assurance release gates, runtime evidence, and regression CI for AI agents.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
7
7
|
"homepage": "https://github.com/Kakarottoooo/agentcert#readme",
|
|
@@ -22,7 +22,10 @@
|
|
|
22
22
|
"llm",
|
|
23
23
|
"mcp",
|
|
24
24
|
"tripwire",
|
|
25
|
-
"evidence"
|
|
25
|
+
"evidence",
|
|
26
|
+
"assurance",
|
|
27
|
+
"authorization",
|
|
28
|
+
"audit"
|
|
26
29
|
],
|
|
27
30
|
"bin": {
|
|
28
31
|
"agentcert": "dist/cli.js"
|