agentcert 0.5.4 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +29 -0
- package/dist/cli.js +44 -1
- package/dist/command-help.js +5 -0
- package/dist/control-plane.js +232 -35
- package/dist/schema-validator.js +68 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -57,6 +57,35 @@ npx agentcert connect --server https://agentcert.app --project your-project-id
|
|
|
57
57
|
npx agentcert push --evidence .agentcert/latest/agentcert-evidence.json
|
|
58
58
|
```
|
|
59
59
|
|
|
60
|
+
Bind a release, pull-request, or nightly run to an issued continuous assurance
|
|
61
|
+
case by declaring the exact reviewed scope:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
npx agentcert run --config agentcert.config.json --push \
|
|
65
|
+
--assurance-case "$AGENTCERT_ASSURANCE_CASE_ID" \
|
|
66
|
+
--assurance-scope agentcert.assurance-scope.json \
|
|
67
|
+
--assurance-trigger auto \
|
|
68
|
+
--require-current auto \
|
|
69
|
+
--continuous-health-out .agentcert/canary/generated-kit-health.json
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Validate the scope before CI uses it:
|
|
73
|
+
|
|
74
|
+
```bash
|
|
75
|
+
npx agentcert schema validate \
|
|
76
|
+
--schema assurance-scope \
|
|
77
|
+
--file agentcert.assurance-scope.json
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
`auto` treats pull requests as prospective, scheduled workflows as nightly,
|
|
81
|
+
and other GitHub runs as release checks. Authoritative failure or scope drift
|
|
82
|
+
sets the Hosted contract to `REVALIDATION_REQUIRED`; only an independently
|
|
83
|
+
issued successor case establishes a new `CURRENT` baseline.
|
|
84
|
+
Release and nightly checks fail unless Hosted returns `CURRENT`. The optional
|
|
85
|
+
health output is redacted and captures the Hosted run/evidence identifiers,
|
|
86
|
+
evidence completeness, freshness transition, and install-to-CURRENT timing for
|
|
87
|
+
external canaries and operational dashboards.
|
|
88
|
+
|
|
60
89
|
Add `--push` to `agentcert run` to run locally and upload the resulting bundle
|
|
61
90
|
in one command. By default, both commands also upload local files referenced by
|
|
62
91
|
the bundle. Reads are confined to `--artifact-root` (the current directory by
|
package/dist/cli.js
CHANGED
|
@@ -9,7 +9,7 @@ import { collectCompanionArtifacts, MAX_REPORTED_COMPANION_ARTIFACT_SKIPS } from
|
|
|
9
9
|
import { recordsFromAgentCertResult, evaluateFailureClassifier, renderCorpusSummary, summarizeCorpus, writeReviewedFailureDataset, } from "./corpus.js";
|
|
10
10
|
import { openCorpusStore, parseCorpusStoreKind } from "./corpus-store.js";
|
|
11
11
|
import { deleteCorpusRecords, exportGovernedCorpus, governCorpusRecords, parseCorpusConsent } from "./corpus-governance.js";
|
|
12
|
-
import { pushEvidenceToControlPlane, verifyControlPlaneConnection } from "./control-plane.js";
|
|
12
|
+
import { pushEvidenceToControlPlane, requireContinuousAssuranceCurrent, verifyControlPlaneConnection } from "./control-plane.js";
|
|
13
13
|
import { runEvidenceConformance } from "./conformance.js";
|
|
14
14
|
import { DEFAULT_AGENTCERT_SERVER, resolveConnection, saveConnection } from "./credentials.js";
|
|
15
15
|
import { applyFailureReviews, appendFailureReview, createFailureReview, findFailurePattern, parseFailureReviewStatus, parseFailureType, parseReviewConfidence, readFailureReviews, } from "./failure-review.js";
|
|
@@ -601,6 +601,11 @@ async function pushHostedEvidence(bundle, bytes, fileName) {
|
|
|
601
601
|
const companions = readBoolFlag("--no-artifacts")
|
|
602
602
|
? undefined
|
|
603
603
|
: await collectCompanionArtifacts(bundle, resolve(readFlag("--artifact-root") ?? process.cwd()));
|
|
604
|
+
const assurance = await readContinuousAssuranceBinding();
|
|
605
|
+
const healthOut = readFlag("--continuous-health-out");
|
|
606
|
+
const requireCurrent = readBoolFlag("--require-current");
|
|
607
|
+
if (requireCurrent && !assurance)
|
|
608
|
+
throw new Error("--require-current needs --assurance-case and --assurance-scope.");
|
|
604
609
|
const result = await pushEvidenceToControlPlane({
|
|
605
610
|
baseUrl: connection.server,
|
|
606
611
|
projectId: connection.projectId,
|
|
@@ -611,6 +616,8 @@ async function pushHostedEvidence(bundle, bytes, fileName) {
|
|
|
611
616
|
externalId: readFlag("--external-id"),
|
|
612
617
|
companionArtifacts: companions?.artifacts,
|
|
613
618
|
skippedCompanionArtifacts: companions?.skipped,
|
|
619
|
+
assurance,
|
|
620
|
+
verifyContinuousAssurance: Boolean(assurance && (healthOut || requireCurrent)),
|
|
614
621
|
});
|
|
615
622
|
process.stdout.write(`Hosted run: ${result.runId}\nHosted evidence: ${result.evidenceId}\n`);
|
|
616
623
|
if (companions) {
|
|
@@ -622,6 +629,42 @@ async function pushHostedEvidence(bundle, bytes, fileName) {
|
|
|
622
629
|
process.stderr.write(`${companions.skipped.length - MAX_REPORTED_COMPANION_ARTIFACT_SKIPS} additional skipped artifacts omitted.\n`);
|
|
623
630
|
}
|
|
624
631
|
}
|
|
632
|
+
if (result.continuousAssuranceHealth) {
|
|
633
|
+
process.stdout.write(`Hosted assurance: ${result.continuousAssuranceHealth.status} (${result.continuousAssuranceHealth.healthy ? "healthy" : "attention required"})\n`);
|
|
634
|
+
if (healthOut) {
|
|
635
|
+
const outputPath = resolve(healthOut);
|
|
636
|
+
await mkdir(dirname(outputPath), { recursive: true });
|
|
637
|
+
await writeFile(outputPath, `${JSON.stringify(result.continuousAssuranceHealth, null, 2)}\n`, "utf8");
|
|
638
|
+
process.stdout.write(`Continuous assurance health: ${outputPath}\n`);
|
|
639
|
+
}
|
|
640
|
+
if (requireCurrent)
|
|
641
|
+
requireContinuousAssuranceCurrent(result.continuousAssuranceHealth);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
async function readContinuousAssuranceBinding() {
|
|
645
|
+
const caseId = readFlag("--assurance-case");
|
|
646
|
+
const scopePath = readFlag("--assurance-scope");
|
|
647
|
+
if (!caseId && !scopePath)
|
|
648
|
+
return undefined;
|
|
649
|
+
if (!caseId || !scopePath)
|
|
650
|
+
throw new Error("--assurance-case and --assurance-scope must be provided together.");
|
|
651
|
+
const scope = await readJson(scopePath);
|
|
652
|
+
const validation = validateAgentCertSchema("assurance-scope", scope);
|
|
653
|
+
if (!validation.valid)
|
|
654
|
+
throw new Error(`Assurance scope is invalid:\n${validation.errors.map((error) => `- ${error}`).join("\n")}`);
|
|
655
|
+
return { caseId, trigger: assuranceTriggerFromEnvironment(readFlag("--assurance-trigger") ?? "auto"), scope: scope };
|
|
656
|
+
}
|
|
657
|
+
function assuranceTriggerFromEnvironment(value) {
|
|
658
|
+
if (value === "pull_request" || value === "release" || value === "nightly")
|
|
659
|
+
return value;
|
|
660
|
+
if (value !== "auto")
|
|
661
|
+
throw new Error("--assurance-trigger must be auto, pull_request, release, or nightly.");
|
|
662
|
+
const event = process.env.GITHUB_EVENT_NAME;
|
|
663
|
+
if (event === "pull_request" || event === "pull_request_target")
|
|
664
|
+
return "pull_request";
|
|
665
|
+
if (event === "schedule")
|
|
666
|
+
return "nightly";
|
|
667
|
+
return "release";
|
|
625
668
|
}
|
|
626
669
|
async function promptValue(label) {
|
|
627
670
|
if (!process.stdin.isTTY || !process.stdout.isTTY) {
|
package/dist/command-help.js
CHANGED
|
@@ -34,6 +34,11 @@ Options:
|
|
|
34
34
|
--project <id> Hosted project ID
|
|
35
35
|
--api-key <key> Project API key (prefer AGENTCERT_API_KEY in CI)
|
|
36
36
|
--external-id <id> Idempotent hosted run ID
|
|
37
|
+
--assurance-case <id> Issued assurance case to reconcile
|
|
38
|
+
--assurance-scope <p> Declared agent/model/prompt/tools/policy/suite scope JSON
|
|
39
|
+
--assurance-trigger <t> auto, pull_request, release, or nightly (default: auto)
|
|
40
|
+
--require-current Fail unless Hosted confirms authoritative CURRENT and complete evidence
|
|
41
|
+
--continuous-health-out <path> Write the redacted Hosted health contract
|
|
37
42
|
--artifact-root <dir> Allowed root for companion artifacts (default: current directory)
|
|
38
43
|
--no-artifacts Upload only the evidence bundle
|
|
39
44
|
--help, -h Show this help
|
package/dist/control-plane.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
1
2
|
import { serializeHostedEvidenceBundle } from "./artifact-manifest.js";
|
|
2
3
|
import { MAX_REPORTED_COMPANION_ARTIFACT_SKIPS, } from "./companion-artifacts.js";
|
|
3
4
|
export class ControlPlaneRequestError extends Error {
|
|
@@ -58,6 +59,8 @@ export async function pushEvidenceToControlPlane(options) {
|
|
|
58
59
|
const externalId = options.externalId ?? bundle.runId;
|
|
59
60
|
const kind = hostedRunKind(bundle);
|
|
60
61
|
const headers = { authorization: `Bearer ${options.apiKey}` };
|
|
62
|
+
const trace = hostedTrace(options.projectId, externalId);
|
|
63
|
+
const observationEvents = buildHostedObservationEvents(bundle, trace);
|
|
61
64
|
const run = await requestJson(request, `${projectUrl}/runs`, {
|
|
62
65
|
method: "POST",
|
|
63
66
|
headers: { ...headers, "content-type": "application/json" },
|
|
@@ -66,28 +69,20 @@ export async function pushEvidenceToControlPlane(options) {
|
|
|
66
69
|
kind,
|
|
67
70
|
schemaVersion: bundle.schemaVersion,
|
|
68
71
|
startedAt: bundle.generatedAt,
|
|
72
|
+
traceId: trace.traceId,
|
|
73
|
+
rootSpanId: trace.rootSpanId,
|
|
69
74
|
metadata: {
|
|
70
75
|
subject: bundle.subject,
|
|
71
76
|
products: bundle.summary.products,
|
|
72
77
|
},
|
|
78
|
+
assurance: options.assurance,
|
|
73
79
|
}),
|
|
74
80
|
});
|
|
75
81
|
await requestJson(request, `${projectUrl}/runs/${encodeURIComponent(run.id)}/events`, {
|
|
76
82
|
method: "POST",
|
|
77
83
|
headers: { ...headers, "content-type": "application/json" },
|
|
78
84
|
body: JSON.stringify({
|
|
79
|
-
events:
|
|
80
|
-
sequence: 0,
|
|
81
|
-
type: "agentcert.evidence.created",
|
|
82
|
-
actor: "agentcert-cli",
|
|
83
|
-
occurredAt: bundle.generatedAt,
|
|
84
|
-
payload: {
|
|
85
|
-
verdict: bundle.verdict,
|
|
86
|
-
totalEvidence: bundle.summary.totalEvidence,
|
|
87
|
-
criticalEvidence: bundle.summary.criticalEvidence,
|
|
88
|
-
highEvidence: bundle.summary.highEvidence,
|
|
89
|
-
},
|
|
90
|
-
}],
|
|
85
|
+
events: observationEvents,
|
|
91
86
|
}),
|
|
92
87
|
});
|
|
93
88
|
const query = new URLSearchParams({
|
|
@@ -118,29 +113,32 @@ export async function pushEvidenceToControlPlane(options) {
|
|
|
118
113
|
});
|
|
119
114
|
artifactBytesUploaded += artifact.bytes.byteLength;
|
|
120
115
|
}
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
}
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
116
|
+
await requestJson(request, `${projectUrl}/runs/${encodeURIComponent(run.id)}/events`, {
|
|
117
|
+
method: "POST",
|
|
118
|
+
headers: { ...headers, "content-type": "application/json" },
|
|
119
|
+
body: JSON.stringify({
|
|
120
|
+
events: [{
|
|
121
|
+
sequence: observationEvents.length,
|
|
122
|
+
type: "agentcert.evidence.uploaded",
|
|
123
|
+
actor: "agentcert-cli",
|
|
124
|
+
occurredAt: bundle.generatedAt,
|
|
125
|
+
payload: {
|
|
126
|
+
evidenceId: evidence.id,
|
|
127
|
+
bundleSchemaVersion: bundle.schemaVersion,
|
|
128
|
+
uploadedCount: companionArtifacts.length,
|
|
129
|
+
skippedCount: skippedArtifacts.length,
|
|
130
|
+
uploadedBytes: artifactBytesUploaded,
|
|
131
|
+
skipped: skippedArtifacts
|
|
132
|
+
.slice(0, MAX_REPORTED_COMPANION_ARTIFACT_SKIPS)
|
|
133
|
+
.map(({ sourcePath, reason }) => ({ sourcePath, reason })),
|
|
134
|
+
skippedDetailsTruncated: skippedArtifacts.length > MAX_REPORTED_COMPANION_ARTIFACT_SKIPS,
|
|
135
|
+
},
|
|
136
|
+
traceId: trace.traceId,
|
|
137
|
+
spanId: stableSpanId(`${trace.traceId}:evidence-uploaded`),
|
|
138
|
+
parentSpanId: trace.rootSpanId,
|
|
139
|
+
}],
|
|
140
|
+
}),
|
|
141
|
+
});
|
|
144
142
|
const firstDivergence = bundle.evidence.find((item) => item.severity === "critical" || item.severity === "high")?.message;
|
|
145
143
|
await requestJson(request, `${projectUrl}/runs/${encodeURIComponent(run.id)}/complete`, {
|
|
146
144
|
method: "POST",
|
|
@@ -161,6 +159,17 @@ export async function pushEvidenceToControlPlane(options) {
|
|
|
161
159
|
},
|
|
162
160
|
}),
|
|
163
161
|
});
|
|
162
|
+
const continuousAssuranceHealth = options.assurance && options.verifyContinuousAssurance
|
|
163
|
+
? await loadContinuousAssuranceHealth({
|
|
164
|
+
baseUrl,
|
|
165
|
+
projectId: options.projectId,
|
|
166
|
+
apiKey: options.apiKey,
|
|
167
|
+
runId: run.id,
|
|
168
|
+
expectedCaseId: options.assurance.caseId,
|
|
169
|
+
expectedScopeFingerprintSha256: scopeFingerprint(options.assurance.scope),
|
|
170
|
+
fetch: request,
|
|
171
|
+
})
|
|
172
|
+
: undefined;
|
|
164
173
|
return {
|
|
165
174
|
runId: run.id,
|
|
166
175
|
evidenceId: evidence.id,
|
|
@@ -168,8 +177,67 @@ export async function pushEvidenceToControlPlane(options) {
|
|
|
168
177
|
artifactsUploaded: companionArtifacts.length,
|
|
169
178
|
artifactsSkipped: skippedArtifacts.length,
|
|
170
179
|
artifactBytesUploaded,
|
|
180
|
+
...(continuousAssuranceHealth ? { continuousAssuranceHealth } : {}),
|
|
181
|
+
};
|
|
182
|
+
}
|
|
183
|
+
export async function loadContinuousAssuranceHealth(options) {
|
|
184
|
+
const baseUrl = options.baseUrl.replace(/\/$/, "");
|
|
185
|
+
const request = options.fetch ?? fetch;
|
|
186
|
+
const analysis = await requestJson(request, `${baseUrl}/v1/projects/${encodeURIComponent(options.projectId)}/runs/${encodeURIComponent(options.runId)}/analysis`, { headers: { authorization: `Bearer ${options.apiKey}` } });
|
|
187
|
+
const run = object(analysis.run);
|
|
188
|
+
const metadata = object(run.metadata);
|
|
189
|
+
const binding = object(metadata.continuousAssurance);
|
|
190
|
+
const reconciliation = object(binding.reconciliation);
|
|
191
|
+
const completeness = object(analysis.evidenceCompleteness);
|
|
192
|
+
const manifest = object(completeness.reconciliation);
|
|
193
|
+
const runStatus = text(run.status) ?? "unknown";
|
|
194
|
+
const nextStatus = assuranceStatus(reconciliation.nextStatus);
|
|
195
|
+
const evidenceStatus = evidenceCompletenessStatus(completeness.status);
|
|
196
|
+
const diagnostics = [];
|
|
197
|
+
if (!text(binding.caseId))
|
|
198
|
+
diagnostics.push(diagnostic("assurance_binding_missing", "Hosted run has no continuous assurance binding.", "Regenerate the kit from an issued assurance case and rerun the workflow."));
|
|
199
|
+
if (options.expectedCaseId && binding.caseId !== options.expectedCaseId)
|
|
200
|
+
diagnostics.push(diagnostic("assurance_case_mismatch", "Hosted run is bound to a different assurance case.", "Use the unmodified generated workflow and case ID."));
|
|
201
|
+
if (options.expectedScopeFingerprintSha256 && binding.scopeFingerprintSha256 !== options.expectedScopeFingerprintSha256)
|
|
202
|
+
diagnostics.push(diagnostic("scope_fingerprint_mismatch", "Hosted scope fingerprint does not match the generated scope file.", "Restore the generated scope file or start a formal revalidation."));
|
|
203
|
+
if (reconciliation.authoritative !== true)
|
|
204
|
+
diagnostics.push(diagnostic("non_authoritative_trigger", "The Hosted reconciliation is prospective, not authoritative.", "Run the release or nightly workflow; pull requests cannot establish CURRENT."));
|
|
205
|
+
if (reconciliation.outcome !== "current" || nextStatus !== "CURRENT")
|
|
206
|
+
diagnostics.push(diagnostic("assurance_not_current", `Hosted reconciliation ended in ${nextStatus}.`, "Review changed scope components or failed tests, then revalidate before release."));
|
|
207
|
+
if (runStatus !== "passed")
|
|
208
|
+
diagnostics.push(diagnostic("run_not_passed", `Hosted run status is ${runStatus}.`, "Open the run trace, fix the first divergence, and rerun the same generated kit."));
|
|
209
|
+
if (evidenceStatus !== "complete")
|
|
210
|
+
diagnostics.push(diagnostic("evidence_incomplete", `Hosted evidence is ${evidenceStatus}.`, "Upload every declared companion artifact and verify manifest hashes before retrying."));
|
|
211
|
+
return {
|
|
212
|
+
schemaVersion: "agentcert.continuous_assurance_health.v0.1",
|
|
213
|
+
healthy: diagnostics.length === 0,
|
|
214
|
+
status: nextStatus,
|
|
215
|
+
checkedAt: new Date().toISOString(),
|
|
216
|
+
run: { id: text(run.id) ?? options.runId, externalId: text(run.externalId) ?? "unknown", status: runStatus },
|
|
217
|
+
assurance: {
|
|
218
|
+
caseId: text(binding.caseId),
|
|
219
|
+
trigger: assuranceTrigger(binding.trigger),
|
|
220
|
+
scopeFingerprintSha256: text(binding.scopeFingerprintSha256),
|
|
221
|
+
authoritative: boolean(reconciliation.authoritative),
|
|
222
|
+
outcome: text(reconciliation.outcome),
|
|
223
|
+
firstAuthoritativeCurrentAt: text(reconciliation.firstAuthoritativeCurrentAt),
|
|
224
|
+
timeToFirstCurrentMs: finiteNumber(reconciliation.timeToFirstCurrentMs),
|
|
225
|
+
},
|
|
226
|
+
evidence: {
|
|
227
|
+
status: evidenceStatus,
|
|
228
|
+
declared: finiteNumber(manifest.declared) ?? 0,
|
|
229
|
+
matched: finiteNumber(manifest.matched) ?? 0,
|
|
230
|
+
reasons: stringArray(completeness.reasons),
|
|
231
|
+
},
|
|
232
|
+
diagnostics,
|
|
171
233
|
};
|
|
172
234
|
}
|
|
235
|
+
export function requireContinuousAssuranceCurrent(health) {
|
|
236
|
+
if (health.healthy)
|
|
237
|
+
return;
|
|
238
|
+
const details = health.diagnostics.map((item) => `${item.code}: ${item.message} ${item.recovery}`).join("\n");
|
|
239
|
+
throw new Error(`Hosted continuous assurance did not reach CURRENT.\n${details}`);
|
|
240
|
+
}
|
|
173
241
|
function hostedRunKind(bundle) {
|
|
174
242
|
const products = new Set(bundle.summary.products);
|
|
175
243
|
if (products.size > 1)
|
|
@@ -182,6 +250,135 @@ function hostedRunKind(bundle) {
|
|
|
182
250
|
return "runtime";
|
|
183
251
|
return "custom";
|
|
184
252
|
}
|
|
253
|
+
function hostedTrace(projectId, externalId) {
|
|
254
|
+
const traceId = createHash("sha256").update(`agentcert:${projectId}:${externalId}`).digest("hex").slice(0, 32);
|
|
255
|
+
return { traceId, rootSpanId: stableSpanId(`${traceId}:root`) };
|
|
256
|
+
}
|
|
257
|
+
function buildHostedObservationEvents(bundle, trace) {
|
|
258
|
+
const events = [{
|
|
259
|
+
sequence: 0,
|
|
260
|
+
type: "agentcert.run.started",
|
|
261
|
+
actor: "agentcert-cli",
|
|
262
|
+
occurredAt: bundle.generatedAt,
|
|
263
|
+
payload: redactedPayload({ subject: bundle.subject, products: bundle.summary.products, schemaVersion: bundle.schemaVersion }),
|
|
264
|
+
traceId: trace.traceId,
|
|
265
|
+
spanId: trace.rootSpanId,
|
|
266
|
+
}];
|
|
267
|
+
for (const [resultIndex, result] of bundle.results.entries()) {
|
|
268
|
+
if (events.length >= 478)
|
|
269
|
+
break;
|
|
270
|
+
const resultSpanId = stableSpanId(`${trace.traceId}:result:${resultIndex}:${result.runId}`);
|
|
271
|
+
events.push({
|
|
272
|
+
sequence: events.length,
|
|
273
|
+
type: "agentcert.product.result",
|
|
274
|
+
actor: result.product,
|
|
275
|
+
occurredAt: result.timestamp || bundle.generatedAt,
|
|
276
|
+
payload: redactedPayload({ product: result.product, phase: result.phase, passed: result.passed, score: result.score, certLevel: result.certLevel, summary: result.summary, sourceRunId: result.runId }),
|
|
277
|
+
traceId: trace.traceId,
|
|
278
|
+
spanId: resultSpanId,
|
|
279
|
+
parentSpanId: trace.rootSpanId,
|
|
280
|
+
});
|
|
281
|
+
for (const [findingIndex, finding] of result.evidence.entries()) {
|
|
282
|
+
if (events.length >= 478)
|
|
283
|
+
break;
|
|
284
|
+
events.push({
|
|
285
|
+
sequence: events.length,
|
|
286
|
+
type: findingEventType(result.product, finding.kind),
|
|
287
|
+
actor: result.product,
|
|
288
|
+
occurredAt: result.timestamp || bundle.generatedAt,
|
|
289
|
+
payload: redactedPayload({
|
|
290
|
+
findingId: finding.id,
|
|
291
|
+
kind: finding.kind,
|
|
292
|
+
severity: finding.severity,
|
|
293
|
+
message: finding.message,
|
|
294
|
+
source: finding.source,
|
|
295
|
+
artifactPath: finding.artifactPath,
|
|
296
|
+
suggestedFix: finding.suggestedFix,
|
|
297
|
+
metadata: finding.metadata,
|
|
298
|
+
passed: !new Set(["critical", "high"]).has(finding.severity),
|
|
299
|
+
}),
|
|
300
|
+
traceId: trace.traceId,
|
|
301
|
+
spanId: stableSpanId(`${trace.traceId}:finding:${resultIndex}:${findingIndex}:${finding.id}`),
|
|
302
|
+
parentSpanId: resultSpanId,
|
|
303
|
+
});
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
events.push({
|
|
307
|
+
sequence: events.length,
|
|
308
|
+
type: "agentcert.run.evaluated",
|
|
309
|
+
actor: "agentcert-cli",
|
|
310
|
+
occurredAt: bundle.generatedAt,
|
|
311
|
+
payload: redactedPayload({ verdict: bundle.verdict, summary: bundle.summary, evidenceStrength: bundle.evidenceStrength?.level }),
|
|
312
|
+
traceId: trace.traceId,
|
|
313
|
+
spanId: stableSpanId(`${trace.traceId}:evaluated`),
|
|
314
|
+
parentSpanId: trace.rootSpanId,
|
|
315
|
+
});
|
|
316
|
+
return events;
|
|
317
|
+
}
|
|
318
|
+
function findingEventType(product, kind) {
|
|
319
|
+
if (product === "tripwire-ci")
|
|
320
|
+
return "tripwire.fault.assertion";
|
|
321
|
+
if (product === "mcpbench")
|
|
322
|
+
return /policy|permission|security|violation/i.test(kind) ? "mcpbench.policy.violation" : "mcpbench.assertion";
|
|
323
|
+
if (product === "onegent-runtime") {
|
|
324
|
+
if (/approval/i.test(kind))
|
|
325
|
+
return "onegent.approval.decision";
|
|
326
|
+
if (/outcome|verification/i.test(kind))
|
|
327
|
+
return "onegent.outcome.verification";
|
|
328
|
+
if (/policy|mandate|authorization/i.test(kind))
|
|
329
|
+
return "onegent.policy.decision";
|
|
330
|
+
return "onegent.action.observation";
|
|
331
|
+
}
|
|
332
|
+
return "agentcert.finding";
|
|
333
|
+
}
|
|
334
|
+
function stableSpanId(seed) {
|
|
335
|
+
return createHash("sha256").update(seed).digest("hex").slice(0, 16);
|
|
336
|
+
}
|
|
337
|
+
function redactedPayload(value, depth = 0) {
|
|
338
|
+
if (depth > 4)
|
|
339
|
+
return "[TRUNCATED]";
|
|
340
|
+
if (typeof value === "string")
|
|
341
|
+
return value.length > 1_000 ? `${value.slice(0, 1_000)}...[TRUNCATED]` : value;
|
|
342
|
+
if (typeof value === "number" || typeof value === "boolean" || value === null)
|
|
343
|
+
return value;
|
|
344
|
+
if (Array.isArray(value))
|
|
345
|
+
return value.slice(0, 50).map((item) => redactedPayload(item, depth + 1));
|
|
346
|
+
if (value && typeof value === "object") {
|
|
347
|
+
const result = {};
|
|
348
|
+
for (const [key, item] of Object.entries(value).slice(0, 50)) {
|
|
349
|
+
result[key] = /(authorization|api.?key|token|secret|password|credential)/i.test(key) ? "[REDACTED]" : redactedPayload(item, depth + 1);
|
|
350
|
+
}
|
|
351
|
+
return result;
|
|
352
|
+
}
|
|
353
|
+
return undefined;
|
|
354
|
+
}
|
|
355
|
+
function scopeFingerprint(scope) {
|
|
356
|
+
return createHash("sha256").update(canonicalJson(scope)).digest("hex");
|
|
357
|
+
}
|
|
358
|
+
function canonicalJson(value) {
|
|
359
|
+
if (Array.isArray(value))
|
|
360
|
+
return `[${value.map(canonicalJson).join(",")}]`;
|
|
361
|
+
if (value && typeof value === "object") {
|
|
362
|
+
const source = value;
|
|
363
|
+
return `{${Object.keys(source).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(source[key])}`).join(",")}}`;
|
|
364
|
+
}
|
|
365
|
+
return JSON.stringify(value) ?? "null";
|
|
366
|
+
}
|
|
367
|
+
function object(value) { return value && typeof value === "object" && !Array.isArray(value) ? value : {}; }
|
|
368
|
+
function text(value) { return typeof value === "string" && value ? value : undefined; }
|
|
369
|
+
function boolean(value) { return typeof value === "boolean" ? value : undefined; }
|
|
370
|
+
function finiteNumber(value) { return typeof value === "number" && Number.isFinite(value) ? value : undefined; }
|
|
371
|
+
function stringArray(value) { return Array.isArray(value) ? value.filter((item) => typeof item === "string") : []; }
|
|
372
|
+
function assuranceStatus(value) {
|
|
373
|
+
return value === "CURRENT" || value === "REVALIDATION_REQUIRED" || value === "SUSPENDED" || value === "EXPIRED" ? value : "UNKNOWN";
|
|
374
|
+
}
|
|
375
|
+
function evidenceCompletenessStatus(value) {
|
|
376
|
+
return value === "complete" || value === "partial" || value === "rejected" ? value : "unknown";
|
|
377
|
+
}
|
|
378
|
+
function assuranceTrigger(value) {
|
|
379
|
+
return value === "pull_request" || value === "release" || value === "nightly" ? value : undefined;
|
|
380
|
+
}
|
|
381
|
+
function diagnostic(code, message, recovery) { return { code, message, recovery }; }
|
|
185
382
|
async function requestJson(request, url, init) {
|
|
186
383
|
let response;
|
|
187
384
|
try {
|
package/dist/schema-validator.js
CHANGED
|
@@ -13,6 +13,7 @@ export function parseSchemaId(input) {
|
|
|
13
13
|
value === "release-gate" ||
|
|
14
14
|
value === "assurance-report" ||
|
|
15
15
|
value === "assurance-delivery" ||
|
|
16
|
+
value === "assurance-scope" ||
|
|
16
17
|
value === "evidence-signature" ||
|
|
17
18
|
value === "evidence-strength" ||
|
|
18
19
|
value === "action-mandate" ||
|
|
@@ -20,7 +21,7 @@ export function parseSchemaId(input) {
|
|
|
20
21
|
value === "trusted-run-receipt") {
|
|
21
22
|
return value;
|
|
22
23
|
}
|
|
23
|
-
throw new Error(`Unsupported schema "${value}". Use evidence-bundle, result, corpus-record, failure-review, classifier-eval, monitor-snapshot, robustness-lab, release-gate, assurance-report, assurance-delivery, evidence-signature, evidence-strength, action-mandate, trusted-action-record, or trusted-run-receipt.`);
|
|
24
|
+
throw new Error(`Unsupported schema "${value}". Use evidence-bundle, result, corpus-record, failure-review, classifier-eval, monitor-snapshot, robustness-lab, release-gate, assurance-report, assurance-delivery, assurance-scope, evidence-signature, evidence-strength, action-mandate, trusted-action-record, or trusted-run-receipt.`);
|
|
24
25
|
}
|
|
25
26
|
export function validateAgentCertSchema(schema, input) {
|
|
26
27
|
const errors = [];
|
|
@@ -52,6 +53,8 @@ export function validateAgentCertSchema(schema, input) {
|
|
|
52
53
|
validateAssuranceReport(value, errors);
|
|
53
54
|
if (schema === "assurance-delivery")
|
|
54
55
|
validateAssuranceDelivery(value, errors);
|
|
56
|
+
if (schema === "assurance-scope")
|
|
57
|
+
validateAssuranceScope(value, errors);
|
|
55
58
|
if (schema === "evidence-strength")
|
|
56
59
|
validateEvidenceStrength(value, errors);
|
|
57
60
|
if (schema === "action-mandate")
|
|
@@ -293,6 +296,7 @@ function validateAssuranceReport(value, errors) {
|
|
|
293
296
|
errors.push("evaluationPlanSha256 must be a lowercase SHA-256 digest.");
|
|
294
297
|
validateTimestamp(value.issuedAt, "issuedAt", errors);
|
|
295
298
|
validateTimestamp(value.expiresAt, "expiresAt", errors);
|
|
299
|
+
validateAssuranceContinuity(value.continuousAssurance, errors);
|
|
296
300
|
}
|
|
297
301
|
function validateAssuranceDelivery(value, errors) {
|
|
298
302
|
requiredConst(value, "schemaVersion", "agentcert.assurance_delivery.v0.1", errors);
|
|
@@ -320,6 +324,69 @@ function validateAssuranceDelivery(value, errors) {
|
|
|
320
324
|
const decision = recordValue(value.decision);
|
|
321
325
|
if (decision)
|
|
322
326
|
requiredEnum(decision, "verdict", ["RELEASE", "RELEASE_WITH_CONTROLS", "BLOCK"], errors);
|
|
327
|
+
validateAssuranceContinuity(value.continuousAssurance, errors);
|
|
328
|
+
}
|
|
329
|
+
function validateAssuranceContinuity(input, errors) {
|
|
330
|
+
if (input === undefined)
|
|
331
|
+
return;
|
|
332
|
+
const value = recordValue(input);
|
|
333
|
+
if (!value) {
|
|
334
|
+
errors.push("continuousAssurance must be an object.");
|
|
335
|
+
return;
|
|
336
|
+
}
|
|
337
|
+
requiredConst(value, "schemaVersion", "agentcert.assurance_continuity.v0.1", errors);
|
|
338
|
+
requiredObject(value, "scope", errors);
|
|
339
|
+
requiredSha256At(value, "scopeFingerprintSha256", "continuousAssurance.scopeFingerprintSha256", errors);
|
|
340
|
+
requiredConst(value, "freshnessAtIssuance", "CURRENT", errors);
|
|
341
|
+
requiredArray(value, "revalidationRequiredWhen", errors);
|
|
342
|
+
stringArray(value.revalidationRequiredWhen, "continuousAssurance.revalidationRequiredWhen", errors);
|
|
343
|
+
const scope = recordValue(value.scope);
|
|
344
|
+
if (scope) {
|
|
345
|
+
const nested = [];
|
|
346
|
+
validateAssuranceScope(scope, nested);
|
|
347
|
+
errors.push(...nested.map((error) => `continuousAssurance.scope.${error}`));
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
function validateAssuranceScope(value, errors) {
|
|
351
|
+
requiredConst(value, "schemaVersion", "agentcert.assurance_scope.v0.1", errors);
|
|
352
|
+
for (const key of ["agent", "model", "prompt", "tools", "policy", "scenarioSuite"])
|
|
353
|
+
requiredObject(value, key, errors);
|
|
354
|
+
const agent = recordValue(value.agent);
|
|
355
|
+
const model = recordValue(value.model);
|
|
356
|
+
const prompt = recordValue(value.prompt);
|
|
357
|
+
const tools = recordValue(value.tools);
|
|
358
|
+
const policy = recordValue(value.policy);
|
|
359
|
+
const suite = recordValue(value.scenarioSuite);
|
|
360
|
+
if (agent) {
|
|
361
|
+
requiredStringAt(agent, "id", "agent.id", errors);
|
|
362
|
+
requiredStringAt(agent, "version", "agent.version", errors);
|
|
363
|
+
optionalSha256At(agent, "artifactSha256", "agent.artifactSha256", errors);
|
|
364
|
+
}
|
|
365
|
+
if (model)
|
|
366
|
+
for (const key of ["provider", "name", "version"])
|
|
367
|
+
requiredStringAt(model, key, `model.${key}`, errors);
|
|
368
|
+
if (prompt)
|
|
369
|
+
requiredSha256At(prompt, "sha256", "prompt.sha256", errors);
|
|
370
|
+
if (tools)
|
|
371
|
+
requiredSha256At(tools, "manifestSha256", "tools.manifestSha256", errors);
|
|
372
|
+
if (policy) {
|
|
373
|
+
requiredStringAt(policy, "id", "policy.id", errors);
|
|
374
|
+
requiredStringAt(policy, "version", "policy.version", errors);
|
|
375
|
+
optionalSha256At(policy, "sha256", "policy.sha256", errors);
|
|
376
|
+
}
|
|
377
|
+
if (suite) {
|
|
378
|
+
requiredStringAt(suite, "id", "scenarioSuite.id", errors);
|
|
379
|
+
requiredStringAt(suite, "version", "scenarioSuite.version", errors);
|
|
380
|
+
requiredSha256At(suite, "sha256", "scenarioSuite.sha256", errors);
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
function requiredSha256At(value, key, path, errors) {
|
|
384
|
+
if (typeof value[key] !== "string" || !/^[0-9a-f]{64}$/.test(value[key]))
|
|
385
|
+
errors.push(`${path} must be a lowercase SHA-256 digest.`);
|
|
386
|
+
}
|
|
387
|
+
function optionalSha256At(value, key, path, errors) {
|
|
388
|
+
if (value[key] !== undefined && (typeof value[key] !== "string" || !/^[0-9a-f]{64}$/.test(value[key])))
|
|
389
|
+
errors.push(`${path} must be a lowercase SHA-256 digest.`);
|
|
323
390
|
}
|
|
324
391
|
function validateFailureReview(value, errors) {
|
|
325
392
|
requiredConst(value, "schemaVersion", "1", errors);
|