agentcert 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -0
- package/dist/cli.js +47 -1
- package/dist/control-plane.js +109 -0
- package/dist/index.js +1 -0
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -34,6 +34,19 @@ Default outputs:
|
|
|
34
34
|
- `.agentcert/latest/reviewed-failure-dataset.jsonl`
|
|
35
35
|
- `.agentcert/latest/monitor.json`
|
|
36
36
|
|
|
37
|
+
Push the validated evidence bundle into a hosted AgentCert project:
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
export AGENTCERT_BASE_URL="https://agentcert.example.com"
|
|
41
|
+
export AGENTCERT_PROJECT_ID="your-project-id"
|
|
42
|
+
export AGENTCERT_API_KEY="ac_live_..."
|
|
43
|
+
npx agentcert push --evidence .agentcert/latest/agentcert-evidence.json
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Add `--push` to `agentcert run` to run locally and upload the resulting bundle
|
|
47
|
+
in one command. Project API keys can create runs, record events, and upload
|
|
48
|
+
evidence, but cannot approve their own runtime actions.
|
|
49
|
+
|
|
37
50
|
Review/export helpers:
|
|
38
51
|
|
|
39
52
|
```bash
|
package/dist/cli.js
CHANGED
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
3
|
-
import { dirname, join, resolve } from "node:path";
|
|
3
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
4
4
|
import { validateEvidenceArtifacts } from "./artifact-validation.js";
|
|
5
5
|
import { renderAgentCertBadge } from "./badge.js";
|
|
6
6
|
import { buildEvidenceBundle } from "./bundle.js";
|
|
7
7
|
import { recordsFromAgentCertResult, evaluateFailureClassifier, renderCorpusSummary, summarizeCorpus, writeReviewedFailureDataset, } from "./corpus.js";
|
|
8
8
|
import { openCorpusStore, parseCorpusStoreKind } from "./corpus-store.js";
|
|
9
|
+
import { pushEvidenceToControlPlane } from "./control-plane.js";
|
|
9
10
|
import { applyFailureReviews, appendFailureReview, createFailureReview, findFailurePattern, parseFailureReviewStatus, parseFailureType, parseReviewConfidence, readFailureReviews, } from "./failure-review.js";
|
|
10
11
|
import { buildMonitorSnapshot, writeMonitorSnapshot } from "./monitor.js";
|
|
11
12
|
import { normalizeMcpBenchResult, normalizeOnegentAuditPacket, normalizeTripwireResult } from "./normalizers.js";
|
|
@@ -269,8 +270,27 @@ else if (command === "run") {
|
|
|
269
270
|
overrides,
|
|
270
271
|
});
|
|
271
272
|
process.stdout.write(renderRunSummary(outcome));
|
|
273
|
+
if (readBoolFlag("--push")) {
|
|
274
|
+
await pushHostedEvidence(outcome.bundle, Buffer.from(`${JSON.stringify(outcome.bundle, null, 2)}\n`), "agentcert-evidence.json");
|
|
275
|
+
}
|
|
272
276
|
process.exitCode = outcome.exitCode;
|
|
273
277
|
}
|
|
278
|
+
else if (command === "push") {
|
|
279
|
+
const evidencePath = resolve(readFlag("--evidence") ?? ".agentcert/latest/agentcert-evidence.json");
|
|
280
|
+
const evidenceBytes = await readFile(evidencePath);
|
|
281
|
+
let bundle;
|
|
282
|
+
try {
|
|
283
|
+
bundle = JSON.parse(evidenceBytes.toString("utf8"));
|
|
284
|
+
}
|
|
285
|
+
catch {
|
|
286
|
+
throw new Error(`Evidence file is not valid JSON: ${evidencePath}`);
|
|
287
|
+
}
|
|
288
|
+
const validation = validateAgentCertSchema("evidence-bundle", bundle);
|
|
289
|
+
if (!validation.valid) {
|
|
290
|
+
throw new Error(`Evidence bundle is invalid:\n${validation.errors.map((error) => `- ${error}`).join("\n")}`);
|
|
291
|
+
}
|
|
292
|
+
await pushHostedEvidence(bundle, evidenceBytes, basename(evidencePath));
|
|
293
|
+
}
|
|
274
294
|
else if (command === "release-gate") {
|
|
275
295
|
const overrides = readRunOverrides();
|
|
276
296
|
const profileName = readFlag("--profile");
|
|
@@ -471,6 +491,8 @@ else {
|
|
|
471
491
|
agentcert monitor build --corpus .agentcert/corpus/corpus.jsonl --out .agentcert/latest/monitor.json --subject my-agent
|
|
472
492
|
agentcert monitor build --store sqlite --sqlite .agentcert/corpus/agentcert.sqlite --out .agentcert/latest/monitor.json --subject my-agent
|
|
473
493
|
agentcert run --profile public-demo
|
|
494
|
+
agentcert push --evidence .agentcert/latest/agentcert-evidence.json --server https://agentcert.example.com --project <project-id>
|
|
495
|
+
agentcert run --tripwire .tripwire/latest/tripwire-result.json --push
|
|
474
496
|
agentcert run --mcpbench .mcpbench/latest/results.json --tripwire .tripwire/latest/tripwire-result.json --onegent .onegent/procurement/audit-packet.json --out .agentcert/latest --corpus .agentcert/corpus/corpus.jsonl --monitor-out .agentcert/latest/monitor.json --reviewed-dataset-out .agentcert/latest/reviewed-failure-dataset.jsonl
|
|
475
497
|
agentcert release-gate --config agentcert.config.json --strict
|
|
476
498
|
agentcert release-gate --evidence .agentcert/latest/agentcert-evidence.json --baseline .agentcert/baselines/main.json
|
|
@@ -492,6 +514,24 @@ async function loadConfig(path) {
|
|
|
492
514
|
}
|
|
493
515
|
return (await readJson(path));
|
|
494
516
|
}
|
|
517
|
+
async function pushHostedEvidence(bundle, bytes, fileName) {
|
|
518
|
+
const baseUrl = readFlag("--server") ?? process.env.AGENTCERT_BASE_URL;
|
|
519
|
+
const projectId = readFlag("--project") ?? process.env.AGENTCERT_PROJECT_ID;
|
|
520
|
+
const apiKey = readFlag("--api-key") ?? process.env.AGENTCERT_API_KEY;
|
|
521
|
+
if (!baseUrl || !projectId || !apiKey) {
|
|
522
|
+
throw new Error("Hosted push requires AGENTCERT_BASE_URL, AGENTCERT_PROJECT_ID, and AGENTCERT_API_KEY (or --server, --project, and --api-key).");
|
|
523
|
+
}
|
|
524
|
+
const result = await pushEvidenceToControlPlane({
|
|
525
|
+
baseUrl,
|
|
526
|
+
projectId,
|
|
527
|
+
apiKey,
|
|
528
|
+
bundle,
|
|
529
|
+
evidenceBytes: bytes,
|
|
530
|
+
fileName,
|
|
531
|
+
externalId: readFlag("--external-id"),
|
|
532
|
+
});
|
|
533
|
+
process.stdout.write(`Hosted run: ${result.runId}\nHosted evidence: ${result.evidenceId}\n`);
|
|
534
|
+
}
|
|
495
535
|
async function readJson(path) {
|
|
496
536
|
const raw = await readFile(resolve(path), "utf8");
|
|
497
537
|
return JSON.parse(raw);
|
|
@@ -683,6 +723,12 @@ function errorHint(message, code) {
|
|
|
683
723
|
if (message.includes("already exists")) {
|
|
684
724
|
return "Hint: re-run with `--force` to overwrite starter files, or choose a different `--out` path.";
|
|
685
725
|
}
|
|
726
|
+
if (message.includes("Hosted push requires")) {
|
|
727
|
+
return "Hint: create a project API key in AgentCert Integrations, then set AGENTCERT_BASE_URL, AGENTCERT_PROJECT_ID, and AGENTCERT_API_KEY.";
|
|
728
|
+
}
|
|
729
|
+
if (message.includes("Authentication required") || message.includes("API key")) {
|
|
730
|
+
return "Hint: check AGENTCERT_API_KEY and confirm the key belongs to AGENTCERT_PROJECT_ID.";
|
|
731
|
+
}
|
|
686
732
|
return undefined;
|
|
687
733
|
}
|
|
688
734
|
function starterTripwireConfig(subject) {
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
export async function pushEvidenceToControlPlane(options) {
|
|
2
|
+
const baseUrl = options.baseUrl.replace(/\/$/, "");
|
|
3
|
+
if (!baseUrl || !options.projectId || !options.apiKey) {
|
|
4
|
+
throw new Error("baseUrl, projectId, and apiKey are required to push evidence.");
|
|
5
|
+
}
|
|
6
|
+
const request = options.fetch ?? fetch;
|
|
7
|
+
const projectUrl = `${baseUrl}/v1/projects/${encodeURIComponent(options.projectId)}`;
|
|
8
|
+
const externalId = options.externalId ?? options.bundle.runId;
|
|
9
|
+
const kind = hostedRunKind(options.bundle);
|
|
10
|
+
const headers = { authorization: `Bearer ${options.apiKey}` };
|
|
11
|
+
const run = await requestJson(request, `${projectUrl}/runs`, {
|
|
12
|
+
method: "POST",
|
|
13
|
+
headers: { ...headers, "content-type": "application/json" },
|
|
14
|
+
body: JSON.stringify({
|
|
15
|
+
externalId,
|
|
16
|
+
kind,
|
|
17
|
+
schemaVersion: options.bundle.schemaVersion,
|
|
18
|
+
startedAt: options.bundle.generatedAt,
|
|
19
|
+
metadata: {
|
|
20
|
+
subject: options.bundle.subject,
|
|
21
|
+
products: options.bundle.summary.products,
|
|
22
|
+
},
|
|
23
|
+
}),
|
|
24
|
+
});
|
|
25
|
+
await requestJson(request, `${projectUrl}/runs/${encodeURIComponent(run.id)}/events`, {
|
|
26
|
+
method: "POST",
|
|
27
|
+
headers: { ...headers, "content-type": "application/json" },
|
|
28
|
+
body: JSON.stringify({
|
|
29
|
+
events: [{
|
|
30
|
+
sequence: 0,
|
|
31
|
+
type: "agentcert.evidence.created",
|
|
32
|
+
actor: "agentcert-cli",
|
|
33
|
+
occurredAt: options.bundle.generatedAt,
|
|
34
|
+
payload: {
|
|
35
|
+
verdict: options.bundle.verdict,
|
|
36
|
+
totalEvidence: options.bundle.summary.totalEvidence,
|
|
37
|
+
criticalEvidence: options.bundle.summary.criticalEvidence,
|
|
38
|
+
highEvidence: options.bundle.summary.highEvidence,
|
|
39
|
+
},
|
|
40
|
+
}],
|
|
41
|
+
}),
|
|
42
|
+
});
|
|
43
|
+
const query = new URLSearchParams({
|
|
44
|
+
fileName: options.fileName ?? "agentcert-evidence.json",
|
|
45
|
+
kind: "evidence_bundle",
|
|
46
|
+
schemaVersion: options.bundle.schemaVersion,
|
|
47
|
+
runId: run.id,
|
|
48
|
+
});
|
|
49
|
+
const evidence = await requestJson(request, `${projectUrl}/evidence?${query}`, {
|
|
50
|
+
method: "POST",
|
|
51
|
+
headers: { ...headers, "content-type": "application/json" },
|
|
52
|
+
body: new Uint8Array(options.evidenceBytes).buffer,
|
|
53
|
+
});
|
|
54
|
+
const firstDivergence = options.bundle.evidence.find((item) => item.severity === "critical" || item.severity === "high")?.message;
|
|
55
|
+
await requestJson(request, `${projectUrl}/runs/${encodeURIComponent(run.id)}/complete`, {
|
|
56
|
+
method: "POST",
|
|
57
|
+
headers: { ...headers, "content-type": "application/json" },
|
|
58
|
+
body: JSON.stringify({
|
|
59
|
+
status: options.bundle.verdict.passed ? "passed" : "failed",
|
|
60
|
+
score: options.bundle.verdict.score,
|
|
61
|
+
summary: `${options.bundle.subject.name}: ${options.bundle.verdict.level}`,
|
|
62
|
+
firstDivergence,
|
|
63
|
+
completedAt: options.bundle.generatedAt,
|
|
64
|
+
metadata: {
|
|
65
|
+
evidenceId: evidence.id,
|
|
66
|
+
evidenceSchemaVersion: options.bundle.schemaVersion,
|
|
67
|
+
},
|
|
68
|
+
}),
|
|
69
|
+
});
|
|
70
|
+
return { runId: run.id, evidenceId: evidence.id, externalId };
|
|
71
|
+
}
|
|
72
|
+
function hostedRunKind(bundle) {
|
|
73
|
+
const products = new Set(bundle.summary.products);
|
|
74
|
+
if (products.size > 1)
|
|
75
|
+
return "release_gate";
|
|
76
|
+
if (products.has("mcpbench"))
|
|
77
|
+
return "mcpbench";
|
|
78
|
+
if (products.has("tripwire-ci"))
|
|
79
|
+
return "tripwire";
|
|
80
|
+
if (products.has("onegent-runtime"))
|
|
81
|
+
return "runtime";
|
|
82
|
+
return "custom";
|
|
83
|
+
}
|
|
84
|
+
async function requestJson(request, url, init) {
|
|
85
|
+
let response;
|
|
86
|
+
try {
|
|
87
|
+
response = await request(url, { ...init, signal: init.signal ?? AbortSignal.timeout(30_000) });
|
|
88
|
+
}
|
|
89
|
+
catch (error) {
|
|
90
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
91
|
+
throw new Error(`AgentCert control plane request failed: ${message}`);
|
|
92
|
+
}
|
|
93
|
+
const text = await response.text();
|
|
94
|
+
let value = {};
|
|
95
|
+
if (text) {
|
|
96
|
+
try {
|
|
97
|
+
value = JSON.parse(text);
|
|
98
|
+
}
|
|
99
|
+
catch {
|
|
100
|
+
if (!response.ok)
|
|
101
|
+
throw new Error(`AgentCert control plane returned HTTP ${response.status}.`);
|
|
102
|
+
throw new Error("AgentCert control plane returned invalid JSON.");
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
if (!response.ok) {
|
|
106
|
+
throw new Error(typeof value.error === "string" ? value.error : `AgentCert control plane returned HTTP ${response.status}.`);
|
|
107
|
+
}
|
|
108
|
+
return value;
|
|
109
|
+
}
|
package/dist/index.js
CHANGED