agentcert 0.1.0 → 0.1.2

package/README.md

@@ -2,12 +2,19 @@
 
 Unified evidence, corpus, monitor, and lab CLI for AgentCert.
 
+AgentCert is regression CI for browser agents. It runs Tripwire robustness
+checks, converts the result into evidence bundles, writes HTML reports and
+badges, and accumulates a local failure corpus.
+
 ## 5-minute local path
 
 ```bash
 npx agentcert init --subject my-browser-agent
 ```
 
+This writes `agentcert.config.json` and `tripwire.yml`. Add `--github-action`
+when you also want `.github/workflows/agentcert-tripwire.yml`.
+
 Edit `tripwire.yml` so `startUrl` and `agent.command` point at your app and
 browser agent. After Tripwire has produced `.tripwire/latest/tripwire-result.json`,
 build the AgentCert outputs:
@@ -33,13 +40,38 @@ Review/export helpers:
 npx agentcert corpus metrics --corpus .agentcert/corpus/corpus.jsonl
 npx agentcert corpus export-reviewed --corpus .agentcert/corpus/corpus.jsonl --out .agentcert/latest/reviewed-failure-dataset.jsonl
 npx agentcert corpus classifier-eval --corpus .agentcert/corpus/corpus.jsonl --out .agentcert/latest/failure-classifier-evaluation.json
+npx agentcert validate .agentcert/latest/agentcert-evidence.json
+npx agentcert validate .agentcert/latest/agentcert-evidence.json --check-artifacts
 npx agentcert schema validate --schema evidence-bundle --file .agentcert/latest/agentcert-evidence.json
 npx agentcert schema validate --schema classifier-eval --file examples/agentcert/classifier-eval.example.json
 ```
 
+Release gate checklist:
+
+```text
+docs/release-gate-checklist.md
+```
+
 CI users can run Tripwire and AgentCert together with
 `Kakarottoooo/agentcert/actions/tripwire@v0`.
 
+The public Real Agent Robustness Lab compares browser-use, Stagehand, and
+Playwright-based agents over the same fault suite:
+
+https://kakarottoooo.github.io/agentcert/public-demo/real-agent-robustness/
+
+Minimal no-key browser-agent example in the GitHub repo:
+
+```text
+examples/minimal-browser-agent/
+```
+
+External integration smoke matrix in the GitHub repo:
+
+```text
+examples/real-agents/external-integration-smokes.md
+```
+
 Corpus storage:
 
 ```bash

package/dist/artifact-validation.js

@@ -0,0 +1,92 @@
+import { access } from "node:fs/promises";
+import { dirname, isAbsolute, resolve } from "node:path";
+export async function validateEvidenceArtifacts(input, artifactRoot) {
+    if (!input || typeof input !== "object" || Array.isArray(input)) {
+        return { checked: 0, missing: [] };
+    }
+    const paths = collectArtifactPaths(input, artifactRoot);
+    const uniquePaths = dedupeArtifactPathEntries(paths).filter((entry) => !looksLikeUrl(entry.path));
+    const missing = [];
+    for (const entry of uniquePaths) {
+        const fullPath = isAbsolute(entry.path) ? entry.path : resolve(entry.root, entry.path);
+        try {
+            await access(fullPath);
+        }
+        catch {
+            missing.push(entry.path);
+        }
+    }
+    return { checked: uniquePaths.length, missing };
+}
+function collectArtifactPaths(bundle, artifactRoot) {
+    const paths = [];
+    const productRoots = productArtifactRoots(bundle, artifactRoot);
+    collectStringValues(bundle.artifacts, paths, artifactRoot);
+    const results = Array.isArray(bundle.results) ? bundle.results : [];
+    for (const result of results) {
+        if (result && typeof result === "object" && !Array.isArray(result)) {
+            const resultRecord = result;
+            collectStringValues(resultRecord.artifacts, paths, artifactRoot);
+        }
+    }
+    const evidence = Array.isArray(bundle.evidence) ? bundle.evidence : [];
+    for (const item of evidence) {
+        if (item && typeof item === "object" && !Array.isArray(item)) {
+            const itemRecord = item;
+            const artifactPath = itemRecord.artifactPath;
+            if (typeof artifactPath === "string" && artifactPath.length > 0) {
+                const source = typeof itemRecord.source === "string" ? itemRecord.source : undefined;
+                paths.push({ path: artifactPath, root: source ? (productRoots.get(source) ?? artifactRoot) : artifactRoot });
+            }
+        }
+    }
+    return paths;
+}
+function productArtifactRoots(bundle, artifactRoot) {
+    const roots = new Map();
+    const results = Array.isArray(bundle.results) ? bundle.results : [];
+    for (const result of results) {
+        if (!result || typeof result !== "object" || Array.isArray(result))
+            continue;
+        const resultRecord = result;
+        const product = resultRecord.product;
+        const artifacts = resultRecord.artifacts;
+        if (typeof product !== "string" || !artifacts || typeof artifacts !== "object" || Array.isArray(artifacts))
+            continue;
+        const artifactRecord = artifacts;
+        const outDir = artifactRecord.outDir;
+        const resultPath = artifactRecord.result ?? artifactRecord.results;
+        if (typeof outDir === "string" && outDir.length > 0) {
+            roots.set(product, isAbsolute(outDir) ? outDir : resolve(artifactRoot, outDir));
+        }
+        else if (typeof resultPath === "string" && resultPath.length > 0) {
+            roots.set(product, dirname(isAbsolute(resultPath) ? resultPath : resolve(artifactRoot, resultPath)));
+        }
+    }
+    return roots;
+}
+function collectStringValues(input, paths, root) {
+    if (!input || typeof input !== "object" || Array.isArray(input)) {
+        return;
+    }
+    for (const value of Object.values(input)) {
+        if (typeof value === "string" && value.length > 0) {
+            paths.push({ path: value, root });
+        }
+    }
+}
+function dedupeArtifactPathEntries(entries) {
+    const seen = new Set();
+    const deduped = [];
+    for (const entry of entries) {
+        const key = `${entry.root}\0${entry.path}`;
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        deduped.push(entry);
+    }
+    return deduped;
+}
+function looksLikeUrl(value) {
+    return /^[a-z][a-z0-9+.-]*:\/\//i.test(value);
+}

package/dist/bundle.js

@@ -1,3 +1,4 @@
+import { AGENTCERT_EVIDENCE_SCHEMA_SEMVER, AGENTCERT_EVIDENCE_SCHEMA_VERSION } from "./types.js";
 export function buildEvidenceBundle(results, subjectName, subjectType = "agent") {
     const evidence = results.flatMap((result) => result.evidence);
     const score = results.length === 0 ? 0 : Math.round(results.reduce((sum, result) => sum + result.score, 0) / results.length);
@@ -12,8 +13,8 @@ export function buildEvidenceBundle(results, subjectName, subjectType = "agent")
     }
     return {
         schemaName: "agentcert.evidence_bundle",
-        schemaVersion: "1",
-        schemaSemver: "1.0.0",
+        schemaVersion: AGENTCERT_EVIDENCE_SCHEMA_VERSION,
+        schemaSemver: AGENTCERT_EVIDENCE_SCHEMA_SEMVER,
         kind: "agentcert.evidence_bundle",
         runId: `agentcert_${Date.now()}`,
         generatedAt: new Date().toISOString(),

package/dist/cli.js

@@ -1,6 +1,7 @@
 #!/usr/bin/env node
 import { mkdir, readFile, writeFile } from "node:fs/promises";
 import { dirname, resolve } from "node:path";
+import { validateEvidenceArtifacts } from "./artifact-validation.js";
 import { renderAgentCertBadge } from "./badge.js";
 import { buildEvidenceBundle } from "./bundle.js";
 import { recordsFromAgentCertResult, evaluateFailureClassifier, renderCorpusSummary, summarizeCorpus, writeReviewedFailureDataset, } from "./corpus.js";
@@ -13,6 +14,8 @@ import { serveAgentCertMonitor } from "./local-server.js";
 import { parseSchemaId, validateAgentCertSchema } from "./schema-validator.js";
 import { loadRunProfile, profileFromArtifactFlags, renderRunSummary, runAgentCertProfile, } from "./runner.js";
 import { buildRobustnessLabSnapshot, readRobustnessLabConfig, renderRobustnessLabSummary, writeRobustnessLabSnapshot } from "./lab.js";
+process.on("uncaughtException", reportFatalError);
+process.on("unhandledRejection", reportFatalError);
 const command = process.argv[2] ?? "help";
 if (command === "init") {
     const outPath = resolve(readFlag("--out") ?? "agentcert.config.json");
@@ -20,6 +23,7 @@ if (command === "init") {
     const githubWorkflowPath = resolve(readFlag("--github-action-out") ?? ".github/workflows/agentcert-tripwire.yml");
     const subject = readFlag("--subject") ?? "my-browser-agent";
     const force = readBoolFlag("--force");
+    const writeGitHubAction = (readBoolFlag("--github-action") || process.argv.includes("--github-action-out")) && !readBoolFlag("--skip-github-action");
     const config = {
         schemaVersion: "1",
         subject: {
@@ -60,15 +64,15 @@ if (command === "init") {
         await writeStarterFile(tripwireConfigPath, starterTripwireConfig(subject), force);
         process.stdout.write(`Wrote ${tripwireConfigPath}\n`);
     }
-    if (!readBoolFlag("--skip-github-action")) {
+    if (writeGitHubAction) {
         await writeStarterFile(githubWorkflowPath, starterGitHubActionWorkflow(subject), force);
         process.stdout.write(`Wrote ${githubWorkflowPath}\n`);
     }
     process.stdout.write(`
 Next:
   1. Edit tripwire.yml so startUrl and agent.command/agent.args match your app and browser agent.
-  2. Commit .github/workflows/agentcert-tripwire.yml to run AgentCert in CI.
-  3. Or run locally after Tripwire writes .tripwire/latest/tripwire-result.json:
+  2. Run in CI with Kakarottoooo/agentcert/actions/tripwire@v0, or re-run init with --github-action to write a workflow template.
+  3. Run locally after Tripwire writes .tripwire/latest/tripwire-result.json:
      npx agentcert run --tripwire .tripwire/latest/tripwire-result.json --subject ${JSON.stringify(subject)} --fail-on-verdict
 `);
 }
@@ -317,10 +321,41 @@ else if (command === "schema") {
 `);
     }
 }
+else if (command === "validate") {
+    const schema = parseSchemaId(readFlag("--schema") ?? "evidence-bundle");
+    const file = readFlag("--file") ?? readFirstPositionalAfterCommand();
+    if (!file) {
+        throw new Error("Missing evidence file. Usage: agentcert validate <file> [--schema evidence-bundle].");
+    }
+    const input = await readJson(file);
+    const result = validateAgentCertSchema(schema, input);
+    if (result.valid) {
+        process.stdout.write(`Valid ${schema}: ${resolve(file)}\n`);
+    }
+    else {
+        process.stdout.write(`Invalid ${schema}: ${resolve(file)}\n`);
+        for (const error of result.errors) {
+            process.stdout.write(`- ${error}\n`);
+        }
+        process.exitCode = 1;
+    }
+    if (result.valid && schema === "evidence-bundle" && readBoolFlag("--check-artifacts")) {
+        const artifactResult = await validateEvidenceArtifacts(input, resolve(readFlag("--artifact-root") ?? process.cwd()));
+        process.stdout.write(`Artifact paths checked: ${artifactResult.checked}\n`);
+        if (artifactResult.missing.length > 0) {
+            process.stdout.write("Missing artifact paths:\n");
+            for (const missing of artifactResult.missing) {
+                process.stdout.write(`- ${missing}\n`);
+            }
+            process.exitCode = 1;
+        }
+    }
+}
 else {
     process.stdout.write(`Usage:
   agentcert init --subject my-browser-agent
   agentcert init --out agentcert.config.json --tripwire-config tripwire.yml --force
+  agentcert init --subject my-browser-agent --github-action
   agentcert report --mcpbench .mcpbench/latest/results.json --tripwire .tripwire/latest/tripwire-result.json --onegent .onegent/procurement/audit-packet.json --out .agentcert/latest --subject my-agent
   agentcert corpus ingest --tripwire .tripwire/latest/tripwire-result.json --out .agentcert/corpus/corpus.jsonl --subject my-agent
   agentcert corpus review --corpus .agentcert/corpus/corpus.jsonl --reviews .agentcert/corpus/failure-reviews.jsonl --pattern-key <failure-key> --type wrong_click --status corrected
@@ -332,6 +367,8 @@ else {
   agentcert run --mcpbench .mcpbench/latest/results.json --tripwire .tripwire/latest/tripwire-result.json --onegent .onegent/procurement/audit-packet.json --out .agentcert/latest --corpus .agentcert/corpus/corpus.jsonl --monitor-out .agentcert/latest/monitor.json --reviewed-dataset-out .agentcert/latest/reviewed-failure-dataset.jsonl
   agentcert lab build --config examples/real-agents/robustness-lab/lab.config.json --out public-demo/real-agent-robustness/evidence/lab-snapshot.json
   agentcert serve --corpus .agentcert/corpus/corpus.jsonl --static public-demo/agentcert-monitor --artifact-root public-demo/browser-agent-robustness/evidence/tripwire-public-demo
+  agentcert validate .agentcert/latest/agentcert-evidence.json
+  agentcert validate .agentcert/latest/agentcert-evidence.json --check-artifacts
   agentcert schema validate --schema evidence-bundle --file .agentcert/latest/agentcert-evidence.json
 `);
 }
@@ -368,6 +405,19 @@ function readFlag(name) {
     const index = process.argv.indexOf(name);
     return index >= 0 ? process.argv[index + 1] : undefined;
 }
+function readFirstPositionalAfterCommand() {
+    for (let index = 3; index < process.argv.length; index += 1) {
+        const value = process.argv[index];
+        if (!value)
+            continue;
+        if (value.startsWith("--")) {
+            index += 1;
+            continue;
+        }
+        return value;
+    }
+    return undefined;
+}
 function readBoolFlag(name) {
     return process.argv.includes(name);
 }
@@ -467,6 +517,38 @@ async function writeStarterFile(path, content, force) {
         throw error;
     }
 }
+function reportFatalError(error) {
+    process.stderr.write(formatCliError(error));
+    process.exit(1);
+}
+function formatCliError(error) {
+    const message = error instanceof Error ? error.message : String(error);
+    const code = error && typeof error === "object" && "code" in error ? String(error.code) : undefined;
+    const detail = [`AgentCert error: ${message}`];
+    const hint = errorHint(message, code);
+    if (hint) {
+        detail.push("", hint);
+    }
+    return `${detail.join("\n")}\n`;
+}
+function errorHint(message, code) {
+    if (code === "ENOENT") {
+        return "Hint: check the artifact path. Run `npx agentcert init` for starter config, then run Tripwire before `npx agentcert run`.";
+    }
+    if (message.includes("agentcert run requires")) {
+        return "Hint: run `npx agentcert init`, pass `--tripwire .tripwire/latest/tripwire-result.json`, or use `--config agentcert.config.json`.";
+    }
+    if (message.includes("No AgentCert artifacts were loaded") || message.includes("No input artifacts were provided")) {
+        return "Hint: provide at least one artifact with `--tripwire`, `--mcpbench`, `--onegent`, or configure `agentcert.config.json`.";
+    }
+    if (message.includes("Unexpected token") || message.includes("JSON")) {
+        return "Hint: AgentCert expected JSON. Check that the artifact file exists and was not replaced by logs or HTML.";
+    }
+    if (message.includes("already exists")) {
+        return "Hint: re-run with `--force` to overwrite starter files, or choose a different `--out` path.";
+    }
+    return undefined;
+}
 function starterTripwireConfig(subject) {
     return `version: "0.1"
 project: ${JSON.stringify(subject)}
@@ -535,6 +617,11 @@ on:
 jobs:
   tripwire:
     runs-on: ubuntu-latest
+    # Uncomment to publish a hosted evidence page + clickable README badge
+    # to the gh-pages branch (also uncomment publish-pages below, then enable
+    # GitHub Pages for the gh-pages branch in the repo settings):
+    # permissions:
+    #   contents: write
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-node@v4
@@ -549,5 +636,6 @@ jobs:
           subject: ${JSON.stringify(subject)}
           agentcert-out: .agentcert/latest
           fail-on-verdict: "true"
+          # publish-pages: "true"
 `;
 }

package/dist/schema-validator.js

@@ -1,3 +1,4 @@
+import { AGENTCERT_EVIDENCE_SCHEMA_SEMVER, AGENTCERT_EVIDENCE_SCHEMA_VERSION } from "./types.js";
 export function parseSchemaId(input) {
     const value = input ?? "evidence-bundle";
     if (value === "evidence-bundle" ||
@@ -34,8 +35,8 @@ export function validateAgentCertSchema(schema, input) {
 }
 function validateEvidenceBundle(value, errors) {
     requiredConst(value, "schemaName", "agentcert.evidence_bundle", errors);
-    requiredConst(value, "schemaVersion", "1", errors);
-    requiredConst(value, "schemaSemver", "1.0.0", errors);
+    requiredConst(value, "schemaVersion", AGENTCERT_EVIDENCE_SCHEMA_VERSION, errors);
+    requiredConst(value, "schemaSemver", AGENTCERT_EVIDENCE_SCHEMA_SEMVER, errors);
     requiredConst(value, "kind", "agentcert.evidence_bundle", errors);
     requiredString(value, "runId", errors);
     requiredString(value, "generatedAt", errors);

package/dist/types.js

@@ -1 +1,2 @@
-export {};
+export const AGENTCERT_EVIDENCE_SCHEMA_VERSION = "agentcert.evidence.v0.1";
+export const AGENTCERT_EVIDENCE_SCHEMA_SEMVER = "0.1.0";

package/package.json

@@ -1,8 +1,29 @@
 {
   "name": "agentcert",
-  "version": "0.1.0",
-  "description": "Unified AgentCert evidence packet and report CLI.",
+  "version": "0.1.2",
+  "description": "Regression CI, evidence reports, and failure corpus tooling for browser agents.",
   "type": "module",
+  "license": "Apache-2.0",
+  "homepage": "https://github.com/Kakarottoooo/agentcert#readme",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Kakarottoooo/agentcert.git",
+    "directory": "packages/agentcert-cli"
+  },
+  "bugs": {
+    "url": "https://github.com/Kakarottoooo/agentcert/issues"
+  },
+  "keywords": [
+    "agent",
+    "browser-agent",
+    "ci",
+    "playwright",
+    "agent-security",
+    "llm",
+    "mcp",
+    "tripwire",
+    "evidence"
+  ],
   "bin": {
     "agentcert": "dist/cli.js"
   },