@attest-ai/core 1.0.0-rc.2 → 1.0.0-rc.3

package/dist/esm/cli.js

@@ -2,13 +2,14 @@
 
 // src/cli.ts
 import { spawnSync as spawnSync2 } from "child_process";
-import * as fs3 from "fs";
+import * as fs4 from "fs";
 import * as os2 from "os";
-import { fileURLToPath } from "url";
+import * as path4 from "path";
+import { fileURLToPath as fileURLToPath2 } from "url";
 
 // src/version.ts
-var VERSION = "1.0.0-rc.2";
-var ENGINE_VERSION = "1.0.0-rc.2";
+var VERSION = "1.0.0-rc.3";
+var ENGINE_VERSION = "1.0.0-rc.3";
 
 // src/engine-manager.ts
 import { spawn, spawnSync } from "child_process";
@@ -76,7 +77,7 @@ function httpsGet(url, redirectCount = 0) {
       new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
     );
   }
-  return new Promise((resolve, reject) => {
+  return new Promise((resolve2, reject) => {
     const client = url.startsWith("https:") ? https : http;
     client.get(url, (res) => {
       const status = res.statusCode ?? 0;
@@ -87,7 +88,7 @@ function httpsGet(url, redirectCount = 0) {
           return;
         }
         res.resume();
-        resolve(httpsGet(location, redirectCount + 1));
+        resolve2(httpsGet(location, redirectCount + 1));
         return;
       }
       if (status < 200 || status >= 300) {
@@ -97,7 +98,7 @@ function httpsGet(url, redirectCount = 0) {
       }
       const chunks = [];
       res.on("data", (chunk) => chunks.push(chunk));
-      res.on("end", () => resolve(Buffer.concat(chunks)));
+      res.on("end", () => resolve2(Buffer.concat(chunks)));
       res.on("error", reject);
     }).on("error", reject);
   });
@@ -228,8 +229,8 @@ Install options:
   }
   return [await downloadEngine(), "download"];
 }
-async function resolveEngine(explicitPath, options = {}) {
-  const [enginePath, source] = await locateEngine(explicitPath, options.allowDownload ?? true);
+async function resolveEngine(explicitPath, options2 = {}) {
+  const [enginePath, source] = await locateEngine(explicitPath, options2.allowDownload ?? true);
   const found = engineVersion(enginePath);
   if (found !== ENGINE_VERSION) {
     throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);
@@ -288,8 +289,8 @@ not_evaluated: warn
 `;
 var GITHUB_WORKFLOW_TEMPLATE = `# Written by \`attest init --ci github\`.
 # The test step records its outcome and continues so the report is always uploaded,
-# the policy evaluated and the pull request annotated; the last step fails the job
-# when the tests failed. A policy warning (exit 2) does not fail the job.
+# the JUnit file written, the policy evaluated and the pull request summarised; the last
+# step fails the job when the tests failed. A policy warning (exit 2) does not fail the job.
 name: attest
 
 on:
@@ -299,11 +300,13 @@ on:
 
 permissions:
   contents: read
-  pull-requests: write
 
 jobs:
   attest:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      pull-requests: write
     steps:
       - uses: actions/checkout@v6
         with:
@@ -317,12 +320,17 @@ jobs:
         id: tests
         continue-on-error: true
         run: npx vitest run ${REPORT_ARGS}
+      - name: Write the JUnit report
+        if: always()
+        run: npx attest report junit --report attest-report.json --out attest-junit.xml
       - name: Upload the Attest report
         if: always()
         uses: actions/upload-artifact@v7
         with:
           name: attest-report
-          path: attest-report.json
+          path: |
+            attest-report.json
+            attest-junit.xml
           if-no-files-found: error
       - name: Evaluate the merge policy
         if: always()
@@ -336,8 +344,9 @@ jobs:
           fi
           exit "$code"
       - name: Annotate the pull request
-        # Fork pull requests get a read-only token that cannot post reviews; the last step
-        # still fails the job when tests failed.
+        # Posts one summary comment (updated on every run) and annotates failures at their
+        # file and line. Fork pull requests get a read-only token that cannot comment; the
+        # last step still fails the job when tests failed.
         if: >-
           always() && github.event_name == 'pull_request' &&
           github.event.pull_request.head.repo.full_name == github.repository
@@ -361,10 +370,10 @@ function write(file, content) {
   process.stdout.write(`Created: ${file}
 `);
 }
-function scaffoldProject(targetDir, options = {}) {
+function scaffoldProject(targetDir, options2 = {}) {
   write(path2.join(targetDir, "tests", "my-agent.test.ts"), SAMPLE_TEST_TEMPLATE);
   write(path2.join(targetDir, "attest.policy.yaml"), POLICY_TEMPLATE);
-  if (options.ci === "github") {
+  if (options2.ci === "github") {
     write(path2.join(targetDir, ".github", "workflows", "attest.yml"), GITHUB_WORKFLOW_TEMPLATE);
   }
   process.stdout.write(
@@ -374,9 +383,232 @@ Next: run \`npx vitest run\`. Then change the line marked ${DEMO_MARKER} in test
   );
 }
 
+// src/capture.ts
+import { spawn as spawn2 } from "child_process";
+import * as fs3 from "fs";
+import * as path3 from "path";
+import { fileURLToPath } from "url";
+var FIXTURE_VERSION = 1;
+var DEFAULT_FIXTURE_DIR = path3.join("tests", "attest", "fixtures");
+var CaptureError = class extends Error {
+  /** The fixture written, when the engine wrote one. */
+  fixture;
+  constructor(message, fixture = null) {
+    super(message);
+    this.name = "CaptureError";
+    this.fixture = fixture;
+  }
+};
+function toPath(p) {
+  return p instanceof URL ? fileURLToPath(p) : p;
+}
+function loadFixture(file) {
+  const p = toPath(file);
+  const doc = JSON.parse(fs3.readFileSync(p, "utf8"));
+  if (!isObject(doc) || doc["fixture_version"] !== FIXTURE_VERSION) {
+    throw new CaptureError(`${p}: not a fixture_version ${FIXTURE_VERSION} capture fixture`);
+  }
+  validate(p, doc);
+  return doc;
+}
+var NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/;
+function isObject(v) {
+  return typeof v === "object" && v !== null && !Array.isArray(v);
+}
+function kindOf(v) {
+  if (Array.isArray(v)) return "array";
+  if (isObject(v)) return "object";
+  if (typeof v === "string" || typeof v === "boolean") return typeof v;
+  return "other";
+}
+function validate(p, doc) {
+  const need = (where, obj, fields) => {
+    if (!isObject(obj)) throw new CaptureError(`${p}: ${where} is not an object`);
+    for (const [key, kind] of Object.entries(fields)) {
+      if (kindOf(obj[key]) !== kind) throw new CaptureError(`${p}: ${where}.${key} is missing or not a ${kind}`);
+    }
+    return obj;
+  };
+  need("fixture", doc, { name: "string", source: "object", trace: "object", exchanges: "array", contracts: "array", verification: "object" });
+  if (!("world" in doc)) throw new CaptureError(`${p}: fixture.world is missing`);
+  if (!NAME_PATTERN.test(doc["name"])) {
+    throw new CaptureError(`${p}: fixture name ${JSON.stringify(doc["name"])} is not a valid fixture name`);
+  }
+  const v = need("fixture.verification", doc["verification"], {
+    verified: "boolean",
+    expect_fail: "boolean",
+    results: "array",
+    expected_failures: "array"
+  });
+  doc["contracts"].forEach(
+    (c, i) => need(`fixture.contracts[${i}]`, c, { id: "string", kind: "string", description: "string", type: "string", spec: "object" })
+  );
+  v["results"].forEach(
+    (r, i) => need(`fixture.verification.results[${i}]`, r, { contract: "string", status: "string", explanation: "string" })
+  );
+}
+function objectKey(k) {
+  if (k === "__proto__") return `[${JSON.stringify(k)}]`;
+  return /^[A-Za-z_$][\w$]*$/.test(k) ? k : JSON.stringify(k);
+}
+function comment(text) {
+  return text.replace(/[\p{Cc}\p{Zl}\p{Zp}]/gu, " ");
+}
+function ts(value) {
+  if (value === null || typeof value === "boolean" || typeof value === "number") {
+    return JSON.stringify(value);
+  }
+  if (typeof value === "string") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(ts).join(", ")}]`;
+  if (typeof value === "object") {
+    const entries = Object.entries(value).map(
+      ([k, v]) => `${objectKey(k)}: ${ts(v)}`
+    );
+    return entries.length === 0 ? "{}" : `{ ${entries.join(", ")} }`;
+  }
+  throw new Error(`value ${String(value)} has no literal`);
+}
+function str(spec, key) {
+  const v = spec[key];
+  if (typeof v !== "string") throw new Error(`missing ${key}`);
+  return v;
+}
+function options(entries) {
+  const set = entries.filter(([, v]) => v !== void 0 && v !== null && v !== false);
+  return set.length === 0 ? "" : `{ ${set.map(([k, v]) => `${k}: ${ts(v)}`).join(", ")} }`;
+}
+function call(method, args, opts) {
+  return `.${method}(${[...args, ...opts ? [opts] : []].join(", ")})`;
+}
+function dslCall(contract) {
+  const s = contract.spec;
+  const soft = s["soft"] === true ? true : void 0;
+  try {
+    if (contract.type === "trace") {
+      const check = s["check"];
+      if (check === "trajectory" && s["mode"] !== "partial_order") {
+        const mode = s["mode"] === "strict" || s["mode"] === void 0 ? void 0 : s["mode"];
+        if (!Array.isArray(s["tools"])) throw new Error("missing tools");
+        return call("trajectory", [ts(s["tools"])], options([["mode", mode], ["soft", soft]]));
+      }
+      if (check === "tool_order") {
+        return call("toolCalledBefore", [ts(str(s, "before")), ts(str(s, "after"))], options([["soft", soft]]));
+      }
+      if (check === "tool_cardinality") {
+        return call(
+          "toolCardinality",
+          [ts(str(s, "tool"))],
+          options([
+            ["per", s["per"]],
+            ["atLeast", s["min"]],
+            ["atMost", s["max"]],
+            ["soft", soft]
+          ])
+        );
+      }
+    }
+    if (contract.type === "schema" && s["target"] === "output.structured") {
+      if (typeof s["schema"] !== "object" || s["schema"] === null) throw new Error("missing schema");
+      return call("outputMatchesSchema", [ts(s["schema"])], "");
+    }
+    if (contract.type === "constraint" && s["field"] === "metadata.cost_usd" && s["operator"] === "lte") {
+      if (typeof s["value"] !== "number") throw new Error("missing value");
+      return call("costUnder", [ts(s["value"])], options([["soft", soft]]));
+    }
+  } catch (err) {
+    throw new Error(`contract ${contract.id}: malformed spec (${err.message})`);
+  }
+  throw new Error(`contract ${contract.id}: no TypeScript DSL call for ${contract.type} spec`);
+}
+function testPathFor(fixturePath) {
+  const name = path3.basename(fixturePath).replace(/\.attest\.json$/, "");
+  return path3.join(path3.dirname(path3.dirname(fixturePath)), `${name}.test.ts`);
+}
+function renderVitest(fixture, fixturePath, testPath) {
+  const calls = [];
+  const skipped = [];
+  for (const c of fixture.contracts) {
+    try {
+      calls.push([c, dslCall(c)]);
+    } catch (err) {
+      skipped.push(err.message);
+    }
+  }
+  const rel = path3.relative(path3.dirname(testPath), fixturePath).split(path3.sep).join("/");
+  const status = new Map(fixture.verification.results.map((r) => [r.contract, r.status]));
+  const expectFail = fixture.verification.expect_fail;
+  const lines = [
+    `// Contracts drafted by \`attest capture\` from fixture ${comment(ts(fixture.name))}.`,
+    "//",
+    "// Review and edit before committing: they check the captured trace without running",
+    "// the agent. Re-capturing does not overwrite this file.",
+    `import { attestExpect, loadFixture, type ExpectChain } from "@attest-ai/core";`,
+    `import { ${expectFail ? "inspect" : "evaluate"} } from "@attest-ai/vitest";`,
+    `import { describe, ${expectFail ? "expect, " : ""}it } from "vitest";`,
+    "",
+    `const FIXTURE = loadFixture(new URL(${ts(rel.startsWith(".") ? rel : `./${rel}`)}, import.meta.url));`
+  ];
+  if (skipped.length > 0) {
+    lines.push("", "// Drafted contracts without a TypeScript DSL call (left out):");
+    for (const s of skipped) lines.push(`// - ${comment(s)}`);
+  }
+  lines.push("", "function contracts(): ExpectChain {", "  return attestExpect(FIXTURE.trace)");
+  for (const [c, dsl] of calls) {
+    lines.push(`    // ${comment(`${c.id}: ${c.description}`)}`, `    ${dsl}`);
+  }
+  lines[lines.length - 1] += ";";
+  lines.push("}");
+  if (expectFail) {
+    lines.push("", "// Status of each contract on the captured incident, in contracts() order.");
+    lines.push("const EXPECTED: Array<[string, string]> = [");
+    for (const [c] of calls) lines.push(`  [${ts(c.id)}, ${ts(status.get(c.id) ?? "pass")}],`);
+    lines.push(
+      "];",
+      "",
+      `describe(${ts(fixture.name)}, () => {`,
+      `  it("reproduces the incident", async () => {`,
+      "    const result = await inspect(contracts());",
+      "",
+      "    expect(result.assertionResults.map((row) => row.status)).toEqual(EXPECTED.map(([, s]) => s));",
+      "  });",
+      "});"
+    );
+  } else {
+    lines.push(
+      "",
+      `describe(${ts(fixture.name)}, () => {`,
+      `  it("keeps its drafted contracts", async () => {`,
+      "    await evaluate(contracts());",
+      "  });",
+      "});"
+    );
+  }
+  return `${lines.join("\n")}
+`;
+}
+function emitVitest(fixturePath) {
+  const testPath = testPathFor(fixturePath);
+  const existing = fs3.lstatSync(testPath, { throwIfNoEntry: false });
+  if (existing?.isSymbolicLink()) {
+    throw new CaptureError(`${testPath} is a symbolic link; refusing to draft a test through it`);
+  }
+  if (existing !== void 0) return null;
+  const source = renderVitest(loadFixture(fixturePath), fixturePath, testPath);
+  fs3.mkdirSync(path3.dirname(testPath), { recursive: true });
+  try {
+    fs3.writeFileSync(testPath, source, { encoding: "utf8", flag: "wx" });
+  } catch (err) {
+    if (err.code === "EEXIST") {
+      throw new CaptureError(`${testPath} appeared while drafting; left unchanged`);
+    }
+    throw err;
+  }
+  return testPath;
+}
+
 // src/cli.ts
 var HELP_FLAGS = /* @__PURE__ */ new Set(["-h", "--help"]);
-var SDK_USAGE = "usage: attest <command> [args]\n\nSDK commands:\n  init [--ci github]  scaffold a sample test and merge policy here\n                      (--ci github also writes .github/workflows/attest.yml)\n\noptions:\n  --version  print the SDK version and the engine version it requires\n  --help     print this help\n";
+var SDK_USAGE = "usage: attest <command> [args]\n\nSDK commands:\n  init [--ci github]  scaffold a sample test and merge policy here\n                      (--ci github also writes .github/workflows/attest.yml)\n\n`attest capture` (engine) also drafts tests/attest/<name>.test.ts from a verified\nfixture, unless that file exists.\n\noptions:\n  --version  print the SDK version and the engine version it requires\n  --help     print this help\n";
 var INIT_USAGE = "usage: attest init [--ci github]  (scaffolds into the current directory)";
 function cmdInit(args) {
   if (args.some((a) => HELP_FLAGS.has(a))) {
@@ -468,10 +700,67 @@ async function forward(args) {
     );
     return 1;
   }
+  if (command === "capture" && captureWritesFixture(args.slice(1))) {
+    return captureAndEmit(engine, args.slice(1));
+  }
   const proc = spawnSync2(engine, args, { stdio: "inherit" });
   if (proc.error) throw proc.error;
   return exitStatus(proc.status, proc.signal);
 }
+var CAPTURE_NO_TEST = /* @__PURE__ */ new Set(["check-redaction", "serve", "h", "help"]);
+function flagName(arg) {
+  if (!arg.startsWith("-") || arg === "-" || arg === "--") return null;
+  return arg.replace(/^-+/, "").split("=", 1)[0] ?? null;
+}
+function captureWritesFixture(args) {
+  for (const a of args) {
+    if (a === "--") break;
+    const name = flagName(a);
+    if (name !== null && CAPTURE_NO_TEST.has(name)) return false;
+  }
+  return true;
+}
+function userResultFile(args) {
+  for (const a of args) {
+    if (a === "--") break;
+    if (flagName(a) === "result-file") return true;
+  }
+  return false;
+}
+function captureAndEmit(engine, args) {
+  if (userResultFile(args)) {
+    process.stderr.write(
+      "attest: capture --result-file is set by the attest CLI; run attest-engine capture directly to choose it\n"
+    );
+    return 2;
+  }
+  const tmp = fs4.mkdtempSync(path4.join(os2.tmpdir(), "attest-capture-"));
+  const resultFile = path4.join(tmp, "result.json");
+  let result;
+  let code;
+  try {
+    const proc = spawnSync2(engine, ["capture", "--result-file", resultFile, ...args], { stdio: "inherit" });
+    if (proc.error) throw proc.error;
+    code = exitStatus(proc.status, proc.signal);
+    if (!fs4.existsSync(resultFile)) return code;
+    result = JSON.parse(fs4.readFileSync(resultFile, "utf8"));
+  } finally {
+    fs4.rmSync(tmp, { recursive: true, force: true });
+  }
+  const fixture = result.fixture;
+  if (!result.verified) {
+    process.stderr.write(`  test: none drafted for unverified ${fixture}
+`);
+    return code;
+  }
+  const written = emitVitest(fixture);
+  process.stderr.write(
+    written === null ? `  test: ${testPathFor(fixture)} exists; left unchanged
+` : `  test: drafted ${written}
+`
+  );
+  return code;
+}
 async function main(argv = process.argv.slice(2)) {
   const [command] = argv;
   if (command === void 0 || HELP_FLAGS.has(command) || command === "help" && argv.length === 1) {
@@ -495,8 +784,8 @@ ${SDK_USAGE}`);
 }
 function isDirectExecution() {
   const script = process.argv[1];
-  if (script === void 0 || !fs3.existsSync(script)) return false;
-  return fs3.realpathSync(script) === fs3.realpathSync(fileURLToPath(import.meta.url));
+  if (script === void 0 || !fs4.existsSync(script)) return false;
+  return fs4.realpathSync(script) === fs4.realpathSync(fileURLToPath2(import.meta.url));
 }
 if (isDirectExecution()) {
   main().then(
@@ -509,6 +798,7 @@ if (isDirectExecution()) {
   );
 }
 export {
+  captureWritesFixture,
   engineCommands,
   exitStatus,
   main