@attest-ai/core 1.0.1 → 1.1.0-rc.1

package/dist/esm/cli.js

@@ -3,30 +3,34 @@
 // src/cli.ts
 import { spawnSync as spawnSync2 } from "child_process";
 import * as fs4 from "fs";
-import * as os2 from "os";
+import * as os3 from "os";
 import * as path4 from "path";
-import { fileURLToPath as fileURLToPath2 } from "url";
+import { fileURLToPath as fileURLToPath3 } from "url";
 
 // src/version.ts
-var VERSION = "1.0.1";
-var ENGINE_VERSION = "1.0.1";
+var VERSION = "1.1.0-rc.1";
+var ENGINE_VERSION = "1.1.0-rc.1";
 
 // src/engine-manager.ts
 import { spawn, spawnSync } from "child_process";
 import { createInterface } from "readline";
-import { existsSync, accessSync, statSync as statSync2, constants as fsConstants } from "fs";
+import { existsSync as existsSync2, accessSync, statSync as statSync2, constants as fsConstants } from "fs";
 import { delimiter, join as join2 } from "path";
 
 // src/engine-downloader.ts
 import * as https from "https";
-import * as http from "http";
 import * as fs from "fs";
 import * as path from "path";
 import * as os from "os";
 import * as crypto from "crypto";
+import { fileURLToPath } from "url";
 var RELEASE_BASE = `https://github.com/attest-framework/attest/releases/download/v${ENGINE_VERSION}`;
-var CHECKSUMS_URL = `${RELEASE_BASE}/checksums-sha256.txt`;
+var TABLE_NAME = "engine-checksums.json";
 var MAX_REDIRECTS = 5;
+var REDIRECT_STATUSES = /* @__PURE__ */ new Set([301, 302, 303, 307, 308]);
+var DEFAULT_CONNECT_TIMEOUT_MS = 1e4;
+var DEFAULT_TOTAL_TIMEOUT_MS = 12e4;
+var SHA256_HEX = /^[0-9a-f]{64}$/;
 function versionedBinDir() {
   return path.join(os.homedir(), ".attest", "bin", ENGINE_VERSION);
 }
@@ -59,73 +63,131 @@ function cachedEnginePath() {
   const binPath = path.join(versionedBinDir(), binaryFilename());
   return fs.statSync(binPath, { throwIfNoEntry: false })?.isFile() ? binPath : null;
 }
-function parseChecksums(text) {
-  const checksums = /* @__PURE__ */ new Map();
-  for (const line of text.split("\n")) {
-    const trimmed = line.trim();
-    if (!trimmed) continue;
-    const match = trimmed.match(/^([a-f0-9]{64})\s+(.+)$/);
-    if (match) {
-      checksums.set(match[2], match[1]);
-    }
+function packageRoot() {
+  let dir = typeof __dirname === "string" ? __dirname : path.dirname(fileURLToPath(import.meta.url));
+  while (!fs.existsSync(path.join(dir, "package.json"))) {
+    const parent = path.dirname(dir);
+    if (parent === dir) return dir;
+    dir = parent;
   }
-  return checksums;
+  return dir;
 }
-function httpsGet(url, redirectCount = 0) {
-  if (redirectCount > MAX_REDIRECTS) {
-    return Promise.reject(
-      new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
+function embeddedHash(key, tablePath) {
+  if (!fs.existsSync(tablePath)) {
+    throw new Error(
+      `This @attest-ai/core install has no embedded engine hash table (${TABLE_NAME}), which is how a source or workspace install looks, so it never downloads an engine. Point ATTEST_ENGINE_PATH at an engine binary, or build one with \`make engine\` and set ATTEST_ENGINE_PATH=./bin/attest-engine.`
     );
   }
-  return new Promise((resolve2, reject) => {
-    const client = url.startsWith("https:") ? https : http;
-    client.get(url, (res) => {
-      const status = res.statusCode ?? 0;
-      if (status === 301 || status === 302) {
-        const location = res.headers.location;
-        if (!location) {
-          reject(new Error(`Redirect ${status} without Location header from ${url}`));
-          return;
-        }
-        res.resume();
-        resolve2(httpsGet(location, redirectCount + 1));
+  let hash;
+  try {
+    const table = JSON.parse(fs.readFileSync(tablePath, "utf-8"));
+    hash = table[ENGINE_VERSION]?.[key];
+  } catch (err) {
+    throw new Error(
+      `@attest-ai/core ${VERSION} ships an unreadable engine hash table at ${tablePath}: ${String(err)}`
+    );
+  }
+  if (typeof hash !== "string" || !SHA256_HEX.test(hash)) {
+    throw new Error(
+      `@attest-ai/core ${VERSION} ships no valid engine hash for v${ENGINE_VERSION} on ${key} in ${tablePath}.`
+    );
+  }
+  return hash;
+}
+function httpsGet(url, o) {
+  return new Promise((resolve3, reject) => {
+    let current;
+    let done = false;
+    const finish = (err, data) => {
+      if (done) return;
+      done = true;
+      clearTimeout(deadline);
+      if (err) {
+        current?.destroy();
+        reject(err);
+      } else {
+        resolve3(data);
+      }
+    };
+    const deadline = setTimeout(
+      () => finish(new Error(`Timed out after ${o.totalTimeoutMs}ms fetching ${url}`)),
+      o.totalTimeoutMs
+    );
+    const hop = (target, redirects) => {
+      if (redirects > MAX_REDIRECTS) {
+        finish(new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`));
+        return;
+      }
+      let parsed;
+      try {
+        parsed = new URL(target);
+      } catch {
+        finish(new Error(`Invalid URL: ${target}`));
         return;
       }
-      if (status < 200 || status >= 300) {
-        res.resume();
-        reject(new Error(`HTTP ${status} fetching ${url}`));
+      if (parsed.protocol !== "https:") {
+        finish(new Error(`Refusing to download over a non-https URL: ${target}`));
         return;
       }
-      const chunks = [];
-      res.on("data", (chunk) => chunks.push(chunk));
-      res.on("end", () => resolve2(Buffer.concat(chunks)));
-      res.on("error", reject);
-    }).on("error", reject);
+      current = https.get(parsed, { ca: o.ca, timeout: o.connectTimeoutMs }, (res) => {
+        const status = res.statusCode ?? 0;
+        if (REDIRECT_STATUSES.has(status)) {
+          res.resume();
+          const location = res.headers.location;
+          if (!location) {
+            finish(new Error(`Redirect ${status} without Location header from ${target}`));
+            return;
+          }
+          let next;
+          try {
+            next = new URL(location, parsed).toString();
+          } catch {
+            finish(new Error(`Invalid redirect Location ${JSON.stringify(location)} from ${target}`));
+            return;
+          }
+          hop(next, redirects + 1);
+          return;
+        }
+        if (status !== 200) {
+          res.resume();
+          finish(new Error(`HTTP ${status} fetching ${target}`));
+          return;
+        }
+        const chunks = [];
+        res.on("data", (chunk) => chunks.push(chunk));
+        res.on("end", () => finish(null, Buffer.concat(chunks)));
+        res.on("error", (err) => finish(err));
+      });
+      current.on(
+        "timeout",
+        () => finish(new Error(`No response within ${o.connectTimeoutMs}ms from ${target}`))
+      );
+      current.on("error", (err) => finish(err));
+    };
+    hop(url, 0);
   });
 }
-async function downloadEngine() {
+async function downloadEngine(options2 = {}) {
   const key = platformKey();
   const filename = `attest-engine-${key}${os.platform() === "win32" ? ".exe" : ""}`;
-  const binaryUrl = `${RELEASE_BASE}/${filename}`;
+  const expectedHash = embeddedHash(key, options2.tablePath ?? path.join(packageRoot(), TABLE_NAME));
+  const binaryUrl = `${options2.releaseBase ?? RELEASE_BASE}/${filename}`;
   process.stderr.write(`[attest] Downloading engine v${ENGINE_VERSION} for ${key}...
 `);
-  process.stderr.write(`[attest] Fetching checksums...
-`);
-  const checksumsText = (await httpsGet(CHECKSUMS_URL)).toString("utf-8");
-  const checksums = parseChecksums(checksumsText);
-  const expectedHash = checksums.get(filename);
-  if (!expectedHash) {
-    throw new Error(
-      `No checksum found for '${filename}' in checksums-sha256.txt. Available files: ${[...checksums.keys()].join(", ")}`
-    );
+  let data;
+  try {
+    data = await httpsGet(binaryUrl, {
+      ca: options2.ca,
+      connectTimeoutMs: options2.connectTimeoutMs ?? DEFAULT_CONNECT_TIMEOUT_MS,
+      totalTimeoutMs: options2.totalTimeoutMs ?? DEFAULT_TOTAL_TIMEOUT_MS
+    });
+  } catch (err) {
+    throw new Error(`Failed to download engine from ${binaryUrl}: ${err.message}`);
   }
-  process.stderr.write(`[attest] Downloading ${binaryUrl}...
-`);
-  const data = await httpsGet(binaryUrl);
   const actualHash = crypto.createHash("sha256").update(data).digest("hex");
   if (actualHash !== expectedHash) {
     throw new Error(
-      `SHA256 mismatch for ${filename}. Expected: ${expectedHash}, got: ${actualHash}. The download may be corrupted. Retry or download manually.`
+      `${filename} does not match the hash shipped in @attest-ai/core ${VERSION}: expected ${expectedHash}, got ${actualHash}. Refusing to install it. Retry, or set ATTEST_ENGINE_PATH to a binary you trust.`
     );
   }
   process.stderr.write(`[attest] Checksum verified.
@@ -194,7 +256,7 @@ function onPath() {
   return null;
 }
 function requireFile(enginePath, label) {
-  if (!existsSync(enginePath) || !statSync2(enginePath).isFile()) {
+  if (!existsSync2(enginePath) || !statSync2(enginePath).isFile()) {
     throw new Error(`${label}=${enginePath} does not exist or is not a file.`);
   }
 }
@@ -402,8 +464,9 @@ Next: run \`npx vitest run\`. Then change the line marked ${DEMO_MARKER} in test
 // src/capture.ts
 import { spawn as spawn2 } from "child_process";
 import * as fs3 from "fs";
+import * as os2 from "os";
 import * as path3 from "path";
-import { fileURLToPath } from "url";
+import { fileURLToPath as fileURLToPath2 } from "url";
 var FIXTURE_VERSION = 1;
 var DEFAULT_FIXTURE_DIR = path3.join("tests", "attest", "fixtures");
 var CaptureError = class extends Error {
@@ -416,7 +479,7 @@ var CaptureError = class extends Error {
   }
 };
 function toPath(p) {
-  return p instanceof URL ? fileURLToPath(p) : p;
+  return p instanceof URL ? fileURLToPath2(p) : p;
 }
 function loadFixture(file) {
   const p = toPath(file);
@@ -540,7 +603,75 @@ function testPathFor(fixturePath) {
   const name = path3.basename(fixturePath).replace(/\.attest\.json$/, "");
   return path3.join(path3.dirname(path3.dirname(fixturePath)), `${name}.test.ts`);
 }
-function renderVitest(fixture, fixturePath, testPath) {
+function relUrl(target, testPath) {
+  const rel = path3.relative(path3.dirname(testPath), target).split(path3.sep).join("/");
+  const url = rel === "" ? "." : rel;
+  return `new URL(${ts(url.startsWith(".") ? url : `./${url}`)}, import.meta.url)`;
+}
+function chainLines(calls, subject, param) {
+  const lines = [`function contracts(${param}): ExpectChain {`, `  return attestExpect(${subject})`];
+  for (const [c, dsl] of calls) {
+    lines.push(`    // ${comment(`${c.id}: ${c.description}`)}`, `    ${dsl}`);
+  }
+  lines[lines.length - 1] += ";";
+  lines.push("}");
+  return lines;
+}
+function skippedLines(skipped) {
+  if (skipped.length === 0) return [];
+  return ["", "// Drafted contracts without a TypeScript DSL call (left out):", ...skipped.map((s) => `// - ${comment(s)}`)];
+}
+function renderReplay(fixture, calls, skipped, fixturePath, testPath, rerun) {
+  const meaning = fixture.verification.expect_fail ? [
+    "// The recording is an incident: these contracts failed on it. The test passes once",
+    "// the new run satisfies them, so it fails again if the agent regresses."
+  ] : ["// The test fails when the agent's behavior stops satisfying them."];
+  const world = rerun.world === void 0 ? "undefined" : relUrl(rerun.world, testPath);
+  const lines = [
+    `// Replay test drafted by \`attest capture\` from fixture ${comment(ts(fixture.name))}.`,
+    "//",
+    "// Re-runs COMMAND, the source command of the fixture, under strict replay (the engine's",
+    "// `replay --mode strict --no-live`): OpenAI- and Anthropic-compatible calls are served",
+    "// from the recording, and one that matches no recording fails the test. Other providers",
+    "// are not blocked: run this test without their credentials. The drafted contracts are",
+    "// evaluated on the NEW run's trace, not on the recording.",
+    "//",
+    "// COMMAND is what this test runs and is reviewed with the test: if the fixture's",
+    "// source.command stops matching it, the test fails instead of running the fixture's.",
+    ...meaning,
+    "//",
+    "// A pass means the new run is consistent with the recording and satisfies the drafted",
+    "// contracts; it does not prove the agent correct. Identities are compared by role",
+    "// (relative identity): a run that swaps who plays which role misses, while the absolute",
+    "// identity is not checked. Review and edit before committing. Re-capturing does not",
+    "// overwrite this file.",
+    `import { attestExpect, replayRun, type ExpectChain } from "@attest-ai/core";`,
+    `import type { Trace } from "@attest-ai/core/proto";`,
+    `import { evaluate } from "@attest-ai/vitest";`,
+    `import { describe, it } from "vitest";`,
+    "",
+    `const FIXTURE = ${relUrl(fixturePath, testPath)};`,
+    "// The directory `attest capture` ran in: the source command's relative paths start here.",
+    `const RUN_DIR = ${relUrl(rerun.cwd, testPath)};`,
+    `const WORLD = ${world};`,
+    "// Reviewed argv: the only command this test runs.",
+    `const COMMAND = ${ts(fixture.source.command ?? [])};`,
+    ...skippedLines(skipped),
+    "",
+    ...chainLines(calls, "trace", "trace: Trace"),
+    "",
+    `describe(${ts(fixture.name)}, () => {`,
+    `  it("new run is consistent with the recording", async () => {`,
+    "    const trace = await replayRun(FIXTURE, { command: COMMAND, cwd: RUN_DIR, world: WORLD });",
+    "",
+    "    await evaluate(contracts(trace));",
+    "  });",
+    "});"
+  ];
+  return `${lines.join("\n")}
+`;
+}
+function renderVitest(fixture, fixturePath, testPath, rerun) {
   const calls = [];
   const skipped = [];
   for (const c of fixture.contracts) {
@@ -550,7 +681,9 @@ function renderVitest(fixture, fixturePath, testPath) {
       skipped.push(err.message);
     }
   }
-  const rel = path3.relative(path3.dirname(testPath), fixturePath).split(path3.sep).join("/");
+  if (rerun !== void 0 && fixture.source.kind === "command" && fixture.source.replay_capable) {
+    return renderReplay(fixture, calls, skipped, fixturePath, testPath, rerun);
+  }
   const status = new Map(fixture.verification.results.map((r) => [r.contract, r.status]));
   const expectFail = fixture.verification.expect_fail;
   const lines = [
@@ -562,18 +695,11 @@ function renderVitest(fixture, fixturePath, testPath) {
     `import { ${expectFail ? "inspect" : "evaluate"} } from "@attest-ai/vitest";`,
     `import { describe, ${expectFail ? "expect, " : ""}it } from "vitest";`,
     "",
-    `const FIXTURE = loadFixture(new URL(${ts(rel.startsWith(".") ? rel : `./${rel}`)}, import.meta.url));`
+    `const FIXTURE = loadFixture(${relUrl(fixturePath, testPath)});`,
+    ...skippedLines(skipped),
+    "",
+    ...chainLines(calls, "FIXTURE.trace", "")
   ];
-  if (skipped.length > 0) {
-    lines.push("", "// Drafted contracts without a TypeScript DSL call (left out):");
-    for (const s of skipped) lines.push(`// - ${comment(s)}`);
-  }
-  lines.push("", "function contracts(): ExpectChain {", "  return attestExpect(FIXTURE.trace)");
-  for (const [c, dsl] of calls) {
-    lines.push(`    // ${comment(`${c.id}: ${c.description}`)}`, `    ${dsl}`);
-  }
-  lines[lines.length - 1] += ";";
-  lines.push("}");
   if (expectFail) {
     lines.push("", "// Status of each contract on the captured incident, in contracts() order.");
     lines.push("const EXPECTED: Array<[string, string]> = [");
@@ -602,14 +728,14 @@ function renderVitest(fixture, fixturePath, testPath) {
   return `${lines.join("\n")}
 `;
 }
-function emitVitest(fixturePath) {
+function emitVitest(fixturePath, rerun) {
   const testPath = testPathFor(fixturePath);
   const existing = fs3.lstatSync(testPath, { throwIfNoEntry: false });
   if (existing?.isSymbolicLink()) {
     throw new CaptureError(`${testPath} is a symbolic link; refusing to draft a test through it`);
   }
   if (existing !== void 0) return null;
-  const source = renderVitest(loadFixture(fixturePath), fixturePath, testPath);
+  const source = renderVitest(loadFixture(fixturePath), fixturePath, testPath, rerun);
   fs3.mkdirSync(path3.dirname(testPath), { recursive: true });
   try {
     fs3.writeFileSync(testPath, source, { encoding: "utf8", flag: "wx" });
@@ -624,7 +750,7 @@ function emitVitest(fixturePath) {
 
 // src/cli.ts
 var HELP_FLAGS = /* @__PURE__ */ new Set(["-h", "--help"]);
-var SDK_USAGE = "usage: attest <command> [args]\n\nSDK commands:\n  init [--ci github] [--skills] [--mcp] [--json]\n                      scaffold a sample test and merge policy here\n                      (--ci github also writes .github/workflows/attest.yml;\n                      --skills installs the agent skills with attest-engine;\n                      --mcp writes the MCP client configs (.mcp.json, .cursor/mcp.json,\n                      .codex/config.toml) with attest-engine;\n                      --json prints {created, updated, skipped} paths as JSON)\n\n`attest capture` (engine) also drafts tests/attest/<name>.test.ts from a verified\nfixture, unless that file exists.\n\noptions:\n  --version  print the SDK version and the engine version it requires\n  --help     print this help\n";
+var SDK_USAGE = "usage: attest <command> [args]\n\nSDK commands:\n  init [--ci github] [--skills] [--mcp] [--json]\n                      scaffold a sample test and merge policy here\n                      (--ci github also writes .github/workflows/attest.yml;\n                      --skills installs the agent skills with attest-engine;\n                      --mcp writes the MCP client configs (.mcp.json, .cursor/mcp.json,\n                      .codex/config.toml) with attest-engine;\n                      --json prints {created, updated, skipped} paths as JSON)\n\n`attest capture` (engine) also drafts tests/attest/<name>.test.ts from a fixture that\nis consistent with the recording, unless that file exists. For a command capture the\ntest re-runs the command written in it under strict replay (OpenAI- and Anthropic-\ncompatible calls are served from the recording; other providers are not blocked) and\nevaluates the contracts on the new run.\n\noptions:\n  --version  print the SDK version and the engine version it requires\n  --help     print this help\n";
 var HELP_PROBE_TIMEOUT_MS = 1e4;
 function errorMessage(err) {
   return err instanceof Error ? err.message : String(err);
@@ -679,7 +805,7 @@ async function printHelp() {
 }
 function exitStatus(status, signal) {
   if (status !== null) return status;
-  if (signal !== null) return 128 + (os2.constants.signals[signal] ?? 0);
+  if (signal !== null) return 128 + (os3.constants.signals[signal] ?? 0);
   return 1;
 }
 async function engineProviding(command) {
@@ -765,6 +891,18 @@ function captureWritesFixture(args) {
   }
   return true;
 }
+function captureWorldPath(args) {
+  for (let i = 0; i < args.length; i++) {
+    const a = args[i];
+    if (a === "--") return void 0;
+    const [name, ...rest] = a.split("=");
+    if (name === "--world" || name === "-world") {
+      const value = rest.length > 0 ? rest.join("=") : args[i + 1];
+      return value === void 0 ? void 0 : path4.resolve(value);
+    }
+  }
+  return void 0;
+}
 function userResultFile(args) {
   for (const a of args) {
     if (a === "--") break;
@@ -779,7 +917,7 @@ function captureAndEmit(engine, args) {
     );
     return 2;
   }
-  const tmp = fs4.mkdtempSync(path4.join(os2.tmpdir(), "attest-capture-"));
+  const tmp = fs4.mkdtempSync(path4.join(os3.tmpdir(), "attest-capture-"));
   const resultFile = path4.join(tmp, "result.json");
   let result;
   let code;
@@ -799,10 +937,10 @@ function captureAndEmit(engine, args) {
 `);
     return code;
   }
-  const written = emitVitest(fixture);
+  const written = emitVitest(fixture, { cwd: process.cwd(), world: captureWorldPath(args) });
   process.stderr.write(
     written === null ? `  test: ${testPathFor(fixture)} exists; left unchanged
-` : `  test: drafted ${written}
+` : `  test: drafted ${written}; for a command capture it re-runs the command under strict replay (other providers are not blocked: run it without their credentials) and evaluates the contracts on the new run
 `
   );
   return code;
@@ -992,7 +1130,7 @@ ${SDK_USAGE}`);
 function isDirectExecution() {
   const script = process.argv[1];
   if (script === void 0 || !fs4.existsSync(script)) return false;
-  return fs4.realpathSync(script) === fs4.realpathSync(fileURLToPath2(import.meta.url));
+  return fs4.realpathSync(script) === fs4.realpathSync(fileURLToPath3(import.meta.url));
 }
 if (isDirectExecution()) {
   main().then(
@@ -1005,6 +1143,7 @@ if (isDirectExecution()) {
   );
 }
 export {
+  captureWorldPath,
   captureWritesFixture,
   engineCommands,
   exitStatus,