@attest-ai/core 0.9.0 → 1.0.0-rc.2

package/dist/esm/cli.js

@@ -1,473 +1,516 @@
 #!/usr/bin/env node
 
 // src/cli.ts
-import * as fs from "fs";
-import * as path from "path";
-import * as os from "os";
+import { spawnSync as spawnSync2 } from "child_process";
+import * as fs3 from "fs";
+import * as os2 from "os";
+import { fileURLToPath } from "url";
 
 // src/version.ts
-var VERSION = "0.9.0";
+var VERSION = "1.0.0-rc.2";
+var ENGINE_VERSION = "1.0.0-rc.2";
 
-// src/calibration.ts
-import { createHash } from "crypto";
-function computeAgreement(pairs, threshold) {
-  if (pairs.length === 0) {
-    throw new Error("no labeled pairs available");
-  }
-  if (threshold <= 0 || threshold >= 1) {
-    throw new Error("threshold must be in (0, 1)");
+// src/engine-manager.ts
+import { spawn, spawnSync } from "child_process";
+import { createInterface } from "readline";
+import { existsSync, accessSync, statSync as statSync2, constants as fsConstants } from "fs";
+import { delimiter, join as join2 } from "path";
+
+// src/engine-downloader.ts
+import * as https from "https";
+import * as http from "http";
+import * as fs from "fs";
+import * as path from "path";
+import * as os from "os";
+import * as crypto from "crypto";
+var RELEASE_BASE = `https://github.com/attest-framework/attest/releases/download/v${ENGINE_VERSION}`;
+var CHECKSUMS_URL = `${RELEASE_BASE}/checksums-sha256.txt`;
+var MAX_REDIRECTS = 5;
+function versionedBinDir() {
+  return path.join(os.homedir(), ".attest", "bin", ENGINE_VERSION);
+}
+function platformKey() {
+  const platform2 = os.platform();
+  const arch2 = os.arch();
+  const mapping = {
+    darwin: { arm64: "darwin-arm64", x64: "darwin-amd64" },
+    linux: { arm64: "linux-arm64", x64: "linux-amd64" },
+    win32: { x64: "windows-amd64", arm64: "windows-arm64" }
+  };
+  const osMap = mapping[platform2];
+  if (!osMap) {
+    throw new Error(
+      `Unsupported platform: ${platform2}. Supported: darwin, linux, win32.`
+    );
   }
-  const n = pairs.length;
-  let humanPos = 0;
-  let judgePos = 0;
-  let bothPos = 0;
-  let bothNeg = 0;
-  for (const p of pairs) {
-    const hPos = p.human >= threshold;
-    const jPos = p.judge >= threshold;
-    if (hPos) humanPos++;
-    if (jPos) judgePos++;
-    if (hPos && jPos) bothPos++;
-    if (!hPos && !jPos) bothNeg++;
+  const key = osMap[arch2];
+  if (!key) {
+    throw new Error(
+      `Unsupported architecture '${arch2}' on platform '${platform2}'. Supported architectures: ${Object.keys(osMap).join(", ")}.`
+    );
   }
-  const humanNeg = n - humanPos;
-  const judgeNeg = n - judgePos;
-  const agreement = (bothPos + bothNeg) / n;
-  const expected = (humanPos * judgePos + humanNeg * judgeNeg) / (n * n);
-  const cohenKappa = expected < 1 ? (agreement - expected) / (1 - expected) : 0;
-  const rocAuc = rocAUC(pairs, threshold);
-  return { threshold, n, agreement, cohenKappa, rocAuc };
+  return key;
 }
-function rocAUC(pairs, threshold) {
-  const rows = pairs.map((p) => ({
-    score: p.judge,
-    pos: p.human >= threshold
-  }));
-  const pos = rows.filter((r) => r.pos).length;
-  const neg = rows.length - pos;
-  if (pos === 0 || neg === 0) return 0;
-  rows.sort((a, b) => a.score - b.score);
-  let rankSumPos = 0;
-  let i = 0;
-  while (i < rows.length) {
-    let j = i;
-    while (j < rows.length && rows[j].score === rows[i].score) {
-      j++;
-    }
-    const avgRank = (i + j + 1) / 2;
-    for (let k = i; k < j; k++) {
-      if (rows[k].pos) rankSumPos += avgRank;
+function binaryFilename() {
+  return os.platform() === "win32" ? "attest-engine.exe" : "attest-engine";
+}
+function cachedEnginePath() {
+  const binPath = path.join(versionedBinDir(), binaryFilename());
+  return fs.statSync(binPath, { throwIfNoEntry: false })?.isFile() ? binPath : null;
+}
+function parseChecksums(text) {
+  const checksums = /* @__PURE__ */ new Map();
+  for (const line of text.split("\n")) {
+    const trimmed = line.trim();
+    if (!trimmed) continue;
+    const match = trimmed.match(/^([a-f0-9]{64})\s+(.+)$/);
+    if (match) {
+      checksums.set(match[2], match[1]);
     }
-    i = j;
   }
-  const auc = (rankSumPos - pos * (pos + 1) / 2) / (pos * neg);
-  if (Number.isNaN(auc) || !Number.isFinite(auc)) return 0;
-  return auc;
+  return checksums;
 }
-function loadLabelsCSV(text) {
-  const rows = parseCSV(text);
-  if (rows.length === 0) {
-    throw new Error("empty CSV");
-  }
-  let start = 0;
-  if (rows[0].length >= 2) {
-    const firstHuman = parseFloatStrict(rows[0][1]);
-    if (firstHuman === null) start = 1;
+function httpsGet(url, redirectCount = 0) {
+  if (redirectCount > MAX_REDIRECTS) {
+    return Promise.reject(
+      new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
+    );
   }
-  const out = [];
-  for (let idx = start; idx < rows.length; idx++) {
-    const row = rows[idx];
-    const lineNum = idx + 1;
-    if (row.length === 0) continue;
-    if (row[0].trimStart().startsWith("#")) continue;
-    if (row.length < 2) {
-      throw new Error(
-        `CSV line ${lineNum}: want at least 2 columns (input, human_label)`
-      );
-    }
-    const human = parseFloatStrict(row[1]);
-    if (human === null) {
-      throw new Error(`CSV line ${lineNum}: human_label not a float: ${row[1]}`);
-    }
-    const rec = {
-      input: row[0],
-      humanLabel: human,
-      judgeScore: 0,
-      judgeKnown: false
-    };
-    if (row.length >= 3 && row[2].trim() !== "") {
-      const judge = parseFloatStrict(row[2]);
-      if (judge === null) {
-        throw new Error(
-          `CSV line ${lineNum}: judge_score not a float: ${row[2]}`
-        );
+  return new Promise((resolve, reject) => {
+    const client = url.startsWith("https:") ? https : http;
+    client.get(url, (res) => {
+      const status = res.statusCode ?? 0;
+      if (status === 301 || status === 302) {
+        const location = res.headers.location;
+        if (!location) {
+          reject(new Error(`Redirect ${status} without Location header from ${url}`));
+          return;
+        }
+        res.resume();
+        resolve(httpsGet(location, redirectCount + 1));
+        return;
       }
-      rec.judgeScore = judge;
-      rec.judgeKnown = true;
-    }
-    out.push(rec);
-  }
-  if (out.length === 0) {
-    throw new Error("CSV contained no labeled rows");
-  }
-  return out;
-}
-function loadLabelsJSONL(text) {
-  const out = [];
-  const lines = text.split(/\r?\n/);
-  for (let i = 0; i < lines.length; i++) {
-    const stripped = lines[i].trim();
-    if (!stripped) continue;
-    let obj;
-    try {
-      obj = JSON.parse(stripped);
-    } catch (err) {
-      throw new Error(`JSONL line ${i + 1}: ${err.message}`);
-    }
-    if (Object.keys(obj).length === 0) continue;
-    if (obj["human_label"] === void 0) {
-      throw new Error(`JSONL line ${i + 1}: missing human_label`);
-    }
-    const human = parseJSONNumber(obj["human_label"]);
-    if (human === null) {
-      throw new Error(`JSONL line ${i + 1}: human_label is not a number`);
-    }
-    const rec = {
-      input: typeof obj["input"] === "string" ? obj["input"] : "",
-      humanLabel: human,
-      judgeScore: 0,
-      judgeKnown: false
-    };
-    if (obj["judge_score"] !== void 0 && obj["judge_score"] !== null) {
-      const judge = parseJSONNumber(obj["judge_score"]);
-      if (judge === null) {
-        throw new Error(`JSONL line ${i + 1}: judge_score is not a number`);
+      if (status < 200 || status >= 300) {
+        res.resume();
+        reject(new Error(`HTTP ${status} fetching ${url}`));
+        return;
       }
-      rec.judgeScore = judge;
-      rec.judgeKnown = true;
-    }
-    out.push(rec);
+      const chunks = [];
+      res.on("data", (chunk) => chunks.push(chunk));
+      res.on("end", () => resolve(Buffer.concat(chunks)));
+      res.on("error", reject);
+    }).on("error", reject);
+  });
+}
+async function downloadEngine() {
+  const key = platformKey();
+  const filename = `attest-engine-${key}${os.platform() === "win32" ? ".exe" : ""}`;
+  const binaryUrl = `${RELEASE_BASE}/${filename}`;
+  process.stderr.write(`[attest] Downloading engine v${ENGINE_VERSION} for ${key}...
+`);
+  process.stderr.write(`[attest] Fetching checksums...
+`);
+  const checksumsText = (await httpsGet(CHECKSUMS_URL)).toString("utf-8");
+  const checksums = parseChecksums(checksumsText);
+  const expectedHash = checksums.get(filename);
+  if (!expectedHash) {
+    throw new Error(
+      `No checksum found for '${filename}' in checksums-sha256.txt. Available files: ${[...checksums.keys()].join(", ")}`
+    );
   }
-  if (out.length === 0) {
-    throw new Error("JSONL contained no labeled rows");
+  process.stderr.write(`[attest] Downloading ${binaryUrl}...
+`);
+  const data = await httpsGet(binaryUrl);
+  const actualHash = crypto.createHash("sha256").update(data).digest("hex");
+  if (actualHash !== expectedHash) {
+    throw new Error(
+      `SHA256 mismatch for ${filename}. Expected: ${expectedHash}, got: ${actualHash}. The download may be corrupted. Retry or download manually.`
+    );
   }
-  return out;
-}
-function parseFloatStrict(raw) {
-  const trimmed = raw.trim();
-  if (trimmed === "") return null;
-  const n = Number(trimmed);
-  if (Number.isNaN(n)) return null;
-  return n;
+  process.stderr.write(`[attest] Checksum verified.
+`);
+  const dir = versionedBinDir();
+  fs.mkdirSync(dir, { recursive: true });
+  const destPath = path.join(dir, binaryFilename());
+  const suffix = `${String(process.pid)}.${crypto.randomBytes(8).toString("hex")}`;
+  const tmpPath = path.join(dir, `.${binaryFilename()}.tmp.${suffix}`);
+  try {
+    fs.writeFileSync(tmpPath, data, { flag: "wx", mode: 493 });
+    fs.chmodSync(tmpPath, 493);
+    fs.renameSync(tmpPath, destPath);
+  } catch (err) {
+    if (err.code !== "EEXIST") fs.rmSync(tmpPath, { force: true });
+    throw err;
+  }
+  process.stderr.write(`[attest] Engine installed at ${destPath}
+`);
+  return destPath;
 }
-function parseJSONNumber(value) {
-  if (typeof value !== "number") return null;
-  if (Number.isNaN(value)) return null;
-  return value;
+
+// src/engine-manager.ts
+var ENGINE_BINARY_NAME = "attest-engine";
+var VERSION_PROBE_TIMEOUT_MS = 1e4;
+var EngineVersionMismatchError = class extends Error {
+  enginePath;
+  source;
+  found;
+  required;
+  constructor(enginePath, source, found, required) {
+    const reported = found === null ? "reports no version" : `reports version ${found}`;
+    super(
+      `attest-engine at ${enginePath} (from ${source}) ${reported}; this SDK requires attest-engine ${required}. Point ATTEST_ENGINE_PATH at a matching engine, or remove the mismatched one from PATH so the SDK can download ${required} into ~/.attest/bin/${required}/.`
+    );
+    this.name = "EngineVersionMismatchError";
+    this.enginePath = enginePath;
+    this.source = source;
+    this.found = found;
+    this.required = required;
+  }
+};
+function engineVersion(enginePath) {
+  const proc = spawnSync(enginePath, ["version"], {
+    stdio: ["ignore", "pipe", "pipe"],
+    encoding: "utf-8",
+    timeout: VERSION_PROBE_TIMEOUT_MS
+  });
+  if (proc.error) throw proc.error;
+  const out = proc.stdout.trim();
+  const prefix = `${ENGINE_BINARY_NAME} `;
+  if (proc.status !== 0 || !out.startsWith(prefix)) return null;
+  return out.slice(prefix.length).trim();
 }
-function parseCSV(text) {
-  const rows = [];
-  let current = [];
-  let field = "";
-  let inQuotes = false;
-  let i = 0;
-  while (i < text.length) {
-    const ch = text[i];
-    if (inQuotes) {
-      if (ch === '"') {
-        if (text[i + 1] === '"') {
-          field += '"';
-          i += 2;
-          continue;
-        }
-        inQuotes = false;
-        i++;
-        continue;
-      }
-      field += ch;
-      i++;
-      continue;
-    }
-    if (ch === '"') {
-      inQuotes = true;
-      i++;
-      continue;
-    }
-    if (ch === ",") {
-      current.push(field);
-      field = "";
-      i++;
-      continue;
-    }
-    if (ch === "\n") {
-      current.push(field);
-      rows.push(current);
-      current = [];
-      field = "";
-      i++;
-      continue;
-    }
-    if (ch === "\r") {
-      i++;
+function onPath() {
+  for (const dir of (process.env["PATH"] ?? "").split(delimiter)) {
+    if (!dir) continue;
+    const candidate = join2(dir, binaryFilename());
+    try {
+      accessSync(candidate, fsConstants.X_OK);
+    } catch {
       continue;
     }
-    field += ch;
-    i++;
-  }
-  if (field.length > 0 || current.length > 0) {
-    current.push(field);
-    rows.push(current);
+    if (statSync2(candidate).isFile()) return candidate;
   }
-  return rows;
+  return null;
 }
-
-// src/cli.ts
-function cacheDir() {
-  const envOverride = process.env["ATTEST_CACHE_DIR"];
-  if (envOverride) return envOverride;
-  return path.join(os.homedir(), ".attest", "cache");
-}
-function cacheDbPath() {
-  return path.join(cacheDir(), "attest.db");
+function requireFile(enginePath, label) {
+  if (!existsSync(enginePath) || !statSync2(enginePath).isFile()) {
+    throw new Error(`${label}=${enginePath} does not exist or is not a file.`);
+  }
 }
-function cmdCacheStats() {
-  const dbPath = cacheDbPath();
-  const exists = fs.existsSync(dbPath);
-  let fileSize = 0;
-  if (exists) {
-    fileSize = fs.statSync(dbPath).size;
+async function locateEngine(explicitPath, allowDownload) {
+  if (explicitPath) {
+    requireFile(explicitPath, "Engine path");
+    return [explicitPath, "explicit engine path"];
+  }
+  const envPath = process.env["ATTEST_ENGINE_PATH"];
+  if (envPath) {
+    requireFile(envPath, "ATTEST_ENGINE_PATH");
+    return [envPath, "ATTEST_ENGINE_PATH"];
+  }
+  const found = onPath();
+  if (found) return [found, "PATH"];
+  const cached = cachedEnginePath();
+  if (cached) return [cached, "engine cache"];
+  if (!allowDownload) {
+    throw new Error(
+      `No '${ENGINE_BINARY_NAME}' ${ENGINE_VERSION} in ATTEST_ENGINE_PATH, PATH or ~/.attest/bin/${ENGINE_VERSION}/.`
+    );
+  }
+  const noDownload = process.env["ATTEST_ENGINE_NO_DOWNLOAD"]?.toLowerCase();
+  if (noDownload === "1" || noDownload === "true" || noDownload === "yes") {
+    throw new Error(
+      `Cannot find '${ENGINE_BINARY_NAME}' ${ENGINE_VERSION} and ATTEST_ENGINE_NO_DOWNLOAD is set.
+Install options:
+  1. Build from source: make engine, then set ATTEST_ENGINE_PATH
+  2. Download release v${ENGINE_VERSION}: https://github.com/attest-framework/attest/releases
+  3. Allow auto-download by unsetting ATTEST_ENGINE_NO_DOWNLOAD`
+    );
   }
-  const stats = { exists, file_size: fileSize, path: dbPath };
-  process.stdout.write(JSON.stringify(stats) + "\n");
+  return [await downloadEngine(), "download"];
 }
-function cmdCacheClear() {
-  const dbPath = cacheDbPath();
-  if (fs.existsSync(dbPath)) {
-    fs.unlinkSync(dbPath);
-    process.stdout.write(`Cleared cache: ${dbPath}
-`);
-  } else {
-    process.stdout.write(`No cache to clear: ${dbPath}
-`);
+async function resolveEngine(explicitPath, options = {}) {
+  const [enginePath, source] = await locateEngine(explicitPath, options.allowDownload ?? true);
+  const found = engineVersion(enginePath);
+  if (found !== ENGINE_VERSION) {
+    throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);
   }
+  return enginePath;
 }
-var CONFTEST_TEMPLATE = `import { describe, it, expect } from "vitest";
-import { attestExpect, TraceBuilder } from "@attest-ai/core";
 
-describe("my agent", () => {
-  it("returns expected output", async () => {
-    const trace = new TraceBuilder("my-agent")
-      .setInput({ user_message: "hello" })
-      .setOutput({ message: "world" })
-      .build();
+// src/scaffold.ts
+import * as fs2 from "fs";
+import * as path2 from "path";
+var DEMO_MARKER = "ATTEST-DEMO";
+var SAMPLE_TEST_TEMPLATE = `// Sample Attest test written by \`attest init\`. Run it with \`npx vitest run\`.
+import { it } from "vitest";
+import { attestExpect, TraceBuilder, type Trace } from "@attest-ai/core";
+import { evaluate } from "@attest-ai/vitest";
 
-    attestExpect(trace).outputContains("world");
-  });
+// The trace of one agent run, built by hand so the sample needs no model, API key
+// or network. Replace it with a trace from your agent: wrap the agent with
+// \`new Agent(...)\` or record it with an adapter such as OpenAIAdapter.
+function myAgentResult(): Trace {
+  return new TraceBuilder("refund-agent")
+    .setInput({ user_message: "Please refund order ORD-1001." })
+    .addToolCall("lookup_order", {
+      args: { order_id: "ORD-1001" },
+      result: { status: "delivered", amount: 42.0 },
+    })
+    .addToolCall("issue_refund", {
+      args: { order_id: "ORD-1001", amount: 42.0 },
+      result: { refund_id: "RF-1001" },
+    })
+    .setOutput({
+      message: "Your refund of $42.00 has been issued.",
+      structured: { refund_id: "RF-1001", amount: 42.0 },
+    })
+    .setMetadata({ total_tokens: 180, cost_usd: 0.002, latency_ms: 900 })
+    .build();
+}
+
+it("my agent issues the refund", async (context) => {
+  const chain = attestExpect(myAgentResult())
+    .outputMatchesSchema({ type: "object", required: ["refund_id"] }) // layer 1: schema
+    .costUnder(0.01) // layer 2: constraint
+    .toolsCalledInOrder(["lookup_order", "issue_refund"]) // layer 3: trace
+    .outputContains("refund"); // ${DEMO_MARKER}: change "refund" to "store credit"
+  await evaluate(chain, { context });
 });
 `;
-function cmdInit(targetDir) {
-  const testsDir = path.join(targetDir, "tests");
-  fs.mkdirSync(testsDir, { recursive: true });
-  process.stdout.write(`Directory: ${testsDir}
-`);
-  const sampleTestPath = path.join(testsDir, "test_my_agent.test.ts");
-  if (fs.existsSync(sampleTestPath)) {
-    process.stderr.write(`Skipped (exists): ${sampleTestPath}
-`);
-  } else {
-    fs.writeFileSync(sampleTestPath, CONFTEST_TEMPLATE);
-    process.stdout.write(`Created: ${sampleTestPath}
+var REPORT_ARGS = "--reporter=default --reporter=@attest-ai/vitest/reporter --outputFile.attest=attest-report.json";
+var POLICY_TEMPLATE = `# Merge policy for \`attest policy evaluate --report <report> --policy attest.policy.yaml\`.
+# Exit 0: pass. Exit 1: a blocking rule failed. Exit 2: warnings only.
+layers:
+  0:  # layer 0 caps every layer together
+    max_hard_fails: 0
+# Assertions that could not run (no judge configured, simulation) warn instead of block.
+not_evaluated: warn
+`;
+var GITHUB_WORKFLOW_TEMPLATE = `# Written by \`attest init --ci github\`.
+# The test step records its outcome and continues so the report is always uploaded,
+# the policy evaluated and the pull request annotated; the last step fails the job
+# when the tests failed. A policy warning (exit 2) does not fail the job.
+name: attest
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+permissions:
+  contents: read
+  pull-requests: write
+
+jobs:
+  attest:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@v6
+        with:
+          node-version: 22
+      - name: Install dependencies
+        run: npm ci
+      - name: Run tests
+        id: tests
+        continue-on-error: true
+        run: npx vitest run ${REPORT_ARGS}
+      - name: Upload the Attest report
+        if: always()
+        uses: actions/upload-artifact@v7
+        with:
+          name: attest-report
+          path: attest-report.json
+          if-no-files-found: error
+      - name: Evaluate the merge policy
+        if: always()
+        run: |
+          set +e
+          npx attest policy evaluate --report attest-report.json --policy attest.policy.yaml
+          code=$?
+          if [ "$code" -eq 2 ]; then
+            echo "::warning::attest policy: warnings only (exit 2)"
+            exit 0
+          fi
+          exit "$code"
+      - name: Annotate the pull request
+        # Fork pull requests get a read-only token that cannot post reviews; the last step
+        # still fails the job when tests failed.
+        if: >-
+          always() && github.event_name == 'pull_request' &&
+          github.event.pull_request.head.repo.full_name == github.repository
+        uses: attest-framework/attest/.github/actions/pr-annotate@v${VERSION}
+        with:
+          report: attest-report.json
+      - name: Fail the job when tests failed
+        if: always() && steps.tests.outcome == 'failure'
+        run: |
+          echo "::error::vitest failed; see the Run tests step"
+          exit 1
+`;
+function write(file, content) {
+  if (fs2.existsSync(file)) {
+    process.stderr.write(`Skipped (exists): ${file}
 `);
+    return;
   }
-}
-function cmdValidate(targetDir) {
-  const testsDir = path.join(targetDir, "tests");
-  let testFiles = [];
-  if (fs.existsSync(testsDir)) {
-    testFiles = fs.readdirSync(testsDir).filter((f) => f.endsWith(".test.ts") || f.endsWith(".test.js"));
-  }
-  if (testFiles.length === 0) {
-    process.stderr.write(
-      `Warning: no test files (*.test.ts / *.test.js) found in ${testsDir}.
-`
-    );
-  } else {
-    process.stdout.write(`Found ${testFiles.length} test file(s) in ${testsDir}
+  fs2.mkdirSync(path2.dirname(file), { recursive: true });
+  fs2.writeFileSync(file, content);
+  process.stdout.write(`Created: ${file}
 `);
-  }
-  const STALE_DAYS = 30;
-  const now = Date.now();
-  const goldenFiles = findGoldenFiles(targetDir);
-  for (const golden of goldenFiles) {
-    const mtime = fs.statSync(golden).mtimeMs;
-    const ageDays = Math.floor((now - mtime) / (1e3 * 60 * 60 * 24));
-    if (ageDays > STALE_DAYS) {
-      process.stderr.write(
-        `Warning: golden trace ${golden} is ${ageDays} days old (>${STALE_DAYS} days). Consider regenerating.
-`
-      );
-    }
-  }
 }
-function findGoldenFiles(dir) {
-  const results = [];
-  if (!fs.existsSync(dir)) return results;
-  const entries = fs.readdirSync(dir, { withFileTypes: true });
-  for (const entry of entries) {
-    const fullPath = path.join(dir, entry.name);
-    if (entry.isDirectory() && entry.name !== "node_modules") {
-      results.push(...findGoldenFiles(fullPath));
-    } else if (entry.isFile() && entry.name.endsWith(".golden")) {
-      results.push(fullPath);
-    }
+function scaffoldProject(targetDir, options = {}) {
+  write(path2.join(targetDir, "tests", "my-agent.test.ts"), SAMPLE_TEST_TEMPLATE);
+  write(path2.join(targetDir, "attest.policy.yaml"), POLICY_TEMPLATE);
+  if (options.ci === "github") {
+    write(path2.join(targetDir, ".github", "workflows", "attest.yml"), GITHUB_WORKFLOW_TEMPLATE);
   }
-  return results;
+  process.stdout.write(
+    `
+Next: run \`npx vitest run\`. Then change the line marked ${DEMO_MARKER} in tests/my-agent.test.ts and run it again to see a failure.
+`
+  );
 }
-function main(argv = process.argv.slice(2)) {
-  const args = argv;
-  if (args[0] === "--version") {
-    process.stdout.write(`attest ${VERSION}
+
+// src/cli.ts
+var HELP_FLAGS = /* @__PURE__ */ new Set(["-h", "--help"]);
+var SDK_USAGE = "usage: attest <command> [args]\n\nSDK commands:\n  init [--ci github]  scaffold a sample test and merge policy here\n                      (--ci github also writes .github/workflows/attest.yml)\n\noptions:\n  --version  print the SDK version and the engine version it requires\n  --help     print this help\n";
+var INIT_USAGE = "usage: attest init [--ci github]  (scaffolds into the current directory)";
+function cmdInit(args) {
+  if (args.some((a) => HELP_FLAGS.has(a))) {
+    process.stdout.write(`${INIT_USAGE}
 `);
-    process.exit(0);
+    return 0;
   }
-  if (args[0] === "cache" && args.length >= 2) {
-    if (args[1] === "stats") {
-      cmdCacheStats();
-      process.exit(0);
-    } else if (args[1] === "clear") {
-      cmdCacheClear();
-      process.exit(0);
-    } else {
-      process.stderr.write(`Unknown cache subcommand: ${args[1]}
+  const joined = args.join(" ");
+  if (args.length > 0 && joined !== "--ci github" && joined !== "--ci=github") {
+    process.stderr.write(`${INIT_USAGE}
 `);
-      process.stderr.write("Available: cache stats, cache clear\n");
-      process.exit(1);
-    }
+    return 2;
   }
-  if (args[0] === "init") {
-    cmdInit(process.cwd());
-    process.exit(0);
-  }
-  if (args[0] === "validate") {
-    cmdValidate(process.cwd());
-    process.exit(0);
-  }
-  if (args[0] === "calibrate") {
-    cmdCalibrate(args.slice(1));
-    process.exit(0);
-  }
-  process.stderr.write(
-    "Usage: attest <command>\n\nCommands:\n  --version       Print version\n  init            Scaffold a test project\n  validate        Validate test suite\n  cache stats     Show cache statistics\n  cache clear     Clear cache database\n  calibrate       Compute judge agreement metrics over labeled data\n"
-  );
-  process.exit(args.length === 0 ? 0 : 1);
+  scaffoldProject(process.cwd(), args.length > 0 ? { ci: "github" } : {});
+  return 0;
 }
-function parseCalibrateArgs(args) {
-  const opts = {
-    labels: "",
-    rubric: "default",
-    rubricVersion: "",
-    threshold: 0.5
-  };
-  for (let i = 0; i < args.length; i++) {
-    const arg = args[i];
-    const next = args[i + 1];
-    if (arg === "--labels") {
-      if (next === void 0) throw new Error("--labels requires a path");
-      opts.labels = next;
-      i++;
-    } else if (arg === "--rubric") {
-      if (next === void 0) throw new Error("--rubric requires a name");
-      opts.rubric = next;
-      i++;
-    } else if (arg === "--rubric-version") {
-      if (next === void 0) throw new Error("--rubric-version requires a value");
-      opts.rubricVersion = next;
-      i++;
-    } else if (arg === "--threshold") {
-      if (next === void 0) throw new Error("--threshold requires a number");
-      const parsed = Number(next);
-      if (next.trim() === "" || Number.isNaN(parsed)) {
-        throw new Error(`--threshold value ${JSON.stringify(next)} is not a number`);
-      }
-      opts.threshold = parsed;
-      i++;
-    } else {
-      throw new Error(`unknown calibrate flag: ${arg}`);
-    }
-  }
-  if (!opts.labels) {
-    throw new Error("--labels is required");
+var HELP_PROBE_TIMEOUT_MS = 1e4;
+function errorMessage(err) {
+  return err instanceof Error ? err.message : String(err);
+}
+function engineCommands(engine, timeoutMs = HELP_PROBE_TIMEOUT_MS) {
+  const proc = spawnSync2(engine, ["help"], {
+    stdio: ["ignore", "pipe", "pipe"],
+    encoding: "utf-8",
+    timeout: timeoutMs
+  });
+  if (proc.error || proc.status !== 0) return null;
+  const lines = proc.stdout.split("\n");
+  const start = lines.indexOf("commands:");
+  if (start === -1) return null;
+  const commands = /* @__PURE__ */ new Map();
+  for (const line of lines.slice(start + 1)) {
+    const match = /^\s*(\S+)\s*(.*)$/.exec(line);
+    if (match?.[1]) commands.set(match[1], (match[2] ?? "").trim());
   }
-  return opts;
+  return commands;
 }
-function cmdCalibrate(args) {
-  let opts;
+async function printHelp() {
+  process.stdout.write(`${SDK_USAGE}
+`);
+  let engine;
   try {
-    opts = parseCalibrateArgs(args);
+    engine = await resolveEngine(void 0, { allowDownload: false });
   } catch (err) {
-    process.stderr.write(`calibrate: ${err.message}
+    const firstLine = (errorMessage(err).split("\n")[0] ?? "").replace(/\.$/, "");
+    process.stdout.write(
+      `engine commands: not available (${firstLine}). Run \`attest doctor\` to download attest-engine ${ENGINE_VERSION}, or set ATTEST_ENGINE_PATH.
+`
+    );
+    return 0;
+  }
+  const commands = engineCommands(engine);
+  if (commands === null) {
+    process.stdout.write(
+      `engine commands: not available (attest-engine at ${engine} lists no commands; it predates the command registry).
+`
+    );
+    return 0;
+  }
+  const width = Math.max(0, ...[...commands.keys()].map((name) => name.length));
+  process.stdout.write(`engine commands (attest-engine ${ENGINE_VERSION}, ${engine}):
+`);
+  for (const [name, summary] of commands) {
+    process.stdout.write(`  ${name.padEnd(width)}  ${summary}
 `);
-    process.exit(2);
-    return;
   }
-  const text = fs.readFileSync(opts.labels, "utf-8");
-  const ext = path.extname(opts.labels).toLowerCase();
-  let records;
+  return 0;
+}
+function exitStatus(status, signal) {
+  if (status !== null) return status;
+  if (signal !== null) return 128 + (os2.constants.signals[signal] ?? 0);
+  return 1;
+}
+async function forward(args) {
+  let engine;
   try {
-    if (ext === ".jsonl" || ext === ".ndjson") {
-      records = loadLabelsJSONL(text);
-    } else if (ext === ".csv") {
-      records = loadLabelsCSV(text);
-    } else if (text.trimStart().startsWith("{")) {
-      records = loadLabelsJSONL(text);
-    } else {
-      records = loadLabelsCSV(text);
-    }
+    engine = await resolveEngine();
   } catch (err) {
-    process.stderr.write(`calibrate: ${err.message}
+    process.stderr.write(`attest: ${errorMessage(err)}
 `);
-    process.exit(1);
-    return;
+    return 1;
   }
-  const pairs = records.filter((r) => r.judgeKnown).map((r) => ({ human: r.humanLabel, judge: r.judgeScore }));
-  const missing = records.filter((r) => !r.judgeKnown).length;
-  if (pairs.length === 0) {
+  const command = args[0] ?? "";
+  const commands = engineCommands(engine);
+  if (commands === null || command !== "help" && !commands.has(command)) {
     process.stderr.write(
-      `calibrate: no rows had judge_score (${missing} rows missing); pre-score the labels file first
+      `attest: attest-engine at ${engine} (version ${ENGINE_VERSION}) predates \`attest ${command}\`: its help does not list that command. Set ATTEST_ENGINE_PATH to an engine built from this release (make engine), or install a release whose engine provides it.
 `
     );
-    process.exit(1);
-    return;
+    return 1;
   }
-  let result;
-  try {
-    result = computeAgreement(pairs, opts.threshold);
-  } catch (err) {
-    process.stderr.write(`calibrate: ${err.message}
+  const proc = spawnSync2(engine, args, { stdio: "inherit" });
+  if (proc.error) throw proc.error;
+  return exitStatus(proc.status, proc.signal);
+}
+async function main(argv = process.argv.slice(2)) {
+  const [command] = argv;
+  if (command === void 0 || HELP_FLAGS.has(command) || command === "help" && argv.length === 1) {
+    return printHelp();
+  }
+  if (command === "--version") {
+    process.stdout.write(`attest ${VERSION} (requires attest-engine ${ENGINE_VERSION})
 `);
-    process.exit(1);
-    return;
+    return 0;
   }
-  const output = {
-    rubric_name: opts.rubric,
-    rubric_version: opts.rubricVersion,
-    threshold: result.threshold,
-    label_count: result.n,
-    agreement: round3(result.agreement),
-    cohen_kappa: round3(result.cohenKappa),
-    roc_auc: round3(result.rocAuc),
-    missing_judge: missing
-  };
-  process.stdout.write(JSON.stringify(output, null, 2) + "\n");
+  if (command === "init") {
+    return cmdInit(argv.slice(1));
+  }
+  if (command.startsWith("-")) {
+    process.stderr.write(`attest: unknown option ${command}
+
+${SDK_USAGE}`);
+    return 2;
+  }
+  return forward(argv);
 }
-function round3(x) {
-  return Math.round(x * 1e3) / 1e3;
+function isDirectExecution() {
+  const script = process.argv[1];
+  if (script === void 0 || !fs3.existsSync(script)) return false;
+  return fs3.realpathSync(script) === fs3.realpathSync(fileURLToPath(import.meta.url));
 }
-var isDirectExecution = typeof import.meta.url === "string" && import.meta.url.startsWith("file:") && process.argv[1] !== void 0 && import.meta.url.endsWith(path.basename(process.argv[1]));
-if (isDirectExecution) {
-  main();
+if (isDirectExecution()) {
+  main().then(
+    (code) => process.exit(code),
+    (err) => {
+      process.stderr.write(`attest: ${errorMessage(err)}
+`);
+      process.exit(1);
+    }
+  );
 }
 export {
+  engineCommands,
+  exitStatus,
   main
 };
 //# sourceMappingURL=cli.js.map