@attest-ai/core 0.5.0 → 0.9.0

package/dist/esm/cli.js

@@ -6,7 +6,220 @@ import * as path from "path";
 import * as os from "os";
 
 // src/version.ts
-var VERSION = "0.5.0";
+var VERSION = "0.9.0";
+
+// src/calibration.ts
+import { createHash } from "crypto";
+function computeAgreement(pairs, threshold) {
+  if (pairs.length === 0) {
+    throw new Error("no labeled pairs available");
+  }
+  if (threshold <= 0 || threshold >= 1) {
+    throw new Error("threshold must be in (0, 1)");
+  }
+  const n = pairs.length;
+  let humanPos = 0;
+  let judgePos = 0;
+  let bothPos = 0;
+  let bothNeg = 0;
+  for (const p of pairs) {
+    const hPos = p.human >= threshold;
+    const jPos = p.judge >= threshold;
+    if (hPos) humanPos++;
+    if (jPos) judgePos++;
+    if (hPos && jPos) bothPos++;
+    if (!hPos && !jPos) bothNeg++;
+  }
+  const humanNeg = n - humanPos;
+  const judgeNeg = n - judgePos;
+  const agreement = (bothPos + bothNeg) / n;
+  const expected = (humanPos * judgePos + humanNeg * judgeNeg) / (n * n);
+  const cohenKappa = expected < 1 ? (agreement - expected) / (1 - expected) : 0;
+  const rocAuc = rocAUC(pairs, threshold);
+  return { threshold, n, agreement, cohenKappa, rocAuc };
+}
+function rocAUC(pairs, threshold) {
+  const rows = pairs.map((p) => ({
+    score: p.judge,
+    pos: p.human >= threshold
+  }));
+  const pos = rows.filter((r) => r.pos).length;
+  const neg = rows.length - pos;
+  if (pos === 0 || neg === 0) return 0;
+  rows.sort((a, b) => a.score - b.score);
+  let rankSumPos = 0;
+  let i = 0;
+  while (i < rows.length) {
+    let j = i;
+    while (j < rows.length && rows[j].score === rows[i].score) {
+      j++;
+    }
+    const avgRank = (i + j + 1) / 2;
+    for (let k = i; k < j; k++) {
+      if (rows[k].pos) rankSumPos += avgRank;
+    }
+    i = j;
+  }
+  const auc = (rankSumPos - pos * (pos + 1) / 2) / (pos * neg);
+  if (Number.isNaN(auc) || !Number.isFinite(auc)) return 0;
+  return auc;
+}
+function loadLabelsCSV(text) {
+  const rows = parseCSV(text);
+  if (rows.length === 0) {
+    throw new Error("empty CSV");
+  }
+  let start = 0;
+  if (rows[0].length >= 2) {
+    const firstHuman = parseFloatStrict(rows[0][1]);
+    if (firstHuman === null) start = 1;
+  }
+  const out = [];
+  for (let idx = start; idx < rows.length; idx++) {
+    const row = rows[idx];
+    const lineNum = idx + 1;
+    if (row.length === 0) continue;
+    if (row[0].trimStart().startsWith("#")) continue;
+    if (row.length < 2) {
+      throw new Error(
+        `CSV line ${lineNum}: want at least 2 columns (input, human_label)`
+      );
+    }
+    const human = parseFloatStrict(row[1]);
+    if (human === null) {
+      throw new Error(`CSV line ${lineNum}: human_label not a float: ${row[1]}`);
+    }
+    const rec = {
+      input: row[0],
+      humanLabel: human,
+      judgeScore: 0,
+      judgeKnown: false
+    };
+    if (row.length >= 3 && row[2].trim() !== "") {
+      const judge = parseFloatStrict(row[2]);
+      if (judge === null) {
+        throw new Error(
+          `CSV line ${lineNum}: judge_score not a float: ${row[2]}`
+        );
+      }
+      rec.judgeScore = judge;
+      rec.judgeKnown = true;
+    }
+    out.push(rec);
+  }
+  if (out.length === 0) {
+    throw new Error("CSV contained no labeled rows");
+  }
+  return out;
+}
+function loadLabelsJSONL(text) {
+  const out = [];
+  const lines = text.split(/\r?\n/);
+  for (let i = 0; i < lines.length; i++) {
+    const stripped = lines[i].trim();
+    if (!stripped) continue;
+    let obj;
+    try {
+      obj = JSON.parse(stripped);
+    } catch (err) {
+      throw new Error(`JSONL line ${i + 1}: ${err.message}`);
+    }
+    if (Object.keys(obj).length === 0) continue;
+    if (obj["human_label"] === void 0) {
+      throw new Error(`JSONL line ${i + 1}: missing human_label`);
+    }
+    const human = parseJSONNumber(obj["human_label"]);
+    if (human === null) {
+      throw new Error(`JSONL line ${i + 1}: human_label is not a number`);
+    }
+    const rec = {
+      input: typeof obj["input"] === "string" ? obj["input"] : "",
+      humanLabel: human,
+      judgeScore: 0,
+      judgeKnown: false
+    };
+    if (obj["judge_score"] !== void 0 && obj["judge_score"] !== null) {
+      const judge = parseJSONNumber(obj["judge_score"]);
+      if (judge === null) {
+        throw new Error(`JSONL line ${i + 1}: judge_score is not a number`);
+      }
+      rec.judgeScore = judge;
+      rec.judgeKnown = true;
+    }
+    out.push(rec);
+  }
+  if (out.length === 0) {
+    throw new Error("JSONL contained no labeled rows");
+  }
+  return out;
+}
+function parseFloatStrict(raw) {
+  const trimmed = raw.trim();
+  if (trimmed === "") return null;
+  const n = Number(trimmed);
+  if (Number.isNaN(n)) return null;
+  return n;
+}
+function parseJSONNumber(value) {
+  if (typeof value !== "number") return null;
+  if (Number.isNaN(value)) return null;
+  return value;
+}
+function parseCSV(text) {
+  const rows = [];
+  let current = [];
+  let field = "";
+  let inQuotes = false;
+  let i = 0;
+  while (i < text.length) {
+    const ch = text[i];
+    if (inQuotes) {
+      if (ch === '"') {
+        if (text[i + 1] === '"') {
+          field += '"';
+          i += 2;
+          continue;
+        }
+        inQuotes = false;
+        i++;
+        continue;
+      }
+      field += ch;
+      i++;
+      continue;
+    }
+    if (ch === '"') {
+      inQuotes = true;
+      i++;
+      continue;
+    }
+    if (ch === ",") {
+      current.push(field);
+      field = "";
+      i++;
+      continue;
+    }
+    if (ch === "\n") {
+      current.push(field);
+      rows.push(current);
+      current = [];
+      field = "";
+      i++;
+      continue;
+    }
+    if (ch === "\r") {
+      i++;
+      continue;
+    }
+    field += ch;
+    i++;
+  }
+  if (field.length > 0 || current.length > 0) {
+    current.push(field);
+    rows.push(current);
+  }
+  return rows;
+}
 
 // src/cli.ts
 function cacheDir() {
@@ -139,11 +352,117 @@ function main(argv = process.argv.slice(2)) {
     cmdValidate(process.cwd());
     process.exit(0);
   }
+  if (args[0] === "calibrate") {
+    cmdCalibrate(args.slice(1));
+    process.exit(0);
+  }
   process.stderr.write(
-    "Usage: attest <command>\n\nCommands:\n  --version       Print version\n  init            Scaffold a test project\n  validate        Validate test suite\n  cache stats     Show cache statistics\n  cache clear     Clear cache database\n"
+    "Usage: attest <command>\n\nCommands:\n  --version       Print version\n  init            Scaffold a test project\n  validate        Validate test suite\n  cache stats     Show cache statistics\n  cache clear     Clear cache database\n  calibrate       Compute judge agreement metrics over labeled data\n"
   );
   process.exit(args.length === 0 ? 0 : 1);
 }
+function parseCalibrateArgs(args) {
+  const opts = {
+    labels: "",
+    rubric: "default",
+    rubricVersion: "",
+    threshold: 0.5
+  };
+  for (let i = 0; i < args.length; i++) {
+    const arg = args[i];
+    const next = args[i + 1];
+    if (arg === "--labels") {
+      if (next === void 0) throw new Error("--labels requires a path");
+      opts.labels = next;
+      i++;
+    } else if (arg === "--rubric") {
+      if (next === void 0) throw new Error("--rubric requires a name");
+      opts.rubric = next;
+      i++;
+    } else if (arg === "--rubric-version") {
+      if (next === void 0) throw new Error("--rubric-version requires a value");
+      opts.rubricVersion = next;
+      i++;
+    } else if (arg === "--threshold") {
+      if (next === void 0) throw new Error("--threshold requires a number");
+      const parsed = Number(next);
+      if (next.trim() === "" || Number.isNaN(parsed)) {
+        throw new Error(`--threshold value ${JSON.stringify(next)} is not a number`);
+      }
+      opts.threshold = parsed;
+      i++;
+    } else {
+      throw new Error(`unknown calibrate flag: ${arg}`);
+    }
+  }
+  if (!opts.labels) {
+    throw new Error("--labels is required");
+  }
+  return opts;
+}
+function cmdCalibrate(args) {
+  let opts;
+  try {
+    opts = parseCalibrateArgs(args);
+  } catch (err) {
+    process.stderr.write(`calibrate: ${err.message}
+`);
+    process.exit(2);
+    return;
+  }
+  const text = fs.readFileSync(opts.labels, "utf-8");
+  const ext = path.extname(opts.labels).toLowerCase();
+  let records;
+  try {
+    if (ext === ".jsonl" || ext === ".ndjson") {
+      records = loadLabelsJSONL(text);
+    } else if (ext === ".csv") {
+      records = loadLabelsCSV(text);
+    } else if (text.trimStart().startsWith("{")) {
+      records = loadLabelsJSONL(text);
+    } else {
+      records = loadLabelsCSV(text);
+    }
+  } catch (err) {
+    process.stderr.write(`calibrate: ${err.message}
+`);
+    process.exit(1);
+    return;
+  }
+  const pairs = records.filter((r) => r.judgeKnown).map((r) => ({ human: r.humanLabel, judge: r.judgeScore }));
+  const missing = records.filter((r) => !r.judgeKnown).length;
+  if (pairs.length === 0) {
+    process.stderr.write(
+      `calibrate: no rows had judge_score (${missing} rows missing); pre-score the labels file first
+`
+    );
+    process.exit(1);
+    return;
+  }
+  let result;
+  try {
+    result = computeAgreement(pairs, opts.threshold);
+  } catch (err) {
+    process.stderr.write(`calibrate: ${err.message}
+`);
+    process.exit(1);
+    return;
+  }
+  const output = {
+    rubric_name: opts.rubric,
+    rubric_version: opts.rubricVersion,
+    threshold: result.threshold,
+    label_count: result.n,
+    agreement: round3(result.agreement),
+    cohen_kappa: round3(result.cohenKappa),
+    roc_auc: round3(result.rocAuc),
+    missing_judge: missing
+  };
+  process.stdout.write(JSON.stringify(output, null, 2) + "\n");
+}
+function round3(x) {
+  return Math.round(x * 1e3) / 1e3;
+}
 var isDirectExecution = typeof import.meta.url === "string" && import.meta.url.startsWith("file:") && process.argv[1] !== void 0 && import.meta.url.endsWith(path.basename(process.argv[1]));
 if (isDirectExecution) {
   main();

package/dist/esm/cli.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/cli.ts","../../src/version.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * CLI entry point for Attest: npx attest or attest (after global install).\n *\n * Ported from Python sdks/python/src/attest/__main__.py\n */\n\nimport * as fs from \"node:fs\";\nimport * as path from \"node:path\";\nimport * as os from \"node:os\";\nimport { VERSION } from \"./version.js\";\n\nfunction cacheDir(): string {\n  const envOverride = process.env[\"ATTEST_CACHE_DIR\"];\n  if (envOverride) return envOverride;\n  return path.join(os.homedir(), \".attest\", \"cache\");\n}\n\nfunction cacheDbPath(): string {\n  return path.join(cacheDir(), \"attest.db\");\n}\n\nfunction cmdCacheStats(): void {\n  const dbPath = cacheDbPath();\n  const exists = fs.existsSync(dbPath);\n  let fileSize = 0;\n  if (exists) {\n    fileSize = fs.statSync(dbPath).size;\n  }\n  const stats = { exists, file_size: fileSize, path: dbPath };\n  process.stdout.write(JSON.stringify(stats) + \"\\n\");\n}\n\nfunction cmdCacheClear(): void {\n  const dbPath = cacheDbPath();\n  if (fs.existsSync(dbPath)) {\n    fs.unlinkSync(dbPath);\n    process.stdout.write(`Cleared cache: ${dbPath}\\n`);\n  } else {\n    process.stdout.write(`No cache to clear: ${dbPath}\\n`);\n  }\n}\n\nconst CONFTEST_TEMPLATE = `import { describe, it, expect } from \"vitest\";\nimport { attestExpect, TraceBuilder } from \"@attest-ai/core\";\n\ndescribe(\"my agent\", () => {\n  it(\"returns expected output\", async () => {\n    const trace = new TraceBuilder(\"my-agent\")\n      .setInput({ user_message: \"hello\" })\n      .setOutput({ message: \"world\" })\n      .build();\n\n    attestExpect(trace).outputContains(\"world\");\n  });\n});\n`;\n\nfunction cmdInit(targetDir: string): void {\n  const testsDir = path.join(targetDir, \"tests\");\n  fs.mkdirSync(testsDir, { recursive: true });\n  process.stdout.write(`Directory: ${testsDir}\\n`);\n\n  const sampleTestPath = path.join(testsDir, \"test_my_agent.test.ts\");\n  if (fs.existsSync(sampleTestPath)) {\n    process.stderr.write(`Skipped (exists): ${sampleTestPath}\\n`);\n  } else {\n    fs.writeFileSync(sampleTestPath, CONFTEST_TEMPLATE);\n    process.stdout.write(`Created: ${sampleTestPath}\\n`);\n  }\n}\n\nfunction cmdValidate(targetDir: string): void {\n  const testsDir = path.join(targetDir, \"tests\");\n\n  let testFiles: string[] = [];\n  if (fs.existsSync(testsDir)) {\n    testFiles = fs\n      .readdirSync(testsDir)\n      .filter((f) => f.endsWith(\".test.ts\") || f.endsWith(\".test.js\"));\n  }\n\n  if (testFiles.length === 0) {\n    process.stderr.write(\n      `Warning: no test files (*.test.ts / *.test.js) found in ${testsDir}.\\n`,\n    );\n  } else {\n    process.stdout.write(`Found ${testFiles.length} test file(s) in ${testsDir}\\n`);\n  }\n\n  // Check for stale golden traces\n  const STALE_DAYS = 30;\n  const now = Date.now();\n  const goldenFiles = findGoldenFiles(targetDir);\n  for (const golden of goldenFiles) {\n    const mtime = fs.statSync(golden).mtimeMs;\n    const ageDays = Math.floor((now - mtime) / (1000 * 60 * 60 * 24));\n    if (ageDays > STALE_DAYS) {\n      process.stderr.write(\n        `Warning: golden trace ${golden} is ${ageDays} days old (>${STALE_DAYS} days). Consider regenerating.\\n`,\n      );\n    }\n  }\n}\n\nfunction findGoldenFiles(dir: string): string[] {\n  const results: string[] = [];\n  if (!fs.existsSync(dir)) return results;\n\n  const entries = fs.readdirSync(dir, { withFileTypes: true });\n  for (const entry of entries) {\n    const fullPath = path.join(dir, entry.name);\n    if (entry.isDirectory() && entry.name !== \"node_modules\") {\n      results.push(...findGoldenFiles(fullPath));\n    } else if (entry.isFile() && entry.name.endsWith(\".golden\")) {\n      results.push(fullPath);\n    }\n  }\n  return results;\n}\n\nexport function main(argv: string[] = process.argv.slice(2)): void {\n  const args = argv;\n\n  if (args[0] === \"--version\") {\n    process.stdout.write(`attest ${VERSION}\\n`);\n    process.exit(0);\n  }\n\n  if (args[0] === \"cache\" && args.length >= 2) {\n    if (args[1] === \"stats\") {\n      cmdCacheStats();\n      process.exit(0);\n    } else if (args[1] === \"clear\") {\n      cmdCacheClear();\n      process.exit(0);\n    } else {\n      process.stderr.write(`Unknown cache subcommand: ${args[1]}\\n`);\n      process.stderr.write(\"Available: cache stats, cache clear\\n\");\n      process.exit(1);\n    }\n  }\n\n  if (args[0] === \"init\") {\n    cmdInit(process.cwd());\n    process.exit(0);\n  }\n\n  if (args[0] === \"validate\") {\n    cmdValidate(process.cwd());\n    process.exit(0);\n  }\n\n  process.stderr.write(\n    \"Usage: attest <command>\\n\\n\" +\n    \"Commands:\\n\" +\n    \"  --version       Print version\\n\" +\n    \"  init            Scaffold a test project\\n\" +\n    \"  validate        Validate test suite\\n\" +\n    \"  cache stats     Show cache statistics\\n\" +\n    \"  cache clear     Clear cache database\\n\",\n  );\n  process.exit(args.length === 0 ? 0 : 1);\n}\n\n// Run when executed directly\nconst isDirectExecution =\n  typeof import.meta.url === \"string\" &&\n  import.meta.url.startsWith(\"file:\") &&\n  process.argv[1] !== undefined &&\n  import.meta.url.endsWith(path.basename(process.argv[1]));\n\nif (isDirectExecution) {\n  main();\n}\n","export const VERSION = \"0.5.0\";\nexport const ENGINE_VERSION = \"0.5.0\";\n"],"mappings":";;;AAOA,YAAY,QAAQ;AACpB,YAAY,UAAU;AACtB,YAAY,QAAQ;;;ACTb,IAAM,UAAU;;;ADYvB,SAAS,WAAmB;AAC1B,QAAM,cAAc,QAAQ,IAAI,kBAAkB;AAClD,MAAI,YAAa,QAAO;AACxB,SAAY,UAAQ,WAAQ,GAAG,WAAW,OAAO;AACnD;AAEA,SAAS,cAAsB;AAC7B,SAAY,UAAK,SAAS,GAAG,WAAW;AAC1C;AAEA,SAAS,gBAAsB;AAC7B,QAAM,SAAS,YAAY;AAC3B,QAAM,SAAY,cAAW,MAAM;AACnC,MAAI,WAAW;AACf,MAAI,QAAQ;AACV,eAAc,YAAS,MAAM,EAAE;AAAA,EACjC;AACA,QAAM,QAAQ,EAAE,QAAQ,WAAW,UAAU,MAAM,OAAO;AAC1D,UAAQ,OAAO,MAAM,KAAK,UAAU,KAAK,IAAI,IAAI;AACnD;AAEA,SAAS,gBAAsB;AAC7B,QAAM,SAAS,YAAY;AAC3B,MAAO,cAAW,MAAM,GAAG;AACzB,IAAG,cAAW,MAAM;AACpB,YAAQ,OAAO,MAAM,kBAAkB,MAAM;AAAA,CAAI;AAAA,EACnD,OAAO;AACL,YAAQ,OAAO,MAAM,sBAAsB,MAAM;AAAA,CAAI;AAAA,EACvD;AACF;AAEA,IAAM,oBAAoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAe1B,SAAS,QAAQ,WAAyB;AACxC,QAAM,WAAgB,UAAK,WAAW,OAAO;AAC7C,EAAG,aAAU,UAAU,EAAE,WAAW,KAAK,CAAC;AAC1C,UAAQ,OAAO,MAAM,cAAc,QAAQ;AAAA,CAAI;AAE/C,QAAM,iBAAsB,UAAK,UAAU,uBAAuB;AAClE,MAAO,cAAW,cAAc,GAAG;AACjC,YAAQ,OAAO,MAAM,qBAAqB,cAAc;AAAA,CAAI;AAAA,EAC9D,OAAO;AACL,IAAG,iBAAc,gBAAgB,iBAAiB;AAClD,YAAQ,OAAO,MAAM,YAAY,cAAc;AAAA,CAAI;AAAA,EACrD;AACF;AAEA,SAAS,YAAY,WAAyB;AAC5C,QAAM,WAAgB,UAAK,WAAW,OAAO;AAE7C,MAAI,YAAsB,CAAC;AAC3B,MAAO,cAAW,QAAQ,GAAG;AAC3B,gBACG,eAAY,QAAQ,EACpB,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,KAAK,EAAE,SAAS,UAAU,CAAC;AAAA,EACnE;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,OAAO;AAAA,MACb,2DAA2D,QAAQ;AAAA;AAAA,IACrE;AAAA,EACF,OAAO;AACL,YAAQ,OAAO,MAAM,SAAS,UAAU,MAAM,oBAAoB,QAAQ;AAAA,CAAI;AAAA,EAChF;AAGA,QAAM,aAAa;AACnB,QAAM,MAAM,KAAK,IAAI;AACrB,QAAM,cAAc,gBAAgB,SAAS;AAC7C,aAAW,UAAU,aAAa;AAChC,UAAM,QAAW,YAAS,MAAM,EAAE;AAClC,UAAM,UAAU,KAAK,OAAO,MAAM,UAAU,MAAO,KAAK,KAAK,GAAG;AAChE,QAAI,UAAU,YAAY;AACxB,cAAQ,OAAO;AAAA,QACb,yBAAyB,MAAM,OAAO,OAAO,eAAe,UAAU;AAAA;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,KAAuB;AAC9C,QAAM,UAAoB,CAAC;AAC3B,MAAI,CAAI,cAAW,GAAG,EAAG,QAAO;AAEhC,QAAM,UAAa,eAAY,KAAK,EAAE,eAAe,KAAK,CAAC;AAC3D,aAAW,SAAS,SAAS;AAC3B,UAAM,WAAgB,UAAK,KAAK,MAAM,IAAI;AAC1C,QAAI,MAAM,YAAY,KAAK,MAAM,SAAS,gBAAgB;AACxD,cAAQ,KAAK,GAAG,gBAAgB,QAAQ,CAAC;AAAA,IAC3C,WAAW,MAAM,OAAO,KAAK,MAAM,KAAK,SAAS,SAAS,GAAG;AAC3D,cAAQ,KAAK,QAAQ;AAAA,IACvB;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,KAAK,OAAiB,QAAQ,KAAK,MAAM,CAAC,GAAS;AACjE,QAAM,OAAO;AAEb,MAAI,KAAK,CAAC,MAAM,aAAa;AAC3B,YAAQ,OAAO,MAAM,UAAU,OAAO;AAAA,CAAI;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW,KAAK,UAAU,GAAG;AAC3C,QAAI,KAAK,CAAC,MAAM,SAAS;AACvB,oBAAc;AACd,cAAQ,KAAK,CAAC;AAAA,IAChB,WAAW,KAAK,CAAC,MAAM,SAAS;AAC9B,oBAAc;AACd,cAAQ,KAAK,CAAC;AAAA,IAChB,OAAO;AACL,cAAQ,OAAO,MAAM,6BAA6B,KAAK,CAAC,CAAC;AAAA,CAAI;AAC7D,cAAQ,OAAO,MAAM,uCAAuC;AAC5D,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AAEA,MAAI,KAAK,CAAC,MAAM,QAAQ;AACtB,YAAQ,QAAQ,IAAI,CAAC;AACrB,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,YAAY;AAC1B,gBAAY,QAAQ,IAAI,CAAC;AACzB,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,UAAQ,OAAO;AAAA,IACb;AAAA,EAOF;AACA,UAAQ,KAAK,KAAK,WAAW,IAAI,IAAI,CAAC;AACxC;AAGA,IAAM,oBACJ,OAAO,YAAY,QAAQ,YAC3B,YAAY,IAAI,WAAW,OAAO,KAClC,QAAQ,KAAK,CAAC,MAAM,UACpB,YAAY,IAAI,SAAc,cAAS,QAAQ,KAAK,CAAC,CAAC,CAAC;AAEzD,IAAI,mBAAmB;AACrB,OAAK;AACP;","names":[]}
+{"version":3,"sources":["../../src/cli.ts","../../src/version.ts","../../src/calibration.ts"],"sourcesContent":["#!/usr/bin/env node\n/**\n * CLI entry point for Attest: npx attest or attest (after global install).\n *\n * Ported from Python sdks/python/src/attest/__main__.py\n */\n\nimport * as fs from \"node:fs\";\nimport * as path from \"node:path\";\nimport * as os from \"node:os\";\nimport { VERSION } from \"./version.js\";\nimport {\n  computeAgreement,\n  loadLabelsCSV,\n  loadLabelsJSONL,\n  type LabelPair,\n} from \"./calibration.js\";\n\nfunction cacheDir(): string {\n  const envOverride = process.env[\"ATTEST_CACHE_DIR\"];\n  if (envOverride) return envOverride;\n  return path.join(os.homedir(), \".attest\", \"cache\");\n}\n\nfunction cacheDbPath(): string {\n  return path.join(cacheDir(), \"attest.db\");\n}\n\nfunction cmdCacheStats(): void {\n  const dbPath = cacheDbPath();\n  const exists = fs.existsSync(dbPath);\n  let fileSize = 0;\n  if (exists) {\n    fileSize = fs.statSync(dbPath).size;\n  }\n  const stats = { exists, file_size: fileSize, path: dbPath };\n  process.stdout.write(JSON.stringify(stats) + \"\\n\");\n}\n\nfunction cmdCacheClear(): void {\n  const dbPath = cacheDbPath();\n  if (fs.existsSync(dbPath)) {\n    fs.unlinkSync(dbPath);\n    process.stdout.write(`Cleared cache: ${dbPath}\\n`);\n  } else {\n    process.stdout.write(`No cache to clear: ${dbPath}\\n`);\n  }\n}\n\nconst CONFTEST_TEMPLATE = `import { describe, it, expect } from \"vitest\";\nimport { attestExpect, TraceBuilder } from \"@attest-ai/core\";\n\ndescribe(\"my agent\", () => {\n  it(\"returns expected output\", async () => {\n    const trace = new TraceBuilder(\"my-agent\")\n      .setInput({ user_message: \"hello\" })\n      .setOutput({ message: \"world\" })\n      .build();\n\n    attestExpect(trace).outputContains(\"world\");\n  });\n});\n`;\n\nfunction cmdInit(targetDir: string): void {\n  const testsDir = path.join(targetDir, \"tests\");\n  fs.mkdirSync(testsDir, { recursive: true });\n  process.stdout.write(`Directory: ${testsDir}\\n`);\n\n  const sampleTestPath = path.join(testsDir, \"test_my_agent.test.ts\");\n  if (fs.existsSync(sampleTestPath)) {\n    process.stderr.write(`Skipped (exists): ${sampleTestPath}\\n`);\n  } else {\n    fs.writeFileSync(sampleTestPath, CONFTEST_TEMPLATE);\n    process.stdout.write(`Created: ${sampleTestPath}\\n`);\n  }\n}\n\nfunction cmdValidate(targetDir: string): void {\n  const testsDir = path.join(targetDir, \"tests\");\n\n  let testFiles: string[] = [];\n  if (fs.existsSync(testsDir)) {\n    testFiles = fs\n      .readdirSync(testsDir)\n      .filter((f) => f.endsWith(\".test.ts\") || f.endsWith(\".test.js\"));\n  }\n\n  if (testFiles.length === 0) {\n    process.stderr.write(\n      `Warning: no test files (*.test.ts / *.test.js) found in ${testsDir}.\\n`,\n    );\n  } else {\n    process.stdout.write(`Found ${testFiles.length} test file(s) in ${testsDir}\\n`);\n  }\n\n  // Check for stale golden traces\n  const STALE_DAYS = 30;\n  const now = Date.now();\n  const goldenFiles = findGoldenFiles(targetDir);\n  for (const golden of goldenFiles) {\n    const mtime = fs.statSync(golden).mtimeMs;\n    const ageDays = Math.floor((now - mtime) / (1000 * 60 * 60 * 24));\n    if (ageDays > STALE_DAYS) {\n      process.stderr.write(\n        `Warning: golden trace ${golden} is ${ageDays} days old (>${STALE_DAYS} days). Consider regenerating.\\n`,\n      );\n    }\n  }\n}\n\nfunction findGoldenFiles(dir: string): string[] {\n  const results: string[] = [];\n  if (!fs.existsSync(dir)) return results;\n\n  const entries = fs.readdirSync(dir, { withFileTypes: true });\n  for (const entry of entries) {\n    const fullPath = path.join(dir, entry.name);\n    if (entry.isDirectory() && entry.name !== \"node_modules\") {\n      results.push(...findGoldenFiles(fullPath));\n    } else if (entry.isFile() && entry.name.endsWith(\".golden\")) {\n      results.push(fullPath);\n    }\n  }\n  return results;\n}\n\nexport function main(argv: string[] = process.argv.slice(2)): void {\n  const args = argv;\n\n  if (args[0] === \"--version\") {\n    process.stdout.write(`attest ${VERSION}\\n`);\n    process.exit(0);\n  }\n\n  if (args[0] === \"cache\" && args.length >= 2) {\n    if (args[1] === \"stats\") {\n      cmdCacheStats();\n      process.exit(0);\n    } else if (args[1] === \"clear\") {\n      cmdCacheClear();\n      process.exit(0);\n    } else {\n      process.stderr.write(`Unknown cache subcommand: ${args[1]}\\n`);\n      process.stderr.write(\"Available: cache stats, cache clear\\n\");\n      process.exit(1);\n    }\n  }\n\n  if (args[0] === \"init\") {\n    cmdInit(process.cwd());\n    process.exit(0);\n  }\n\n  if (args[0] === \"validate\") {\n    cmdValidate(process.cwd());\n    process.exit(0);\n  }\n\n  if (args[0] === \"calibrate\") {\n    cmdCalibrate(args.slice(1));\n    process.exit(0);\n  }\n\n  process.stderr.write(\n    \"Usage: attest <command>\\n\\n\" +\n    \"Commands:\\n\" +\n    \"  --version       Print version\\n\" +\n    \"  init            Scaffold a test project\\n\" +\n    \"  validate        Validate test suite\\n\" +\n    \"  cache stats     Show cache statistics\\n\" +\n    \"  cache clear     Clear cache database\\n\" +\n    \"  calibrate       Compute judge agreement metrics over labeled data\\n\",\n  );\n  process.exit(args.length === 0 ? 0 : 1);\n}\n\ninterface CalibrateOptions {\n  labels: string;\n  rubric: string;\n  rubricVersion: string;\n  threshold: number;\n}\n\nfunction parseCalibrateArgs(args: string[]): CalibrateOptions {\n  const opts: CalibrateOptions = {\n    labels: \"\",\n    rubric: \"default\",\n    rubricVersion: \"\",\n    threshold: 0.5,\n  };\n  for (let i = 0; i < args.length; i++) {\n    const arg = args[i]!;\n    const next = args[i + 1];\n    if (arg === \"--labels\") {\n      if (next === undefined) throw new Error(\"--labels requires a path\");\n      opts.labels = next;\n      i++;\n    } else if (arg === \"--rubric\") {\n      if (next === undefined) throw new Error(\"--rubric requires a name\");\n      opts.rubric = next;\n      i++;\n    } else if (arg === \"--rubric-version\") {\n      if (next === undefined) throw new Error(\"--rubric-version requires a value\");\n      opts.rubricVersion = next;\n      i++;\n    } else if (arg === \"--threshold\") {\n      if (next === undefined) throw new Error(\"--threshold requires a number\");\n      const parsed = Number(next);\n      if (next.trim() === \"\" || Number.isNaN(parsed)) {\n        throw new Error(`--threshold value ${JSON.stringify(next)} is not a number`);\n      }\n      opts.threshold = parsed;\n      i++;\n    } else {\n      throw new Error(`unknown calibrate flag: ${arg}`);\n    }\n  }\n  if (!opts.labels) {\n    throw new Error(\"--labels is required\");\n  }\n  return opts;\n}\n\nfunction cmdCalibrate(args: string[]): void {\n  let opts: CalibrateOptions;\n  try {\n    opts = parseCalibrateArgs(args);\n  } catch (err) {\n    process.stderr.write(`calibrate: ${(err as Error).message}\\n`);\n    process.exit(2);\n    return;\n  }\n\n  const text = fs.readFileSync(opts.labels, \"utf-8\");\n  const ext = path.extname(opts.labels).toLowerCase();\n  let records;\n  try {\n    if (ext === \".jsonl\" || ext === \".ndjson\") {\n      records = loadLabelsJSONL(text);\n    } else if (ext === \".csv\") {\n      records = loadLabelsCSV(text);\n    } else if (text.trimStart().startsWith(\"{\")) {\n      records = loadLabelsJSONL(text);\n    } else {\n      records = loadLabelsCSV(text);\n    }\n  } catch (err) {\n    process.stderr.write(`calibrate: ${(err as Error).message}\\n`);\n    process.exit(1);\n    return;\n  }\n\n  const pairs: LabelPair[] = records\n    .filter((r) => r.judgeKnown)\n    .map((r) => ({ human: r.humanLabel, judge: r.judgeScore }));\n  const missing = records.filter((r) => !r.judgeKnown).length;\n  if (pairs.length === 0) {\n    process.stderr.write(\n      `calibrate: no rows had judge_score (${missing} rows missing); pre-score the labels file first\\n`,\n    );\n    process.exit(1);\n    return;\n  }\n\n  let result;\n  try {\n    result = computeAgreement(pairs, opts.threshold);\n  } catch (err) {\n    process.stderr.write(`calibrate: ${(err as Error).message}\\n`);\n    process.exit(1);\n    return;\n  }\n\n  const output = {\n    rubric_name: opts.rubric,\n    rubric_version: opts.rubricVersion,\n    threshold: result.threshold,\n    label_count: result.n,\n    agreement: round3(result.agreement),\n    cohen_kappa: round3(result.cohenKappa),\n    roc_auc: round3(result.rocAuc),\n    missing_judge: missing,\n  };\n  process.stdout.write(JSON.stringify(output, null, 2) + \"\\n\");\n}\n\nfunction round3(x: number): number {\n  return Math.round(x * 1000) / 1000;\n}\n\n// Run when executed directly\nconst isDirectExecution =\n  typeof import.meta.url === \"string\" &&\n  import.meta.url.startsWith(\"file:\") &&\n  process.argv[1] !== undefined &&\n  import.meta.url.endsWith(path.basename(process.argv[1]));\n\nif (isDirectExecution) {\n  main();\n}\n","export const VERSION = \"0.9.0\";\nexport const ENGINE_VERSION = \"0.9.0\";\n","/**\n * Calibration helpers — TS parity with engine/internal/assertion/judge/calibration.go\n * and sdks/python/src/attest/calibration.py.\n *\n * Computes Cohen's κ, agreement %, and ROC-AUC over (human, judge) score\n * pairs and parses CSV/JSONL labels files. Metrics match the engine\n * byte-for-byte on the same input.\n */\n\nimport { createHash } from \"node:crypto\";\n\nexport interface LabelPair {\n  readonly human: number;\n  readonly judge: number;\n}\n\nexport interface AgreementResult {\n  readonly threshold: number;\n  readonly n: number;\n  readonly agreement: number;\n  readonly cohenKappa: number;\n  readonly rocAuc: number;\n}\n\nexport interface LabeledRecord {\n  input: string;\n  humanLabel: number;\n  judgeScore: number;\n  judgeKnown: boolean;\n}\n\n/**\n * Compute Cohen's κ, agreement %, and ROC-AUC over labeled pairs.\n *\n * Throws when pairs is empty or threshold is outside (0, 1). Cohen's κ\n * degenerates to 0 when all labels land in a single class for both\n * raters; ROC-AUC is reported as 0 when only one human-class is\n * present.\n */\nexport function computeAgreement(\n  pairs: readonly LabelPair[],\n  threshold: number,\n): AgreementResult {\n  if (pairs.length === 0) {\n    throw new Error(\"no labeled pairs available\");\n  }\n  if (threshold <= 0 || threshold >= 1) {\n    throw new Error(\"threshold must be in (0, 1)\");\n  }\n\n  const n = pairs.length;\n  let humanPos = 0;\n  let judgePos = 0;\n  let bothPos = 0;\n  let bothNeg = 0;\n  for (const p of pairs) {\n    const hPos = p.human >= threshold;\n    const jPos = p.judge >= threshold;\n    if (hPos) humanPos++;\n    if (jPos) judgePos++;\n    if (hPos && jPos) bothPos++;\n    if (!hPos && !jPos) bothNeg++;\n  }\n  const humanNeg = n - humanPos;\n  const judgeNeg = n - judgePos;\n\n  const agreement = (bothPos + bothNeg) / n;\n  const expected =\n    (humanPos * judgePos + humanNeg * judgeNeg) / (n * n);\n  const cohenKappa =\n    expected < 1 ? (agreement - expected) / (1 - expected) : 0;\n\n  const rocAuc = rocAUC(pairs, threshold);\n\n  return { threshold, n, agreement, cohenKappa, rocAuc };\n}\n\nfunction rocAUC(pairs: readonly LabelPair[], threshold: number): number {\n  const rows: { score: number; pos: boolean }[] = pairs.map((p) => ({\n    score: p.judge,\n    pos: p.human >= threshold,\n  }));\n  const pos = rows.filter((r) => r.pos).length;\n  const neg = rows.length - pos;\n  if (pos === 0 || neg === 0) return 0;\n\n  rows.sort((a, b) => a.score - b.score);\n\n  let rankSumPos = 0;\n  let i = 0;\n  while (i < rows.length) {\n    let j = i;\n    while (j < rows.length && rows[j]!.score === rows[i]!.score) {\n      j++;\n    }\n    const avgRank = (i + j + 1) / 2;\n    for (let k = i; k < j; k++) {\n      if (rows[k]!.pos) rankSumPos += avgRank;\n    }\n    i = j;\n  }\n\n  const auc = (rankSumPos - (pos * (pos + 1)) / 2) / (pos * neg);\n  if (Number.isNaN(auc) || !Number.isFinite(auc)) return 0;\n  return auc;\n}\n\n/**\n * Parse a 2- or 3-column CSV (input, human_label[, judge_score]).\n * Header detected when row[0][1] does not parse as a float. Lines\n * beginning with `#` are skipped.\n */\nexport function loadLabelsCSV(text: string): LabeledRecord[] {\n  const rows = parseCSV(text);\n  if (rows.length === 0) {\n    throw new Error(\"empty CSV\");\n  }\n  let start = 0;\n  if (rows[0]!.length >= 2) {\n    const firstHuman = parseFloatStrict(rows[0]![1]!);\n    if (firstHuman === null) start = 1;\n  }\n  const out: LabeledRecord[] = [];\n  for (let idx = start; idx < rows.length; idx++) {\n    const row = rows[idx]!;\n    const lineNum = idx + 1;\n    if (row.length === 0) continue;\n    if (row[0]!.trimStart().startsWith(\"#\")) continue;\n    if (row.length < 2) {\n      throw new Error(\n        `CSV line ${lineNum}: want at least 2 columns (input, human_label)`,\n      );\n    }\n    const human = parseFloatStrict(row[1]!);\n    if (human === null) {\n      throw new Error(`CSV line ${lineNum}: human_label not a float: ${row[1]}`);\n    }\n    const rec: LabeledRecord = {\n      input: row[0]!,\n      humanLabel: human,\n      judgeScore: 0,\n      judgeKnown: false,\n    };\n    if (row.length >= 3 && row[2]!.trim() !== \"\") {\n      const judge = parseFloatStrict(row[2]!);\n      if (judge === null) {\n        throw new Error(\n          `CSV line ${lineNum}: judge_score not a float: ${row[2]}`,\n        );\n      }\n      rec.judgeScore = judge;\n      rec.judgeKnown = true;\n    }\n    out.push(rec);\n  }\n  if (out.length === 0) {\n    throw new Error(\"CSV contained no labeled rows\");\n  }\n  return out;\n}\n\n/**\n * Parse a newline-delimited JSON file. Each line is an object with the\n * shape {\"input\": \"...\", \"human_label\": 0.9, \"judge_score\": 0.8}.\n * `judge_score` is optional.\n */\nexport function loadLabelsJSONL(text: string): LabeledRecord[] {\n  const out: LabeledRecord[] = [];\n  const lines = text.split(/\\r?\\n/);\n  for (let i = 0; i < lines.length; i++) {\n    const stripped = lines[i]!.trim();\n    if (!stripped) continue;\n    let obj: Record<string, unknown>;\n    try {\n      obj = JSON.parse(stripped) as Record<string, unknown>;\n    } catch (err) {\n      throw new Error(`JSONL line ${i + 1}: ${(err as Error).message}`);\n    }\n    if (Object.keys(obj).length === 0) continue;\n    if (obj[\"human_label\"] === undefined) {\n      throw new Error(`JSONL line ${i + 1}: missing human_label`);\n    }\n    const human = parseJSONNumber(obj[\"human_label\"]);\n    if (human === null) {\n      throw new Error(`JSONL line ${i + 1}: human_label is not a number`);\n    }\n    const rec: LabeledRecord = {\n      input: typeof obj[\"input\"] === \"string\" ? obj[\"input\"] : \"\",\n      humanLabel: human,\n      judgeScore: 0,\n      judgeKnown: false,\n    };\n    if (obj[\"judge_score\"] !== undefined && obj[\"judge_score\"] !== null) {\n      const judge = parseJSONNumber(obj[\"judge_score\"]);\n      if (judge === null) {\n        throw new Error(`JSONL line ${i + 1}: judge_score is not a number`);\n      }\n      rec.judgeScore = judge;\n      rec.judgeKnown = true;\n    }\n    out.push(rec);\n  }\n  if (out.length === 0) {\n    throw new Error(\"JSONL contained no labeled rows\");\n  }\n  return out;\n}\n\n/**\n * Return the 16-character SHA-256 prefix used by the engine for prompt\n * hashes. Mirrors promptHash in engine/internal/assertion/judge_eval.go\n * so SDK-recorded calibration rows align with JudgeMetadata.prompt_hash.\n */\nexport function promptHash(text: string): string {\n  return createHash(\"sha256\").update(text).digest(\"hex\").slice(0, 16);\n}\n\n/**\n * Parse a string as a number, rejecting empty input and values that\n * JavaScript's `Number()` would silently coerce to 0. Returns null on\n * rejection so callers can throw a useful error. Matches Go's\n * strconv.ParseFloat and Python's float() — both error on the empty\n * string, while bare `Number(\"\")` returns 0.\n *\n * Permits \"Infinity\" so the loader stays consistent with strconv.ParseFloat;\n * downstream consumers (CalibrationStore.Record, computeAgreement) handle\n * the [0, 1] range check separately.\n */\nfunction parseFloatStrict(raw: string): number | null {\n  const trimmed = raw.trim();\n  if (trimmed === \"\") return null;\n  const n = Number(trimmed);\n  if (Number.isNaN(n)) return null;\n  return n;\n}\n\n/**\n * Coerce a JSON value to a finite number. Accepts only `number` (rejects\n * stringly-typed numbers and booleans) so calibration files cannot\n * accidentally smuggle non-numeric labels past type-loose JSON parsing.\n */\nfunction parseJSONNumber(value: unknown): number | null {\n  if (typeof value !== \"number\") return null;\n  if (Number.isNaN(value)) return null;\n  return value;\n}\n\n/**\n * Minimal RFC-4180-ish CSV parser. Handles quoted fields with embedded\n * commas and CRLF line endings. Sufficient for calibration label files;\n * callers needing full CSV support should bring their own parser.\n */\nfunction parseCSV(text: string): string[][] {\n  const rows: string[][] = [];\n  let current: string[] = [];\n  let field = \"\";\n  let inQuotes = false;\n  let i = 0;\n  while (i < text.length) {\n    const ch = text[i]!;\n    if (inQuotes) {\n      if (ch === '\"') {\n        if (text[i + 1] === '\"') {\n          field += '\"';\n          i += 2;\n          continue;\n        }\n        inQuotes = false;\n        i++;\n        continue;\n      }\n      field += ch;\n      i++;\n      continue;\n    }\n    if (ch === '\"') {\n      inQuotes = true;\n      i++;\n      continue;\n    }\n    if (ch === \",\") {\n      current.push(field);\n      field = \"\";\n      i++;\n      continue;\n    }\n    if (ch === \"\\n\") {\n      current.push(field);\n      rows.push(current);\n      current = [];\n      field = \"\";\n      i++;\n      continue;\n    }\n    if (ch === \"\\r\") {\n      i++;\n      continue;\n    }\n    field += ch;\n    i++;\n  }\n  if (field.length > 0 || current.length > 0) {\n    current.push(field);\n    rows.push(current);\n  }\n  return rows;\n}\n"],"mappings":";;;AAOA,YAAY,QAAQ;AACpB,YAAY,UAAU;AACtB,YAAY,QAAQ;;;ACTb,IAAM,UAAU;;;ACSvB,SAAS,kBAAkB;AA8BpB,SAAS,iBACd,OACA,WACiB;AACjB,MAAI,MAAM,WAAW,GAAG;AACtB,UAAM,IAAI,MAAM,4BAA4B;AAAA,EAC9C;AACA,MAAI,aAAa,KAAK,aAAa,GAAG;AACpC,UAAM,IAAI,MAAM,6BAA6B;AAAA,EAC/C;AAEA,QAAM,IAAI,MAAM;AAChB,MAAI,WAAW;AACf,MAAI,WAAW;AACf,MAAI,UAAU;AACd,MAAI,UAAU;AACd,aAAW,KAAK,OAAO;AACrB,UAAM,OAAO,EAAE,SAAS;AACxB,UAAM,OAAO,EAAE,SAAS;AACxB,QAAI,KAAM;AACV,QAAI,KAAM;AACV,QAAI,QAAQ,KAAM;AAClB,QAAI,CAAC,QAAQ,CAAC,KAAM;AAAA,EACtB;AACA,QAAM,WAAW,IAAI;AACrB,QAAM,WAAW,IAAI;AAErB,QAAM,aAAa,UAAU,WAAW;AACxC,QAAM,YACH,WAAW,WAAW,WAAW,aAAa,IAAI;AACrD,QAAM,aACJ,WAAW,KAAK,YAAY,aAAa,IAAI,YAAY;AAE3D,QAAM,SAAS,OAAO,OAAO,SAAS;AAEtC,SAAO,EAAE,WAAW,GAAG,WAAW,YAAY,OAAO;AACvD;AAEA,SAAS,OAAO,OAA6B,WAA2B;AACtE,QAAM,OAA0C,MAAM,IAAI,CAAC,OAAO;AAAA,IAChE,OAAO,EAAE;AAAA,IACT,KAAK,EAAE,SAAS;AAAA,EAClB,EAAE;AACF,QAAM,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,GAAG,EAAE;AACtC,QAAM,MAAM,KAAK,SAAS;AAC1B,MAAI,QAAQ,KAAK,QAAQ,EAAG,QAAO;AAEnC,OAAK,KAAK,CAAC,GAAG,MAAM,EAAE,QAAQ,EAAE,KAAK;AAErC,MAAI,aAAa;AACjB,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,QAAQ;AACtB,QAAI,IAAI;AACR,WAAO,IAAI,KAAK,UAAU,KAAK,CAAC,EAAG,UAAU,KAAK,CAAC,EAAG,OAAO;AAC3D;AAAA,IACF;AACA,UAAM,WAAW,IAAI,IAAI,KAAK;AAC9B,aAAS,IAAI,GAAG,IAAI,GAAG,KAAK;AAC1B,UAAI,KAAK,CAAC,EAAG,IAAK,eAAc;AAAA,IAClC;AACA,QAAI;AAAA,EACN;AAEA,QAAM,OAAO,aAAc,OAAO,MAAM,KAAM,MAAM,MAAM;AAC1D,MAAI,OAAO,MAAM,GAAG,KAAK,CAAC,OAAO,SAAS,GAAG,EAAG,QAAO;AACvD,SAAO;AACT;AAOO,SAAS,cAAc,MAA+B;AAC3D,QAAM,OAAO,SAAS,IAAI;AAC1B,MAAI,KAAK,WAAW,GAAG;AACrB,UAAM,IAAI,MAAM,WAAW;AAAA,EAC7B;AACA,MAAI,QAAQ;AACZ,MAAI,KAAK,CAAC,EAAG,UAAU,GAAG;AACxB,UAAM,aAAa,iBAAiB,KAAK,CAAC,EAAG,CAAC,CAAE;AAChD,QAAI,eAAe,KAAM,SAAQ;AAAA,EACnC;AACA,QAAM,MAAuB,CAAC;AAC9B,WAAS,MAAM,OAAO,MAAM,KAAK,QAAQ,OAAO;AAC9C,UAAM,MAAM,KAAK,GAAG;AACpB,UAAM,UAAU,MAAM;AACtB,QAAI,IAAI,WAAW,EAAG;AACtB,QAAI,IAAI,CAAC,EAAG,UAAU,EAAE,WAAW,GAAG,EAAG;AACzC,QAAI,IAAI,SAAS,GAAG;AAClB,YAAM,IAAI;AAAA,QACR,YAAY,OAAO;AAAA,MACrB;AAAA,IACF;AACA,UAAM,QAAQ,iBAAiB,IAAI,CAAC,CAAE;AACtC,QAAI,UAAU,MAAM;AAClB,YAAM,IAAI,MAAM,YAAY,OAAO,8BAA8B,IAAI,CAAC,CAAC,EAAE;AAAA,IAC3E;AACA,UAAM,MAAqB;AAAA,MACzB,OAAO,IAAI,CAAC;AAAA,MACZ,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,YAAY;AAAA,IACd;AACA,QAAI,IAAI,UAAU,KAAK,IAAI,CAAC,EAAG,KAAK,MAAM,IAAI;AAC5C,YAAM,QAAQ,iBAAiB,IAAI,CAAC,CAAE;AACtC,UAAI,UAAU,MAAM;AAClB,cAAM,IAAI;AAAA,UACR,YAAY,OAAO,8BAA8B,IAAI,CAAC,CAAC;AAAA,QACzD;AAAA,MACF;AACA,UAAI,aAAa;AACjB,UAAI,aAAa;AAAA,IACnB;AACA,QAAI,KAAK,GAAG;AAAA,EACd;AACA,MAAI,IAAI,WAAW,GAAG;AACpB,UAAM,IAAI,MAAM,+BAA+B;AAAA,EACjD;AACA,SAAO;AACT;AAOO,SAAS,gBAAgB,MAA+B;AAC7D,QAAM,MAAuB,CAAC;AAC9B,QAAM,QAAQ,KAAK,MAAM,OAAO;AAChC,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,UAAM,WAAW,MAAM,CAAC,EAAG,KAAK;AAChC,QAAI,CAAC,SAAU;AACf,QAAI;AACJ,QAAI;AACF,YAAM,KAAK,MAAM,QAAQ;AAAA,IAC3B,SAAS,KAAK;AACZ,YAAM,IAAI,MAAM,cAAc,IAAI,CAAC,KAAM,IAAc,OAAO,EAAE;AAAA,IAClE;AACA,QAAI,OAAO,KAAK,GAAG,EAAE,WAAW,EAAG;AACnC,QAAI,IAAI,aAAa,MAAM,QAAW;AACpC,YAAM,IAAI,MAAM,cAAc,IAAI,CAAC,uBAAuB;AAAA,IAC5D;AACA,UAAM,QAAQ,gBAAgB,IAAI,aAAa,CAAC;AAChD,QAAI,UAAU,MAAM;AAClB,YAAM,IAAI,MAAM,cAAc,IAAI,CAAC,+BAA+B;AAAA,IACpE;AACA,UAAM,MAAqB;AAAA,MACzB,OAAO,OAAO,IAAI,OAAO,MAAM,WAAW,IAAI,OAAO,IAAI;AAAA,MACzD,YAAY;AAAA,MACZ,YAAY;AAAA,MACZ,YAAY;AAAA,IACd;AACA,QAAI,IAAI,aAAa,MAAM,UAAa,IAAI,aAAa,MAAM,MAAM;AACnE,YAAM,QAAQ,gBAAgB,IAAI,aAAa,CAAC;AAChD,UAAI,UAAU,MAAM;AAClB,cAAM,IAAI,MAAM,cAAc,IAAI,CAAC,+BAA+B;AAAA,MACpE;AACA,UAAI,aAAa;AACjB,UAAI,aAAa;AAAA,IACnB;AACA,QAAI,KAAK,GAAG;AAAA,EACd;AACA,MAAI,IAAI,WAAW,GAAG;AACpB,UAAM,IAAI,MAAM,iCAAiC;AAAA,EACnD;AACA,SAAO;AACT;AAsBA,SAAS,iBAAiB,KAA4B;AACpD,QAAM,UAAU,IAAI,KAAK;AACzB,MAAI,YAAY,GAAI,QAAO;AAC3B,QAAM,IAAI,OAAO,OAAO;AACxB,MAAI,OAAO,MAAM,CAAC,EAAG,QAAO;AAC5B,SAAO;AACT;AAOA,SAAS,gBAAgB,OAA+B;AACtD,MAAI,OAAO,UAAU,SAAU,QAAO;AACtC,MAAI,OAAO,MAAM,KAAK,EAAG,QAAO;AAChC,SAAO;AACT;AAOA,SAAS,SAAS,MAA0B;AAC1C,QAAM,OAAmB,CAAC;AAC1B,MAAI,UAAoB,CAAC;AACzB,MAAI,QAAQ;AACZ,MAAI,WAAW;AACf,MAAI,IAAI;AACR,SAAO,IAAI,KAAK,QAAQ;AACtB,UAAM,KAAK,KAAK,CAAC;AACjB,QAAI,UAAU;AACZ,UAAI,OAAO,KAAK;AACd,YAAI,KAAK,IAAI,CAAC,MAAM,KAAK;AACvB,mBAAS;AACT,eAAK;AACL;AAAA,QACF;AACA,mBAAW;AACX;AACA;AAAA,MACF;AACA,eAAS;AACT;AACA;AAAA,IACF;AACA,QAAI,OAAO,KAAK;AACd,iBAAW;AACX;AACA;AAAA,IACF;AACA,QAAI,OAAO,KAAK;AACd,cAAQ,KAAK,KAAK;AAClB,cAAQ;AACR;AACA;AAAA,IACF;AACA,QAAI,OAAO,MAAM;AACf,cAAQ,KAAK,KAAK;AAClB,WAAK,KAAK,OAAO;AACjB,gBAAU,CAAC;AACX,cAAQ;AACR;AACA;AAAA,IACF;AACA,QAAI,OAAO,MAAM;AACf;AACA;AAAA,IACF;AACA,aAAS;AACT;AAAA,EACF;AACA,MAAI,MAAM,SAAS,KAAK,QAAQ,SAAS,GAAG;AAC1C,YAAQ,KAAK,KAAK;AAClB,SAAK,KAAK,OAAO;AAAA,EACnB;AACA,SAAO;AACT;;;AFhSA,SAAS,WAAmB;AAC1B,QAAM,cAAc,QAAQ,IAAI,kBAAkB;AAClD,MAAI,YAAa,QAAO;AACxB,SAAY,UAAQ,WAAQ,GAAG,WAAW,OAAO;AACnD;AAEA,SAAS,cAAsB;AAC7B,SAAY,UAAK,SAAS,GAAG,WAAW;AAC1C;AAEA,SAAS,gBAAsB;AAC7B,QAAM,SAAS,YAAY;AAC3B,QAAM,SAAY,cAAW,MAAM;AACnC,MAAI,WAAW;AACf,MAAI,QAAQ;AACV,eAAc,YAAS,MAAM,EAAE;AAAA,EACjC;AACA,QAAM,QAAQ,EAAE,QAAQ,WAAW,UAAU,MAAM,OAAO;AAC1D,UAAQ,OAAO,MAAM,KAAK,UAAU,KAAK,IAAI,IAAI;AACnD;AAEA,SAAS,gBAAsB;AAC7B,QAAM,SAAS,YAAY;AAC3B,MAAO,cAAW,MAAM,GAAG;AACzB,IAAG,cAAW,MAAM;AACpB,YAAQ,OAAO,MAAM,kBAAkB,MAAM;AAAA,CAAI;AAAA,EACnD,OAAO;AACL,YAAQ,OAAO,MAAM,sBAAsB,MAAM;AAAA,CAAI;AAAA,EACvD;AACF;AAEA,IAAM,oBAAoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAe1B,SAAS,QAAQ,WAAyB;AACxC,QAAM,WAAgB,UAAK,WAAW,OAAO;AAC7C,EAAG,aAAU,UAAU,EAAE,WAAW,KAAK,CAAC;AAC1C,UAAQ,OAAO,MAAM,cAAc,QAAQ;AAAA,CAAI;AAE/C,QAAM,iBAAsB,UAAK,UAAU,uBAAuB;AAClE,MAAO,cAAW,cAAc,GAAG;AACjC,YAAQ,OAAO,MAAM,qBAAqB,cAAc;AAAA,CAAI;AAAA,EAC9D,OAAO;AACL,IAAG,iBAAc,gBAAgB,iBAAiB;AAClD,YAAQ,OAAO,MAAM,YAAY,cAAc;AAAA,CAAI;AAAA,EACrD;AACF;AAEA,SAAS,YAAY,WAAyB;AAC5C,QAAM,WAAgB,UAAK,WAAW,OAAO;AAE7C,MAAI,YAAsB,CAAC;AAC3B,MAAO,cAAW,QAAQ,GAAG;AAC3B,gBACG,eAAY,QAAQ,EACpB,OAAO,CAAC,MAAM,EAAE,SAAS,UAAU,KAAK,EAAE,SAAS,UAAU,CAAC;AAAA,EACnE;AAEA,MAAI,UAAU,WAAW,GAAG;AAC1B,YAAQ,OAAO;AAAA,MACb,2DAA2D,QAAQ;AAAA;AAAA,IACrE;AAAA,EACF,OAAO;AACL,YAAQ,OAAO,MAAM,SAAS,UAAU,MAAM,oBAAoB,QAAQ;AAAA,CAAI;AAAA,EAChF;AAGA,QAAM,aAAa;AACnB,QAAM,MAAM,KAAK,IAAI;AACrB,QAAM,cAAc,gBAAgB,SAAS;AAC7C,aAAW,UAAU,aAAa;AAChC,UAAM,QAAW,YAAS,MAAM,EAAE;AAClC,UAAM,UAAU,KAAK,OAAO,MAAM,UAAU,MAAO,KAAK,KAAK,GAAG;AAChE,QAAI,UAAU,YAAY;AACxB,cAAQ,OAAO;AAAA,QACb,yBAAyB,MAAM,OAAO,OAAO,eAAe,UAAU;AAAA;AAAA,MACxE;AAAA,IACF;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,KAAuB;AAC9C,QAAM,UAAoB,CAAC;AAC3B,MAAI,CAAI,cAAW,GAAG,EAAG,QAAO;AAEhC,QAAM,UAAa,eAAY,KAAK,EAAE,eAAe,KAAK,CAAC;AAC3D,aAAW,SAAS,SAAS;AAC3B,UAAM,WAAgB,UAAK,KAAK,MAAM,IAAI;AAC1C,QAAI,MAAM,YAAY,KAAK,MAAM,SAAS,gBAAgB;AACxD,cAAQ,KAAK,GAAG,gBAAgB,QAAQ,CAAC;AAAA,IAC3C,WAAW,MAAM,OAAO,KAAK,MAAM,KAAK,SAAS,SAAS,GAAG;AAC3D,cAAQ,KAAK,QAAQ;AAAA,IACvB;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,KAAK,OAAiB,QAAQ,KAAK,MAAM,CAAC,GAAS;AACjE,QAAM,OAAO;AAEb,MAAI,KAAK,CAAC,MAAM,aAAa;AAC3B,YAAQ,OAAO,MAAM,UAAU,OAAO;AAAA,CAAI;AAC1C,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,WAAW,KAAK,UAAU,GAAG;AAC3C,QAAI,KAAK,CAAC,MAAM,SAAS;AACvB,oBAAc;AACd,cAAQ,KAAK,CAAC;AAAA,IAChB,WAAW,KAAK,CAAC,MAAM,SAAS;AAC9B,oBAAc;AACd,cAAQ,KAAK,CAAC;AAAA,IAChB,OAAO;AACL,cAAQ,OAAO,MAAM,6BAA6B,KAAK,CAAC,CAAC;AAAA,CAAI;AAC7D,cAAQ,OAAO,MAAM,uCAAuC;AAC5D,cAAQ,KAAK,CAAC;AAAA,IAChB;AAAA,EACF;AAEA,MAAI,KAAK,CAAC,MAAM,QAAQ;AACtB,YAAQ,QAAQ,IAAI,CAAC;AACrB,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,YAAY;AAC1B,gBAAY,QAAQ,IAAI,CAAC;AACzB,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,MAAI,KAAK,CAAC,MAAM,aAAa;AAC3B,iBAAa,KAAK,MAAM,CAAC,CAAC;AAC1B,YAAQ,KAAK,CAAC;AAAA,EAChB;AAEA,UAAQ,OAAO;AAAA,IACb;AAAA,EAQF;AACA,UAAQ,KAAK,KAAK,WAAW,IAAI,IAAI,CAAC;AACxC;AASA,SAAS,mBAAmB,MAAkC;AAC5D,QAAM,OAAyB;AAAA,IAC7B,QAAQ;AAAA,IACR,QAAQ;AAAA,IACR,eAAe;AAAA,IACf,WAAW;AAAA,EACb;AACA,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,UAAM,OAAO,KAAK,IAAI,CAAC;AACvB,QAAI,QAAQ,YAAY;AACtB,UAAI,SAAS,OAAW,OAAM,IAAI,MAAM,0BAA0B;AAClE,WAAK,SAAS;AACd;AAAA,IACF,WAAW,QAAQ,YAAY;AAC7B,UAAI,SAAS,OAAW,OAAM,IAAI,MAAM,0BAA0B;AAClE,WAAK,SAAS;AACd;AAAA,IACF,WAAW,QAAQ,oBAAoB;AACrC,UAAI,SAAS,OAAW,OAAM,IAAI,MAAM,mCAAmC;AAC3E,WAAK,gBAAgB;AACrB;AAAA,IACF,WAAW,QAAQ,eAAe;AAChC,UAAI,SAAS,OAAW,OAAM,IAAI,MAAM,+BAA+B;AACvE,YAAM,SAAS,OAAO,IAAI;AAC1B,UAAI,KAAK,KAAK,MAAM,MAAM,OAAO,MAAM,MAAM,GAAG;AAC9C,cAAM,IAAI,MAAM,qBAAqB,KAAK,UAAU,IAAI,CAAC,kBAAkB;AAAA,MAC7E;AACA,WAAK,YAAY;AACjB;AAAA,IACF,OAAO;AACL,YAAM,IAAI,MAAM,2BAA2B,GAAG,EAAE;AAAA,IAClD;AAAA,EACF;AACA,MAAI,CAAC,KAAK,QAAQ;AAChB,UAAM,IAAI,MAAM,sBAAsB;AAAA,EACxC;AACA,SAAO;AACT;AAEA,SAAS,aAAa,MAAsB;AAC1C,MAAI;AACJ,MAAI;AACF,WAAO,mBAAmB,IAAI;AAAA,EAChC,SAAS,KAAK;AACZ,YAAQ,OAAO,MAAM,cAAe,IAAc,OAAO;AAAA,CAAI;AAC7D,YAAQ,KAAK,CAAC;AACd;AAAA,EACF;AAEA,QAAM,OAAU,gBAAa,KAAK,QAAQ,OAAO;AACjD,QAAM,MAAW,aAAQ,KAAK,MAAM,EAAE,YAAY;AAClD,MAAI;AACJ,MAAI;AACF,QAAI,QAAQ,YAAY,QAAQ,WAAW;AACzC,gBAAU,gBAAgB,IAAI;AAAA,IAChC,WAAW,QAAQ,QAAQ;AACzB,gBAAU,cAAc,IAAI;AAAA,IAC9B,WAAW,KAAK,UAAU,EAAE,WAAW,GAAG,GAAG;AAC3C,gBAAU,gBAAgB,IAAI;AAAA,IAChC,OAAO;AACL,gBAAU,cAAc,IAAI;AAAA,IAC9B;AAAA,EACF,SAAS,KAAK;AACZ,YAAQ,OAAO,MAAM,cAAe,IAAc,OAAO;AAAA,CAAI;AAC7D,YAAQ,KAAK,CAAC;AACd;AAAA,EACF;AAEA,QAAM,QAAqB,QACxB,OAAO,CAAC,MAAM,EAAE,UAAU,EAC1B,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,YAAY,OAAO,EAAE,WAAW,EAAE;AAC5D,QAAM,UAAU,QAAQ,OAAO,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE;AACrD,MAAI,MAAM,WAAW,GAAG;AACtB,YAAQ,OAAO;AAAA,MACb,uCAAuC,OAAO;AAAA;AAAA,IAChD;AACA,YAAQ,KAAK,CAAC;AACd;AAAA,EACF;AAEA,MAAI;AACJ,MAAI;AACF,aAAS,iBAAiB,OAAO,KAAK,SAAS;AAAA,EACjD,SAAS,KAAK;AACZ,YAAQ,OAAO,MAAM,cAAe,IAAc,OAAO;AAAA,CAAI;AAC7D,YAAQ,KAAK,CAAC;AACd;AAAA,EACF;AAEA,QAAM,SAAS;AAAA,IACb,aAAa,KAAK;AAAA,IAClB,gBAAgB,KAAK;AAAA,IACrB,WAAW,OAAO;AAAA,IAClB,aAAa,OAAO;AAAA,IACpB,WAAW,OAAO,OAAO,SAAS;AAAA,IAClC,aAAa,OAAO,OAAO,UAAU;AAAA,IACrC,SAAS,OAAO,OAAO,MAAM;AAAA,IAC7B,eAAe;AAAA,EACjB;AACA,UAAQ,OAAO,MAAM,KAAK,UAAU,QAAQ,MAAM,CAAC,IAAI,IAAI;AAC7D;AAEA,SAAS,OAAO,GAAmB;AACjC,SAAO,KAAK,MAAM,IAAI,GAAI,IAAI;AAChC;AAGA,IAAM,oBACJ,OAAO,YAAY,QAAQ,YAC3B,YAAY,IAAI,WAAW,OAAO,KAClC,QAAQ,KAAK,CAAC,MAAM,UACpB,YAAY,IAAI,SAAc,cAAS,QAAQ,KAAK,CAAC,CAAC,CAAC;AAEzD,IAAI,mBAAmB;AACrB,OAAK;AACP;","names":[]}

package/dist/esm/index.d.ts

@@ -1,12 +1,12 @@
-import { InitializeResult, Trace, Assertion, EvaluateBatchResult, Step, AssertionResult } from './proto/index.js';
-export { AgentId, AssertionId, AssertionSpec, ConstraintSpec, ContentSpec, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, EmbeddingSpec, EngineTimeoutError, ErrorData, EvaluateBatchParams, InitializeParams, LlmJudgeSpec, PluginSpec, ProtocolError, RPCError, RPCRequest, RPCResponse, STATUS_HARD_FAIL, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, SchemaSpec, ShutdownResult, SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, TraceId, TraceMetadata, TraceSpec, TraceTreeSpec, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, traceId } from './proto/index.js';
+import { InitializeResult, ProtocolLogger, ProtocolDiagnostic, Trace, Assertion, EvaluateBatchResult, DriftReport, SimulatePersona, ConversationMessage, SimulateFaultConfig, Step, AssertionResult } from './proto/index.js';
+export { AgentId, AssertionId, AssertionSpec, BiasProbe, ConstraintSpec, ContentSpec, ERR_ASSERTION_ERROR, ERR_ENGINE_ERROR, ERR_INVALID_TRACE, ERR_PROVIDER_ERROR, ERR_SESSION_ERROR, ERR_TIMEOUT, EmbeddingSpec, EngineTimeoutError, ErrorData, EvaluateBatchParams, GenerateUserMessageParams, GenerateUserMessageResult, InitializeParams, JudgeAgreement, JudgeMetadata, LlmJudgeSpec, PluginSpec, ProtocolDesyncError, ProtocolDiagnosticBuffer, ProtocolDiagnosticKind, ProtocolError, QueryDriftParams, QueryDriftResult, RPCError, RPCRequest, RPCResponse, STATUS_HARD_FAIL, STATUS_PASS, STATUS_SOFT_FAIL, STEP_AGENT_CALL, STEP_LLM_CALL, STEP_RETRIEVAL, STEP_TOOL_CALL, SchemaSpec, ShutdownResult, SubmitPluginResultParams, TYPE_CONSTRAINT, TYPE_CONTENT, TYPE_EMBEDDING, TYPE_LLM_JUDGE, TYPE_PLUGIN, TYPE_SCHEMA, TYPE_TRACE, TYPE_TRACE_TREE, TraceId, TraceMetadata, TraceSpec, TraceTreeSpec, agentId, assertionId, decodeResponse, encodeRequest, extractId, extractResult, layerForType, previewLine, traceId } from './proto/index.js';
 import { ChildProcess } from 'node:child_process';
 import { Interface } from 'node:readline';
 import { AsyncLocalStorage } from 'node:async_hooks';
 import { EventEmitter } from 'node:events';
 
-declare const VERSION = "0.5.0";
-declare const ENGINE_VERSION = "0.5.0";
+declare const VERSION = "0.9.0";
+declare const ENGINE_VERSION = "0.9.0";
 
 declare class EngineManager {
     private enginePath;
@@ -31,6 +31,28 @@ declare class EngineManager {
     get readlineInterface(): Interface | null;
 }
 
+interface AttestClientOptions {
+    /** Logger override. When omitted, warnings only emit if ATTEST_DEBUG_PROTOCOL=1. */
+    logger?: ProtocolLogger;
+    /** Listener invoked for every protocol diagnostic. */
+    onDiagnostic?: (diagnostic: ProtocolDiagnostic) => void;
+    /** Diagnostic ring-buffer capacity. Default 32. */
+    diagnosticBufferSize?: number;
+    /** Number of diagnostics in `desyncWindowMs` that triggers desync rejection. Default 3. */
+    desyncThreshold?: number;
+    /** Window length for desync detection in milliseconds. Default 1000. */
+    desyncWindowMs?: number;
+}
+/**
+ * AttestClient owns request/response correlation over the engine's
+ * NDJSON protocol.
+ *
+ * Desync detection is a one-way latch: once the diagnostic rate breaches
+ * `desyncThreshold` within `desyncWindowMs` the client refuses further
+ * sends for the lifetime of the instance. There is no reset hook by
+ * design — recovery requires constructing a new client over a fresh
+ * engine subprocess.
+ */
 declare class AttestClient {
     private readonly engine;
     private requestId;
@@ -38,10 +60,23 @@ declare class AttestClient {
     private writeChain;
     private readerActive;
     private lineHandler;
-    constructor(engine: EngineManager);
+    private readonly logger;
+    private readonly diagnosticBuffer;
+    private readonly diagnosticListeners;
+    private readonly desyncThreshold;
+    private readonly desyncWindowMs;
+    private desynced;
+    constructor(engine: EngineManager, options?: AttestClientOptions);
+    /** Subscribe to protocol diagnostics. Returns an unsubscribe function. */
+    onProtocolDiagnostic(listener: (diagnostic: ProtocolDiagnostic) => void): () => void;
+    /** Snapshot of the diagnostic ring buffer (most recent last). */
+    protocolDiagnostics(): readonly ProtocolDiagnostic[];
     startReader(): void;
     stopReader(): void;
+    private recordDiagnostic;
+    private maybeTriggerDesync;
     private handleLine;
+    private handleDecodeFailure;
     private failAll;
     sendRequest(method: string, params: Record<string, unknown>, timeoutMs?: number): Promise<unknown>;
     evaluateBatch(trace: Trace, assertions: readonly Assertion[], options?: {
@@ -50,6 +85,12 @@ declare class AttestClient {
     submitPluginResult(traceId: string, pluginName: string, assertionId: string, status: string, score: number, explanation: string, options?: {
         timeout?: number;
     }): Promise<boolean>;
+    queryDrift(assertionId: string, windowSize?: number, options?: {
+        timeout?: number;
+    }): Promise<DriftReport>;
+    generateUserMessage(persona: SimulatePersona, conversationHistory: readonly ConversationMessage[], faultConfig?: SimulateFaultConfig, options?: {
+        timeout?: number;
+    }): Promise<string>;
 }
 
 declare class TraceBuilder {
@@ -106,6 +147,29 @@ declare class AgentResult {
     get failCount(): number;
 }
 
+/**
+ * Terminal-grade rendering of Attest assertion failures.
+ *
+ * Mirrors the Python attest.cli.diagnostics module so PR comments,
+ * pytest output, and vitest output show the same fields. Use as the
+ * message argument to a vitest `expect(result.passed, ...)` so
+ * reviewers see trace path, expected vs actual, and judge metadata.
+ */
+
+interface RenderOptions {
+    /** Emit ANSI colour escapes. Default false (plain text). */
+    color?: boolean;
+    /** Source location annotation appended after each block. */
+    testFile?: string;
+    /** Render passing assertions in addition to failures. Default false. */
+    includePassing?: boolean;
+}
+declare function renderAssertionFailure(result: AssertionResult, options?: RenderOptions): string;
+declare function renderSummary(result: AgentResult, options?: {
+    color?: boolean;
+}): string;
+declare function renderDiagnostics(result: AgentResult, options?: RenderOptions): string;
+
 declare class TraceTree {
     readonly root: Trace;
     constructor(root: Trace);
@@ -517,7 +581,7 @@ declare class ExpectChain {
      * ```
      */
     outputSimilarTo(reference: string, opts?: {
-        threshold?: number;
+        threshold?: number | "dynamic";
         model?: string;
         soft?: boolean;
     }): this;
@@ -538,7 +602,7 @@ declare class ExpectChain {
      */
     passesJudge(criteria: string, opts?: {
         rubric?: string;
-        threshold?: number;
+        threshold?: number | "dynamic";
         model?: string;
         soft?: boolean;
     }): this;
@@ -805,21 +869,36 @@ declare function tier(level: number): <T extends (...args: never[]) => unknown>(
  * @param options.simulation - Enable simulation mode. All `evaluateBatch` calls
  *   return deterministic pass results without spawning the engine or calling any
  *   LLM. Equivalent to setting `ATTEST_SIMULATION=1` in the environment.
+ * @param options.sampleRate - Sampling rate for continuous evaluation (0.0–1.0).
+ *   Equivalent to setting `ATTEST_SAMPLE_RATE` in the environment.
  */
 declare function config(options: {
     simulation?: boolean;
+    sampleRate?: number;
 }): void;
 /**
  * Returns true when simulation mode is active via either the programmatic API
  * or the `ATTEST_SIMULATION` environment variable.
  */
 declare function isSimulationMode(): boolean;
+/**
+ * Returns the sampling rate for continuous evaluation. Priority:
+ * programmatic `config({ sampleRate })` > `ATTEST_SAMPLE_RATE` env var > 0.
+ */
+declare function getSampleRate(): number;
 /**
  * Reset all programmatic config to defaults. Primarily for use in tests.
  * Environment variables are not affected.
  */
 declare function resetConfig(): void;
 
+/**
+ * Return a deterministic all-pass result for every assertion without
+ * contacting the engine or any LLM provider. Used when simulation mode
+ * is active (`ATTEST_SIMULATION=1` or `config({ simulation: true })`).
+ */
+declare function simulationEvaluateBatch(assertions: readonly Assertion[]): EvaluateBatchResult;
+
 interface PluginResult {
     readonly status: string;
     readonly score: number;
@@ -882,6 +961,64 @@ declare class ContinuousEvalRunner {
     private scheduleNext;
 }
 
+/**
+ * Calibration helpers — TS parity with engine/internal/assertion/judge/calibration.go
+ * and sdks/python/src/attest/calibration.py.
+ *
+ * Computes Cohen's κ, agreement %, and ROC-AUC over (human, judge) score
+ * pairs and parses CSV/JSONL labels files. Metrics match the engine
+ * byte-for-byte on the same input.
+ */
+interface LabelPair {
+    readonly human: number;
+    readonly judge: number;
+}
+interface AgreementResult {
+    readonly threshold: number;
+    readonly n: number;
+    readonly agreement: number;
+    readonly cohenKappa: number;
+    readonly rocAuc: number;
+}
+interface LabeledRecord {
+    input: string;
+    humanLabel: number;
+    judgeScore: number;
+    judgeKnown: boolean;
+}
+/**
+ * Compute Cohen's κ, agreement %, and ROC-AUC over labeled pairs.
+ *
+ * Throws when pairs is empty or threshold is outside (0, 1). Cohen's κ
+ * degenerates to 0 when all labels land in a single class for both
+ * raters; ROC-AUC is reported as 0 when only one human-class is
+ * present.
+ */
+declare function computeAgreement(pairs: readonly LabelPair[], threshold: number): AgreementResult;
+/**
+ * Parse a 2- or 3-column CSV (input, human_label[, judge_score]).
+ * Header detected when row[0][1] does not parse as a float. Lines
+ * beginning with `#` are skipped.
+ */
+declare function loadLabelsCSV(text: string): LabeledRecord[];
+/**
+ * Parse a newline-delimited JSON file. Each line is an object with the
+ * shape {"input": "...", "human_label": 0.9, "judge_score": 0.8}.
+ * `judge_score` is optional.
+ */
+declare function loadLabelsJSONL(text: string): LabeledRecord[];
+/**
+ * Return the 16-character SHA-256 prefix used by the engine for prompt
+ * hashes. Mirrors promptHash in engine/internal/assertion/judge_eval.go
+ * so SDK-recorded calibration rows align with JudgeMetadata.prompt_hash.
+ */
+declare function promptHash(text: string): string;
+
+declare const FRIENDLY_USER: SimulatePersona;
+declare const ADVERSARIAL_USER: SimulatePersona;
+declare const CONFUSED_USER: SimulatePersona;
+declare const COOPERATIVE_USER: SimulatePersona;
+
 declare class ManualAdapter {
     private readonly agentId;
     constructor(agentId?: string);
@@ -1082,4 +1219,4 @@ interface TraceAdapter {
     traceFromResponse(response: unknown, options?: Record<string, unknown>): Trace;
 }
 
-export { Agent, AgentResult, AlertDispatcher, type AlertPayload, AnthropicAdapter, Assertion, AssertionResult, AttestClient, type AttestPlugin, ContinuousEvalRunner, ENGINE_VERSION, EngineManager, EvaluateBatchResult, ExpectChain, GeminiAdapter, InitializeResult, LangChainAdapter, ManualAdapter, OTelAdapter, OllamaAdapter, OpenAIAdapter, PluginRegistry, type PluginResult, Sampler, Step, TIER_1, TIER_2, TIER_3, Trace, type TraceAdapter, TraceBuilder, TraceTree, VERSION, activeBuilder, agent, attestExpect, config, delegate, executePluginAssertion, isSimulationMode, resetConfig, tier };
+export { ADVERSARIAL_USER, Agent, AgentResult, type AgreementResult, AlertDispatcher, type AlertPayload, AnthropicAdapter, Assertion, AssertionResult, AttestClient, type AttestClientOptions, type AttestPlugin, CONFUSED_USER, COOPERATIVE_USER, ContinuousEvalRunner, ConversationMessage, DriftReport, ENGINE_VERSION, EngineManager, EvaluateBatchResult, ExpectChain, FRIENDLY_USER, GeminiAdapter, InitializeResult, type LabelPair, type LabeledRecord, LangChainAdapter, ManualAdapter, OTelAdapter, OllamaAdapter, OpenAIAdapter, PluginRegistry, type PluginResult, ProtocolDiagnostic, ProtocolLogger, type RenderOptions, Sampler, SimulateFaultConfig, SimulatePersona, Step, TIER_1, TIER_2, TIER_3, Trace, type TraceAdapter, TraceBuilder, TraceTree, VERSION, activeBuilder, agent, attestExpect, computeAgreement, config, delegate, executePluginAssertion, getSampleRate, isSimulationMode, loadLabelsCSV, loadLabelsJSONL, promptHash, renderAssertionFailure, renderDiagnostics, renderSummary, resetConfig, simulationEvaluateBatch, tier };