@attest-ai/core 0.9.0 → 1.0.0-rc.2

package/dist/esm/index.js

@@ -6,6 +6,17 @@ var STEP_AGENT_CALL = "agent_call";
 var STATUS_PASS = "pass";
 var STATUS_SOFT_FAIL = "soft_fail";
 var STATUS_HARD_FAIL = "hard_fail";
+var STATUS_NOT_EVALUATED = "not_evaluated";
+var REASON_PROVIDER_UNAVAILABLE = "provider_unavailable";
+var REASON_PROVIDER_NOT_CONFIGURED = "provider_not_configured";
+var REASON_SIMULATION = "simulation";
+var REASON_MISSING_EVIDENCE = "missing_evidence";
+var REASON_SHORT_CIRCUITED = "short_circuited";
+var REASON_BUDGET_EXHAUSTED = "budget_exhausted";
+var REASON_JUDGE_ABSTAINED = "judge_abstained";
+var OUTCOME_PASS = "pass";
+var OUTCOME_FAIL = "fail";
+var OUTCOME_SKIP = "skip";
 var TYPE_SCHEMA = "schema";
 var TYPE_CONSTRAINT = "constraint";
 var TYPE_TRACE = "trace";
@@ -31,6 +42,13 @@ function assertionId(id) {
 function agentId(id) {
   return id;
 }
+var FailureClass = {
+  BrokenCode: "broken_code",
+  FlakyJudge: "flaky_judge",
+  BadRubric: "bad_rubric",
+  MissingTraceData: "missing_trace_data",
+  StochasticVariance: "stochastic_variance"
+};
 function layerForType(assertionType) {
   switch (assertionType) {
     case "schema":
@@ -86,6 +104,12 @@ var ProtocolDesyncError = class extends Error {
     this.diagnostics = diagnostics;
   }
 };
+var OTLPConversionError = class extends Error {
+  constructor(message) {
+    super(message);
+    this.name = "OTLPConversionError";
+  }
+};
 
 // src/proto/diagnostics.ts
 var DEFAULT_BUFFER_SIZE = 32;
@@ -192,15 +216,14 @@ function extractId(response) {
 }
 
 // src/version.ts
-var VERSION = "0.9.0";
-var ENGINE_VERSION = "0.9.0";
+var VERSION = "1.0.0-rc.2";
+var ENGINE_VERSION = "1.0.0-rc.2";
 
 // src/engine-manager.ts
-import { spawn, execSync } from "child_process";
+import { spawn, spawnSync } from "child_process";
 import { createInterface } from "readline";
-import { existsSync as existsSync2, accessSync, constants as fsConstants } from "fs";
-import { resolve, join as join2, dirname } from "path";
-import { fileURLToPath } from "url";
+import { existsSync, accessSync, statSync as statSync2, constants as fsConstants } from "fs";
+import { delimiter, join as join2 } from "path";
 
 // src/engine-downloader.ts
 import * as https from "https";
@@ -212,10 +235,8 @@ import * as crypto from "crypto";
 var RELEASE_BASE = `https://github.com/attest-framework/attest/releases/download/v${ENGINE_VERSION}`;
 var CHECKSUMS_URL = `${RELEASE_BASE}/checksums-sha256.txt`;
 var MAX_REDIRECTS = 5;
-function attestBinDir() {
-  const dir = path.join(os.homedir(), ".attest", "bin");
-  fs.mkdirSync(dir, { recursive: true });
-  return dir;
+function versionedBinDir() {
+  return path.join(os.homedir(), ".attest", "bin", ENGINE_VERSION);
 }
 function platformKey() {
   const platform2 = os.platform();
@@ -243,20 +264,8 @@ function binaryFilename() {
   return os.platform() === "win32" ? "attest-engine.exe" : "attest-engine";
 }
 function cachedEnginePath() {
-  const dir = path.join(os.homedir(), ".attest", "bin");
-  const binPath = path.join(dir, binaryFilename());
-  const versionFile = path.join(dir, ".engine-version");
-  if (!fs.existsSync(binPath)) {
-    return null;
-  }
-  if (!fs.existsSync(versionFile)) {
-    return null;
-  }
-  const cachedVersion = fs.readFileSync(versionFile, "utf-8").trim();
-  if (cachedVersion !== ENGINE_VERSION) {
-    return null;
-  }
-  return binPath;
+  const binPath = path.join(versionedBinDir(), binaryFilename());
+  return fs.statSync(binPath, { throwIfNoEntry: false })?.isFile() ? binPath : null;
 }
 function parseChecksums(text) {
   const checksums = /* @__PURE__ */ new Map();
@@ -276,7 +285,7 @@ function httpsGet(url, redirectCount = 0) {
       new Error(`Too many redirects (>${MAX_REDIRECTS}) following ${url}`)
     );
   }
-  return new Promise((resolve2, reject) => {
+  return new Promise((resolve, reject) => {
     const client = url.startsWith("https:") ? https : http;
     client.get(url, (res) => {
       const status = res.statusCode ?? 0;
@@ -287,7 +296,7 @@ function httpsGet(url, redirectCount = 0) {
           return;
         }
         res.resume();
-        resolve2(httpsGet(location, redirectCount + 1));
+        resolve(httpsGet(location, redirectCount + 1));
         return;
       }
       if (status < 200 || status >= 300) {
@@ -297,14 +306,14 @@ function httpsGet(url, redirectCount = 0) {
       }
       const chunks = [];
       res.on("data", (chunk) => chunks.push(chunk));
-      res.on("end", () => resolve2(Buffer.concat(chunks)));
+      res.on("end", () => resolve(Buffer.concat(chunks)));
       res.on("error", reject);
     }).on("error", reject);
   });
 }
 async function downloadEngine() {
   const key = platformKey();
-  const filename = `attest-engine-${key}`;
+  const filename = `attest-engine-${key}${os.platform() === "win32" ? ".exe" : ""}`;
   const binaryUrl = `${RELEASE_BASE}/${filename}`;
   process.stderr.write(`[attest] Downloading engine v${ENGINE_VERSION} for ${key}...
 `);
@@ -324,19 +333,24 @@ async function downloadEngine() {
   const actualHash = crypto.createHash("sha256").update(data).digest("hex");
   if (actualHash !== expectedHash) {
     throw new Error(
-      `SHA256 mismatch for ${filename}. Expected: ${expectedHash}, got: ${actualHash}. The download may be corrupted. Delete ~/.attest/bin/ and retry.`
+      `SHA256 mismatch for ${filename}. Expected: ${expectedHash}, got: ${actualHash}. The download may be corrupted. Retry or download manually.`
     );
   }
   process.stderr.write(`[attest] Checksum verified.
 `);
-  const dir = attestBinDir();
-  const destFilename = binaryFilename();
-  const destPath = path.join(dir, destFilename);
-  const tmpPath = path.join(dir, `.${destFilename}.tmp.${process.pid}`);
-  fs.writeFileSync(tmpPath, data);
-  fs.chmodSync(tmpPath, 493);
-  fs.renameSync(tmpPath, destPath);
-  fs.writeFileSync(path.join(dir, ".engine-version"), ENGINE_VERSION);
+  const dir = versionedBinDir();
+  fs.mkdirSync(dir, { recursive: true });
+  const destPath = path.join(dir, binaryFilename());
+  const suffix = `${String(process.pid)}.${crypto.randomBytes(8).toString("hex")}`;
+  const tmpPath = path.join(dir, `.${binaryFilename()}.tmp.${suffix}`);
+  try {
+    fs.writeFileSync(tmpPath, data, { flag: "wx", mode: 493 });
+    fs.chmodSync(tmpPath, 493);
+    fs.renameSync(tmpPath, destPath);
+  } catch (err) {
+    if (err.code !== "EEXIST") fs.rmSync(tmpPath, { force: true });
+    throw err;
+  }
   process.stderr.write(`[attest] Engine installed at ${destPath}
 `);
   return destPath;
@@ -344,45 +358,92 @@ async function downloadEngine() {
 
 // src/engine-manager.ts
 var ENGINE_BINARY_NAME = "attest-engine";
-async function findEngineBinary() {
-  const envPath = process.env["ATTEST_ENGINE_PATH"];
-  if (envPath) {
-    if (!existsSync2(envPath)) {
-      throw new Error(
-        `ATTEST_ENGINE_PATH is set to '${envPath}' but the file does not exist.`
-      );
-    }
-    return envPath;
-  }
-  try {
-    const cmd = process.platform === "win32" ? "where" : "which";
-    const found = execSync(`${cmd} ${ENGINE_BINARY_NAME}`, { encoding: "utf-8" }).trim();
-    if (found) return found;
-  } catch {
+var VERSION_PROBE_TIMEOUT_MS = 1e4;
+var EngineVersionMismatchError = class extends Error {
+  enginePath;
+  source;
+  found;
+  required;
+  constructor(enginePath, source, found, required) {
+    const reported = found === null ? "reports no version" : `reports version ${found}`;
+    super(
+      `attest-engine at ${enginePath} (from ${source}) ${reported}; this SDK requires attest-engine ${required}. Point ATTEST_ENGINE_PATH at a matching engine, or remove the mismatched one from PATH so the SDK can download ${required} into ~/.attest/bin/${required}/.`
+    );
+    this.name = "EngineVersionMismatchError";
+    this.enginePath = enginePath;
+    this.source = source;
+    this.found = found;
+    this.required = required;
   }
-  const cached = cachedEnginePath();
-  if (cached) return cached;
-  const candidates = [
-    join2(dirname(fileURLToPath(import.meta.url)), "..", "..", "..", "..", "..", "bin", ENGINE_BINARY_NAME),
-    join2(process.cwd(), "bin", ENGINE_BINARY_NAME)
-  ];
-  for (const candidate of candidates) {
-    const resolved = resolve(candidate);
+};
+function engineVersion(enginePath) {
+  const proc = spawnSync(enginePath, ["version"], {
+    stdio: ["ignore", "pipe", "pipe"],
+    encoding: "utf-8",
+    timeout: VERSION_PROBE_TIMEOUT_MS
+  });
+  if (proc.error) throw proc.error;
+  const out = proc.stdout.trim();
+  const prefix = `${ENGINE_BINARY_NAME} `;
+  if (proc.status !== 0 || !out.startsWith(prefix)) return null;
+  return out.slice(prefix.length).trim();
+}
+function onPath() {
+  for (const dir of (process.env["PATH"] ?? "").split(delimiter)) {
+    if (!dir) continue;
+    const candidate = join2(dir, binaryFilename());
     try {
-      accessSync(resolved, fsConstants.X_OK);
-      if (existsSync2(resolved)) return resolved;
+      accessSync(candidate, fsConstants.X_OK);
     } catch {
+      continue;
     }
+    if (statSync2(candidate).isFile()) return candidate;
+  }
+  return null;
+}
+function requireFile(enginePath, label) {
+  if (!existsSync(enginePath) || !statSync2(enginePath).isFile()) {
+    throw new Error(`${label}=${enginePath} does not exist or is not a file.`);
+  }
+}
+async function locateEngine(explicitPath, allowDownload) {
+  if (explicitPath) {
+    requireFile(explicitPath, "Engine path");
+    return [explicitPath, "explicit engine path"];
   }
-  const noDownload = process.env["ATTEST_ENGINE_NO_DOWNLOAD"];
+  const envPath = process.env["ATTEST_ENGINE_PATH"];
+  if (envPath) {
+    requireFile(envPath, "ATTEST_ENGINE_PATH");
+    return [envPath, "ATTEST_ENGINE_PATH"];
+  }
+  const found = onPath();
+  if (found) return [found, "PATH"];
+  const cached = cachedEnginePath();
+  if (cached) return [cached, "engine cache"];
+  if (!allowDownload) {
+    throw new Error(
+      `No '${ENGINE_BINARY_NAME}' ${ENGINE_VERSION} in ATTEST_ENGINE_PATH, PATH or ~/.attest/bin/${ENGINE_VERSION}/.`
+    );
+  }
+  const noDownload = process.env["ATTEST_ENGINE_NO_DOWNLOAD"]?.toLowerCase();
   if (noDownload === "1" || noDownload === "true" || noDownload === "yes") {
     throw new Error(
-      `Cannot find '${ENGINE_BINARY_NAME}' binary and auto-download is disabled (ATTEST_ENGINE_NO_DOWNLOAD is set). Install the engine manually:
-  1. Download from https://github.com/attest-framework/attest/releases
-  2. Place the binary on your PATH or set ATTEST_ENGINE_PATH`
+      `Cannot find '${ENGINE_BINARY_NAME}' ${ENGINE_VERSION} and ATTEST_ENGINE_NO_DOWNLOAD is set.
+Install options:
+  1. Build from source: make engine, then set ATTEST_ENGINE_PATH
+  2. Download release v${ENGINE_VERSION}: https://github.com/attest-framework/attest/releases
+  3. Allow auto-download by unsetting ATTEST_ENGINE_NO_DOWNLOAD`
     );
   }
-  return downloadEngine();
+  return [await downloadEngine(), "download"];
+}
+async function resolveEngine(explicitPath, options = {}) {
+  const [enginePath, source] = await locateEngine(explicitPath, options.allowDownload ?? true);
+  const found = engineVersion(enginePath);
+  if (found !== ENGINE_VERSION) {
+    throw new EngineVersionMismatchError(enginePath, source, found, ENGINE_VERSION);
+  }
+  return enginePath;
 }
 var EngineManager = class {
   enginePath;
@@ -397,28 +458,41 @@ var EngineManager = class {
     this.logLevel = options?.logLevel ?? "warn";
   }
   async start() {
-    if (!this.enginePath) {
-      this.enginePath = await findEngineBinary();
-    }
+    this.enginePath = await resolveEngine(this.enginePath || void 0);
     this.process = spawn(this.enginePath, [`--log-level=${this.logLevel}`], {
       stdio: ["pipe", "pipe", "pipe"]
     });
     this.reader = createInterface({ input: this.process.stdout });
-    const result = await this.sendRequestInternal("initialize", {
+    try {
+      this.initResult = await this.initialize();
+    } catch (err) {
+      await this.stop();
+      throw err;
+    }
+    this.initialized = true;
+    return this.initResult;
+  }
+  /** Send initialize and check the engine's version and capabilities. */
+  async initialize() {
+    const init = await this.sendRequestInternal("initialize", {
       sdk_name: "attest-typescript",
       sdk_version: VERSION,
       protocol_version: 1,
       required_capabilities: ["layers_1_4"],
       preferred_encoding: "json"
     });
-    this.initResult = result;
-    if (!this.initResult.compatible) {
-      throw new Error(
-        `Engine incompatible. Missing capabilities: ${JSON.stringify(this.initResult.missing)}`
+    if (init.engine_version !== ENGINE_VERSION) {
+      throw new EngineVersionMismatchError(
+        this.enginePath,
+        "initialize",
+        init.engine_version,
+        ENGINE_VERSION
       );
     }
-    this.initialized = true;
-    return this.initResult;
+    if (!init.compatible) {
+      throw new Error(`Engine incompatible. Missing capabilities: ${JSON.stringify(init.missing)}`);
+    }
+    return init;
   }
   async stop() {
     if (this.process === null) return;
@@ -459,11 +533,11 @@ var EngineManager = class {
     }
     this.requestId += 1;
     const requestStr = encodeRequest(this.requestId, method, params);
-    return new Promise((resolve2, reject) => {
+    return new Promise((resolve, reject) => {
       const onLine = (line) => {
         try {
           const response = decodeResponse(line);
-          resolve2(extractResult(response));
+          resolve(extractResult(response));
         } catch (err) {
           reject(err);
         }
@@ -511,7 +585,11 @@ function config(options) {
   }
 }
 function isSimulationMode() {
-  return _simulationMode || process.env["ATTEST_SIMULATION"] === "1" || process.env["ATTEST_SIMULATION"] === "true" || process.env["ATTEST_SIMULATION"] === "yes";
+  return _simulationMode || simulationEnvEnabled();
+}
+function simulationEnvEnabled() {
+  const env = (process.env["ATTEST_SIMULATION"] ?? "").trim().toLowerCase();
+  return env === "1" || env === "true" || env === "yes";
 }
 function getSampleRate() {
   if (_sampleRate > 0) return _sampleRate;
@@ -532,23 +610,75 @@ function resetConfig() {
   _sampleRate = 0;
 }
 
-// src/simulation.ts
-function simulationEvaluateBatch(assertions) {
-  const results = assertions.map((assertion) => ({
-    assertion_id: assertion.assertion_id,
-    status: "pass",
-    score: 1,
-    explanation: `[simulation] assertion '${assertion.assertion_id}' passed (type: ${assertion.type})`,
-    cost: 0,
-    duration_ms: 0
-  }));
+// src/usage.ts
+var INT_FIELDS = [
+  "model_calls",
+  "judge_calls",
+  "input_tokens",
+  "output_tokens",
+  "cached_tokens",
+  "unpriced_calls",
+  "budget_refused"
+];
+var USD_FIELDS = ["cost_usd", "retry_cost_usd"];
+function zeroUsage() {
+  return {
+    model_calls: 0,
+    judge_calls: 0,
+    input_tokens: 0,
+    output_tokens: 0,
+    cached_tokens: 0,
+    cost_usd: 0,
+    retry_cost_usd: 0,
+    unpriced_calls: 0,
+    budget_refused: 0
+  };
+}
+function addUsage(a, b) {
   return {
-    results,
-    total_cost: 0,
-    total_duration_ms: 0,
-    simulated: true
+    model_calls: a.model_calls + b.model_calls,
+    judge_calls: a.judge_calls + b.judge_calls,
+    input_tokens: a.input_tokens + b.input_tokens,
+    output_tokens: a.output_tokens + b.output_tokens,
+    cached_tokens: a.cached_tokens + b.cached_tokens,
+    cost_usd: a.cost_usd + b.cost_usd,
+    retry_cost_usd: a.retry_cost_usd + b.retry_cost_usd,
+    unpriced_calls: a.unpriced_calls + b.unpriced_calls,
+    budget_refused: a.budget_refused + b.budget_refused
   };
 }
+function sumUsage(usages) {
+  let total = zeroUsage();
+  for (const u of usages) total = addUsage(total, u);
+  return total;
+}
+function parseUsage(raw) {
+  if (typeof raw !== "object" || raw === null || Array.isArray(raw)) {
+    throw new TypeError(`evaluate_batch result: usage must be an object, got ${JSON.stringify(raw)}`);
+  }
+  const obj = raw;
+  const field = (name, integer) => {
+    const v = obj[name];
+    if (typeof v !== "number" || !Number.isFinite(v) || v < 0 || integer && !Number.isInteger(v)) {
+      const kind = integer ? "a non-negative integer" : "a non-negative number";
+      throw new TypeError(`evaluate_batch result: usage.${name} must be ${kind}, got ${JSON.stringify(v)}`);
+    }
+    return v;
+  };
+  const ints = Object.fromEntries(INT_FIELDS.map((f) => [f, field(f, true)]));
+  const usd = Object.fromEntries(USD_FIELDS.map((f) => [f, field(f, false)]));
+  return { ...ints, ...usd };
+}
+function formatUsage(u) {
+  let line = `${u.model_calls} model calls \xB7 ${u.input_tokens + u.output_tokens} tokens (${u.cached_tokens} cached) \xB7 $${u.cost_usd.toFixed(4)} \xB7 retry cost $${u.retry_cost_usd.toFixed(4)}`;
+  if (u.budget_refused > 0) line += ` \xB7 ${u.budget_refused} calls refused by budget`;
+  if (u.unpriced_calls > 0) line += ` \xB7 ${u.unpriced_calls} unpriced calls not in $`;
+  return line;
+}
+function usageSummaryLine(u) {
+  if (u.model_calls === 0 && u.budget_refused === 0) return void 0;
+  return `attest usage: ${formatUsage(u)}`;
+}
 
 // src/client.ts
 var DEFAULT_TIMEOUT_MS = 3e4;
@@ -774,12 +904,12 @@ var AttestClient = class {
       throw new Error("Engine process not started.");
     }
     let capturedReqId;
-    const requestPromise = new Promise((resolve2, reject) => {
+    const requestPromise = new Promise((resolve, reject) => {
       this.writeChain = this.writeChain.then(() => {
         this.requestId += 1;
         const reqId = this.requestId;
         capturedReqId = reqId;
-        this.pending.set(reqId, { resolve: resolve2, reject });
+        this.pending.set(reqId, { resolve, reject });
         const requestStr = encodeRequest(reqId, method, params);
         cp.stdin.write(requestStr, (err) => {
           if (err) {
@@ -799,18 +929,19 @@ var AttestClient = class {
   }
   async evaluateBatch(trace, assertions, options) {
     if (isSimulationMode()) {
-      return simulationEvaluateBatch(assertions);
+      requireSimulationCapability(this.engine["initResult"]);
     }
     const params = {
       trace,
-      assertions: [...assertions]
+      assertions: [...assertions],
+      ...options?.budget !== void 0 ? { budget: options.budget } : {}
     };
     const raw = await this.sendRequest(
       "evaluate_batch",
       params,
       options?.timeout
     );
-    return raw;
+    return { ...raw, usage: parseUsage(raw["usage"]) };
   }
   async submitPluginResult(traceId2, pluginName, assertionId2, status, score, explanation, options) {
     const result = {
@@ -838,10 +969,10 @@ var AttestClient = class {
     if (isSimulationMode()) {
       return {
         assertion_id: assertionId2,
-        mean: 1,
+        mean: 0,
         stddev: 0,
         count: 0,
-        latest_score: 1,
+        latest_score: 0,
         deviation: 0,
         status: "no_data"
       };
@@ -879,6 +1010,16 @@ function classifyDecodeError(message) {
   if (message.startsWith("empty response line")) return "malformed_json";
   return "malformed_json";
 }
+function requireSimulationCapability(init) {
+  if (init === null || init === void 0) {
+    throw new Error("simulation mode: the engine has not been initialized");
+  }
+  if (!(init.capabilities ?? []).includes("simulation_mode")) {
+    throw new Error(
+      `simulation mode is on but attest-engine ${init.engine_version} was not started in simulation mode (capability 'simulation_mode' missing), so it would call real providers. Set ATTEST_SIMULATION=1 before the engine starts (the Vitest fixtures do this for you) and use attest-engine 1.0 or later.`
+    );
+  }
+}
 
 // src/trace.ts
 import { randomUUID } from "crypto";
@@ -969,22 +1110,55 @@ var TraceBuilder = class {
 };
 
 // src/result.ts
+var KNOWN_STATUSES = /* @__PURE__ */ new Set([
+  STATUS_PASS,
+  STATUS_SOFT_FAIL,
+  STATUS_HARD_FAIL,
+  STATUS_NOT_EVALUATED
+]);
+function normalizeAssertionResult(r) {
+  if (KNOWN_STATUSES.has(r.status)) return r;
+  return {
+    ...r,
+    status: STATUS_NOT_EVALUATED,
+    explanation: `[unknown status '${r.status}'] ${r.explanation}`
+  };
+}
 var AgentResult = class {
   trace;
   assertionResults;
   totalCost;
   totalDurationMs;
-  constructor(trace, assertionResults = [], totalCost = 0, totalDurationMs = 0) {
+  /** Attest model usage of the evaluation that produced this result. */
+  usage;
+  constructor(trace, assertionResults = [], totalCost = 0, totalDurationMs = 0, usage = zeroUsage()) {
     this.trace = trace;
-    this.assertionResults = assertionResults;
+    this.assertionResults = assertionResults.map(normalizeAssertionResult);
     this.totalCost = totalCost;
     this.totalDurationMs = totalDurationMs;
+    this.usage = usage;
   }
+  /** True if every assertion has status "pass". */
   get passed() {
     return this.assertionResults.every((r) => r.status === STATUS_PASS);
   }
+  /**
+   * Test outcome after the whole batch. Any hard_fail fails; otherwise any
+   * not_evaluated skips (fails when `strict`); otherwise pass, including
+   * when only soft_fail rows are present (see `softFailures`).
+   */
+  outcome(options) {
+    if (this.hardFailures.length > 0) return OUTCOME_FAIL;
+    if (this.notEvaluated.length > 0) {
+      return options?.strict === true ? OUTCOME_FAIL : OUTCOME_SKIP;
+    }
+    return OUTCOME_PASS;
+  }
+  /** Failed assertions: hard_fail and soft_fail only. */
   get failedAssertions() {
-    return this.assertionResults.filter((r) => r.status !== STATUS_PASS);
+    return this.assertionResults.filter(
+      (r) => r.status === STATUS_HARD_FAIL || r.status === STATUS_SOFT_FAIL
+    );
   }
   get hardFailures() {
     return this.assertionResults.filter((r) => r.status === STATUS_HARD_FAIL);
@@ -992,11 +1166,40 @@ var AgentResult = class {
   get softFailures() {
     return this.assertionResults.filter((r) => r.status === STATUS_SOFT_FAIL);
   }
+  /** Assertions the engine did not evaluate; each carries a `reason`. */
+  get notEvaluated() {
+    return this.assertionResults.filter((r) => r.status === STATUS_NOT_EVALUATED);
+  }
+  /** Count of not_evaluated assertions per reason, in first-seen order. */
+  get notEvaluatedReasons() {
+    const counts = {};
+    for (const r of this.notEvaluated) {
+      const key = r.reason || "unknown";
+      counts[key] = (counts[key] ?? 0) + 1;
+    }
+    return counts;
+  }
+  /**
+   * Passed / evaluated assertions; null when nothing was evaluated.
+   * not_evaluated rows are excluded from the denominator.
+   */
+  get passRate() {
+    const evaluated = this.assertionResults.length - this.notEvaluated.length;
+    return evaluated === 0 ? null : this.passCount / evaluated;
+  }
   get passCount() {
     return this.assertionResults.filter((r) => r.status === STATUS_PASS).length;
   }
+  /** Number of failed assertions (hard_fail + soft_fail). */
   get failCount() {
-    return this.assertionResults.length - this.passCount;
+    return this.failedAssertions.length;
+  }
+  /** "not evaluated: <reason>[, <reason> ×n ...]" for skipped tests. */
+  skipReason() {
+    const parts = Object.entries(this.notEvaluatedReasons).map(
+      ([reason, n]) => n === 1 ? reason : `${reason} \xD7${n}`
+    );
+    return `not evaluated: ${parts.join(", ")}`;
   }
 };
 
@@ -1036,6 +1239,9 @@ function statusGlyph(status, color) {
   } else if (status === STATUS_HARD_FAIL) {
     glyph = "FAIL";
     c = ANSI_RED;
+  } else if (status === STATUS_NOT_EVALUATED) {
+    glyph = "N/E";
+    c = ANSI_DIM;
   } else {
     glyph = status.toUpperCase() || "????";
     c = ANSI_DIM;
@@ -1063,10 +1269,15 @@ function renderAssertionFailure(result, options = {}) {
   lines.push(
     `  ${statusGlyph(result.status, color)} ${bold(result.assertion_id, color)} ${dim(`\u2014 ${label}`, color)}`
   );
+  if (result.reason) {
+    lines.push(`      reason:     not evaluated: ${result.reason}`);
+  }
   if (result.trace_node_path) {
     lines.push(`      trace path: ${result.trace_node_path}`);
   }
-  lines.push(`      score:      ${result.score.toFixed(3)}`);
+  if (result.status !== STATUS_NOT_EVALUATED) {
+    lines.push(`      score:      ${result.score.toFixed(3)}`);
+  }
   if (result.expected) {
     lines.push(`      expected:   ${truncate(result.expected)}`);
   }
@@ -1079,6 +1290,9 @@ function renderAssertionFailure(result, options = {}) {
   if (result.threshold_source && result.threshold_source !== "static") {
     lines.push(`      threshold:  ${result.threshold_source}`);
   }
+  if (result.failure_class) {
+    lines.push(`      class:      ${result.failure_class}`);
+  }
   if (result.judge_metadata) {
     appendJudgeMetadata(lines, result.judge_metadata);
   }
@@ -1101,7 +1315,8 @@ function appendJudgeMetadata(lines, meta) {
     if (meta.rubric_version) {
       rubric = `${rubric} @ ${meta.rubric_version}`;
     }
-    lines.push(`      judge:      ${meta.model} / ${rubric}`);
+    const judge = meta.provider ? `${meta.provider}/${meta.model}` : meta.model;
+    lines.push(`      judge:      ${judge} / ${rubric}`);
   }
   if (meta.prompt_hash) {
     lines.push(`      prompt:     #${meta.prompt_hash}`);
@@ -1141,6 +1356,7 @@ function renderSummary(result, options = {}) {
     `${statusGlyph(STATUS_PASS, color)} ${passN}`,
     `${statusGlyph(STATUS_SOFT_FAIL, color)} ${soft}`,
     `${statusGlyph(STATUS_HARD_FAIL, color)} ${hard}`,
+    `${statusGlyph(STATUS_NOT_EVALUATED, color)} ${result.notEvaluated.length}`,
     `cost $${result.totalCost.toFixed(6)}`,
     `dur ${result.totalDurationMs}ms`
   ];
@@ -1149,23 +1365,27 @@ function renderSummary(result, options = {}) {
 function renderDiagnostics(result, options = {}) {
   const color = options.color ?? false;
   const includePassing = options.includePassing ?? false;
-  const rows = includePassing ? result.assertionResults : result.failedAssertions;
+  const rows = includePassing ? result.assertionResults : result.assertionResults.filter((r) => r.status !== STATUS_PASS);
   if (rows.length === 0) {
     return dim("  (no failures)\n", color);
   }
   const out = [""];
-  out.push(
-    bold(
-      `Attest diagnostic \u2014 ${rows.length} of ${result.assertionResults.length} assertions failed:`,
-      color
-    )
-  );
+  out.push(bold(`Attest diagnostic \u2014 ${headerCounts(result)}:`, color));
   for (const r of rows) {
     out.push(renderAssertionFailure(r, options));
   }
   out.push(dim(`  Summary: ${renderSummary(result, { color })}`, color));
   return out.join("\n");
 }
+function headerCounts(result) {
+  const total = result.assertionResults.length;
+  const notEvaluated = result.notEvaluated.length;
+  if (result.failCount === 0 && notEvaluated > 0) {
+    return `${notEvaluated} of ${total} assertions not evaluated`;
+  }
+  const text = `${result.failCount} of ${total} assertions failed`;
+  return notEvaluated > 0 ? `${text}, ${notEvaluated} not evaluated` : text;
+}
 
 // src/trace-tree.ts
 var TraceTree = class {
@@ -1620,6 +1840,86 @@ var ExpectChain = class {
       soft: opts?.soft ?? false
     });
   }
+  // -- Layer 3: Behavior contracts (tool calls only, retries counted) --
+  /**
+   * Assert how many times a tool is called, per value of argument `per` when set.
+   *
+   * @example
+   * ```ts
+   * // fails with "issue_refund called 2× for order_id=O-1042 (max 1)"
+   * expect(result).toolCardinality("issue_refund", { per: "order_id", atMost: 1 });
+   * ```
+   */
+  toolCardinality(tool, opts) {
+    if (opts.atMost === void 0 && opts.atLeast === void 0) {
+      throw new Error("toolCardinality needs atMost, atLeast or both");
+    }
+    const spec = { check: "tool_cardinality", tool };
+    if (opts.per !== void 0) spec.per = opts.per;
+    if (opts.atLeast !== void 0) spec.min = opts.atLeast;
+    if (opts.atMost !== void 0) spec.max = opts.atMost;
+    spec.soft = opts.soft ?? false;
+    return this.add(TYPE_TRACE, spec);
+  }
+  /**
+   * Assert the tool-call trajectory matches `tools`: `strict` (same sequence,
+   * the default), `unordered` (same calls, any order), `subset` (no call
+   * outside `tools`) or `superset` (every call in `tools` made). Use
+   * {@link trajectoryPartialOrder} for ordering constraints.
+   */
+  trajectory(tools, opts) {
+    return this.add(TYPE_TRACE, {
+      check: "trajectory",
+      mode: opts?.mode ?? "strict",
+      tools,
+      soft: opts?.soft ?? false
+    });
+  }
+  /**
+   * Assert every tool in `tools` and `order` is called and each
+   * `[before, after]` pair holds as in {@link toolCalledBefore}.
+   */
+  trajectoryPartialOrder(order, opts) {
+    const spec = {
+      check: "trajectory",
+      mode: "partial_order",
+      order: order.map(([before, after]) => [before, after])
+    };
+    if (opts?.tools !== void 0) spec.tools = opts.tools;
+    spec.soft = opts?.soft ?? false;
+    return this.add(TYPE_TRACE, spec);
+  }
+  /** Assert every call to `after` has an earlier call to `before`; holds when `after` is never called. */
+  toolCalledBefore(before, after, opts) {
+    return this.add(TYPE_TRACE, { check: "tool_order", before, after, soft: opts?.soft ?? false });
+  }
+  /** Assert no block of `period` (1–4) tool calls repeats `minRepeats` (default 2) times in a row; A→B→A→B fails period 2. */
+  noToolCycles(period, opts) {
+    return this.add(TYPE_TRACE, {
+      check: "cycle",
+      period,
+      min_repeats: opts?.minRepeats ?? 2,
+      soft: opts?.soft ?? false
+    });
+  }
+  /** Assert at most `maxStreak` consecutive tool calls end in error (trace v2). */
+  maxToolFailureStreak(maxStreak, opts) {
+    return this.add(TYPE_TRACE, { check: "failure_streak", max: maxStreak, soft: opts?.soft ?? false });
+  }
+  /**
+   * Assert every call to `tool` has argument `arg` (a dot path) satisfying
+   * `op value`; holds when `tool` is never called.
+   */
+  toolArgSatisfies(tool, arg, op, value, opts) {
+    return this.add(TYPE_TRACE, {
+      check: "arg_predicate",
+      tool,
+      arg,
+      op,
+      value,
+      soft: opts?.soft ?? false
+    });
+  }
   // -- Layer 4: Content --
   /**
    * Assert the output message contains the given string.
@@ -2790,119 +3090,113 @@ var OllamaAdapter = class {
 };
 
 // src/adapters/otel.ts
+import { spawnSync as spawnSync2 } from "child_process";
+var CONVERT_TIMEOUT_MS = 6e4;
+var MAX_TRACE_OUTPUT_BYTES = 64 * 1024 * 1024;
+var STATUS_CODE_ERROR = 2;
 var OTelAdapter = class _OTelAdapter {
-  agentId;
-  constructor(agentId2) {
-    this.agentId = agentId2;
-  }
-  static fromSpans(spans, agentId2) {
-    const adapter = new _OTelAdapter(agentId2);
-    return adapter.buildTrace(spans);
+  /**
+   * Build a trace from finished OpenTelemetry spans of one trace.
+   *
+   * @param agentId Trace `agent_id`; defaults to the first turn's `gen_ai.agent.name`.
+   * @throws OTLPConversionError when the engine rejects the spans.
+   */
+  static async fromSpans(spans, agentId2) {
+    return _OTelAdapter.fromOtlp(spansToOtlp(spans), agentId2);
   }
-  buildTrace(spans) {
-    const sorted = [...spans].sort((a, b) => {
-      const aTime = a.startTime[0] * 1e9 + a.startTime[1];
-      const bTime = b.startTime[0] * 1e9 + b.startTime[1];
-      return aTime - bTime;
+  /**
+   * Convert an OTLP/JSON export (`resourceSpans` → `scopeSpans` → `spans`).
+   *
+   * @param agentId Trace `agent_id`; defaults to the first turn's `gen_ai.agent.name`.
+   * @throws OTLPConversionError when the engine rejects the export (malformed JSON, invalid
+   *   ids or timestamps, more than one trace, non-numeric token counts, ...).
+   * @throws RangeError when a parsed export holds a non-finite number.
+   * @throws OTLPConversionError when the engine times out, overflows the output buffer or
+   *   prints no trace.
+   * @throws Error from `spawnSync` when the engine cannot start.
+   */
+  static async fromOtlp(otlp, agentId2) {
+    const payload = typeof otlp === "string" || otlp instanceof Uint8Array ? otlp : JSON.stringify(otlp, (_key, v) => {
+      if (typeof v === "number" && !Number.isFinite(v)) {
+        throw new RangeError(`OTLP export holds a non-finite number: ${String(v)}`);
+      }
+      return v;
     });
-    const builder = new TraceBuilder(this.agentId);
-    const rootSpan = this.findRootSpan(sorted);
-    if (rootSpan) {
-      const traceId2 = rootSpan.spanContext().traceId;
-      if (traceId2) {
-        builder.setTraceId(`otel_${traceId2.slice(0, 16)}`);
+    const args = ["trace", "convert"];
+    if (agentId2 !== void 0) args.push("--agent-id", agentId2);
+    args.push("-");
+    const proc = spawnSync2(await resolveEngine(), args, {
+      input: payload,
+      timeout: CONVERT_TIMEOUT_MS,
+      maxBuffer: MAX_TRACE_OUTPUT_BYTES
+    });
+    if (proc.error) {
+      const code = proc.error.code;
+      if (code === "ETIMEDOUT") {
+        throw new OTLPConversionError(
+          `attest-engine trace convert timed out after ${CONVERT_TIMEOUT_MS / 1e3}s`
+        );
       }
-    }
-    let outputMessage = "";
-    let totalTokens;
-    let latencyMs;
-    let model;
-    for (const span of sorted) {
-      const attrs = span.attributes ?? {};
-      const stepType = this.classifySpan(attrs, span.name);
-      if (stepType === "llm_call") {
-        const [stepArgs, stepResult] = this.extractLlmStep(attrs);
-        builder.addLlmCall(span.name, {
-          args: stepArgs,
-          result: stepResult,
-          metadata: this.spanMetadata(span)
-        });
-        const completion = attrs["gen_ai.completion"];
-        if (completion) outputMessage = String(completion);
-        const inputTokens = Number(attrs["gen_ai.usage.input_tokens"] ?? 0);
-        const outputTokens = Number(attrs["gen_ai.usage.output_tokens"] ?? 0);
-        const spanTokens = inputTokens + outputTokens;
-        if (spanTokens > 0) totalTokens = (totalTokens ?? 0) + spanTokens;
-        if (model === void 0) {
-          const responseModel = attrs["gen_ai.response.model"];
-          const requestModel = attrs["gen_ai.request.model"];
-          if (responseModel) model = String(responseModel);
-          else if (requestModel) model = String(requestModel);
-        }
-      } else if (stepType === "tool_call") {
-        const [stepArgs, stepResult] = this.extractToolStep(attrs);
-        const toolName = String(attrs["gen_ai.tool.name"] ?? span.name);
-        builder.addToolCall(toolName, {
-          args: stepArgs,
-          result: stepResult,
-          metadata: this.spanMetadata(span)
-        });
+      if (code === "ENOBUFS") {
+        throw new OTLPConversionError(
+          `attest-engine trace convert printed more than ${MAX_TRACE_OUTPUT_BYTES} bytes`
+        );
       }
+      throw proc.error;
     }
-    if (rootSpan) {
-      const startNs = rootSpan.startTime[0] * 1e9 + rootSpan.startTime[1];
-      const endNs = rootSpan.endTime[0] * 1e9 + rootSpan.endTime[1];
-      latencyMs = Math.floor((endNs - startNs) / 1e6);
+    if (proc.status !== 0) {
+      const message = proc.stderr.toString("utf8").trim();
+      throw new OTLPConversionError(
+        message || `attest-engine trace convert exited ${String(proc.status ?? proc.signal)}`
+      );
     }
-    builder.setOutput({ message: outputMessage });
-    builder.setMetadata({ total_tokens: totalTokens, latency_ms: latencyMs, model });
-    return builder.build();
-  }
-  findRootSpan(spans) {
-    for (const span of spans) {
-      if (!span.parentSpanId) return span;
+    try {
+      return JSON.parse(proc.stdout.toString("utf8"));
+    } catch (err) {
+      throw new OTLPConversionError(
+        `attest-engine trace convert printed output that is not JSON: ${String(err)}`
+      );
     }
-    return spans[0];
   }
-  classifySpan(attrs, name) {
-    const op = String(attrs["gen_ai.operation.name"] ?? "");
-    if (op === "chat" || op === "completion" || op === "generate_content" || "gen_ai.completion" in attrs) {
-      return "llm_call";
-    }
-    if (op === "tool" || "gen_ai.tool.name" in attrs) {
-      return "tool_call";
-    }
-    const lower = name.toLowerCase();
-    if (lower.includes("completion") || lower.includes("chat")) return "llm_call";
-    if (lower.includes("tool")) return "tool_call";
-    return void 0;
+};
+function hrTimeToNanos(time, spanName) {
+  if (!Number.isSafeInteger(time[0]) || !Number.isSafeInteger(time[1])) {
+    throw new RangeError(`span ${JSON.stringify(spanName)} has a non-integer HrTime`);
   }
-  extractLlmStep(attrs) {
-    const args = {};
-    const result = {};
-    if ("gen_ai.request.model" in attrs) args.model = String(attrs["gen_ai.request.model"]);
-    if ("gen_ai.system" in attrs) args.system = String(attrs["gen_ai.system"]);
-    if ("gen_ai.prompt" in attrs) args.prompt = String(attrs["gen_ai.prompt"]);
-    if ("gen_ai.completion" in attrs) result.completion = String(attrs["gen_ai.completion"]);
-    if ("gen_ai.usage.input_tokens" in attrs) result.input_tokens = Number(attrs["gen_ai.usage.input_tokens"]);
-    if ("gen_ai.usage.output_tokens" in attrs) result.output_tokens = Number(attrs["gen_ai.usage.output_tokens"]);
-    if ("gen_ai.response.model" in attrs) result.model = String(attrs["gen_ai.response.model"]);
-    return [args, result];
-  }
-  extractToolStep(attrs) {
-    const args = {};
-    const result = {};
-    if ("gen_ai.tool.call.id" in attrs) args.call_id = String(attrs["gen_ai.tool.call.id"]);
-    if ("gen_ai.tool.parameters" in attrs) args.parameters = attrs["gen_ai.tool.parameters"];
-    if ("gen_ai.tool.output" in attrs) result.output = attrs["gen_ai.tool.output"];
-    return [args, result];
-  }
-  spanMetadata(span) {
-    const startNs = span.startTime[0] * 1e9 + span.startTime[1];
-    const endNs = span.endTime[0] * 1e9 + span.endTime[1];
-    return { duration_ms: Math.floor((endNs - startNs) / 1e6) };
+  return (BigInt(time[0]) * 1000000000n + BigInt(time[1])).toString();
+}
+function anyValue(value) {
+  if (value === null || value === void 0) return {};
+  if (typeof value === "boolean") return { boolValue: value };
+  if (typeof value === "string") return { stringValue: value };
+  if (typeof value === "number") {
+    if (!Number.isFinite(value)) {
+      throw new RangeError(`OTel attribute value must be finite, got ${String(value)}`);
+    }
+    return Number.isSafeInteger(value) ? { intValue: String(value) } : { doubleValue: value };
   }
-};
+  return { arrayValue: { values: value.map((v) => anyValue(v)) } };
+}
+function spansToOtlp(spans) {
+  const encoded = spans.map((span) => {
+    const ctx = span.spanContext();
+    const out = {
+      traceId: ctx.traceId,
+      spanId: ctx.spanId,
+      name: span.name,
+      startTimeUnixNano: hrTimeToNanos(span.startTime, span.name),
+      endTimeUnixNano: hrTimeToNanos(span.endTime, span.name),
+      attributes: Object.entries(span.attributes ?? {}).filter(([, v]) => v !== void 0).map(([key, v]) => ({ key, value: anyValue(v) }))
+    };
+    const parent = span.parentSpanContext?.spanId ?? span.parentSpanId;
+    if (parent) out["parentSpanId"] = parent;
+    if (span.status?.code === STATUS_CODE_ERROR) {
+      out["status"] = { code: STATUS_CODE_ERROR, message: span.status.message ?? "" };
+    }
+    return out;
+  });
+  return { resourceSpans: [{ scopeSpans: [{ spans: encoded }] }] };
+}
 
 // src/adapters/langchain.ts
 var LangChainAdapter = class {
@@ -3021,17 +3315,30 @@ export {
   EngineTimeoutError,
   ExpectChain,
   FRIENDLY_USER,
+  FailureClass,
   GeminiAdapter,
   LangChainAdapter,
   ManualAdapter,
+  OTLPConversionError,
   OTelAdapter,
+  OUTCOME_FAIL,
+  OUTCOME_PASS,
+  OUTCOME_SKIP,
   OllamaAdapter,
   OpenAIAdapter,
   PluginRegistry,
   ProtocolDesyncError,
   ProtocolDiagnosticBuffer,
   ProtocolError,
+  REASON_BUDGET_EXHAUSTED,
+  REASON_JUDGE_ABSTAINED,
+  REASON_MISSING_EVIDENCE,
+  REASON_PROVIDER_NOT_CONFIGURED,
+  REASON_PROVIDER_UNAVAILABLE,
+  REASON_SHORT_CIRCUITED,
+  REASON_SIMULATION,
   STATUS_HARD_FAIL,
+  STATUS_NOT_EVALUATED,
   STATUS_PASS,
   STATUS_SOFT_FAIL,
   STEP_AGENT_CALL,
@@ -3054,6 +3361,7 @@ export {
   TraceTree,
   VERSION,
   activeBuilder,
+  addUsage,
   agent,
   agentId,
   assertionId,
@@ -3066,19 +3374,25 @@ export {
   executePluginAssertion,
   extractId,
   extractResult,
+  formatUsage,
   getSampleRate,
   isSimulationMode,
   layerForType,
   loadLabelsCSV,
   loadLabelsJSONL,
+  parseUsage,
   previewLine,
   promptHash,
   renderAssertionFailure,
   renderDiagnostics,
   renderSummary,
+  requireSimulationCapability,
   resetConfig,
-  simulationEvaluateBatch,
+  simulationEnvEnabled,
+  sumUsage,
   tier,
-  traceId
+  traceId,
+  usageSummaryLine,
+  zeroUsage
 };
 //# sourceMappingURL=index.js.map