@tangle-network/agent-eval 0.120.2 → 0.120.4

package/dist/chunk-K6N6XJJX.js

@@ -0,0 +1,306 @@
+import {
+  TraceEmitter
+} from "./chunk-VQMK5FMP.js";
+import {
+  ConfigError
+} from "./chunk-ONWEPEDO.js";
+
+// src/sandbox-harness.ts
+var vitestTestParser = {
+  id: "vitest",
+  parse(stdout) {
+    const m = stdout.match(/Tests\s+(\d+)\s+(passed|failed)(?:\s*\|\s*(\d+)\s+(passed|failed))?/i);
+    if (!m) return void 0;
+    let passed = 0;
+    let failed = 0;
+    const a = parseInt(m[1], 10);
+    const aLabel = m[2].toLowerCase();
+    if (aLabel === "passed") passed += a;
+    else failed += a;
+    if (m[3] && m[4]) {
+      const b = parseInt(m[3], 10);
+      if (m[4].toLowerCase() === "passed") passed += b;
+      else failed += b;
+    }
+    return { testsTotal: passed + failed, testsPassed: passed };
+  }
+};
+var pytestTestParser = {
+  id: "pytest",
+  parse(stdout) {
+    const total = stdout.match(/collected\s+(\d+)\s+items?/i);
+    const passed = stdout.match(/(\d+)\s+passed/);
+    if (!total || !passed) return void 0;
+    return { testsTotal: parseInt(total[1], 10), testsPassed: parseInt(passed[1], 10) };
+  }
+};
+var jestTestParser = {
+  id: "jest",
+  parse(stdout) {
+    const m = stdout.match(/Tests:\s+(?:(\d+)\s+failed[^,]*,\s*)?(\d+)\s+passed,\s+(\d+)\s+total/i);
+    if (!m) return void 0;
+    return { testsTotal: parseInt(m[3], 10), testsPassed: parseInt(m[2], 10) };
+  }
+};
+function composeParsers(...parsers) {
+  return {
+    id: parsers.map((p) => p.id).join("|"),
+    parse(stdout, stderr, exitCode) {
+      for (const p of parsers) {
+        const res = p.parse(stdout, stderr, exitCode);
+        if (res) return res;
+      }
+      return void 0;
+    }
+  };
+}
+var SubprocessSandboxDriver = class {
+  id = "subprocess";
+  defaultCwd;
+  defaultEnv;
+  constructor(options = {}) {
+    this.defaultCwd = options.cwd;
+    this.defaultEnv = options.env;
+  }
+  async exec(phase, command, config) {
+    const { spawn } = await import("child_process");
+    const start = Date.now();
+    const effectiveCwd = config.cwd ?? this.defaultCwd;
+    const effectiveEnv = { ...process.env, ...this.defaultEnv ?? {}, ...config.env ?? {} };
+    const maxOutputBytes = config.maxOutputBytes ?? 16 * 1024 * 1024;
+    return await new Promise((resolve) => {
+      const child = spawn(command, {
+        shell: true,
+        cwd: effectiveCwd,
+        env: effectiveEnv,
+        // Own process group so a timeout can SIGKILL the whole tree, not
+        // just the shell — otherwise a runaway grandchild keeps the pipes
+        // open and `close` never fires (the timeout silently does nothing).
+        detached: process.platform !== "win32"
+      });
+      let stdout = "";
+      let stderr = "";
+      let outputBytes = 0;
+      let outputTruncated = false;
+      let killedByTimeout = false;
+      let settled = false;
+      const onStdout = (d) => {
+        const chunk = capture(String(d));
+        if (chunk) stdout += chunk;
+      };
+      const onStderr = (d) => {
+        const chunk = capture(String(d));
+        if (chunk) stderr += chunk;
+      };
+      function capture(s) {
+        if (outputBytes >= maxOutputBytes) {
+          outputTruncated = true;
+          return "";
+        }
+        const room = maxOutputBytes - outputBytes;
+        if (s.length > room) {
+          outputBytes = maxOutputBytes;
+          outputTruncated = true;
+          return s.slice(0, room);
+        }
+        outputBytes += s.length;
+        return s;
+      }
+      child.stdout?.on("data", onStdout);
+      child.stderr?.on("data", onStderr);
+      const cleanup = () => {
+        clearTimeout(timeout);
+        if (forceResolve) clearTimeout(forceResolve);
+        child.stdout?.off("data", onStdout);
+        child.stderr?.off("data", onStderr);
+        child.removeAllListeners("close");
+        child.removeAllListeners("error");
+      };
+      const finish = (outcome) => {
+        if (settled) return;
+        settled = true;
+        cleanup();
+        const wallMs = Date.now() - start;
+        const exitCode = killedByTimeout ? outcome.code && outcome.code !== 0 ? outcome.code : 124 : outcome.code ?? 1;
+        const parsed = !killedByTimeout && phase === "test" && config.testParser ? config.testParser.parse(stdout, stderr, exitCode) : void 0;
+        resolve({
+          phase,
+          exitCode,
+          stdout,
+          stderr: outcome.runnerError ? stderr + outcome.runnerError : stderr,
+          wallMs,
+          testsTotal: parsed?.testsTotal,
+          testsPassed: parsed?.testsPassed,
+          killedByTimeout: killedByTimeout || void 0,
+          outputTruncated: outputTruncated || void 0
+        });
+      };
+      const killTree = () => {
+        try {
+          if (process.platform !== "win32" && typeof child.pid === "number") {
+            process.kill(-child.pid, "SIGKILL");
+          } else {
+            child.kill("SIGKILL");
+          }
+        } catch (err) {
+          console.warn("[sandbox-harness] SIGKILL on timeout failed:", err);
+        }
+      };
+      let forceResolve;
+      const timeout = setTimeout(
+        () => {
+          killedByTimeout = true;
+          killTree();
+          forceResolve = setTimeout(() => finish({ code: null }), 2e3);
+        },
+        config.timeoutMs ?? 10 * 6e4
+      );
+      child.on("close", (code) => {
+        if (forceResolve) clearTimeout(forceResolve);
+        finish({ code });
+      });
+      child.on("error", (err) => {
+        if (forceResolve) clearTimeout(forceResolve);
+        finish({ code: 127, runnerError: String(err) });
+      });
+    });
+  }
+};
+var DockerSandboxDriver = class {
+  id = "docker";
+  async exec(phase, command, config) {
+    if (!config.image) throw new ConfigError("DockerSandboxDriver requires config.image");
+    const sub = new SubprocessSandboxDriver();
+    const envArgs = Object.entries(config.env ?? {}).map(([k, v]) => `-e ${shellQuote(k)}=${shellQuote(v)}`).join(" ");
+    const wrapped = `docker run --rm ${envArgs} ${shellQuote(config.image)} sh -c ${shellQuote(command)}`;
+    return sub.exec(phase, wrapped, { ...config, env: void 0 });
+  }
+};
+function shellQuote(v) {
+  if (/^[A-Za-z0-9_\-/.@:=]+$/.test(v)) return v;
+  return `'${v.replace(/'/g, `'\\''`)}'`;
+}
+var SandboxHarness = class {
+  driver;
+  constructor(driver = new SubprocessSandboxDriver()) {
+    this.driver = driver;
+  }
+  async run(config, emitter) {
+    const handle = await emitter.sandbox({
+      name: `sandbox(${this.driver.id})`,
+      image: config.image,
+      command: [config.setupCommand, config.runCommand, config.testCommand].filter(Boolean).join(" && ")
+    });
+    const result = { passed: false, totalWallMs: 0, score: 0 };
+    try {
+      if (config.setupCommand) {
+        result.setup = await this.driver.exec("setup", config.setupCommand, config);
+        result.totalWallMs += result.setup.wallMs;
+        if (result.setup.killedByTimeout || result.setup.exitCode !== 0) {
+          const reason = result.setup.killedByTimeout ? `setup timed out after ${result.setup.wallMs}ms` : `setup failed (exit ${result.setup.exitCode})`;
+          await handle.fail(reason, {
+            exitCode: result.setup.exitCode,
+            wallMs: result.totalWallMs
+          });
+          return result;
+        }
+      }
+      if (config.runCommand) {
+        result.run = await this.driver.exec("run", config.runCommand, config);
+        result.totalWallMs += result.run.wallMs;
+        if (result.run.killedByTimeout || result.run.exitCode !== 0) {
+          const reason = result.run.killedByTimeout ? `run timed out after ${result.run.wallMs}ms` : `run failed (exit ${result.run.exitCode})`;
+          await handle.fail(reason, {
+            exitCode: result.run.exitCode,
+            wallMs: result.totalWallMs
+          });
+          return result;
+        }
+      }
+      if (config.testCommand) {
+        result.test = await this.driver.exec("test", config.testCommand, config);
+        result.totalWallMs += result.test.wallMs;
+        const passed = !result.test.killedByTimeout && result.test.exitCode === 0;
+        result.passed = passed;
+        if (result.test.testsTotal !== void 0 && result.test.testsTotal > 0) {
+          result.score = (result.test.testsPassed ?? 0) / result.test.testsTotal;
+        } else {
+          result.score = passed ? 1 : 0;
+        }
+        await handle.end({
+          exitCode: result.test.exitCode,
+          testsTotal: result.test.testsTotal,
+          testsPassed: result.test.testsPassed,
+          wallMs: result.totalWallMs,
+          status: passed ? "ok" : "error"
+        });
+      } else {
+        result.passed = true;
+        result.score = 1;
+        await handle.end({ wallMs: result.totalWallMs });
+      }
+    } catch (err) {
+      await handle.fail(err instanceof Error ? err : String(err));
+      throw err;
+    }
+    return result;
+  }
+};
+
+// src/test-graded-scenario.ts
+async function runTestGradedScenario(scenario, store, options = {}) {
+  const emitter = new TraceEmitter(store);
+  await emitter.startRun({
+    scenarioId: scenario.id,
+    variantId: options.variantId,
+    datasetVersion: scenario.datasetVersion,
+    tags: scenario.tags,
+    ...options.provenance
+  });
+  const harness = new SandboxHarness(options.driver);
+  const result = await harness.run(scenario.harness, emitter);
+  const threshold = scenario.passThreshold ?? 1;
+  const pass = result.passed && result.score >= threshold;
+  const setupFailed = result.setup !== void 0 && result.setup.exitCode !== 0;
+  const runFailed = result.run !== void 0 && result.run.exitCode !== 0;
+  const testFailed = result.test !== void 0 && result.test.exitCode !== 0;
+  const failureClass = pass ? "success" : setupFailed || runFailed ? "sandbox_failure" : testFailed ? "format_drift" : "unknown";
+  await emitter.endRun({
+    pass,
+    score: result.score,
+    failureClass,
+    notes: pass ? void 0 : reasonForFailure(result)
+  });
+  return {
+    runId: emitter.runId,
+    scenario,
+    harness: result,
+    pass,
+    score: result.score,
+    failureClass
+  };
+}
+function reasonForFailure(result) {
+  if (result.setup && result.setup.exitCode !== 0)
+    return `setup failed: exit ${result.setup.exitCode}`;
+  if (result.run && result.run.exitCode !== 0) return `run failed: exit ${result.run.exitCode}`;
+  if (result.test) {
+    if (result.test.testsTotal !== void 0) {
+      return `tests: ${result.test.testsPassed ?? 0}/${result.test.testsTotal}`;
+    }
+    return `test exit ${result.test.exitCode}`;
+  }
+  return "no test command";
+}
+
+export {
+  vitestTestParser,
+  pytestTestParser,
+  jestTestParser,
+  composeParsers,
+  SubprocessSandboxDriver,
+  DockerSandboxDriver,
+  SandboxHarness,
+  runTestGradedScenario
+};
+//# sourceMappingURL=chunk-K6N6XJJX.js.map

package/dist/chunk-K6N6XJJX.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/sandbox-harness.ts","../src/test-graded-scenario.ts"],"sourcesContent":["/**\n * SandboxHarness — executes a scenario in an isolated environment and\n * emits a rich SandboxSpan into the trace.\n *\n * Two built-in drivers:\n *   - `SubprocessSandboxDriver` — spawn in a local cwd with env vars.\n *     Fast, no dependencies, fine for unit tests and most CI gates.\n *   - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;\n *     shells out to `docker run`. Stronger isolation, slower startup.\n *\n * Consumers implement `SandboxDriver` for custom backends (Firecracker,\n * Cloudflare sandbox product, etc.). The harness doesn't care which.\n */\n\nimport { ConfigError } from './errors'\nimport type { TraceEmitter } from './trace/emitter'\nimport type { SandboxSpan } from './trace/schema'\n\nexport interface HarnessConfig {\n  /** Setup command (e.g. \"pnpm install\"). Non-zero exit fails the run. */\n  setupCommand?: string\n  /** Run command (e.g. \"pnpm build\"). */\n  runCommand?: string\n  /** Test command (e.g. \"pnpm test --run\"). Drives the test count + pass count. */\n  testCommand?: string\n  /** Absolute cwd for the subprocess driver. Ignored by docker driver. */\n  cwd?: string\n  /** Max wall-clock per phase in ms. Default 10 minutes. */\n  timeoutMs?: number\n  /**\n   * Cap on captured stdout+stderr bytes per phase. A runaway process can\n   * otherwise grow the in-memory buffer without bound. Once hit, further\n   * output is dropped and `outputTruncated` is set. Default 16 MiB.\n   */\n  maxOutputBytes?: number\n  /** Image for the docker driver. */\n  image?: string\n  /** Extra env vars (validated; shell-escaped). */\n  env?: Record<string, string>\n  /** Parser for the test output — maps stdout/stderr/exit code → pass count. */\n  testParser?: TestOutputParser\n}\n\nexport interface TestOutputParser {\n  id: string\n  parse(\n    stdout: string,\n    stderr: string,\n    exitCode: number,\n  ): { testsTotal: number; testsPassed: number } | undefined\n}\n\nexport interface SandboxResult {\n  phase: 'setup' | 'run' | 'test'\n  exitCode: number\n  stdout: string\n  stderr: string\n  wallMs: number\n  testsTotal?: number\n  testsPassed?: number\n  /**\n   * True when the process was killed because it exceeded `timeoutMs`. A\n   * SIGKILLed child can still close with exit code 0; callers MUST treat\n   * a timed-out phase as a hard failure regardless of `exitCode`, never\n   * as a pass. `undefined`/`false` means the process completed on its own.\n   */\n  killedByTimeout?: boolean\n  /**\n   * True when captured stdout/stderr hit `maxOutputBytes` and further\n   * output was dropped. The result is still returned (the process was\n   * not killed for this), but downstream parsers see truncated text.\n   */\n  outputTruncated?: boolean\n}\n\nexport interface SandboxDriver {\n  id: string\n  exec(\n    phase: SandboxResult['phase'],\n    command: string,\n    config: HarnessConfig,\n  ): Promise<SandboxResult>\n}\n\n// ── Parsers ──────────────────────────────────────────────────────────\n\n/** Vitest default summary line: \"Tests  X passed | Y failed\". */\nexport const vitestTestParser: TestOutputParser = {\n  id: 'vitest',\n  parse(stdout) {\n    const m = stdout.match(/Tests\\s+(\\d+)\\s+(passed|failed)(?:\\s*\\|\\s*(\\d+)\\s+(passed|failed))?/i)\n    if (!m) return undefined\n    let passed = 0\n    let failed = 0\n    const a = parseInt(m[1]!, 10)\n    const aLabel = m[2]!.toLowerCase()\n    if (aLabel === 'passed') passed += a\n    else failed += a\n    if (m[3] && m[4]) {\n      const b = parseInt(m[3], 10)\n      if (m[4].toLowerCase() === 'passed') passed += b\n      else failed += b\n    }\n    return { testsTotal: passed + failed, testsPassed: passed }\n  },\n}\n\n/** Pytest default: \"collected N items\" + \" X passed, Y failed\". */\nexport const pytestTestParser: TestOutputParser = {\n  id: 'pytest',\n  parse(stdout) {\n    const total = stdout.match(/collected\\s+(\\d+)\\s+items?/i)\n    const passed = stdout.match(/(\\d+)\\s+passed/)\n    if (!total || !passed) return undefined\n    return { testsTotal: parseInt(total[1]!, 10), testsPassed: parseInt(passed[1]!, 10) }\n  },\n}\n\n/** Jest: \"Tests: X passed, Y total\" (and optional failed). */\nexport const jestTestParser: TestOutputParser = {\n  id: 'jest',\n  parse(stdout) {\n    const m = stdout.match(/Tests:\\s+(?:(\\d+)\\s+failed[^,]*,\\s*)?(\\d+)\\s+passed,\\s+(\\d+)\\s+total/i)\n    if (!m) return undefined\n    return { testsTotal: parseInt(m[3]!, 10), testsPassed: parseInt(m[2]!, 10) }\n  },\n}\n\n/** Composite parser — tries a list of parsers in order. */\nexport function composeParsers(...parsers: TestOutputParser[]): TestOutputParser {\n  return {\n    id: parsers.map((p) => p.id).join('|'),\n    parse(stdout, stderr, exitCode) {\n      for (const p of parsers) {\n        const res = p.parse(stdout, stderr, exitCode)\n        if (res) return res\n      }\n      return undefined\n    },\n  }\n}\n\n// ── Drivers ──────────────────────────────────────────────────────────\n\nexport interface SubprocessSandboxDriverOptions {\n  /**\n   * Default cwd for all `exec` calls. Used when the per-call `HarnessConfig`\n   * does not set its own `cwd`. Lets callers bind the driver to a working\n   * directory once instead of spreading cwd into every harness config —\n   * useful when the harness config is constructed far from the call site\n   * (e.g. starter-foundry's promoter passes a static HarnessConfig per\n   * family taxonomy but needs a per-run composed-scaffold cwd).\n   */\n  cwd?: string\n  /**\n   * Default env merged into every `exec` call's env (per-call `HarnessConfig.env`\n   * still wins on key collision). Same ergonomic rationale as `cwd` above.\n   */\n  env?: Record<string, string>\n}\n\nexport class SubprocessSandboxDriver implements SandboxDriver {\n  id = 'subprocess'\n  private defaultCwd?: string\n  private defaultEnv?: Record<string, string>\n\n  constructor(options: SubprocessSandboxDriverOptions = {}) {\n    this.defaultCwd = options.cwd\n    this.defaultEnv = options.env\n  }\n\n  async exec(\n    phase: SandboxResult['phase'],\n    command: string,\n    config: HarnessConfig,\n  ): Promise<SandboxResult> {\n    const { spawn } = await import('node:child_process')\n    const start = Date.now()\n    // Per-call config wins; fall back to constructor defaults. Honoring\n    // the constructor `cwd` keeps the subprocess from inheriting Node's\n    // cwd when only the constructor arg is supplied.\n    const effectiveCwd = config.cwd ?? this.defaultCwd\n    const effectiveEnv = { ...process.env, ...(this.defaultEnv ?? {}), ...(config.env ?? {}) }\n    const maxOutputBytes = config.maxOutputBytes ?? 16 * 1024 * 1024\n    return await new Promise<SandboxResult>((resolve) => {\n      const child = spawn(command, {\n        shell: true,\n        cwd: effectiveCwd,\n        env: effectiveEnv,\n        // Own process group so a timeout can SIGKILL the whole tree, not\n        // just the shell — otherwise a runaway grandchild keeps the pipes\n        // open and `close` never fires (the timeout silently does nothing).\n        detached: process.platform !== 'win32',\n      })\n      let stdout = ''\n      let stderr = ''\n      let outputBytes = 0\n      let outputTruncated = false\n      let killedByTimeout = false\n      let settled = false\n\n      const onStdout = (d: unknown) => {\n        const chunk = capture(String(d))\n        if (chunk) stdout += chunk\n      }\n      const onStderr = (d: unknown) => {\n        const chunk = capture(String(d))\n        if (chunk) stderr += chunk\n      }\n      // Bound the in-memory buffer so a runaway process can't OOM the\n      // harness. Once the cap is hit we keep draining the streams (to let\n      // the child reach `close`) but discard the bytes.\n      function capture(s: string): string {\n        if (outputBytes >= maxOutputBytes) {\n          outputTruncated = true\n          return ''\n        }\n        const room = maxOutputBytes - outputBytes\n        if (s.length > room) {\n          outputBytes = maxOutputBytes\n          outputTruncated = true\n          return s.slice(0, room)\n        }\n        outputBytes += s.length\n        return s\n      }\n\n      child.stdout?.on('data', onStdout)\n      child.stderr?.on('data', onStderr)\n\n      const cleanup = () => {\n        clearTimeout(timeout)\n        if (forceResolve) clearTimeout(forceResolve)\n        child.stdout?.off('data', onStdout)\n        child.stderr?.off('data', onStderr)\n        child.removeAllListeners('close')\n        child.removeAllListeners('error')\n      }\n\n      // Resolve exactly once. `code`/`signal` come from `close`; on the\n      // timeout path we force a non-zero exit so a SIGKILLed child that\n      // reports exit 0 can never read as a clean pass downstream.\n      const finish = (outcome: { code: number | null; runnerError?: string }) => {\n        if (settled) return\n        settled = true\n        cleanup()\n        const wallMs = Date.now() - start\n        const exitCode = killedByTimeout\n          ? outcome.code && outcome.code !== 0\n            ? outcome.code\n            : 124\n          : (outcome.code ?? 1)\n        const parsed =\n          !killedByTimeout && phase === 'test' && config.testParser\n            ? config.testParser.parse(stdout, stderr, exitCode)\n            : undefined\n        resolve({\n          phase,\n          exitCode,\n          stdout,\n          stderr: outcome.runnerError ? stderr + outcome.runnerError : stderr,\n          wallMs,\n          testsTotal: parsed?.testsTotal,\n          testsPassed: parsed?.testsPassed,\n          killedByTimeout: killedByTimeout || undefined,\n          outputTruncated: outputTruncated || undefined,\n        })\n      }\n\n      // Kill the whole process group on win32-less platforms (we spawned\n      // detached). On a clean close this is a no-op.\n      const killTree = () => {\n        try {\n          if (process.platform !== 'win32' && typeof child.pid === 'number') {\n            process.kill(-child.pid, 'SIGKILL')\n          } else {\n            child.kill('SIGKILL')\n          }\n        } catch (err) {\n          console.warn('[sandbox-harness] SIGKILL on timeout failed:', err)\n        }\n      }\n\n      let forceResolve: ReturnType<typeof setTimeout> | undefined\n      const timeout = setTimeout(\n        () => {\n          killedByTimeout = true\n          killTree()\n          // Give `close` a brief window to fire (flushing any final output)\n          // after the group dies; if an orphaned grandchild keeps the pipes\n          // open, force-resolve so the harness can't hang on a runaway.\n          forceResolve = setTimeout(() => finish({ code: null }), 2000)\n        },\n        config.timeoutMs ?? 10 * 60_000,\n      )\n\n      child.on('close', (code) => {\n        if (forceResolve) clearTimeout(forceResolve)\n        finish({ code })\n      })\n      child.on('error', (err) => {\n        if (forceResolve) clearTimeout(forceResolve)\n        finish({ code: 127, runnerError: String(err) })\n      })\n    })\n  }\n}\n\nexport class DockerSandboxDriver implements SandboxDriver {\n  id = 'docker'\n\n  async exec(\n    phase: SandboxResult['phase'],\n    command: string,\n    config: HarnessConfig,\n  ): Promise<SandboxResult> {\n    if (!config.image) throw new ConfigError('DockerSandboxDriver requires config.image')\n    const sub = new SubprocessSandboxDriver()\n    const envArgs = Object.entries(config.env ?? {})\n      .map(([k, v]) => `-e ${shellQuote(k)}=${shellQuote(v)}`)\n      .join(' ')\n    const wrapped = `docker run --rm ${envArgs} ${shellQuote(config.image)} sh -c ${shellQuote(command)}`\n    return sub.exec(phase, wrapped, { ...config, env: undefined })\n  }\n}\n\nfunction shellQuote(v: string): string {\n  if (/^[A-Za-z0-9_\\-/.@:=]+$/.test(v)) return v\n  return `'${v.replace(/'/g, `'\\\\''`)}'`\n}\n\n// ── Harness orchestration ────────────────────────────────────────────\n\nexport interface SandboxHarnessResult {\n  passed: boolean\n  setup?: SandboxResult\n  run?: SandboxResult\n  test?: SandboxResult\n  totalWallMs: number\n  /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */\n  score: number\n}\n\nexport class SandboxHarness {\n  private driver: SandboxDriver\n  constructor(driver: SandboxDriver = new SubprocessSandboxDriver()) {\n    this.driver = driver\n  }\n\n  async run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult> {\n    const handle = await emitter.sandbox({\n      name: `sandbox(${this.driver.id})`,\n      image: config.image,\n      command: [config.setupCommand, config.runCommand, config.testCommand]\n        .filter(Boolean)\n        .join(' && '),\n    })\n    const result: SandboxHarnessResult = { passed: false, totalWallMs: 0, score: 0 }\n    try {\n      if (config.setupCommand) {\n        result.setup = await this.driver.exec('setup', config.setupCommand, config)\n        result.totalWallMs += result.setup.wallMs\n        if (result.setup.killedByTimeout || result.setup.exitCode !== 0) {\n          const reason = result.setup.killedByTimeout\n            ? `setup timed out after ${result.setup.wallMs}ms`\n            : `setup failed (exit ${result.setup.exitCode})`\n          await handle.fail(reason, {\n            exitCode: result.setup.exitCode,\n            wallMs: result.totalWallMs,\n          } as Partial<SandboxSpan>)\n          return result\n        }\n      }\n      if (config.runCommand) {\n        result.run = await this.driver.exec('run', config.runCommand, config)\n        result.totalWallMs += result.run.wallMs\n        if (result.run.killedByTimeout || result.run.exitCode !== 0) {\n          const reason = result.run.killedByTimeout\n            ? `run timed out after ${result.run.wallMs}ms`\n            : `run failed (exit ${result.run.exitCode})`\n          await handle.fail(reason, {\n            exitCode: result.run.exitCode,\n            wallMs: result.totalWallMs,\n          } as Partial<SandboxSpan>)\n          return result\n        }\n      }\n      if (config.testCommand) {\n        result.test = await this.driver.exec('test', config.testCommand, config)\n        result.totalWallMs += result.test.wallMs\n        // A timed-out test phase is a hard fail regardless of the exit code\n        // a SIGKILLed child happens to report — never let it score as a pass.\n        const passed = !result.test.killedByTimeout && result.test.exitCode === 0\n        result.passed = passed\n        if (result.test.testsTotal !== undefined && result.test.testsTotal > 0) {\n          result.score = (result.test.testsPassed ?? 0) / result.test.testsTotal\n        } else {\n          result.score = passed ? 1 : 0\n        }\n        await handle.end({\n          exitCode: result.test.exitCode,\n          testsTotal: result.test.testsTotal,\n          testsPassed: result.test.testsPassed,\n          wallMs: result.totalWallMs,\n          status: passed ? 'ok' : 'error',\n        } as Partial<SandboxSpan>)\n      } else {\n        result.passed = true\n        result.score = 1\n        await handle.end({ wallMs: result.totalWallMs } as Partial<SandboxSpan>)\n      }\n    } catch (err) {\n      await handle.fail(err instanceof Error ? err : String(err))\n      throw err\n    }\n    return result\n  }\n}\n","/**\n * TestGradedScenario — a scenario whose score comes from a test suite.\n *\n * This is the SWE-bench pattern generalized. The scenario ships:\n *   - fixture data (setup instructions)\n *   - a test command the harness runs\n *   - optional assertion overrides\n *\n * The runner emits a run, delegates to SandboxHarness, records the\n * outcome, and returns a structured verdict. Consumers bind their own\n * agent execution to this contract.\n */\n\nimport type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from './sandbox-harness'\nimport { SandboxHarness } from './sandbox-harness'\nimport { TraceEmitter } from './trace/emitter'\nimport type { FailureClass, Run } from './trace/schema'\nimport type { TraceStore } from './trace/store'\n\nexport interface TestGradedScenario {\n  id: string\n  description?: string\n  harness: HarnessConfig\n  /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */\n  passThreshold?: number\n  /** Provenance for dataset tracking. */\n  datasetVersion?: string\n  /** Free-form tags (difficulty, category, etc.). */\n  tags?: Record<string, string>\n}\n\nexport interface TestGradedRunOptions {\n  variantId?: string\n  driver?: SandboxDriver\n  /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */\n  provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>\n}\n\nexport interface TestGradedRunResult {\n  runId: string\n  scenario: TestGradedScenario\n  harness: SandboxHarnessResult\n  pass: boolean\n  score: number\n  failureClass?: FailureClass\n}\n\nexport async function runTestGradedScenario(\n  scenario: TestGradedScenario,\n  store: TraceStore,\n  options: TestGradedRunOptions = {},\n): Promise<TestGradedRunResult> {\n  const emitter = new TraceEmitter(store)\n  await emitter.startRun({\n    scenarioId: scenario.id,\n    variantId: options.variantId,\n    datasetVersion: scenario.datasetVersion,\n    tags: scenario.tags,\n    ...options.provenance,\n  })\n  const harness = new SandboxHarness(options.driver)\n  const result = await harness.run(scenario.harness, emitter)\n  const threshold = scenario.passThreshold ?? 1.0\n  const pass = result.passed && result.score >= threshold\n  const setupFailed = result.setup !== undefined && result.setup.exitCode !== 0\n  const runFailed = result.run !== undefined && result.run.exitCode !== 0\n  const testFailed = result.test !== undefined && result.test.exitCode !== 0\n  const failureClass: FailureClass | undefined = pass\n    ? 'success'\n    : setupFailed || runFailed\n      ? 'sandbox_failure'\n      : testFailed\n        ? 'format_drift'\n        : 'unknown'\n  await emitter.endRun({\n    pass,\n    score: result.score,\n    failureClass,\n    notes: pass ? undefined : reasonForFailure(result),\n  })\n  return {\n    runId: emitter.runId,\n    scenario,\n    harness: result,\n    pass,\n    score: result.score,\n    failureClass,\n  }\n}\n\nfunction reasonForFailure(result: SandboxHarnessResult): string {\n  if (result.setup && result.setup.exitCode !== 0)\n    return `setup failed: exit ${result.setup.exitCode}`\n  if (result.run && result.run.exitCode !== 0) return `run failed: exit ${result.run.exitCode}`\n  if (result.test) {\n    if (result.test.testsTotal !== undefined) {\n      return `tests: ${result.test.testsPassed ?? 0}/${result.test.testsTotal}`\n    }\n    return `test exit ${result.test.exitCode}`\n  }\n  return 'no test command'\n}\n"],"mappings":";;;;;;;;AAuFO,IAAM,mBAAqC;AAAA,EAChD,IAAI;AAAA,EACJ,MAAM,QAAQ;AACZ,UAAM,IAAI,OAAO,MAAM,sEAAsE;AAC7F,QAAI,CAAC,EAAG,QAAO;AACf,QAAI,SAAS;AACb,QAAI,SAAS;AACb,UAAM,IAAI,SAAS,EAAE,CAAC,GAAI,EAAE;AAC5B,UAAM,SAAS,EAAE,CAAC,EAAG,YAAY;AACjC,QAAI,WAAW,SAAU,WAAU;AAAA,QAC9B,WAAU;AACf,QAAI,EAAE,CAAC,KAAK,EAAE,CAAC,GAAG;AAChB,YAAM,IAAI,SAAS,EAAE,CAAC,GAAG,EAAE;AAC3B,UAAI,EAAE,CAAC,EAAE,YAAY,MAAM,SAAU,WAAU;AAAA,UAC1C,WAAU;AAAA,IACjB;AACA,WAAO,EAAE,YAAY,SAAS,QAAQ,aAAa,OAAO;AAAA,EAC5D;AACF;AAGO,IAAM,mBAAqC;AAAA,EAChD,IAAI;AAAA,EACJ,MAAM,QAAQ;AACZ,UAAM,QAAQ,OAAO,MAAM,6BAA6B;AACxD,UAAM,SAAS,OAAO,MAAM,gBAAgB;AAC5C,QAAI,CAAC,SAAS,CAAC,OAAQ,QAAO;AAC9B,WAAO,EAAE,YAAY,SAAS,MAAM,CAAC,GAAI,EAAE,GAAG,aAAa,SAAS,OAAO,CAAC,GAAI,EAAE,EAAE;AAAA,EACtF;AACF;AAGO,IAAM,iBAAmC;AAAA,EAC9C,IAAI;AAAA,EACJ,MAAM,QAAQ;AACZ,UAAM,IAAI,OAAO,MAAM,uEAAuE;AAC9F,QAAI,CAAC,EAAG,QAAO;AACf,WAAO,EAAE,YAAY,SAAS,EAAE,CAAC,GAAI,EAAE,GAAG,aAAa,SAAS,EAAE,CAAC,GAAI,EAAE,EAAE;AAAA,EAC7E;AACF;AAGO,SAAS,kBAAkB,SAA+C;AAC/E,SAAO;AAAA,IACL,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,KAAK,GAAG;AAAA,IACrC,MAAM,QAAQ,QAAQ,UAAU;AAC9B,iBAAW,KAAK,SAAS;AACvB,cAAM,MAAM,EAAE,MAAM,QAAQ,QAAQ,QAAQ;AAC5C,YAAI,IAAK,QAAO;AAAA,MAClB;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAqBO,IAAM,0BAAN,MAAuD;AAAA,EAC5D,KAAK;AAAA,EACG;AAAA,EACA;AAAA,EAER,YAAY,UAA0C,CAAC,GAAG;AACxD,SAAK,aAAa,QAAQ;AAC1B,SAAK,aAAa,QAAQ;AAAA,EAC5B;AAAA,EAEA,MAAM,KACJ,OACA,SACA,QACwB;AACxB,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,eAAoB;AACnD,UAAM,QAAQ,KAAK,IAAI;AAIvB,UAAM,eAAe,OAAO,OAAO,KAAK;AACxC,UAAM,eAAe,EAAE,GAAG,QAAQ,KAAK,GAAI,KAAK,cAAc,CAAC,GAAI,GAAI,OAAO,OAAO,CAAC,EAAG;AACzF,UAAM,iBAAiB,OAAO,kBAAkB,KAAK,OAAO;AAC5D,WAAO,MAAM,IAAI,QAAuB,CAAC,YAAY;AACnD,YAAM,QAAQ,MAAM,SAAS;AAAA,QAC3B,OAAO;AAAA,QACP,KAAK;AAAA,QACL,KAAK;AAAA;AAAA;AAAA;AAAA,QAIL,UAAU,QAAQ,aAAa;AAAA,MACjC,CAAC;AACD,UAAI,SAAS;AACb,UAAI,SAAS;AACb,UAAI,cAAc;AAClB,UAAI,kBAAkB;AACtB,UAAI,kBAAkB;AACtB,UAAI,UAAU;AAEd,YAAM,WAAW,CAAC,MAAe;AAC/B,cAAM,QAAQ,QAAQ,OAAO,CAAC,CAAC;AAC/B,YAAI,MAAO,WAAU;AAAA,MACvB;AACA,YAAM,WAAW,CAAC,MAAe;AAC/B,cAAM,QAAQ,QAAQ,OAAO,CAAC,CAAC;AAC/B,YAAI,MAAO,WAAU;AAAA,MACvB;AAIA,eAAS,QAAQ,GAAmB;AAClC,YAAI,eAAe,gBAAgB;AACjC,4BAAkB;AAClB,iBAAO;AAAA,QACT;AACA,cAAM,OAAO,iBAAiB;AAC9B,YAAI,EAAE,SAAS,MAAM;AACnB,wBAAc;AACd,4BAAkB;AAClB,iBAAO,EAAE,MAAM,GAAG,IAAI;AAAA,QACxB;AACA,uBAAe,EAAE;AACjB,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,GAAG,QAAQ,QAAQ;AACjC,YAAM,QAAQ,GAAG,QAAQ,QAAQ;AAEjC,YAAM,UAAU,MAAM;AACpB,qBAAa,OAAO;AACpB,YAAI,aAAc,cAAa,YAAY;AAC3C,cAAM,QAAQ,IAAI,QAAQ,QAAQ;AAClC,cAAM,QAAQ,IAAI,QAAQ,QAAQ;AAClC,cAAM,mBAAmB,OAAO;AAChC,cAAM,mBAAmB,OAAO;AAAA,MAClC;AAKA,YAAM,SAAS,CAAC,YAA2D;AACzE,YAAI,QAAS;AACb,kBAAU;AACV,gBAAQ;AACR,cAAM,SAAS,KAAK,IAAI,IAAI;AAC5B,cAAM,WAAW,kBACb,QAAQ,QAAQ,QAAQ,SAAS,IAC/B,QAAQ,OACR,MACD,QAAQ,QAAQ;AACrB,cAAM,SACJ,CAAC,mBAAmB,UAAU,UAAU,OAAO,aAC3C,OAAO,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAChD;AACN,gBAAQ;AAAA,UACN;AAAA,UACA;AAAA,UACA;AAAA,UACA,QAAQ,QAAQ,cAAc,SAAS,QAAQ,cAAc;AAAA,UAC7D;AAAA,UACA,YAAY,QAAQ;AAAA,UACpB,aAAa,QAAQ;AAAA,UACrB,iBAAiB,mBAAmB;AAAA,UACpC,iBAAiB,mBAAmB;AAAA,QACtC,CAAC;AAAA,MACH;AAIA,YAAM,WAAW,MAAM;AACrB,YAAI;AACF,cAAI,QAAQ,aAAa,WAAW,OAAO,MAAM,QAAQ,UAAU;AACjE,oBAAQ,KAAK,CAAC,MAAM,KAAK,SAAS;AAAA,UACpC,OAAO;AACL,kBAAM,KAAK,SAAS;AAAA,UACtB;AAAA,QACF,SAAS,KAAK;AACZ,kBAAQ,KAAK,gDAAgD,GAAG;AAAA,QAClE;AAAA,MACF;AAEA,UAAI;AACJ,YAAM,UAAU;AAAA,QACd,MAAM;AACJ,4BAAkB;AAClB,mBAAS;AAIT,yBAAe,WAAW,MAAM,OAAO,EAAE,MAAM,KAAK,CAAC,GAAG,GAAI;AAAA,QAC9D;AAAA,QACA,OAAO,aAAa,KAAK;AAAA,MAC3B;AAEA,YAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,YAAI,aAAc,cAAa,YAAY;AAC3C,eAAO,EAAE,KAAK,CAAC;AAAA,MACjB,CAAC;AACD,YAAM,GAAG,SAAS,CAAC,QAAQ;AACzB,YAAI,aAAc,cAAa,YAAY;AAC3C,eAAO,EAAE,MAAM,KAAK,aAAa,OAAO,GAAG,EAAE,CAAC;AAAA,MAChD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACF;AAEO,IAAM,sBAAN,MAAmD;AAAA,EACxD,KAAK;AAAA,EAEL,MAAM,KACJ,OACA,SACA,QACwB;AACxB,QAAI,CAAC,OAAO,MAAO,OAAM,IAAI,YAAY,2CAA2C;AACpF,UAAM,MAAM,IAAI,wBAAwB;AACxC,UAAM,UAAU,OAAO,QAAQ,OAAO,OAAO,CAAC,CAAC,EAC5C,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI,WAAW,CAAC,CAAC,EAAE,EACtD,KAAK,GAAG;AACX,UAAM,UAAU,mBAAmB,OAAO,IAAI,WAAW,OAAO,KAAK,CAAC,UAAU,WAAW,OAAO,CAAC;AACnG,WAAO,IAAI,KAAK,OAAO,SAAS,EAAE,GAAG,QAAQ,KAAK,OAAU,CAAC;AAAA,EAC/D;AACF;AAEA,SAAS,WAAW,GAAmB;AACrC,MAAI,yBAAyB,KAAK,CAAC,EAAG,QAAO;AAC7C,SAAO,IAAI,EAAE,QAAQ,MAAM,OAAO,CAAC;AACrC;AAcO,IAAM,iBAAN,MAAqB;AAAA,EAClB;AAAA,EACR,YAAY,SAAwB,IAAI,wBAAwB,GAAG;AACjE,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,IAAI,QAAuB,SAAsD;AACrF,UAAM,SAAS,MAAM,QAAQ,QAAQ;AAAA,MACnC,MAAM,WAAW,KAAK,OAAO,EAAE;AAAA,MAC/B,OAAO,OAAO;AAAA,MACd,SAAS,CAAC,OAAO,cAAc,OAAO,YAAY,OAAO,WAAW,EACjE,OAAO,OAAO,EACd,KAAK,MAAM;AAAA,IAChB,CAAC;AACD,UAAM,SAA+B,EAAE,QAAQ,OAAO,aAAa,GAAG,OAAO,EAAE;AAC/E,QAAI;AACF,UAAI,OAAO,cAAc;AACvB,eAAO,QAAQ,MAAM,KAAK,OAAO,KAAK,SAAS,OAAO,cAAc,MAAM;AAC1E,eAAO,eAAe,OAAO,MAAM;AACnC,YAAI,OAAO,MAAM,mBAAmB,OAAO,MAAM,aAAa,GAAG;AAC/D,gBAAM,SAAS,OAAO,MAAM,kBACxB,yBAAyB,OAAO,MAAM,MAAM,OAC5C,sBAAsB,OAAO,MAAM,QAAQ;AAC/C,gBAAM,OAAO,KAAK,QAAQ;AAAA,YACxB,UAAU,OAAO,MAAM;AAAA,YACvB,QAAQ,OAAO;AAAA,UACjB,CAAyB;AACzB,iBAAO;AAAA,QACT;AAAA,MACF;AACA,UAAI,OAAO,YAAY;AACrB,eAAO,MAAM,MAAM,KAAK,OAAO,KAAK,OAAO,OAAO,YAAY,MAAM;AACpE,eAAO,eAAe,OAAO,IAAI;AACjC,YAAI,OAAO,IAAI,mBAAmB,OAAO,IAAI,aAAa,GAAG;AAC3D,gBAAM,SAAS,OAAO,IAAI,kBACtB,uBAAuB,OAAO,IAAI,MAAM,OACxC,oBAAoB,OAAO,IAAI,QAAQ;AAC3C,gBAAM,OAAO,KAAK,QAAQ;AAAA,YACxB,UAAU,OAAO,IAAI;AAAA,YACrB,QAAQ,OAAO;AAAA,UACjB,CAAyB;AACzB,iBAAO;AAAA,QACT;AAAA,MACF;AACA,UAAI,OAAO,aAAa;AACtB,eAAO,OAAO,MAAM,KAAK,OAAO,KAAK,QAAQ,OAAO,aAAa,MAAM;AACvE,eAAO,eAAe,OAAO,KAAK;AAGlC,cAAM,SAAS,CAAC,OAAO,KAAK,mBAAmB,OAAO,KAAK,aAAa;AACxE,eAAO,SAAS;AAChB,YAAI,OAAO,KAAK,eAAe,UAAa,OAAO,KAAK,aAAa,GAAG;AACtE,iBAAO,SAAS,OAAO,KAAK,eAAe,KAAK,OAAO,KAAK;AAAA,QAC9D,OAAO;AACL,iBAAO,QAAQ,SAAS,IAAI;AAAA,QAC9B;AACA,cAAM,OAAO,IAAI;AAAA,UACf,UAAU,OAAO,KAAK;AAAA,UACtB,YAAY,OAAO,KAAK;AAAA,UACxB,aAAa,OAAO,KAAK;AAAA,UACzB,QAAQ,OAAO;AAAA,UACf,QAAQ,SAAS,OAAO;AAAA,QAC1B,CAAyB;AAAA,MAC3B,OAAO;AACL,eAAO,SAAS;AAChB,eAAO,QAAQ;AACf,cAAM,OAAO,IAAI,EAAE,QAAQ,OAAO,YAAY,CAAyB;AAAA,MACzE;AAAA,IACF,SAAS,KAAK;AACZ,YAAM,OAAO,KAAK,eAAe,QAAQ,MAAM,OAAO,GAAG,CAAC;AAC1D,YAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AACF;;;AClXA,eAAsB,sBACpB,UACA,OACA,UAAgC,CAAC,GACH;AAC9B,QAAM,UAAU,IAAI,aAAa,KAAK;AACtC,QAAM,QAAQ,SAAS;AAAA,IACrB,YAAY,SAAS;AAAA,IACrB,WAAW,QAAQ;AAAA,IACnB,gBAAgB,SAAS;AAAA,IACzB,MAAM,SAAS;AAAA,IACf,GAAG,QAAQ;AAAA,EACb,CAAC;AACD,QAAM,UAAU,IAAI,eAAe,QAAQ,MAAM;AACjD,QAAM,SAAS,MAAM,QAAQ,IAAI,SAAS,SAAS,OAAO;AAC1D,QAAM,YAAY,SAAS,iBAAiB;AAC5C,QAAM,OAAO,OAAO,UAAU,OAAO,SAAS;AAC9C,QAAM,cAAc,OAAO,UAAU,UAAa,OAAO,MAAM,aAAa;AAC5E,QAAM,YAAY,OAAO,QAAQ,UAAa,OAAO,IAAI,aAAa;AACtE,QAAM,aAAa,OAAO,SAAS,UAAa,OAAO,KAAK,aAAa;AACzE,QAAM,eAAyC,OAC3C,YACA,eAAe,YACb,oBACA,aACE,iBACA;AACR,QAAM,QAAQ,OAAO;AAAA,IACnB;AAAA,IACA,OAAO,OAAO;AAAA,IACd;AAAA,IACA,OAAO,OAAO,SAAY,iBAAiB,MAAM;AAAA,EACnD,CAAC;AACD,SAAO;AAAA,IACL,OAAO,QAAQ;AAAA,IACf;AAAA,IACA,SAAS;AAAA,IACT;AAAA,IACA,OAAO,OAAO;AAAA,IACd;AAAA,EACF;AACF;AAEA,SAAS,iBAAiB,QAAsC;AAC9D,MAAI,OAAO,SAAS,OAAO,MAAM,aAAa;AAC5C,WAAO,sBAAsB,OAAO,MAAM,QAAQ;AACpD,MAAI,OAAO,OAAO,OAAO,IAAI,aAAa,EAAG,QAAO,oBAAoB,OAAO,IAAI,QAAQ;AAC3F,MAAI,OAAO,MAAM;AACf,QAAI,OAAO,KAAK,eAAe,QAAW;AACxC,aAAO,UAAU,OAAO,KAAK,eAAe,CAAC,IAAI,OAAO,KAAK,UAAU;AAAA,IACzE;AACA,WAAO,aAAa,OAAO,KAAK,QAAQ;AAAA,EAC1C;AACA,SAAO;AACT;","names":[]}

package/dist/chunk-MA6HLL3S.js

@@ -0,0 +1,65 @@
+// src/trace/schema.ts
+var TRACE_SCHEMA_VERSION = "1.0.0";
+var FAILURE_CLASSES = [
+  "success",
+  "reasoning_error",
+  "tool_selection_error",
+  "tool_argument_error",
+  "tool_recovery_failure",
+  "hallucination",
+  "instruction_following",
+  "safety_refusal_miss",
+  "policy_violation",
+  "budget_exceeded",
+  "format_drift",
+  "permission_escalation",
+  "pii_leak",
+  "cost_overrun",
+  "timeout",
+  "sandbox_failure",
+  "missing_user_data",
+  "missing_domain_data",
+  "missing_codebase_context",
+  "missing_runtime_context",
+  "missing_credentials",
+  "missing_integration_connection",
+  "missing_integration_scope",
+  "integration_approval_required",
+  "integration_auth_expired",
+  "integration_provider_failure",
+  "bad_integration_manifest",
+  "unsafe_integration_write_denied",
+  "stale_external_data",
+  "bad_retrieval",
+  "insufficient_evidence",
+  "contradictory_evidence",
+  "ambiguous_user_intent",
+  "knowledge_readiness_blocked",
+  "unknown"
+];
+function isLlmSpan(s) {
+  return s.kind === "llm";
+}
+function isToolSpan(s) {
+  return s.kind === "tool";
+}
+function isRetrievalSpan(s) {
+  return s.kind === "retrieval";
+}
+function isJudgeSpan(s) {
+  return s.kind === "judge";
+}
+function isSandboxSpan(s) {
+  return s.kind === "sandbox";
+}
+
+export {
+  TRACE_SCHEMA_VERSION,
+  FAILURE_CLASSES,
+  isLlmSpan,
+  isToolSpan,
+  isRetrievalSpan,
+  isJudgeSpan,
+  isSandboxSpan
+};
+//# sourceMappingURL=chunk-MA6HLL3S.js.map

package/dist/chunk-MA6HLL3S.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/trace/schema.ts"],"sourcesContent":["/**\n * TraceSchema v1 — the canonical data model for agent-eval.\n *\n * Every score, every failure class, every pipeline in the framework is\n * a view over this data. Shape it once, live with it.\n *\n * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)\n * but extended with agent-specific span kinds (llm, tool, retrieval,\n * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict\n * entities that OTEL leaves as free-form attributes.\n */\n\nexport const TRACE_SCHEMA_VERSION = '1.0.0'\n\n// ── Run ──────────────────────────────────────────────────────────────\n\nexport type RunStatus = 'running' | 'completed' | 'failed' | 'aborted'\n\nexport interface BudgetSpec {\n  tokens?: number\n  wallMs?: number\n  calls?: number\n  usd?: number\n}\n\nexport interface RunOutcome {\n  score?: number\n  pass?: boolean\n  failureClass?: FailureClass\n  notes?: string\n}\n\n/**\n * Layer — optional classification in a nested build workflow.\n * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).\n * `app-build`: sandbox harness that compiled + tested the generated scaffold.\n * `app-runtime`: a run of the generated agent against a domain scenario.\n * `meta`: any meta-eval (judge replay, correlation analysis).\n */\nexport type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom'\n\nexport interface Run {\n  runId: string\n  /**\n   * Stable identifier of the scenario being executed.\n   *\n   * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts\n   * input WITHOUT this field, substituting a sensible default\n   * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no\n   * curated scenario to anchor to (runtime / operator / meta-eval runs). This\n   * keeps the persisted shape unambiguous for downstream filters + aggregations\n   * while removing the boilerplate of inventing placeholder ids at the call site.\n   */\n  scenarioId: string\n  variantId?: string\n  datasetVersion?: string\n  /** Git SHA of agent code at run time. */\n  codeSha?: string\n  /** Hash of the prompt template + any system prompt. */\n  promptSha?: string\n  /** Model id + date + system-prompt hash, concatenated. */\n  modelFingerprint?: string\n  seed?: number\n  /** Arbitrary environment markers (shell, docker version, tz). */\n  envFingerprint?: Record<string, string>\n  /** Version of the redaction rules applied to this run. */\n  redactionVersion?: string\n  /** Parent run in a nested build workflow. A builder run's children are\n   *  app-build runs; those children are app-runtime runs. */\n  parentRunId?: string\n  /** Stable project identifier — groups runs across chats + sessions. */\n  projectId?: string\n  /** Chat/conversation identifier within a project. */\n  chatId?: string\n  /** Layer classification — hint for aggregation; not enforced. */\n  layer?: RunLayer\n  startedAt: number\n  endedAt?: number\n  status: RunStatus\n  outcome?: RunOutcome\n  budget?: BudgetSpec\n  /** Free-form labels for downstream grouping. */\n  tags?: Record<string, string>\n}\n\n// ── Spans (hierarchical work units) ──────────────────────────────────\n\nexport type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom'\n\nexport type SpanStatus = 'ok' | 'error'\n\nexport interface SpanBase {\n  spanId: string\n  parentSpanId?: string\n  runId: string\n  kind: SpanKind\n  name: string\n  startedAt: number\n  endedAt?: number\n  status?: SpanStatus\n  error?: string\n  /** Anything not covered by typed fields. Kept deliberately free-form. */\n  attributes?: Record<string, unknown>\n}\n\nexport interface Message {\n  role: 'system' | 'user' | 'assistant' | 'tool'\n  content: string\n  tokens?: number\n  /** Multi-modal content descriptors; blobs themselves live in Artifacts. */\n  images?: Array<{ artifactId?: string; url?: string; mime?: string }>\n}\n\nexport interface LlmSpan extends SpanBase {\n  kind: 'llm'\n  model: string\n  messages: Message[]\n  output?: string\n  inputTokens?: number\n  /** All generated tokens, including the reasoning subset when present. */\n  outputTokens?: number\n  cachedTokens?: number\n  cacheWriteTokens?: number\n  /** Reasoning-token subset of `outputTokens`. */\n  reasoningTokens?: number\n  costUsd?: number\n  finishReason?: string\n}\n\nexport interface ToolSpan extends SpanBase {\n  kind: 'tool'\n  toolName: string\n  args: unknown\n  /** False when the source observed the call but did not capture its arguments. */\n  argsCaptured?: boolean\n  result?: unknown\n  latencyMs?: number\n}\n\nexport interface RetrievalSpan extends SpanBase {\n  kind: 'retrieval'\n  query: string\n  hits: Array<{ docId: string; score: number; content?: string }>\n}\n\nexport interface JudgeSpan extends SpanBase {\n  kind: 'judge'\n  judgeId: string\n  /** Span this judgment applies to. */\n  targetSpanId: string\n  dimension: string\n  /** Numeric score (free-range; interpretation up to the judge). */\n  score: number\n  rationale?: string\n  evidence?: string\n}\n\nexport interface SandboxSpan extends SpanBase {\n  kind: 'sandbox'\n  image?: string\n  command?: string\n  exitCode?: number\n  testsTotal?: number\n  testsPassed?: number\n  stdoutHash?: string\n  stderrHash?: string\n  /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */\n  wallMs?: number\n}\n\nexport interface GenericSpan extends SpanBase {\n  kind: 'agent' | 'custom'\n}\n\nexport type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan\n\n// ── Events (point-in-time occurrences within a span) ─────────────────\n\nexport type EventKind =\n  | 'log'\n  | 'error'\n  | 'budget_decrement'\n  | 'budget_breach'\n  | 'state_mutation'\n  | 'policy_violation'\n  | 'redaction_applied'\n  | 'custom'\n\nexport interface TraceEvent {\n  eventId: string\n  runId: string\n  spanId?: string\n  kind: EventKind\n  timestamp: number\n  payload: Record<string, unknown>\n}\n\n// ── Budget ledger (running token/wall/call/$ accounting) ─────────────\n\nexport interface BudgetLedgerEntry {\n  runId: string\n  dimension: keyof BudgetSpec\n  limit: number\n  consumed: number\n  remaining: number\n  timestamp: number\n  breached: boolean\n  /** Span that triggered this entry, if any. */\n  spanId?: string\n}\n\n// ── Artifacts (blobs addressed by hash) ──────────────────────────────\n\nexport interface Artifact {\n  artifactId: string\n  runId: string\n  spanId?: string\n  contentType: string\n  sizeBytes: number\n  /** sha256 in hex. */\n  hash: string\n  /** External storage URL (R2, S3, filesystem path). */\n  storageUrl?: string\n  /** Inline content for small blobs — keep under ~64KB. */\n  inlineContent?: string\n}\n\n// ── Failure taxonomy ─────────────────────────────────────────────────\n\nexport type FailureClass =\n  | 'success'\n  | 'reasoning_error'\n  | 'tool_selection_error'\n  | 'tool_argument_error'\n  | 'tool_recovery_failure'\n  | 'hallucination'\n  | 'instruction_following'\n  | 'safety_refusal_miss'\n  | 'policy_violation'\n  | 'budget_exceeded'\n  | 'format_drift'\n  | 'permission_escalation'\n  | 'pii_leak'\n  | 'cost_overrun'\n  | 'timeout'\n  | 'sandbox_failure'\n  | 'missing_user_data'\n  | 'missing_domain_data'\n  | 'missing_codebase_context'\n  | 'missing_runtime_context'\n  | 'missing_credentials'\n  | 'missing_integration_connection'\n  | 'missing_integration_scope'\n  | 'integration_approval_required'\n  | 'integration_auth_expired'\n  | 'integration_provider_failure'\n  | 'bad_integration_manifest'\n  | 'unsafe_integration_write_denied'\n  | 'stale_external_data'\n  | 'bad_retrieval'\n  | 'insufficient_evidence'\n  | 'contradictory_evidence'\n  | 'ambiguous_user_intent'\n  | 'knowledge_readiness_blocked'\n  | 'unknown'\n\nexport const FAILURE_CLASSES: readonly FailureClass[] = [\n  'success',\n  'reasoning_error',\n  'tool_selection_error',\n  'tool_argument_error',\n  'tool_recovery_failure',\n  'hallucination',\n  'instruction_following',\n  'safety_refusal_miss',\n  'policy_violation',\n  'budget_exceeded',\n  'format_drift',\n  'permission_escalation',\n  'pii_leak',\n  'cost_overrun',\n  'timeout',\n  'sandbox_failure',\n  'missing_user_data',\n  'missing_domain_data',\n  'missing_codebase_context',\n  'missing_runtime_context',\n  'missing_credentials',\n  'missing_integration_connection',\n  'missing_integration_scope',\n  'integration_approval_required',\n  'integration_auth_expired',\n  'integration_provider_failure',\n  'bad_integration_manifest',\n  'unsafe_integration_write_denied',\n  'stale_external_data',\n  'bad_retrieval',\n  'insufficient_evidence',\n  'contradictory_evidence',\n  'ambiguous_user_intent',\n  'knowledge_readiness_blocked',\n  'unknown',\n] as const\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nexport function isLlmSpan(s: Span): s is LlmSpan {\n  return s.kind === 'llm'\n}\nexport function isToolSpan(s: Span): s is ToolSpan {\n  return s.kind === 'tool'\n}\nexport function isRetrievalSpan(s: Span): s is RetrievalSpan {\n  return s.kind === 'retrieval'\n}\nexport function isJudgeSpan(s: Span): s is JudgeSpan {\n  return s.kind === 'judge'\n}\nexport function isSandboxSpan(s: Span): s is SandboxSpan {\n  return s.kind === 'sandbox'\n}\n"],"mappings":";AAYO,IAAM,uBAAuB;AA8P7B,IAAM,kBAA2C;AAAA,EACtD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAIO,SAAS,UAAU,GAAuB;AAC/C,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,WAAW,GAAwB;AACjD,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,gBAAgB,GAA6B;AAC3D,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,YAAY,GAAyB;AACnD,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,cAAc,GAA2B;AACvD,SAAO,EAAE,SAAS;AACpB;","names":[]}

package/dist/chunk-MAZ26DC7.js

@@ -0,0 +1,99 @@
+// src/sequential.ts
+function pairedEvalueSequence(deltas, opts = {}) {
+  const c = opts.bound ?? 1;
+  const alpha = opts.alpha ?? 0.05;
+  const initialShrink = opts.initialBetShrinkage ?? 0.5;
+  const rope = opts.rope ?? null;
+  if (c <= 0) throw new Error("pairedEvalueSequence: bound must be > 0");
+  if (alpha <= 0 || alpha >= 1) throw new Error("pairedEvalueSequence: alpha must be in (0,1)");
+  if (rope && !(Number.isFinite(rope.low) && Number.isFinite(rope.high) && rope.low <= rope.high)) {
+    throw new Error("pairedEvalueSequence: rope must satisfy low \u2264 high");
+  }
+  const steps = [];
+  let clipped = false;
+  let evalue = 1;
+  let decisionFiredAt = null;
+  let sum = 0;
+  let sumSq = 0;
+  let count = 0;
+  for (let i = 0; i < deltas.length; i++) {
+    let d = deltas[i];
+    if (d < -c || d > c) {
+      d = Math.max(-c, Math.min(c, d));
+      clipped = true;
+    }
+    const muHat = count === 0 ? 0 : sum / count;
+    const varHat = count === 0 ? c * c : Math.max(1e-12, sumSq / count - muHat * muHat);
+    const t = i + 1;
+    const shrink = initialShrink * Math.min(1, count / 32);
+    let lambda = muHat / (varHat + c * c) * shrink;
+    const lambdaMax = 0.99 / c;
+    if (lambda > lambdaMax) lambda = lambdaMax;
+    if (lambda < -lambdaMax) lambda = -lambdaMax;
+    evalue = evalue * (1 + lambda * d);
+    if (!Number.isFinite(evalue) || evalue < 0) evalue = 0;
+    sum += d;
+    sumSq += d * d;
+    count += 1;
+    const pValue = Math.min(1, 1 / Math.max(evalue, 1e-300));
+    const cs = empiricalBernsteinCs(sum, sumSq, count, c, alpha);
+    let decision = "continue";
+    if (rope && cs.low >= rope.low && cs.high <= rope.high) decision = "equivalent";
+    else if (evalue >= 2 / alpha && muHat > 0) decision = "promote_now";
+    else if (evalue >= 2 / alpha && muHat < 0) decision = "reject_now";
+    else if (rope && cs.high < rope.low) decision = "reject_now";
+    if (decision !== "continue" && decisionFiredAt === null) decisionFiredAt = t;
+    steps.push({ t, delta: d, evalue, pValue, csLow: cs.low, csHigh: cs.high, decision });
+  }
+  const finalDecision = steps.length === 0 ? "continue" : steps[steps.length - 1].decision;
+  return { steps, finalDecision, decisionFiredAt, clipped };
+}
+function evaluateInterimReleaseConfidence(input) {
+  const candidates = input.deltaSeries.map((s) => {
+    const seq = pairedEvalueSequence(s.deltas, {
+      alpha: input.alpha,
+      bound: input.bound,
+      rope: input.rope
+    });
+    const last = seq.steps[seq.steps.length - 1];
+    return {
+      candidateId: s.candidateId,
+      decision: seq.finalDecision,
+      decisionFiredAt: seq.decisionFiredAt,
+      finalEvalue: last?.evalue ?? 1,
+      finalPValue: last?.pValue ?? 1,
+      pairs: seq.steps.length,
+      csLow: last?.csLow ?? Number.NEGATIVE_INFINITY,
+      csHigh: last?.csHigh ?? Number.POSITIVE_INFINITY
+    };
+  });
+  const promote = candidates.find((c) => c.decision === "promote_now");
+  if (promote)
+    return {
+      candidates,
+      recommendation: { decision: "promote_now", candidateId: promote.candidateId }
+    };
+  const live = candidates.find((c) => c.decision === "continue");
+  if (live) return { candidates, recommendation: { decision: "continue", candidateId: null } };
+  const equiv = candidates.find((c) => c.decision === "equivalent");
+  if (equiv)
+    return {
+      candidates,
+      recommendation: { decision: "equivalent", candidateId: equiv.candidateId }
+    };
+  return { candidates, recommendation: { decision: "reject_now", candidateId: null } };
+}
+function empiricalBernsteinCs(sum, sumSq, n, bound, alpha) {
+  if (n === 0) return { low: -bound, high: bound };
+  const mean = sum / n;
+  const variance = Math.max(0, sumSq / n - mean * mean);
+  const psi = Math.log(2 / alpha) + 1.7 * Math.log(Math.log(Math.max(Math.E, n)) + 1);
+  const radius = Math.sqrt(2 * variance * psi / n) + 3 * bound * psi / n;
+  return { low: mean - radius, high: mean + radius };
+}
+
+export {
+  pairedEvalueSequence,
+  evaluateInterimReleaseConfidence
+};
+//# sourceMappingURL=chunk-MAZ26DC7.js.map

package/dist/chunk-MAZ26DC7.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/sequential.ts"],"sourcesContent":["/**\n * Always-valid sequential evaluation.\n *\n * `researchReport` assumes a single pre-specified analysis. Real\n * consumers run campaigns weekly / nightly / per-PR; each new run silently\n * inflates the false-discovery rate, because the BH-FDR guarantee is for\n * the *first* look, not the 47th. Without time-uniform inference,\n * launch-decision teams either (a) don't peek, which forfeits the cost\n * advantage of stop-when-decisive, or (b) peek and pretend they didn't,\n * which forfeits scientific validity.\n *\n * This module ships **e-value-based confidence sequences** for paired\n * bounded outcomes. The methodology is the predictable plug-in betting\n * martingale of Waudby-Smith & Ramdas (2024) — provably valid at *any*\n * stopping time. Concretely:\n *\n *   For paired deltas D_1, D_2, … ∈ [-c, c] with the null H_0: E[D] ≤ 0,\n *   a betting fraction λ_i is chosen using only D_{1..i-1} (predictable\n *   plug-in), and the running e-value is\n *\n *     E_t = ∏_{i=1}^{t} (1 + λ_i · D_i)\n *\n *   E_t is a non-negative martingale under H_0 with E[E_t] ≤ 1, so by\n *   Ville's inequality, P(∃ t : E_t ≥ 1/α) ≤ α — we can reject the null\n *   at any time without inflating the type-I error.\n *\n * Combined with `runEvalCampaign`, every consumer running rolling\n * campaigns gains the ability to ship the moment evidence is decisive,\n * stop-early on dead-on-arrival variants, and accumulate evidence across\n * partial runs without spending the FDR budget. No new sweep is wasted.\n *\n * References:\n *   - Howard, S. R., Ramdas, A., McAuliffe, J., Sekhon, J. (2021).\n *     Time-uniform, nonparametric, nonasymptotic confidence sequences.\n *     Annals of Statistics, 49(2), 1055–1080.\n *   - Waudby-Smith, I., Ramdas, A. (2024). Estimating means of bounded\n *     random variables by betting. JRSS B, 86(1), 1–27.\n */\n\nexport type SequentialDecision = 'promote_now' | 'continue' | 'reject_now' | 'equivalent'\n\nexport interface PairedEvalueOptions {\n  /**\n   * Bound on |delta|. Default 1 (matching most score scales). Must satisfy\n   * c > 0; deltas outside [-c, c] are clipped with a warning attached to\n   * the return value.\n   */\n  bound?: number\n  /** Target Type-I error. Default 0.05. */\n  alpha?: number\n  /**\n   * Region of Practical Equivalence on the *mean* paired delta. When\n   * supplied, the verdict can return `'equivalent'` once the running\n   * confidence sequence on the mean is fully contained in [low, high].\n   */\n  rope?: { low: number; high: number }\n  /** Initial bet shrinkage (0 < scale ≤ 1). Default 0.5 — empirically robust. */\n  initialBetShrinkage?: number\n}\n\nexport interface PairedEvalueStep {\n  /** 1-indexed observation count. */\n  t: number\n  delta: number\n  /** Running e-value E_t = ∏ (1 + λ_i · D_i). */\n  evalue: number\n  /** Time-uniform p-value at stopping time t. */\n  pValue: number\n  /** Lower bound of the empirical Bernstein confidence sequence at level 1-α. */\n  csLow: number\n  csHigh: number\n  /** Verdict at this stopping time. */\n  decision: SequentialDecision\n}\n\nexport interface PairedEvalueSequence {\n  steps: PairedEvalueStep[]\n  /** The decision at the final step. */\n  finalDecision: SequentialDecision\n  /** Index (1-based) at which a non-`continue` decision first fired, or null. */\n  decisionFiredAt: number | null\n  /** True if any deltas were clipped to [-bound, bound]. */\n  clipped: boolean\n}\n\n/**\n * Run the paired e-value sequence over an in-order delta stream.\n *\n * Use for *streaming* / interim analyses: pass the deltas you have so\n * far, get the verdict at every prefix length. The decision is\n * monotone-stable in the sense that once `'reject_now'` or `'promote_now'`\n * fires, the verdict at later steps remains decisive (the e-value is a\n * non-negative martingale; once it crosses the threshold, it's crossed).\n */\nexport function pairedEvalueSequence(\n  deltas: number[],\n  opts: PairedEvalueOptions = {},\n): PairedEvalueSequence {\n  const c = opts.bound ?? 1\n  const alpha = opts.alpha ?? 0.05\n  const initialShrink = opts.initialBetShrinkage ?? 0.5\n  const rope = opts.rope ?? null\n  if (c <= 0) throw new Error('pairedEvalueSequence: bound must be > 0')\n  if (alpha <= 0 || alpha >= 1) throw new Error('pairedEvalueSequence: alpha must be in (0,1)')\n  if (rope && !(Number.isFinite(rope.low) && Number.isFinite(rope.high) && rope.low <= rope.high)) {\n    throw new Error('pairedEvalueSequence: rope must satisfy low ≤ high')\n  }\n\n  const steps: PairedEvalueStep[] = []\n  let clipped = false\n  let evalue = 1\n  let decisionFiredAt: number | null = null\n\n  // Running statistics (using only D_{1..i-1} for the bet → predictable plug-in).\n  let sum = 0\n  let sumSq = 0\n  let count = 0\n\n  for (let i = 0; i < deltas.length; i++) {\n    let d = deltas[i]!\n    if (d < -c || d > c) {\n      d = Math.max(-c, Math.min(c, d))\n      clipped = true\n    }\n\n    // Predictable plug-in bet (positive λ tests for E[D] > 0; we run a two-sided\n    // test by tracking the symmetric e-value via |bet|).\n    // λ_i ∝ mean / (variance + bound^2). Shrink early to avoid overbetting.\n    const muHat = count === 0 ? 0 : sum / count\n    const varHat = count === 0 ? c * c : Math.max(1e-12, sumSq / count - muHat * muHat)\n    const t = i + 1\n    const shrink = initialShrink * Math.min(1, count / 32) // anneal toward 1\n    let lambda = (muHat / (varHat + c * c)) * shrink\n    // Clip to ensure 1 + λ·D > 0 for all |D| ≤ c (so the e-value stays non-negative).\n    const lambdaMax = 0.99 / c\n    if (lambda > lambdaMax) lambda = lambdaMax\n    if (lambda < -lambdaMax) lambda = -lambdaMax\n\n    evalue = evalue * (1 + lambda * d)\n    if (!Number.isFinite(evalue) || evalue < 0) evalue = 0\n\n    sum += d\n    sumSq += d * d\n    count += 1\n\n    const pValue = Math.min(1, 1 / Math.max(evalue, 1e-300))\n\n    // Empirical Bernstein confidence sequence on the mean. Howard et al.\n    // (2021), Theorem 4.4 with σ̂² the running sample variance and a\n    // calibration constant tuned for two-sided coverage at level 1 - α.\n    const cs = empiricalBernsteinCs(sum, sumSq, count, c, alpha)\n\n    let decision: SequentialDecision = 'continue'\n    if (rope && cs.low >= rope.low && cs.high <= rope.high) decision = 'equivalent'\n    else if (evalue >= 2 / alpha && muHat > 0) decision = 'promote_now'\n    else if (evalue >= 2 / alpha && muHat < 0) decision = 'reject_now'\n    else if (rope && cs.high < rope.low) decision = 'reject_now'\n\n    if (decision !== 'continue' && decisionFiredAt === null) decisionFiredAt = t\n\n    steps.push({ t, delta: d, evalue, pValue, csLow: cs.low, csHigh: cs.high, decision })\n  }\n\n  const finalDecision = steps.length === 0 ? 'continue' : steps[steps.length - 1]!.decision\n  return { steps, finalDecision, decisionFiredAt, clipped }\n}\n\nexport interface InterimReleaseConfidenceInput {\n  /**\n   * One delta series per candidate (paired deltas vs comparator). Order\n   * within a series is the order the campaigns were run.\n   */\n  deltaSeries: Array<{ candidateId: string; deltas: number[] }>\n  alpha?: number\n  bound?: number\n  rope?: { low: number; high: number }\n}\n\nexport interface InterimReleaseConfidence {\n  candidates: Array<{\n    candidateId: string\n    decision: SequentialDecision\n    decisionFiredAt: number | null\n    finalEvalue: number\n    finalPValue: number\n    pairs: number\n    csLow: number\n    csHigh: number\n  }>\n  /**\n   * Campaign-level recommendation: pick the strongest 'promote_now', else\n   * 'continue' if any candidate is still live, else 'reject_now' if every\n   * candidate is dead, else 'equivalent'.\n   */\n  recommendation: { decision: SequentialDecision; candidateId: string | null }\n}\n\n/**\n * Run interim sequential analyses across many candidates at once,\n * preserving the time-uniform α guarantee for each candidate's series and\n * synthesising a campaign-level recommendation. Designed to be called on\n * every campaign tick — the recommendation is anytime-valid.\n */\nexport function evaluateInterimReleaseConfidence(\n  input: InterimReleaseConfidenceInput,\n): InterimReleaseConfidence {\n  const candidates = input.deltaSeries.map((s) => {\n    const seq = pairedEvalueSequence(s.deltas, {\n      alpha: input.alpha,\n      bound: input.bound,\n      rope: input.rope,\n    })\n    const last = seq.steps[seq.steps.length - 1]\n    return {\n      candidateId: s.candidateId,\n      decision: seq.finalDecision,\n      decisionFiredAt: seq.decisionFiredAt,\n      finalEvalue: last?.evalue ?? 1,\n      finalPValue: last?.pValue ?? 1,\n      pairs: seq.steps.length,\n      csLow: last?.csLow ?? Number.NEGATIVE_INFINITY,\n      csHigh: last?.csHigh ?? Number.POSITIVE_INFINITY,\n    }\n  })\n\n  const promote = candidates.find((c) => c.decision === 'promote_now')\n  if (promote)\n    return {\n      candidates,\n      recommendation: { decision: 'promote_now', candidateId: promote.candidateId },\n    }\n  const live = candidates.find((c) => c.decision === 'continue')\n  if (live) return { candidates, recommendation: { decision: 'continue', candidateId: null } }\n  const equiv = candidates.find((c) => c.decision === 'equivalent')\n  if (equiv)\n    return {\n      candidates,\n      recommendation: { decision: 'equivalent', candidateId: equiv.candidateId },\n    }\n  return { candidates, recommendation: { decision: 'reject_now', candidateId: null } }\n}\n\n// ── Internals ────────────────────────────────────────────────────────────\n\n/**\n * Empirical Bernstein confidence sequence on the mean of bounded variables.\n * Adapted from Howard et al. (2021) §4.4. Provides a time-uniform CI on\n * the running mean; valid at every stopping time.\n */\nfunction empiricalBernsteinCs(\n  sum: number,\n  sumSq: number,\n  n: number,\n  bound: number,\n  alpha: number,\n): { low: number; high: number } {\n  if (n === 0) return { low: -bound, high: bound }\n  const mean = sum / n\n  const variance = Math.max(0, sumSq / n - mean * mean)\n  // Iterated-log calibration constant. The 1.7 exponent matches the\n  // recommended choice in Howard et al. for two-sided coverage at level\n  // 1 - α with mild log-corrections; tightening further requires a\n  // tuned mixture and is out of scope.\n  const psi = Math.log(2 / alpha) + 1.7 * Math.log(Math.log(Math.max(Math.E, n)) + 1)\n  const radius = Math.sqrt((2 * variance * psi) / n) + (3 * bound * psi) / n\n  return { low: mean - radius, high: mean + radius }\n}\n"],"mappings":";AA8FO,SAAS,qBACd,QACA,OAA4B,CAAC,GACP;AACtB,QAAM,IAAI,KAAK,SAAS;AACxB,QAAM,QAAQ,KAAK,SAAS;AAC5B,QAAM,gBAAgB,KAAK,uBAAuB;AAClD,QAAM,OAAO,KAAK,QAAQ;AAC1B,MAAI,KAAK,EAAG,OAAM,IAAI,MAAM,yCAAyC;AACrE,MAAI,SAAS,KAAK,SAAS,EAAG,OAAM,IAAI,MAAM,8CAA8C;AAC5F,MAAI,QAAQ,EAAE,OAAO,SAAS,KAAK,GAAG,KAAK,OAAO,SAAS,KAAK,IAAI,KAAK,KAAK,OAAO,KAAK,OAAO;AAC/F,UAAM,IAAI,MAAM,yDAAoD;AAAA,EACtE;AAEA,QAAM,QAA4B,CAAC;AACnC,MAAI,UAAU;AACd,MAAI,SAAS;AACb,MAAI,kBAAiC;AAGrC,MAAI,MAAM;AACV,MAAI,QAAQ;AACZ,MAAI,QAAQ;AAEZ,WAAS,IAAI,GAAG,IAAI,OAAO,QAAQ,KAAK;AACtC,QAAI,IAAI,OAAO,CAAC;AAChB,QAAI,IAAI,CAAC,KAAK,IAAI,GAAG;AACnB,UAAI,KAAK,IAAI,CAAC,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AAC/B,gBAAU;AAAA,IACZ;AAKA,UAAM,QAAQ,UAAU,IAAI,IAAI,MAAM;AACtC,UAAM,SAAS,UAAU,IAAI,IAAI,IAAI,KAAK,IAAI,OAAO,QAAQ,QAAQ,QAAQ,KAAK;AAClF,UAAM,IAAI,IAAI;AACd,UAAM,SAAS,gBAAgB,KAAK,IAAI,GAAG,QAAQ,EAAE;AACrD,QAAI,SAAU,SAAS,SAAS,IAAI,KAAM;AAE1C,UAAM,YAAY,OAAO;AACzB,QAAI,SAAS,UAAW,UAAS;AACjC,QAAI,SAAS,CAAC,UAAW,UAAS,CAAC;AAEnC,aAAS,UAAU,IAAI,SAAS;AAChC,QAAI,CAAC,OAAO,SAAS,MAAM,KAAK,SAAS,EAAG,UAAS;AAErD,WAAO;AACP,aAAS,IAAI;AACb,aAAS;AAET,UAAM,SAAS,KAAK,IAAI,GAAG,IAAI,KAAK,IAAI,QAAQ,MAAM,CAAC;AAKvD,UAAM,KAAK,qBAAqB,KAAK,OAAO,OAAO,GAAG,KAAK;AAE3D,QAAI,WAA+B;AACnC,QAAI,QAAQ,GAAG,OAAO,KAAK,OAAO,GAAG,QAAQ,KAAK,KAAM,YAAW;AAAA,aAC1D,UAAU,IAAI,SAAS,QAAQ,EAAG,YAAW;AAAA,aAC7C,UAAU,IAAI,SAAS,QAAQ,EAAG,YAAW;AAAA,aAC7C,QAAQ,GAAG,OAAO,KAAK,IAAK,YAAW;AAEhD,QAAI,aAAa,cAAc,oBAAoB,KAAM,mBAAkB;AAE3E,UAAM,KAAK,EAAE,GAAG,OAAO,GAAG,QAAQ,QAAQ,OAAO,GAAG,KAAK,QAAQ,GAAG,MAAM,SAAS,CAAC;AAAA,EACtF;AAEA,QAAM,gBAAgB,MAAM,WAAW,IAAI,aAAa,MAAM,MAAM,SAAS,CAAC,EAAG;AACjF,SAAO,EAAE,OAAO,eAAe,iBAAiB,QAAQ;AAC1D;AAsCO,SAAS,iCACd,OAC0B;AAC1B,QAAM,aAAa,MAAM,YAAY,IAAI,CAAC,MAAM;AAC9C,UAAM,MAAM,qBAAqB,EAAE,QAAQ;AAAA,MACzC,OAAO,MAAM;AAAA,MACb,OAAO,MAAM;AAAA,MACb,MAAM,MAAM;AAAA,IACd,CAAC;AACD,UAAM,OAAO,IAAI,MAAM,IAAI,MAAM,SAAS,CAAC;AAC3C,WAAO;AAAA,MACL,aAAa,EAAE;AAAA,MACf,UAAU,IAAI;AAAA,MACd,iBAAiB,IAAI;AAAA,MACrB,aAAa,MAAM,UAAU;AAAA,MAC7B,aAAa,MAAM,UAAU;AAAA,MAC7B,OAAO,IAAI,MAAM;AAAA,MACjB,OAAO,MAAM,SAAS,OAAO;AAAA,MAC7B,QAAQ,MAAM,UAAU,OAAO;AAAA,IACjC;AAAA,EACF,CAAC;AAED,QAAM,UAAU,WAAW,KAAK,CAAC,MAAM,EAAE,aAAa,aAAa;AACnE,MAAI;AACF,WAAO;AAAA,MACL;AAAA,MACA,gBAAgB,EAAE,UAAU,eAAe,aAAa,QAAQ,YAAY;AAAA,IAC9E;AACF,QAAM,OAAO,WAAW,KAAK,CAAC,MAAM,EAAE,aAAa,UAAU;AAC7D,MAAI,KAAM,QAAO,EAAE,YAAY,gBAAgB,EAAE,UAAU,YAAY,aAAa,KAAK,EAAE;AAC3F,QAAM,QAAQ,WAAW,KAAK,CAAC,MAAM,EAAE,aAAa,YAAY;AAChE,MAAI;AACF,WAAO;AAAA,MACL;AAAA,MACA,gBAAgB,EAAE,UAAU,cAAc,aAAa,MAAM,YAAY;AAAA,IAC3E;AACF,SAAO,EAAE,YAAY,gBAAgB,EAAE,UAAU,cAAc,aAAa,KAAK,EAAE;AACrF;AASA,SAAS,qBACP,KACA,OACA,GACA,OACA,OAC+B;AAC/B,MAAI,MAAM,EAAG,QAAO,EAAE,KAAK,CAAC,OAAO,MAAM,MAAM;AAC/C,QAAM,OAAO,MAAM;AACnB,QAAM,WAAW,KAAK,IAAI,GAAG,QAAQ,IAAI,OAAO,IAAI;AAKpD,QAAM,MAAM,KAAK,IAAI,IAAI,KAAK,IAAI,MAAM,KAAK,IAAI,KAAK,IAAI,KAAK,IAAI,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC;AAClF,QAAM,SAAS,KAAK,KAAM,IAAI,WAAW,MAAO,CAAC,IAAK,IAAI,QAAQ,MAAO;AACzE,SAAO,EAAE,KAAK,OAAO,QAAQ,MAAM,OAAO,OAAO;AACnD;","names":[]}