@tangle-network/agent-eval 0.120.0 → 0.120.2

package/dist/chunk-K4DBDHLK.js

@@ -1,158 +0,0 @@
-// src/trace/attribute-vocabulary.ts
-import {
-  GEN_AI_INPUT_TOKEN_KEYS,
-  GEN_AI_MODEL_KEYS,
-  GEN_AI_OUTPUT_TOKEN_KEYS
-} from "@tangle-network/agent-core/telemetry";
-var OPENINFERENCE_SPAN_KIND = "openinference.span.kind";
-var LLM_MODEL_NAME = "llm.model_name";
-var LLM_INPUT_TOKENS = "llm.token_count.prompt";
-var LLM_OUTPUT_TOKENS = "llm.token_count.completion";
-var LLM_REASONING_TOKENS = "llm.token_count.reasoning";
-var LLM_CACHED_TOKENS = "llm.token_count.prompt_cache_hit";
-var LLM_CACHE_WRITE_TOKENS = "llm.token_count.prompt_cache_write";
-var LLM_CONTEXT_TOKENS = "tangle.llm.context_tokens";
-var LLM_COST_USD = "llm.cost_usd";
-var TOOL_NAME = "tool.name";
-var TOOL_ARGS_CAPTURED = "tool.args_captured";
-var TOOL_LATENCY_MS = "tool.latency_ms";
-var INPUT_VALUE = "input.value";
-var OUTPUT_VALUE = "output.value";
-var SPAN_KIND_ATTR_KEYS = [OPENINFERENCE_SPAN_KIND, "inference.observation_kind"];
-var LLM_MODEL_ATTR_KEYS = [
-  LLM_MODEL_NAME,
-  "inference.llm.model_name",
-  "llm.model",
-  ...GEN_AI_MODEL_KEYS,
-  "tangle.model"
-];
-var LLM_INPUT_TOKEN_ATTR_KEYS = [
-  LLM_INPUT_TOKENS,
-  "inference.llm.input_tokens",
-  "llm.input_tokens",
-  ...GEN_AI_INPUT_TOKEN_KEYS,
-  "tangle.tokens.in",
-  "tokens.in"
-];
-var LLM_OUTPUT_TOKEN_ATTR_KEYS = [
-  LLM_OUTPUT_TOKENS,
-  "inference.llm.output_tokens",
-  "llm.output_tokens",
-  ...GEN_AI_OUTPUT_TOKEN_KEYS,
-  "tangle.tokens.out",
-  "tokens.out"
-];
-var LLM_REASONING_TOKEN_ATTR_KEYS = [
-  LLM_REASONING_TOKENS,
-  "inference.llm.reasoning_tokens",
-  "llm.reasoning_tokens",
-  "gen_ai.usage.reasoning_tokens",
-  "gen_ai.usage.reasoning_output_tokens",
-  "reasoning_tokens",
-  "reasoning_output_tokens",
-  "tangle.tokens.reasoning",
-  "gen_ai.usage.output_tokens_details.reasoning_tokens",
-  "gen_ai.usage.completion_tokens_details.reasoning_tokens"
-];
-var LLM_CACHED_TOKEN_ATTR_KEYS = [
-  LLM_CACHED_TOKENS,
-  "inference.llm.cached_tokens",
-  "llm.cached_tokens",
-  "gen_ai.usage.cached_tokens",
-  "gen_ai.usage.prompt_tokens_details.cached_tokens",
-  "gen_ai.usage.input_tokens_details.cached_tokens",
-  "gen_ai.usage.cache_read_tokens",
-  "gen_ai.usage.cache_read_input_tokens",
-  "cache_read_tokens",
-  "cache_read_input_tokens",
-  "input_cache_read",
-  "tangle.tokens.cached"
-];
-var LLM_CACHE_WRITE_TOKEN_ATTR_KEYS = [
-  LLM_CACHE_WRITE_TOKENS,
-  "inference.llm.cache_write_tokens",
-  "llm.cache_write_tokens",
-  "gen_ai.usage.cache_creation_tokens",
-  "gen_ai.usage.cache_creation_input_tokens",
-  "cache_creation_tokens",
-  "cache_creation_input_tokens",
-  "input_cache_creation",
-  "tangle.tokens.cache_write"
-];
-var LLM_COST_ATTR_KEYS = [
-  LLM_COST_USD,
-  "inference.llm.cost.total",
-  "llm.cost.total",
-  "gen_ai.usage.cost",
-  "gen_ai.usage.cost_usd",
-  "tangle.cost.usd",
-  "cost.usd",
-  "cost"
-];
-var RUN_COST_ATTR_KEYS = ["tangle.cost.usd", "cost.usd"];
-var TOOL_NAME_ATTR_KEYS = [TOOL_NAME, "inference.tool.name"];
-function asNumber(value) {
-  if (typeof value === "number") return Number.isFinite(value) ? value : null;
-  if (typeof value === "string" && value.length > 0) {
-    const parsed = Number(value);
-    return Number.isFinite(parsed) ? parsed : null;
-  }
-  return null;
-}
-function firstNumberAttr(attributes, keys) {
-  for (const key of keys) {
-    const value = asNumber(attributes[key]);
-    if (value !== null) return value;
-  }
-  return null;
-}
-function contextInputTokens(usage) {
-  if (usage.inputTokens === void 0 || usage.inputTokens === null) return void 0;
-  return usage.inputTokens + (usage.cachedTokens ?? 0) + (usage.cacheWriteTokens ?? 0);
-}
-function applyLlmSpanOtlpAttributes(attributes, span) {
-  if (span.model !== void 0) attributes[LLM_MODEL_NAME] = span.model;
-  if (span.inputTokens !== void 0) attributes[LLM_INPUT_TOKENS] = span.inputTokens;
-  if (span.outputTokens !== void 0) attributes[LLM_OUTPUT_TOKENS] = span.outputTokens;
-  if (span.reasoningTokens !== void 0) attributes[LLM_REASONING_TOKENS] = span.reasoningTokens;
-  if (span.cachedTokens !== void 0) attributes[LLM_CACHED_TOKENS] = span.cachedTokens;
-  if (span.cacheWriteTokens !== void 0) {
-    attributes[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens;
-  }
-  const contextTokens = contextInputTokens(span);
-  if (contextTokens !== void 0) attributes[LLM_CONTEXT_TOKENS] = contextTokens;
-  if (span.costUsd !== void 0) attributes[LLM_COST_USD] = span.costUsd;
-  if (span.finishReason) attributes["llm.finish_reason"] = span.finishReason;
-}
-
-export {
-  OPENINFERENCE_SPAN_KIND,
-  LLM_MODEL_NAME,
-  LLM_INPUT_TOKENS,
-  LLM_OUTPUT_TOKENS,
-  LLM_REASONING_TOKENS,
-  LLM_CACHED_TOKENS,
-  LLM_CACHE_WRITE_TOKENS,
-  LLM_CONTEXT_TOKENS,
-  LLM_COST_USD,
-  TOOL_NAME,
-  TOOL_ARGS_CAPTURED,
-  TOOL_LATENCY_MS,
-  INPUT_VALUE,
-  OUTPUT_VALUE,
-  SPAN_KIND_ATTR_KEYS,
-  LLM_MODEL_ATTR_KEYS,
-  LLM_INPUT_TOKEN_ATTR_KEYS,
-  LLM_OUTPUT_TOKEN_ATTR_KEYS,
-  LLM_REASONING_TOKEN_ATTR_KEYS,
-  LLM_CACHED_TOKEN_ATTR_KEYS,
-  LLM_CACHE_WRITE_TOKEN_ATTR_KEYS,
-  LLM_COST_ATTR_KEYS,
-  RUN_COST_ATTR_KEYS,
-  TOOL_NAME_ATTR_KEYS,
-  asNumber,
-  firstNumberAttr,
-  contextInputTokens,
-  applyLlmSpanOtlpAttributes
-};
-//# sourceMappingURL=chunk-K4DBDHLK.js.map

package/dist/chunk-K4DBDHLK.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/trace/attribute-vocabulary.ts"],"sourcesContent":["/** Canonical OpenInference-over-OTLP attribute names used at trace boundaries. */\n\nimport {\n  GEN_AI_INPUT_TOKEN_KEYS,\n  GEN_AI_MODEL_KEYS,\n  GEN_AI_OUTPUT_TOKEN_KEYS,\n} from '@tangle-network/agent-core/telemetry'\n\nexport const OPENINFERENCE_SPAN_KIND = 'openinference.span.kind'\nexport const LLM_MODEL_NAME = 'llm.model_name'\nexport const LLM_INPUT_TOKENS = 'llm.token_count.prompt'\nexport const LLM_OUTPUT_TOKENS = 'llm.token_count.completion'\nexport const LLM_REASONING_TOKENS = 'llm.token_count.reasoning'\nexport const LLM_CACHED_TOKENS = 'llm.token_count.prompt_cache_hit'\nexport const LLM_CACHE_WRITE_TOKENS = 'llm.token_count.prompt_cache_write'\n/** Exact prompt context after summing mutually exclusive input/cache categories. */\nexport const LLM_CONTEXT_TOKENS = 'tangle.llm.context_tokens'\nexport const LLM_COST_USD = 'llm.cost_usd'\nexport const TOOL_NAME = 'tool.name'\nexport const TOOL_ARGS_CAPTURED = 'tool.args_captured'\nexport const TOOL_LATENCY_MS = 'tool.latency_ms'\nexport const INPUT_VALUE = 'input.value'\nexport const OUTPUT_VALUE = 'output.value'\n\nexport const SPAN_KIND_ATTR_KEYS = [OPENINFERENCE_SPAN_KIND, 'inference.observation_kind'] as const\n\nexport const LLM_MODEL_ATTR_KEYS = [\n  LLM_MODEL_NAME,\n  'inference.llm.model_name',\n  'llm.model',\n  ...GEN_AI_MODEL_KEYS,\n  'tangle.model',\n] as const\n\nexport const LLM_INPUT_TOKEN_ATTR_KEYS = [\n  LLM_INPUT_TOKENS,\n  'inference.llm.input_tokens',\n  'llm.input_tokens',\n  ...GEN_AI_INPUT_TOKEN_KEYS,\n  'tangle.tokens.in',\n  'tokens.in',\n] as const\n\nexport const LLM_OUTPUT_TOKEN_ATTR_KEYS = [\n  LLM_OUTPUT_TOKENS,\n  'inference.llm.output_tokens',\n  'llm.output_tokens',\n  ...GEN_AI_OUTPUT_TOKEN_KEYS,\n  'tangle.tokens.out',\n  'tokens.out',\n] as const\n\n/** Reasoning-token subset of output, when a producer exposes it separately. */\nexport const LLM_REASONING_TOKEN_ATTR_KEYS = [\n  LLM_REASONING_TOKENS,\n  'inference.llm.reasoning_tokens',\n  'llm.reasoning_tokens',\n  'gen_ai.usage.reasoning_tokens',\n  'gen_ai.usage.reasoning_output_tokens',\n  'reasoning_tokens',\n  'reasoning_output_tokens',\n  'tangle.tokens.reasoning',\n  'gen_ai.usage.output_tokens_details.reasoning_tokens',\n  'gen_ai.usage.completion_tokens_details.reasoning_tokens',\n] as const\n\nexport const LLM_CACHED_TOKEN_ATTR_KEYS = [\n  LLM_CACHED_TOKENS,\n  'inference.llm.cached_tokens',\n  'llm.cached_tokens',\n  'gen_ai.usage.cached_tokens',\n  'gen_ai.usage.prompt_tokens_details.cached_tokens',\n  'gen_ai.usage.input_tokens_details.cached_tokens',\n  'gen_ai.usage.cache_read_tokens',\n  'gen_ai.usage.cache_read_input_tokens',\n  'cache_read_tokens',\n  'cache_read_input_tokens',\n  'input_cache_read',\n  'tangle.tokens.cached',\n] as const\n\nexport const LLM_CACHE_WRITE_TOKEN_ATTR_KEYS = [\n  LLM_CACHE_WRITE_TOKENS,\n  'inference.llm.cache_write_tokens',\n  'llm.cache_write_tokens',\n  'gen_ai.usage.cache_creation_tokens',\n  'gen_ai.usage.cache_creation_input_tokens',\n  'cache_creation_tokens',\n  'cache_creation_input_tokens',\n  'input_cache_creation',\n  'tangle.tokens.cache_write',\n] as const\n\nexport const LLM_COST_ATTR_KEYS = [\n  LLM_COST_USD,\n  'inference.llm.cost.total',\n  'llm.cost.total',\n  'gen_ai.usage.cost',\n  'gen_ai.usage.cost_usd',\n  'tangle.cost.usd',\n  'cost.usd',\n  'cost',\n] as const\n\n/** Explicit run-total cost keys safe to preserve on an untyped span. */\nexport const RUN_COST_ATTR_KEYS = ['tangle.cost.usd', 'cost.usd'] as const\n\nexport const TOOL_NAME_ATTR_KEYS = [TOOL_NAME, 'inference.tool.name'] as const\n\n/** Read a numeric attribute, tolerating numeric strings; `null` if absent or invalid. */\nexport function asNumber(value: unknown): number | null {\n  if (typeof value === 'number') return Number.isFinite(value) ? value : null\n  if (typeof value === 'string' && value.length > 0) {\n    const parsed = Number(value)\n    return Number.isFinite(parsed) ? parsed : null\n  }\n  return null\n}\n\n/** First finite numeric value across a list of candidate attribute keys. */\nexport function firstNumberAttr(\n  attributes: Record<string, unknown>,\n  keys: readonly string[],\n): number | null {\n  for (const key of keys) {\n    const value = asNumber(attributes[key])\n    if (value !== null) return value\n  }\n  return null\n}\n\n/** Sum producer-supplied, mutually exclusive prompt-token categories. */\nexport function contextInputTokens(usage: {\n  inputTokens?: number | null\n  cachedTokens?: number | null\n  cacheWriteTokens?: number | null\n}): number | undefined {\n  if (usage.inputTokens === undefined || usage.inputTokens === null) return undefined\n  return usage.inputTokens + (usage.cachedTokens ?? 0) + (usage.cacheWriteTokens ?? 0)\n}\n\nexport interface LlmSpanOtlpInput {\n  model?: string\n  inputTokens?: number\n  outputTokens?: number\n  reasoningTokens?: number\n  cachedTokens?: number\n  cacheWriteTokens?: number\n  costUsd?: number\n  finishReason?: string\n}\n\n/** Write the canonical LLM attributes shared by trace producers. */\nexport function applyLlmSpanOtlpAttributes(\n  attributes: Record<string, unknown>,\n  span: LlmSpanOtlpInput,\n): void {\n  if (span.model !== undefined) attributes[LLM_MODEL_NAME] = span.model\n  if (span.inputTokens !== undefined) attributes[LLM_INPUT_TOKENS] = span.inputTokens\n  if (span.outputTokens !== undefined) attributes[LLM_OUTPUT_TOKENS] = span.outputTokens\n  if (span.reasoningTokens !== undefined) attributes[LLM_REASONING_TOKENS] = span.reasoningTokens\n  if (span.cachedTokens !== undefined) attributes[LLM_CACHED_TOKENS] = span.cachedTokens\n  if (span.cacheWriteTokens !== undefined) {\n    attributes[LLM_CACHE_WRITE_TOKENS] = span.cacheWriteTokens\n  }\n  const contextTokens = contextInputTokens(span)\n  if (contextTokens !== undefined) attributes[LLM_CONTEXT_TOKENS] = contextTokens\n  if (span.costUsd !== undefined) attributes[LLM_COST_USD] = span.costUsd\n  if (span.finishReason) attributes['llm.finish_reason'] = span.finishReason\n}\n"],"mappings":";AAEA;AAAA,EACE;AAAA,EACA;AAAA,EACA;AAAA,OACK;AAEA,IAAM,0BAA0B;AAChC,IAAM,iBAAiB;AACvB,IAAM,mBAAmB;AACzB,IAAM,oBAAoB;AAC1B,IAAM,uBAAuB;AAC7B,IAAM,oBAAoB;AAC1B,IAAM,yBAAyB;AAE/B,IAAM,qBAAqB;AAC3B,IAAM,eAAe;AACrB,IAAM,YAAY;AAClB,IAAM,qBAAqB;AAC3B,IAAM,kBAAkB;AACxB,IAAM,cAAc;AACpB,IAAM,eAAe;AAErB,IAAM,sBAAsB,CAAC,yBAAyB,4BAA4B;AAElF,IAAM,sBAAsB;AAAA,EACjC;AAAA,EACA;AAAA,EACA;AAAA,EACA,GAAG;AAAA,EACH;AACF;AAEO,IAAM,4BAA4B;AAAA,EACvC;AAAA,EACA;AAAA,EACA;AAAA,EACA,GAAG;AAAA,EACH;AAAA,EACA;AACF;AAEO,IAAM,6BAA6B;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA,GAAG;AAAA,EACH;AAAA,EACA;AACF;AAGO,IAAM,gCAAgC;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,IAAM,6BAA6B;AAAA,EACxC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,IAAM,kCAAkC;AAAA,EAC7C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,IAAM,qBAAqB;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAGO,IAAM,qBAAqB,CAAC,mBAAmB,UAAU;AAEzD,IAAM,sBAAsB,CAAC,WAAW,qBAAqB;AAG7D,SAAS,SAAS,OAA+B;AACtD,MAAI,OAAO,UAAU,SAAU,QAAO,OAAO,SAAS,KAAK,IAAI,QAAQ;AACvE,MAAI,OAAO,UAAU,YAAY,MAAM,SAAS,GAAG;AACjD,UAAM,SAAS,OAAO,KAAK;AAC3B,WAAO,OAAO,SAAS,MAAM,IAAI,SAAS;AAAA,EAC5C;AACA,SAAO;AACT;AAGO,SAAS,gBACd,YACA,MACe;AACf,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,SAAS,WAAW,GAAG,CAAC;AACtC,QAAI,UAAU,KAAM,QAAO;AAAA,EAC7B;AACA,SAAO;AACT;AAGO,SAAS,mBAAmB,OAIZ;AACrB,MAAI,MAAM,gBAAgB,UAAa,MAAM,gBAAgB,KAAM,QAAO;AAC1E,SAAO,MAAM,eAAe,MAAM,gBAAgB,MAAM,MAAM,oBAAoB;AACpF;AAcO,SAAS,2BACd,YACA,MACM;AACN,MAAI,KAAK,UAAU,OAAW,YAAW,cAAc,IAAI,KAAK;AAChE,MAAI,KAAK,gBAAgB,OAAW,YAAW,gBAAgB,IAAI,KAAK;AACxE,MAAI,KAAK,iBAAiB,OAAW,YAAW,iBAAiB,IAAI,KAAK;AAC1E,MAAI,KAAK,oBAAoB,OAAW,YAAW,oBAAoB,IAAI,KAAK;AAChF,MAAI,KAAK,iBAAiB,OAAW,YAAW,iBAAiB,IAAI,KAAK;AAC1E,MAAI,KAAK,qBAAqB,QAAW;AACvC,eAAW,sBAAsB,IAAI,KAAK;AAAA,EAC5C;AACA,QAAM,gBAAgB,mBAAmB,IAAI;AAC7C,MAAI,kBAAkB,OAAW,YAAW,kBAAkB,IAAI;AAClE,MAAI,KAAK,YAAY,OAAW,YAAW,YAAY,IAAI,KAAK;AAChE,MAAI,KAAK,aAAc,YAAW,mBAAmB,IAAI,KAAK;AAChE;","names":[]}

package/dist/chunk-K6N6XJJX.js

@@ -1,306 +0,0 @@
-import {
-  TraceEmitter
-} from "./chunk-VQMK5FMP.js";
-import {
-  ConfigError
-} from "./chunk-ONWEPEDO.js";
-
-// src/sandbox-harness.ts
-var vitestTestParser = {
-  id: "vitest",
-  parse(stdout) {
-    const m = stdout.match(/Tests\s+(\d+)\s+(passed|failed)(?:\s*\|\s*(\d+)\s+(passed|failed))?/i);
-    if (!m) return void 0;
-    let passed = 0;
-    let failed = 0;
-    const a = parseInt(m[1], 10);
-    const aLabel = m[2].toLowerCase();
-    if (aLabel === "passed") passed += a;
-    else failed += a;
-    if (m[3] && m[4]) {
-      const b = parseInt(m[3], 10);
-      if (m[4].toLowerCase() === "passed") passed += b;
-      else failed += b;
-    }
-    return { testsTotal: passed + failed, testsPassed: passed };
-  }
-};
-var pytestTestParser = {
-  id: "pytest",
-  parse(stdout) {
-    const total = stdout.match(/collected\s+(\d+)\s+items?/i);
-    const passed = stdout.match(/(\d+)\s+passed/);
-    if (!total || !passed) return void 0;
-    return { testsTotal: parseInt(total[1], 10), testsPassed: parseInt(passed[1], 10) };
-  }
-};
-var jestTestParser = {
-  id: "jest",
-  parse(stdout) {
-    const m = stdout.match(/Tests:\s+(?:(\d+)\s+failed[^,]*,\s*)?(\d+)\s+passed,\s+(\d+)\s+total/i);
-    if (!m) return void 0;
-    return { testsTotal: parseInt(m[3], 10), testsPassed: parseInt(m[2], 10) };
-  }
-};
-function composeParsers(...parsers) {
-  return {
-    id: parsers.map((p) => p.id).join("|"),
-    parse(stdout, stderr, exitCode) {
-      for (const p of parsers) {
-        const res = p.parse(stdout, stderr, exitCode);
-        if (res) return res;
-      }
-      return void 0;
-    }
-  };
-}
-var SubprocessSandboxDriver = class {
-  id = "subprocess";
-  defaultCwd;
-  defaultEnv;
-  constructor(options = {}) {
-    this.defaultCwd = options.cwd;
-    this.defaultEnv = options.env;
-  }
-  async exec(phase, command, config) {
-    const { spawn } = await import("child_process");
-    const start = Date.now();
-    const effectiveCwd = config.cwd ?? this.defaultCwd;
-    const effectiveEnv = { ...process.env, ...this.defaultEnv ?? {}, ...config.env ?? {} };
-    const maxOutputBytes = config.maxOutputBytes ?? 16 * 1024 * 1024;
-    return await new Promise((resolve) => {
-      const child = spawn(command, {
-        shell: true,
-        cwd: effectiveCwd,
-        env: effectiveEnv,
-        // Own process group so a timeout can SIGKILL the whole tree, not
-        // just the shell — otherwise a runaway grandchild keeps the pipes
-        // open and `close` never fires (the timeout silently does nothing).
-        detached: process.platform !== "win32"
-      });
-      let stdout = "";
-      let stderr = "";
-      let outputBytes = 0;
-      let outputTruncated = false;
-      let killedByTimeout = false;
-      let settled = false;
-      const onStdout = (d) => {
-        const chunk = capture(String(d));
-        if (chunk) stdout += chunk;
-      };
-      const onStderr = (d) => {
-        const chunk = capture(String(d));
-        if (chunk) stderr += chunk;
-      };
-      function capture(s) {
-        if (outputBytes >= maxOutputBytes) {
-          outputTruncated = true;
-          return "";
-        }
-        const room = maxOutputBytes - outputBytes;
-        if (s.length > room) {
-          outputBytes = maxOutputBytes;
-          outputTruncated = true;
-          return s.slice(0, room);
-        }
-        outputBytes += s.length;
-        return s;
-      }
-      child.stdout?.on("data", onStdout);
-      child.stderr?.on("data", onStderr);
-      const cleanup = () => {
-        clearTimeout(timeout);
-        if (forceResolve) clearTimeout(forceResolve);
-        child.stdout?.off("data", onStdout);
-        child.stderr?.off("data", onStderr);
-        child.removeAllListeners("close");
-        child.removeAllListeners("error");
-      };
-      const finish = (outcome) => {
-        if (settled) return;
-        settled = true;
-        cleanup();
-        const wallMs = Date.now() - start;
-        const exitCode = killedByTimeout ? outcome.code && outcome.code !== 0 ? outcome.code : 124 : outcome.code ?? 1;
-        const parsed = !killedByTimeout && phase === "test" && config.testParser ? config.testParser.parse(stdout, stderr, exitCode) : void 0;
-        resolve({
-          phase,
-          exitCode,
-          stdout,
-          stderr: outcome.runnerError ? stderr + outcome.runnerError : stderr,
-          wallMs,
-          testsTotal: parsed?.testsTotal,
-          testsPassed: parsed?.testsPassed,
-          killedByTimeout: killedByTimeout || void 0,
-          outputTruncated: outputTruncated || void 0
-        });
-      };
-      const killTree = () => {
-        try {
-          if (process.platform !== "win32" && typeof child.pid === "number") {
-            process.kill(-child.pid, "SIGKILL");
-          } else {
-            child.kill("SIGKILL");
-          }
-        } catch (err) {
-          console.warn("[sandbox-harness] SIGKILL on timeout failed:", err);
-        }
-      };
-      let forceResolve;
-      const timeout = setTimeout(
-        () => {
-          killedByTimeout = true;
-          killTree();
-          forceResolve = setTimeout(() => finish({ code: null }), 2e3);
-        },
-        config.timeoutMs ?? 10 * 6e4
-      );
-      child.on("close", (code) => {
-        if (forceResolve) clearTimeout(forceResolve);
-        finish({ code });
-      });
-      child.on("error", (err) => {
-        if (forceResolve) clearTimeout(forceResolve);
-        finish({ code: 127, runnerError: String(err) });
-      });
-    });
-  }
-};
-var DockerSandboxDriver = class {
-  id = "docker";
-  async exec(phase, command, config) {
-    if (!config.image) throw new ConfigError("DockerSandboxDriver requires config.image");
-    const sub = new SubprocessSandboxDriver();
-    const envArgs = Object.entries(config.env ?? {}).map(([k, v]) => `-e ${shellQuote(k)}=${shellQuote(v)}`).join(" ");
-    const wrapped = `docker run --rm ${envArgs} ${shellQuote(config.image)} sh -c ${shellQuote(command)}`;
-    return sub.exec(phase, wrapped, { ...config, env: void 0 });
-  }
-};
-function shellQuote(v) {
-  if (/^[A-Za-z0-9_\-/.@:=]+$/.test(v)) return v;
-  return `'${v.replace(/'/g, `'\\''`)}'`;
-}
-var SandboxHarness = class {
-  driver;
-  constructor(driver = new SubprocessSandboxDriver()) {
-    this.driver = driver;
-  }
-  async run(config, emitter) {
-    const handle = await emitter.sandbox({
-      name: `sandbox(${this.driver.id})`,
-      image: config.image,
-      command: [config.setupCommand, config.runCommand, config.testCommand].filter(Boolean).join(" && ")
-    });
-    const result = { passed: false, totalWallMs: 0, score: 0 };
-    try {
-      if (config.setupCommand) {
-        result.setup = await this.driver.exec("setup", config.setupCommand, config);
-        result.totalWallMs += result.setup.wallMs;
-        if (result.setup.killedByTimeout || result.setup.exitCode !== 0) {
-          const reason = result.setup.killedByTimeout ? `setup timed out after ${result.setup.wallMs}ms` : `setup failed (exit ${result.setup.exitCode})`;
-          await handle.fail(reason, {
-            exitCode: result.setup.exitCode,
-            wallMs: result.totalWallMs
-          });
-          return result;
-        }
-      }
-      if (config.runCommand) {
-        result.run = await this.driver.exec("run", config.runCommand, config);
-        result.totalWallMs += result.run.wallMs;
-        if (result.run.killedByTimeout || result.run.exitCode !== 0) {
-          const reason = result.run.killedByTimeout ? `run timed out after ${result.run.wallMs}ms` : `run failed (exit ${result.run.exitCode})`;
-          await handle.fail(reason, {
-            exitCode: result.run.exitCode,
-            wallMs: result.totalWallMs
-          });
-          return result;
-        }
-      }
-      if (config.testCommand) {
-        result.test = await this.driver.exec("test", config.testCommand, config);
-        result.totalWallMs += result.test.wallMs;
-        const passed = !result.test.killedByTimeout && result.test.exitCode === 0;
-        result.passed = passed;
-        if (result.test.testsTotal !== void 0 && result.test.testsTotal > 0) {
-          result.score = (result.test.testsPassed ?? 0) / result.test.testsTotal;
-        } else {
-          result.score = passed ? 1 : 0;
-        }
-        await handle.end({
-          exitCode: result.test.exitCode,
-          testsTotal: result.test.testsTotal,
-          testsPassed: result.test.testsPassed,
-          wallMs: result.totalWallMs,
-          status: passed ? "ok" : "error"
-        });
-      } else {
-        result.passed = true;
-        result.score = 1;
-        await handle.end({ wallMs: result.totalWallMs });
-      }
-    } catch (err) {
-      await handle.fail(err instanceof Error ? err : String(err));
-      throw err;
-    }
-    return result;
-  }
-};
-
-// src/test-graded-scenario.ts
-async function runTestGradedScenario(scenario, store, options = {}) {
-  const emitter = new TraceEmitter(store);
-  await emitter.startRun({
-    scenarioId: scenario.id,
-    variantId: options.variantId,
-    datasetVersion: scenario.datasetVersion,
-    tags: scenario.tags,
-    ...options.provenance
-  });
-  const harness = new SandboxHarness(options.driver);
-  const result = await harness.run(scenario.harness, emitter);
-  const threshold = scenario.passThreshold ?? 1;
-  const pass = result.passed && result.score >= threshold;
-  const setupFailed = result.setup !== void 0 && result.setup.exitCode !== 0;
-  const runFailed = result.run !== void 0 && result.run.exitCode !== 0;
-  const testFailed = result.test !== void 0 && result.test.exitCode !== 0;
-  const failureClass = pass ? "success" : setupFailed || runFailed ? "sandbox_failure" : testFailed ? "format_drift" : "unknown";
-  await emitter.endRun({
-    pass,
-    score: result.score,
-    failureClass,
-    notes: pass ? void 0 : reasonForFailure(result)
-  });
-  return {
-    runId: emitter.runId,
-    scenario,
-    harness: result,
-    pass,
-    score: result.score,
-    failureClass
-  };
-}
-function reasonForFailure(result) {
-  if (result.setup && result.setup.exitCode !== 0)
-    return `setup failed: exit ${result.setup.exitCode}`;
-  if (result.run && result.run.exitCode !== 0) return `run failed: exit ${result.run.exitCode}`;
-  if (result.test) {
-    if (result.test.testsTotal !== void 0) {
-      return `tests: ${result.test.testsPassed ?? 0}/${result.test.testsTotal}`;
-    }
-    return `test exit ${result.test.exitCode}`;
-  }
-  return "no test command";
-}
-
-export {
-  vitestTestParser,
-  pytestTestParser,
-  jestTestParser,
-  composeParsers,
-  SubprocessSandboxDriver,
-  DockerSandboxDriver,
-  SandboxHarness,
-  runTestGradedScenario
-};
-//# sourceMappingURL=chunk-K6N6XJJX.js.map

package/dist/chunk-K6N6XJJX.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/sandbox-harness.ts","../src/test-graded-scenario.ts"],"sourcesContent":["/**\n * SandboxHarness — executes a scenario in an isolated environment and\n * emits a rich SandboxSpan into the trace.\n *\n * Two built-in drivers:\n *   - `SubprocessSandboxDriver` — spawn in a local cwd with env vars.\n *     Fast, no dependencies, fine for unit tests and most CI gates.\n *   - `DockerSandboxDriver` — lifted from tangle-router's sandbox path;\n *     shells out to `docker run`. Stronger isolation, slower startup.\n *\n * Consumers implement `SandboxDriver` for custom backends (Firecracker,\n * Cloudflare sandbox product, etc.). The harness doesn't care which.\n */\n\nimport { ConfigError } from './errors'\nimport type { TraceEmitter } from './trace/emitter'\nimport type { SandboxSpan } from './trace/schema'\n\nexport interface HarnessConfig {\n  /** Setup command (e.g. \"pnpm install\"). Non-zero exit fails the run. */\n  setupCommand?: string\n  /** Run command (e.g. \"pnpm build\"). */\n  runCommand?: string\n  /** Test command (e.g. \"pnpm test --run\"). Drives the test count + pass count. */\n  testCommand?: string\n  /** Absolute cwd for the subprocess driver. Ignored by docker driver. */\n  cwd?: string\n  /** Max wall-clock per phase in ms. Default 10 minutes. */\n  timeoutMs?: number\n  /**\n   * Cap on captured stdout+stderr bytes per phase. A runaway process can\n   * otherwise grow the in-memory buffer without bound. Once hit, further\n   * output is dropped and `outputTruncated` is set. Default 16 MiB.\n   */\n  maxOutputBytes?: number\n  /** Image for the docker driver. */\n  image?: string\n  /** Extra env vars (validated; shell-escaped). */\n  env?: Record<string, string>\n  /** Parser for the test output — maps stdout/stderr/exit code → pass count. */\n  testParser?: TestOutputParser\n}\n\nexport interface TestOutputParser {\n  id: string\n  parse(\n    stdout: string,\n    stderr: string,\n    exitCode: number,\n  ): { testsTotal: number; testsPassed: number } | undefined\n}\n\nexport interface SandboxResult {\n  phase: 'setup' | 'run' | 'test'\n  exitCode: number\n  stdout: string\n  stderr: string\n  wallMs: number\n  testsTotal?: number\n  testsPassed?: number\n  /**\n   * True when the process was killed because it exceeded `timeoutMs`. A\n   * SIGKILLed child can still close with exit code 0; callers MUST treat\n   * a timed-out phase as a hard failure regardless of `exitCode`, never\n   * as a pass. `undefined`/`false` means the process completed on its own.\n   */\n  killedByTimeout?: boolean\n  /**\n   * True when captured stdout/stderr hit `maxOutputBytes` and further\n   * output was dropped. The result is still returned (the process was\n   * not killed for this), but downstream parsers see truncated text.\n   */\n  outputTruncated?: boolean\n}\n\nexport interface SandboxDriver {\n  id: string\n  exec(\n    phase: SandboxResult['phase'],\n    command: string,\n    config: HarnessConfig,\n  ): Promise<SandboxResult>\n}\n\n// ── Parsers ──────────────────────────────────────────────────────────\n\n/** Vitest default summary line: \"Tests  X passed | Y failed\". */\nexport const vitestTestParser: TestOutputParser = {\n  id: 'vitest',\n  parse(stdout) {\n    const m = stdout.match(/Tests\\s+(\\d+)\\s+(passed|failed)(?:\\s*\\|\\s*(\\d+)\\s+(passed|failed))?/i)\n    if (!m) return undefined\n    let passed = 0\n    let failed = 0\n    const a = parseInt(m[1]!, 10)\n    const aLabel = m[2]!.toLowerCase()\n    if (aLabel === 'passed') passed += a\n    else failed += a\n    if (m[3] && m[4]) {\n      const b = parseInt(m[3], 10)\n      if (m[4].toLowerCase() === 'passed') passed += b\n      else failed += b\n    }\n    return { testsTotal: passed + failed, testsPassed: passed }\n  },\n}\n\n/** Pytest default: \"collected N items\" + \" X passed, Y failed\". */\nexport const pytestTestParser: TestOutputParser = {\n  id: 'pytest',\n  parse(stdout) {\n    const total = stdout.match(/collected\\s+(\\d+)\\s+items?/i)\n    const passed = stdout.match(/(\\d+)\\s+passed/)\n    if (!total || !passed) return undefined\n    return { testsTotal: parseInt(total[1]!, 10), testsPassed: parseInt(passed[1]!, 10) }\n  },\n}\n\n/** Jest: \"Tests: X passed, Y total\" (and optional failed). */\nexport const jestTestParser: TestOutputParser = {\n  id: 'jest',\n  parse(stdout) {\n    const m = stdout.match(/Tests:\\s+(?:(\\d+)\\s+failed[^,]*,\\s*)?(\\d+)\\s+passed,\\s+(\\d+)\\s+total/i)\n    if (!m) return undefined\n    return { testsTotal: parseInt(m[3]!, 10), testsPassed: parseInt(m[2]!, 10) }\n  },\n}\n\n/** Composite parser — tries a list of parsers in order. */\nexport function composeParsers(...parsers: TestOutputParser[]): TestOutputParser {\n  return {\n    id: parsers.map((p) => p.id).join('|'),\n    parse(stdout, stderr, exitCode) {\n      for (const p of parsers) {\n        const res = p.parse(stdout, stderr, exitCode)\n        if (res) return res\n      }\n      return undefined\n    },\n  }\n}\n\n// ── Drivers ──────────────────────────────────────────────────────────\n\nexport interface SubprocessSandboxDriverOptions {\n  /**\n   * Default cwd for all `exec` calls. Used when the per-call `HarnessConfig`\n   * does not set its own `cwd`. Lets callers bind the driver to a working\n   * directory once instead of spreading cwd into every harness config —\n   * useful when the harness config is constructed far from the call site\n   * (e.g. starter-foundry's promoter passes a static HarnessConfig per\n   * family taxonomy but needs a per-run composed-scaffold cwd).\n   */\n  cwd?: string\n  /**\n   * Default env merged into every `exec` call's env (per-call `HarnessConfig.env`\n   * still wins on key collision). Same ergonomic rationale as `cwd` above.\n   */\n  env?: Record<string, string>\n}\n\nexport class SubprocessSandboxDriver implements SandboxDriver {\n  id = 'subprocess'\n  private defaultCwd?: string\n  private defaultEnv?: Record<string, string>\n\n  constructor(options: SubprocessSandboxDriverOptions = {}) {\n    this.defaultCwd = options.cwd\n    this.defaultEnv = options.env\n  }\n\n  async exec(\n    phase: SandboxResult['phase'],\n    command: string,\n    config: HarnessConfig,\n  ): Promise<SandboxResult> {\n    const { spawn } = await import('node:child_process')\n    const start = Date.now()\n    // Per-call config wins; fall back to constructor defaults. Honoring\n    // the constructor `cwd` keeps the subprocess from inheriting Node's\n    // cwd when only the constructor arg is supplied.\n    const effectiveCwd = config.cwd ?? this.defaultCwd\n    const effectiveEnv = { ...process.env, ...(this.defaultEnv ?? {}), ...(config.env ?? {}) }\n    const maxOutputBytes = config.maxOutputBytes ?? 16 * 1024 * 1024\n    return await new Promise<SandboxResult>((resolve) => {\n      const child = spawn(command, {\n        shell: true,\n        cwd: effectiveCwd,\n        env: effectiveEnv,\n        // Own process group so a timeout can SIGKILL the whole tree, not\n        // just the shell — otherwise a runaway grandchild keeps the pipes\n        // open and `close` never fires (the timeout silently does nothing).\n        detached: process.platform !== 'win32',\n      })\n      let stdout = ''\n      let stderr = ''\n      let outputBytes = 0\n      let outputTruncated = false\n      let killedByTimeout = false\n      let settled = false\n\n      const onStdout = (d: unknown) => {\n        const chunk = capture(String(d))\n        if (chunk) stdout += chunk\n      }\n      const onStderr = (d: unknown) => {\n        const chunk = capture(String(d))\n        if (chunk) stderr += chunk\n      }\n      // Bound the in-memory buffer so a runaway process can't OOM the\n      // harness. Once the cap is hit we keep draining the streams (to let\n      // the child reach `close`) but discard the bytes.\n      function capture(s: string): string {\n        if (outputBytes >= maxOutputBytes) {\n          outputTruncated = true\n          return ''\n        }\n        const room = maxOutputBytes - outputBytes\n        if (s.length > room) {\n          outputBytes = maxOutputBytes\n          outputTruncated = true\n          return s.slice(0, room)\n        }\n        outputBytes += s.length\n        return s\n      }\n\n      child.stdout?.on('data', onStdout)\n      child.stderr?.on('data', onStderr)\n\n      const cleanup = () => {\n        clearTimeout(timeout)\n        if (forceResolve) clearTimeout(forceResolve)\n        child.stdout?.off('data', onStdout)\n        child.stderr?.off('data', onStderr)\n        child.removeAllListeners('close')\n        child.removeAllListeners('error')\n      }\n\n      // Resolve exactly once. `code`/`signal` come from `close`; on the\n      // timeout path we force a non-zero exit so a SIGKILLed child that\n      // reports exit 0 can never read as a clean pass downstream.\n      const finish = (outcome: { code: number | null; runnerError?: string }) => {\n        if (settled) return\n        settled = true\n        cleanup()\n        const wallMs = Date.now() - start\n        const exitCode = killedByTimeout\n          ? outcome.code && outcome.code !== 0\n            ? outcome.code\n            : 124\n          : (outcome.code ?? 1)\n        const parsed =\n          !killedByTimeout && phase === 'test' && config.testParser\n            ? config.testParser.parse(stdout, stderr, exitCode)\n            : undefined\n        resolve({\n          phase,\n          exitCode,\n          stdout,\n          stderr: outcome.runnerError ? stderr + outcome.runnerError : stderr,\n          wallMs,\n          testsTotal: parsed?.testsTotal,\n          testsPassed: parsed?.testsPassed,\n          killedByTimeout: killedByTimeout || undefined,\n          outputTruncated: outputTruncated || undefined,\n        })\n      }\n\n      // Kill the whole process group on win32-less platforms (we spawned\n      // detached). On a clean close this is a no-op.\n      const killTree = () => {\n        try {\n          if (process.platform !== 'win32' && typeof child.pid === 'number') {\n            process.kill(-child.pid, 'SIGKILL')\n          } else {\n            child.kill('SIGKILL')\n          }\n        } catch (err) {\n          console.warn('[sandbox-harness] SIGKILL on timeout failed:', err)\n        }\n      }\n\n      let forceResolve: ReturnType<typeof setTimeout> | undefined\n      const timeout = setTimeout(\n        () => {\n          killedByTimeout = true\n          killTree()\n          // Give `close` a brief window to fire (flushing any final output)\n          // after the group dies; if an orphaned grandchild keeps the pipes\n          // open, force-resolve so the harness can't hang on a runaway.\n          forceResolve = setTimeout(() => finish({ code: null }), 2000)\n        },\n        config.timeoutMs ?? 10 * 60_000,\n      )\n\n      child.on('close', (code) => {\n        if (forceResolve) clearTimeout(forceResolve)\n        finish({ code })\n      })\n      child.on('error', (err) => {\n        if (forceResolve) clearTimeout(forceResolve)\n        finish({ code: 127, runnerError: String(err) })\n      })\n    })\n  }\n}\n\nexport class DockerSandboxDriver implements SandboxDriver {\n  id = 'docker'\n\n  async exec(\n    phase: SandboxResult['phase'],\n    command: string,\n    config: HarnessConfig,\n  ): Promise<SandboxResult> {\n    if (!config.image) throw new ConfigError('DockerSandboxDriver requires config.image')\n    const sub = new SubprocessSandboxDriver()\n    const envArgs = Object.entries(config.env ?? {})\n      .map(([k, v]) => `-e ${shellQuote(k)}=${shellQuote(v)}`)\n      .join(' ')\n    const wrapped = `docker run --rm ${envArgs} ${shellQuote(config.image)} sh -c ${shellQuote(command)}`\n    return sub.exec(phase, wrapped, { ...config, env: undefined })\n  }\n}\n\nfunction shellQuote(v: string): string {\n  if (/^[A-Za-z0-9_\\-/.@:=]+$/.test(v)) return v\n  return `'${v.replace(/'/g, `'\\\\''`)}'`\n}\n\n// ── Harness orchestration ────────────────────────────────────────────\n\nexport interface SandboxHarnessResult {\n  passed: boolean\n  setup?: SandboxResult\n  run?: SandboxResult\n  test?: SandboxResult\n  totalWallMs: number\n  /** Final score — 0 when no tests; otherwise testsPassed/testsTotal. */\n  score: number\n}\n\nexport class SandboxHarness {\n  private driver: SandboxDriver\n  constructor(driver: SandboxDriver = new SubprocessSandboxDriver()) {\n    this.driver = driver\n  }\n\n  async run(config: HarnessConfig, emitter: TraceEmitter): Promise<SandboxHarnessResult> {\n    const handle = await emitter.sandbox({\n      name: `sandbox(${this.driver.id})`,\n      image: config.image,\n      command: [config.setupCommand, config.runCommand, config.testCommand]\n        .filter(Boolean)\n        .join(' && '),\n    })\n    const result: SandboxHarnessResult = { passed: false, totalWallMs: 0, score: 0 }\n    try {\n      if (config.setupCommand) {\n        result.setup = await this.driver.exec('setup', config.setupCommand, config)\n        result.totalWallMs += result.setup.wallMs\n        if (result.setup.killedByTimeout || result.setup.exitCode !== 0) {\n          const reason = result.setup.killedByTimeout\n            ? `setup timed out after ${result.setup.wallMs}ms`\n            : `setup failed (exit ${result.setup.exitCode})`\n          await handle.fail(reason, {\n            exitCode: result.setup.exitCode,\n            wallMs: result.totalWallMs,\n          } as Partial<SandboxSpan>)\n          return result\n        }\n      }\n      if (config.runCommand) {\n        result.run = await this.driver.exec('run', config.runCommand, config)\n        result.totalWallMs += result.run.wallMs\n        if (result.run.killedByTimeout || result.run.exitCode !== 0) {\n          const reason = result.run.killedByTimeout\n            ? `run timed out after ${result.run.wallMs}ms`\n            : `run failed (exit ${result.run.exitCode})`\n          await handle.fail(reason, {\n            exitCode: result.run.exitCode,\n            wallMs: result.totalWallMs,\n          } as Partial<SandboxSpan>)\n          return result\n        }\n      }\n      if (config.testCommand) {\n        result.test = await this.driver.exec('test', config.testCommand, config)\n        result.totalWallMs += result.test.wallMs\n        // A timed-out test phase is a hard fail regardless of the exit code\n        // a SIGKILLed child happens to report — never let it score as a pass.\n        const passed = !result.test.killedByTimeout && result.test.exitCode === 0\n        result.passed = passed\n        if (result.test.testsTotal !== undefined && result.test.testsTotal > 0) {\n          result.score = (result.test.testsPassed ?? 0) / result.test.testsTotal\n        } else {\n          result.score = passed ? 1 : 0\n        }\n        await handle.end({\n          exitCode: result.test.exitCode,\n          testsTotal: result.test.testsTotal,\n          testsPassed: result.test.testsPassed,\n          wallMs: result.totalWallMs,\n          status: passed ? 'ok' : 'error',\n        } as Partial<SandboxSpan>)\n      } else {\n        result.passed = true\n        result.score = 1\n        await handle.end({ wallMs: result.totalWallMs } as Partial<SandboxSpan>)\n      }\n    } catch (err) {\n      await handle.fail(err instanceof Error ? err : String(err))\n      throw err\n    }\n    return result\n  }\n}\n","/**\n * TestGradedScenario — a scenario whose score comes from a test suite.\n *\n * This is the SWE-bench pattern generalized. The scenario ships:\n *   - fixture data (setup instructions)\n *   - a test command the harness runs\n *   - optional assertion overrides\n *\n * The runner emits a run, delegates to SandboxHarness, records the\n * outcome, and returns a structured verdict. Consumers bind their own\n * agent execution to this contract.\n */\n\nimport type { HarnessConfig, SandboxDriver, SandboxHarnessResult } from './sandbox-harness'\nimport { SandboxHarness } from './sandbox-harness'\nimport { TraceEmitter } from './trace/emitter'\nimport type { FailureClass, Run } from './trace/schema'\nimport type { TraceStore } from './trace/store'\n\nexport interface TestGradedScenario {\n  id: string\n  description?: string\n  harness: HarnessConfig\n  /** Optional pass threshold in 0..1 (default 1.0 = all tests must pass). */\n  passThreshold?: number\n  /** Provenance for dataset tracking. */\n  datasetVersion?: string\n  /** Free-form tags (difficulty, category, etc.). */\n  tags?: Record<string, string>\n}\n\nexport interface TestGradedRunOptions {\n  variantId?: string\n  driver?: SandboxDriver\n  /** Metadata recorded on the Run (codeSha, promptSha, modelFingerprint, seed). */\n  provenance?: Pick<Run, 'codeSha' | 'promptSha' | 'modelFingerprint' | 'seed' | 'envFingerprint'>\n}\n\nexport interface TestGradedRunResult {\n  runId: string\n  scenario: TestGradedScenario\n  harness: SandboxHarnessResult\n  pass: boolean\n  score: number\n  failureClass?: FailureClass\n}\n\nexport async function runTestGradedScenario(\n  scenario: TestGradedScenario,\n  store: TraceStore,\n  options: TestGradedRunOptions = {},\n): Promise<TestGradedRunResult> {\n  const emitter = new TraceEmitter(store)\n  await emitter.startRun({\n    scenarioId: scenario.id,\n    variantId: options.variantId,\n    datasetVersion: scenario.datasetVersion,\n    tags: scenario.tags,\n    ...options.provenance,\n  })\n  const harness = new SandboxHarness(options.driver)\n  const result = await harness.run(scenario.harness, emitter)\n  const threshold = scenario.passThreshold ?? 1.0\n  const pass = result.passed && result.score >= threshold\n  const setupFailed = result.setup !== undefined && result.setup.exitCode !== 0\n  const runFailed = result.run !== undefined && result.run.exitCode !== 0\n  const testFailed = result.test !== undefined && result.test.exitCode !== 0\n  const failureClass: FailureClass | undefined = pass\n    ? 'success'\n    : setupFailed || runFailed\n      ? 'sandbox_failure'\n      : testFailed\n        ? 'format_drift'\n        : 'unknown'\n  await emitter.endRun({\n    pass,\n    score: result.score,\n    failureClass,\n    notes: pass ? undefined : reasonForFailure(result),\n  })\n  return {\n    runId: emitter.runId,\n    scenario,\n    harness: result,\n    pass,\n    score: result.score,\n    failureClass,\n  }\n}\n\nfunction reasonForFailure(result: SandboxHarnessResult): string {\n  if (result.setup && result.setup.exitCode !== 0)\n    return `setup failed: exit ${result.setup.exitCode}`\n  if (result.run && result.run.exitCode !== 0) return `run failed: exit ${result.run.exitCode}`\n  if (result.test) {\n    if (result.test.testsTotal !== undefined) {\n      return `tests: ${result.test.testsPassed ?? 0}/${result.test.testsTotal}`\n    }\n    return `test exit ${result.test.exitCode}`\n  }\n  return 'no test command'\n}\n"],"mappings":";;;;;;;;AAuFO,IAAM,mBAAqC;AAAA,EAChD,IAAI;AAAA,EACJ,MAAM,QAAQ;AACZ,UAAM,IAAI,OAAO,MAAM,sEAAsE;AAC7F,QAAI,CAAC,EAAG,QAAO;AACf,QAAI,SAAS;AACb,QAAI,SAAS;AACb,UAAM,IAAI,SAAS,EAAE,CAAC,GAAI,EAAE;AAC5B,UAAM,SAAS,EAAE,CAAC,EAAG,YAAY;AACjC,QAAI,WAAW,SAAU,WAAU;AAAA,QAC9B,WAAU;AACf,QAAI,EAAE,CAAC,KAAK,EAAE,CAAC,GAAG;AAChB,YAAM,IAAI,SAAS,EAAE,CAAC,GAAG,EAAE;AAC3B,UAAI,EAAE,CAAC,EAAE,YAAY,MAAM,SAAU,WAAU;AAAA,UAC1C,WAAU;AAAA,IACjB;AACA,WAAO,EAAE,YAAY,SAAS,QAAQ,aAAa,OAAO;AAAA,EAC5D;AACF;AAGO,IAAM,mBAAqC;AAAA,EAChD,IAAI;AAAA,EACJ,MAAM,QAAQ;AACZ,UAAM,QAAQ,OAAO,MAAM,6BAA6B;AACxD,UAAM,SAAS,OAAO,MAAM,gBAAgB;AAC5C,QAAI,CAAC,SAAS,CAAC,OAAQ,QAAO;AAC9B,WAAO,EAAE,YAAY,SAAS,MAAM,CAAC,GAAI,EAAE,GAAG,aAAa,SAAS,OAAO,CAAC,GAAI,EAAE,EAAE;AAAA,EACtF;AACF;AAGO,IAAM,iBAAmC;AAAA,EAC9C,IAAI;AAAA,EACJ,MAAM,QAAQ;AACZ,UAAM,IAAI,OAAO,MAAM,uEAAuE;AAC9F,QAAI,CAAC,EAAG,QAAO;AACf,WAAO,EAAE,YAAY,SAAS,EAAE,CAAC,GAAI,EAAE,GAAG,aAAa,SAAS,EAAE,CAAC,GAAI,EAAE,EAAE;AAAA,EAC7E;AACF;AAGO,SAAS,kBAAkB,SAA+C;AAC/E,SAAO;AAAA,IACL,IAAI,QAAQ,IAAI,CAAC,MAAM,EAAE,EAAE,EAAE,KAAK,GAAG;AAAA,IACrC,MAAM,QAAQ,QAAQ,UAAU;AAC9B,iBAAW,KAAK,SAAS;AACvB,cAAM,MAAM,EAAE,MAAM,QAAQ,QAAQ,QAAQ;AAC5C,YAAI,IAAK,QAAO;AAAA,MAClB;AACA,aAAO;AAAA,IACT;AAAA,EACF;AACF;AAqBO,IAAM,0BAAN,MAAuD;AAAA,EAC5D,KAAK;AAAA,EACG;AAAA,EACA;AAAA,EAER,YAAY,UAA0C,CAAC,GAAG;AACxD,SAAK,aAAa,QAAQ;AAC1B,SAAK,aAAa,QAAQ;AAAA,EAC5B;AAAA,EAEA,MAAM,KACJ,OACA,SACA,QACwB;AACxB,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,eAAoB;AACnD,UAAM,QAAQ,KAAK,IAAI;AAIvB,UAAM,eAAe,OAAO,OAAO,KAAK;AACxC,UAAM,eAAe,EAAE,GAAG,QAAQ,KAAK,GAAI,KAAK,cAAc,CAAC,GAAI,GAAI,OAAO,OAAO,CAAC,EAAG;AACzF,UAAM,iBAAiB,OAAO,kBAAkB,KAAK,OAAO;AAC5D,WAAO,MAAM,IAAI,QAAuB,CAAC,YAAY;AACnD,YAAM,QAAQ,MAAM,SAAS;AAAA,QAC3B,OAAO;AAAA,QACP,KAAK;AAAA,QACL,KAAK;AAAA;AAAA;AAAA;AAAA,QAIL,UAAU,QAAQ,aAAa;AAAA,MACjC,CAAC;AACD,UAAI,SAAS;AACb,UAAI,SAAS;AACb,UAAI,cAAc;AAClB,UAAI,kBAAkB;AACtB,UAAI,kBAAkB;AACtB,UAAI,UAAU;AAEd,YAAM,WAAW,CAAC,MAAe;AAC/B,cAAM,QAAQ,QAAQ,OAAO,CAAC,CAAC;AAC/B,YAAI,MAAO,WAAU;AAAA,MACvB;AACA,YAAM,WAAW,CAAC,MAAe;AAC/B,cAAM,QAAQ,QAAQ,OAAO,CAAC,CAAC;AAC/B,YAAI,MAAO,WAAU;AAAA,MACvB;AAIA,eAAS,QAAQ,GAAmB;AAClC,YAAI,eAAe,gBAAgB;AACjC,4BAAkB;AAClB,iBAAO;AAAA,QACT;AACA,cAAM,OAAO,iBAAiB;AAC9B,YAAI,EAAE,SAAS,MAAM;AACnB,wBAAc;AACd,4BAAkB;AAClB,iBAAO,EAAE,MAAM,GAAG,IAAI;AAAA,QACxB;AACA,uBAAe,EAAE;AACjB,eAAO;AAAA,MACT;AAEA,YAAM,QAAQ,GAAG,QAAQ,QAAQ;AACjC,YAAM,QAAQ,GAAG,QAAQ,QAAQ;AAEjC,YAAM,UAAU,MAAM;AACpB,qBAAa,OAAO;AACpB,YAAI,aAAc,cAAa,YAAY;AAC3C,cAAM,QAAQ,IAAI,QAAQ,QAAQ;AAClC,cAAM,QAAQ,IAAI,QAAQ,QAAQ;AAClC,cAAM,mBAAmB,OAAO;AAChC,cAAM,mBAAmB,OAAO;AAAA,MAClC;AAKA,YAAM,SAAS,CAAC,YAA2D;AACzE,YAAI,QAAS;AACb,kBAAU;AACV,gBAAQ;AACR,cAAM,SAAS,KAAK,IAAI,IAAI;AAC5B,cAAM,WAAW,kBACb,QAAQ,QAAQ,QAAQ,SAAS,IAC/B,QAAQ,OACR,MACD,QAAQ,QAAQ;AACrB,cAAM,SACJ,CAAC,mBAAmB,UAAU,UAAU,OAAO,aAC3C,OAAO,WAAW,MAAM,QAAQ,QAAQ,QAAQ,IAChD;AACN,gBAAQ;AAAA,UACN;AAAA,UACA;AAAA,UACA;AAAA,UACA,QAAQ,QAAQ,cAAc,SAAS,QAAQ,cAAc;AAAA,UAC7D;AAAA,UACA,YAAY,QAAQ;AAAA,UACpB,aAAa,QAAQ;AAAA,UACrB,iBAAiB,mBAAmB;AAAA,UACpC,iBAAiB,mBAAmB;AAAA,QACtC,CAAC;AAAA,MACH;AAIA,YAAM,WAAW,MAAM;AACrB,YAAI;AACF,cAAI,QAAQ,aAAa,WAAW,OAAO,MAAM,QAAQ,UAAU;AACjE,oBAAQ,KAAK,CAAC,MAAM,KAAK,SAAS;AAAA,UACpC,OAAO;AACL,kBAAM,KAAK,SAAS;AAAA,UACtB;AAAA,QACF,SAAS,KAAK;AACZ,kBAAQ,KAAK,gDAAgD,GAAG;AAAA,QAClE;AAAA,MACF;AAEA,UAAI;AACJ,YAAM,UAAU;AAAA,QACd,MAAM;AACJ,4BAAkB;AAClB,mBAAS;AAIT,yBAAe,WAAW,MAAM,OAAO,EAAE,MAAM,KAAK,CAAC,GAAG,GAAI;AAAA,QAC9D;AAAA,QACA,OAAO,aAAa,KAAK;AAAA,MAC3B;AAEA,YAAM,GAAG,SAAS,CAAC,SAAS;AAC1B,YAAI,aAAc,cAAa,YAAY;AAC3C,eAAO,EAAE,KAAK,CAAC;AAAA,MACjB,CAAC;AACD,YAAM,GAAG,SAAS,CAAC,QAAQ;AACzB,YAAI,aAAc,cAAa,YAAY;AAC3C,eAAO,EAAE,MAAM,KAAK,aAAa,OAAO,GAAG,EAAE,CAAC;AAAA,MAChD,CAAC;AAAA,IACH,CAAC;AAAA,EACH;AACF;AAEO,IAAM,sBAAN,MAAmD;AAAA,EACxD,KAAK;AAAA,EAEL,MAAM,KACJ,OACA,SACA,QACwB;AACxB,QAAI,CAAC,OAAO,MAAO,OAAM,IAAI,YAAY,2CAA2C;AACpF,UAAM,MAAM,IAAI,wBAAwB;AACxC,UAAM,UAAU,OAAO,QAAQ,OAAO,OAAO,CAAC,CAAC,EAC5C,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,MAAM,WAAW,CAAC,CAAC,IAAI,WAAW,CAAC,CAAC,EAAE,EACtD,KAAK,GAAG;AACX,UAAM,UAAU,mBAAmB,OAAO,IAAI,WAAW,OAAO,KAAK,CAAC,UAAU,WAAW,OAAO,CAAC;AACnG,WAAO,IAAI,KAAK,OAAO,SAAS,EAAE,GAAG,QAAQ,KAAK,OAAU,CAAC;AAAA,EAC/D;AACF;AAEA,SAAS,WAAW,GAAmB;AACrC,MAAI,yBAAyB,KAAK,CAAC,EAAG,QAAO;AAC7C,SAAO,IAAI,EAAE,QAAQ,MAAM,OAAO,CAAC;AACrC;AAcO,IAAM,iBAAN,MAAqB;AAAA,EAClB;AAAA,EACR,YAAY,SAAwB,IAAI,wBAAwB,GAAG;AACjE,SAAK,SAAS;AAAA,EAChB;AAAA,EAEA,MAAM,IAAI,QAAuB,SAAsD;AACrF,UAAM,SAAS,MAAM,QAAQ,QAAQ;AAAA,MACnC,MAAM,WAAW,KAAK,OAAO,EAAE;AAAA,MAC/B,OAAO,OAAO;AAAA,MACd,SAAS,CAAC,OAAO,cAAc,OAAO,YAAY,OAAO,WAAW,EACjE,OAAO,OAAO,EACd,KAAK,MAAM;AAAA,IAChB,CAAC;AACD,UAAM,SAA+B,EAAE,QAAQ,OAAO,aAAa,GAAG,OAAO,EAAE;AAC/E,QAAI;AACF,UAAI,OAAO,cAAc;AACvB,eAAO,QAAQ,MAAM,KAAK,OAAO,KAAK,SAAS,OAAO,cAAc,MAAM;AAC1E,eAAO,eAAe,OAAO,MAAM;AACnC,YAAI,OAAO,MAAM,mBAAmB,OAAO,MAAM,aAAa,GAAG;AAC/D,gBAAM,SAAS,OAAO,MAAM,kBACxB,yBAAyB,OAAO,MAAM,MAAM,OAC5C,sBAAsB,OAAO,MAAM,QAAQ;AAC/C,gBAAM,OAAO,KAAK,QAAQ;AAAA,YACxB,UAAU,OAAO,MAAM;AAAA,YACvB,QAAQ,OAAO;AAAA,UACjB,CAAyB;AACzB,iBAAO;AAAA,QACT;AAAA,MACF;AACA,UAAI,OAAO,YAAY;AACrB,eAAO,MAAM,MAAM,KAAK,OAAO,KAAK,OAAO,OAAO,YAAY,MAAM;AACpE,eAAO,eAAe,OAAO,IAAI;AACjC,YAAI,OAAO,IAAI,mBAAmB,OAAO,IAAI,aAAa,GAAG;AAC3D,gBAAM,SAAS,OAAO,IAAI,kBACtB,uBAAuB,OAAO,IAAI,MAAM,OACxC,oBAAoB,OAAO,IAAI,QAAQ;AAC3C,gBAAM,OAAO,KAAK,QAAQ;AAAA,YACxB,UAAU,OAAO,IAAI;AAAA,YACrB,QAAQ,OAAO;AAAA,UACjB,CAAyB;AACzB,iBAAO;AAAA,QACT;AAAA,MACF;AACA,UAAI,OAAO,aAAa;AACtB,eAAO,OAAO,MAAM,KAAK,OAAO,KAAK,QAAQ,OAAO,aAAa,MAAM;AACvE,eAAO,eAAe,OAAO,KAAK;AAGlC,cAAM,SAAS,CAAC,OAAO,KAAK,mBAAmB,OAAO,KAAK,aAAa;AACxE,eAAO,SAAS;AAChB,YAAI,OAAO,KAAK,eAAe,UAAa,OAAO,KAAK,aAAa,GAAG;AACtE,iBAAO,SAAS,OAAO,KAAK,eAAe,KAAK,OAAO,KAAK;AAAA,QAC9D,OAAO;AACL,iBAAO,QAAQ,SAAS,IAAI;AAAA,QAC9B;AACA,cAAM,OAAO,IAAI;AAAA,UACf,UAAU,OAAO,KAAK;AAAA,UACtB,YAAY,OAAO,KAAK;AAAA,UACxB,aAAa,OAAO,KAAK;AAAA,UACzB,QAAQ,OAAO;AAAA,UACf,QAAQ,SAAS,OAAO;AAAA,QAC1B,CAAyB;AAAA,MAC3B,OAAO;AACL,eAAO,SAAS;AAChB,eAAO,QAAQ;AACf,cAAM,OAAO,IAAI,EAAE,QAAQ,OAAO,YAAY,CAAyB;AAAA,MACzE;AAAA,IACF,SAAS,KAAK;AACZ,YAAM,OAAO,KAAK,eAAe,QAAQ,MAAM,OAAO,GAAG,CAAC;AAC1D,YAAM;AAAA,IACR;AACA,WAAO;AAAA,EACT;AACF;;;AClXA,eAAsB,sBACpB,UACA,OACA,UAAgC,CAAC,GACH;AAC9B,QAAM,UAAU,IAAI,aAAa,KAAK;AACtC,QAAM,QAAQ,SAAS;AAAA,IACrB,YAAY,SAAS;AAAA,IACrB,WAAW,QAAQ;AAAA,IACnB,gBAAgB,SAAS;AAAA,IACzB,MAAM,SAAS;AAAA,IACf,GAAG,QAAQ;AAAA,EACb,CAAC;AACD,QAAM,UAAU,IAAI,eAAe,QAAQ,MAAM;AACjD,QAAM,SAAS,MAAM,QAAQ,IAAI,SAAS,SAAS,OAAO;AAC1D,QAAM,YAAY,SAAS,iBAAiB;AAC5C,QAAM,OAAO,OAAO,UAAU,OAAO,SAAS;AAC9C,QAAM,cAAc,OAAO,UAAU,UAAa,OAAO,MAAM,aAAa;AAC5E,QAAM,YAAY,OAAO,QAAQ,UAAa,OAAO,IAAI,aAAa;AACtE,QAAM,aAAa,OAAO,SAAS,UAAa,OAAO,KAAK,aAAa;AACzE,QAAM,eAAyC,OAC3C,YACA,eAAe,YACb,oBACA,aACE,iBACA;AACR,QAAM,QAAQ,OAAO;AAAA,IACnB;AAAA,IACA,OAAO,OAAO;AAAA,IACd;AAAA,IACA,OAAO,OAAO,SAAY,iBAAiB,MAAM;AAAA,EACnD,CAAC;AACD,SAAO;AAAA,IACL,OAAO,QAAQ;AAAA,IACf;AAAA,IACA,SAAS;AAAA,IACT;AAAA,IACA,OAAO,OAAO;AAAA,IACd;AAAA,EACF;AACF;AAEA,SAAS,iBAAiB,QAAsC;AAC9D,MAAI,OAAO,SAAS,OAAO,MAAM,aAAa;AAC5C,WAAO,sBAAsB,OAAO,MAAM,QAAQ;AACpD,MAAI,OAAO,OAAO,OAAO,IAAI,aAAa,EAAG,QAAO,oBAAoB,OAAO,IAAI,QAAQ;AAC3F,MAAI,OAAO,MAAM;AACf,QAAI,OAAO,KAAK,eAAe,QAAW;AACxC,aAAO,UAAU,OAAO,KAAK,eAAe,CAAC,IAAI,OAAO,KAAK,UAAU;AAAA,IACzE;AACA,WAAO,aAAa,OAAO,KAAK,QAAQ;AAAA,EAC1C;AACA,SAAO;AACT;","names":[]}

package/dist/chunk-MA6HLL3S.js

@@ -1,65 +0,0 @@
-// src/trace/schema.ts
-var TRACE_SCHEMA_VERSION = "1.0.0";
-var FAILURE_CLASSES = [
-  "success",
-  "reasoning_error",
-  "tool_selection_error",
-  "tool_argument_error",
-  "tool_recovery_failure",
-  "hallucination",
-  "instruction_following",
-  "safety_refusal_miss",
-  "policy_violation",
-  "budget_exceeded",
-  "format_drift",
-  "permission_escalation",
-  "pii_leak",
-  "cost_overrun",
-  "timeout",
-  "sandbox_failure",
-  "missing_user_data",
-  "missing_domain_data",
-  "missing_codebase_context",
-  "missing_runtime_context",
-  "missing_credentials",
-  "missing_integration_connection",
-  "missing_integration_scope",
-  "integration_approval_required",
-  "integration_auth_expired",
-  "integration_provider_failure",
-  "bad_integration_manifest",
-  "unsafe_integration_write_denied",
-  "stale_external_data",
-  "bad_retrieval",
-  "insufficient_evidence",
-  "contradictory_evidence",
-  "ambiguous_user_intent",
-  "knowledge_readiness_blocked",
-  "unknown"
-];
-function isLlmSpan(s) {
-  return s.kind === "llm";
-}
-function isToolSpan(s) {
-  return s.kind === "tool";
-}
-function isRetrievalSpan(s) {
-  return s.kind === "retrieval";
-}
-function isJudgeSpan(s) {
-  return s.kind === "judge";
-}
-function isSandboxSpan(s) {
-  return s.kind === "sandbox";
-}
-
-export {
-  TRACE_SCHEMA_VERSION,
-  FAILURE_CLASSES,
-  isLlmSpan,
-  isToolSpan,
-  isRetrievalSpan,
-  isJudgeSpan,
-  isSandboxSpan
-};
-//# sourceMappingURL=chunk-MA6HLL3S.js.map

package/dist/chunk-MA6HLL3S.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/trace/schema.ts"],"sourcesContent":["/**\n * TraceSchema v1 — the canonical data model for agent-eval.\n *\n * Every score, every failure class, every pipeline in the framework is\n * a view over this data. Shape it once, live with it.\n *\n * Wire-compatible with OpenTelemetry span semantics (see trace/otel.ts)\n * but extended with agent-specific span kinds (llm, tool, retrieval,\n * judge, sandbox) and first-class BudgetLedger / Artifact / JudgeVerdict\n * entities that OTEL leaves as free-form attributes.\n */\n\nexport const TRACE_SCHEMA_VERSION = '1.0.0'\n\n// ── Run ──────────────────────────────────────────────────────────────\n\nexport type RunStatus = 'running' | 'completed' | 'failed' | 'aborted'\n\nexport interface BudgetSpec {\n  tokens?: number\n  wallMs?: number\n  calls?: number\n  usd?: number\n}\n\nexport interface RunOutcome {\n  score?: number\n  pass?: boolean\n  failureClass?: FailureClass\n  notes?: string\n}\n\n/**\n * Layer — optional classification in a nested build workflow.\n * `builder`: the meta-agent editing a project (e.g. agent-builder Forge chat).\n * `app-build`: sandbox harness that compiled + tested the generated scaffold.\n * `app-runtime`: a run of the generated agent against a domain scenario.\n * `meta`: any meta-eval (judge replay, correlation analysis).\n */\nexport type RunLayer = 'builder' | 'app-build' | 'app-runtime' | 'meta' | 'custom'\n\nexport interface Run {\n  runId: string\n  /**\n   * Stable identifier of the scenario being executed.\n   *\n   * Always populated on the persisted Run — but `TraceEmitter.startRun` accepts\n   * input WITHOUT this field, substituting a sensible default\n   * (`run.layer ?? run.tags?.['kind'] ?? 'runtime'`) when the caller has no\n   * curated scenario to anchor to (runtime / operator / meta-eval runs). This\n   * keeps the persisted shape unambiguous for downstream filters + aggregations\n   * while removing the boilerplate of inventing placeholder ids at the call site.\n   */\n  scenarioId: string\n  variantId?: string\n  datasetVersion?: string\n  /** Git SHA of agent code at run time. */\n  codeSha?: string\n  /** Hash of the prompt template + any system prompt. */\n  promptSha?: string\n  /** Model id + date + system-prompt hash, concatenated. */\n  modelFingerprint?: string\n  seed?: number\n  /** Arbitrary environment markers (shell, docker version, tz). */\n  envFingerprint?: Record<string, string>\n  /** Version of the redaction rules applied to this run. */\n  redactionVersion?: string\n  /** Parent run in a nested build workflow. A builder run's children are\n   *  app-build runs; those children are app-runtime runs. */\n  parentRunId?: string\n  /** Stable project identifier — groups runs across chats + sessions. */\n  projectId?: string\n  /** Chat/conversation identifier within a project. */\n  chatId?: string\n  /** Layer classification — hint for aggregation; not enforced. */\n  layer?: RunLayer\n  startedAt: number\n  endedAt?: number\n  status: RunStatus\n  outcome?: RunOutcome\n  budget?: BudgetSpec\n  /** Free-form labels for downstream grouping. */\n  tags?: Record<string, string>\n}\n\n// ── Spans (hierarchical work units) ──────────────────────────────────\n\nexport type SpanKind = 'agent' | 'llm' | 'tool' | 'retrieval' | 'judge' | 'sandbox' | 'custom'\n\nexport type SpanStatus = 'ok' | 'error'\n\nexport interface SpanBase {\n  spanId: string\n  parentSpanId?: string\n  runId: string\n  kind: SpanKind\n  name: string\n  startedAt: number\n  endedAt?: number\n  status?: SpanStatus\n  error?: string\n  /** Anything not covered by typed fields. Kept deliberately free-form. */\n  attributes?: Record<string, unknown>\n}\n\nexport interface Message {\n  role: 'system' | 'user' | 'assistant' | 'tool'\n  content: string\n  tokens?: number\n  /** Multi-modal content descriptors; blobs themselves live in Artifacts. */\n  images?: Array<{ artifactId?: string; url?: string; mime?: string }>\n}\n\nexport interface LlmSpan extends SpanBase {\n  kind: 'llm'\n  model: string\n  messages: Message[]\n  output?: string\n  inputTokens?: number\n  /** All generated tokens, including the reasoning subset when present. */\n  outputTokens?: number\n  cachedTokens?: number\n  cacheWriteTokens?: number\n  /** Reasoning-token subset of `outputTokens`. */\n  reasoningTokens?: number\n  costUsd?: number\n  finishReason?: string\n}\n\nexport interface ToolSpan extends SpanBase {\n  kind: 'tool'\n  toolName: string\n  args: unknown\n  /** False when the source observed the call but did not capture its arguments. */\n  argsCaptured?: boolean\n  result?: unknown\n  latencyMs?: number\n}\n\nexport interface RetrievalSpan extends SpanBase {\n  kind: 'retrieval'\n  query: string\n  hits: Array<{ docId: string; score: number; content?: string }>\n}\n\nexport interface JudgeSpan extends SpanBase {\n  kind: 'judge'\n  judgeId: string\n  /** Span this judgment applies to. */\n  targetSpanId: string\n  dimension: string\n  /** Numeric score (free-range; interpretation up to the judge). */\n  score: number\n  rationale?: string\n  evidence?: string\n}\n\nexport interface SandboxSpan extends SpanBase {\n  kind: 'sandbox'\n  image?: string\n  command?: string\n  exitCode?: number\n  testsTotal?: number\n  testsPassed?: number\n  stdoutHash?: string\n  stderrHash?: string\n  /** Duration in ms; the harness fills this explicitly (endedAt - startedAt may miss setup). */\n  wallMs?: number\n}\n\nexport interface GenericSpan extends SpanBase {\n  kind: 'agent' | 'custom'\n}\n\nexport type Span = LlmSpan | ToolSpan | RetrievalSpan | JudgeSpan | SandboxSpan | GenericSpan\n\n// ── Events (point-in-time occurrences within a span) ─────────────────\n\nexport type EventKind =\n  | 'log'\n  | 'error'\n  | 'budget_decrement'\n  | 'budget_breach'\n  | 'state_mutation'\n  | 'policy_violation'\n  | 'redaction_applied'\n  | 'custom'\n\nexport interface TraceEvent {\n  eventId: string\n  runId: string\n  spanId?: string\n  kind: EventKind\n  timestamp: number\n  payload: Record<string, unknown>\n}\n\n// ── Budget ledger (running token/wall/call/$ accounting) ─────────────\n\nexport interface BudgetLedgerEntry {\n  runId: string\n  dimension: keyof BudgetSpec\n  limit: number\n  consumed: number\n  remaining: number\n  timestamp: number\n  breached: boolean\n  /** Span that triggered this entry, if any. */\n  spanId?: string\n}\n\n// ── Artifacts (blobs addressed by hash) ──────────────────────────────\n\nexport interface Artifact {\n  artifactId: string\n  runId: string\n  spanId?: string\n  contentType: string\n  sizeBytes: number\n  /** sha256 in hex. */\n  hash: string\n  /** External storage URL (R2, S3, filesystem path). */\n  storageUrl?: string\n  /** Inline content for small blobs — keep under ~64KB. */\n  inlineContent?: string\n}\n\n// ── Failure taxonomy ─────────────────────────────────────────────────\n\nexport type FailureClass =\n  | 'success'\n  | 'reasoning_error'\n  | 'tool_selection_error'\n  | 'tool_argument_error'\n  | 'tool_recovery_failure'\n  | 'hallucination'\n  | 'instruction_following'\n  | 'safety_refusal_miss'\n  | 'policy_violation'\n  | 'budget_exceeded'\n  | 'format_drift'\n  | 'permission_escalation'\n  | 'pii_leak'\n  | 'cost_overrun'\n  | 'timeout'\n  | 'sandbox_failure'\n  | 'missing_user_data'\n  | 'missing_domain_data'\n  | 'missing_codebase_context'\n  | 'missing_runtime_context'\n  | 'missing_credentials'\n  | 'missing_integration_connection'\n  | 'missing_integration_scope'\n  | 'integration_approval_required'\n  | 'integration_auth_expired'\n  | 'integration_provider_failure'\n  | 'bad_integration_manifest'\n  | 'unsafe_integration_write_denied'\n  | 'stale_external_data'\n  | 'bad_retrieval'\n  | 'insufficient_evidence'\n  | 'contradictory_evidence'\n  | 'ambiguous_user_intent'\n  | 'knowledge_readiness_blocked'\n  | 'unknown'\n\nexport const FAILURE_CLASSES: readonly FailureClass[] = [\n  'success',\n  'reasoning_error',\n  'tool_selection_error',\n  'tool_argument_error',\n  'tool_recovery_failure',\n  'hallucination',\n  'instruction_following',\n  'safety_refusal_miss',\n  'policy_violation',\n  'budget_exceeded',\n  'format_drift',\n  'permission_escalation',\n  'pii_leak',\n  'cost_overrun',\n  'timeout',\n  'sandbox_failure',\n  'missing_user_data',\n  'missing_domain_data',\n  'missing_codebase_context',\n  'missing_runtime_context',\n  'missing_credentials',\n  'missing_integration_connection',\n  'missing_integration_scope',\n  'integration_approval_required',\n  'integration_auth_expired',\n  'integration_provider_failure',\n  'bad_integration_manifest',\n  'unsafe_integration_write_denied',\n  'stale_external_data',\n  'bad_retrieval',\n  'insufficient_evidence',\n  'contradictory_evidence',\n  'ambiguous_user_intent',\n  'knowledge_readiness_blocked',\n  'unknown',\n] as const\n\n// ── Helpers ──────────────────────────────────────────────────────────\n\nexport function isLlmSpan(s: Span): s is LlmSpan {\n  return s.kind === 'llm'\n}\nexport function isToolSpan(s: Span): s is ToolSpan {\n  return s.kind === 'tool'\n}\nexport function isRetrievalSpan(s: Span): s is RetrievalSpan {\n  return s.kind === 'retrieval'\n}\nexport function isJudgeSpan(s: Span): s is JudgeSpan {\n  return s.kind === 'judge'\n}\nexport function isSandboxSpan(s: Span): s is SandboxSpan {\n  return s.kind === 'sandbox'\n}\n"],"mappings":";AAYO,IAAM,uBAAuB;AA8P7B,IAAM,kBAA2C;AAAA,EACtD;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAIO,SAAS,UAAU,GAAuB;AAC/C,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,WAAW,GAAwB;AACjD,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,gBAAgB,GAA6B;AAC3D,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,YAAY,GAAyB;AACnD,SAAO,EAAE,SAAS;AACpB;AACO,SAAS,cAAc,GAA2B;AACvD,SAAO,EAAE,SAAS;AACpB;","names":[]}