@tangle-network/agent-eval 0.119.0 → 0.120.0

package/dist/{chunk-FIUFRXP3.js → chunk-WW2A73HW.js}

@@ -1,8 +1,8 @@
 import {
   OtlpFileTraceStore,
-  TraceFileMissingError,
-  buildTraceAnalystTools
-} from "./chunk-6QIM2EAP.js";
+  buildTraceAnalystTools,
+  runTraceAnalysisLoop
+} from "./chunk-PXD6ZFNY.js";
 
 // src/trace-analyst/prompts.ts
 var TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \`traces\` namespace.
@@ -32,51 +32,33 @@ DISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol \u2014 follow exactly:
 
 9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.
 
-10. If maxDepth > 0 and the question splits into independent semantic branches, delegate well-defined subtasks to subagents using \`await llmQuery(...)\`. Pass narrow context and a focused query. Examples:
+10. If the question splits into independent reasoning branches, use bounded \`llmQuery(...)\` calls over evidence you already loaded. Subqueries cannot inspect the trace store, so pass the exact trace excerpts they need. Example:
 
     const reviews = await llmQuery([
-      { query: 'Drill into trace abc123 \u2014 what tool calls preceded the failure?', context: { trace_id: 'abc123' } },
-      { query: 'Drill into trace def456 \u2014 same failure mode?', context: { trace_id: 'def456' } },
+      { query: 'Classify the failure mechanism in this excerpt.', context: traceAbcExcerpt },
+      { query: 'Classify the failure mechanism in this excerpt.', context: traceDefExcerpt },
     ]);
 
 OBSERVABILITY rules:
-- Each non-final actor turn must emit at least one \`console.log(...)\` for evidence. Up to 3 logs per turn is fine when correlating multiple data sources (e.g. one log for findings list, one for source-file content, one for derived analysis).
-- Do NOT combine \`console.log\` with \`final(...)\` or \`askClarification(...)\` in the same turn \u2014 finish gathering data first, then call final on its own turn.
+- Each discovery turn must emit at least one concise \`console.log(...)\` showing what evidence was learned.
+- Finish gathering evidence before submitting the analysis.
 - Reuse runtime variables across turns; don't recompute.
-- When done, call \`await final(answer)\` with the fully-formed report. The responder rewrites the answer into output fields; if you only pass a vague summary string the responder has nothing concrete to format.
-
-CRITICAL \u2014 \`final()\` payload contract for evidence-grounded analysis tasks:
-- Pass a STRUCTURED object as the second arg with the actual data the responder needs to format the answer. Do NOT pass abstract instructions; pass evidence.
-- Example for per-item verdict tasks:
-  \`\`\`js
-  await final("Format the per-item verdict report from the evidence below.", {
-    findings: [
-      { id: 'sub-1-finding-1', claim: '...', verdict: 'TRUE-POSITIVE', evidence: 'lines 42-45 of contracts/X.sol show ...' },
-      ...all items
-    ],
-    systemic_summary: '3 sentences I wrote based on the evidence above'
-  });
-  \`\`\`
-- Calling \`final("answer", {})\` with no evidence is a failure mode \u2014 the responder will hallucinate or echo back the field names. Always include the gathered data.
-- Premature final after a single viewSpans call is INSUFFICIENT for per-finding analysis tasks. Read the requested attributes (e.g. \`spans[i].attributes['redteam.finding.title']\`), and for each one perform the requested cross-reference (e.g. read the source SPAN's \`attributes['source.content']\`).
 
 OUTPUT contract \u2014 your final answer must include:
 - A clear prose conclusion answering the user's question.
 - Trace ids and span ids cited as evidence for each claim.
 - Failure modes named in the user's domain language, with frequency and concrete examples.
+- A concise findings array containing only claims supported by inspected evidence.
 
 Do NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.`;
-var TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v5-2026-05-06";
-var TRACE_ANALYST_SUBAGENT_DESCRIPTION = `You are a trace-analyst subagent. Your parent has delegated a focused trace-inspection question. Use the same DISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol but stay tightly scoped: do exactly what was asked, return a concise compact answer, do NOT spawn further subagents unless the parent's question is genuinely multi-branch.
-
-Cite trace ids and span ids for every claim. Do NOT invent ids.`;
+var TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v6-2026-07-14";
 
 // src/trace-analyst/analyst.ts
-import { AxJSRuntime, agent } from "@ax-llm/ax";
 async function analyzeTraces(input, options) {
   if (!input.question || typeof input.question !== "string") {
     throw new TypeError("analyzeTraces: input.question must be a non-empty string");
   }
+  rejectRemovedOptions(options);
   const store = typeof options.source === "string" ? new OtlpFileTraceStore({ path: options.source }) : options.source;
   if (store instanceof OtlpFileTraceStore) {
     await store.ensureIndexed();
@@ -93,6 +75,7 @@ async function analyzeTraces(input, options) {
   }
   const actorTurnCallback = async (turn) => {
     const snap = {
+      stage: turn.stage,
       turn: turn.turn,
       isError: turn.isError,
       code: turn.code,
@@ -109,81 +92,57 @@ async function analyzeTraces(input, options) {
     }
     if (options.onTurn) await options.onTurn(snap);
   };
-  const maxDepth = options.maxDepth ?? 1;
+  const maxSubqueries = options.maxSubqueries ?? 4;
   const maxTurns = options.maxTurns ?? 12;
-  const maxParallelSubagents = options.maxParallelSubagents ?? 2;
+  const maxParallelSubqueries = options.maxParallelSubqueries ?? 2;
   const maxRuntimeChars = options.maxRuntimeChars ?? 6e3;
-  const functions = tools;
-  const analyst = agent(
-    // `reasoning!` is an internal (Ax `!`) scratchpad field: generated first to
-    // force reason-before-conclude, stripped from the returned output — so the
-    // consumed shape stays { answer, findings }. Brings the trace-analyst to the
-    // same prose-first CoT ordering the kind-factory gets from its `report` field.
-    "question:string -> reasoning!:string, answer:string, findings:string[]",
-    {
-      agentIdentity: {
-        name: "TraceAnalyst",
-        description: "Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes."
-      },
-      contextFields: ["question"],
-      runtime: new AxJSRuntime({
-        permissions: [],
-        blockDynamicImport: true,
-        allowedModules: [],
-        freezeIntrinsics: true,
-        blockShadowRealm: true,
-        // RLM stdout mode relies on runtime bindings persisting across turns.
-        preventGlobalThisExtensions: false
-      }),
-      mode: maxDepth > 0 ? "advanced" : "simple",
-      recursionOptions: maxDepth > 0 ? { maxDepth } : void 0,
+  let completed;
+  try {
+    completed = await runTraceAnalysisLoop({
+      id: "TraceAnalyst",
+      description: "Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes.",
+      prompt: `${options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION}
+
+The report must answer the user's question, and findings must be an array of concise evidence-backed strings.`,
+      question: input.question,
+      ai: options.ai,
+      ...options.model ? { model: options.model } : {},
+      tools,
+      findingType: "string",
+      maxSubqueries,
+      maxParallelSubqueries,
       maxTurns,
       maxRuntimeChars,
-      maxBatchedLlmQueryConcurrency: maxParallelSubagents,
-      promptLevel: "detailed",
-      // Trace analysis depends on exact prior tool results and runtime variables.
-      contextPolicy: { preset: "full", budget: "balanced" },
-      functions,
-      actorOptions: {
-        description: options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION,
-        ...options.model ? { model: options.model } : {},
-        // Keep actor messages tool-call/content shaped across reasoning models.
-        showThoughts: false,
-        thinkingTokenBudget: "none"
-      },
-      responderOptions: {
-        ...options.model ? { model: options.model } : {},
-        description: options.subagentDescription ?? TRACE_ANALYST_SUBAGENT_DESCRIPTION,
-        showThoughts: false
-      },
-      actorTurnCallback,
-      bubbleErrors: [TraceFileMissingError]
-    }
-  );
-  let result;
-  try {
-    result = await analyst.forward(options.ai, { question: input.question });
+      ...options.signal ? { signal: options.signal } : {},
+      onTurn: actorTurnCallback
+    });
   } finally {
     if (progressFs) {
       await new Promise((resolve) => progressFs.end(() => resolve()));
     }
   }
   return {
-    answer: typeof result.answer === "string" ? result.answer : String(result.answer ?? ""),
-    findings: Array.isArray(result.findings) ? result.findings.filter((s) => typeof s === "string") : [],
+    answer: completed.report,
+    findings: completed.findings,
     turns,
     turnCount: turns.length,
-    usage: normalizeRoleArrays(analyst.getUsage()),
-    chatLog: normalizeRoleArrays(analyst.getChatLog()),
+    usage: { actor: normalizeRecordArray(completed.usage), responder: [] },
+    chatLog: { actor: normalizeRecordArray(completed.chatLog), responder: [] },
     actorPromptVersion: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION
   };
 }
-function normalizeRoleArrays(value) {
-  const record = value && typeof value === "object" ? value : {};
-  return {
-    actor: normalizeRecordArray(record.actor),
-    responder: normalizeRecordArray(record.responder)
-  };
+function rejectRemovedOptions(options) {
+  const supplied = options;
+  const migrations = [
+    ["maxDepth", "maxSubqueries"],
+    ["maxParallelSubagents", "maxParallelSubqueries"],
+    ["subagentDescription", "actorDescription"]
+  ];
+  for (const [removed, replacement] of migrations) {
+    if (removed in supplied) {
+      throw new TypeError(`analyzeTraces: '${removed}' is unsupported; use '${replacement}'`);
+    }
+  }
 }
 function normalizeRecordArray(value) {
   if (!Array.isArray(value)) return [];
@@ -195,7 +154,6 @@ function normalizeRecordArray(value) {
 export {
   TRACE_ANALYST_ACTOR_DESCRIPTION,
   TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
-  TRACE_ANALYST_SUBAGENT_DESCRIPTION,
   analyzeTraces
 };
-//# sourceMappingURL=chunk-FIUFRXP3.js.map
+//# sourceMappingURL=chunk-WW2A73HW.js.map

package/dist/chunk-WW2A73HW.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../src/trace-analyst/prompts.ts","../src/trace-analyst/analyst.ts"],"sourcesContent":["/** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */\n\nexport const TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \\`traces\\` namespace.\n\nDISCOVERY → NARROW → DEEP-READ protocol — follow exactly:\n\n1. ALWAYS call \\`traces.getDatasetOverview({})\\` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids — never fabricate one).\n\n2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. \\`filters.regex_pattern\\` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces — narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.\n\n3. To list more traces than the sample, call \\`traces.queryTraces({ filters?, limit, offset? })\\`. Each summary carries raw_jsonl_bytes — use it to choose between viewTrace and searchTrace BEFORE calling either.\n\n4. Per-trace inspection:\n   - SMALL trace (raw_jsonl_bytes well under 150_000): call \\`traces.viewTrace({ trace_id })\\`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large \\`input.value\\` / \\`output.value\\` / \\`llm.input_messages\\` will show a \\`[trace-analyst truncated: N bytes]\\` marker.\n   - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an \\`oversized\\` response): use \\`traces.searchTrace({ trace_id, regex_pattern })\\` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call \\`traces.viewSpans({ trace_id, span_ids: [...] })\\` for surgical reads (~16KB cap, 4× higher than discovery), or \\`traces.searchSpan({ trace_id, span_id, regex_pattern })\\` for one large span. Stays bounded regardless of trace size.\n   - Useful regex patterns: \\`STATUS_CODE_ERROR\\` (failures), tool names like \\`grep\\` or \\`view_trace\\`, error strings like \\`MaxTurnsExceeded\\`, model names, attribute keys.\n\n5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.\n\n5a. **Result-shape contract** — searchTrace and searchSpan return \\`{ trace_id, hits, total_matches, has_more }\\`. Iterate \\`result.hits\\` (NOT result.matches). Each hit has \\`{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }\\`. viewTrace returns \\`{ trace_id, spans }\\` (or \\`oversized\\`). viewSpans returns \\`{ trace_id, spans, missing_span_ids, truncated_attribute_count }\\`. Never assume a field name — log the result shape first if unsure.\n\n6. If viewTrace returns an \\`oversized\\` summary instead of \\`spans\\`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.\n\n7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.\n\n8. If a tool errors (invalid regex, range error), STOP and reconsider — don't retry with a guessed id or argument. Use the discovery tools above to recover.\n\n9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.\n\n10. If the question splits into independent reasoning branches, use bounded \\`llmQuery(...)\\` calls over evidence you already loaded. Subqueries cannot inspect the trace store, so pass the exact trace excerpts they need. Example:\n\n    const reviews = await llmQuery([\n      { query: 'Classify the failure mechanism in this excerpt.', context: traceAbcExcerpt },\n      { query: 'Classify the failure mechanism in this excerpt.', context: traceDefExcerpt },\n    ]);\n\nOBSERVABILITY rules:\n- Each discovery turn must emit at least one concise \\`console.log(...)\\` showing what evidence was learned.\n- Finish gathering evidence before submitting the analysis.\n- Reuse runtime variables across turns; don't recompute.\n\nOUTPUT contract — your final answer must include:\n- A clear prose conclusion answering the user's question.\n- Trace ids and span ids cited as evidence for each claim.\n- Failure modes named in the user's domain language, with frequency and concrete examples.\n- A concise findings array containing only claims supported by inspected evidence.\n\nDo NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.`\n\nexport const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = 'trace-analyst-actor-v6-2026-07-14'\n","import type { AxAgentActorTurnCallbackArgs, AxAIService, AxFunction } from '@ax-llm/ax'\nimport { runTraceAnalysisLoop, type TraceAnalysisLoopResult } from './loop'\nimport { TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION } from './prompts'\nimport type { TraceAnalysisStore } from './store'\nimport { OtlpFileTraceStore } from './store-otlp'\nimport { buildTraceAnalystTools } from './tools'\n\nexport interface AnalyzeTracesInput {\n  /** The user-facing question. Domain framing belongs here, not in the\n   *  actor description. */\n  question: string\n}\n\nexport interface AnalyzeTracesResult {\n  /** The actor's submitted prose answer. */\n  answer: string\n  /** Bulleted findings from the actor's structured completion. */\n  findings: string[]\n  /** Per-turn snapshots captured via `actorTurnCallback`. */\n  turns: AnalyzeTracesTurnSnapshot[]\n  /** Total turns the actor took. */\n  turnCount: number\n  /** Token usage by role. */\n  usage: TraceAnalystUsage\n  /** Full system + assistant + tool message log by role. */\n  chatLog: TraceAnalystChatLog\n  /** Prompt version that produced this run. */\n  actorPromptVersion: string\n}\n\nexport interface TraceAnalystUsage {\n  actor: TraceAnalystUsageEntry[]\n  responder: TraceAnalystUsageEntry[]\n}\n\nexport interface TraceAnalystUsageEntry {\n  [key: string]: unknown\n}\n\nexport interface TraceAnalystChatLog {\n  actor: TraceAnalystChatMessage[]\n  responder: TraceAnalystChatMessage[]\n}\n\nexport interface TraceAnalystChatMessage {\n  [key: string]: unknown\n}\n\nexport interface AnalyzeTracesTurnSnapshot {\n  stage: AxAgentActorTurnCallbackArgs['stage']\n  turn: number\n  isError: boolean\n  /** The JS code the actor produced for this turn. */\n  code: string\n  /** The formatted action-log entry the actor sees on the next turn. */\n  output: string\n  /** Provider thought (when `executorOptions.showThoughts` is true and the\n   *  provider returns it). */\n  thought?: string\n}\n\nexport interface AnalyzeTracesOptions {\n  /** Trace data source. Pass either an OTLP-JSONL path or a custom store. */\n  source: string | TraceAnalysisStore\n  /** Caller-provided AxAIService. */\n  ai: AxAIService\n  /** Model id forwarded to the actor. */\n  model?: string\n  /** Maximum model subqueries. 0 disables model fan-out. Default 4. */\n  maxSubqueries?: number\n  /** Maximum actor turns. Default 12. */\n  maxTurns?: number\n  /** Maximum parallel model subqueries. Default 2. */\n  maxParallelSubqueries?: number\n  /** Cancels in-flight model and tool work. */\n  signal?: AbortSignal\n  /** Override the actor description. */\n  actorDescription?: string\n  /** Per-turn observability hook. */\n  onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise<void>\n  /** Override max runtime characters per turn. Default 6000. */\n  maxRuntimeChars?: number\n  /** When set, every turn's snapshot is appended to this JSONL file\n   *  immediately. If the analyst crashes mid-loop (provider 503,\n   *  network error, validator reject) the partial reasoning is still\n   *  on disk for diagnosis and recovery. */\n  progressLogPath?: string\n}\n\n/**\n * Run the trace analyst.\n *\n * Throws:\n *   - `TraceFileMissingError` if `source` is a path and doesn't exist.\n *   - `AxAgentClarificationError` if the analyst asks for clarification.\n *   - Provider errors (auth, rate limits) propagate from the AI service.\n */\nexport async function analyzeTraces(\n  input: AnalyzeTracesInput,\n  options: AnalyzeTracesOptions,\n): Promise<AnalyzeTracesResult> {\n  if (!input.question || typeof input.question !== 'string') {\n    throw new TypeError('analyzeTraces: input.question must be a non-empty string')\n  }\n  rejectRemovedOptions(options)\n\n  const store: TraceAnalysisStore =\n    typeof options.source === 'string'\n      ? new OtlpFileTraceStore({ path: options.source })\n      : options.source\n\n  // Pre-warm file stores so missing inputs fail before the RLM starts.\n  if (store instanceof OtlpFileTraceStore) {\n    await store.ensureIndexed()\n  }\n\n  const tools: AxFunction[] = buildTraceAnalystTools({ store })\n  const turns: AnalyzeTracesTurnSnapshot[] = []\n\n  // Persist each turn as JSONL so interrupted analyst runs keep useful evidence.\n  let progressFs: import('node:fs').WriteStream | undefined\n  if (options.progressLogPath) {\n    const { createWriteStream } = await import('node:fs')\n    const { mkdir } = await import('node:fs/promises')\n    const { dirname } = await import('node:path')\n    await mkdir(dirname(options.progressLogPath), { recursive: true })\n    progressFs = createWriteStream(options.progressLogPath, { flags: 'a' })\n  }\n\n  const actorTurnCallback = async (turn: AxAgentActorTurnCallbackArgs): Promise<void> => {\n    const snap: AnalyzeTracesTurnSnapshot = {\n      stage: turn.stage,\n      turn: turn.turn,\n      isError: turn.isError,\n      code: turn.code,\n      output: turn.output,\n      thought: turn.thought,\n    }\n    turns.push(snap)\n    if (progressFs) {\n      try {\n        progressFs.write(`${JSON.stringify({ ...snap, ts: Date.now() })}\\n`)\n      } catch {\n        // Progress logging must never fail the analyst.\n      }\n    }\n    if (options.onTurn) await options.onTurn(snap)\n  }\n\n  const maxSubqueries = options.maxSubqueries ?? 4\n  const maxTurns = options.maxTurns ?? 12\n  const maxParallelSubqueries = options.maxParallelSubqueries ?? 2\n  const maxRuntimeChars = options.maxRuntimeChars ?? 6000\n  let completed: TraceAnalysisLoopResult<string>\n  try {\n    completed = await runTraceAnalysisLoop({\n      id: 'TraceAnalyst',\n      description:\n        'Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes.',\n      prompt: `${options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION}\n\nThe report must answer the user's question, and findings must be an array of concise evidence-backed strings.`,\n      question: input.question,\n      ai: options.ai,\n      ...(options.model ? { model: options.model } : {}),\n      tools,\n      findingType: 'string',\n      maxSubqueries,\n      maxParallelSubqueries,\n      maxTurns,\n      maxRuntimeChars,\n      ...(options.signal ? { signal: options.signal } : {}),\n      onTurn: actorTurnCallback,\n    })\n  } finally {\n    if (progressFs) {\n      await new Promise<void>((resolve) => progressFs!.end(() => resolve()))\n    }\n  }\n\n  return {\n    answer: completed.report,\n    findings: completed.findings,\n    turns,\n    turnCount: turns.length,\n    usage: { actor: normalizeRecordArray(completed.usage), responder: [] },\n    chatLog: { actor: normalizeRecordArray(completed.chatLog), responder: [] },\n    actorPromptVersion: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,\n  }\n}\n\nfunction rejectRemovedOptions(options: AnalyzeTracesOptions): void {\n  const supplied = options as unknown as Record<string, unknown>\n  const migrations = [\n    ['maxDepth', 'maxSubqueries'],\n    ['maxParallelSubagents', 'maxParallelSubqueries'],\n    ['subagentDescription', 'actorDescription'],\n  ] as const\n  for (const [removed, replacement] of migrations) {\n    if (removed in supplied) {\n      throw new TypeError(`analyzeTraces: '${removed}' is unsupported; use '${replacement}'`)\n    }\n  }\n}\n\nfunction normalizeRecordArray(value: unknown): Record<string, unknown>[] {\n  if (!Array.isArray(value)) return []\n  return value.map((item) =>\n    item && typeof item === 'object' ? { ...(item as Record<string, unknown>) } : { value: item },\n  )\n}\n"],"mappings":";;;;;;;AAEO,IAAM,kCAAkC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA+CxC,IAAM,0CAA0C;;;ACgDvD,eAAsB,cACpB,OACA,SAC8B;AAC9B,MAAI,CAAC,MAAM,YAAY,OAAO,MAAM,aAAa,UAAU;AACzD,UAAM,IAAI,UAAU,0DAA0D;AAAA,EAChF;AACA,uBAAqB,OAAO;AAE5B,QAAM,QACJ,OAAO,QAAQ,WAAW,WACtB,IAAI,mBAAmB,EAAE,MAAM,QAAQ,OAAO,CAAC,IAC/C,QAAQ;AAGd,MAAI,iBAAiB,oBAAoB;AACvC,UAAM,MAAM,cAAc;AAAA,EAC5B;AAEA,QAAM,QAAsB,uBAAuB,EAAE,MAAM,CAAC;AAC5D,QAAM,QAAqC,CAAC;AAG5C,MAAI;AACJ,MAAI,QAAQ,iBAAiB;AAC3B,UAAM,EAAE,kBAAkB,IAAI,MAAM,OAAO,IAAS;AACpD,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,aAAkB;AACjD,UAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,MAAW;AAC5C,UAAM,MAAM,QAAQ,QAAQ,eAAe,GAAG,EAAE,WAAW,KAAK,CAAC;AACjE,iBAAa,kBAAkB,QAAQ,iBAAiB,EAAE,OAAO,IAAI,CAAC;AAAA,EACxE;AAEA,QAAM,oBAAoB,OAAO,SAAsD;AACrF,UAAM,OAAkC;AAAA,MACtC,OAAO,KAAK;AAAA,MACZ,MAAM,KAAK;AAAA,MACX,SAAS,KAAK;AAAA,MACd,MAAM,KAAK;AAAA,MACX,QAAQ,KAAK;AAAA,MACb,SAAS,KAAK;AAAA,IAChB;AACA,UAAM,KAAK,IAAI;AACf,QAAI,YAAY;AACd,UAAI;AACF,mBAAW,MAAM,GAAG,KAAK,UAAU,EAAE,GAAG,MAAM,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC;AAAA,CAAI;AAAA,MACrE,QAAQ;AAAA,MAER;AAAA,IACF;AACA,QAAI,QAAQ,OAAQ,OAAM,QAAQ,OAAO,IAAI;AAAA,EAC/C;AAEA,QAAM,gBAAgB,QAAQ,iBAAiB;AAC/C,QAAM,WAAW,QAAQ,YAAY;AACrC,QAAM,wBAAwB,QAAQ,yBAAyB;AAC/D,QAAM,kBAAkB,QAAQ,mBAAmB;AACnD,MAAI;AACJ,MAAI;AACF,gBAAY,MAAM,qBAAqB;AAAA,MACrC,IAAI;AAAA,MACJ,aACE;AAAA,MACF,QAAQ,GAAG,QAAQ,oBAAoB,+BAA+B;AAAA;AAAA;AAAA,MAGtE,UAAU,MAAM;AAAA,MAChB,IAAI,QAAQ;AAAA,MACZ,GAAI,QAAQ,QAAQ,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA,MAChD;AAAA,MACA,aAAa;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,GAAI,QAAQ,SAAS,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;AAAA,MACnD,QAAQ;AAAA,IACV,CAAC;AAAA,EACH,UAAE;AACA,QAAI,YAAY;AACd,YAAM,IAAI,QAAc,CAAC,YAAY,WAAY,IAAI,MAAM,QAAQ,CAAC,CAAC;AAAA,IACvE;AAAA,EACF;AAEA,SAAO;AAAA,IACL,QAAQ,UAAU;AAAA,IAClB,UAAU,UAAU;AAAA,IACpB;AAAA,IACA,WAAW,MAAM;AAAA,IACjB,OAAO,EAAE,OAAO,qBAAqB,UAAU,KAAK,GAAG,WAAW,CAAC,EAAE;AAAA,IACrE,SAAS,EAAE,OAAO,qBAAqB,UAAU,OAAO,GAAG,WAAW,CAAC,EAAE;AAAA,IACzE,oBAAoB;AAAA,EACtB;AACF;AAEA,SAAS,qBAAqB,SAAqC;AACjE,QAAM,WAAW;AACjB,QAAM,aAAa;AAAA,IACjB,CAAC,YAAY,eAAe;AAAA,IAC5B,CAAC,wBAAwB,uBAAuB;AAAA,IAChD,CAAC,uBAAuB,kBAAkB;AAAA,EAC5C;AACA,aAAW,CAAC,SAAS,WAAW,KAAK,YAAY;AAC/C,QAAI,WAAW,UAAU;AACvB,YAAM,IAAI,UAAU,mBAAmB,OAAO,0BAA0B,WAAW,GAAG;AAAA,IACxF;AAAA,EACF;AACF;AAEA,SAAS,qBAAqB,OAA2C;AACvE,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,SAAO,MAAM;AAAA,IAAI,CAAC,SAChB,QAAQ,OAAO,SAAS,WAAW,EAAE,GAAI,KAAiC,IAAI,EAAE,OAAO,KAAK;AAAA,EAC9F;AACF;","names":[]}

package/dist/{chunk-PAHNGS65.js → chunk-XDIRG3TO.js}

@@ -21,13 +21,25 @@ var BackendIntegrityError = class extends AgentEvalError {
   }
   report;
 };
-function isStubRecord(rec) {
-  return rec.tokenUsage.input === 0 && rec.tokenUsage.output === 0;
+function summarizeBackendIntegrity(records) {
+  return summarizeBackendUsage(
+    records.map((record) => ({
+      inputTokens: record.tokenUsage.input,
+      outputTokens: record.tokenUsage.output,
+      costUsd: record.costUsd
+    }))
+  );
 }
-function isUncostedRecord(rec) {
-  return rec.tokenUsage.output > 0 && rec.costUsd === 0;
+function summarizeAgentReceiptIntegrity(receipts) {
+  return summarizeBackendUsage(
+    receipts.filter((receipt) => receipt.channel === "agent").map((receipt) => ({
+      inputTokens: receipt.inputTokens,
+      outputTokens: receipt.outputTokens,
+      costUsd: receipt.costUsd
+    }))
+  );
 }
-function summarizeBackendIntegrity(records) {
+function summarizeBackendUsage(records) {
   const totalRecords = records.length;
   let stubRecords = 0;
   let realRecords = 0;
@@ -36,12 +48,12 @@ function summarizeBackendIntegrity(records) {
   let totalOutputTokens = 0;
   let totalCostUsd = 0;
   for (const rec of records) {
-    totalInputTokens += rec.tokenUsage.input;
-    totalOutputTokens += rec.tokenUsage.output;
+    totalInputTokens += rec.inputTokens;
+    totalOutputTokens += rec.outputTokens;
     totalCostUsd += rec.costUsd;
-    if (isStubRecord(rec)) stubRecords++;
+    if (rec.inputTokens === 0 && rec.outputTokens === 0) stubRecords++;
     else realRecords++;
-    if (isUncostedRecord(rec)) uncostedRecords++;
+    if (rec.outputTokens > 0 && rec.costUsd === 0) uncostedRecords++;
   }
   const verdict = totalRecords === 0 ? "stub" : stubRecords === totalRecords ? "stub" : stubRecords === 0 ? "real" : "mixed";
   const diagnosis = buildDiagnosis({
@@ -100,6 +112,13 @@ function buildDiagnosis(r) {
 }
 function assertRealBackend(records, opts = {}) {
   const report = summarizeBackendIntegrity(records);
+  return assertBackendReport(report, opts);
+}
+function assertRealAgentReceipts(receipts, opts = {}) {
+  const report = summarizeAgentReceiptIntegrity(receipts);
+  return assertBackendReport(report, opts);
+}
+function assertBackendReport(report, opts) {
   const allowMixed = opts.allowMixed ?? true;
   if (report.verdict === "stub") {
     throw new BackendIntegrityError(
@@ -245,6 +264,135 @@ function cachedJudge(judge, store, options) {
   return wrapped;
 }
 
+// src/campaign/coverage.ts
+function assertCampaignDesign(scenarios, reps) {
+  if (!Number.isSafeInteger(reps) || reps < 1) {
+    throw new Error("campaign design requires reps to be a positive safe integer");
+  }
+  const scenarioIds = /* @__PURE__ */ new Set();
+  for (const scenario of scenarios) {
+    if (typeof scenario.id !== "string" || scenario.id.trim().length === 0) {
+      throw new Error("campaign design requires every scenario to have a non-empty id");
+    }
+    if (scenarioIds.has(scenario.id)) {
+      throw new Error(`campaign design contains duplicate scenario id '${scenario.id}'`);
+    }
+    if (typeof scenario.kind !== "string" || scenario.kind.trim().length === 0) {
+      throw new Error("campaign design requires every scenario to have a non-empty kind");
+    }
+    scenarioIds.add(scenario.id);
+  }
+}
+function campaignScenarioIdentity(scenario) {
+  assertCampaignDesign([scenario], 1);
+  return {
+    id: scenario.id,
+    kind: scenario.kind,
+    scenarioDigest: `sha256:${contentHash(scenario)}`
+  };
+}
+function campaignSplitDigestFromIdentities(scenarios, reps) {
+  assertCampaignDesign(scenarios, reps);
+  for (const scenario of scenarios) {
+    if (!/^sha256:[a-f0-9]{64}$/.test(scenario.scenarioDigest)) {
+      throw new Error(`campaign scenario '${scenario.id}' has an invalid digest`);
+    }
+  }
+  return `sha256:${contentHash({
+    schema: "tangle.campaign-split",
+    scenarios: scenarios.map(({ id, kind, scenarioDigest }) => ({ id, kind, scenarioDigest })),
+    reps
+  })}`;
+}
+function campaignSplitDigest(scenarios, reps) {
+  assertCampaignDesign(scenarios, reps);
+  return campaignSplitDigestFromIdentities(scenarios.map(campaignScenarioIdentity), reps);
+}
+function assertCampaignSplitIdentity(scenarios, reps, splitDigest) {
+  if (campaignSplitDigestFromIdentities(scenarios, reps) !== splitDigest) {
+    throw new Error("campaign split digest does not match its retained scenario identities");
+  }
+}
+function campaignCoverage(cells, scenarios, reps, requireJudgeScore) {
+  assertCampaignDesign(scenarios, reps);
+  const expectedCellIds = designedCellIds(scenarios, reps);
+  const cellsById = /* @__PURE__ */ new Map();
+  for (const cell of cells) {
+    const matches = cellsById.get(cell.cellId) ?? [];
+    matches.push(cell);
+    cellsById.set(cell.cellId, matches);
+  }
+  const scorableCellIds = [];
+  const unscorableCells = [];
+  for (const cellId of expectedCellIds) {
+    const matches = cellsById.get(cellId) ?? [];
+    if (matches.length === 0) {
+      unscorableCells.push({ cellId, reason: "missing campaign cell" });
+      continue;
+    }
+    if (matches.length > 1) {
+      unscorableCells.push({ cellId, reason: `duplicate campaign cell (${matches.length})` });
+      continue;
+    }
+    const cell = matches[0];
+    const scoreEntries = Object.entries(cell.judgeScores);
+    const successfulScores = scoreEntries.map(([, score]) => score).filter((score) => score.failed !== true && Number.isFinite(score.composite));
+    const nonFiniteScores = scoreEntries.filter(
+      ([, score]) => score.failed !== true && (!Number.isFinite(score.composite) || Object.values(score.dimensions).some((value) => !Number.isFinite(value)))
+    );
+    const reasons = [];
+    if (cell.error) reasons.push(cell.error);
+    if (cell.artifact === null || cell.artifact === void 0) reasons.push("missing artifact");
+    if (!cell.error && requireJudgeScore && successfulScores.length === 0) {
+      reasons.push("no successful finite judge score");
+    }
+    if (scoreEntries.some(([, score]) => score.failed === true)) {
+      reasons.push("judge score marked failed");
+    }
+    if (nonFiniteScores.length > 0) {
+      reasons.push(
+        `non-finite judge score: ${nonFiniteScores.map(([name]) => name).sort().join(", ")}`
+      );
+    }
+    if (reasons.length > 0) {
+      unscorableCells.push({ cellId, reason: reasons.join("; ") });
+    } else {
+      scorableCellIds.push(cellId);
+    }
+  }
+  const expected = new Set(expectedCellIds);
+  for (const cell of cells) {
+    if (cell.cellId !== `${cell.scenarioId}:${cell.rep}`) {
+      unscorableCells.push({
+        cellId: cell.cellId,
+        reason: "campaign cell id does not match scenario id and rep"
+      });
+      continue;
+    }
+    if (!expected.has(cell.cellId)) {
+      unscorableCells.push({ cellId: cell.cellId, reason: "unexpected campaign cell" });
+    }
+  }
+  return {
+    complete: unscorableCells.length === 0 && scorableCellIds.length === expectedCellIds.length,
+    expectedCellIds,
+    scorableCellIds,
+    unscorableCells
+  };
+}
+function formatCoverageFailures(coverage) {
+  const shown = coverage.unscorableCells.slice(0, 3).map((cell) => `${cell.cellId}: ${cell.reason}`).join("; ");
+  const remainder = coverage.unscorableCells.length - Math.min(3, coverage.unscorableCells.length);
+  return remainder > 0 ? `${shown}; +${remainder} more` : shown || "unknown coverage failure";
+}
+function designedCellIds(scenarios, reps) {
+  const ids = [];
+  for (const scenario of scenarios) {
+    for (let rep = 0; rep < reps; rep++) ids.push(`${scenario.id}:${rep}`);
+  }
+  return ids;
+}
+
 // src/campaign/run-dir.ts
 import { homedir } from "os";
 import { basename, isAbsolute, join } from "path";
@@ -527,6 +675,7 @@ async function runCampaign(opts) {
   const judges = opts.judges ?? [];
   const storage = opts.storage ?? fsCampaignStorage();
   const costPhase = opts.costPhase ?? "campaign";
+  assertCampaignDesign(opts.scenarios, reps);
   if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) {
     throw new Error("runCampaign: runDir is required and must be a non-empty string");
   }
@@ -548,6 +697,7 @@ async function runCampaign(opts) {
     seed,
     reps
   });
+  const splitDigest = campaignSplitDigest(opts.scenarios, reps);
   const startedAt = now();
   const runAttemptId = globalThis.crypto.randomUUID();
   const cells = [];
@@ -609,7 +759,9 @@ async function runCampaign(opts) {
   );
   return {
     manifestHash,
+    splitDigest,
     seed,
+    reps,
     startedAt: startedAt.toISOString(),
     endedAt: endedAt.toISOString(),
     durationMs: endedAt.getTime() - startedAt.getTime(),
@@ -617,7 +769,7 @@ async function runCampaign(opts) {
     aggregates,
     runDir: opts.runDir,
     artifactsByPath,
-    scenarios: opts.scenarios.map((s) => ({ id: s.id, kind: s.kind }))
+    scenarios: opts.scenarios.map(campaignScenarioIdentity)
   };
 }
 async function executeCell(args) {
@@ -826,6 +978,7 @@ function planCampaignRun(opts) {
   const reps = opts.reps ?? 1;
   const resumable = opts.resumable ?? true;
   const storage = opts.storage ?? fsCampaignStorage();
+  assertCampaignDesign(opts.scenarios, reps);
   if (typeof opts.runDir !== "string" || opts.runDir.trim().length === 0) {
     throw new Error("planCampaignRun: runDir is required and must be a non-empty string");
   }
@@ -837,6 +990,7 @@ function planCampaignRun(opts) {
     seed,
     reps
   });
+  const splitDigest = campaignSplitDigest(opts.scenarios, reps);
   const cells = buildCellSchedule(opts.scenarios, seed, reps).map((slot) => {
     const cachePath = join3(
       opts.runDir,
@@ -883,6 +1037,7 @@ function planCampaignRun(opts) {
   const cellsCached = cells.filter((cell) => cell.status === "cached").length;
   return {
     manifestHash,
+    splitDigest,
     totalCells: cells.length,
     cellsCached,
     cellsToRun: cells.length - cellsCached,
@@ -1080,12 +1235,21 @@ function aggregate(samples, seed) {
 export {
   BackendIntegrityError,
   summarizeBackendIntegrity,
+  summarizeAgentReceiptIntegrity,
   assertRealBackend,
+  assertRealAgentReceipts,
   canonicalJson,
   contentHash,
   inMemoryVerdictCache,
   fileVerdictCache,
   cachedJudge,
+  assertCampaignDesign,
+  campaignScenarioIdentity,
+  campaignSplitDigestFromIdentities,
+  campaignSplitDigest,
+  assertCampaignSplitIdentity,
+  campaignCoverage,
+  formatCoverageFailures,
   tangleTracesRoot,
   resolveRunDir,
   SearchLedgerError,
@@ -1099,4 +1263,4 @@ export {
   runCampaign,
   planCampaignRun
 };
-//# sourceMappingURL=chunk-PAHNGS65.js.map
+//# sourceMappingURL=chunk-XDIRG3TO.js.map