@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/chunk-5DTSBUL2.js

@@ -1,159 +0,0 @@
-import {
-  OtlpFileTraceStore,
-  buildTraceAnalystTools,
-  runTraceAnalysisLoop
-} from "./chunk-P6FYH6K4.js";
-
-// src/trace-analyst/prompts.ts
-var TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \`traces\` namespace.
-
-DISCOVERY \u2192 NARROW \u2192 DEEP-READ protocol \u2014 follow exactly:
-
-1. ALWAYS call \`traces.getDatasetOverview({})\` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids \u2014 never fabricate one).
-
-2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. \`filters.regex_pattern\` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces \u2014 narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.
-
-3. To list more traces than the sample, call \`traces.queryTraces({ filters?, limit, offset? })\`. Each summary carries raw_jsonl_bytes \u2014 use it to choose between viewTrace and searchTrace BEFORE calling either.
-
-4. Per-trace inspection:
-   - SMALL trace (raw_jsonl_bytes well under 150_000): call \`traces.viewTrace({ trace_id })\`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large \`input.value\` / \`output.value\` / \`llm.input_messages\` will show a \`[trace-analyst truncated: N bytes]\` marker.
-   - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an \`oversized\` response): use \`traces.searchTrace({ trace_id, regex_pattern })\` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call \`traces.viewSpans({ trace_id, span_ids: [...] })\` for surgical reads (~16KB cap, 4\xD7 higher than discovery), or \`traces.searchSpan({ trace_id, span_id, regex_pattern })\` for one large span. Stays bounded regardless of trace size.
-   - Useful regex patterns: \`STATUS_CODE_ERROR\` (failures), tool names like \`grep\` or \`view_trace\`, error strings like \`MaxTurnsExceeded\`, model names, attribute keys.
-
-5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.
-
-5a. **Result-shape contract** \u2014 searchTrace and searchSpan return \`{ trace_id, hits, total_matches, has_more }\`. Iterate \`result.hits\` (NOT result.matches). Each hit has \`{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }\`. viewTrace returns \`{ trace_id, spans }\` (or \`oversized\`). viewSpans returns \`{ trace_id, spans, missing_span_ids, truncated_attribute_count }\`. Never assume a field name \u2014 log the result shape first if unsure.
-
-6. If viewTrace returns an \`oversized\` summary instead of \`spans\`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.
-
-7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.
-
-8. If a tool errors (invalid regex, range error), STOP and reconsider \u2014 don't retry with a guessed id or argument. Use the discovery tools above to recover.
-
-9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.
-
-10. If the question splits into independent reasoning branches, use bounded \`llmQuery(...)\` calls over evidence you already loaded. Subqueries cannot inspect the trace store, so pass the exact trace excerpts they need. Example:
-
-    const reviews = await llmQuery([
-      { query: 'Classify the failure mechanism in this excerpt.', context: traceAbcExcerpt },
-      { query: 'Classify the failure mechanism in this excerpt.', context: traceDefExcerpt },
-    ]);
-
-OBSERVABILITY rules:
-- Each discovery turn must emit at least one concise \`console.log(...)\` showing what evidence was learned.
-- Finish gathering evidence before submitting the analysis.
-- Reuse runtime variables across turns; don't recompute.
-
-OUTPUT contract \u2014 your final answer must include:
-- A clear prose conclusion answering the user's question.
-- Trace ids and span ids cited as evidence for each claim.
-- Failure modes named in the user's domain language, with frequency and concrete examples.
-- A concise findings array containing only claims supported by inspected evidence.
-
-Do NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.`;
-var TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = "trace-analyst-actor-v6-2026-07-14";
-
-// src/trace-analyst/analyst.ts
-async function analyzeTraces(input, options) {
-  if (!input.question || typeof input.question !== "string") {
-    throw new TypeError("analyzeTraces: input.question must be a non-empty string");
-  }
-  rejectRemovedOptions(options);
-  const store = typeof options.source === "string" ? new OtlpFileTraceStore({ path: options.source }) : options.source;
-  if (store instanceof OtlpFileTraceStore) {
-    await store.ensureIndexed();
-  }
-  const tools = buildTraceAnalystTools({ store });
-  const turns = [];
-  let progressFs;
-  if (options.progressLogPath) {
-    const { createWriteStream } = await import("fs");
-    const { mkdir } = await import("fs/promises");
-    const { dirname } = await import("path");
-    await mkdir(dirname(options.progressLogPath), { recursive: true });
-    progressFs = createWriteStream(options.progressLogPath, { flags: "a" });
-  }
-  const actorTurnCallback = async (turn) => {
-    const snap = {
-      stage: turn.stage,
-      turn: turn.turn,
-      isError: turn.isError,
-      code: turn.code,
-      output: turn.output,
-      thought: turn.thought
-    };
-    turns.push(snap);
-    if (progressFs) {
-      try {
-        progressFs.write(`${JSON.stringify({ ...snap, ts: Date.now() })}
-`);
-      } catch {
-      }
-    }
-    if (options.onTurn) await options.onTurn(snap);
-  };
-  const maxSubqueries = options.maxSubqueries ?? 4;
-  const maxTurns = options.maxTurns ?? 12;
-  const maxParallelSubqueries = options.maxParallelSubqueries ?? 2;
-  const maxRuntimeChars = options.maxRuntimeChars ?? 6e3;
-  let completed;
-  try {
-    completed = await runTraceAnalysisLoop({
-      id: "TraceAnalyst",
-      description: "Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes.",
-      prompt: `${options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION}
-
-The report must answer the user's question, and findings must be an array of concise evidence-backed strings.`,
-      question: input.question,
-      ai: options.ai,
-      ...options.model ? { model: options.model } : {},
-      tools,
-      findingType: "string",
-      maxSubqueries,
-      maxParallelSubqueries,
-      maxTurns,
-      maxRuntimeChars,
-      ...options.signal ? { signal: options.signal } : {},
-      onTurn: actorTurnCallback
-    });
-  } finally {
-    if (progressFs) {
-      await new Promise((resolve) => progressFs.end(() => resolve()));
-    }
-  }
-  return {
-    answer: completed.report,
-    findings: completed.findings,
-    turns,
-    turnCount: turns.length,
-    usage: { actor: normalizeRecordArray(completed.usage), responder: [] },
-    chatLog: { actor: normalizeRecordArray(completed.chatLog), responder: [] },
-    actorPromptVersion: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION
-  };
-}
-function rejectRemovedOptions(options) {
-  const supplied = options;
-  const migrations = [
-    ["maxDepth", "maxSubqueries"],
-    ["maxParallelSubagents", "maxParallelSubqueries"],
-    ["subagentDescription", "actorDescription"]
-  ];
-  for (const [removed, replacement] of migrations) {
-    if (removed in supplied) {
-      throw new TypeError(`analyzeTraces: '${removed}' is unsupported; use '${replacement}'`);
-    }
-  }
-}
-function normalizeRecordArray(value) {
-  if (!Array.isArray(value)) return [];
-  return value.map(
-    (item) => item && typeof item === "object" ? { ...item } : { value: item }
-  );
-}
-
-export {
-  TRACE_ANALYST_ACTOR_DESCRIPTION,
-  TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,
-  analyzeTraces
-};
-//# sourceMappingURL=chunk-5DTSBUL2.js.map

package/dist/chunk-5DTSBUL2.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/trace-analyst/prompts.ts","../src/trace-analyst/analyst.ts"],"sourcesContent":["/** Ax RLM prompt for bounded trace discovery and evidence-backed analysis. */\n\nexport const TRACE_ANALYST_ACTOR_DESCRIPTION = `You answer questions about an OTLP-shaped JSONL trace dataset using the trace tools provided in the \\`traces\\` namespace.\n\nDISCOVERY → NARROW → DEEP-READ protocol — follow exactly:\n\n1. ALWAYS call \\`traces.getDatasetOverview({})\\` FIRST without a regex_pattern. The result tells you total_traces, raw_jsonl_bytes, services, agents, models, and sample_trace_ids (real ids — never fabricate one).\n\n2. Use raw_jsonl_bytes to gauge how expensive raw scans will be. \\`filters.regex_pattern\\` is the one scan-heavy filter on getDatasetOverview / queryTraces / countTraces — narrow with indexed fields (has_errors, model_names, service_names, agent_names, time bounds) BEFORE adding a regex on a large dataset.\n\n3. To list more traces than the sample, call \\`traces.queryTraces({ filters?, limit, offset? })\\`. Each summary carries raw_jsonl_bytes — use it to choose between viewTrace and searchTrace BEFORE calling either.\n\n4. Per-trace inspection:\n   - SMALL trace (raw_jsonl_bytes well under 150_000): call \\`traces.viewTrace({ trace_id })\\`. Returns all spans. Per-attribute payloads are head-capped at ~4KB; large \\`input.value\\` / \\`output.value\\` / \\`llm.input_messages\\` will show a \\`[trace-analyst truncated: N bytes]\\` marker.\n   - LARGE trace (raw_jsonl_bytes near or above 150_000, or you saw an \\`oversized\\` response): use \\`traces.searchTrace({ trace_id, regex_pattern })\\` to get bounded SpanMatchRecords (span metadata + matched text + surrounding context). Then call \\`traces.viewSpans({ trace_id, span_ids: [...] })\\` for surgical reads (~16KB cap, 4× higher than discovery), or \\`traces.searchSpan({ trace_id, span_id, regex_pattern })\\` for one large span. Stays bounded regardless of trace size.\n   - Useful regex patterns: \\`STATUS_CODE_ERROR\\` (failures), tool names like \\`grep\\` or \\`view_trace\\`, error strings like \\`MaxTurnsExceeded\\`, model names, attribute keys.\n\n5. ONLY call viewTrace / viewSpans / searchTrace / searchSpan with trace/span ids you have already seen in sample_trace_ids, a queryTraces page, or a previous search result. Never invent ids.\n\n5a. **Result-shape contract** — searchTrace and searchSpan return \\`{ trace_id, hits, total_matches, has_more }\\`. Iterate \\`result.hits\\` (NOT result.matches). Each hit has \\`{ span_id, span_name, span_kind, attribute_path, matched_text, context_before, context_after, match_offset }\\`. viewTrace returns \\`{ trace_id, spans }\\` (or \\`oversized\\`). viewSpans returns \\`{ trace_id, spans, missing_span_ids, truncated_attribute_count }\\`. Never assume a field name — log the result shape first if unsure.\n\n6. If viewTrace returns an \\`oversized\\` summary instead of \\`spans\\`, DO NOT retry the same call. Read the summary's top_span_names, span_count, span_response_bytes_max, error_span_count to plan a follow-up: switch to searchTrace (or searchSpan for one large span), then viewSpans on a smaller, surgical span_ids set.\n\n7. If searchTrace or searchSpan returns has_more=true, REFINE the regex to be more specific rather than blindly raising max_matches.\n\n8. If a tool errors (invalid regex, range error), STOP and reconsider — don't retry with a guessed id or argument. Use the discovery tools above to recover.\n\n9. If a ~4KB-truncated payload from viewTrace / searchTrace matters for your answer, first try viewSpans on that span id (~16KB cap). If a 16KB-truncated payload from viewSpans still matters, narrow further with searchSpan against a more specific regex rather than asking for the full payload again.\n\n10. If the question splits into independent reasoning branches, use bounded \\`llmQuery(...)\\` calls over evidence you already loaded. Subqueries cannot inspect the trace store, so pass the exact trace excerpts they need. Example:\n\n    const reviews = await llmQuery([\n      { query: 'Classify the failure mechanism in this excerpt.', context: traceAbcExcerpt },\n      { query: 'Classify the failure mechanism in this excerpt.', context: traceDefExcerpt },\n    ]);\n\nOBSERVABILITY rules:\n- Each discovery turn must emit at least one concise \\`console.log(...)\\` showing what evidence was learned.\n- Finish gathering evidence before submitting the analysis.\n- Reuse runtime variables across turns; don't recompute.\n\nOUTPUT contract — your final answer must include:\n- A clear prose conclusion answering the user's question.\n- Trace ids and span ids cited as evidence for each claim.\n- Failure modes named in the user's domain language, with frequency and concrete examples.\n- A concise findings array containing only claims supported by inspected evidence.\n\nDo NOT invent trace ids, span ids, error messages, or model names. Every fact must be traceable to a tool result.`\n\nexport const TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION = 'trace-analyst-actor-v6-2026-07-14'\n","import type { AxAgentActorTurnCallbackArgs, AxAIService, AxFunction } from '@ax-llm/ax'\nimport { runTraceAnalysisLoop, type TraceAnalysisLoopResult } from './loop'\nimport { TRACE_ANALYST_ACTOR_DESCRIPTION, TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION } from './prompts'\nimport type { TraceAnalysisStore } from './store'\nimport { OtlpFileTraceStore } from './store-otlp'\nimport { buildTraceAnalystTools } from './tools'\n\nexport interface AnalyzeTracesInput {\n  /** The user-facing question. Domain framing belongs here, not in the\n   *  actor description. */\n  question: string\n}\n\nexport interface AnalyzeTracesResult {\n  /** The actor's submitted prose answer. */\n  answer: string\n  /** Bulleted findings from the actor's structured completion. */\n  findings: string[]\n  /** Per-turn snapshots captured via `actorTurnCallback`. */\n  turns: AnalyzeTracesTurnSnapshot[]\n  /** Total turns the actor took. */\n  turnCount: number\n  /** Token usage by role. */\n  usage: TraceAnalystUsage\n  /** Full system + assistant + tool message log by role. */\n  chatLog: TraceAnalystChatLog\n  /** Prompt version that produced this run. */\n  actorPromptVersion: string\n}\n\nexport interface TraceAnalystUsage {\n  actor: TraceAnalystUsageEntry[]\n  responder: TraceAnalystUsageEntry[]\n}\n\nexport interface TraceAnalystUsageEntry {\n  [key: string]: unknown\n}\n\nexport interface TraceAnalystChatLog {\n  actor: TraceAnalystChatMessage[]\n  responder: TraceAnalystChatMessage[]\n}\n\nexport interface TraceAnalystChatMessage {\n  [key: string]: unknown\n}\n\nexport interface AnalyzeTracesTurnSnapshot {\n  stage: AxAgentActorTurnCallbackArgs['stage']\n  turn: number\n  isError: boolean\n  /** The JS code the actor produced for this turn. */\n  code: string\n  /** The formatted action-log entry the actor sees on the next turn. */\n  output: string\n  /** Provider thought (when `executorOptions.showThoughts` is true and the\n   *  provider returns it). */\n  thought?: string\n}\n\nexport interface AnalyzeTracesOptions {\n  /** Trace data source. Pass either an OTLP-JSONL path or a custom store. */\n  source: string | TraceAnalysisStore\n  /** Caller-provided AxAIService. */\n  ai: AxAIService\n  /** Model id forwarded to the actor. */\n  model?: string\n  /** Maximum model subqueries. 0 disables model fan-out. Default 4. */\n  maxSubqueries?: number\n  /** Maximum actor turns. Default 12. */\n  maxTurns?: number\n  /** Maximum parallel model subqueries. Default 2. */\n  maxParallelSubqueries?: number\n  /** Cancels in-flight model and tool work. */\n  signal?: AbortSignal\n  /** Override the actor description. */\n  actorDescription?: string\n  /** Per-turn observability hook. */\n  onTurn?: (turn: AnalyzeTracesTurnSnapshot) => void | Promise<void>\n  /** Override max runtime characters per turn. Default 6000. */\n  maxRuntimeChars?: number\n  /** When set, every turn's snapshot is appended to this JSONL file\n   *  immediately. If the analyst crashes mid-loop (provider 503,\n   *  network error, validator reject) the partial reasoning is still\n   *  on disk for diagnosis and recovery. */\n  progressLogPath?: string\n}\n\n/**\n * Run the trace analyst.\n *\n * Throws:\n *   - `TraceFileMissingError` if `source` is a path and doesn't exist.\n *   - `AxAgentClarificationError` if the analyst asks for clarification.\n *   - Provider errors (auth, rate limits) propagate from the AI service.\n */\nexport async function analyzeTraces(\n  input: AnalyzeTracesInput,\n  options: AnalyzeTracesOptions,\n): Promise<AnalyzeTracesResult> {\n  if (!input.question || typeof input.question !== 'string') {\n    throw new TypeError('analyzeTraces: input.question must be a non-empty string')\n  }\n  rejectRemovedOptions(options)\n\n  const store: TraceAnalysisStore =\n    typeof options.source === 'string'\n      ? new OtlpFileTraceStore({ path: options.source })\n      : options.source\n\n  // Pre-warm file stores so missing inputs fail before the RLM starts.\n  if (store instanceof OtlpFileTraceStore) {\n    await store.ensureIndexed()\n  }\n\n  const tools: AxFunction[] = buildTraceAnalystTools({ store })\n  const turns: AnalyzeTracesTurnSnapshot[] = []\n\n  // Persist each turn as JSONL so interrupted analyst runs keep useful evidence.\n  let progressFs: import('node:fs').WriteStream | undefined\n  if (options.progressLogPath) {\n    const { createWriteStream } = await import('node:fs')\n    const { mkdir } = await import('node:fs/promises')\n    const { dirname } = await import('node:path')\n    await mkdir(dirname(options.progressLogPath), { recursive: true })\n    progressFs = createWriteStream(options.progressLogPath, { flags: 'a' })\n  }\n\n  const actorTurnCallback = async (turn: AxAgentActorTurnCallbackArgs): Promise<void> => {\n    const snap: AnalyzeTracesTurnSnapshot = {\n      stage: turn.stage,\n      turn: turn.turn,\n      isError: turn.isError,\n      code: turn.code,\n      output: turn.output,\n      thought: turn.thought,\n    }\n    turns.push(snap)\n    if (progressFs) {\n      try {\n        progressFs.write(`${JSON.stringify({ ...snap, ts: Date.now() })}\\n`)\n      } catch {\n        // Progress logging must never fail the analyst.\n      }\n    }\n    if (options.onTurn) await options.onTurn(snap)\n  }\n\n  const maxSubqueries = options.maxSubqueries ?? 4\n  const maxTurns = options.maxTurns ?? 12\n  const maxParallelSubqueries = options.maxParallelSubqueries ?? 2\n  const maxRuntimeChars = options.maxRuntimeChars ?? 6000\n  let completed: TraceAnalysisLoopResult<string>\n  try {\n    completed = await runTraceAnalysisLoop({\n      id: 'TraceAnalyst',\n      description:\n        'Analyzes OTLP-shaped JSONL traces using bounded discovery tools to identify systemic failure modes.',\n      prompt: `${options.actorDescription ?? TRACE_ANALYST_ACTOR_DESCRIPTION}\n\nThe report must answer the user's question, and findings must be an array of concise evidence-backed strings.`,\n      question: input.question,\n      ai: options.ai,\n      ...(options.model ? { model: options.model } : {}),\n      tools,\n      findingType: 'string',\n      maxSubqueries,\n      maxParallelSubqueries,\n      maxTurns,\n      maxRuntimeChars,\n      ...(options.signal ? { signal: options.signal } : {}),\n      onTurn: actorTurnCallback,\n    })\n  } finally {\n    if (progressFs) {\n      await new Promise<void>((resolve) => progressFs!.end(() => resolve()))\n    }\n  }\n\n  return {\n    answer: completed.report,\n    findings: completed.findings,\n    turns,\n    turnCount: turns.length,\n    usage: { actor: normalizeRecordArray(completed.usage), responder: [] },\n    chatLog: { actor: normalizeRecordArray(completed.chatLog), responder: [] },\n    actorPromptVersion: TRACE_ANALYST_ACTOR_DESCRIPTION_VERSION,\n  }\n}\n\nfunction rejectRemovedOptions(options: AnalyzeTracesOptions): void {\n  const supplied = options as unknown as Record<string, unknown>\n  const migrations = [\n    ['maxDepth', 'maxSubqueries'],\n    ['maxParallelSubagents', 'maxParallelSubqueries'],\n    ['subagentDescription', 'actorDescription'],\n  ] as const\n  for (const [removed, replacement] of migrations) {\n    if (removed in supplied) {\n      throw new TypeError(`analyzeTraces: '${removed}' is unsupported; use '${replacement}'`)\n    }\n  }\n}\n\nfunction normalizeRecordArray(value: unknown): Record<string, unknown>[] {\n  if (!Array.isArray(value)) return []\n  return value.map((item) =>\n    item && typeof item === 'object' ? { ...(item as Record<string, unknown>) } : { value: item },\n  )\n}\n"],"mappings":";;;;;;;AAEO,IAAM,kCAAkC;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AA+CxC,IAAM,0CAA0C;;;ACgDvD,eAAsB,cACpB,OACA,SAC8B;AAC9B,MAAI,CAAC,MAAM,YAAY,OAAO,MAAM,aAAa,UAAU;AACzD,UAAM,IAAI,UAAU,0DAA0D;AAAA,EAChF;AACA,uBAAqB,OAAO;AAE5B,QAAM,QACJ,OAAO,QAAQ,WAAW,WACtB,IAAI,mBAAmB,EAAE,MAAM,QAAQ,OAAO,CAAC,IAC/C,QAAQ;AAGd,MAAI,iBAAiB,oBAAoB;AACvC,UAAM,MAAM,cAAc;AAAA,EAC5B;AAEA,QAAM,QAAsB,uBAAuB,EAAE,MAAM,CAAC;AAC5D,QAAM,QAAqC,CAAC;AAG5C,MAAI;AACJ,MAAI,QAAQ,iBAAiB;AAC3B,UAAM,EAAE,kBAAkB,IAAI,MAAM,OAAO,IAAS;AACpD,UAAM,EAAE,MAAM,IAAI,MAAM,OAAO,aAAkB;AACjD,UAAM,EAAE,QAAQ,IAAI,MAAM,OAAO,MAAW;AAC5C,UAAM,MAAM,QAAQ,QAAQ,eAAe,GAAG,EAAE,WAAW,KAAK,CAAC;AACjE,iBAAa,kBAAkB,QAAQ,iBAAiB,EAAE,OAAO,IAAI,CAAC;AAAA,EACxE;AAEA,QAAM,oBAAoB,OAAO,SAAsD;AACrF,UAAM,OAAkC;AAAA,MACtC,OAAO,KAAK;AAAA,MACZ,MAAM,KAAK;AAAA,MACX,SAAS,KAAK;AAAA,MACd,MAAM,KAAK;AAAA,MACX,QAAQ,KAAK;AAAA,MACb,SAAS,KAAK;AAAA,IAChB;AACA,UAAM,KAAK,IAAI;AACf,QAAI,YAAY;AACd,UAAI;AACF,mBAAW,MAAM,GAAG,KAAK,UAAU,EAAE,GAAG,MAAM,IAAI,KAAK,IAAI,EAAE,CAAC,CAAC;AAAA,CAAI;AAAA,MACrE,QAAQ;AAAA,MAER;AAAA,IACF;AACA,QAAI,QAAQ,OAAQ,OAAM,QAAQ,OAAO,IAAI;AAAA,EAC/C;AAEA,QAAM,gBAAgB,QAAQ,iBAAiB;AAC/C,QAAM,WAAW,QAAQ,YAAY;AACrC,QAAM,wBAAwB,QAAQ,yBAAyB;AAC/D,QAAM,kBAAkB,QAAQ,mBAAmB;AACnD,MAAI;AACJ,MAAI;AACF,gBAAY,MAAM,qBAAqB;AAAA,MACrC,IAAI;AAAA,MACJ,aACE;AAAA,MACF,QAAQ,GAAG,QAAQ,oBAAoB,+BAA+B;AAAA;AAAA;AAAA,MAGtE,UAAU,MAAM;AAAA,MAChB,IAAI,QAAQ;AAAA,MACZ,GAAI,QAAQ,QAAQ,EAAE,OAAO,QAAQ,MAAM,IAAI,CAAC;AAAA,MAChD;AAAA,MACA,aAAa;AAAA,MACb;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA,GAAI,QAAQ,SAAS,EAAE,QAAQ,QAAQ,OAAO,IAAI,CAAC;AAAA,MACnD,QAAQ;AAAA,IACV,CAAC;AAAA,EACH,UAAE;AACA,QAAI,YAAY;AACd,YAAM,IAAI,QAAc,CAAC,YAAY,WAAY,IAAI,MAAM,QAAQ,CAAC,CAAC;AAAA,IACvE;AAAA,EACF;AAEA,SAAO;AAAA,IACL,QAAQ,UAAU;AAAA,IAClB,UAAU,UAAU;AAAA,IACpB;AAAA,IACA,WAAW,MAAM;AAAA,IACjB,OAAO,EAAE,OAAO,qBAAqB,UAAU,KAAK,GAAG,WAAW,CAAC,EAAE;AAAA,IACrE,SAAS,EAAE,OAAO,qBAAqB,UAAU,OAAO,GAAG,WAAW,CAAC,EAAE;AAAA,IACzE,oBAAoB;AAAA,EACtB;AACF;AAEA,SAAS,qBAAqB,SAAqC;AACjE,QAAM,WAAW;AACjB,QAAM,aAAa;AAAA,IACjB,CAAC,YAAY,eAAe;AAAA,IAC5B,CAAC,wBAAwB,uBAAuB;AAAA,IAChD,CAAC,uBAAuB,kBAAkB;AAAA,EAC5C;AACA,aAAW,CAAC,SAAS,WAAW,KAAK,YAAY;AAC/C,QAAI,WAAW,UAAU;AACvB,YAAM,IAAI,UAAU,mBAAmB,OAAO,0BAA0B,WAAW,GAAG;AAAA,IACxF;AAAA,EACF;AACF;AAEA,SAAS,qBAAqB,OAA2C;AACvE,MAAI,CAAC,MAAM,QAAQ,KAAK,EAAG,QAAO,CAAC;AACnC,SAAO,MAAM;AAAA,IAAI,CAAC,SAChB,QAAQ,OAAO,SAAS,WAAW,EAAE,GAAI,KAAiC,IAAI,EAAE,OAAO,KAAK;AAAA,EAC9F;AACF;","names":[]}

package/dist/chunk-7FO3TNPI.js

@@ -1,232 +0,0 @@
-import {
-  mcnemar,
-  pairedBootstrap,
-  pairedRiskDifference,
-  wilcoxonSignedRank
-} from "./chunk-ZHTZ4EYI.js";
-import {
-  ValidationError
-} from "./chunk-ONWEPEDO.js";
-
-// src/paired-arms.ts
-function pairArms(rows, opts) {
-  const { baselineArm, treatmentArm } = opts;
-  if (baselineArm === treatmentArm) {
-    throw new ValidationError(
-      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' \u2014 an arm cannot be compared to itself`
-    );
-  }
-  const byArm = /* @__PURE__ */ new Map();
-  const armsSeen = /* @__PURE__ */ new Set();
-  for (const row of rows) {
-    armsSeen.add(row.arm);
-    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue;
-    const byKey = byArm.get(row.arm) ?? /* @__PURE__ */ new Map();
-    const group = byKey.get(row.pairKey) ?? [];
-    group.push(row);
-    byKey.set(row.pairKey, group);
-    byArm.set(row.arm, byKey);
-  }
-  for (const arm of [baselineArm, treatmentArm]) {
-    if (!byArm.has(arm)) {
-      const seen = [...armsSeen].sort().join(", ") || "<none>";
-      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`);
-    }
-  }
-  const baselineByKey = byArm.get(baselineArm);
-  const treatmentByKey = byArm.get(treatmentArm);
-  const allKeys = [.../* @__PURE__ */ new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort();
-  const pairs = [];
-  const unpairedBaseline = [];
-  const unpairedTreatment = [];
-  for (const pairKey of allKeys) {
-    const b = baselineByKey.get(pairKey) ?? [];
-    const t = treatmentByKey.get(pairKey) ?? [];
-    if (b.length <= 1 && t.length <= 1) {
-      if (b.length === 1 && t.length === 1) {
-        const baseline = b[0];
-        const treatment = t[0];
-        if (baseline.repKey !== void 0 || treatment.repKey !== void 0) {
-          if (baseline.repKey === void 0 || treatment.repKey === void 0 || baseline.repKey !== treatment.repKey) {
-            unpairedBaseline.push(baseline);
-            unpairedTreatment.push(treatment);
-            continue;
-          }
-        }
-        pairs.push({ pairKey, repIndex: 0, baseline, treatment });
-      } else {
-        unpairedBaseline.push(...b);
-        unpairedTreatment.push(...t);
-      }
-      continue;
-    }
-    const bByRep = indexByRepKey(b, pairKey, baselineArm);
-    const tByRep = indexByRepKey(t, pairKey, treatmentArm);
-    const repKeys = [.../* @__PURE__ */ new Set([...bByRep.keys(), ...tByRep.keys()])].sort();
-    let repIndex = 0;
-    for (const repKey of repKeys) {
-      const baseline = bByRep.get(repKey);
-      const treatment = tByRep.get(repKey);
-      if (baseline !== void 0 && treatment !== void 0) {
-        pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment });
-      } else if (baseline !== void 0) {
-        unpairedBaseline.push(baseline);
-      } else if (treatment !== void 0) {
-        unpairedTreatment.push(treatment);
-      }
-    }
-  }
-  return { pairs, unpairedBaseline, unpairedTreatment };
-}
-function indexByRepKey(group, pairKey, arm) {
-  const byRep = /* @__PURE__ */ new Map();
-  for (const row of group) {
-    if (row.repKey === void 0) {
-      throw new ValidationError(
-        `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' is missing repKey \u2014 multi-rep items require an explicit repKey on every row so reps pair by identity (pairing reps by outcome or by index would bias the paired statistics)`
-      );
-    }
-    if (byRep.has(row.repKey)) {
-      throw new ValidationError(
-        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' \u2014 (pairKey, repKey) must uniquely identify a rep within an arm`
-      );
-    }
-    byRep.set(row.repKey, row);
-  }
-  return byRep;
-}
-function comparePairedArms(rows, opts) {
-  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts);
-  let correctness = null;
-  const baselinePass = [];
-  const treatmentPass = [];
-  for (const pair of pairs) {
-    if (pair.baseline.pass === void 0 || pair.treatment.pass === void 0) continue;
-    baselinePass.push(pair.baseline.pass ? 1 : 0);
-    treatmentPass.push(pair.treatment.pass ? 1 : 0);
-  }
-  if (baselinePass.length > 0) {
-    const mc = mcnemar(baselinePass, treatmentPass);
-    correctness = {
-      b10: mc.b,
-      b01: mc.c,
-      mcnemar: mc,
-      riskDifference: pairedRiskDifference(baselinePass, treatmentPass)
-    };
-  }
-  const metricNames = opts.metricNames ?? [
-    ...new Set(
-      pairs.flatMap((p) => [
-        ...Object.keys(p.baseline.metrics ?? {}),
-        ...Object.keys(p.treatment.metrics ?? {})
-      ])
-    )
-  ].sort();
-  const metricDeltas = metricNames.map((name) => {
-    const before = [];
-    const after = [];
-    let nMissing = 0;
-    for (const pair of pairs) {
-      const b = metricValue(pair.baseline, name);
-      const t = metricValue(pair.treatment, name);
-      if (b === void 0 || t === void 0) {
-        nMissing++;
-        continue;
-      }
-      before.push(b);
-      after.push(t);
-    }
-    const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap);
-    return {
-      name,
-      n: before.length,
-      nMissing,
-      medianDelta: bootstrapCi?.median ?? null,
-      meanDelta: bootstrapCi?.mean ?? null,
-      bootstrapCi,
-      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after)
-    };
-  });
-  return {
-    nPairs: pairs.length,
-    nUnpairedBaseline: unpairedBaseline.length,
-    nUnpairedTreatment: unpairedTreatment.length,
-    correctness,
-    metricDeltas
-  };
-}
-function pairRunRecords(baselineRuns, treatmentRuns) {
-  const baselineRows = runRecordArmRows(baselineRuns, "baseline");
-  const treatmentRows = runRecordArmRows(treatmentRuns, "treatment");
-  validateRunRecordArmRows(baselineRows, "baseline");
-  validateRunRecordArmRows(treatmentRows, "treatment");
-  if (baselineRows.length === 0 || treatmentRows.length === 0) {
-    return {
-      pairs: [],
-      unpairedBaseline: baselineRows.map((row) => row.run),
-      unpairedTreatment: treatmentRows.map((row) => row.run)
-    };
-  }
-  const result = pairArms([...baselineRows, ...treatmentRows], {
-    baselineArm: "baseline",
-    treatmentArm: "treatment"
-  });
-  return {
-    pairs: result.pairs.map((pair) => {
-      const baseline = pair.baseline;
-      const treatment = pair.treatment;
-      return {
-        pairKey: pair.pairKey,
-        repKey: baseline.repKey,
-        baseline: baseline.run,
-        treatment: treatment.run
-      };
-    }),
-    unpairedBaseline: result.unpairedBaseline.map((row) => row.run),
-    unpairedTreatment: result.unpairedTreatment.map((row) => row.run)
-  };
-}
-function runRecordArmRows(runs, arm) {
-  return runs.map((run) => {
-    const scenarioId = run.scenarioId.trim();
-    if (!scenarioId) {
-      throw new ValidationError(
-        `pairRunRecords: run '${run.runId}' is missing scenarioId; paired comparisons require explicit scenario identity`
-      );
-    }
-    return {
-      pairKey: JSON.stringify([run.experimentId, scenarioId]),
-      repKey: String(run.seed),
-      arm,
-      run
-    };
-  });
-}
-function validateRunRecordArmRows(rows, arm) {
-  const byPairKey = /* @__PURE__ */ new Map();
-  for (const row of rows) {
-    const group = byPairKey.get(row.pairKey) ?? [];
-    group.push(row);
-    byPairKey.set(row.pairKey, group);
-  }
-  for (const [pairKey, group] of byPairKey) {
-    if (group.length > 1) indexByRepKey(group, pairKey, arm);
-  }
-}
-function metricValue(row, name) {
-  const v = row.metrics?.[name];
-  if (v === void 0) return void 0;
-  if (!Number.isFinite(v)) {
-    throw new ValidationError(
-      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`
-    );
-  }
-  return v;
-}
-
-export {
-  pairArms,
-  comparePairedArms,
-  pairRunRecords
-};
-//# sourceMappingURL=chunk-7FO3TNPI.js.map

package/dist/chunk-7FO3TNPI.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/paired-arms.ts"],"sourcesContent":["/**\n * Matched-pair arm comparison — \"did the treatment arm beat the baseline arm\n * on the SAME work items?\"\n *\n * An arm A/B over run records is only trustworthy when it is PAIRED: the same\n * task/scenario/seed evaluated under both arms, compared item-by-item, so\n * inter-item difficulty variance cancels instead of masquerading as an arm\n * effect. This module owns the two error-prone steps every consumer otherwise\n * hand-rolls:\n *\n *   1. Pairing — matching rows across arms by `pairKey` (and by `repKey`\n *      within multi-rep items), with leftovers REPORTED rather than silently\n *      dropped (a silently unbalanced pairing biases every paired statistic\n *      downstream). Pairing never keys on outcome content: matching reps by\n *      their outcomes deflates discordant-pair counts and makes McNemar\n *      anti-conservative, so reps pair only by (`pairKey`, `repKey`) identity.\n *   2. Composition — feeding the matched pairs to the correct paired\n *      estimators that already live in `statistics`: `mcnemar` +\n *      `pairedRiskDifference` for pass/fail, `pairedBootstrap` +\n *      `wilcoxonSignedRank` for continuous metrics. No statistic is\n *      re-implemented here.\n *\n * The row shape is deliberately structural — callers project a `RunRecord`\n * (or any record) into `{ pairKey, arm, pass?, metrics? }`. Arm names are\n * caller-supplied parameters; the module ships no domain literal.\n */\n\nimport { ValidationError } from './errors'\nimport type { RunRecord } from './run-record'\nimport type { McNemarResult, PairedBootstrapOptions, PairedBootstrapResult } from './statistics'\nimport {\n  mcnemar,\n  pairedBootstrap,\n  pairedRiskDifference,\n  type RiskDifferenceResult,\n  wilcoxonSignedRank,\n} from './statistics'\n\n/** One arm observation of one work item. Structural on purpose: callers\n *  project their own record type (e.g. a `RunRecord`) into this shape. */\nexport interface PairedArmRow {\n  /** Matching key — rows sharing a `pairKey` across both arms form pairs\n   *  (typically the task/scenario/seed identity). */\n  pairKey: string\n  /** Rep identity within a `pairKey` (e.g. a seed or rep number). Required on\n   *  every row of a `pairKey` that has more than one rep in either arm; reps\n   *  then pair only on exact (`pairKey`, `repKey`) match, never on outcome\n   *  content. Optional when each arm has at most one rep of the item. */\n  repKey?: string\n  /** Arm label this row was produced under. */\n  arm: string\n  /** Binary outcome; omit when the comparison has no pass/fail notion. */\n  pass?: boolean\n  /** Named numeric measurements (score, cost, latency, …). */\n  metrics?: Record<string, number>\n}\n\nexport interface PairArmsOptions {\n  /** Arm treated as the control side of every pair. */\n  baselineArm: string\n  /** Arm treated as the treatment side of every pair. */\n  treatmentArm: string\n}\n\n/** One matched (baseline, treatment) observation of the same work item. */\nexport interface MatchedPair {\n  pairKey: string\n  /** 0-based position of this pair within its `pairKey`, ordered by sorted\n   *  `repKey` (always 0 for a single-rep item). The rep identity itself is on\n   *  the rows (`baseline.repKey` / `treatment.repKey`). */\n  repIndex: number\n  baseline: PairedArmRow\n  treatment: PairedArmRow\n}\n\nexport interface PairArmsResult {\n  /** Matched pairs, ordered by (`pairKey`, `repIndex`). */\n  pairs: MatchedPair[]\n  /** Baseline rows left without a treatment counterpart — reported, never\n   *  silently dropped. */\n  unpairedBaseline: PairedArmRow[]\n  /** Treatment rows left without a baseline counterpart. */\n  unpairedTreatment: PairedArmRow[]\n}\n\n/**\n * Match rows across two arms into (baseline, treatment) pairs by `pairKey`.\n *\n * A `pairKey` with at most one row per arm pairs directly, no `repKey`\n * needed. A `pairKey` with multiple reps in either arm requires `repKey` on\n * every one of its rows, and reps pair only on exact (`pairKey`, `repKey`)\n * match — pairing is keyed purely on row identity, never on outcome content\n * (outcome-keyed matching deflates discordant counts and biases McNemar), and\n * is therefore independent of input order. Reps whose `repKey` has no\n * counterpart in the other arm, and items present in only one arm, land in\n * the unpaired lists — reported, never truncated.\n *\n * Fail-loud: throws when either named arm has zero rows (an unknown arm\n * name would otherwise read as \"everything unpaired\"), when the two arm\n * names are equal, when a multi-rep `pairKey` has a row without `repKey`, or\n * when a (`pairKey`, arm) group repeats a `repKey` (the match would be\n * ambiguous).\n */\nexport function pairArms(rows: readonly PairedArmRow[], opts: PairArmsOptions): PairArmsResult {\n  const { baselineArm, treatmentArm } = opts\n  if (baselineArm === treatmentArm) {\n    throw new ValidationError(\n      `pairArms: baselineArm and treatmentArm are both '${baselineArm}' — an arm cannot be compared to itself`,\n    )\n  }\n\n  // arm → pairKey → rows\n  const byArm = new Map<string, Map<string, PairedArmRow[]>>()\n  const armsSeen = new Set<string>()\n  for (const row of rows) {\n    armsSeen.add(row.arm)\n    if (row.arm !== baselineArm && row.arm !== treatmentArm) continue\n    const byKey = byArm.get(row.arm) ?? new Map<string, PairedArmRow[]>()\n    const group = byKey.get(row.pairKey) ?? []\n    group.push(row)\n    byKey.set(row.pairKey, group)\n    byArm.set(row.arm, byKey)\n  }\n\n  for (const arm of [baselineArm, treatmentArm]) {\n    if (!byArm.has(arm)) {\n      const seen = [...armsSeen].sort().join(', ') || '<none>'\n      throw new ValidationError(`pairArms: no rows for arm '${arm}' (arms present: ${seen})`)\n    }\n  }\n\n  const baselineByKey = byArm.get(baselineArm)!\n  const treatmentByKey = byArm.get(treatmentArm)!\n\n  const allKeys = [...new Set([...baselineByKey.keys(), ...treatmentByKey.keys()])].sort()\n  const pairs: MatchedPair[] = []\n  const unpairedBaseline: PairedArmRow[] = []\n  const unpairedTreatment: PairedArmRow[] = []\n  for (const pairKey of allKeys) {\n    const b = baselineByKey.get(pairKey) ?? []\n    const t = treatmentByKey.get(pairKey) ?? []\n\n    if (b.length <= 1 && t.length <= 1) {\n      if (b.length === 1 && t.length === 1) {\n        const baseline = b[0]!\n        const treatment = t[0]!\n        if (baseline.repKey !== undefined || treatment.repKey !== undefined) {\n          if (\n            baseline.repKey === undefined ||\n            treatment.repKey === undefined ||\n            baseline.repKey !== treatment.repKey\n          ) {\n            unpairedBaseline.push(baseline)\n            unpairedTreatment.push(treatment)\n            continue\n          }\n        }\n        pairs.push({ pairKey, repIndex: 0, baseline, treatment })\n      } else {\n        unpairedBaseline.push(...b)\n        unpairedTreatment.push(...t)\n      }\n      continue\n    }\n\n    const bByRep = indexByRepKey(b, pairKey, baselineArm)\n    const tByRep = indexByRepKey(t, pairKey, treatmentArm)\n    const repKeys = [...new Set([...bByRep.keys(), ...tByRep.keys()])].sort()\n    let repIndex = 0\n    for (const repKey of repKeys) {\n      const baseline = bByRep.get(repKey)\n      const treatment = tByRep.get(repKey)\n      if (baseline !== undefined && treatment !== undefined) {\n        pairs.push({ pairKey, repIndex: repIndex++, baseline, treatment })\n      } else if (baseline !== undefined) {\n        unpairedBaseline.push(baseline)\n      } else if (treatment !== undefined) {\n        unpairedTreatment.push(treatment)\n      }\n    }\n  }\n\n  return { pairs, unpairedBaseline, unpairedTreatment }\n}\n\n/** Index a multi-rep (pairKey, arm) group by `repKey`, enforcing that every\n *  row carries one and that no repKey repeats within the group. */\nfunction indexByRepKey(\n  group: readonly PairedArmRow[],\n  pairKey: string,\n  arm: string,\n): Map<string, PairedArmRow> {\n  const byRep = new Map<string, PairedArmRow>()\n  for (const row of group) {\n    if (row.repKey === undefined) {\n      throw new ValidationError(\n        `pairArms: pairKey '${pairKey}' has multiple reps in an arm, but a row in arm '${arm}' ` +\n          `is missing repKey — multi-rep items require an explicit repKey on every row so reps ` +\n          `pair by identity (pairing reps by outcome or by index would bias the paired statistics)`,\n      )\n    }\n    if (byRep.has(row.repKey)) {\n      throw new ValidationError(\n        `pairArms: duplicate repKey '${row.repKey}' for pairKey '${pairKey}' in arm '${arm}' — ` +\n          `(pairKey, repKey) must uniquely identify a rep within an arm`,\n      )\n    }\n    byRep.set(row.repKey, row)\n  }\n  return byRep\n}\n\n/** Paired pass/fail comparison over the pairs where BOTH sides carry `pass`. */\nexport interface PairedCorrectness {\n  /** Discordant pairs where the treatment passed and the baseline failed. */\n  b10: number\n  /** Discordant pairs where the baseline passed and the treatment failed. */\n  b01: number\n  /** Exact McNemar significance over the paired outcomes (`b === b10`, `c === b01`). */\n  mcnemar: McNemarResult\n  /** Paired effect size: p(treatment) − p(baseline) with a paired-variance CI. */\n  riskDifference: RiskDifferenceResult\n}\n\n/** Paired delta summary for one named metric (delta = treatment − baseline). */\nexport interface PairedMetricDelta {\n  name: string\n  /** Pairs where BOTH sides carry a finite value for this metric. */\n  n: number\n  /** Pairs where at least one side does not carry the metric. */\n  nMissing: number\n  /** Median paired delta, or null when `n === 0`. */\n  medianDelta: number | null\n  /** Mean paired delta, or null when `n === 0`. */\n  meanDelta: number | null\n  /** Bootstrap CI on the paired delta (`pairedBootstrap`); null when\n   *  `n === 0` — a zero-width [0, 0] interval on no data would read as a\n   *  measured tight null. */\n  bootstrapCi: PairedBootstrapResult | null\n  /** Wilcoxon signed-rank test on the paired deltas; null when `n === 0`. */\n  wilcoxon: { w: number; p: number } | null\n}\n\nexport interface ComparePairedArmsOptions extends PairArmsOptions {\n  /** Metrics to compare. Default: every metric name observed on any matched\n   *  pair, sorted. A name that appears on no pair is still reported (with\n   *  `n = 0`) so a misspelled metric is visible instead of vanishing. */\n  metricNames?: string[]\n  /** Passed through to `pairedBootstrap` — set `seed` for reproducible CIs. */\n  bootstrap?: PairedBootstrapOptions\n}\n\nexport interface PairedArmsComparison {\n  nPairs: number\n  nUnpairedBaseline: number\n  nUnpairedTreatment: number\n  /** null when no matched pair carries `pass` on both sides — a pass/fail\n   *  verdict over rows that never measured pass/fail would be fabricated. */\n  correctness: PairedCorrectness | null\n  metricDeltas: PairedMetricDelta[]\n}\n\n/**\n * Full matched-pair arm comparison: pair via {@link pairArms}, then compose\n * the paired estimators from `statistics` over the matched pairs.\n *\n * Correctness uses only the pairs where both sides carry `pass` (`mcnemar.n`\n * is that subset's size); each metric uses only the pairs where both sides\n * carry a finite value for it, with the remainder counted in `nMissing`.\n * Deltas are treatment − baseline throughout.\n *\n * Fail-loud: inherits {@link pairArms}'s unknown-arm throw, and throws on a\n * non-finite metric value — silently treating corrupt telemetry as \"metric\n * absent\" would misreport it as missing coverage.\n */\nexport function comparePairedArms(\n  rows: readonly PairedArmRow[],\n  opts: ComparePairedArmsOptions,\n): PairedArmsComparison {\n  const { pairs, unpairedBaseline, unpairedTreatment } = pairArms(rows, opts)\n\n  let correctness: PairedCorrectness | null = null\n  const baselinePass: number[] = []\n  const treatmentPass: number[] = []\n  for (const pair of pairs) {\n    if (pair.baseline.pass === undefined || pair.treatment.pass === undefined) continue\n    baselinePass.push(pair.baseline.pass ? 1 : 0)\n    treatmentPass.push(pair.treatment.pass ? 1 : 0)\n  }\n  if (baselinePass.length > 0) {\n    const mc = mcnemar(baselinePass, treatmentPass)\n    correctness = {\n      b10: mc.b,\n      b01: mc.c,\n      mcnemar: mc,\n      riskDifference: pairedRiskDifference(baselinePass, treatmentPass),\n    }\n  }\n\n  const metricNames =\n    opts.metricNames ??\n    [\n      ...new Set(\n        pairs.flatMap((p) => [\n          ...Object.keys(p.baseline.metrics ?? {}),\n          ...Object.keys(p.treatment.metrics ?? {}),\n        ]),\n      ),\n    ].sort()\n\n  const metricDeltas: PairedMetricDelta[] = metricNames.map((name) => {\n    const before: number[] = []\n    const after: number[] = []\n    let nMissing = 0\n    for (const pair of pairs) {\n      const b = metricValue(pair.baseline, name)\n      const t = metricValue(pair.treatment, name)\n      if (b === undefined || t === undefined) {\n        nMissing++\n        continue\n      }\n      before.push(b)\n      after.push(t)\n    }\n    const bootstrapCi = before.length === 0 ? null : pairedBootstrap(before, after, opts.bootstrap)\n    return {\n      name,\n      n: before.length,\n      nMissing,\n      medianDelta: bootstrapCi?.median ?? null,\n      meanDelta: bootstrapCi?.mean ?? null,\n      bootstrapCi,\n      wilcoxon: before.length === 0 ? null : wilcoxonSignedRank(before, after),\n    }\n  })\n\n  return {\n    nPairs: pairs.length,\n    nUnpairedBaseline: unpairedBaseline.length,\n    nUnpairedTreatment: unpairedTreatment.length,\n    correctness,\n    metricDeltas,\n  }\n}\n\nexport interface MatchedRunRecordPair {\n  pairKey: string\n  repKey: string\n  baseline: RunRecord\n  treatment: RunRecord\n}\n\nexport interface PairRunRecordsResult {\n  pairs: MatchedRunRecordPair[]\n  unpairedBaseline: RunRecord[]\n  unpairedTreatment: RunRecord[]\n}\n\ninterface RunRecordArmRow extends PairedArmRow {\n  run: RunRecord\n  repKey: string\n}\n\n/**\n * Pair two RunRecord arms by the identity of the evaluated work:\n * `(experimentId, scenarioId, seed)`.\n *\n * Falling back to array order, candidate id, or experiment id can compare\n * different tasks and fabricate lift. Duplicate identities throw.\n */\nexport function pairRunRecords(\n  baselineRuns: readonly RunRecord[],\n  treatmentRuns: readonly RunRecord[],\n): PairRunRecordsResult {\n  const baselineRows = runRecordArmRows(baselineRuns, 'baseline')\n  const treatmentRows = runRecordArmRows(treatmentRuns, 'treatment')\n  validateRunRecordArmRows(baselineRows, 'baseline')\n  validateRunRecordArmRows(treatmentRows, 'treatment')\n  if (baselineRows.length === 0 || treatmentRows.length === 0) {\n    return {\n      pairs: [],\n      unpairedBaseline: baselineRows.map((row) => row.run),\n      unpairedTreatment: treatmentRows.map((row) => row.run),\n    }\n  }\n\n  const result = pairArms([...baselineRows, ...treatmentRows], {\n    baselineArm: 'baseline',\n    treatmentArm: 'treatment',\n  })\n  return {\n    pairs: result.pairs.map((pair) => {\n      const baseline = pair.baseline as RunRecordArmRow\n      const treatment = pair.treatment as RunRecordArmRow\n      return {\n        pairKey: pair.pairKey,\n        repKey: baseline.repKey,\n        baseline: baseline.run,\n        treatment: treatment.run,\n      }\n    }),\n    unpairedBaseline: result.unpairedBaseline.map((row) => (row as RunRecordArmRow).run),\n    unpairedTreatment: result.unpairedTreatment.map((row) => (row as RunRecordArmRow).run),\n  }\n}\n\nfunction runRecordArmRows(runs: readonly RunRecord[], arm: string): RunRecordArmRow[] {\n  return runs.map((run) => {\n    const scenarioId = run.scenarioId.trim()\n    if (!scenarioId) {\n      throw new ValidationError(\n        `pairRunRecords: run '${run.runId}' is missing scenarioId; paired comparisons require explicit scenario identity`,\n      )\n    }\n    return {\n      pairKey: JSON.stringify([run.experimentId, scenarioId]),\n      repKey: String(run.seed),\n      arm,\n      run,\n    }\n  })\n}\n\nfunction validateRunRecordArmRows(rows: readonly RunRecordArmRow[], arm: string): void {\n  const byPairKey = new Map<string, RunRecordArmRow[]>()\n  for (const row of rows) {\n    const group = byPairKey.get(row.pairKey) ?? []\n    group.push(row)\n    byPairKey.set(row.pairKey, group)\n  }\n  for (const [pairKey, group] of byPairKey) {\n    if (group.length > 1) indexByRepKey(group, pairKey, arm)\n  }\n}\n\nfunction metricValue(row: PairedArmRow, name: string): number | undefined {\n  const v = row.metrics?.[name]\n  if (v === undefined) return undefined\n  if (!Number.isFinite(v)) {\n    throw new ValidationError(\n      `comparePairedArms: non-finite value for metric '${name}' on pairKey '${row.pairKey}' (arm '${row.arm}'): ${v}`,\n    )\n  }\n  return v\n}\n"],"mappings":";;;;;;;;;;;AAuGO,SAAS,SAAS,MAA+B,MAAuC;AAC7F,QAAM,EAAE,aAAa,aAAa,IAAI;AACtC,MAAI,gBAAgB,cAAc;AAChC,UAAM,IAAI;AAAA,MACR,oDAAoD,WAAW;AAAA,IACjE;AAAA,EACF;AAGA,QAAM,QAAQ,oBAAI,IAAyC;AAC3D,QAAM,WAAW,oBAAI,IAAY;AACjC,aAAW,OAAO,MAAM;AACtB,aAAS,IAAI,IAAI,GAAG;AACpB,QAAI,IAAI,QAAQ,eAAe,IAAI,QAAQ,aAAc;AACzD,UAAM,QAAQ,MAAM,IAAI,IAAI,GAAG,KAAK,oBAAI,IAA4B;AACpE,UAAM,QAAQ,MAAM,IAAI,IAAI,OAAO,KAAK,CAAC;AACzC,UAAM,KAAK,GAAG;AACd,UAAM,IAAI,IAAI,SAAS,KAAK;AAC5B,UAAM,IAAI,IAAI,KAAK,KAAK;AAAA,EAC1B;AAEA,aAAW,OAAO,CAAC,aAAa,YAAY,GAAG;AAC7C,QAAI,CAAC,MAAM,IAAI,GAAG,GAAG;AACnB,YAAM,OAAO,CAAC,GAAG,QAAQ,EAAE,KAAK,EAAE,KAAK,IAAI,KAAK;AAChD,YAAM,IAAI,gBAAgB,8BAA8B,GAAG,oBAAoB,IAAI,GAAG;AAAA,IACxF;AAAA,EACF;AAEA,QAAM,gBAAgB,MAAM,IAAI,WAAW;AAC3C,QAAM,iBAAiB,MAAM,IAAI,YAAY;AAE7C,QAAM,UAAU,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,cAAc,KAAK,GAAG,GAAG,eAAe,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK;AACvF,QAAM,QAAuB,CAAC;AAC9B,QAAM,mBAAmC,CAAC;AAC1C,QAAM,oBAAoC,CAAC;AAC3C,aAAW,WAAW,SAAS;AAC7B,UAAM,IAAI,cAAc,IAAI,OAAO,KAAK,CAAC;AACzC,UAAM,IAAI,eAAe,IAAI,OAAO,KAAK,CAAC;AAE1C,QAAI,EAAE,UAAU,KAAK,EAAE,UAAU,GAAG;AAClC,UAAI,EAAE,WAAW,KAAK,EAAE,WAAW,GAAG;AACpC,cAAM,WAAW,EAAE,CAAC;AACpB,cAAM,YAAY,EAAE,CAAC;AACrB,YAAI,SAAS,WAAW,UAAa,UAAU,WAAW,QAAW;AACnE,cACE,SAAS,WAAW,UACpB,UAAU,WAAW,UACrB,SAAS,WAAW,UAAU,QAC9B;AACA,6BAAiB,KAAK,QAAQ;AAC9B,8BAAkB,KAAK,SAAS;AAChC;AAAA,UACF;AAAA,QACF;AACA,cAAM,KAAK,EAAE,SAAS,UAAU,GAAG,UAAU,UAAU,CAAC;AAAA,MAC1D,OAAO;AACL,yBAAiB,KAAK,GAAG,CAAC;AAC1B,0BAAkB,KAAK,GAAG,CAAC;AAAA,MAC7B;AACA;AAAA,IACF;AAEA,UAAM,SAAS,cAAc,GAAG,SAAS,WAAW;AACpD,UAAM,SAAS,cAAc,GAAG,SAAS,YAAY;AACrD,UAAM,UAAU,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,OAAO,KAAK,GAAG,GAAG,OAAO,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK;AACxE,QAAI,WAAW;AACf,eAAW,UAAU,SAAS;AAC5B,YAAM,WAAW,OAAO,IAAI,MAAM;AAClC,YAAM,YAAY,OAAO,IAAI,MAAM;AACnC,UAAI,aAAa,UAAa,cAAc,QAAW;AACrD,cAAM,KAAK,EAAE,SAAS,UAAU,YAAY,UAAU,UAAU,CAAC;AAAA,MACnE,WAAW,aAAa,QAAW;AACjC,yBAAiB,KAAK,QAAQ;AAAA,MAChC,WAAW,cAAc,QAAW;AAClC,0BAAkB,KAAK,SAAS;AAAA,MAClC;AAAA,IACF;AAAA,EACF;AAEA,SAAO,EAAE,OAAO,kBAAkB,kBAAkB;AACtD;AAIA,SAAS,cACP,OACA,SACA,KAC2B;AAC3B,QAAM,QAAQ,oBAAI,IAA0B;AAC5C,aAAW,OAAO,OAAO;AACvB,QAAI,IAAI,WAAW,QAAW;AAC5B,YAAM,IAAI;AAAA,QACR,sBAAsB,OAAO,oDAAoD,GAAG;AAAA,MAGtF;AAAA,IACF;AACA,QAAI,MAAM,IAAI,IAAI,MAAM,GAAG;AACzB,YAAM,IAAI;AAAA,QACR,+BAA+B,IAAI,MAAM,kBAAkB,OAAO,aAAa,GAAG;AAAA,MAEpF;AAAA,IACF;AACA,UAAM,IAAI,IAAI,QAAQ,GAAG;AAAA,EAC3B;AACA,SAAO;AACT;AAiEO,SAAS,kBACd,MACA,MACsB;AACtB,QAAM,EAAE,OAAO,kBAAkB,kBAAkB,IAAI,SAAS,MAAM,IAAI;AAE1E,MAAI,cAAwC;AAC5C,QAAM,eAAyB,CAAC;AAChC,QAAM,gBAA0B,CAAC;AACjC,aAAW,QAAQ,OAAO;AACxB,QAAI,KAAK,SAAS,SAAS,UAAa,KAAK,UAAU,SAAS,OAAW;AAC3E,iBAAa,KAAK,KAAK,SAAS,OAAO,IAAI,CAAC;AAC5C,kBAAc,KAAK,KAAK,UAAU,OAAO,IAAI,CAAC;AAAA,EAChD;AACA,MAAI,aAAa,SAAS,GAAG;AAC3B,UAAM,KAAK,QAAQ,cAAc,aAAa;AAC9C,kBAAc;AAAA,MACZ,KAAK,GAAG;AAAA,MACR,KAAK,GAAG;AAAA,MACR,SAAS;AAAA,MACT,gBAAgB,qBAAqB,cAAc,aAAa;AAAA,IAClE;AAAA,EACF;AAEA,QAAM,cACJ,KAAK,eACL;AAAA,IACE,GAAG,IAAI;AAAA,MACL,MAAM,QAAQ,CAAC,MAAM;AAAA,QACnB,GAAG,OAAO,KAAK,EAAE,SAAS,WAAW,CAAC,CAAC;AAAA,QACvC,GAAG,OAAO,KAAK,EAAE,UAAU,WAAW,CAAC,CAAC;AAAA,MAC1C,CAAC;AAAA,IACH;AAAA,EACF,EAAE,KAAK;AAET,QAAM,eAAoC,YAAY,IAAI,CAAC,SAAS;AAClE,UAAM,SAAmB,CAAC;AAC1B,UAAM,QAAkB,CAAC;AACzB,QAAI,WAAW;AACf,eAAW,QAAQ,OAAO;AACxB,YAAM,IAAI,YAAY,KAAK,UAAU,IAAI;AACzC,YAAM,IAAI,YAAY,KAAK,WAAW,IAAI;AAC1C,UAAI,MAAM,UAAa,MAAM,QAAW;AACtC;AACA;AAAA,MACF;AACA,aAAO,KAAK,CAAC;AACb,YAAM,KAAK,CAAC;AAAA,IACd;AACA,UAAM,cAAc,OAAO,WAAW,IAAI,OAAO,gBAAgB,QAAQ,OAAO,KAAK,SAAS;AAC9F,WAAO;AAAA,MACL;AAAA,MACA,GAAG,OAAO;AAAA,MACV;AAAA,MACA,aAAa,aAAa,UAAU;AAAA,MACpC,WAAW,aAAa,QAAQ;AAAA,MAChC;AAAA,MACA,UAAU,OAAO,WAAW,IAAI,OAAO,mBAAmB,QAAQ,KAAK;AAAA,IACzE;AAAA,EACF,CAAC;AAED,SAAO;AAAA,IACL,QAAQ,MAAM;AAAA,IACd,mBAAmB,iBAAiB;AAAA,IACpC,oBAAoB,kBAAkB;AAAA,IACtC;AAAA,IACA;AAAA,EACF;AACF;AA2BO,SAAS,eACd,cACA,eACsB;AACtB,QAAM,eAAe,iBAAiB,cAAc,UAAU;AAC9D,QAAM,gBAAgB,iBAAiB,eAAe,WAAW;AACjE,2BAAyB,cAAc,UAAU;AACjD,2BAAyB,eAAe,WAAW;AACnD,MAAI,aAAa,WAAW,KAAK,cAAc,WAAW,GAAG;AAC3D,WAAO;AAAA,MACL,OAAO,CAAC;AAAA,MACR,kBAAkB,aAAa,IAAI,CAAC,QAAQ,IAAI,GAAG;AAAA,MACnD,mBAAmB,cAAc,IAAI,CAAC,QAAQ,IAAI,GAAG;AAAA,IACvD;AAAA,EACF;AAEA,QAAM,SAAS,SAAS,CAAC,GAAG,cAAc,GAAG,aAAa,GAAG;AAAA,IAC3D,aAAa;AAAA,IACb,cAAc;AAAA,EAChB,CAAC;AACD,SAAO;AAAA,IACL,OAAO,OAAO,MAAM,IAAI,CAAC,SAAS;AAChC,YAAM,WAAW,KAAK;AACtB,YAAM,YAAY,KAAK;AACvB,aAAO;AAAA,QACL,SAAS,KAAK;AAAA,QACd,QAAQ,SAAS;AAAA,QACjB,UAAU,SAAS;AAAA,QACnB,WAAW,UAAU;AAAA,MACvB;AAAA,IACF,CAAC;AAAA,IACD,kBAAkB,OAAO,iBAAiB,IAAI,CAAC,QAAS,IAAwB,GAAG;AAAA,IACnF,mBAAmB,OAAO,kBAAkB,IAAI,CAAC,QAAS,IAAwB,GAAG;AAAA,EACvF;AACF;AAEA,SAAS,iBAAiB,MAA4B,KAAgC;AACpF,SAAO,KAAK,IAAI,CAAC,QAAQ;AACvB,UAAM,aAAa,IAAI,WAAW,KAAK;AACvC,QAAI,CAAC,YAAY;AACf,YAAM,IAAI;AAAA,QACR,wBAAwB,IAAI,KAAK;AAAA,MACnC;AAAA,IACF;AACA,WAAO;AAAA,MACL,SAAS,KAAK,UAAU,CAAC,IAAI,cAAc,UAAU,CAAC;AAAA,MACtD,QAAQ,OAAO,IAAI,IAAI;AAAA,MACvB;AAAA,MACA;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAEA,SAAS,yBAAyB,MAAkC,KAAmB;AACrF,QAAM,YAAY,oBAAI,IAA+B;AACrD,aAAW,OAAO,MAAM;AACtB,UAAM,QAAQ,UAAU,IAAI,IAAI,OAAO,KAAK,CAAC;AAC7C,UAAM,KAAK,GAAG;AACd,cAAU,IAAI,IAAI,SAAS,KAAK;AAAA,EAClC;AACA,aAAW,CAAC,SAAS,KAAK,KAAK,WAAW;AACxC,QAAI,MAAM,SAAS,EAAG,eAAc,OAAO,SAAS,GAAG;AAAA,EACzD;AACF;AAEA,SAAS,YAAY,KAAmB,MAAkC;AACxE,QAAM,IAAI,IAAI,UAAU,IAAI;AAC5B,MAAI,MAAM,OAAW,QAAO;AAC5B,MAAI,CAAC,OAAO,SAAS,CAAC,GAAG;AACvB,UAAM,IAAI;AAAA,MACR,mDAAmD,IAAI,iBAAiB,IAAI,OAAO,WAAW,IAAI,GAAG,OAAO,CAAC;AAAA,IAC/G;AAAA,EACF;AACA,SAAO;AACT;","names":[]}