@tangle-network/agent-eval 0.161.1 → 0.170.0

package/dist/contract/index.js

@@ -1,23 +1,23 @@
 import { s as ValidationError } from "../errors-Dngq5h35.js";
-import { i as hashCanonical } from "../canonical-IL-Bu-14.js";
-import { r as pairedBootstrap } from "../paired-tests-BHIhYVdu.js";
-import { a as summarizeExecution, i as analyzeRuns, n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-h-s-sI-v.js";
-import { a as parseRunRecordSafe } from "../run-record-BC0ebuRP.js";
-import { E as compareOptimizationMethods, Y as defaultProductionGate, ft as campaignSplitDigest, i as transientDispatchFailure, m as runImprovementLoop, nt as runCampaign, t as llmJudge, tt as runEval } from "../llm-judge-BhasIPFT.js";
+import { a as hashCanonical } from "../canonical-DPyQ_rpt.js";
+import { r as pairedBootstrap } from "../paired-tests-C8iCsioC.js";
+import { a as summarizeExecution, i as analyzeRuns, n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-Clj-8igZ.js";
+import { a as parseRunRecordSafe } from "../run-record-DLORoL7t.js";
+import { O as compareOptimizationMethods, Z as defaultProductionGate, g as runImprovementLoop, i as transientDispatchFailure, it as runCampaign, mt as campaignSplitDigest, rt as runEval, t as llmJudge } from "../llm-judge-DbJdo8Nj.js";
 import { i as isModelPriced, n as estimateCost } from "../metrics-Qv-cpptD.js";
 import { i as CostLedger } from "../cost-ledger-B1qx30B4.js";
-import { d as mapConcurrentRange } from "../ledger-core-BOzlRygb.js";
-import { S as inMemoryCampaignStorage, x as fsCampaignStorage } from "../external-optimizer-subprocess-BIWbHpgD.js";
-import { a as heldoutSignificance, s as decidePairedPromotion } from "../power-preflight-DEw-uC7q.js";
-import { r as makeProposalFinding } from "../types-BI4fT3HN.js";
-import { G as isOtlpModelCall, W as classifyOtlpSpanRole } from "../kind-factory-DY8FdoXf.js";
-import { n as buildDefaultAnalystRegistry, t as createChatClient } from "../chat-client-DlMlAeYI.js";
+import { d as mapConcurrentRange } from "../ledger-core-PIfjCbKn.js";
+import { C as inMemoryCampaignStorage, S as fsCampaignStorage } from "../external-optimizer-subprocess-Cex8Da2i.js";
+import { a as heldoutSignificance, s as decidePairedPromotion } from "../power-preflight-CFXm0Vjo.js";
+import { i as makeProposalFinding } from "../types-CiWITkGo.js";
+import { G as isOtlpModelCall, W as classifyOtlpSpanRole } from "../kind-factory-DMeEoMQZ.js";
+import { n as buildDefaultAnalystRegistry, t as createChatClient } from "../chat-client-DEtybj5i.js";
 import { LLM_MODEL_ATTR_KEYS, SPAN_KIND_ATTR_KEYS } from "../trace-attributes.js";
 import { t as extractUsage } from "../extract-usage-BrQ8mCLX.js";
 import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-ChBKlTd_.js";
-import { i as summarizeTraceErrors, n as recordAggregateMeasurements, r as summarizeExecutionMeasurements, t as readTaskFailureLabels } from "../task-failure-attributes-DTl-7-Kw.js";
-import { c as externalTextOptimizationMethod, d as REFERENCE_EQUIVALENCE_INPUT_LIMITS, f as REFERENCE_EQUIVALENCE_JUDGE_VERSION, m as runReferenceEquivalenceJudge, n as gepaOptimizationMethod, o as heldOutGate, p as createReferenceEquivalenceJudge, s as composeGate, t as skillOptOptimizationMethod } from "../skillopt-optimization-method-DbaekMcn.js";
-import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-xzA40Evo.js";
+import { i as summarizeTraceErrors, n as recordAggregateMeasurements, r as summarizeExecutionMeasurements, t as readTaskFailureLabels } from "../task-failure-attributes-CBGtLS_H.js";
+import { c as externalTextOptimizationMethod, d as REFERENCE_EQUIVALENCE_INPUT_LIMITS, f as REFERENCE_EQUIVALENCE_JUDGE_VERSION, m as runReferenceEquivalenceJudge, n as gepaOptimizationMethod, o as heldOutGate, p as createReferenceEquivalenceJudge, s as composeGate, t as skillOptOptimizationMethod } from "../skillopt-optimization-method-B2R9C5aG.js";
+import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-LY9mVQ7W.js";
 import { createHash } from "node:crypto";
 import { agentCandidateBenchmarkSuiteSchema, agentCandidateBenchmarkTaskSchema, agentCandidateBundleSchema, agentCandidateEvaluationPolicySchema, agentCandidateExperimentSchema, agentImprovementMeasuredComparisonSchema, agentProfileImprovementExperimentSchema, agentProfileImprovementMeasuredComparisonSchema, agentProfileImprovementRunCellSchema, agentProfileImprovementRunReceiptSchema, agentProfileImprovementSuiteInputsSchema, agentProfileImprovementSuiteSchema, agentProfileImprovementTaskSchema, candidateExecutionEvidenceSchema, canonicalCandidateDigest, canonicalCandidateJson, numbersApproximatelyEqual, omitTopLevelDigest } from "@tangle-network/agent-interface";
 import { createReadStream } from "node:fs";

package/dist/{counterfactual-BaFUWK3H.d.ts → counterfactual-Bee5_BIn.d.ts}

@@ -1,6 +1,6 @@
-import { s as TraceStore } from "./store-B06JdC56.js";
-import { i as TraceEmitter } from "./emitter-D_jYSGRd.js";
-import { n as TrajectoryStep, t as Trajectory } from "./trajectory-Bi157Gun.js";
+import { s as TraceStore } from "./store-Cq9oOrI1.js";
+import { i as TraceEmitter } from "./emitter-Bvnu0VzL.js";
+import { n as TrajectoryStep, t as Trajectory } from "./trajectory-r1bQqvBQ.js";
 //#region src/counterfactual.d.ts
 type CounterfactualMutation = {
   kind: 'swap-model';
@@ -57,4 +57,4 @@ interface CounterfactualRunner {
 declare function runCounterfactual(store: TraceStore, originalRunId: string, mutation: CounterfactualMutation, runner: CounterfactualRunner): Promise<CounterfactualResult>;
 //#endregion
 export { runCounterfactual as a, CounterfactualRunner as i, CounterfactualMutation as n, CounterfactualResult as r, CounterfactualContext as t };
-//# sourceMappingURL=counterfactual-BaFUWK3H.d.ts.map
+//# sourceMappingURL=counterfactual-Bee5_BIn.d.ts.map

package/dist/{counterfactual-BaFUWK3H.d.ts.map → counterfactual-Bee5_BIn.d.ts.map}

@@ -1 +1 @@
-{"version":3,"file":"counterfactual-BaFUWK3H.d.ts","names":[],"sources":["../src/counterfactual.ts"],"mappings":";;;;KAoBY;EACN;EAAoB;EAAY;;EAChC;EAA0B;EAAY;;EACtC;EAAwB;;EACxB;EAA+B;EAAY;;EAE3C;EACA;EACA;EACA,QAAQ,MAAM,mBAAmB;;UAGtB;EACf;EACA,oBAAoB;;;EAGpB,QAAQ;EACR,UAAU;;;;EAIV,aAAa;;UAGE;EACf;EACA;EACA,UAAU;;EAEV;IACE;IACA;IACA;;;UAIa;;;;;;EAMf,cAAc,KAAK,uBAAuB,SAAS,iBAAiB;;iBAGhD,kBACpB,OAAO,YACP,uBACA,UAAU,wBACV,QAAQ,uBACP,QAAQ"}
+{"version":3,"file":"counterfactual-Bee5_BIn.d.ts","names":[],"sources":["../src/counterfactual.ts"],"mappings":";;;;KAoBY;EACN;EAAoB;EAAY;;EAChC;EAA0B;EAAY;;EACtC;EAAwB;;EACxB;EAA+B;EAAY;;EAE3C;EACA;EACA;EACA,QAAQ,MAAM,mBAAmB;;UAGtB;EACf;EACA,oBAAoB;;;EAGpB,QAAQ;EACR,UAAU;;;;EAIV,aAAa;;UAGE;EACf;EACA;EACA,UAAU;;EAEV;IACE;IACA;IACA;;;UAIa;;;;;;EAMf,cAAc,KAAK,uBAAuB,SAAS,iBAAiB;;iBAGhD,kBACpB,OAAO,YACP,uBACA,UAAU,wBACV,QAAQ,uBACP,QAAQ"}

package/dist/{counterfactual-D_VWavVm.js → counterfactual-Bjq1mlUu.js}

@@ -1,5 +1,5 @@
 import { o as NotFoundError, s as ValidationError } from "./errors-Dngq5h35.js";
-import { t as TraceEmitter } from "./emitter-BpYFQPj4.js";
+import { t as TraceEmitter } from "./emitter-DeQHiDMm.js";
 import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
 //#region src/counterfactual.ts
 /**
@@ -95,4 +95,4 @@ function applyMutation(step, mutation) {
 //#endregion
 export { runCounterfactual as t };
 
-//# sourceMappingURL=counterfactual-D_VWavVm.js.map
+//# sourceMappingURL=counterfactual-Bjq1mlUu.js.map

package/dist/{counterfactual-D_VWavVm.js.map → counterfactual-Bjq1mlUu.js.map}

@@ -1 +1 @@
-{"version":3,"file":"counterfactual-D_VWavVm.js","names":[],"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n  | { kind: 'swap-model'; at: number; newModel: string }\n  | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n  | { kind: 'truncate-after'; at: number }\n  | { kind: 'inject-system-message'; at: number; content: string }\n  | {\n      kind: 'custom'\n      at: number\n      describe: string\n      apply: (step: TrajectoryStep) => TrajectoryStep\n    }\n\nexport interface CounterfactualContext {\n  originalRunId: string\n  originalTrajectory: Trajectory\n  /** Steps up to (but not including) the mutation point — the prefix the\n   *  replayed agent inherits as its prior conversation/tool history. */\n  prefix: TrajectoryStep[]\n  mutation: CounterfactualMutation\n  /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n   *  as the FIRST step the replayed agent emits (they decide whether to\n   *  re-emit it or continue from there). */\n  mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n  counterfactualRunId: string\n  originalRunId: string\n  mutation: CounterfactualMutation\n  /** Structured delta summary — caller can extend via scoring. */\n  delta: {\n    originalOutcomeScore: number | null\n    counterfactualOutcomeScore: number | null\n    deltaScore: number | null\n  }\n}\n\nexport interface CounterfactualRunner {\n  /**\n   * Execute the agent from `ctx.prefix` with the mutation applied.\n   * MUST emit spans into the provided emitter so they become part of\n   * the counterfactual run. MUST call emitter.endRun() with a verdict.\n   */\n  executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n  store: TraceStore,\n  originalRunId: string,\n  mutation: CounterfactualMutation,\n  runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n  const originalRun = await store.getRun(originalRunId)\n  if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n  const trajectory = await buildTrajectory(store, originalRunId)\n  if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n    throw new ValidationError(\n      `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n    )\n  }\n  const targetStep = trajectory.steps[mutation.at]!\n  const mutatedStep = applyMutation(targetStep, mutation)\n\n  const cfEmitter = new TraceEmitter(store)\n  await cfEmitter.startRun({\n    scenarioId: originalRun.scenarioId,\n    variantId: originalRun.variantId\n      ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n      : `cf:${mutation.kind}@${mutation.at}`,\n    projectId: originalRun.projectId,\n    parentRunId: originalRunId,\n    layer: 'meta',\n    tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n  })\n\n  await runner.executeFrom(\n    {\n      originalRunId,\n      originalTrajectory: trajectory,\n      prefix: trajectory.steps.slice(0, mutation.at),\n      mutation,\n      mutatedStep,\n    },\n    cfEmitter,\n  )\n\n  const counterfactual = await store.getRun(cfEmitter.runId)\n  const delta = {\n    originalOutcomeScore: originalRun.outcome?.score ?? null,\n    counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n    deltaScore:\n      originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n        ? counterfactual.outcome.score - originalRun.outcome.score\n        : null,\n  }\n  return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n  if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return { ...step, span: { ...llm, model: mutation.newModel } }\n  }\n  if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n    const tool = step.span as ToolSpan\n    return { ...step, span: { ...tool, result: mutation.newResult } }\n  }\n  if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return {\n      ...step,\n      span: {\n        ...llm,\n        messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n      },\n    }\n  }\n  if (mutation.kind === 'custom') return mutation.apply(step)\n  // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n  return step\n}\n\n/**\n * Aggregate a batch of counterfactuals into a simple attribution table:\n * which mutation kinds move outcomes most? (Useful when you run a grid\n * over the same trajectory — swap-model at every llm span, swap-tool\n * at every tool span — and want a ranked summary.)\n */\nexport function attributeCounterfactuals(results: CounterfactualResult[]): Array<{\n  mutationKind: CounterfactualMutation['kind']\n  n: number\n  meanAbsDelta: number\n  meanSignedDelta: number\n}> {\n  const grouped = new Map<string, CounterfactualResult[]>()\n  for (const r of results) {\n    const arr = grouped.get(r.mutation.kind) ?? []\n    arr.push(r)\n    grouped.set(r.mutation.kind, arr)\n  }\n  const out: Array<{\n    mutationKind: CounterfactualMutation['kind']\n    n: number\n    meanAbsDelta: number\n    meanSignedDelta: number\n  }> = []\n  for (const [kind, items] of grouped) {\n    const deltas = items\n      .map((i) => i.delta.deltaScore)\n      .filter((d): d is number => typeof d === 'number')\n    if (deltas.length === 0) continue\n    const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length\n    const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length\n    out.push({\n      mutationKind: kind as CounterfactualMutation['kind'],\n      n: deltas.length,\n      meanAbsDelta: meanAbs,\n      meanSignedDelta: meanSigned,\n    })\n  }\n  return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta)\n}\n\n// Re-export Span type for consumer ergonomics.\nexport type { Span }\n"],"mappings":";;;;;;;;;;;;;;;;;AAkEA,eAAsB,kBACpB,OACA,eACA,UACA,QAC+B;CAC/B,MAAM,cAAc,MAAM,MAAM,OAAO,aAAa;CACpD,IAAI,CAAC,aAAa,MAAM,IAAI,cAAc,uBAAuB,cAAc,WAAW;CAC1F,MAAM,aAAa,MAAM,gBAAgB,OAAO,aAAa;CAC7D,IAAI,SAAS,KAAK,KAAK,SAAS,MAAM,WAAW,MAAM,QACrD,MAAM,IAAI,gBACR,+BAA+B,SAAS,GAAG,oBAAoB,WAAW,MAAM,OAAO,EACzF;CAEF,MAAM,aAAa,WAAW,MAAM,SAAS;CAC7C,MAAM,cAAc,cAAc,YAAY,QAAQ;CAEtD,MAAM,YAAY,IAAI,aAAa,KAAK;CACxC,MAAM,UAAU,SAAS;EACvB,YAAY,YAAY;EACxB,WAAW,YAAY,YACnB,GAAG,YAAY,UAAU,MAAM,SAAS,KAAK,GAAG,SAAS,OACzD,MAAM,SAAS,KAAK,GAAG,SAAS;EACpC,WAAW,YAAY;EACvB,aAAa;EACb,OAAO;EACP,MAAM;GAAE,gBAAgB;GAAQ,cAAc,SAAS;GAAM,YAAY,OAAO,SAAS,EAAE;EAAE;CAC/F,CAAC;CAED,MAAM,OAAO,YACX;EACE;EACA,oBAAoB;EACpB,QAAQ,WAAW,MAAM,MAAM,GAAG,SAAS,EAAE;EAC7C;EACA;CACF,GACA,SACF;CAEA,MAAM,iBAAiB,MAAM,MAAM,OAAO,UAAU,KAAK;CACzD,MAAM,QAAQ;EACZ,sBAAsB,YAAY,SAAS,SAAS;EACpD,4BAA4B,gBAAgB,SAAS,SAAS;EAC9D,YACE,YAAY,SAAS,UAAU,KAAA,KAAa,gBAAgB,SAAS,UAAU,KAAA,IAC3E,eAAe,QAAQ,QAAQ,YAAY,QAAQ,QACnD;CACR;CACA,OAAO;EAAE,qBAAqB,UAAU;EAAO;EAAe;EAAU;CAAM;AAChF;AAEA,SAAS,cAAc,MAAsB,UAAkD;CAC7F,IAAI,SAAS,SAAS,gBAAgB,KAAK,KAAK,SAAS,OAAO;EAC9D,MAAM,MAAM,KAAK;EACjB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAK,OAAO,SAAS;GAAS;EAAE;CAC/D;CACA,IAAI,SAAS,SAAS,sBAAsB,KAAK,KAAK,SAAS,QAAQ;EACrE,MAAM,OAAO,KAAK;EAClB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAM,QAAQ,SAAS;GAAU;EAAE;CAClE;CACA,IAAI,SAAS,SAAS,2BAA2B,KAAK,KAAK,SAAS,OAAO;EACzE,MAAM,MAAM,KAAK;EACjB,OAAO;GACL,GAAG;GACH,MAAM;IACJ,GAAG;IACH,UAAU,CAAC;KAAE,MAAM;KAAU,SAAS,SAAS;IAAQ,GAAG,GAAG,IAAI,QAAQ;GAC3E;EACF;CACF;CACA,IAAI,SAAS,SAAS,UAAU,OAAO,SAAS,MAAM,IAAI;CAE1D,OAAO;AACT"}
+{"version":3,"file":"counterfactual-Bjq1mlUu.js","names":[],"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n  | { kind: 'swap-model'; at: number; newModel: string }\n  | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n  | { kind: 'truncate-after'; at: number }\n  | { kind: 'inject-system-message'; at: number; content: string }\n  | {\n      kind: 'custom'\n      at: number\n      describe: string\n      apply: (step: TrajectoryStep) => TrajectoryStep\n    }\n\nexport interface CounterfactualContext {\n  originalRunId: string\n  originalTrajectory: Trajectory\n  /** Steps up to (but not including) the mutation point — the prefix the\n   *  replayed agent inherits as its prior conversation/tool history. */\n  prefix: TrajectoryStep[]\n  mutation: CounterfactualMutation\n  /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n   *  as the FIRST step the replayed agent emits (they decide whether to\n   *  re-emit it or continue from there). */\n  mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n  counterfactualRunId: string\n  originalRunId: string\n  mutation: CounterfactualMutation\n  /** Structured delta summary — caller can extend via scoring. */\n  delta: {\n    originalOutcomeScore: number | null\n    counterfactualOutcomeScore: number | null\n    deltaScore: number | null\n  }\n}\n\nexport interface CounterfactualRunner {\n  /**\n   * Execute the agent from `ctx.prefix` with the mutation applied.\n   * MUST emit spans into the provided emitter so they become part of\n   * the counterfactual run. MUST call emitter.endRun() with a verdict.\n   */\n  executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n  store: TraceStore,\n  originalRunId: string,\n  mutation: CounterfactualMutation,\n  runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n  const originalRun = await store.getRun(originalRunId)\n  if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n  const trajectory = await buildTrajectory(store, originalRunId)\n  if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n    throw new ValidationError(\n      `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n    )\n  }\n  const targetStep = trajectory.steps[mutation.at]!\n  const mutatedStep = applyMutation(targetStep, mutation)\n\n  const cfEmitter = new TraceEmitter(store)\n  await cfEmitter.startRun({\n    scenarioId: originalRun.scenarioId,\n    variantId: originalRun.variantId\n      ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n      : `cf:${mutation.kind}@${mutation.at}`,\n    projectId: originalRun.projectId,\n    parentRunId: originalRunId,\n    layer: 'meta',\n    tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n  })\n\n  await runner.executeFrom(\n    {\n      originalRunId,\n      originalTrajectory: trajectory,\n      prefix: trajectory.steps.slice(0, mutation.at),\n      mutation,\n      mutatedStep,\n    },\n    cfEmitter,\n  )\n\n  const counterfactual = await store.getRun(cfEmitter.runId)\n  const delta = {\n    originalOutcomeScore: originalRun.outcome?.score ?? null,\n    counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n    deltaScore:\n      originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n        ? counterfactual.outcome.score - originalRun.outcome.score\n        : null,\n  }\n  return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n  if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return { ...step, span: { ...llm, model: mutation.newModel } }\n  }\n  if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n    const tool = step.span as ToolSpan\n    return { ...step, span: { ...tool, result: mutation.newResult } }\n  }\n  if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return {\n      ...step,\n      span: {\n        ...llm,\n        messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n      },\n    }\n  }\n  if (mutation.kind === 'custom') return mutation.apply(step)\n  // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n  return step\n}\n\n/**\n * Aggregate a batch of counterfactuals into a simple attribution table:\n * which mutation kinds move outcomes most? (Useful when you run a grid\n * over the same trajectory — swap-model at every llm span, swap-tool\n * at every tool span — and want a ranked summary.)\n */\nexport function attributeCounterfactuals(results: CounterfactualResult[]): Array<{\n  mutationKind: CounterfactualMutation['kind']\n  n: number\n  meanAbsDelta: number\n  meanSignedDelta: number\n}> {\n  const grouped = new Map<string, CounterfactualResult[]>()\n  for (const r of results) {\n    const arr = grouped.get(r.mutation.kind) ?? []\n    arr.push(r)\n    grouped.set(r.mutation.kind, arr)\n  }\n  const out: Array<{\n    mutationKind: CounterfactualMutation['kind']\n    n: number\n    meanAbsDelta: number\n    meanSignedDelta: number\n  }> = []\n  for (const [kind, items] of grouped) {\n    const deltas = items\n      .map((i) => i.delta.deltaScore)\n      .filter((d): d is number => typeof d === 'number')\n    if (deltas.length === 0) continue\n    const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length\n    const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length\n    out.push({\n      mutationKind: kind as CounterfactualMutation['kind'],\n      n: deltas.length,\n      meanAbsDelta: meanAbs,\n      meanSignedDelta: meanSigned,\n    })\n  }\n  return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta)\n}\n\n// Re-export Span type for consumer ergonomics.\nexport type { Span }\n"],"mappings":";;;;;;;;;;;;;;;;;AAkEA,eAAsB,kBACpB,OACA,eACA,UACA,QAC+B;CAC/B,MAAM,cAAc,MAAM,MAAM,OAAO,aAAa;CACpD,IAAI,CAAC,aAAa,MAAM,IAAI,cAAc,uBAAuB,cAAc,WAAW;CAC1F,MAAM,aAAa,MAAM,gBAAgB,OAAO,aAAa;CAC7D,IAAI,SAAS,KAAK,KAAK,SAAS,MAAM,WAAW,MAAM,QACrD,MAAM,IAAI,gBACR,+BAA+B,SAAS,GAAG,oBAAoB,WAAW,MAAM,OAAO,EACzF;CAEF,MAAM,aAAa,WAAW,MAAM,SAAS;CAC7C,MAAM,cAAc,cAAc,YAAY,QAAQ;CAEtD,MAAM,YAAY,IAAI,aAAa,KAAK;CACxC,MAAM,UAAU,SAAS;EACvB,YAAY,YAAY;EACxB,WAAW,YAAY,YACnB,GAAG,YAAY,UAAU,MAAM,SAAS,KAAK,GAAG,SAAS,OACzD,MAAM,SAAS,KAAK,GAAG,SAAS;EACpC,WAAW,YAAY;EACvB,aAAa;EACb,OAAO;EACP,MAAM;GAAE,gBAAgB;GAAQ,cAAc,SAAS;GAAM,YAAY,OAAO,SAAS,EAAE;EAAE;CAC/F,CAAC;CAED,MAAM,OAAO,YACX;EACE;EACA,oBAAoB;EACpB,QAAQ,WAAW,MAAM,MAAM,GAAG,SAAS,EAAE;EAC7C;EACA;CACF,GACA,SACF;CAEA,MAAM,iBAAiB,MAAM,MAAM,OAAO,UAAU,KAAK;CACzD,MAAM,QAAQ;EACZ,sBAAsB,YAAY,SAAS,SAAS;EACpD,4BAA4B,gBAAgB,SAAS,SAAS;EAC9D,YACE,YAAY,SAAS,UAAU,KAAA,KAAa,gBAAgB,SAAS,UAAU,KAAA,IAC3E,eAAe,QAAQ,QAAQ,YAAY,QAAQ,QACnD;CACR;CACA,OAAO;EAAE,qBAAqB,UAAU;EAAO;EAAe;EAAU;CAAM;AAChF;AAEA,SAAS,cAAc,MAAsB,UAAkD;CAC7F,IAAI,SAAS,SAAS,gBAAgB,KAAK,KAAK,SAAS,OAAO;EAC9D,MAAM,MAAM,KAAK;EACjB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAK,OAAO,SAAS;GAAS;EAAE;CAC/D;CACA,IAAI,SAAS,SAAS,sBAAsB,KAAK,KAAK,SAAS,QAAQ;EACrE,MAAM,OAAO,KAAK;EAClB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAM,QAAQ,SAAS;GAAU;EAAE;CAClE;CACA,IAAI,SAAS,SAAS,2BAA2B,KAAK,KAAK,SAAS,OAAO;EACzE,MAAM,MAAM,KAAK;EACjB,OAAO;GACL,GAAG;GACH,MAAM;IACJ,GAAG;IACH,UAAU,CAAC;KAAE,MAAM;KAAU,SAAS,SAAS;IAAQ,GAAG,GAAG,IAAI,QAAQ;GAC3E;EACF;CACF;CACA,IAAI,SAAS,SAAS,UAAU,OAAO,SAAS,MAAM,IAAI;CAE1D,OAAO;AACT"}

package/dist/{default-registry-G9CKMNkc.d.ts → default-registry-ovxrOP0_.d.ts}

@@ -1,8 +1,8 @@
-import { i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-D9ssmxKL.js";
-import { l as ExactCapableAnalyst } from "./exact-types-qnexxJ1Z.js";
-import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-8You7OK1.js";
-import { c as RawAnalystFinding, i as TraceAnalystLimits, r as TraceAnalysisEngineResult, t as TraceAnalysisEngine } from "./engine-Cu5qD5Fc.js";
-import { t as TraceToolGroupName } from "./tool-groups-Ci8i9ErB.js";
+import { i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-DMoNFDWi.js";
+import { l as ExactCapableAnalyst } from "./exact-types-BEecmnWm.js";
+import { n as AnalystRegistry, r as AnalystRegistryOptions } from "./registry-xEb_xfns.js";
+import { c as RawAnalystFinding, i as TraceAnalystLimits, r as TraceAnalysisEngineResult, t as TraceAnalysisEngine } from "./engine-D12Rb6WB.js";
+import { t as TraceToolGroupName } from "./tool-groups-BnXlCJZQ.js";
 //#region src/trace-analyst/behavioral-metrics.d.ts
 type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
 interface SuboptimalSignal {
@@ -110,4 +110,4 @@ interface DefaultAnalystRegistryOptions {
 declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
 //#endregion
 export { createTraceAnalyst as a, BehavioralAnalystOptions as c, TraceAnalystDefinition as i, behavioralAnalyst as l, buildDefaultAnalystRegistry as n, renderPriorFindings as o, CreateTraceAnalystOptions as r, runTraceAnalyst as s, DefaultAnalystRegistryOptions as t, deriveEfficiencyFindings as u };
-//# sourceMappingURL=default-registry-G9CKMNkc.d.ts.map
+//# sourceMappingURL=default-registry-ovxrOP0_.d.ts.map

package/dist/{default-registry-G9CKMNkc.d.ts.map → default-registry-ovxrOP0_.d.ts.map}

@@ -1 +1 @@
-{"version":3,"file":"default-registry-G9CKMNkc.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;UC1BR;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;;;UCpZ1B;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}
+{"version":3,"file":"default-registry-ovxrOP0_.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;UC1BR;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;;;UCpZ1B;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}

package/dist/{define-agent-eval-h-s-sI-v.js → define-agent-eval-Clj-8igZ.js}

@@ -1,20 +1,20 @@
 import { s as ValidationError } from "./errors-Dngq5h35.js";
-import { a as spearmanR, r as pearsonR } from "./descriptive-jDOuI6mz.js";
-import { r as continuousAgreement } from "./judge-calibration-zZjLz8hr.js";
+import { a as spearmanR, r as pearsonR } from "./descriptive-1V17A-qa.js";
+import { r as continuousAgreement } from "./judge-calibration-BnpVKtnb.js";
 import { i as pairedCohensDz } from "./effect-sizes-DiH8MGOH.js";
-import { i as requiredPairedSampleSize, r as pairedMde } from "./power-and-mde-CHIrXJll.js";
-import { r as pairRunRecords } from "./paired-arms-D-XRF_fy.js";
-import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-BHIhYVdu.js";
-import { r as welchsTTest } from "./baseline-BhPRQBVn.js";
-import "./query-CHmMP42p.js";
+import { i as requiredPairedSampleSize, r as pairedMde } from "./power-and-mde-B8F2RdcD.js";
+import { r as pairRunRecords } from "./paired-arms-D4aeIHUy.js";
+import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-C8iCsioC.js";
+import { r as welchsTTest } from "./baseline-BC-eBZ7U.js";
+import "./query-BPGMVlbM.js";
 import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
-import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-BC0ebuRP.js";
-import { r as paretoChart } from "./summary-report-BI5hUtvK.js";
-import { N as surfaceContentHash, P as surfaceHash, Y as defaultProductionGate, c as emitLoopProvenance, it as resolveRunDir, l as loopProvenanceArgsFromResult, m as runImprovementLoop, tt as runEval, w as assertOptimizationResult } from "./llm-judge-BhasIPFT.js";
-import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-t4lB1yt8.js";
-import { S as inMemoryCampaignStorage, b as createRunCostLedger, x as fsCampaignStorage } from "./external-optimizer-subprocess-BIWbHpgD.js";
-import { t as powerPreflight } from "./power-preflight-DEw-uC7q.js";
-import { t as createHostedClient } from "./client-CX7KqIdB.js";
+import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-DLORoL7t.js";
+import { r as paretoChart } from "./summary-report-Bgh8CpNK.js";
+import { E as assertOptimizationResult, F as surfaceContentHash, I as surfaceHash, Z as defaultProductionGate, d as loopProvenanceArgsFromResult, g as runImprovementLoop, ot as resolveRunDir, rt as runEval, u as emitLoopProvenance } from "./llm-judge-DbJdo8Nj.js";
+import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-SkxYgT0x.js";
+import { C as inMemoryCampaignStorage, S as fsCampaignStorage, x as createRunCostLedger } from "./external-optimizer-subprocess-Cex8Da2i.js";
+import { t as powerPreflight } from "./power-preflight-CFXm0Vjo.js";
+import { t as createHostedClient } from "./client-BvwNkIRN.js";
 //#region src/contamination-guard.ts
 function checkCanaries(output, scenarios) {
 	const leaks = [];
@@ -1023,6 +1023,13 @@ function splitMethodPartitions(searchScenarios, explicitSelection, fraction) {
 function safeRunComponent(value) {
 	return value.replace(/[^a-zA-Z0-9._-]/g, "_");
 }
+/** 32-bit FNV-1a over raw UTF-16 code units, read as an unsigned int and used
+*  only to order scenarios deterministically.
+*
+*  Frozen: the order it produces assigns scenarios to partitions, so a change
+*  re-partitions decisions already recorded. It masks no byte, unlike
+*  `fnv1a32` in `src/partition-held-out.ts`, so the two disagree on any
+*  non-ASCII id and cannot be shared. */
 function stableScenarioHash(value) {
 	let hash = 2166136261;
 	for (let index = 0; index < value.length; index++) {
@@ -1364,6 +1371,11 @@ function averageComposite(campaign) {
 	const aggs = Object.values(campaign.aggregates.byScenario);
 	return aggs.length === 0 ? 0 : aggs.reduce((s, a) => s + a.meanComposite, 0) / aggs.length;
 }
+/** 32-bit FNV-1a over raw UTF-16 code units, rendered as hex for a cell key.
+*
+*  Frozen: the key names a persisted cell, so a change orphans every cell
+*  already written. Same loop as `stableScenarioHash` above but a different
+*  return form; neither is a general-purpose hash. */
 function hashString(s) {
 	let h = 2166136261;
 	for (let i = 0; i < s.length; i++) {
@@ -1497,4 +1509,4 @@ function requirePositiveInteger(value, field) {
 //#endregion
 export { summarizeExecution as a, analyzeRuns as i, SelfImproveRunError as n, checkCanaries as o, selfImprove as r, defineAgentEval as t };
 
-//# sourceMappingURL=define-agent-eval-h-s-sI-v.js.map
+//# sourceMappingURL=define-agent-eval-Clj-8igZ.js.map