@tangle-network/agent-eval 0.144.5 → 0.144.7

package/dist/control.d.ts

@@ -1,3 +1,3 @@
-import { $ as ControlRunResult, J as ControlActionOutcome, Q as ControlEvalResult, X as ControlContext, Y as ControlBudget, Z as ControlDecision, at as StopDecision, ct as runAgentControlLoop, dt as subjectiveEval, et as ControlRuntimeConfig, it as ControlStopPolicies, lt as stopOnNoProgress, nt as ControlSeverity, ot as allCriticalPassed, q as ControlActionFailureMode, rt as ControlStep, st as objectiveEval, tt as ControlRuntimeError, ut as stopOnRepeatedAction } from "./feedback-trajectory-WK7x4mhy.js";
-import { A as ActionExecutionPolicy, M as evaluateActionPolicy, _ as ProposeReviewReport, a as ProposeReviewControlAction, c as ProposeReviewControlState, g as ProposeReviewConfig, i as scoreFromEvals, j as ActionPolicyDecision, k as runProposeReview, n as RunEvidenceMetadata, o as ProposeReviewControlConfig, r as controlRunToRunRecord, s as ProposeReviewControlResult, t as ControlRunToRunRecordOptions, u as runProposeReviewAsControlLoop } from "./run-evidence-j5Ynww6L.js";
+import { $ as ControlRunResult, J as ControlActionOutcome, Q as ControlEvalResult, X as ControlContext, Y as ControlBudget, Z as ControlDecision, at as StopDecision, ct as runAgentControlLoop, dt as subjectiveEval, et as ControlRuntimeConfig, it as ControlStopPolicies, lt as stopOnNoProgress, nt as ControlSeverity, ot as allCriticalPassed, q as ControlActionFailureMode, rt as ControlStep, st as objectiveEval, tt as ControlRuntimeError, ut as stopOnRepeatedAction } from "./feedback-trajectory-Rh280oXo.js";
+import { A as ActionExecutionPolicy, M as evaluateActionPolicy, _ as ProposeReviewReport, a as ProposeReviewControlAction, c as ProposeReviewControlState, g as ProposeReviewConfig, i as scoreFromEvals, j as ActionPolicyDecision, k as runProposeReview, n as RunEvidenceMetadata, o as ProposeReviewControlConfig, r as controlRunToRunRecord, s as ProposeReviewControlResult, t as ControlRunToRunRecordOptions, u as runProposeReviewAsControlLoop } from "./run-evidence-BDFFai9R.js";
 export { type ActionExecutionPolicy, type ActionPolicyDecision, type ControlActionFailureMode, type ControlActionOutcome, type ControlBudget, type ControlContext, type ControlDecision, type ControlEvalResult, type ControlRunResult, type ControlRunToRunRecordOptions, type ControlRuntimeConfig, type ControlRuntimeError, type ControlSeverity, type ControlStep, type ControlStopPolicies, type ProposeReviewConfig, type ProposeReviewControlAction, type ProposeReviewControlConfig, type ProposeReviewControlResult, type ProposeReviewControlState, type ProposeReviewReport, type RunEvidenceMetadata, type StopDecision, allCriticalPassed, controlRunToRunRecord, evaluateActionPolicy, objectiveEval, runAgentControlLoop, runProposeReview, runProposeReviewAsControlLoop, scoreFromEvals, stopOnNoProgress, stopOnRepeatedAction, subjectiveEval };

package/dist/control.js

@@ -1,2 +1,2 @@
-import { c as scoreFromEvals, d as runAgentControlLoop, f as stopOnNoProgress, l as allCriticalPassed, m as subjectiveEval, n as runProposeReviewAsControlLoop, o as runProposeReview, p as stopOnRepeatedAction, s as controlRunToRunRecord, u as objectiveEval, y as evaluateActionPolicy } from "./propose-review-control-BciUCZoh.js";
+import { c as scoreFromEvals, d as runAgentControlLoop, f as stopOnNoProgress, l as allCriticalPassed, m as subjectiveEval, n as runProposeReviewAsControlLoop, o as runProposeReview, p as stopOnRepeatedAction, s as controlRunToRunRecord, u as objectiveEval, y as evaluateActionPolicy } from "./propose-review-control-qgWLA6E9.js";
 export { allCriticalPassed, controlRunToRunRecord, evaluateActionPolicy, objectiveEval, runAgentControlLoop, runProposeReview, runProposeReviewAsControlLoop, scoreFromEvals, stopOnNoProgress, stopOnRepeatedAction, subjectiveEval };

package/dist/{cost-ledger-FuQvHxPm.d.ts → cost-ledger-Bv_e8XHY.d.ts}

@@ -1,4 +1,4 @@
-import { l as ValidationError } from "./errors-DkfjIDvD.js";
+import { l as ValidationError } from "./errors-CKPfb2aH.js";
 //#region src/cost-ledger.d.ts
 type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
 /** How a USD amount was obtained. */
@@ -259,4 +259,4 @@ declare function costForUsage(model: string, usage: CostUsage): CostResult;
 declare function costForTokenPricing(pricing: CustomTokenPricing, usage: Pick<CostUsage, 'inputTokens' | 'outputTokens' | 'cachedTokens' | 'cacheWriteTokens'>): number;
 //#endregion
 export { PendingCostCall as C, costForUsage as D, costForTokenPricing as E, modelPriceKey as O, PaidCallResult as S, RunPaidCallInput as T, CostReservationExceededError as _, CostChannel as a, CustomTokenPricing as b, CostLedgerHandle as c, CostLedgerPersistenceError as d, CostLedgerSummary as f, CostReceiptInput as g, CostReceiptCaptureError as h, CostCeilingReachedError as i, CostLedgerOptions as l, CostReceipt as m, CostAccountingIncompleteError as n, CostLedger as o, CostProvenance as p, CostCallConflictError as r, CostLedgerFilter as s, ChannelRollup as t, CostLedgerPersistence as u, CostResult as v, PendingCostCallView as w, MaximumCharge as x, CostUsage as y };
-//# sourceMappingURL=cost-ledger-FuQvHxPm.d.ts.map
+//# sourceMappingURL=cost-ledger-Bv_e8XHY.d.ts.map

package/dist/{cost-ledger-FuQvHxPm.d.ts.map → cost-ledger-Bv_e8XHY.d.ts.map}

@@ -1 +1 @@
-{"version":3,"file":"cost-ledger-FuQvHxPm.d.ts","names":[],"sources":["../src/cost-ledger.ts"],"mappings":";;KAIY;;KAGA;EACN;EAAkB;;EAClB;EAAmB;;EACnB;EAAoB;;UAET;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;UAGQ;EACR;EACA,SAAS;EACT;EACA;EACA;EACA;EACA,OAAO;EACP;;UAGe,wBAAwB;EACvC;;UAGe,4BAA4B;EAC3C;;UAGe,oBAAoB,cAAc;EACjD;EACA;EACA;EACA;;EAEA;IACE;IACA;IACA;IACA;;;EAGF;;EAEA;;;EAGA;;UAKe,yBAAyB;EACxC;;EAEA,qBAAqB;EACrB;;EAEA;EACA;EACA;;;UAIe;;EAEf;;EAEA;;EAEA;EACA;;KAGU;EACN;;EAEA,oBAAoB;IAClB,KAAK;EACN;IAAkB;UAER,iBAAiB;EAChC;EACA,SAAS;EACT;EACA;;EAEA;EACA,OAAO;EACP,SAAS;;EAET,gBAAgB;;EAEhB,QAAQ,QAAQ,aAAa,iBAAiB,QAAQ;EACtD,QAAQ,OAAO,IAAI;EACnB,kBAAkB,OAAO,QAAQ;;KAGvB,eAAe;EACrB;EAAiB;EAAgB,OAAO;EAAG,SAAS;;EACpD;EAAkB;EAAiB,OAAO;EAAO,UAAU;;UAEhD;EACf,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,gBAAgB;EAChB,WAAW;EACX;EACA;EACA;EACA;EACA;;UAGe;EACf,UAAU;EACV;EACA,OAAO;;UAGQ;;EAEf;;EAEA,SAAS;;;UAIM;EACf;IAAU;IAAkB;;EAC5B,OAAO,0BAA0B;;UAGlB;EACf;EACA,cAAc;;EAEd,oBAAoB;;cAGT,gCAAgC;EAC3C,YACE,oBACA,sBACA,sBACA,eACA;;cAQS,sCAAsC;cAEtC,qCAAqC;EAChD,YAAY,eAAe,mBAAmB;;cAOnC,8BAA8B;WAChC;WACA,UAAU;EAEnB,YACE,iBACA;IAAW;IAAiB,UAAU;IAAa;;;cAQ1C,mCAAmC;WACrC;WACA,UAAU;EAEnB,YAAY,gBAAgB,iBAAiB,UAAU;;cAU5C,gCAAgC;WAClC;WACA,UAAU;WACV,cAAc;EAEvB,YAAY,gBAAgB,gBAAgB,uBAAuB,UAAU;;;cAgClE;mBACM;mBACA;mBACA;mBACA;UACT;UACA;UACA;WACC;mBACQ;EAEjB,YAAY,iBAAiB;EAwDvB,YAAY,GAAG,OAAO,iBAAiB,KAAK,QAAQ,eAAe;;EA2GnE,YAAY,UAAS,wBAA6B;;EA0BxD,UACE,gBACA,UAAU,kBACV;IAAW;IAAkB;MAC5B;EAkBH,KAAK,SAAS,mBAAmB;;EAQjC,YAAY,SAAS,mBAAmB;EAcxC,QAAQ,SAAS,mBAAmB;EAkFpC,cAAc;EAYd;UAIc;UAoDN;UAsCA;UAKM;UAoBN;UAmBA;UAkBA;UAcA;UA+BA;;;;;;UAcA;UASA;UAYA;UAMA;;;;;;;;KA0BE,mBAAmB,KAC7B,YACA,cAAc,8CAEd,QAAQ,KAAK;;iBAGC,cAAc;UAIb;EACf;EACA;;iBAGc,aAAa,eAAe,OAAO,YAAY;;iBAc/C,oBACd,SAAS,oBACT,OAAO,KAAK"}
+{"version":3,"file":"cost-ledger-Bv_e8XHY.d.ts","names":[],"sources":["../src/cost-ledger.ts"],"mappings":";;KAIY;;KAGA;EACN;EAAkB;;EAClB;EAAmB;;EACnB;EAAoB;;UAET;EACf;;EAEA;;EAEA;;EAEA;;EAEA;;UAGQ;EACR;EACA,SAAS;EACT;EACA;EACA;EACA;EACA,OAAO;EACP;;UAGe,wBAAwB;EACvC;;UAGe,4BAA4B;EAC3C;;UAGe,oBAAoB,cAAc;EACjD;EACA;EACA;EACA;;EAEA;IACE;IACA;IACA;IACA;;;EAGF;;EAEA;;;EAGA;;UAKe,yBAAyB;EACxC;;EAEA,qBAAqB;EACrB;;EAEA;EACA;EACA;;;UAIe;;EAEf;;EAEA;;EAEA;EACA;;KAGU;EACN;;EAEA,oBAAoB;IAClB,KAAK;EACN;IAAkB;UAER,iBAAiB;EAChC;EACA,SAAS;EACT;EACA;;EAEA;EACA,OAAO;EACP,SAAS;;EAET,gBAAgB;;EAEhB,QAAQ,QAAQ,aAAa,iBAAiB,QAAQ;EACtD,QAAQ,OAAO,IAAI;EACnB,kBAAkB,OAAO,QAAQ;;KAGvB,eAAe;EACrB;EAAiB;EAAgB,OAAO;EAAG,SAAS;;EACpD;EAAkB;EAAiB,OAAO;EAAO,UAAU;;UAEhD;EACf,SAAS;EACT;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA;EACA,gBAAgB;EAChB,WAAW;EACX;EACA;EACA;EACA;EACA;;UAGe;EACf,UAAU;EACV;EACA,OAAO;;UAGQ;;EAEf;;EAEA,SAAS;;;UAIM;EACf;IAAU;IAAkB;;EAC5B,OAAO,0BAA0B;;UAGlB;EACf;EACA,cAAc;;EAEd,oBAAoB;;cAGT,gCAAgC;EAC3C,YACE,oBACA,sBACA,sBACA,eACA;;cAQS,sCAAsC;cAEtC,qCAAqC;EAChD,YAAY,eAAe,mBAAmB;;cAOnC,8BAA8B;WAChC;WACA,UAAU;EAEnB,YACE,iBACA;IAAW;IAAiB,UAAU;IAAa;;;cAQ1C,mCAAmC;WACrC;WACA,UAAU;EAEnB,YAAY,gBAAgB,iBAAiB,UAAU;;cAU5C,gCAAgC;WAClC;WACA,UAAU;WACV,cAAc;EAEvB,YAAY,gBAAgB,gBAAgB,uBAAuB,UAAU;;;cAgClE;mBACM;mBACA;mBACA;mBACA;UACT;UACA;UACA;WACC;mBACQ;EAEjB,YAAY,iBAAiB;EAwDvB,YAAY,GAAG,OAAO,iBAAiB,KAAK,QAAQ,eAAe;;EA2GnE,YAAY,UAAS,wBAA6B;;EA0BxD,UACE,gBACA,UAAU,kBACV;IAAW;IAAkB;MAC5B;EAkBH,KAAK,SAAS,mBAAmB;;EAQjC,YAAY,SAAS,mBAAmB;EAcxC,QAAQ,SAAS,mBAAmB;EAkFpC,cAAc;EAYd;UAIc;UAoDN;UAsCA;UAKM;UAoBN;UAmBA;UAkBA;UAcA;UA+BA;;;;;;UAcA;UASA;UAYA;UAMA;;;;;;;;KA0BE,mBAAmB,KAC7B,YACA,cAAc,8CAEd,QAAQ,KAAK;;iBAGC,cAAc;UAIb;EACf;EACA;;iBAGc,aAAa,eAAe,OAAO,YAAY;;iBAc/C,oBACd,SAAS,oBACT,OAAO,KAAK"}

package/dist/counterfactual-CWPTrMH7.js

@@ -0,0 +1,126 @@
+import { c as ValidationError, o as NotFoundError } from "./errors-D-LKuDhb.js";
+import { t as TraceEmitter } from "./emitter-CPBAhxum.js";
+import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
+//#region src/counterfactual.ts
+/**
+* Counterfactual replay — "what would have happened if we'd changed
+* exactly one thing at turn N?"
+*
+* The framework does NOT drive the agent — it sets up the replay
+* context (prior spans, prior state, mutation spec) and records the
+* resulting divergence. Consumers supply an `executeFrom(ctx)` callback
+* that runs their agent starting from turn N with the mutation applied.
+*
+* Counterfactual runs are recorded as a new Run with `layer='meta'` and
+* `parentRunId = originalRunId`, so downstream diff + correlation
+* pipelines see them natively.
+*/
+async function runCounterfactual(store, originalRunId, mutation, runner) {
+	const originalRun = await store.getRun(originalRunId);
+	if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`);
+	const trajectory = await buildTrajectory(store, originalRunId);
+	if (mutation.at < 0 || mutation.at >= trajectory.steps.length) throw new ValidationError(`counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`);
+	const targetStep = trajectory.steps[mutation.at];
+	const mutatedStep = applyMutation(targetStep, mutation);
+	const cfEmitter = new TraceEmitter(store);
+	await cfEmitter.startRun({
+		scenarioId: originalRun.scenarioId,
+		variantId: originalRun.variantId ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}` : `cf:${mutation.kind}@${mutation.at}`,
+		projectId: originalRun.projectId,
+		parentRunId: originalRunId,
+		layer: "meta",
+		tags: {
+			counterfactual: "true",
+			mutationKind: mutation.kind,
+			mutationAt: String(mutation.at)
+		}
+	});
+	await runner.executeFrom({
+		originalRunId,
+		originalTrajectory: trajectory,
+		prefix: trajectory.steps.slice(0, mutation.at),
+		mutation,
+		mutatedStep
+	}, cfEmitter);
+	const counterfactual = await store.getRun(cfEmitter.runId);
+	const delta = {
+		originalOutcomeScore: originalRun.outcome?.score ?? null,
+		counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,
+		deltaScore: originalRun.outcome?.score !== void 0 && counterfactual?.outcome?.score !== void 0 ? counterfactual.outcome.score - originalRun.outcome.score : null
+	};
+	return {
+		counterfactualRunId: cfEmitter.runId,
+		originalRunId,
+		mutation,
+		delta
+	};
+}
+function applyMutation(step, mutation) {
+	if (mutation.kind === "swap-model" && step.span.kind === "llm") {
+		const llm = step.span;
+		return {
+			...step,
+			span: {
+				...llm,
+				model: mutation.newModel
+			}
+		};
+	}
+	if (mutation.kind === "swap-tool-result" && step.span.kind === "tool") {
+		const tool = step.span;
+		return {
+			...step,
+			span: {
+				...tool,
+				result: mutation.newResult
+			}
+		};
+	}
+	if (mutation.kind === "inject-system-message" && step.span.kind === "llm") {
+		const llm = step.span;
+		return {
+			...step,
+			span: {
+				...llm,
+				messages: [{
+					role: "system",
+					content: mutation.content
+				}, ...llm.messages]
+			}
+		};
+	}
+	if (mutation.kind === "custom") return mutation.apply(step);
+	return step;
+}
+/**
+* Aggregate a batch of counterfactuals into a simple attribution table:
+* which mutation kinds move outcomes most? (Useful when you run a grid
+* over the same trajectory — swap-model at every llm span, swap-tool
+* at every tool span — and want a ranked summary.)
+*/
+function attributeCounterfactuals(results) {
+	const grouped = /* @__PURE__ */ new Map();
+	for (const r of results) {
+		const arr = grouped.get(r.mutation.kind) ?? [];
+		arr.push(r);
+		grouped.set(r.mutation.kind, arr);
+	}
+	const out = [];
+	for (const [kind, items] of grouped) {
+		const deltas = items.map((i) => i.delta.deltaScore).filter((d) => typeof d === "number");
+		if (deltas.length === 0) continue;
+		const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length;
+		const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length;
+		out.push({
+			mutationKind: kind,
+			n: deltas.length,
+			meanAbsDelta: meanAbs,
+			meanSignedDelta: meanSigned
+		});
+	}
+	return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta);
+}
+//#endregion
+export { runCounterfactual as n, attributeCounterfactuals as t };
+
+//# sourceMappingURL=counterfactual-CWPTrMH7.js.map

package/dist/counterfactual-CWPTrMH7.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"counterfactual-CWPTrMH7.js","names":[],"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n  | { kind: 'swap-model'; at: number; newModel: string }\n  | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n  | { kind: 'truncate-after'; at: number }\n  | { kind: 'inject-system-message'; at: number; content: string }\n  | {\n      kind: 'custom'\n      at: number\n      describe: string\n      apply: (step: TrajectoryStep) => TrajectoryStep\n    }\n\nexport interface CounterfactualContext {\n  originalRunId: string\n  originalTrajectory: Trajectory\n  /** Steps up to (but not including) the mutation point — the prefix the\n   *  replayed agent inherits as its prior conversation/tool history. */\n  prefix: TrajectoryStep[]\n  mutation: CounterfactualMutation\n  /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n   *  as the FIRST step the replayed agent emits (they decide whether to\n   *  re-emit it or continue from there). */\n  mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n  counterfactualRunId: string\n  originalRunId: string\n  mutation: CounterfactualMutation\n  /** Structured delta summary — caller can extend via scoring. */\n  delta: {\n    originalOutcomeScore: number | null\n    counterfactualOutcomeScore: number | null\n    deltaScore: number | null\n  }\n}\n\nexport interface CounterfactualRunner {\n  /**\n   * Execute the agent from `ctx.prefix` with the mutation applied.\n   * MUST emit spans into the provided emitter so they become part of\n   * the counterfactual run. MUST call emitter.endRun() with a verdict.\n   */\n  executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n  store: TraceStore,\n  originalRunId: string,\n  mutation: CounterfactualMutation,\n  runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n  const originalRun = await store.getRun(originalRunId)\n  if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n  const trajectory = await buildTrajectory(store, originalRunId)\n  if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n    throw new ValidationError(\n      `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n    )\n  }\n  const targetStep = trajectory.steps[mutation.at]!\n  const mutatedStep = applyMutation(targetStep, mutation)\n\n  const cfEmitter = new TraceEmitter(store)\n  await cfEmitter.startRun({\n    scenarioId: originalRun.scenarioId,\n    variantId: originalRun.variantId\n      ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n      : `cf:${mutation.kind}@${mutation.at}`,\n    projectId: originalRun.projectId,\n    parentRunId: originalRunId,\n    layer: 'meta',\n    tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n  })\n\n  await runner.executeFrom(\n    {\n      originalRunId,\n      originalTrajectory: trajectory,\n      prefix: trajectory.steps.slice(0, mutation.at),\n      mutation,\n      mutatedStep,\n    },\n    cfEmitter,\n  )\n\n  const counterfactual = await store.getRun(cfEmitter.runId)\n  const delta = {\n    originalOutcomeScore: originalRun.outcome?.score ?? null,\n    counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n    deltaScore:\n      originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n        ? counterfactual.outcome.score - originalRun.outcome.score\n        : null,\n  }\n  return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n  if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return { ...step, span: { ...llm, model: mutation.newModel } }\n  }\n  if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n    const tool = step.span as ToolSpan\n    return { ...step, span: { ...tool, result: mutation.newResult } }\n  }\n  if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return {\n      ...step,\n      span: {\n        ...llm,\n        messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n      },\n    }\n  }\n  if (mutation.kind === 'custom') return mutation.apply(step)\n  // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n  return step\n}\n\n/**\n * Aggregate a batch of counterfactuals into a simple attribution table:\n * which mutation kinds move outcomes most? (Useful when you run a grid\n * over the same trajectory — swap-model at every llm span, swap-tool\n * at every tool span — and want a ranked summary.)\n */\nexport function attributeCounterfactuals(results: CounterfactualResult[]): Array<{\n  mutationKind: CounterfactualMutation['kind']\n  n: number\n  meanAbsDelta: number\n  meanSignedDelta: number\n}> {\n  const grouped = new Map<string, CounterfactualResult[]>()\n  for (const r of results) {\n    const arr = grouped.get(r.mutation.kind) ?? []\n    arr.push(r)\n    grouped.set(r.mutation.kind, arr)\n  }\n  const out: Array<{\n    mutationKind: CounterfactualMutation['kind']\n    n: number\n    meanAbsDelta: number\n    meanSignedDelta: number\n  }> = []\n  for (const [kind, items] of grouped) {\n    const deltas = items\n      .map((i) => i.delta.deltaScore)\n      .filter((d): d is number => typeof d === 'number')\n    if (deltas.length === 0) continue\n    const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length\n    const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length\n    out.push({\n      mutationKind: kind as CounterfactualMutation['kind'],\n      n: deltas.length,\n      meanAbsDelta: meanAbs,\n      meanSignedDelta: meanSigned,\n    })\n  }\n  return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta)\n}\n\n// Re-export Span type for consumer ergonomics.\nexport type { Span }\n"],"mappings":";;;;;;;;;;;;;;;;;AAkEA,eAAsB,kBACpB,OACA,eACA,UACA,QAC+B;CAC/B,MAAM,cAAc,MAAM,MAAM,OAAO,aAAa;CACpD,IAAI,CAAC,aAAa,MAAM,IAAI,cAAc,uBAAuB,cAAc,WAAW;CAC1F,MAAM,aAAa,MAAM,gBAAgB,OAAO,aAAa;CAC7D,IAAI,SAAS,KAAK,KAAK,SAAS,MAAM,WAAW,MAAM,QACrD,MAAM,IAAI,gBACR,+BAA+B,SAAS,GAAG,oBAAoB,WAAW,MAAM,OAAO,EACzF;CAEF,MAAM,aAAa,WAAW,MAAM,SAAS;CAC7C,MAAM,cAAc,cAAc,YAAY,QAAQ;CAEtD,MAAM,YAAY,IAAI,aAAa,KAAK;CACxC,MAAM,UAAU,SAAS;EACvB,YAAY,YAAY;EACxB,WAAW,YAAY,YACnB,GAAG,YAAY,UAAU,MAAM,SAAS,KAAK,GAAG,SAAS,OACzD,MAAM,SAAS,KAAK,GAAG,SAAS;EACpC,WAAW,YAAY;EACvB,aAAa;EACb,OAAO;EACP,MAAM;GAAE,gBAAgB;GAAQ,cAAc,SAAS;GAAM,YAAY,OAAO,SAAS,EAAE;EAAE;CAC/F,CAAC;CAED,MAAM,OAAO,YACX;EACE;EACA,oBAAoB;EACpB,QAAQ,WAAW,MAAM,MAAM,GAAG,SAAS,EAAE;EAC7C;EACA;CACF,GACA,SACF;CAEA,MAAM,iBAAiB,MAAM,MAAM,OAAO,UAAU,KAAK;CACzD,MAAM,QAAQ;EACZ,sBAAsB,YAAY,SAAS,SAAS;EACpD,4BAA4B,gBAAgB,SAAS,SAAS;EAC9D,YACE,YAAY,SAAS,UAAU,KAAA,KAAa,gBAAgB,SAAS,UAAU,KAAA,IAC3E,eAAe,QAAQ,QAAQ,YAAY,QAAQ,QACnD;CACR;CACA,OAAO;EAAE,qBAAqB,UAAU;EAAO;EAAe;EAAU;CAAM;AAChF;AAEA,SAAS,cAAc,MAAsB,UAAkD;CAC7F,IAAI,SAAS,SAAS,gBAAgB,KAAK,KAAK,SAAS,OAAO;EAC9D,MAAM,MAAM,KAAK;EACjB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAK,OAAO,SAAS;GAAS;EAAE;CAC/D;CACA,IAAI,SAAS,SAAS,sBAAsB,KAAK,KAAK,SAAS,QAAQ;EACrE,MAAM,OAAO,KAAK;EAClB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAM,QAAQ,SAAS;GAAU;EAAE;CAClE;CACA,IAAI,SAAS,SAAS,2BAA2B,KAAK,KAAK,SAAS,OAAO;EACzE,MAAM,MAAM,KAAK;EACjB,OAAO;GACL,GAAG;GACH,MAAM;IACJ,GAAG;IACH,UAAU,CAAC;KAAE,MAAM;KAAU,SAAS,SAAS;IAAQ,GAAG,GAAG,IAAI,QAAQ;GAC3E;EACF;CACF;CACA,IAAI,SAAS,SAAS,UAAU,OAAO,SAAS,MAAM,IAAI;CAE1D,OAAO;AACT;;;;;;;AAQA,SAAgB,yBAAyB,SAKtC;CACD,MAAM,0BAAU,IAAI,IAAoC;CACxD,KAAK,MAAM,KAAK,SAAS;EACvB,MAAM,MAAM,QAAQ,IAAI,EAAE,SAAS,IAAI,KAAK,CAAC;EAC7C,IAAI,KAAK,CAAC;EACV,QAAQ,IAAI,EAAE,SAAS,MAAM,GAAG;CAClC;CACA,MAAM,MAKD,CAAC;CACN,KAAK,MAAM,CAAC,MAAM,UAAU,SAAS;EACnC,MAAM,SAAS,MACZ,KAAK,MAAM,EAAE,MAAM,UAAU,CAAC,CAC9B,QAAQ,MAAmB,OAAO,MAAM,QAAQ;EACnD,IAAI,OAAO,WAAW,GAAG;EACzB,MAAM,UAAU,OAAO,QAAQ,GAAG,MAAM,IAAI,KAAK,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO;EACrE,MAAM,aAAa,OAAO,QAAQ,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;EAC9D,IAAI,KAAK;GACP,cAAc;GACd,GAAG,OAAO;GACV,cAAc;GACd,iBAAiB;EACnB,CAAC;CACH;CACA,OAAO,IAAI,MAAM,GAAG,MAAM,EAAE,eAAe,EAAE,YAAY;AAC3D"}

package/dist/counterfactual-CxmxAONP.d.ts

@@ -0,0 +1,72 @@
+import { s as TraceStore } from "./store-CT9YIIve.js";
+import { i as TraceEmitter } from "./emitter-DGQGoLyj.js";
+import { n as TrajectoryStep, t as Trajectory } from "./trajectory-YC15QDYQ.js";
+//#region src/counterfactual.d.ts
+type CounterfactualMutation = {
+  kind: 'swap-model';
+  at: number;
+  newModel: string;
+} | {
+  kind: 'swap-tool-result';
+  at: number;
+  newResult: unknown;
+} | {
+  kind: 'truncate-after';
+  at: number;
+} | {
+  kind: 'inject-system-message';
+  at: number;
+  content: string;
+} | {
+  kind: 'custom';
+  at: number;
+  describe: string;
+  apply: (step: TrajectoryStep) => TrajectoryStep;
+};
+interface CounterfactualContext {
+  originalRunId: string;
+  originalTrajectory: Trajectory;
+  /** Steps up to (but not including) the mutation point — the prefix the
+   *  replayed agent inherits as its prior conversation/tool history. */
+  prefix: TrajectoryStep[];
+  mutation: CounterfactualMutation;
+  /** Pre-applied mutation on the step at `mutation.at`. Consumers use this
+   *  as the FIRST step the replayed agent emits (they decide whether to
+   *  re-emit it or continue from there). */
+  mutatedStep: TrajectoryStep;
+}
+interface CounterfactualResult {
+  counterfactualRunId: string;
+  originalRunId: string;
+  mutation: CounterfactualMutation;
+  /** Structured delta summary — caller can extend via scoring. */
+  delta: {
+    originalOutcomeScore: number | null;
+    counterfactualOutcomeScore: number | null;
+    deltaScore: number | null;
+  };
+}
+interface CounterfactualRunner {
+  /**
+   * Execute the agent from `ctx.prefix` with the mutation applied.
+   * MUST emit spans into the provided emitter so they become part of
+   * the counterfactual run. MUST call emitter.endRun() with a verdict.
+   */
+  executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>;
+}
+declare function runCounterfactual(store: TraceStore, originalRunId: string, mutation: CounterfactualMutation, runner: CounterfactualRunner): Promise<CounterfactualResult>;
+/**
+ * Aggregate a batch of counterfactuals into a simple attribution table:
+ * which mutation kinds move outcomes most? (Useful when you run a grid
+ * over the same trajectory — swap-model at every llm span, swap-tool
+ * at every tool span — and want a ranked summary.)
+ */
+declare function attributeCounterfactuals(results: CounterfactualResult[]): Array<{
+  mutationKind: CounterfactualMutation['kind'];
+  n: number;
+  meanAbsDelta: number;
+  meanSignedDelta: number;
+}>;
+//#endregion
+export { attributeCounterfactuals as a, CounterfactualRunner as i, CounterfactualMutation as n, runCounterfactual as o, CounterfactualResult as r, CounterfactualContext as t };
+//# sourceMappingURL=counterfactual-CxmxAONP.d.ts.map

package/dist/counterfactual-CxmxAONP.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"counterfactual-CxmxAONP.d.ts","names":[],"sources":["../src/counterfactual.ts"],"mappings":";;;;KAoBY;EACN;EAAoB;EAAY;;EAChC;EAA0B;EAAY;;EACtC;EAAwB;;EACxB;EAA+B;EAAY;;EAE3C;EACA;EACA;EACA,QAAQ,MAAM,mBAAmB;;UAGtB;EACf;EACA,oBAAoB;;;EAGpB,QAAQ;EACR,UAAU;;;;EAIV,aAAa;;UAGE;EACf;EACA;EACA,UAAU;;EAEV;IACE;IACA;IACA;;;UAIa;;;;;;EAMf,cAAc,KAAK,uBAAuB,SAAS,iBAAiB;;iBAGhD,kBACpB,OAAO,YACP,uBACA,UAAU,wBACV,QAAQ,uBACP,QAAQ;;;;;;;iBA6EK,yBAAyB,SAAS,yBAAyB;EACzE,cAAc;EACd;EACA;EACA"}

package/dist/{dataset-v_Y5902-.d.ts → dataset-C8xaLXdY.d.ts}

@@ -1,4 +1,4 @@
-import { l as ValidationError } from "./errors-DkfjIDvD.js";
+import { l as ValidationError } from "./errors-CKPfb2aH.js";
 //#region src/dataset.d.ts
 /**
  * Dataset — versioned, sliceable, content-hashed scenario collection.
@@ -112,4 +112,4 @@ declare class Dataset {
 declare function hashScenarios(scenarios: DatasetScenario[]): Promise<string>;
 //#endregion
 export { DatasetScenario as a, SliceOptions as c, DatasetProvenance as i, hashScenarios as l, DatasetDifficulty as n, DatasetSplit as o, DatasetManifest as r, HoldoutLockedError as s, Dataset as t };
-//# sourceMappingURL=dataset-v_Y5902-.d.ts.map
+//# sourceMappingURL=dataset-C8xaLXdY.d.ts.map

package/dist/{dataset-v_Y5902-.d.ts.map → dataset-C8xaLXdY.d.ts.map}

@@ -1 +1 @@
-{"version":3,"file":"dataset-v_Y5902-.d.ts","names":[],"sources":["../src/dataset.ts"],"mappings":";;;;;;;;;;;;;;;;;;KAiBY;KACA;UAEK;EACf;;EAEA;EACA,QAAQ;EACR,aAAa;;EAEb;;;;;;;;;;EAUA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA,YAAY;;EAEZ;EACA;EACA,aAAa,OAAO;;UAGL;EACf,QAAQ;EACR,aAAa;;EAEb;EACA;;EAEA,UAAU,UAAU;;EAEpB;;;cAMW,2BAA2B;EACtC,YAAY;;cAOD;WACF;WACA,YAAY;UACb;UACA;EAER,YAAY;IACV;IACA,YAAY;IACZ,WAAW;IACX;;;EASF,gBAAgB;MAIZ;;;;;EAQJ,MAAM,UAAS,eAAoB;;;;;EAqB7B,YAAY,QAAQ;;EAgB1B,MAAM,YAAW;IAAU;IAAc;OAA0B;EAWnE;EAIA,IAAI,UAAU;EAQd,OAAO;;;;;EAWP;SAQO,UACL,eACA,UAAU,KAAK,oEACd;;iBAaiB,cAAc,WAAW,oBAAoB"}
+{"version":3,"file":"dataset-C8xaLXdY.d.ts","names":[],"sources":["../src/dataset.ts"],"mappings":";;;;;;;;;;;;;;;;;;KAiBY;KACA;UAEK;EACf;;EAEA;EACA,QAAQ;EACR,aAAa;;EAEb;;;;;;;;;;EAUA;EACA,OAAO;;UAGQ;EACf;EACA;EACA;EACA;EACA;;EAEA;;UAGe;EACf;EACA,YAAY;;EAEZ;EACA;EACA,aAAa,OAAO;;UAGL;EACf,QAAQ;EACR,aAAa;;EAEb;EACA;;EAEA,UAAU,UAAU;;EAEpB;;;cAMW,2BAA2B;EACtC,YAAY;;cAOD;WACF;WACA,YAAY;UACb;UACA;EAER,YAAY;IACV;IACA,YAAY;IACZ,WAAW;IACX;;;EASF,gBAAgB;MAIZ;;;;;EAQJ,MAAM,UAAS,eAAoB;;;;;EAqB7B,YAAY,QAAQ;;EAgB1B,MAAM,YAAW;IAAU;IAAc;OAA0B;EAWnE;EAIA,IAAI,UAAU;EAQd,OAAO;;;;;EAWP;SAQO,UACL,eACA,UAAU,KAAK,oEACd;;iBAaiB,cAAc,WAAW,oBAAoB"}

package/dist/{default-registry-BwbZ9N9v.d.ts → default-registry-BZhStdjl.d.ts}

@@ -1,7 +1,8 @@
-import { N as TraceAnalystSpan, i as AnalystFinding, n as AnalystContext, x as TraceAnalysisStore } from "./types-CZt1PBIk.js";
-import { l as ExactCapableAnalyst } from "./exact-types-BQ7W90C4.js";
-import { D as AnalystRegistryOptions, E as AnalystRegistry } from "./completion-verifier-VvpHRu78.js";
-import { i as TraceAnalysisEngine, o as TraceAnalysisEngineResult, p as RawAnalystFinding, s as TraceAnalystLimits, t as TraceToolGroupName } from "./tool-groups-ByZiqpVk.js";
+import { I as TraceAnalystSpan, i as AnalystFinding, n as AnalystContext, w as TraceAnalysisStore } from "./types-DF_Udrp-.js";
+import { l as ExactCapableAnalyst } from "./exact-types-Djvzosly.js";
+import { D as AnalystRegistryOptions, E as AnalystRegistry } from "./completion-verifier-foUCLif_.js";
+import { a as TraceAnalystLimits, d as RawAnalystFinding, i as TraceAnalysisEngineResult, n as TraceAnalysisEngine } from "./engine-nB64f48I.js";
+import { t as TraceToolGroupName } from "./tool-groups-DIVBnJyl.js";
 //#region src/trace-analyst/behavioral-metrics.d.ts
 type SuboptimalCode = 'monotonic-input-growth' | 'output-length-decay' | 'single-tool-dependency' | 'no-self-verification';
 interface SuboptimalSignal {
@@ -115,4 +116,4 @@ interface DefaultAnalystRegistryOptions {
 declare function buildDefaultAnalystRegistry(options?: DefaultAnalystRegistryOptions): AnalystRegistry;
 //#endregion
 export { createTraceAnalyst as a, runTraceAnalyst as c, deriveEfficiencyFindings as d, BehavioralMetrics as f, computeTraceMetrics as g, SuboptimalSignal as h, TraceAnalystDefinition as i, BehavioralAnalystOptions as l, SuboptimalCode as m, buildDefaultAnalystRegistry as n, renderPriorFindings as o, BehavioralTokenSequence as p, CreateTraceAnalystOptions as r, renderUpstreamFindings as s, DefaultAnalystRegistryOptions as t, behavioralAnalyst as u };
-//# sourceMappingURL=default-registry-BwbZ9N9v.d.ts.map
+//# sourceMappingURL=default-registry-BZhStdjl.d.ts.map

package/dist/default-registry-BZhStdjl.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"default-registry-BZhStdjl.d.ts","names":[],"sources":["../src/trace-analyst/behavioral-metrics.ts","../src/analyst/behavioral-analyst.ts","../src/analyst/kind-factory.ts","../src/analyst/default-registry.ts"],"mappings":";;;;;;KAoBY;UAMK;EACf,MAAM;EACN;;EAEA;;EAEA,UAAU;;UAGK;;EAEf;EACA;;EAEA,gBAAgB;;EAEhB;EACA;EACA,eAAe;EACf;EACA;;EAEA;EACA;EACA,SAAS;;UAGM;EACf;EACA;EACA,sBAAsB;EACtB,uBAAuB;;;;;;iBAoFT,oBAAoB,gBAAgB,qBAAqB;;;UC9GxD;;EAEf;;EAEA;;;;;;iBA+Dc,yBACd,SAAS,mBACT;EAAQ;EAAoB;IAC3B;;iBAkCa,kBACd,UAAS,2BACR,oBAAoB;;;;UCrHN;EACf;EACA;EACA;EACA;;EAEA,sBAAsB,SAAS;;EAE/B;;EAEA,WAAW;;EAEX,kBACE,OAAO,oBACP,SAAS,wCACe;EAC1B,SAAS,QAAQ;EACjB,eAAe,KAAK,mBAAmB,SAAS,mBAAmB;EACnE;EACA;;UAGe;EACf,QAAQ;;EAER;EACA;;;iBAIoB,gBAAgB;EACpC,YAAY;EACZ,QAAQ;EACR,OAAO;EACP,SAAS;EACT;IACE,QAAQ;;iBAgFI,mBACd,YAAY,wBACZ,SAAS,4BACR,oBAAoB;iBA+QP,oBAAoB,OAAO;iBAe3B,uBAAuB,UAAU;;;UCnahC;;;;;;EAMf,SAAS;EACT,uBAAuB;;EAEvB;EACA,aAAa;EACb,WAAW;;iBAGG,4BACd,UAAS,gCACR"}

package/dist/{default-registry-RLNNoeEP.js → default-registry-BaQXW1Ow.js}

@@ -1,4 +1,4 @@
-import { n as LlmClient } from "./llm-client-D3EoChAU.js";
+import { n as LlmClient } from "./llm-client-DzvMUsS_.js";
 import { LLM_CONTEXT_TOKENS, LLM_INPUT_TOKEN_ATTR_KEYS, LLM_OUTPUT_TOKEN_ATTR_KEYS, TOOL_NAME_ATTR_KEYS } from "./trace-attributes.js";
 import { t as executionTrackByLane } from "./execution-tracks-CpgFPpS5.js";
 import { $ as spanEpochMillis, H as snapshotExactExecutionPlan, R as findingSubjectGrammarPromptFor, U as deepFreezeCanonicalJson, V as snapshotExactExecutionComponentIdentity, t as createTraceAnalyst } from "./kind-factory-BHIgPmzS.js";
@@ -2449,4 +2449,4 @@ function buildDefaultAnalystRegistry(options = {}) {
 //#endregion
 export { createChatClient as C, computeTraceMetrics as S, CONTROL_INTEGRITY_ANALYST as _, analystFindingDigest as a, behavioralAnalyst as b, completedAnalystReviewQuality as c, validateAnalystReviewDecisions as d, DEFAULT_TRACE_ANALYST_KINDS as f, FAILURE_MODE_KIND_SPEC as g, IMPROVEMENT_KIND_SPEC as h, assertExactRegistryRunOpts as i, readAnalystReview as l, KNOWLEDGE_GAP_KIND_SPEC as m, AnalystRegistry as n, analystRunDigest as o, KNOWLEDGE_POISONING_KIND_SPEC as p, ExactAnalystRunExecutionError as r, assertUniqueFindingIds as s, buildDefaultAnalystRegistry as t, snapshotAnalystRun as u, ControlIntegrityAnalyst as v, deriveEfficiencyFindings as x, emitControlIntegrityFindings as y };
 
-//# sourceMappingURL=default-registry-RLNNoeEP.js.map
+//# sourceMappingURL=default-registry-BaQXW1Ow.js.map