@tangle-network/agent-eval 0.161.0 → 0.163.2

package/dist/{chat-json-call-6g5sJobJ.js → chat-json-call-5Jxna-aV.js}

@@ -1,4 +1,4 @@
-import { c as extractJsonPayload, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-hgDieDNN.js";
+import { c as extractJsonPayload, o as costReceiptFromLlm, s as costReceiptFromLlmError, u as maximumChargeForLlmRequest } from "./llm-client-BFMRpmqb.js";
 //#region src/chat-json-call.ts
 /** Parse a JSON answer out of a model response. The transport may fence it. */
 function parseJsonAnswer(response, actor) {
@@ -50,4 +50,4 @@ async function paidJsonChat(input) {
 //#endregion
 export { paidJsonChat as t };
 
-//# sourceMappingURL=chat-json-call-6g5sJobJ.js.map
+//# sourceMappingURL=chat-json-call-5Jxna-aV.js.map

package/dist/{chat-json-call-6g5sJobJ.js.map → chat-json-call-5Jxna-aV.js.map}

@@ -1 +1 @@
-{"version":3,"file":"chat-json-call-6g5sJobJ.js","names":[],"sources":["../src/chat-json-call.ts"],"sourcesContent":["/**\n * One paid JSON model call through a caller-owned `ChatClient`, metered by the\n * cost ledger.\n *\n * agent-eval executes no paid model: the transport is supplied by the caller\n * and the credential never enters this package. What stays here is the\n * accounting around the call — the priced maximum reserved before it runs, the\n * stable call id forwarded as the provider idempotency key, the receipt\n * settled from the response, and an honest unknown-usage receipt when the\n * transport failed.\n *\n * The judges and the wire judge endpoint all make the same call in the same\n * order; this is the one copy of that sequence.\n */\n\nimport type { ChatClient, ChatResponse } from './analyst/chat-client'\nimport type { CostChannel, CostLedgerHandle, CostReceipt, CustomTokenPricing } from './cost-ledger'\nimport {\n  costReceiptFromLlm,\n  costReceiptFromLlmError,\n  extractJsonPayload,\n  type LlmCallRequest,\n  maximumChargeForLlmRequest,\n} from './llm-client'\n\nexport interface PaidJsonChatInput {\n  /** Caller-owned transport. One `chat()` call. */\n  chat: ChatClient\n  /** The exact canonical request, including its JSON-mode or schema fields. */\n  request: LlmCallRequest\n  ledger: CostLedgerHandle\n  channel: CostChannel\n  phase: string\n  actor: string\n  tags?: Record<string, string>\n  signal?: AbortSignal\n  /** Endpoint rates used when the transport reports no billed amount. */\n  pricing?: CustomTokenPricing\n}\n\nexport type PaidJsonChatResult<T> =\n  | { succeeded: true; value: T; response: ChatResponse; receipt: CostReceipt }\n  | { succeeded: false; error: Error; receipt?: CostReceipt }\n\n/** Parse a JSON answer out of a model response. The transport may fence it. */\nfunction parseJsonAnswer<T>(response: ChatResponse, actor: string): T {\n  try {\n    return JSON.parse(extractJsonPayload(response.content)) as T\n  } catch (error) {\n    throw new Error(\n      `${actor}: model answer was not JSON — ${error instanceof Error ? error.message : String(error)}`,\n    )\n  }\n}\n\nexport async function paidJsonChat<T>(input: PaidJsonChatInput): Promise<PaidJsonChatResult<T>> {\n  const paid = await input.ledger.runPaidCall({\n    channel: input.channel,\n    phase: input.phase,\n    actor: input.actor,\n    model: input.request.model,\n    ...(input.tags && Object.keys(input.tags).length > 0 ? { tags: input.tags } : {}),\n    maximumCharge: maximumChargeForLlmRequest(input.request, {\n      ...(input.chat.maximumAttempts === undefined\n        ? {}\n        : { maximumAttempts: input.chat.maximumAttempts }),\n      ...(input.pricing ? { customTokenPricing: input.pricing } : {}),\n    }),\n    ...(input.signal ? { signal: input.signal } : {}),\n    execute: (signal, callId) => input.chat.chat(input.request, { signal, idempotencyKey: callId }),\n    receipt: (response) => costReceiptFromLlm(response, input.pricing),\n    receiptFromError: (error) => costReceiptFromLlmError(error, input.pricing),\n  })\n  if (!paid.succeeded) {\n    return {\n      succeeded: false,\n      error: paid.error,\n      ...(paid.receipt ? { receipt: paid.receipt } : {}),\n    }\n  }\n  // The call completed and was billed. A malformed answer is a contract\n  // failure AFTER the money was spent, so it keeps the settled receipt instead\n  // of reporting the spend as unknown.\n  try {\n    return {\n      succeeded: true,\n      value: parseJsonAnswer<T>(paid.value, input.actor),\n      response: paid.value,\n      receipt: paid.receipt,\n    }\n  } catch (error) {\n    return {\n      succeeded: false,\n      error: error instanceof Error ? error : new Error(String(error)),\n      receipt: paid.receipt,\n    }\n  }\n}\n"],"mappings":";;;AA6CA,SAAS,gBAAmB,UAAwB,OAAkB;CACpE,IAAI;EACF,OAAO,KAAK,MAAM,mBAAmB,SAAS,OAAO,CAAC;CACxD,SAAS,OAAO;EACd,MAAM,IAAI,MACR,GAAG,MAAM,gCAAgC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,GAChG;CACF;AACF;AAEA,eAAsB,aAAgB,OAA0D;CAC9F,MAAM,OAAO,MAAM,MAAM,OAAO,YAAY;EAC1C,SAAS,MAAM;EACf,OAAO,MAAM;EACb,OAAO,MAAM;EACb,OAAO,MAAM,QAAQ;EACrB,GAAI,MAAM,QAAQ,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,SAAS,IAAI,EAAE,MAAM,MAAM,KAAK,IAAI,CAAC;EAC/E,eAAe,2BAA2B,MAAM,SAAS;GACvD,GAAI,MAAM,KAAK,oBAAoB,KAAA,IAC/B,CAAC,IACD,EAAE,iBAAiB,MAAM,KAAK,gBAAgB;GAClD,GAAI,MAAM,UAAU,EAAE,oBAAoB,MAAM,QAAQ,IAAI,CAAC;EAC/D,CAAC;EACD,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAO,IAAI,CAAC;EAC/C,UAAU,QAAQ,WAAW,MAAM,KAAK,KAAK,MAAM,SAAS;GAAE;GAAQ,gBAAgB;EAAO,CAAC;EAC9F,UAAU,aAAa,mBAAmB,UAAU,MAAM,OAAO;EACjE,mBAAmB,UAAU,wBAAwB,OAAO,MAAM,OAAO;CAC3E,CAAC;CACD,IAAI,CAAC,KAAK,WACR,OAAO;EACL,WAAW;EACX,OAAO,KAAK;EACZ,GAAI,KAAK,UAAU,EAAE,SAAS,KAAK,QAAQ,IAAI,CAAC;CAClD;CAKF,IAAI;EACF,OAAO;GACL,WAAW;GACX,OAAO,gBAAmB,KAAK,OAAO,MAAM,KAAK;GACjD,UAAU,KAAK;GACf,SAAS,KAAK;EAChB;CACF,SAAS,OAAO;EACd,OAAO;GACL,WAAW;GACX,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GAC/D,SAAS,KAAK;EAChB;CACF;AACF"}
+{"version":3,"file":"chat-json-call-5Jxna-aV.js","names":[],"sources":["../src/chat-json-call.ts"],"sourcesContent":["/**\n * One paid JSON model call through a caller-owned `ChatClient`, metered by the\n * cost ledger.\n *\n * agent-eval executes no paid model: the transport is supplied by the caller\n * and the credential never enters this package. What stays here is the\n * accounting around the call — the priced maximum reserved before it runs, the\n * stable call id forwarded as the provider idempotency key, the receipt\n * settled from the response, and an honest unknown-usage receipt when the\n * transport failed.\n *\n * The judges and the wire judge endpoint all make the same call in the same\n * order; this is the one copy of that sequence.\n */\n\nimport type { ChatClient, ChatResponse } from './analyst/chat-client'\nimport type { CostChannel, CostLedgerHandle, CostReceipt, CustomTokenPricing } from './cost-ledger'\nimport {\n  costReceiptFromLlm,\n  costReceiptFromLlmError,\n  extractJsonPayload,\n  type LlmCallRequest,\n  maximumChargeForLlmRequest,\n} from './llm-client'\n\nexport interface PaidJsonChatInput {\n  /** Caller-owned transport. One `chat()` call. */\n  chat: ChatClient\n  /** The exact canonical request, including its JSON-mode or schema fields. */\n  request: LlmCallRequest\n  ledger: CostLedgerHandle\n  channel: CostChannel\n  phase: string\n  actor: string\n  tags?: Record<string, string>\n  signal?: AbortSignal\n  /** Endpoint rates used when the transport reports no billed amount. */\n  pricing?: CustomTokenPricing\n}\n\nexport type PaidJsonChatResult<T> =\n  | { succeeded: true; value: T; response: ChatResponse; receipt: CostReceipt }\n  | { succeeded: false; error: Error; receipt?: CostReceipt }\n\n/** Parse a JSON answer out of a model response. The transport may fence it. */\nfunction parseJsonAnswer<T>(response: ChatResponse, actor: string): T {\n  try {\n    return JSON.parse(extractJsonPayload(response.content)) as T\n  } catch (error) {\n    throw new Error(\n      `${actor}: model answer was not JSON — ${error instanceof Error ? error.message : String(error)}`,\n    )\n  }\n}\n\nexport async function paidJsonChat<T>(input: PaidJsonChatInput): Promise<PaidJsonChatResult<T>> {\n  const paid = await input.ledger.runPaidCall({\n    channel: input.channel,\n    phase: input.phase,\n    actor: input.actor,\n    model: input.request.model,\n    ...(input.tags && Object.keys(input.tags).length > 0 ? { tags: input.tags } : {}),\n    maximumCharge: maximumChargeForLlmRequest(input.request, {\n      ...(input.chat.maximumAttempts === undefined\n        ? {}\n        : { maximumAttempts: input.chat.maximumAttempts }),\n      ...(input.pricing ? { customTokenPricing: input.pricing } : {}),\n    }),\n    ...(input.signal ? { signal: input.signal } : {}),\n    execute: (signal, callId) => input.chat.chat(input.request, { signal, idempotencyKey: callId }),\n    receipt: (response) => costReceiptFromLlm(response, input.pricing),\n    receiptFromError: (error) => costReceiptFromLlmError(error, input.pricing),\n  })\n  if (!paid.succeeded) {\n    return {\n      succeeded: false,\n      error: paid.error,\n      ...(paid.receipt ? { receipt: paid.receipt } : {}),\n    }\n  }\n  // The call completed and was billed. A malformed answer is a contract\n  // failure AFTER the money was spent, so it keeps the settled receipt instead\n  // of reporting the spend as unknown.\n  try {\n    return {\n      succeeded: true,\n      value: parseJsonAnswer<T>(paid.value, input.actor),\n      response: paid.value,\n      receipt: paid.receipt,\n    }\n  } catch (error) {\n    return {\n      succeeded: false,\n      error: error instanceof Error ? error : new Error(String(error)),\n      receipt: paid.receipt,\n    }\n  }\n}\n"],"mappings":";;;AA6CA,SAAS,gBAAmB,UAAwB,OAAkB;CACpE,IAAI;EACF,OAAO,KAAK,MAAM,mBAAmB,SAAS,OAAO,CAAC;CACxD,SAAS,OAAO;EACd,MAAM,IAAI,MACR,GAAG,MAAM,gCAAgC,iBAAiB,QAAQ,MAAM,UAAU,OAAO,KAAK,GAChG;CACF;AACF;AAEA,eAAsB,aAAgB,OAA0D;CAC9F,MAAM,OAAO,MAAM,MAAM,OAAO,YAAY;EAC1C,SAAS,MAAM;EACf,OAAO,MAAM;EACb,OAAO,MAAM;EACb,OAAO,MAAM,QAAQ;EACrB,GAAI,MAAM,QAAQ,OAAO,KAAK,MAAM,IAAI,CAAC,CAAC,SAAS,IAAI,EAAE,MAAM,MAAM,KAAK,IAAI,CAAC;EAC/E,eAAe,2BAA2B,MAAM,SAAS;GACvD,GAAI,MAAM,KAAK,oBAAoB,KAAA,IAC/B,CAAC,IACD,EAAE,iBAAiB,MAAM,KAAK,gBAAgB;GAClD,GAAI,MAAM,UAAU,EAAE,oBAAoB,MAAM,QAAQ,IAAI,CAAC;EAC/D,CAAC;EACD,GAAI,MAAM,SAAS,EAAE,QAAQ,MAAM,OAAO,IAAI,CAAC;EAC/C,UAAU,QAAQ,WAAW,MAAM,KAAK,KAAK,MAAM,SAAS;GAAE;GAAQ,gBAAgB;EAAO,CAAC;EAC9F,UAAU,aAAa,mBAAmB,UAAU,MAAM,OAAO;EACjE,mBAAmB,UAAU,wBAAwB,OAAO,MAAM,OAAO;CAC3E,CAAC;CACD,IAAI,CAAC,KAAK,WACR,OAAO;EACL,WAAW;EACX,OAAO,KAAK;EACZ,GAAI,KAAK,UAAU,EAAE,SAAS,KAAK,QAAQ,IAAI,CAAC;CAClD;CAKF,IAAI;EACF,OAAO;GACL,WAAW;GACX,OAAO,gBAAmB,KAAK,OAAO,MAAM,KAAK;GACjD,UAAU,KAAK;GACf,SAAS,KAAK;EAChB;CACF,SAAS,OAAO;EACd,OAAO;GACL,WAAW;GACX,OAAO,iBAAiB,QAAQ,QAAQ,IAAI,MAAM,OAAO,KAAK,CAAC;GAC/D,SAAS,KAAK;EAChB;CACF;AACF"}

package/dist/cli.js

@@ -1,8 +1,8 @@
 #!/usr/bin/env node
 import { i as runRolloutReleaseCli } from "./hf-dataset-D8_RNIis.js";
-import { n as LlmClient } from "./llm-client-hgDieDNN.js";
-import { t as runAnalystBenchmarkCommand } from "./benchmark-command-BVtaq_ve.js";
-import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-BtFd4uzB.js";
+import { n as LlmClient } from "./llm-client-BFMRpmqb.js";
+import { t as runAnalystBenchmarkCommand } from "./benchmark-command-CF-4GEWZ.js";
+import { a as runRpcBatch, o as runRpcOnce, p as handleVersion, r as startServerAsync, s as buildOpenApi } from "./server-BjYiJHoJ.js";
 import { writeFileSync } from "node:fs";
 //#region src/cli-config.ts
 /**

package/dist/contract/index.js

@@ -1,14 +1,14 @@
 import { s as ValidationError } from "../errors-Dngq5h35.js";
 import { i as hashCanonical } from "../canonical-IL-Bu-14.js";
-import { r as pairedBootstrap } from "../paired-tests-BHIhYVdu.js";
-import { a as summarizeExecution, i as analyzeRuns, n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-h-s-sI-v.js";
+import { r as pairedBootstrap } from "../paired-tests-C8iCsioC.js";
+import { a as summarizeExecution, i as analyzeRuns, n as SelfImproveRunError, r as selfImprove, t as defineAgentEval } from "../define-agent-eval-D08pWIJb.js";
 import { a as parseRunRecordSafe } from "../run-record-BC0ebuRP.js";
-import { E as compareOptimizationMethods, Y as defaultProductionGate, ft as campaignSplitDigest, i as transientDispatchFailure, m as runImprovementLoop, nt as runCampaign, t as llmJudge, tt as runEval } from "../llm-judge-BhasIPFT.js";
+import { E as compareOptimizationMethods, Y as defaultProductionGate, ft as campaignSplitDigest, i as transientDispatchFailure, m as runImprovementLoop, nt as runCampaign, t as llmJudge, tt as runEval } from "../llm-judge-Du7WQPh7.js";
 import { i as isModelPriced, n as estimateCost } from "../metrics-Qv-cpptD.js";
 import { i as CostLedger } from "../cost-ledger-B1qx30B4.js";
 import { d as mapConcurrentRange } from "../ledger-core-BOzlRygb.js";
-import { S as inMemoryCampaignStorage, x as fsCampaignStorage } from "../external-optimizer-subprocess-BIWbHpgD.js";
-import { a as heldoutSignificance, s as decidePairedPromotion } from "../power-preflight-DEw-uC7q.js";
+import { S as inMemoryCampaignStorage, x as fsCampaignStorage } from "../external-optimizer-subprocess-CqLMW3nh.js";
+import { a as heldoutSignificance, s as decidePairedPromotion } from "../power-preflight-CFXm0Vjo.js";
 import { r as makeProposalFinding } from "../types-BI4fT3HN.js";
 import { G as isOtlpModelCall, W as classifyOtlpSpanRole } from "../kind-factory-DY8FdoXf.js";
 import { n as buildDefaultAnalystRegistry, t as createChatClient } from "../chat-client-DlMlAeYI.js";
@@ -16,8 +16,8 @@ import { LLM_MODEL_ATTR_KEYS, SPAN_KIND_ATTR_KEYS } from "../trace-attributes.js
 import { t as extractUsage } from "../extract-usage-BrQ8mCLX.js";
 import { n as InMemoryOutcomeStore, t as FileSystemOutcomeStore } from "../outcome-store-ChBKlTd_.js";
 import { i as summarizeTraceErrors, n as recordAggregateMeasurements, r as summarizeExecutionMeasurements, t as readTaskFailureLabels } from "../task-failure-attributes-DTl-7-Kw.js";
-import { c as externalTextOptimizationMethod, d as REFERENCE_EQUIVALENCE_INPUT_LIMITS, f as REFERENCE_EQUIVALENCE_JUDGE_VERSION, m as runReferenceEquivalenceJudge, n as gepaOptimizationMethod, o as heldOutGate, p as createReferenceEquivalenceJudge, s as composeGate, t as skillOptOptimizationMethod } from "../skillopt-optimization-method-DbaekMcn.js";
-import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-xzA40Evo.js";
+import { c as externalTextOptimizationMethod, d as REFERENCE_EQUIVALENCE_INPUT_LIMITS, f as REFERENCE_EQUIVALENCE_JUDGE_VERSION, m as runReferenceEquivalenceJudge, n as gepaOptimizationMethod, o as heldOutGate, p as createReferenceEquivalenceJudge, s as composeGate, t as skillOptOptimizationMethod } from "../skillopt-optimization-method-UArRo-nr.js";
+import { n as paretoPolicy, r as paretoSignificanceGate, t as buildEvidenceVector } from "../promotion-policy-LY9mVQ7W.js";
 import { createHash } from "node:crypto";
 import { agentCandidateBenchmarkSuiteSchema, agentCandidateBenchmarkTaskSchema, agentCandidateBundleSchema, agentCandidateEvaluationPolicySchema, agentCandidateExperimentSchema, agentImprovementMeasuredComparisonSchema, agentProfileImprovementExperimentSchema, agentProfileImprovementMeasuredComparisonSchema, agentProfileImprovementRunCellSchema, agentProfileImprovementRunReceiptSchema, agentProfileImprovementSuiteInputsSchema, agentProfileImprovementSuiteSchema, agentProfileImprovementTaskSchema, candidateExecutionEvidenceSchema, canonicalCandidateDigest, canonicalCandidateJson, numbersApproximatelyEqual, omitTopLevelDigest } from "@tangle-network/agent-interface";
 import { createReadStream } from "node:fs";

package/dist/{counterfactual-D_VWavVm.js → counterfactual-Bjq1mlUu.js}

@@ -1,5 +1,5 @@
 import { o as NotFoundError, s as ValidationError } from "./errors-Dngq5h35.js";
-import { t as TraceEmitter } from "./emitter-BpYFQPj4.js";
+import { t as TraceEmitter } from "./emitter-DeQHiDMm.js";
 import { t as buildTrajectory } from "./trajectory-D_7rLrvE.js";
 //#region src/counterfactual.ts
 /**
@@ -95,4 +95,4 @@ function applyMutation(step, mutation) {
 //#endregion
 export { runCounterfactual as t };
 
-//# sourceMappingURL=counterfactual-D_VWavVm.js.map
+//# sourceMappingURL=counterfactual-Bjq1mlUu.js.map

package/dist/{counterfactual-D_VWavVm.js.map → counterfactual-Bjq1mlUu.js.map}

@@ -1 +1 @@
-{"version":3,"file":"counterfactual-D_VWavVm.js","names":[],"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n  | { kind: 'swap-model'; at: number; newModel: string }\n  | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n  | { kind: 'truncate-after'; at: number }\n  | { kind: 'inject-system-message'; at: number; content: string }\n  | {\n      kind: 'custom'\n      at: number\n      describe: string\n      apply: (step: TrajectoryStep) => TrajectoryStep\n    }\n\nexport interface CounterfactualContext {\n  originalRunId: string\n  originalTrajectory: Trajectory\n  /** Steps up to (but not including) the mutation point — the prefix the\n   *  replayed agent inherits as its prior conversation/tool history. */\n  prefix: TrajectoryStep[]\n  mutation: CounterfactualMutation\n  /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n   *  as the FIRST step the replayed agent emits (they decide whether to\n   *  re-emit it or continue from there). */\n  mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n  counterfactualRunId: string\n  originalRunId: string\n  mutation: CounterfactualMutation\n  /** Structured delta summary — caller can extend via scoring. */\n  delta: {\n    originalOutcomeScore: number | null\n    counterfactualOutcomeScore: number | null\n    deltaScore: number | null\n  }\n}\n\nexport interface CounterfactualRunner {\n  /**\n   * Execute the agent from `ctx.prefix` with the mutation applied.\n   * MUST emit spans into the provided emitter so they become part of\n   * the counterfactual run. MUST call emitter.endRun() with a verdict.\n   */\n  executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n  store: TraceStore,\n  originalRunId: string,\n  mutation: CounterfactualMutation,\n  runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n  const originalRun = await store.getRun(originalRunId)\n  if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n  const trajectory = await buildTrajectory(store, originalRunId)\n  if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n    throw new ValidationError(\n      `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n    )\n  }\n  const targetStep = trajectory.steps[mutation.at]!\n  const mutatedStep = applyMutation(targetStep, mutation)\n\n  const cfEmitter = new TraceEmitter(store)\n  await cfEmitter.startRun({\n    scenarioId: originalRun.scenarioId,\n    variantId: originalRun.variantId\n      ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n      : `cf:${mutation.kind}@${mutation.at}`,\n    projectId: originalRun.projectId,\n    parentRunId: originalRunId,\n    layer: 'meta',\n    tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n  })\n\n  await runner.executeFrom(\n    {\n      originalRunId,\n      originalTrajectory: trajectory,\n      prefix: trajectory.steps.slice(0, mutation.at),\n      mutation,\n      mutatedStep,\n    },\n    cfEmitter,\n  )\n\n  const counterfactual = await store.getRun(cfEmitter.runId)\n  const delta = {\n    originalOutcomeScore: originalRun.outcome?.score ?? null,\n    counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n    deltaScore:\n      originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n        ? counterfactual.outcome.score - originalRun.outcome.score\n        : null,\n  }\n  return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n  if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return { ...step, span: { ...llm, model: mutation.newModel } }\n  }\n  if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n    const tool = step.span as ToolSpan\n    return { ...step, span: { ...tool, result: mutation.newResult } }\n  }\n  if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return {\n      ...step,\n      span: {\n        ...llm,\n        messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n      },\n    }\n  }\n  if (mutation.kind === 'custom') return mutation.apply(step)\n  // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n  return step\n}\n\n/**\n * Aggregate a batch of counterfactuals into a simple attribution table:\n * which mutation kinds move outcomes most? (Useful when you run a grid\n * over the same trajectory — swap-model at every llm span, swap-tool\n * at every tool span — and want a ranked summary.)\n */\nexport function attributeCounterfactuals(results: CounterfactualResult[]): Array<{\n  mutationKind: CounterfactualMutation['kind']\n  n: number\n  meanAbsDelta: number\n  meanSignedDelta: number\n}> {\n  const grouped = new Map<string, CounterfactualResult[]>()\n  for (const r of results) {\n    const arr = grouped.get(r.mutation.kind) ?? []\n    arr.push(r)\n    grouped.set(r.mutation.kind, arr)\n  }\n  const out: Array<{\n    mutationKind: CounterfactualMutation['kind']\n    n: number\n    meanAbsDelta: number\n    meanSignedDelta: number\n  }> = []\n  for (const [kind, items] of grouped) {\n    const deltas = items\n      .map((i) => i.delta.deltaScore)\n      .filter((d): d is number => typeof d === 'number')\n    if (deltas.length === 0) continue\n    const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length\n    const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length\n    out.push({\n      mutationKind: kind as CounterfactualMutation['kind'],\n      n: deltas.length,\n      meanAbsDelta: meanAbs,\n      meanSignedDelta: meanSigned,\n    })\n  }\n  return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta)\n}\n\n// Re-export Span type for consumer ergonomics.\nexport type { Span }\n"],"mappings":";;;;;;;;;;;;;;;;;AAkEA,eAAsB,kBACpB,OACA,eACA,UACA,QAC+B;CAC/B,MAAM,cAAc,MAAM,MAAM,OAAO,aAAa;CACpD,IAAI,CAAC,aAAa,MAAM,IAAI,cAAc,uBAAuB,cAAc,WAAW;CAC1F,MAAM,aAAa,MAAM,gBAAgB,OAAO,aAAa;CAC7D,IAAI,SAAS,KAAK,KAAK,SAAS,MAAM,WAAW,MAAM,QACrD,MAAM,IAAI,gBACR,+BAA+B,SAAS,GAAG,oBAAoB,WAAW,MAAM,OAAO,EACzF;CAEF,MAAM,aAAa,WAAW,MAAM,SAAS;CAC7C,MAAM,cAAc,cAAc,YAAY,QAAQ;CAEtD,MAAM,YAAY,IAAI,aAAa,KAAK;CACxC,MAAM,UAAU,SAAS;EACvB,YAAY,YAAY;EACxB,WAAW,YAAY,YACnB,GAAG,YAAY,UAAU,MAAM,SAAS,KAAK,GAAG,SAAS,OACzD,MAAM,SAAS,KAAK,GAAG,SAAS;EACpC,WAAW,YAAY;EACvB,aAAa;EACb,OAAO;EACP,MAAM;GAAE,gBAAgB;GAAQ,cAAc,SAAS;GAAM,YAAY,OAAO,SAAS,EAAE;EAAE;CAC/F,CAAC;CAED,MAAM,OAAO,YACX;EACE;EACA,oBAAoB;EACpB,QAAQ,WAAW,MAAM,MAAM,GAAG,SAAS,EAAE;EAC7C;EACA;CACF,GACA,SACF;CAEA,MAAM,iBAAiB,MAAM,MAAM,OAAO,UAAU,KAAK;CACzD,MAAM,QAAQ;EACZ,sBAAsB,YAAY,SAAS,SAAS;EACpD,4BAA4B,gBAAgB,SAAS,SAAS;EAC9D,YACE,YAAY,SAAS,UAAU,KAAA,KAAa,gBAAgB,SAAS,UAAU,KAAA,IAC3E,eAAe,QAAQ,QAAQ,YAAY,QAAQ,QACnD;CACR;CACA,OAAO;EAAE,qBAAqB,UAAU;EAAO;EAAe;EAAU;CAAM;AAChF;AAEA,SAAS,cAAc,MAAsB,UAAkD;CAC7F,IAAI,SAAS,SAAS,gBAAgB,KAAK,KAAK,SAAS,OAAO;EAC9D,MAAM,MAAM,KAAK;EACjB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAK,OAAO,SAAS;GAAS;EAAE;CAC/D;CACA,IAAI,SAAS,SAAS,sBAAsB,KAAK,KAAK,SAAS,QAAQ;EACrE,MAAM,OAAO,KAAK;EAClB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAM,QAAQ,SAAS;GAAU;EAAE;CAClE;CACA,IAAI,SAAS,SAAS,2BAA2B,KAAK,KAAK,SAAS,OAAO;EACzE,MAAM,MAAM,KAAK;EACjB,OAAO;GACL,GAAG;GACH,MAAM;IACJ,GAAG;IACH,UAAU,CAAC;KAAE,MAAM;KAAU,SAAS,SAAS;IAAQ,GAAG,GAAG,IAAI,QAAQ;GAC3E;EACF;CACF;CACA,IAAI,SAAS,SAAS,UAAU,OAAO,SAAS,MAAM,IAAI;CAE1D,OAAO;AACT"}
+{"version":3,"file":"counterfactual-Bjq1mlUu.js","names":[],"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n  | { kind: 'swap-model'; at: number; newModel: string }\n  | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n  | { kind: 'truncate-after'; at: number }\n  | { kind: 'inject-system-message'; at: number; content: string }\n  | {\n      kind: 'custom'\n      at: number\n      describe: string\n      apply: (step: TrajectoryStep) => TrajectoryStep\n    }\n\nexport interface CounterfactualContext {\n  originalRunId: string\n  originalTrajectory: Trajectory\n  /** Steps up to (but not including) the mutation point — the prefix the\n   *  replayed agent inherits as its prior conversation/tool history. */\n  prefix: TrajectoryStep[]\n  mutation: CounterfactualMutation\n  /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n   *  as the FIRST step the replayed agent emits (they decide whether to\n   *  re-emit it or continue from there). */\n  mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n  counterfactualRunId: string\n  originalRunId: string\n  mutation: CounterfactualMutation\n  /** Structured delta summary — caller can extend via scoring. */\n  delta: {\n    originalOutcomeScore: number | null\n    counterfactualOutcomeScore: number | null\n    deltaScore: number | null\n  }\n}\n\nexport interface CounterfactualRunner {\n  /**\n   * Execute the agent from `ctx.prefix` with the mutation applied.\n   * MUST emit spans into the provided emitter so they become part of\n   * the counterfactual run. MUST call emitter.endRun() with a verdict.\n   */\n  executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n  store: TraceStore,\n  originalRunId: string,\n  mutation: CounterfactualMutation,\n  runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n  const originalRun = await store.getRun(originalRunId)\n  if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n  const trajectory = await buildTrajectory(store, originalRunId)\n  if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n    throw new ValidationError(\n      `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n    )\n  }\n  const targetStep = trajectory.steps[mutation.at]!\n  const mutatedStep = applyMutation(targetStep, mutation)\n\n  const cfEmitter = new TraceEmitter(store)\n  await cfEmitter.startRun({\n    scenarioId: originalRun.scenarioId,\n    variantId: originalRun.variantId\n      ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n      : `cf:${mutation.kind}@${mutation.at}`,\n    projectId: originalRun.projectId,\n    parentRunId: originalRunId,\n    layer: 'meta',\n    tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n  })\n\n  await runner.executeFrom(\n    {\n      originalRunId,\n      originalTrajectory: trajectory,\n      prefix: trajectory.steps.slice(0, mutation.at),\n      mutation,\n      mutatedStep,\n    },\n    cfEmitter,\n  )\n\n  const counterfactual = await store.getRun(cfEmitter.runId)\n  const delta = {\n    originalOutcomeScore: originalRun.outcome?.score ?? null,\n    counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n    deltaScore:\n      originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n        ? counterfactual.outcome.score - originalRun.outcome.score\n        : null,\n  }\n  return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n  if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return { ...step, span: { ...llm, model: mutation.newModel } }\n  }\n  if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n    const tool = step.span as ToolSpan\n    return { ...step, span: { ...tool, result: mutation.newResult } }\n  }\n  if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return {\n      ...step,\n      span: {\n        ...llm,\n        messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n      },\n    }\n  }\n  if (mutation.kind === 'custom') return mutation.apply(step)\n  // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n  return step\n}\n\n/**\n * Aggregate a batch of counterfactuals into a simple attribution table:\n * which mutation kinds move outcomes most? (Useful when you run a grid\n * over the same trajectory — swap-model at every llm span, swap-tool\n * at every tool span — and want a ranked summary.)\n */\nexport function attributeCounterfactuals(results: CounterfactualResult[]): Array<{\n  mutationKind: CounterfactualMutation['kind']\n  n: number\n  meanAbsDelta: number\n  meanSignedDelta: number\n}> {\n  const grouped = new Map<string, CounterfactualResult[]>()\n  for (const r of results) {\n    const arr = grouped.get(r.mutation.kind) ?? []\n    arr.push(r)\n    grouped.set(r.mutation.kind, arr)\n  }\n  const out: Array<{\n    mutationKind: CounterfactualMutation['kind']\n    n: number\n    meanAbsDelta: number\n    meanSignedDelta: number\n  }> = []\n  for (const [kind, items] of grouped) {\n    const deltas = items\n      .map((i) => i.delta.deltaScore)\n      .filter((d): d is number => typeof d === 'number')\n    if (deltas.length === 0) continue\n    const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length\n    const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length\n    out.push({\n      mutationKind: kind as CounterfactualMutation['kind'],\n      n: deltas.length,\n      meanAbsDelta: meanAbs,\n      meanSignedDelta: meanSigned,\n    })\n  }\n  return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta)\n}\n\n// Re-export Span type for consumer ergonomics.\nexport type { Span }\n"],"mappings":";;;;;;;;;;;;;;;;;AAkEA,eAAsB,kBACpB,OACA,eACA,UACA,QAC+B;CAC/B,MAAM,cAAc,MAAM,MAAM,OAAO,aAAa;CACpD,IAAI,CAAC,aAAa,MAAM,IAAI,cAAc,uBAAuB,cAAc,WAAW;CAC1F,MAAM,aAAa,MAAM,gBAAgB,OAAO,aAAa;CAC7D,IAAI,SAAS,KAAK,KAAK,SAAS,MAAM,WAAW,MAAM,QACrD,MAAM,IAAI,gBACR,+BAA+B,SAAS,GAAG,oBAAoB,WAAW,MAAM,OAAO,EACzF;CAEF,MAAM,aAAa,WAAW,MAAM,SAAS;CAC7C,MAAM,cAAc,cAAc,YAAY,QAAQ;CAEtD,MAAM,YAAY,IAAI,aAAa,KAAK;CACxC,MAAM,UAAU,SAAS;EACvB,YAAY,YAAY;EACxB,WAAW,YAAY,YACnB,GAAG,YAAY,UAAU,MAAM,SAAS,KAAK,GAAG,SAAS,OACzD,MAAM,SAAS,KAAK,GAAG,SAAS;EACpC,WAAW,YAAY;EACvB,aAAa;EACb,OAAO;EACP,MAAM;GAAE,gBAAgB;GAAQ,cAAc,SAAS;GAAM,YAAY,OAAO,SAAS,EAAE;EAAE;CAC/F,CAAC;CAED,MAAM,OAAO,YACX;EACE;EACA,oBAAoB;EACpB,QAAQ,WAAW,MAAM,MAAM,GAAG,SAAS,EAAE;EAC7C;EACA;CACF,GACA,SACF;CAEA,MAAM,iBAAiB,MAAM,MAAM,OAAO,UAAU,KAAK;CACzD,MAAM,QAAQ;EACZ,sBAAsB,YAAY,SAAS,SAAS;EACpD,4BAA4B,gBAAgB,SAAS,SAAS;EAC9D,YACE,YAAY,SAAS,UAAU,KAAA,KAAa,gBAAgB,SAAS,UAAU,KAAA,IAC3E,eAAe,QAAQ,QAAQ,YAAY,QAAQ,QACnD;CACR;CACA,OAAO;EAAE,qBAAqB,UAAU;EAAO;EAAe;EAAU;CAAM;AAChF;AAEA,SAAS,cAAc,MAAsB,UAAkD;CAC7F,IAAI,SAAS,SAAS,gBAAgB,KAAK,KAAK,SAAS,OAAO;EAC9D,MAAM,MAAM,KAAK;EACjB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAK,OAAO,SAAS;GAAS;EAAE;CAC/D;CACA,IAAI,SAAS,SAAS,sBAAsB,KAAK,KAAK,SAAS,QAAQ;EACrE,MAAM,OAAO,KAAK;EAClB,OAAO;GAAE,GAAG;GAAM,MAAM;IAAE,GAAG;IAAM,QAAQ,SAAS;GAAU;EAAE;CAClE;CACA,IAAI,SAAS,SAAS,2BAA2B,KAAK,KAAK,SAAS,OAAO;EACzE,MAAM,MAAM,KAAK;EACjB,OAAO;GACL,GAAG;GACH,MAAM;IACJ,GAAG;IACH,UAAU,CAAC;KAAE,MAAM;KAAU,SAAS,SAAS;IAAQ,GAAG,GAAG,IAAI,QAAQ;GAC3E;EACF;CACF;CACA,IAAI,SAAS,SAAS,UAAU,OAAO,SAAS,MAAM,IAAI;CAE1D,OAAO;AACT"}

package/dist/{define-agent-eval-h-s-sI-v.js → define-agent-eval-D08pWIJb.js}

@@ -1,19 +1,19 @@
 import { s as ValidationError } from "./errors-Dngq5h35.js";
-import { a as spearmanR, r as pearsonR } from "./descriptive-jDOuI6mz.js";
-import { r as continuousAgreement } from "./judge-calibration-zZjLz8hr.js";
+import { a as spearmanR, r as pearsonR } from "./descriptive-1V17A-qa.js";
+import { r as continuousAgreement } from "./judge-calibration-BnpVKtnb.js";
 import { i as pairedCohensDz } from "./effect-sizes-DiH8MGOH.js";
-import { i as requiredPairedSampleSize, r as pairedMde } from "./power-and-mde-CHIrXJll.js";
-import { r as pairRunRecords } from "./paired-arms-D-XRF_fy.js";
-import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-BHIhYVdu.js";
-import { r as welchsTTest } from "./baseline-BhPRQBVn.js";
-import "./query-CHmMP42p.js";
+import { i as requiredPairedSampleSize, r as pairedMde } from "./power-and-mde-B8F2RdcD.js";
+import { r as pairRunRecords } from "./paired-arms-D4aeIHUy.js";
+import { o as pairedTTest, r as pairedBootstrap } from "./paired-tests-C8iCsioC.js";
+import { r as welchsTTest } from "./baseline-BC-eBZ7U.js";
+import "./query-_5g6re3_.js";
 import { r as observedSplitScore } from "./reward-nw2xZGZG.js";
 import { c as validateRunRecord, i as modelHasSnapshot } from "./run-record-BC0ebuRP.js";
-import { r as paretoChart } from "./summary-report-BI5hUtvK.js";
-import { N as surfaceContentHash, P as surfaceHash, Y as defaultProductionGate, c as emitLoopProvenance, it as resolveRunDir, l as loopProvenanceArgsFromResult, m as runImprovementLoop, tt as runEval, w as assertOptimizationResult } from "./llm-judge-BhasIPFT.js";
-import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-t4lB1yt8.js";
-import { S as inMemoryCampaignStorage, b as createRunCostLedger, x as fsCampaignStorage } from "./external-optimizer-subprocess-BIWbHpgD.js";
-import { t as powerPreflight } from "./power-preflight-DEw-uC7q.js";
+import { r as paretoChart } from "./summary-report-BXeQ5Ues.js";
+import { N as surfaceContentHash, P as surfaceHash, Y as defaultProductionGate, c as emitLoopProvenance, it as resolveRunDir, l as loopProvenanceArgsFromResult, m as runImprovementLoop, tt as runEval, w as assertOptimizationResult } from "./llm-judge-Du7WQPh7.js";
+import { c as campaignCellTaskScore, l as campaignCellToRunRecord, o as campaignCellExecutionEvidence, s as campaignCellJudgeDimensions } from "./reward-hacking-O5zKDANP.js";
+import { S as inMemoryCampaignStorage, b as createRunCostLedger, x as fsCampaignStorage } from "./external-optimizer-subprocess-CqLMW3nh.js";
+import { t as powerPreflight } from "./power-preflight-CFXm0Vjo.js";
 import { t as createHostedClient } from "./client-CX7KqIdB.js";
 //#region src/contamination-guard.ts
 function checkCanaries(output, scenarios) {
@@ -1497,4 +1497,4 @@ function requirePositiveInteger(value, field) {
 //#endregion
 export { summarizeExecution as a, analyzeRuns as i, SelfImproveRunError as n, checkCanaries as o, selfImprove as r, defineAgentEval as t };
 
-//# sourceMappingURL=define-agent-eval-h-s-sI-v.js.map
+//# sourceMappingURL=define-agent-eval-D08pWIJb.js.map