@tangle-network/agent-eval 0.108.1 → 0.110.0

package/dist/chunk-6SK5VFYK.js

@@ -1,100 +0,0 @@
-import {
-  buildTrajectory
-} from "./chunk-RZTMDUO7.js";
-import {
-  TraceEmitter
-} from "./chunk-TVVP3ZZQ.js";
-import {
-  NotFoundError,
-  ValidationError
-} from "./chunk-ONWEPEDO.js";
-
-// src/counterfactual.ts
-async function runCounterfactual(store, originalRunId, mutation, runner) {
-  const originalRun = await store.getRun(originalRunId);
-  if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`);
-  const trajectory = await buildTrajectory(store, originalRunId);
-  if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {
-    throw new ValidationError(
-      `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`
-    );
-  }
-  const targetStep = trajectory.steps[mutation.at];
-  const mutatedStep = applyMutation(targetStep, mutation);
-  const cfEmitter = new TraceEmitter(store);
-  await cfEmitter.startRun({
-    scenarioId: originalRun.scenarioId,
-    variantId: originalRun.variantId ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}` : `cf:${mutation.kind}@${mutation.at}`,
-    projectId: originalRun.projectId,
-    parentRunId: originalRunId,
-    layer: "meta",
-    tags: { counterfactual: "true", mutationKind: mutation.kind, mutationAt: String(mutation.at) }
-  });
-  await runner.executeFrom(
-    {
-      originalRunId,
-      originalTrajectory: trajectory,
-      prefix: trajectory.steps.slice(0, mutation.at),
-      mutation,
-      mutatedStep
-    },
-    cfEmitter
-  );
-  const counterfactual = await store.getRun(cfEmitter.runId);
-  const delta = {
-    originalOutcomeScore: originalRun.outcome?.score ?? null,
-    counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,
-    deltaScore: originalRun.outcome?.score !== void 0 && counterfactual?.outcome?.score !== void 0 ? counterfactual.outcome.score - originalRun.outcome.score : null
-  };
-  return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta };
-}
-function applyMutation(step, mutation) {
-  if (mutation.kind === "swap-model" && step.span.kind === "llm") {
-    const llm = step.span;
-    return { ...step, span: { ...llm, model: mutation.newModel } };
-  }
-  if (mutation.kind === "swap-tool-result" && step.span.kind === "tool") {
-    const tool = step.span;
-    return { ...step, span: { ...tool, result: mutation.newResult } };
-  }
-  if (mutation.kind === "inject-system-message" && step.span.kind === "llm") {
-    const llm = step.span;
-    return {
-      ...step,
-      span: {
-        ...llm,
-        messages: [{ role: "system", content: mutation.content }, ...llm.messages]
-      }
-    };
-  }
-  if (mutation.kind === "custom") return mutation.apply(step);
-  return step;
-}
-function attributeCounterfactuals(results) {
-  const grouped = /* @__PURE__ */ new Map();
-  for (const r of results) {
-    const arr = grouped.get(r.mutation.kind) ?? [];
-    arr.push(r);
-    grouped.set(r.mutation.kind, arr);
-  }
-  const out = [];
-  for (const [kind, items] of grouped) {
-    const deltas = items.map((i) => i.delta.deltaScore).filter((d) => typeof d === "number");
-    if (deltas.length === 0) continue;
-    const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length;
-    const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length;
-    out.push({
-      mutationKind: kind,
-      n: deltas.length,
-      meanAbsDelta: meanAbs,
-      meanSignedDelta: meanSigned
-    });
-  }
-  return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta);
-}
-
-export {
-  runCounterfactual,
-  attributeCounterfactuals
-};
-//# sourceMappingURL=chunk-6SK5VFYK.js.map

package/dist/chunk-6SK5VFYK.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/counterfactual.ts"],"sourcesContent":["/**\n * Counterfactual replay — \"what would have happened if we'd changed\n * exactly one thing at turn N?\"\n *\n * The framework does NOT drive the agent — it sets up the replay\n * context (prior spans, prior state, mutation spec) and records the\n * resulting divergence. Consumers supply an `executeFrom(ctx)` callback\n * that runs their agent starting from turn N with the mutation applied.\n *\n * Counterfactual runs are recorded as a new Run with `layer='meta'` and\n * `parentRunId = originalRunId`, so downstream diff + correlation\n * pipelines see them natively.\n */\n\nimport { NotFoundError, ValidationError } from './errors'\nimport { TraceEmitter } from './trace/emitter'\nimport type { LlmSpan, Span, ToolSpan } from './trace/schema'\nimport type { TraceStore } from './trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from './trajectory'\n\nexport type CounterfactualMutation =\n  | { kind: 'swap-model'; at: number; newModel: string }\n  | { kind: 'swap-tool-result'; at: number; newResult: unknown }\n  | { kind: 'truncate-after'; at: number }\n  | { kind: 'inject-system-message'; at: number; content: string }\n  | {\n      kind: 'custom'\n      at: number\n      describe: string\n      apply: (step: TrajectoryStep) => TrajectoryStep\n    }\n\nexport interface CounterfactualContext {\n  originalRunId: string\n  originalTrajectory: Trajectory\n  /** Steps up to (but not including) the mutation point — the prefix the\n   *  replayed agent inherits as its prior conversation/tool history. */\n  prefix: TrajectoryStep[]\n  mutation: CounterfactualMutation\n  /** Pre-applied mutation on the step at `mutation.at`. Consumers use this\n   *  as the FIRST step the replayed agent emits (they decide whether to\n   *  re-emit it or continue from there). */\n  mutatedStep: TrajectoryStep\n}\n\nexport interface CounterfactualResult {\n  counterfactualRunId: string\n  originalRunId: string\n  mutation: CounterfactualMutation\n  /** Structured delta summary — caller can extend via scoring. */\n  delta: {\n    originalOutcomeScore: number | null\n    counterfactualOutcomeScore: number | null\n    deltaScore: number | null\n  }\n}\n\nexport interface CounterfactualRunner {\n  /**\n   * Execute the agent from `ctx.prefix` with the mutation applied.\n   * MUST emit spans into the provided emitter so they become part of\n   * the counterfactual run. MUST call emitter.endRun() with a verdict.\n   */\n  executeFrom: (ctx: CounterfactualContext, emitter: TraceEmitter) => Promise<void>\n}\n\nexport async function runCounterfactual(\n  store: TraceStore,\n  originalRunId: string,\n  mutation: CounterfactualMutation,\n  runner: CounterfactualRunner,\n): Promise<CounterfactualResult> {\n  const originalRun = await store.getRun(originalRunId)\n  if (!originalRun) throw new NotFoundError(`counterfactual: run ${originalRunId} not found`)\n  const trajectory = await buildTrajectory(store, originalRunId)\n  if (mutation.at < 0 || mutation.at >= trajectory.steps.length) {\n    throw new ValidationError(\n      `counterfactual: mutation.at=${mutation.at} out of range [0, ${trajectory.steps.length})`,\n    )\n  }\n  const targetStep = trajectory.steps[mutation.at]!\n  const mutatedStep = applyMutation(targetStep, mutation)\n\n  const cfEmitter = new TraceEmitter(store)\n  await cfEmitter.startRun({\n    scenarioId: originalRun.scenarioId,\n    variantId: originalRun.variantId\n      ? `${originalRun.variantId}+cf:${mutation.kind}@${mutation.at}`\n      : `cf:${mutation.kind}@${mutation.at}`,\n    projectId: originalRun.projectId,\n    parentRunId: originalRunId,\n    layer: 'meta',\n    tags: { counterfactual: 'true', mutationKind: mutation.kind, mutationAt: String(mutation.at) },\n  })\n\n  await runner.executeFrom(\n    {\n      originalRunId,\n      originalTrajectory: trajectory,\n      prefix: trajectory.steps.slice(0, mutation.at),\n      mutation,\n      mutatedStep,\n    },\n    cfEmitter,\n  )\n\n  const counterfactual = await store.getRun(cfEmitter.runId)\n  const delta = {\n    originalOutcomeScore: originalRun.outcome?.score ?? null,\n    counterfactualOutcomeScore: counterfactual?.outcome?.score ?? null,\n    deltaScore:\n      originalRun.outcome?.score !== undefined && counterfactual?.outcome?.score !== undefined\n        ? counterfactual.outcome.score - originalRun.outcome.score\n        : null,\n  }\n  return { counterfactualRunId: cfEmitter.runId, originalRunId, mutation, delta }\n}\n\nfunction applyMutation(step: TrajectoryStep, mutation: CounterfactualMutation): TrajectoryStep {\n  if (mutation.kind === 'swap-model' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return { ...step, span: { ...llm, model: mutation.newModel } }\n  }\n  if (mutation.kind === 'swap-tool-result' && step.span.kind === 'tool') {\n    const tool = step.span as ToolSpan\n    return { ...step, span: { ...tool, result: mutation.newResult } }\n  }\n  if (mutation.kind === 'inject-system-message' && step.span.kind === 'llm') {\n    const llm = step.span as LlmSpan\n    return {\n      ...step,\n      span: {\n        ...llm,\n        messages: [{ role: 'system', content: mutation.content }, ...llm.messages],\n      },\n    }\n  }\n  if (mutation.kind === 'custom') return mutation.apply(step)\n  // swap-tool-result on non-tool span / swap-model on non-llm / truncate-after: no step-level change.\n  return step\n}\n\n/**\n * Aggregate a batch of counterfactuals into a simple attribution table:\n * which mutation kinds move outcomes most? (Useful when you run a grid\n * over the same trajectory — swap-model at every llm span, swap-tool\n * at every tool span — and want a ranked summary.)\n */\nexport function attributeCounterfactuals(results: CounterfactualResult[]): Array<{\n  mutationKind: CounterfactualMutation['kind']\n  n: number\n  meanAbsDelta: number\n  meanSignedDelta: number\n}> {\n  const grouped = new Map<string, CounterfactualResult[]>()\n  for (const r of results) {\n    const arr = grouped.get(r.mutation.kind) ?? []\n    arr.push(r)\n    grouped.set(r.mutation.kind, arr)\n  }\n  const out: Array<{\n    mutationKind: CounterfactualMutation['kind']\n    n: number\n    meanAbsDelta: number\n    meanSignedDelta: number\n  }> = []\n  for (const [kind, items] of grouped) {\n    const deltas = items\n      .map((i) => i.delta.deltaScore)\n      .filter((d): d is number => typeof d === 'number')\n    if (deltas.length === 0) continue\n    const meanAbs = deltas.reduce((a, b) => a + Math.abs(b), 0) / deltas.length\n    const meanSigned = deltas.reduce((a, b) => a + b, 0) / deltas.length\n    out.push({\n      mutationKind: kind as CounterfactualMutation['kind'],\n      n: deltas.length,\n      meanAbsDelta: meanAbs,\n      meanSignedDelta: meanSigned,\n    })\n  }\n  return out.sort((a, b) => b.meanAbsDelta - a.meanAbsDelta)\n}\n\n// Re-export Span type for consumer ergonomics.\nexport type { Span }\n"],"mappings":";;;;;;;;;;;;AAkEA,eAAsB,kBACpB,OACA,eACA,UACA,QAC+B;AAC/B,QAAM,cAAc,MAAM,MAAM,OAAO,aAAa;AACpD,MAAI,CAAC,YAAa,OAAM,IAAI,cAAc,uBAAuB,aAAa,YAAY;AAC1F,QAAM,aAAa,MAAM,gBAAgB,OAAO,aAAa;AAC7D,MAAI,SAAS,KAAK,KAAK,SAAS,MAAM,WAAW,MAAM,QAAQ;AAC7D,UAAM,IAAI;AAAA,MACR,+BAA+B,SAAS,EAAE,qBAAqB,WAAW,MAAM,MAAM;AAAA,IACxF;AAAA,EACF;AACA,QAAM,aAAa,WAAW,MAAM,SAAS,EAAE;AAC/C,QAAM,cAAc,cAAc,YAAY,QAAQ;AAEtD,QAAM,YAAY,IAAI,aAAa,KAAK;AACxC,QAAM,UAAU,SAAS;AAAA,IACvB,YAAY,YAAY;AAAA,IACxB,WAAW,YAAY,YACnB,GAAG,YAAY,SAAS,OAAO,SAAS,IAAI,IAAI,SAAS,EAAE,KAC3D,MAAM,SAAS,IAAI,IAAI,SAAS,EAAE;AAAA,IACtC,WAAW,YAAY;AAAA,IACvB,aAAa;AAAA,IACb,OAAO;AAAA,IACP,MAAM,EAAE,gBAAgB,QAAQ,cAAc,SAAS,MAAM,YAAY,OAAO,SAAS,EAAE,EAAE;AAAA,EAC/F,CAAC;AAED,QAAM,OAAO;AAAA,IACX;AAAA,MACE;AAAA,MACA,oBAAoB;AAAA,MACpB,QAAQ,WAAW,MAAM,MAAM,GAAG,SAAS,EAAE;AAAA,MAC7C;AAAA,MACA;AAAA,IACF;AAAA,IACA;AAAA,EACF;AAEA,QAAM,iBAAiB,MAAM,MAAM,OAAO,UAAU,KAAK;AACzD,QAAM,QAAQ;AAAA,IACZ,sBAAsB,YAAY,SAAS,SAAS;AAAA,IACpD,4BAA4B,gBAAgB,SAAS,SAAS;AAAA,IAC9D,YACE,YAAY,SAAS,UAAU,UAAa,gBAAgB,SAAS,UAAU,SAC3E,eAAe,QAAQ,QAAQ,YAAY,QAAQ,QACnD;AAAA,EACR;AACA,SAAO,EAAE,qBAAqB,UAAU,OAAO,eAAe,UAAU,MAAM;AAChF;AAEA,SAAS,cAAc,MAAsB,UAAkD;AAC7F,MAAI,SAAS,SAAS,gBAAgB,KAAK,KAAK,SAAS,OAAO;AAC9D,UAAM,MAAM,KAAK;AACjB,WAAO,EAAE,GAAG,MAAM,MAAM,EAAE,GAAG,KAAK,OAAO,SAAS,SAAS,EAAE;AAAA,EAC/D;AACA,MAAI,SAAS,SAAS,sBAAsB,KAAK,KAAK,SAAS,QAAQ;AACrE,UAAM,OAAO,KAAK;AAClB,WAAO,EAAE,GAAG,MAAM,MAAM,EAAE,GAAG,MAAM,QAAQ,SAAS,UAAU,EAAE;AAAA,EAClE;AACA,MAAI,SAAS,SAAS,2BAA2B,KAAK,KAAK,SAAS,OAAO;AACzE,UAAM,MAAM,KAAK;AACjB,WAAO;AAAA,MACL,GAAG;AAAA,MACH,MAAM;AAAA,QACJ,GAAG;AAAA,QACH,UAAU,CAAC,EAAE,MAAM,UAAU,SAAS,SAAS,QAAQ,GAAG,GAAG,IAAI,QAAQ;AAAA,MAC3E;AAAA,IACF;AAAA,EACF;AACA,MAAI,SAAS,SAAS,SAAU,QAAO,SAAS,MAAM,IAAI;AAE1D,SAAO;AACT;AAQO,SAAS,yBAAyB,SAKtC;AACD,QAAM,UAAU,oBAAI,IAAoC;AACxD,aAAW,KAAK,SAAS;AACvB,UAAM,MAAM,QAAQ,IAAI,EAAE,SAAS,IAAI,KAAK,CAAC;AAC7C,QAAI,KAAK,CAAC;AACV,YAAQ,IAAI,EAAE,SAAS,MAAM,GAAG;AAAA,EAClC;AACA,QAAM,MAKD,CAAC;AACN,aAAW,CAAC,MAAM,KAAK,KAAK,SAAS;AACnC,UAAM,SAAS,MACZ,IAAI,CAAC,MAAM,EAAE,MAAM,UAAU,EAC7B,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACnD,QAAI,OAAO,WAAW,EAAG;AACzB,UAAM,UAAU,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,KAAK,IAAI,CAAC,GAAG,CAAC,IAAI,OAAO;AACrE,UAAM,aAAa,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,OAAO;AAC9D,QAAI,KAAK;AAAA,MACP,cAAc;AAAA,MACd,GAAG,OAAO;AAAA,MACV,cAAc;AAAA,MACd,iBAAiB;AAAA,IACnB,CAAC;AAAA,EACH;AACA,SAAO,IAAI,KAAK,CAAC,GAAG,MAAM,EAAE,eAAe,EAAE,YAAY;AAC3D;","names":[]}