@tangle-network/agent-eval 0.23.1 → 0.25.0

package/dist/prm/index.d.ts

@@ -0,0 +1,99 @@
+import { P as PrmGradedTrace, S as StepRubric, a as PrmGrader } from '../rubric-D5tjHNJQ.js';
+export { G as GradedStep, b as StepContext, i as isPrmVerdict } from '../rubric-D5tjHNJQ.js';
+import { S as Span, T as TraceStore } from '../store-Db2Bv8Cf.js';
+import '../trajectory-CnoBo-JY.js';
+
+/**
+ * Export PRM-graded traces as training data for downstream reward-model
+ * fine-tuning. Canonical format is NDJSON of
+ * `{ trajectory_text, step_index, rubric, score }` so a small model can
+ * learn to predict step rewards from step context.
+ *
+ * The framework doesn't train the model — we emit the data; callers
+ * plug it into their preferred trainer (TRL, Unsloth, custom).
+ */
+
+interface PrmTrainingSample {
+    runId: string;
+    spanId: string;
+    rubricId: string;
+    score: number;
+    /** Serialized step context — step + surrounding conversation. */
+    context: {
+        priorTurns: Array<{
+            role: string;
+            content: string;
+        }>;
+        step: {
+            kind: Span['kind'];
+            text: string;
+        };
+    };
+    /** Optional evidence + rationale for auditability. */
+    rationale?: string;
+    evidence?: string;
+}
+declare function exportTrainingData(store: TraceStore, graded: PrmGradedTrace[], options?: {
+    contextWindow?: number;
+}): Promise<PrmTrainingSample[]>;
+/** NDJSON serialization — write to file or stream directly to a trainer. */
+declare function toNdjson(samples: PrmTrainingSample[]): string;
+
+/**
+ * Built-in reference rubrics. Consumers combine these with domain
+ * rubrics. All are deterministic, rule-based — cheap to run + easy
+ * to unit-test. LLM-based rubrics are trivially authored by
+ * following the StepRubric contract.
+ */
+
+/** Penalize very short or very long assistant outputs. */
+declare function outputLengthRubric(args?: {
+    minChars?: number;
+    maxChars?: number;
+    weight?: number;
+}): StepRubric;
+/** Reward tool calls that succeeded (status='ok') with an informative result. */
+declare function toolSuccessRubric(args?: {
+    weight?: number;
+}): StepRubric;
+/** Penalize tool calls that duplicate a prior call with identical args. */
+declare function toolNonRedundantRubric(args?: {
+    weight?: number;
+}): StepRubric;
+/** Penalize LLM outputs that contain common refusal markers when a refusal
+ *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */
+declare function nonRefusalRubric(args?: {
+    markers?: RegExp[];
+    weight?: number;
+}): StepRubric;
+/** Reward outputs that invoke the next-step tool the trajectory actually uses
+ *  (i.e. the LLM span announced "I will call X" and the following tool span IS X). */
+declare function toolIntentAlignmentRubric(args?: {
+    weight?: number;
+}): StepRubric;
+
+/**
+ * Inference-time PRM scoring — pick the best of N candidate trajectories
+ * using a trained reward model (or a rule-based PRM as a proxy).
+ *
+ * The canonical Best-of-N pattern: generate N completions, score each
+ * with a PRM, pick the winner. Here the scoring loop is framework-agnostic
+ * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.
+ */
+
+interface BestOfNResult {
+    winner: PrmGradedTrace;
+    ranked: PrmGradedTrace[];
+    /** Standard deviation of aggregate scores — small = candidates were homogenous. */
+    stdDev: number;
+}
+declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[]): Promise<BestOfNResult>;
+/**
+ * Weighted vote across multiple graders — use when you want a PRM ensemble
+ * (e.g. rule-based + LLM-based + trained model). Each grader produces its
+ * own ranking; we aggregate via rank-sum (Borda count) so no single grader
+ * dominates via a different score scale.
+ */
+declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[]): Promise<BestOfNResult>;
+
+export { type BestOfNResult, PrmGradedTrace, PrmGrader, type PrmTrainingSample, StepRubric, exportTrainingData, nonRefusalRubric, outputLengthRubric, prmBestOfN, prmEnsembleBestOfN, toNdjson, toolIntentAlignmentRubric, toolNonRedundantRubric, toolSuccessRubric };

package/dist/prm/index.js

@@ -0,0 +1,222 @@
+import {
+  exportTrainingData,
+  toNdjson
+} from "../chunk-KMPRBJK4.js";
+import {
+  buildTrajectory
+} from "../chunk-RZTMDUO7.js";
+import "../chunk-5BKGXME7.js";
+import {
+  TraceEmitter
+} from "../chunk-TVVP3ZZQ.js";
+import "../chunk-PZ5AY32C.js";
+
+// src/prm/builtin-rubrics.ts
+function outputLengthRubric(args = {}) {
+  const min = args.minChars ?? 20;
+  const max = args.maxChars ?? 8e3;
+  return {
+    id: "output-length",
+    kinds: ["llm"],
+    weight: args.weight ?? 0.5,
+    async grade({ step }) {
+      const llm = step.span;
+      const len = (llm.output ?? "").length;
+      if (len === 0) return { score: 0, rationale: "empty output" };
+      if (len < min)
+        return { score: Math.max(0, len / min), rationale: `below min (${len} < ${min})` };
+      if (len > max)
+        return {
+          score: Math.max(0, 1 - (len - max) / max),
+          rationale: `above max (${len} > ${max})`
+        };
+      return { score: 1, rationale: `${len} chars in bounds` };
+    }
+  };
+}
+function toolSuccessRubric(args = {}) {
+  return {
+    id: "tool-success",
+    kinds: ["tool"],
+    weight: args.weight ?? 1,
+    async grade({ step }) {
+      const tool = step.span;
+      if (tool.status === "error")
+        return { score: 0, rationale: `error: ${tool.error ?? "unknown"}` };
+      const r = tool.result;
+      if (r === null || r === void 0) return { score: 0.3, rationale: "empty result" };
+      const asText = typeof r === "string" ? r : JSON.stringify(r);
+      if (asText.length < 4) return { score: 0.5, rationale: "tiny result" };
+      return { score: 1, rationale: `${tool.toolName} ok` };
+    }
+  };
+}
+function toolNonRedundantRubric(args = {}) {
+  const weight = args.weight ?? 0.5;
+  return {
+    id: "tool-non-redundant",
+    kinds: ["tool"],
+    weight,
+    async grade({ step, prior }) {
+      const tool = step.span;
+      const priorMatches = prior.filter((p) => {
+        if (p.span.kind !== "tool") return false;
+        const pt = p.span;
+        return pt.toolName === tool.toolName && stableStringify(pt.args) === stableStringify(tool.args);
+      });
+      if (priorMatches.length === 0) return { score: 1, rationale: "novel call" };
+      return {
+        score: Math.max(0, 1 - priorMatches.length * 0.5),
+        rationale: `${priorMatches.length} duplicate(s)`
+      };
+    }
+  };
+}
+function nonRefusalRubric(args = {}) {
+  const weight = args.weight ?? 1;
+  const markers = args.markers ?? [
+    /\bi\s+(?:can(?:not|'t)|won't|will\s+not)\b/i,
+    /\b(?:as\s+an?\s+)?ai\b.*?\b(?:can't|cannot)\b/i
+  ];
+  return {
+    id: "non-refusal",
+    kinds: ["llm"],
+    weight,
+    async grade({ step }) {
+      const llm = step.span;
+      const out = llm.output ?? "";
+      const refused = markers.some((re) => re.test(out));
+      return refused ? { score: 0, rationale: "refusal marker present" } : { score: 1, rationale: "no refusal" };
+    }
+  };
+}
+function toolIntentAlignmentRubric(args = {}) {
+  return {
+    id: "tool-intent-alignment",
+    kinds: ["llm"],
+    weight: args.weight ?? 0.5,
+    async grade({ step, next }) {
+      const llm = step.span;
+      const nextTool = next.find((s) => s.span.kind === "tool");
+      if (!nextTool) return null;
+      const toolName = nextTool.span.toolName;
+      const out = (llm.output ?? "").toLowerCase();
+      const mentioned = out.includes(toolName.toLowerCase());
+      return mentioned ? { score: 1, rationale: `mentioned "${toolName}" before calling it` } : { score: 0.5, rationale: `called "${toolName}" without announcing it` };
+    }
+  };
+}
+function stableStringify(value) {
+  if (value === null || typeof value !== "object") return JSON.stringify(value);
+  if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`;
+  const keys = Object.keys(value).sort();
+  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify(value[k])}`).join(",")}}`;
+}
+
+// src/prm/inference.ts
+async function prmBestOfN(store, grader, runIds) {
+  if (runIds.length === 0) throw new Error("prmBestOfN: at least 1 candidate required");
+  const graded = await Promise.all(runIds.map((id) => grader.grade(store, id)));
+  const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore);
+  const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length;
+  const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length;
+  return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance) };
+}
+async function prmEnsembleBestOfN(store, graders, runIds) {
+  if (graders.length === 0) throw new Error("prmEnsembleBestOfN: at least 1 grader");
+  const perGrader = await Promise.all(
+    graders.map(async (g) => {
+      const graded = await Promise.all(runIds.map((id) => g.grade(store, id)));
+      return graded.sort((a, b) => b.aggregateScore - a.aggregateScore);
+    })
+  );
+  const bordaScores = /* @__PURE__ */ new Map();
+  for (const ranking of perGrader) {
+    ranking.forEach((g, rank) => {
+      bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank));
+    });
+  }
+  const canonical = perGrader[0];
+  const byRun = new Map(canonical.map((g) => [g.runId, g]));
+  const ranked = [...byRun.values()].sort(
+    (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0)
+  );
+  const mean = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length;
+  const variance = ranked.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / ranked.length;
+  return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance) };
+}
+
+// src/prm/rubric.ts
+var PrmGrader = class {
+  constructor(rubrics) {
+    this.rubrics = rubrics;
+    if (rubrics.length === 0) throw new Error("PrmGrader: at least 1 rubric required");
+  }
+  rubrics;
+  /**
+   * Grade every eligible span in a run. Emits a JudgeVerdict span for each
+   * (rubric × span) verdict so the result is visible to downstream pipelines
+   * (judgeAgreementView, etc.) — PRM is just "a judge that runs per span."
+   */
+  async grade(store, runId) {
+    const trajectory = await buildTrajectory(store, runId);
+    const emitter = new TraceEmitter(store, { runId });
+    const steps = [];
+    let ungraded = 0;
+    for (let i = 0; i < trajectory.steps.length; i++) {
+      const step = trajectory.steps[i];
+      const ctx = {
+        trajectory,
+        step,
+        prior: trajectory.steps.slice(0, i),
+        next: trajectory.steps.slice(i + 1)
+      };
+      let gradedThis = false;
+      for (const rubric of this.rubrics) {
+        if (rubric.kinds && !rubric.kinds.includes(step.span.kind)) continue;
+        const verdict = await rubric.grade(ctx);
+        if (verdict === null) continue;
+        const weight = rubric.weight ?? 1;
+        steps.push({
+          spanId: step.span.spanId,
+          rubricId: rubric.id,
+          score: verdict.score,
+          weight,
+          rationale: verdict.rationale,
+          evidence: verdict.evidence
+        });
+        gradedThis = true;
+        await emitter.recordJudge({
+          judgeId: `prm:${rubric.id}`,
+          targetSpanId: step.span.spanId,
+          dimension: "step_quality",
+          score: verdict.score,
+          rationale: verdict.rationale,
+          evidence: verdict.evidence,
+          name: `prm:${rubric.id}`
+        });
+      }
+      if (!gradedThis) ungraded++;
+    }
+    const totalWeight = steps.reduce((a, s) => a + s.weight, 0);
+    const aggregateScore = totalWeight === 0 ? 0 : steps.reduce((a, s) => a + s.score * s.weight, 0) / totalWeight;
+    return { runId, steps, aggregateScore, gradedCount: steps.length, ungradedCount: ungraded };
+  }
+};
+function isPrmVerdict(verdict) {
+  return verdict.judgeId.startsWith("prm:");
+}
+export {
+  PrmGrader,
+  exportTrainingData,
+  isPrmVerdict,
+  nonRefusalRubric,
+  outputLengthRubric,
+  prmBestOfN,
+  prmEnsembleBestOfN,
+  toNdjson,
+  toolIntentAlignmentRubric,
+  toolNonRedundantRubric,
+  toolSuccessRubric
+};
+//# sourceMappingURL=index.js.map

package/dist/prm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/prm/builtin-rubrics.ts","../../src/prm/inference.ts","../../src/prm/rubric.ts"],"sourcesContent":["/**\n * Built-in reference rubrics. Consumers combine these with domain\n * rubrics. All are deterministic, rule-based — cheap to run + easy\n * to unit-test. LLM-based rubrics are trivially authored by\n * following the StepRubric contract.\n */\n\nimport type { LlmSpan, ToolSpan } from '../trace/schema'\nimport type { StepRubric } from './rubric'\n\n/** Penalize very short or very long assistant outputs. */\nexport function outputLengthRubric(\n  args: { minChars?: number; maxChars?: number; weight?: number } = {},\n): StepRubric {\n  const min = args.minChars ?? 20\n  const max = args.maxChars ?? 8000\n  return {\n    id: 'output-length',\n    kinds: ['llm'],\n    weight: args.weight ?? 0.5,\n    async grade({ step }) {\n      const llm = step.span as LlmSpan\n      const len = (llm.output ?? '').length\n      if (len === 0) return { score: 0, rationale: 'empty output' }\n      if (len < min)\n        return { score: Math.max(0, len / min), rationale: `below min (${len} < ${min})` }\n      if (len > max)\n        return {\n          score: Math.max(0, 1 - (len - max) / max),\n          rationale: `above max (${len} > ${max})`,\n        }\n      return { score: 1, rationale: `${len} chars in bounds` }\n    },\n  }\n}\n\n/** Reward tool calls that succeeded (status='ok') with an informative result. */\nexport function toolSuccessRubric(args: { weight?: number } = {}): StepRubric {\n  return {\n    id: 'tool-success',\n    kinds: ['tool'],\n    weight: args.weight ?? 1,\n    async grade({ step }) {\n      const tool = step.span as ToolSpan\n      if (tool.status === 'error')\n        return { score: 0, rationale: `error: ${tool.error ?? 'unknown'}` }\n      const r = tool.result\n      if (r === null || r === undefined) return { score: 0.3, rationale: 'empty result' }\n      const asText = typeof r === 'string' ? r : JSON.stringify(r)\n      if (asText.length < 4) return { score: 0.5, rationale: 'tiny result' }\n      return { score: 1, rationale: `${tool.toolName} ok` }\n    },\n  }\n}\n\n/** Penalize tool calls that duplicate a prior call with identical args. */\nexport function toolNonRedundantRubric(args: { weight?: number } = {}): StepRubric {\n  const weight = args.weight ?? 0.5\n  return {\n    id: 'tool-non-redundant',\n    kinds: ['tool'],\n    weight,\n    async grade({ step, prior }) {\n      const tool = step.span as ToolSpan\n      const priorMatches = prior.filter((p) => {\n        if (p.span.kind !== 'tool') return false\n        const pt = p.span as ToolSpan\n        return (\n          pt.toolName === tool.toolName && stableStringify(pt.args) === stableStringify(tool.args)\n        )\n      })\n      if (priorMatches.length === 0) return { score: 1, rationale: 'novel call' }\n      return {\n        score: Math.max(0, 1 - priorMatches.length * 0.5),\n        rationale: `${priorMatches.length} duplicate(s)`,\n      }\n    },\n  }\n}\n\n/** Penalize LLM outputs that contain common refusal markers when a refusal\n *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */\nexport function nonRefusalRubric(args: { markers?: RegExp[]; weight?: number } = {}): StepRubric {\n  const weight = args.weight ?? 1\n  const markers = args.markers ?? [\n    /\\bi\\s+(?:can(?:not|'t)|won't|will\\s+not)\\b/i,\n    /\\b(?:as\\s+an?\\s+)?ai\\b.*?\\b(?:can't|cannot)\\b/i,\n  ]\n  return {\n    id: 'non-refusal',\n    kinds: ['llm'],\n    weight,\n    async grade({ step }) {\n      const llm = step.span as LlmSpan\n      const out = llm.output ?? ''\n      const refused = markers.some((re) => re.test(out))\n      return refused\n        ? { score: 0, rationale: 'refusal marker present' }\n        : { score: 1, rationale: 'no refusal' }\n    },\n  }\n}\n\n/** Reward outputs that invoke the next-step tool the trajectory actually uses\n *  (i.e. the LLM span announced \"I will call X\" and the following tool span IS X). */\nexport function toolIntentAlignmentRubric(args: { weight?: number } = {}): StepRubric {\n  return {\n    id: 'tool-intent-alignment',\n    kinds: ['llm'],\n    weight: args.weight ?? 0.5,\n    async grade({ step, next }) {\n      const llm = step.span as LlmSpan\n      const nextTool = next.find((s) => s.span.kind === 'tool')\n      if (!nextTool) return null\n      const toolName = (nextTool.span as ToolSpan).toolName\n      const out = (llm.output ?? '').toLowerCase()\n      const mentioned = out.includes(toolName.toLowerCase())\n      return mentioned\n        ? { score: 1, rationale: `mentioned \"${toolName}\" before calling it` }\n        : { score: 0.5, rationale: `called \"${toolName}\" without announcing it` }\n    },\n  }\n}\n\nfunction stableStringify(value: unknown): string {\n  if (value === null || typeof value !== 'object') return JSON.stringify(value)\n  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`\n  const keys = Object.keys(value as Record<string, unknown>).sort()\n  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify((value as Record<string, unknown>)[k])}`).join(',')}}`\n}\n","/**\n * Inference-time PRM scoring — pick the best of N candidate trajectories\n * using a trained reward model (or a rule-based PRM as a proxy).\n *\n * The canonical Best-of-N pattern: generate N completions, score each\n * with a PRM, pick the winner. Here the scoring loop is framework-agnostic\n * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.\n */\n\nimport type { TraceStore } from '../trace/store'\nimport type { PrmGradedTrace, PrmGrader } from './rubric'\n\nexport interface BestOfNResult {\n  winner: PrmGradedTrace\n  ranked: PrmGradedTrace[]\n  /** Standard deviation of aggregate scores — small = candidates were homogenous. */\n  stdDev: number\n}\n\nexport async function prmBestOfN(\n  store: TraceStore,\n  grader: PrmGrader,\n  runIds: string[],\n): Promise<BestOfNResult> {\n  if (runIds.length === 0) throw new Error('prmBestOfN: at least 1 candidate required')\n  const graded = await Promise.all(runIds.map((id) => grader.grade(store, id)))\n  const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore)\n  const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length\n  const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length\n  return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) }\n}\n\n/**\n * Weighted vote across multiple graders — use when you want a PRM ensemble\n * (e.g. rule-based + LLM-based + trained model). Each grader produces its\n * own ranking; we aggregate via rank-sum (Borda count) so no single grader\n * dominates via a different score scale.\n */\nexport async function prmEnsembleBestOfN(\n  store: TraceStore,\n  graders: PrmGrader[],\n  runIds: string[],\n): Promise<BestOfNResult> {\n  if (graders.length === 0) throw new Error('prmEnsembleBestOfN: at least 1 grader')\n  const perGrader = await Promise.all(\n    graders.map(async (g) => {\n      const graded = await Promise.all(runIds.map((id) => g.grade(store, id)))\n      return graded.sort((a, b) => b.aggregateScore - a.aggregateScore)\n    }),\n  )\n  // Borda: rank-sum across graders.\n  const bordaScores = new Map<string, number>()\n  for (const ranking of perGrader) {\n    ranking.forEach((g, rank) => {\n      bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank))\n    })\n  }\n  // Return a synthesized ranking using the first grader's graded traces\n  // ordered by Borda score. aggregateScore field kept for UX.\n  const canonical = perGrader[0]!\n  const byRun = new Map(canonical.map((g) => [g.runId, g]))\n  const ranked = [...byRun.values()].sort(\n    (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0),\n  )\n  const mean = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length\n  const variance = ranked.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / ranked.length\n  return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) }\n}\n","/**\n * Process Reward Modeling — per-step rubric grading.\n *\n * A StepRubric inspects one span and returns a score + rationale.\n * PrmGrader applies an array of rubrics to every LLM span in a\n * trajectory (consumers can broaden to tool/retrieval spans via the\n * `kind` filter on each rubric).\n *\n * Why this matters: outcome-only eval (did the final artifact work?)\n * gives sparse reward — most agent turns are unattributable. PRMs\n * densify the signal so optimizers and RL fine-tuning can assign\n * credit per turn.\n */\n\nimport { TraceEmitter } from '../trace/emitter'\nimport type { JudgeSpan, Span } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory'\n\nexport interface StepContext {\n  trajectory: Trajectory\n  step: TrajectoryStep\n  /** Steps preceding `step` in trajectory order. */\n  prior: TrajectoryStep[]\n  /** Steps following `step`. */\n  next: TrajectoryStep[]\n}\n\nexport interface StepRubric {\n  id: string\n  /** Only grade spans of these kinds (default: all). */\n  kinds?: Array<Span['kind']>\n  /** Weight in the aggregate score. Default 1. */\n  weight?: number\n  /** Returns score in 0..1 + optional rationale/evidence. Return `null` to\n   *  skip grading (rubric doesn't apply to this step). */\n  grade: (\n    ctx: StepContext,\n  ) => Promise<{ score: number; rationale?: string; evidence?: string } | null>\n}\n\nexport interface GradedStep {\n  spanId: string\n  rubricId: string\n  score: number\n  weight: number\n  rationale?: string\n  evidence?: string\n}\n\nexport interface PrmGradedTrace {\n  runId: string\n  steps: GradedStep[]\n  /** Weighted mean of all graded steps; 0..1. */\n  aggregateScore: number\n  /** Number of spans graded — useful for sanity-checking coverage. */\n  gradedCount: number\n  /** Number of spans in the trajectory that no rubric matched. */\n  ungradedCount: number\n}\n\nexport class PrmGrader {\n  constructor(private rubrics: StepRubric[]) {\n    if (rubrics.length === 0) throw new Error('PrmGrader: at least 1 rubric required')\n  }\n\n  /**\n   * Grade every eligible span in a run. Emits a JudgeVerdict span for each\n   * (rubric × span) verdict so the result is visible to downstream pipelines\n   * (judgeAgreementView, etc.) — PRM is just \"a judge that runs per span.\"\n   */\n  async grade(store: TraceStore, runId: string): Promise<PrmGradedTrace> {\n    const trajectory = await buildTrajectory(store, runId)\n    const emitter = new TraceEmitter(store, { runId })\n    const steps: GradedStep[] = []\n    let ungraded = 0\n    for (let i = 0; i < trajectory.steps.length; i++) {\n      const step = trajectory.steps[i]!\n      const ctx: StepContext = {\n        trajectory,\n        step,\n        prior: trajectory.steps.slice(0, i),\n        next: trajectory.steps.slice(i + 1),\n      }\n      let gradedThis = false\n      for (const rubric of this.rubrics) {\n        if (rubric.kinds && !rubric.kinds.includes(step.span.kind)) continue\n        const verdict = await rubric.grade(ctx)\n        if (verdict === null) continue\n        const weight = rubric.weight ?? 1\n        steps.push({\n          spanId: step.span.spanId,\n          rubricId: rubric.id,\n          score: verdict.score,\n          weight,\n          rationale: verdict.rationale,\n          evidence: verdict.evidence,\n        })\n        gradedThis = true\n        // Persist the verdict as a JudgeSpan so the query pipelines see it\n        await emitter.recordJudge({\n          judgeId: `prm:${rubric.id}`,\n          targetSpanId: step.span.spanId,\n          dimension: 'step_quality',\n          score: verdict.score,\n          rationale: verdict.rationale,\n          evidence: verdict.evidence,\n          name: `prm:${rubric.id}`,\n        })\n      }\n      if (!gradedThis) ungraded++\n    }\n\n    const totalWeight = steps.reduce((a, s) => a + s.weight, 0)\n    const aggregateScore =\n      totalWeight === 0 ? 0 : steps.reduce((a, s) => a + s.score * s.weight, 0) / totalWeight\n\n    return { runId, steps, aggregateScore, gradedCount: steps.length, ungradedCount: ungraded }\n  }\n}\n\n/** Helper: reads JudgeVerdict spans that PRM emitted so downstream pipelines\n *  can distinguish PRM verdicts from human or top-level LLM judges. */\nexport function isPrmVerdict(verdict: JudgeSpan): boolean {\n  return verdict.judgeId.startsWith('prm:')\n}\n"],"mappings":";;;;;;;;;;;;;;AAWO,SAAS,mBACd,OAAkE,CAAC,GACvD;AACZ,QAAM,MAAM,KAAK,YAAY;AAC7B,QAAM,MAAM,KAAK,YAAY;AAC7B,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,MAAM,KAAK;AACjB,YAAM,OAAO,IAAI,UAAU,IAAI;AAC/B,UAAI,QAAQ,EAAG,QAAO,EAAE,OAAO,GAAG,WAAW,eAAe;AAC5D,UAAI,MAAM;AACR,eAAO,EAAE,OAAO,KAAK,IAAI,GAAG,MAAM,GAAG,GAAG,WAAW,cAAc,GAAG,MAAM,GAAG,IAAI;AACnF,UAAI,MAAM;AACR,eAAO;AAAA,UACL,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,GAAG;AAAA,UACxC,WAAW,cAAc,GAAG,MAAM,GAAG;AAAA,QACvC;AACF,aAAO,EAAE,OAAO,GAAG,WAAW,GAAG,GAAG,mBAAmB;AAAA,IACzD;AAAA,EACF;AACF;AAGO,SAAS,kBAAkB,OAA4B,CAAC,GAAe;AAC5E,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,MAAM;AAAA,IACd,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,OAAO,KAAK;AAClB,UAAI,KAAK,WAAW;AAClB,eAAO,EAAE,OAAO,GAAG,WAAW,UAAU,KAAK,SAAS,SAAS,GAAG;AACpE,YAAM,IAAI,KAAK;AACf,UAAI,MAAM,QAAQ,MAAM,OAAW,QAAO,EAAE,OAAO,KAAK,WAAW,eAAe;AAClF,YAAM,SAAS,OAAO,MAAM,WAAW,IAAI,KAAK,UAAU,CAAC;AAC3D,UAAI,OAAO,SAAS,EAAG,QAAO,EAAE,OAAO,KAAK,WAAW,cAAc;AACrE,aAAO,EAAE,OAAO,GAAG,WAAW,GAAG,KAAK,QAAQ,MAAM;AAAA,IACtD;AAAA,EACF;AACF;AAGO,SAAS,uBAAuB,OAA4B,CAAC,GAAe;AACjF,QAAM,SAAS,KAAK,UAAU;AAC9B,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,MAAM;AAAA,IACd;AAAA,IACA,MAAM,MAAM,EAAE,MAAM,MAAM,GAAG;AAC3B,YAAM,OAAO,KAAK;AAClB,YAAM,eAAe,MAAM,OAAO,CAAC,MAAM;AACvC,YAAI,EAAE,KAAK,SAAS,OAAQ,QAAO;AACnC,cAAM,KAAK,EAAE;AACb,eACE,GAAG,aAAa,KAAK,YAAY,gBAAgB,GAAG,IAAI,MAAM,gBAAgB,KAAK,IAAI;AAAA,MAE3F,CAAC;AACD,UAAI,aAAa,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,WAAW,aAAa;AAC1E,aAAO;AAAA,QACL,OAAO,KAAK,IAAI,GAAG,IAAI,aAAa,SAAS,GAAG;AAAA,QAChD,WAAW,GAAG,aAAa,MAAM;AAAA,MACnC;AAAA,IACF;AAAA,EACF;AACF;AAIO,SAAS,iBAAiB,OAAgD,CAAC,GAAe;AAC/F,QAAM,SAAS,KAAK,UAAU;AAC9B,QAAM,UAAU,KAAK,WAAW;AAAA,IAC9B;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb;AAAA,IACA,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,MAAM,KAAK;AACjB,YAAM,MAAM,IAAI,UAAU;AAC1B,YAAM,UAAU,QAAQ,KAAK,CAAC,OAAO,GAAG,KAAK,GAAG,CAAC;AACjD,aAAO,UACH,EAAE,OAAO,GAAG,WAAW,yBAAyB,IAChD,EAAE,OAAO,GAAG,WAAW,aAAa;AAAA,IAC1C;AAAA,EACF;AACF;AAIO,SAAS,0BAA0B,OAA4B,CAAC,GAAe;AACpF,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,MAAM,KAAK,GAAG;AAC1B,YAAM,MAAM,KAAK;AACjB,YAAM,WAAW,KAAK,KAAK,CAAC,MAAM,EAAE,KAAK,SAAS,MAAM;AACxD,UAAI,CAAC,SAAU,QAAO;AACtB,YAAM,WAAY,SAAS,KAAkB;AAC7C,YAAM,OAAO,IAAI,UAAU,IAAI,YAAY;AAC3C,YAAM,YAAY,IAAI,SAAS,SAAS,YAAY,CAAC;AACrD,aAAO,YACH,EAAE,OAAO,GAAG,WAAW,cAAc,QAAQ,sBAAsB,IACnE,EAAE,OAAO,KAAK,WAAW,WAAW,QAAQ,0BAA0B;AAAA,IAC5E;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,OAAwB;AAC/C,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK;AAC5E,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,IAAI,MAAM,IAAI,eAAe,EAAE,KAAK,GAAG,CAAC;AACzE,QAAM,OAAO,OAAO,KAAK,KAAgC,EAAE,KAAK;AAChE,SAAO,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,gBAAiB,MAAkC,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC;AACxH;;;AC9GA,eAAsB,WACpB,OACA,QACA,QACwB;AACxB,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,2CAA2C;AACpF,QAAM,SAAS,MAAM,QAAQ,IAAI,OAAO,IAAI,CAAC,OAAO,OAAO,MAAM,OAAO,EAAE,CAAC,CAAC;AAC5E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAC7E,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,gBAAgB,CAAC,IAAI,OAAO;AACvE,QAAM,WAAW,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,iBAAiB,SAAS,GAAG,CAAC,IAAI,OAAO;AACzF,SAAO,EAAE,QAAQ,OAAO,CAAC,GAAI,QAAQ,QAAQ,KAAK,KAAK,QAAQ,EAAE;AACnE;AAQA,eAAsB,mBACpB,OACA,SACA,QACwB;AACxB,MAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC;AACjF,QAAM,YAAY,MAAM,QAAQ;AAAA,IAC9B,QAAQ,IAAI,OAAO,MAAM;AACvB,YAAM,SAAS,MAAM,QAAQ,IAAI,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,OAAO,EAAE,CAAC,CAAC;AACvE,aAAO,OAAO,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAAA,IAClE,CAAC;AAAA,EACH;AAEA,QAAM,cAAc,oBAAI,IAAoB;AAC5C,aAAW,WAAW,WAAW;AAC/B,YAAQ,QAAQ,CAAC,GAAG,SAAS;AAC3B,kBAAY,IAAI,EAAE,QAAQ,YAAY,IAAI,EAAE,KAAK,KAAK,MAAM,QAAQ,SAAS,KAAK;AAAA,IACpF,CAAC;AAAA,EACH;AAGA,QAAM,YAAY,UAAU,CAAC;AAC7B,QAAM,QAAQ,IAAI,IAAI,UAAU,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AACxD,QAAM,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,EAAE;AAAA,IACjC,CAAC,GAAG,OAAO,YAAY,IAAI,EAAE,KAAK,KAAK,MAAM,YAAY,IAAI,EAAE,KAAK,KAAK;AAAA,EAC3E;AACA,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,gBAAgB,CAAC,IAAI,OAAO;AACvE,QAAM,WAAW,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,iBAAiB,SAAS,GAAG,CAAC,IAAI,OAAO;AACzF,SAAO,EAAE,QAAQ,OAAO,CAAC,GAAI,QAAQ,QAAQ,KAAK,KAAK,QAAQ,EAAE;AACnE;;;ACNO,IAAM,YAAN,MAAgB;AAAA,EACrB,YAAoB,SAAuB;AAAvB;AAClB,QAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC;AAAA,EACnF;AAAA,EAFoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASpB,MAAM,MAAM,OAAmB,OAAwC;AACrE,UAAM,aAAa,MAAM,gBAAgB,OAAO,KAAK;AACrD,UAAM,UAAU,IAAI,aAAa,OAAO,EAAE,MAAM,CAAC;AACjD,UAAM,QAAsB,CAAC;AAC7B,QAAI,WAAW;AACf,aAAS,IAAI,GAAG,IAAI,WAAW,MAAM,QAAQ,KAAK;AAChD,YAAM,OAAO,WAAW,MAAM,CAAC;AAC/B,YAAM,MAAmB;AAAA,QACvB;AAAA,QACA;AAAA,QACA,OAAO,WAAW,MAAM,MAAM,GAAG,CAAC;AAAA,QAClC,MAAM,WAAW,MAAM,MAAM,IAAI,CAAC;AAAA,MACpC;AACA,UAAI,aAAa;AACjB,iBAAW,UAAU,KAAK,SAAS;AACjC,YAAI,OAAO,SAAS,CAAC,OAAO,MAAM,SAAS,KAAK,KAAK,IAAI,EAAG;AAC5D,cAAM,UAAU,MAAM,OAAO,MAAM,GAAG;AACtC,YAAI,YAAY,KAAM;AACtB,cAAM,SAAS,OAAO,UAAU;AAChC,cAAM,KAAK;AAAA,UACT,QAAQ,KAAK,KAAK;AAAA,UAClB,UAAU,OAAO;AAAA,UACjB,OAAO,QAAQ;AAAA,UACf;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB,UAAU,QAAQ;AAAA,QACpB,CAAC;AACD,qBAAa;AAEb,cAAM,QAAQ,YAAY;AAAA,UACxB,SAAS,OAAO,OAAO,EAAE;AAAA,UACzB,cAAc,KAAK,KAAK;AAAA,UACxB,WAAW;AAAA,UACX,OAAO,QAAQ;AAAA,UACf,WAAW,QAAQ;AAAA,UACnB,UAAU,QAAQ;AAAA,UAClB,MAAM,OAAO,OAAO,EAAE;AAAA,QACxB,CAAC;AAAA,MACH;AACA,UAAI,CAAC,WAAY;AAAA,IACnB;AAEA,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,CAAC;AAC1D,UAAM,iBACJ,gBAAgB,IAAI,IAAI,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI;AAE9E,WAAO,EAAE,OAAO,OAAO,gBAAgB,aAAa,MAAM,QAAQ,eAAe,SAAS;AAAA,EAC5F;AACF;AAIO,SAAS,aAAa,SAA6B;AACxD,SAAO,QAAQ,QAAQ,WAAW,MAAM;AAC1C;","names":[]}

package/dist/query-DODUYdPg.d.ts

@@ -0,0 +1,30 @@
+import { L as LlmSpan, T as TraceStore, J as JudgeSpan, R as Run, F as FailureClass, a as ToolSpan } from './store-Db2Bv8Cf.js';
+
+/**
+ * Typed query helpers over TraceStore.
+ *
+ * Not a full SQL engine — a minimal, composable set of operators that
+ * cover the canned-pipeline use cases. For ad-hoc analytics, persist to
+ * NDJSON and point DuckDB at it; the schema is stable so external SQL
+ * tooling works out of the box.
+ */
+
+declare function runsForScenario(store: TraceStore, scenarioId: string): Promise<Run[]>;
+declare function llmSpans(store: TraceStore, runId?: string): Promise<LlmSpan[]>;
+declare function toolSpans(store: TraceStore, runId?: string, toolName?: string): Promise<ToolSpan[]>;
+declare function judgeSpans(store: TraceStore, runId?: string): Promise<JudgeSpan[]>;
+/** Group spans by any key selector. */
+declare function groupBy<T, K extends string | number>(items: T[], key: (t: T) => K): Map<K, T[]>;
+/** Hash tool arguments to an orderless-key-stable string for de-duplication. */
+declare function argHash(args: unknown): string;
+/** Sum an LLM-span array into aggregate token + cost. */
+declare function aggregateLlm(spans: LlmSpan[]): {
+    inputTokens: number;
+    outputTokens: number;
+    cachedTokens: number;
+    costUsd: number;
+};
+/** Pick the outcome's failure class when present, else derive 'success' from run status. */
+declare function runFailureClass(run: Run): FailureClass;
+
+export { aggregateLlm as a, argHash as b, runsForScenario as c, groupBy as g, judgeSpans as j, llmSpans as l, runFailureClass as r, toolSpans as t };