@tangle-network/agent-eval 0.93.0 → 0.95.0

package/dist/prm/builtin-rubrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"builtin-rubrics.d.ts","sourceRoot":"","sources":["../../src/prm/builtin-rubrics.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AAE1C,0DAA0D;AAC1D,wBAAgB,kBAAkB,CAChC,IAAI,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GACnE,UAAU,CAqBZ;AAED,iFAAiF;AACjF,wBAAgB,iBAAiB,CAAC,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,CAgB5E;AAED,2EAA2E;AAC3E,wBAAgB,sBAAsB,CAAC,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,CAsBjF;AAED;uFACuF;AACvF,wBAAgB,gBAAgB,CAAC,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,CAmB/F;AAED;sFACsF;AACtF,wBAAgB,yBAAyB,CAAC,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,CAiBpF"}

package/dist/prm/index.d.ts

@@ -1,100 +1,5 @@
-import { P as PrmGradedTrace, S as StepRubric, a as PrmGrader } from '../rubric-BOfxn4ja.js';
-export { G as GradedStep, b as StepContext, i as isPrmVerdict } from '../rubric-BOfxn4ja.js';
-import { T as TraceStore } from '../store-CKUAgsJz.js';
-import { S as Span } from '../schema-m0gsnbt3.js';
-import '../trajectory-GEdXJCL5.js';
-
-/**
- * Export PRM-graded traces as training data for downstream reward-model
- * fine-tuning. Canonical format is NDJSON of
- * `{ trajectory_text, step_index, rubric, score }` so a small model can
- * learn to predict step rewards from step context.
- *
- * The framework doesn't train the model — we emit the data; callers
- * plug it into their preferred trainer (TRL, Unsloth, custom).
- */
-
-interface PrmTrainingSample {
-    runId: string;
-    spanId: string;
-    rubricId: string;
-    score: number;
-    /** Serialized step context — step + surrounding conversation. */
-    context: {
-        priorTurns: Array<{
-            role: string;
-            content: string;
-        }>;
-        step: {
-            kind: Span['kind'];
-            text: string;
-        };
-    };
-    /** Optional evidence + rationale for auditability. */
-    rationale?: string;
-    evidence?: string;
-}
-declare function exportTrainingData(store: TraceStore, graded: PrmGradedTrace[], options?: {
-    contextWindow?: number;
-}): Promise<PrmTrainingSample[]>;
-/** NDJSON serialization — write to file or stream directly to a trainer. */
-declare function toNdjson(samples: PrmTrainingSample[]): string;
-
-/**
- * Built-in reference rubrics. Consumers combine these with domain
- * rubrics. All are deterministic, rule-based — cheap to run + easy
- * to unit-test. LLM-based rubrics are trivially authored by
- * following the StepRubric contract.
- */
-
-/** Penalize very short or very long assistant outputs. */
-declare function outputLengthRubric(args?: {
-    minChars?: number;
-    maxChars?: number;
-    weight?: number;
-}): StepRubric;
-/** Reward tool calls that succeeded (status='ok') with an informative result. */
-declare function toolSuccessRubric(args?: {
-    weight?: number;
-}): StepRubric;
-/** Penalize tool calls that duplicate a prior call with identical args. */
-declare function toolNonRedundantRubric(args?: {
-    weight?: number;
-}): StepRubric;
-/** Penalize LLM outputs that contain common refusal markers when a refusal
- *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */
-declare function nonRefusalRubric(args?: {
-    markers?: RegExp[];
-    weight?: number;
-}): StepRubric;
-/** Reward outputs that invoke the next-step tool the trajectory actually uses
- *  (i.e. the LLM span announced "I will call X" and the following tool span IS X). */
-declare function toolIntentAlignmentRubric(args?: {
-    weight?: number;
-}): StepRubric;
-
-/**
- * Inference-time PRM scoring — pick the best of N candidate trajectories
- * using a trained reward model (or a rule-based PRM as a proxy).
- *
- * The canonical Best-of-N pattern: generate N completions, score each
- * with a PRM, pick the winner. Here the scoring loop is framework-agnostic
- * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.
- */
-
-interface BestOfNResult {
-    winner: PrmGradedTrace;
-    ranked: PrmGradedTrace[];
-    /** Standard deviation of aggregate scores — small = candidates were homogenous. */
-    stdDev: number;
-}
-declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[]): Promise<BestOfNResult>;
-/**
- * Weighted vote across multiple graders — use when you want a PRM ensemble
- * (e.g. rule-based + LLM-based + trained model). Each grader produces its
- * own ranking; we aggregate via rank-sum (Borda count) so no single grader
- * dominates via a different score scale.
- */
-declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[]): Promise<BestOfNResult>;
-
-export { type BestOfNResult, PrmGradedTrace, PrmGrader, type PrmTrainingSample, StepRubric, exportTrainingData, nonRefusalRubric, outputLengthRubric, prmBestOfN, prmEnsembleBestOfN, toNdjson, toolIntentAlignmentRubric, toolNonRedundantRubric, toolSuccessRubric };
+export * from './builtin-rubrics';
+export * from './inference';
+export * from './rubric';
+export * from './training-export';
+//# sourceMappingURL=index.d.ts.map

package/dist/prm/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/prm/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA;AACjC,cAAc,aAAa,CAAA;AAC3B,cAAc,UAAU,CAAA;AACxB,cAAc,mBAAmB,CAAA"}

package/dist/prm/index.js

@@ -114,30 +114,73 @@ function stableStringify(value) {
 }
 
 // src/prm/inference.ts
-async function prmBestOfN(store, grader, runIds) {
+var DEFAULT_PRM_CONCURRENCY = 4;
+async function prmBestOfN(store, grader, runIds, options = {}) {
   if (runIds.length === 0) throw new Error("prmBestOfN: at least 1 candidate required");
-  const graded = await Promise.all(runIds.map((id) => grader.grade(store, id)));
+  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY);
+  const graded = new Array(runIds.length);
+  let cursor = 0;
+  async function worker() {
+    while (true) {
+      const i = cursor++;
+      if (i >= runIds.length) return;
+      graded[i] = await grader.grade(store, runIds[i]);
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, runIds.length) }, () => worker()));
   const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore);
   const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length;
   const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length;
   return { winner: ranked[0], ranked, stdDev: Math.sqrt(variance) };
 }
-async function prmEnsembleBestOfN(store, graders, runIds) {
+async function prmEnsembleBestOfN(store, graders, runIds, options = {}) {
   if (graders.length === 0) throw new Error("prmEnsembleBestOfN: at least 1 grader");
-  const perGrader = await Promise.all(
-    graders.map(async (g) => {
-      const graded = await Promise.all(runIds.map((id) => g.grade(store, id)));
-      return graded.sort((a, b) => b.aggregateScore - a.aggregateScore);
-    })
-  );
+  if (runIds.length === 0) throw new Error("prmEnsembleBestOfN: at least 1 candidate required");
+  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY);
+  const jobs = [];
+  for (let gi = 0; gi < graders.length; gi++) {
+    for (const runId of runIds) jobs.push({ graderIdx: gi, runId });
+  }
+  const settled = new Array(jobs.length);
+  let cursor = 0;
+  async function worker() {
+    while (true) {
+      const i = cursor++;
+      if (i >= jobs.length) return;
+      const job = jobs[i];
+      settled[i] = await graders[job.graderIdx].grade(store, job.runId).then((value) => ({ status: "fulfilled", value })).catch((reason) => ({ status: "rejected", reason }));
+    }
+  }
+  await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, () => worker()));
+  const perGrader = graders.map(() => []);
+  const failures = [];
+  for (let i = 0; i < jobs.length; i++) {
+    const job = jobs[i];
+    const result = settled[i];
+    if (result.status === "fulfilled") perGrader[job.graderIdx].push(result.value);
+    else {
+      const error = result.reason instanceof Error ? result.reason.message : String(result.reason);
+      failures.push({ graderIdx: job.graderIdx, runId: job.runId, error });
+    }
+  }
+  const survivingGraders = perGrader.filter((ranking) => ranking.length > 0);
+  if (survivingGraders.length === 0) {
+    throw new Error(
+      `prmEnsembleBestOfN: every grader failed on every candidate (${failures.length} call(s)). First error: ${failures[0]?.error ?? "unknown"}`
+    );
+  }
+  for (const ranking of survivingGraders)
+    ranking.sort((a, b) => b.aggregateScore - a.aggregateScore);
   const bordaScores = /* @__PURE__ */ new Map();
-  for (const ranking of perGrader) {
+  for (const ranking of survivingGraders) {
     ranking.forEach((g, rank) => {
       bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank));
     });
   }
-  const canonical = perGrader[0];
-  const byRun = new Map(canonical.map((g) => [g.runId, g]));
+  const byRun = /* @__PURE__ */ new Map();
+  for (const ranking of survivingGraders) {
+    for (const g of ranking) if (!byRun.has(g.runId)) byRun.set(g.runId, g);
+  }
   const ranked = [...byRun.values()].sort(
     (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0)
   );

package/dist/prm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["../../src/prm/builtin-rubrics.ts","../../src/prm/inference.ts","../../src/prm/rubric.ts"],"sourcesContent":["/**\n * Built-in reference rubrics. Consumers combine these with domain\n * rubrics. All are deterministic, rule-based — cheap to run + easy\n * to unit-test. LLM-based rubrics are trivially authored by\n * following the StepRubric contract.\n */\n\nimport type { LlmSpan, ToolSpan } from '../trace/schema'\nimport type { StepRubric } from './rubric'\n\n/** Penalize very short or very long assistant outputs. */\nexport function outputLengthRubric(\n  args: { minChars?: number; maxChars?: number; weight?: number } = {},\n): StepRubric {\n  const min = args.minChars ?? 20\n  const max = args.maxChars ?? 8000\n  return {\n    id: 'output-length',\n    kinds: ['llm'],\n    weight: args.weight ?? 0.5,\n    async grade({ step }) {\n      const llm = step.span as LlmSpan\n      const len = (llm.output ?? '').length\n      if (len === 0) return { score: 0, rationale: 'empty output' }\n      if (len < min)\n        return { score: Math.max(0, len / min), rationale: `below min (${len} < ${min})` }\n      if (len > max)\n        return {\n          score: Math.max(0, 1 - (len - max) / max),\n          rationale: `above max (${len} > ${max})`,\n        }\n      return { score: 1, rationale: `${len} chars in bounds` }\n    },\n  }\n}\n\n/** Reward tool calls that succeeded (status='ok') with an informative result. */\nexport function toolSuccessRubric(args: { weight?: number } = {}): StepRubric {\n  return {\n    id: 'tool-success',\n    kinds: ['tool'],\n    weight: args.weight ?? 1,\n    async grade({ step }) {\n      const tool = step.span as ToolSpan\n      if (tool.status === 'error')\n        return { score: 0, rationale: `error: ${tool.error ?? 'unknown'}` }\n      const r = tool.result\n      if (r === null || r === undefined) return { score: 0.3, rationale: 'empty result' }\n      const asText = typeof r === 'string' ? r : JSON.stringify(r)\n      if (asText.length < 4) return { score: 0.5, rationale: 'tiny result' }\n      return { score: 1, rationale: `${tool.toolName} ok` }\n    },\n  }\n}\n\n/** Penalize tool calls that duplicate a prior call with identical args. */\nexport function toolNonRedundantRubric(args: { weight?: number } = {}): StepRubric {\n  const weight = args.weight ?? 0.5\n  return {\n    id: 'tool-non-redundant',\n    kinds: ['tool'],\n    weight,\n    async grade({ step, prior }) {\n      const tool = step.span as ToolSpan\n      const priorMatches = prior.filter((p) => {\n        if (p.span.kind !== 'tool') return false\n        const pt = p.span as ToolSpan\n        return (\n          pt.toolName === tool.toolName && stableStringify(pt.args) === stableStringify(tool.args)\n        )\n      })\n      if (priorMatches.length === 0) return { score: 1, rationale: 'novel call' }\n      return {\n        score: Math.max(0, 1 - priorMatches.length * 0.5),\n        rationale: `${priorMatches.length} duplicate(s)`,\n      }\n    },\n  }\n}\n\n/** Penalize LLM outputs that contain common refusal markers when a refusal\n *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */\nexport function nonRefusalRubric(args: { markers?: RegExp[]; weight?: number } = {}): StepRubric {\n  const weight = args.weight ?? 1\n  const markers = args.markers ?? [\n    /\\bi\\s+(?:can(?:not|'t)|won't|will\\s+not)\\b/i,\n    /\\b(?:as\\s+an?\\s+)?ai\\b.*?\\b(?:can't|cannot)\\b/i,\n  ]\n  return {\n    id: 'non-refusal',\n    kinds: ['llm'],\n    weight,\n    async grade({ step }) {\n      const llm = step.span as LlmSpan\n      const out = llm.output ?? ''\n      const refused = markers.some((re) => re.test(out))\n      return refused\n        ? { score: 0, rationale: 'refusal marker present' }\n        : { score: 1, rationale: 'no refusal' }\n    },\n  }\n}\n\n/** Reward outputs that invoke the next-step tool the trajectory actually uses\n *  (i.e. the LLM span announced \"I will call X\" and the following tool span IS X). */\nexport function toolIntentAlignmentRubric(args: { weight?: number } = {}): StepRubric {\n  return {\n    id: 'tool-intent-alignment',\n    kinds: ['llm'],\n    weight: args.weight ?? 0.5,\n    async grade({ step, next }) {\n      const llm = step.span as LlmSpan\n      const nextTool = next.find((s) => s.span.kind === 'tool')\n      if (!nextTool) return null\n      const toolName = (nextTool.span as ToolSpan).toolName\n      const out = (llm.output ?? '').toLowerCase()\n      const mentioned = out.includes(toolName.toLowerCase())\n      return mentioned\n        ? { score: 1, rationale: `mentioned \"${toolName}\" before calling it` }\n        : { score: 0.5, rationale: `called \"${toolName}\" without announcing it` }\n    },\n  }\n}\n\nfunction stableStringify(value: unknown): string {\n  if (value === null || typeof value !== 'object') return JSON.stringify(value)\n  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`\n  const keys = Object.keys(value as Record<string, unknown>).sort()\n  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify((value as Record<string, unknown>)[k])}`).join(',')}}`\n}\n","/**\n * Inference-time PRM scoring — pick the best of N candidate trajectories\n * using a trained reward model (or a rule-based PRM as a proxy).\n *\n * The canonical Best-of-N pattern: generate N completions, score each\n * with a PRM, pick the winner. Here the scoring loop is framework-agnostic\n * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.\n */\n\nimport type { TraceStore } from '../trace/store'\nimport type { PrmGradedTrace, PrmGrader } from './rubric'\n\nexport interface BestOfNResult {\n  winner: PrmGradedTrace\n  ranked: PrmGradedTrace[]\n  /** Standard deviation of aggregate scores — small = candidates were homogenous. */\n  stdDev: number\n}\n\nexport async function prmBestOfN(\n  store: TraceStore,\n  grader: PrmGrader,\n  runIds: string[],\n): Promise<BestOfNResult> {\n  if (runIds.length === 0) throw new Error('prmBestOfN: at least 1 candidate required')\n  const graded = await Promise.all(runIds.map((id) => grader.grade(store, id)))\n  const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore)\n  const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length\n  const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length\n  return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) }\n}\n\n/**\n * Weighted vote across multiple graders — use when you want a PRM ensemble\n * (e.g. rule-based + LLM-based + trained model). Each grader produces its\n * own ranking; we aggregate via rank-sum (Borda count) so no single grader\n * dominates via a different score scale.\n */\nexport async function prmEnsembleBestOfN(\n  store: TraceStore,\n  graders: PrmGrader[],\n  runIds: string[],\n): Promise<BestOfNResult> {\n  if (graders.length === 0) throw new Error('prmEnsembleBestOfN: at least 1 grader')\n  const perGrader = await Promise.all(\n    graders.map(async (g) => {\n      const graded = await Promise.all(runIds.map((id) => g.grade(store, id)))\n      return graded.sort((a, b) => b.aggregateScore - a.aggregateScore)\n    }),\n  )\n  // Borda: rank-sum across graders.\n  const bordaScores = new Map<string, number>()\n  for (const ranking of perGrader) {\n    ranking.forEach((g, rank) => {\n      bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank))\n    })\n  }\n  // Return a synthesized ranking using the first grader's graded traces\n  // ordered by Borda score. aggregateScore field kept for UX.\n  const canonical = perGrader[0]!\n  const byRun = new Map(canonical.map((g) => [g.runId, g]))\n  const ranked = [...byRun.values()].sort(\n    (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0),\n  )\n  const mean = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length\n  const variance = ranked.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / ranked.length\n  return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) }\n}\n","/**\n * Process Reward Modeling — per-step rubric grading.\n *\n * A StepRubric inspects one span and returns a score + rationale.\n * PrmGrader applies an array of rubrics to every LLM span in a\n * trajectory (consumers can broaden to tool/retrieval spans via the\n * `kind` filter on each rubric).\n *\n * Why this matters: outcome-only eval (did the final artifact work?)\n * gives sparse reward — most agent turns are unattributable. PRMs\n * densify the signal so optimizers and RL fine-tuning can assign\n * credit per turn.\n */\n\nimport { TraceEmitter } from '../trace/emitter'\nimport type { JudgeSpan, Span } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory'\n\nexport interface StepContext {\n  trajectory: Trajectory\n  step: TrajectoryStep\n  /** Steps preceding `step` in trajectory order. */\n  prior: TrajectoryStep[]\n  /** Steps following `step`. */\n  next: TrajectoryStep[]\n}\n\nexport interface StepRubric {\n  id: string\n  /** Only grade spans of these kinds (default: all). */\n  kinds?: Array<Span['kind']>\n  /** Weight in the aggregate score. Default 1. */\n  weight?: number\n  /** Returns score in 0..1 + optional rationale/evidence. Return `null` to\n   *  skip grading (rubric doesn't apply to this step). */\n  grade: (\n    ctx: StepContext,\n  ) => Promise<{ score: number; rationale?: string; evidence?: string } | null>\n}\n\nexport interface GradedStep {\n  spanId: string\n  rubricId: string\n  score: number\n  weight: number\n  rationale?: string\n  evidence?: string\n}\n\nexport interface PrmGradedTrace {\n  runId: string\n  steps: GradedStep[]\n  /** Weighted mean of all graded steps; 0..1. */\n  aggregateScore: number\n  /** Number of spans graded — useful for sanity-checking coverage. */\n  gradedCount: number\n  /** Number of spans in the trajectory that no rubric matched. */\n  ungradedCount: number\n}\n\nexport class PrmGrader {\n  constructor(private rubrics: StepRubric[]) {\n    if (rubrics.length === 0) throw new Error('PrmGrader: at least 1 rubric required')\n  }\n\n  /**\n   * Grade every eligible span in a run. Emits a JudgeVerdict span for each\n   * (rubric × span) verdict so the result is visible to downstream pipelines\n   * (judgeAgreementView, etc.) — PRM is just \"a judge that runs per span.\"\n   */\n  async grade(store: TraceStore, runId: string): Promise<PrmGradedTrace> {\n    const trajectory = await buildTrajectory(store, runId)\n    const emitter = new TraceEmitter(store, { runId })\n    const steps: GradedStep[] = []\n    let ungraded = 0\n    for (let i = 0; i < trajectory.steps.length; i++) {\n      const step = trajectory.steps[i]!\n      const ctx: StepContext = {\n        trajectory,\n        step,\n        prior: trajectory.steps.slice(0, i),\n        next: trajectory.steps.slice(i + 1),\n      }\n      let gradedThis = false\n      for (const rubric of this.rubrics) {\n        if (rubric.kinds && !rubric.kinds.includes(step.span.kind)) continue\n        const verdict = await rubric.grade(ctx)\n        if (verdict === null) continue\n        const weight = rubric.weight ?? 1\n        steps.push({\n          spanId: step.span.spanId,\n          rubricId: rubric.id,\n          score: verdict.score,\n          weight,\n          rationale: verdict.rationale,\n          evidence: verdict.evidence,\n        })\n        gradedThis = true\n        // Persist the verdict as a JudgeSpan so the query pipelines see it\n        await emitter.recordJudge({\n          judgeId: `prm:${rubric.id}`,\n          targetSpanId: step.span.spanId,\n          dimension: 'step_quality',\n          score: verdict.score,\n          rationale: verdict.rationale,\n          evidence: verdict.evidence,\n          name: `prm:${rubric.id}`,\n        })\n      }\n      if (!gradedThis) ungraded++\n    }\n\n    const totalWeight = steps.reduce((a, s) => a + s.weight, 0)\n    const aggregateScore =\n      totalWeight === 0 ? 0 : steps.reduce((a, s) => a + s.score * s.weight, 0) / totalWeight\n\n    return { runId, steps, aggregateScore, gradedCount: steps.length, ungradedCount: ungraded }\n  }\n}\n\n/** Helper: reads JudgeVerdict spans that PRM emitted so downstream pipelines\n *  can distinguish PRM verdicts from human or top-level LLM judges. */\nexport function isPrmVerdict(verdict: JudgeSpan): boolean {\n  return verdict.judgeId.startsWith('prm:')\n}\n"],"mappings":";;;;;;;;;;;;;;AAWO,SAAS,mBACd,OAAkE,CAAC,GACvD;AACZ,QAAM,MAAM,KAAK,YAAY;AAC7B,QAAM,MAAM,KAAK,YAAY;AAC7B,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,MAAM,KAAK;AACjB,YAAM,OAAO,IAAI,UAAU,IAAI;AAC/B,UAAI,QAAQ,EAAG,QAAO,EAAE,OAAO,GAAG,WAAW,eAAe;AAC5D,UAAI,MAAM;AACR,eAAO,EAAE,OAAO,KAAK,IAAI,GAAG,MAAM,GAAG,GAAG,WAAW,cAAc,GAAG,MAAM,GAAG,IAAI;AACnF,UAAI,MAAM;AACR,eAAO;AAAA,UACL,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,GAAG;AAAA,UACxC,WAAW,cAAc,GAAG,MAAM,GAAG;AAAA,QACvC;AACF,aAAO,EAAE,OAAO,GAAG,WAAW,GAAG,GAAG,mBAAmB;AAAA,IACzD;AAAA,EACF;AACF;AAGO,SAAS,kBAAkB,OAA4B,CAAC,GAAe;AAC5E,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,MAAM;AAAA,IACd,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,OAAO,KAAK;AAClB,UAAI,KAAK,WAAW;AAClB,eAAO,EAAE,OAAO,GAAG,WAAW,UAAU,KAAK,SAAS,SAAS,GAAG;AACpE,YAAM,IAAI,KAAK;AACf,UAAI,MAAM,QAAQ,MAAM,OAAW,QAAO,EAAE,OAAO,KAAK,WAAW,eAAe;AAClF,YAAM,SAAS,OAAO,MAAM,WAAW,IAAI,KAAK,UAAU,CAAC;AAC3D,UAAI,OAAO,SAAS,EAAG,QAAO,EAAE,OAAO,KAAK,WAAW,cAAc;AACrE,aAAO,EAAE,OAAO,GAAG,WAAW,GAAG,KAAK,QAAQ,MAAM;AAAA,IACtD;AAAA,EACF;AACF;AAGO,SAAS,uBAAuB,OAA4B,CAAC,GAAe;AACjF,QAAM,SAAS,KAAK,UAAU;AAC9B,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,MAAM;AAAA,IACd;AAAA,IACA,MAAM,MAAM,EAAE,MAAM,MAAM,GAAG;AAC3B,YAAM,OAAO,KAAK;AAClB,YAAM,eAAe,MAAM,OAAO,CAAC,MAAM;AACvC,YAAI,EAAE,KAAK,SAAS,OAAQ,QAAO;AACnC,cAAM,KAAK,EAAE;AACb,eACE,GAAG,aAAa,KAAK,YAAY,gBAAgB,GAAG,IAAI,MAAM,gBAAgB,KAAK,IAAI;AAAA,MAE3F,CAAC;AACD,UAAI,aAAa,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,WAAW,aAAa;AAC1E,aAAO;AAAA,QACL,OAAO,KAAK,IAAI,GAAG,IAAI,aAAa,SAAS,GAAG;AAAA,QAChD,WAAW,GAAG,aAAa,MAAM;AAAA,MACnC;AAAA,IACF;AAAA,EACF;AACF;AAIO,SAAS,iBAAiB,OAAgD,CAAC,GAAe;AAC/F,QAAM,SAAS,KAAK,UAAU;AAC9B,QAAM,UAAU,KAAK,WAAW;AAAA,IAC9B;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb;AAAA,IACA,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,MAAM,KAAK;AACjB,YAAM,MAAM,IAAI,UAAU;AAC1B,YAAM,UAAU,QAAQ,KAAK,CAAC,OAAO,GAAG,KAAK,GAAG,CAAC;AACjD,aAAO,UACH,EAAE,OAAO,GAAG,WAAW,yBAAyB,IAChD,EAAE,OAAO,GAAG,WAAW,aAAa;AAAA,IAC1C;AAAA,EACF;AACF;AAIO,SAAS,0BAA0B,OAA4B,CAAC,GAAe;AACpF,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,MAAM,KAAK,GAAG;AAC1B,YAAM,MAAM,KAAK;AACjB,YAAM,WAAW,KAAK,KAAK,CAAC,MAAM,EAAE,KAAK,SAAS,MAAM;AACxD,UAAI,CAAC,SAAU,QAAO;AACtB,YAAM,WAAY,SAAS,KAAkB;AAC7C,YAAM,OAAO,IAAI,UAAU,IAAI,YAAY;AAC3C,YAAM,YAAY,IAAI,SAAS,SAAS,YAAY,CAAC;AACrD,aAAO,YACH,EAAE,OAAO,GAAG,WAAW,cAAc,QAAQ,sBAAsB,IACnE,EAAE,OAAO,KAAK,WAAW,WAAW,QAAQ,0BAA0B;AAAA,IAC5E;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,OAAwB;AAC/C,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK;AAC5E,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,IAAI,MAAM,IAAI,eAAe,EAAE,KAAK,GAAG,CAAC;AACzE,QAAM,OAAO,OAAO,KAAK,KAAgC,EAAE,KAAK;AAChE,SAAO,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,gBAAiB,MAAkC,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC;AACxH;;;AC9GA,eAAsB,WACpB,OACA,QACA,QACwB;AACxB,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,2CAA2C;AACpF,QAAM,SAAS,MAAM,QAAQ,IAAI,OAAO,IAAI,CAAC,OAAO,OAAO,MAAM,OAAO,EAAE,CAAC,CAAC;AAC5E,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAC7E,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,gBAAgB,CAAC,IAAI,OAAO;AACvE,QAAM,WAAW,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,iBAAiB,SAAS,GAAG,CAAC,IAAI,OAAO;AACzF,SAAO,EAAE,QAAQ,OAAO,CAAC,GAAI,QAAQ,QAAQ,KAAK,KAAK,QAAQ,EAAE;AACnE;AAQA,eAAsB,mBACpB,OACA,SACA,QACwB;AACxB,MAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC;AACjF,QAAM,YAAY,MAAM,QAAQ;AAAA,IAC9B,QAAQ,IAAI,OAAO,MAAM;AACvB,YAAM,SAAS,MAAM,QAAQ,IAAI,OAAO,IAAI,CAAC,OAAO,EAAE,MAAM,OAAO,EAAE,CAAC,CAAC;AACvE,aAAO,OAAO,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAAA,IAClE,CAAC;AAAA,EACH;AAEA,QAAM,cAAc,oBAAI,IAAoB;AAC5C,aAAW,WAAW,WAAW;AAC/B,YAAQ,QAAQ,CAAC,GAAG,SAAS;AAC3B,kBAAY,IAAI,EAAE,QAAQ,YAAY,IAAI,EAAE,KAAK,KAAK,MAAM,QAAQ,SAAS,KAAK;AAAA,IACpF,CAAC;AAAA,EACH;AAGA,QAAM,YAAY,UAAU,CAAC;AAC7B,QAAM,QAAQ,IAAI,IAAI,UAAU,IAAI,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;AACxD,QAAM,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,EAAE;AAAA,IACjC,CAAC,GAAG,OAAO,YAAY,IAAI,EAAE,KAAK,KAAK,MAAM,YAAY,IAAI,EAAE,KAAK,KAAK;AAAA,EAC3E;AACA,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,gBAAgB,CAAC,IAAI,OAAO;AACvE,QAAM,WAAW,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,iBAAiB,SAAS,GAAG,CAAC,IAAI,OAAO;AACzF,SAAO,EAAE,QAAQ,OAAO,CAAC,GAAI,QAAQ,QAAQ,KAAK,KAAK,QAAQ,EAAE;AACnE;;;ACNO,IAAM,YAAN,MAAgB;AAAA,EACrB,YAAoB,SAAuB;AAAvB;AAClB,QAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC;AAAA,EACnF;AAAA,EAFoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASpB,MAAM,MAAM,OAAmB,OAAwC;AACrE,UAAM,aAAa,MAAM,gBAAgB,OAAO,KAAK;AACrD,UAAM,UAAU,IAAI,aAAa,OAAO,EAAE,MAAM,CAAC;AACjD,UAAM,QAAsB,CAAC;AAC7B,QAAI,WAAW;AACf,aAAS,IAAI,GAAG,IAAI,WAAW,MAAM,QAAQ,KAAK;AAChD,YAAM,OAAO,WAAW,MAAM,CAAC;AAC/B,YAAM,MAAmB;AAAA,QACvB;AAAA,QACA;AAAA,QACA,OAAO,WAAW,MAAM,MAAM,GAAG,CAAC;AAAA,QAClC,MAAM,WAAW,MAAM,MAAM,IAAI,CAAC;AAAA,MACpC;AACA,UAAI,aAAa;AACjB,iBAAW,UAAU,KAAK,SAAS;AACjC,YAAI,OAAO,SAAS,CAAC,OAAO,MAAM,SAAS,KAAK,KAAK,IAAI,EAAG;AAC5D,cAAM,UAAU,MAAM,OAAO,MAAM,GAAG;AACtC,YAAI,YAAY,KAAM;AACtB,cAAM,SAAS,OAAO,UAAU;AAChC,cAAM,KAAK;AAAA,UACT,QAAQ,KAAK,KAAK;AAAA,UAClB,UAAU,OAAO;AAAA,UACjB,OAAO,QAAQ;AAAA,UACf;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB,UAAU,QAAQ;AAAA,QACpB,CAAC;AACD,qBAAa;AAEb,cAAM,QAAQ,YAAY;AAAA,UACxB,SAAS,OAAO,OAAO,EAAE;AAAA,UACzB,cAAc,KAAK,KAAK;AAAA,UACxB,WAAW;AAAA,UACX,OAAO,QAAQ;AAAA,UACf,WAAW,QAAQ;AAAA,UACnB,UAAU,QAAQ;AAAA,UAClB,MAAM,OAAO,OAAO,EAAE;AAAA,QACxB,CAAC;AAAA,MACH;AACA,UAAI,CAAC,WAAY;AAAA,IACnB;AAEA,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,CAAC;AAC1D,UAAM,iBACJ,gBAAgB,IAAI,IAAI,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI;AAE9E,WAAO,EAAE,OAAO,OAAO,gBAAgB,aAAa,MAAM,QAAQ,eAAe,SAAS;AAAA,EAC5F;AACF;AAIO,SAAS,aAAa,SAA6B;AACxD,SAAO,QAAQ,QAAQ,WAAW,MAAM;AAC1C;","names":[]}
+{"version":3,"sources":["../../src/prm/builtin-rubrics.ts","../../src/prm/inference.ts","../../src/prm/rubric.ts"],"sourcesContent":["/**\n * Built-in reference rubrics. Consumers combine these with domain\n * rubrics. All are deterministic, rule-based — cheap to run + easy\n * to unit-test. LLM-based rubrics are trivially authored by\n * following the StepRubric contract.\n */\n\nimport type { LlmSpan, ToolSpan } from '../trace/schema'\nimport type { StepRubric } from './rubric'\n\n/** Penalize very short or very long assistant outputs. */\nexport function outputLengthRubric(\n  args: { minChars?: number; maxChars?: number; weight?: number } = {},\n): StepRubric {\n  const min = args.minChars ?? 20\n  const max = args.maxChars ?? 8000\n  return {\n    id: 'output-length',\n    kinds: ['llm'],\n    weight: args.weight ?? 0.5,\n    async grade({ step }) {\n      const llm = step.span as LlmSpan\n      const len = (llm.output ?? '').length\n      if (len === 0) return { score: 0, rationale: 'empty output' }\n      if (len < min)\n        return { score: Math.max(0, len / min), rationale: `below min (${len} < ${min})` }\n      if (len > max)\n        return {\n          score: Math.max(0, 1 - (len - max) / max),\n          rationale: `above max (${len} > ${max})`,\n        }\n      return { score: 1, rationale: `${len} chars in bounds` }\n    },\n  }\n}\n\n/** Reward tool calls that succeeded (status='ok') with an informative result. */\nexport function toolSuccessRubric(args: { weight?: number } = {}): StepRubric {\n  return {\n    id: 'tool-success',\n    kinds: ['tool'],\n    weight: args.weight ?? 1,\n    async grade({ step }) {\n      const tool = step.span as ToolSpan\n      if (tool.status === 'error')\n        return { score: 0, rationale: `error: ${tool.error ?? 'unknown'}` }\n      const r = tool.result\n      if (r === null || r === undefined) return { score: 0.3, rationale: 'empty result' }\n      const asText = typeof r === 'string' ? r : JSON.stringify(r)\n      if (asText.length < 4) return { score: 0.5, rationale: 'tiny result' }\n      return { score: 1, rationale: `${tool.toolName} ok` }\n    },\n  }\n}\n\n/** Penalize tool calls that duplicate a prior call with identical args. */\nexport function toolNonRedundantRubric(args: { weight?: number } = {}): StepRubric {\n  const weight = args.weight ?? 0.5\n  return {\n    id: 'tool-non-redundant',\n    kinds: ['tool'],\n    weight,\n    async grade({ step, prior }) {\n      const tool = step.span as ToolSpan\n      const priorMatches = prior.filter((p) => {\n        if (p.span.kind !== 'tool') return false\n        const pt = p.span as ToolSpan\n        return (\n          pt.toolName === tool.toolName && stableStringify(pt.args) === stableStringify(tool.args)\n        )\n      })\n      if (priorMatches.length === 0) return { score: 1, rationale: 'novel call' }\n      return {\n        score: Math.max(0, 1 - priorMatches.length * 0.5),\n        rationale: `${priorMatches.length} duplicate(s)`,\n      }\n    },\n  }\n}\n\n/** Penalize LLM outputs that contain common refusal markers when a refusal\n *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */\nexport function nonRefusalRubric(args: { markers?: RegExp[]; weight?: number } = {}): StepRubric {\n  const weight = args.weight ?? 1\n  const markers = args.markers ?? [\n    /\\bi\\s+(?:can(?:not|'t)|won't|will\\s+not)\\b/i,\n    /\\b(?:as\\s+an?\\s+)?ai\\b.*?\\b(?:can't|cannot)\\b/i,\n  ]\n  return {\n    id: 'non-refusal',\n    kinds: ['llm'],\n    weight,\n    async grade({ step }) {\n      const llm = step.span as LlmSpan\n      const out = llm.output ?? ''\n      const refused = markers.some((re) => re.test(out))\n      return refused\n        ? { score: 0, rationale: 'refusal marker present' }\n        : { score: 1, rationale: 'no refusal' }\n    },\n  }\n}\n\n/** Reward outputs that invoke the next-step tool the trajectory actually uses\n *  (i.e. the LLM span announced \"I will call X\" and the following tool span IS X). */\nexport function toolIntentAlignmentRubric(args: { weight?: number } = {}): StepRubric {\n  return {\n    id: 'tool-intent-alignment',\n    kinds: ['llm'],\n    weight: args.weight ?? 0.5,\n    async grade({ step, next }) {\n      const llm = step.span as LlmSpan\n      const nextTool = next.find((s) => s.span.kind === 'tool')\n      if (!nextTool) return null\n      const toolName = (nextTool.span as ToolSpan).toolName\n      const out = (llm.output ?? '').toLowerCase()\n      const mentioned = out.includes(toolName.toLowerCase())\n      return mentioned\n        ? { score: 1, rationale: `mentioned \"${toolName}\" before calling it` }\n        : { score: 0.5, rationale: `called \"${toolName}\" without announcing it` }\n    },\n  }\n}\n\nfunction stableStringify(value: unknown): string {\n  if (value === null || typeof value !== 'object') return JSON.stringify(value)\n  if (Array.isArray(value)) return `[${value.map(stableStringify).join(',')}]`\n  const keys = Object.keys(value as Record<string, unknown>).sort()\n  return `{${keys.map((k) => `${JSON.stringify(k)}:${stableStringify((value as Record<string, unknown>)[k])}`).join(',')}}`\n}\n","/**\n * Inference-time PRM scoring — pick the best of N candidate trajectories\n * using a trained reward model (or a rule-based PRM as a proxy).\n *\n * The canonical Best-of-N pattern: generate N completions, score each\n * with a PRM, pick the winner. Here the scoring loop is framework-agnostic\n * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.\n */\n\nimport type { TraceStore } from '../trace/store'\nimport type { PrmGradedTrace, PrmGrader } from './rubric'\n\nexport interface BestOfNResult {\n  winner: PrmGradedTrace\n  ranked: PrmGradedTrace[]\n  /** Standard deviation of aggregate scores — small = candidates were homogenous. */\n  stdDev: number\n}\n\n/** Default max concurrent grader calls. Bounds LLM fan-out so a wide ensemble\n *  doesn't trigger a provider rate-limit storm. */\nconst DEFAULT_PRM_CONCURRENCY = 4\n\nexport interface PrmBestOfNOptions {\n  /** Max concurrent `grader.grade` calls. Default 4. */\n  concurrency?: number\n}\n\nexport async function prmBestOfN(\n  store: TraceStore,\n  grader: PrmGrader,\n  runIds: string[],\n  options: PrmBestOfNOptions = {},\n): Promise<BestOfNResult> {\n  if (runIds.length === 0) throw new Error('prmBestOfN: at least 1 candidate required')\n  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY)\n  const graded: PrmGradedTrace[] = new Array(runIds.length)\n  let cursor = 0\n  async function worker(): Promise<void> {\n    while (true) {\n      const i = cursor++\n      if (i >= runIds.length) return\n      graded[i] = await grader.grade(store, runIds[i]!)\n    }\n  }\n  await Promise.all(Array.from({ length: Math.min(concurrency, runIds.length) }, () => worker()))\n  const ranked = [...graded].sort((a, b) => b.aggregateScore - a.aggregateScore)\n  const mean = graded.reduce((a, g) => a + g.aggregateScore, 0) / graded.length\n  const variance = graded.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / graded.length\n  return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) }\n}\n\n/**\n * Weighted vote across multiple graders — use when you want a PRM ensemble\n * (e.g. rule-based + LLM-based + trained model). Each grader produces its\n * own ranking; we aggregate via rank-sum (Borda count) so no single grader\n * dominates via a different score scale.\n */\nexport async function prmEnsembleBestOfN(\n  store: TraceStore,\n  graders: PrmGrader[],\n  runIds: string[],\n  options: PrmBestOfNOptions = {},\n): Promise<BestOfNResult> {\n  if (graders.length === 0) throw new Error('prmEnsembleBestOfN: at least 1 grader')\n  if (runIds.length === 0) throw new Error('prmEnsembleBestOfN: at least 1 candidate required')\n  const concurrency = Math.max(1, options.concurrency ?? DEFAULT_PRM_CONCURRENCY)\n\n  // Flatten the (grader, runId) product and run it through a single bounded\n  // pool. Nested unbounded fan-out (graders × runIds) would launch every LLM\n  // call at once. allSettled isolates failures: one grader (or one runId)\n  // failing doesn't void the whole ensemble.\n  type Job = { graderIdx: number; runId: string }\n  const jobs: Job[] = []\n  for (let gi = 0; gi < graders.length; gi++) {\n    for (const runId of runIds) jobs.push({ graderIdx: gi, runId })\n  }\n  const settled: PromiseSettledResult<PrmGradedTrace>[] = new Array(jobs.length)\n  let cursor = 0\n  async function worker(): Promise<void> {\n    while (true) {\n      const i = cursor++\n      if (i >= jobs.length) return\n      const job = jobs[i]!\n      settled[i] = await graders[job.graderIdx]!.grade(store, job.runId)\n        .then((value): PromiseSettledResult<PrmGradedTrace> => ({ status: 'fulfilled', value }))\n        .catch((reason): PromiseSettledResult<PrmGradedTrace> => ({ status: 'rejected', reason }))\n    }\n  }\n  await Promise.all(Array.from({ length: Math.min(concurrency, jobs.length) }, () => worker()))\n\n  // Regroup fulfilled results into per-grader rankings. A grader contributes\n  // only the candidates it successfully graded; a grader that graded nothing\n  // is dropped from the vote rather than skewing it with phantom zeros.\n  const perGrader: PrmGradedTrace[][] = graders.map(() => [])\n  const failures: { graderIdx: number; runId: string; error: string }[] = []\n  for (let i = 0; i < jobs.length; i++) {\n    const job = jobs[i]!\n    const result = settled[i]!\n    if (result.status === 'fulfilled') perGrader[job.graderIdx]!.push(result.value)\n    else {\n      const error = result.reason instanceof Error ? result.reason.message : String(result.reason)\n      failures.push({ graderIdx: job.graderIdx, runId: job.runId, error })\n    }\n  }\n  const survivingGraders = perGrader.filter((ranking) => ranking.length > 0)\n  if (survivingGraders.length === 0) {\n    throw new Error(\n      `prmEnsembleBestOfN: every grader failed on every candidate (${failures.length} call(s)). First error: ${failures[0]?.error ?? 'unknown'}`,\n    )\n  }\n  for (const ranking of survivingGraders)\n    ranking.sort((a, b) => b.aggregateScore - a.aggregateScore)\n\n  // Borda: rank-sum across surviving graders.\n  const bordaScores = new Map<string, number>()\n  for (const ranking of survivingGraders) {\n    ranking.forEach((g, rank) => {\n      bordaScores.set(g.runId, (bordaScores.get(g.runId) ?? 0) + (ranking.length - rank))\n    })\n  }\n  // Synthesize a ranking from the union of every successfully-graded trace,\n  // ordered by Borda score. aggregateScore field kept for UX. Using the union\n  // (not just the first grader) keeps a candidate that one grader dropped but\n  // another graded.\n  const byRun = new Map<string, PrmGradedTrace>()\n  for (const ranking of survivingGraders) {\n    for (const g of ranking) if (!byRun.has(g.runId)) byRun.set(g.runId, g)\n  }\n  const ranked = [...byRun.values()].sort(\n    (a, b) => (bordaScores.get(b.runId) ?? 0) - (bordaScores.get(a.runId) ?? 0),\n  )\n  const mean = ranked.reduce((a, g) => a + g.aggregateScore, 0) / ranked.length\n  const variance = ranked.reduce((a, g) => a + (g.aggregateScore - mean) ** 2, 0) / ranked.length\n  return { winner: ranked[0]!, ranked, stdDev: Math.sqrt(variance) }\n}\n","/**\n * Process Reward Modeling — per-step rubric grading.\n *\n * A StepRubric inspects one span and returns a score + rationale.\n * PrmGrader applies an array of rubrics to every LLM span in a\n * trajectory (consumers can broaden to tool/retrieval spans via the\n * `kind` filter on each rubric).\n *\n * Why this matters: outcome-only eval (did the final artifact work?)\n * gives sparse reward — most agent turns are unattributable. PRMs\n * densify the signal so optimizers and RL fine-tuning can assign\n * credit per turn.\n */\n\nimport { TraceEmitter } from '../trace/emitter'\nimport type { JudgeSpan, Span } from '../trace/schema'\nimport type { TraceStore } from '../trace/store'\nimport { buildTrajectory, type Trajectory, type TrajectoryStep } from '../trajectory'\n\nexport interface StepContext {\n  trajectory: Trajectory\n  step: TrajectoryStep\n  /** Steps preceding `step` in trajectory order. */\n  prior: TrajectoryStep[]\n  /** Steps following `step`. */\n  next: TrajectoryStep[]\n}\n\nexport interface StepRubric {\n  id: string\n  /** Only grade spans of these kinds (default: all). */\n  kinds?: Array<Span['kind']>\n  /** Weight in the aggregate score. Default 1. */\n  weight?: number\n  /** Returns score in 0..1 + optional rationale/evidence. Return `null` to\n   *  skip grading (rubric doesn't apply to this step). */\n  grade: (\n    ctx: StepContext,\n  ) => Promise<{ score: number; rationale?: string; evidence?: string } | null>\n}\n\nexport interface GradedStep {\n  spanId: string\n  rubricId: string\n  score: number\n  weight: number\n  rationale?: string\n  evidence?: string\n}\n\nexport interface PrmGradedTrace {\n  runId: string\n  steps: GradedStep[]\n  /** Weighted mean of all graded steps; 0..1. */\n  aggregateScore: number\n  /** Number of spans graded — useful for sanity-checking coverage. */\n  gradedCount: number\n  /** Number of spans in the trajectory that no rubric matched. */\n  ungradedCount: number\n}\n\nexport class PrmGrader {\n  constructor(private rubrics: StepRubric[]) {\n    if (rubrics.length === 0) throw new Error('PrmGrader: at least 1 rubric required')\n  }\n\n  /**\n   * Grade every eligible span in a run. Emits a JudgeVerdict span for each\n   * (rubric × span) verdict so the result is visible to downstream pipelines\n   * (judgeAgreementView, etc.) — PRM is just \"a judge that runs per span.\"\n   */\n  async grade(store: TraceStore, runId: string): Promise<PrmGradedTrace> {\n    const trajectory = await buildTrajectory(store, runId)\n    const emitter = new TraceEmitter(store, { runId })\n    const steps: GradedStep[] = []\n    let ungraded = 0\n    for (let i = 0; i < trajectory.steps.length; i++) {\n      const step = trajectory.steps[i]!\n      const ctx: StepContext = {\n        trajectory,\n        step,\n        prior: trajectory.steps.slice(0, i),\n        next: trajectory.steps.slice(i + 1),\n      }\n      let gradedThis = false\n      for (const rubric of this.rubrics) {\n        if (rubric.kinds && !rubric.kinds.includes(step.span.kind)) continue\n        const verdict = await rubric.grade(ctx)\n        if (verdict === null) continue\n        const weight = rubric.weight ?? 1\n        steps.push({\n          spanId: step.span.spanId,\n          rubricId: rubric.id,\n          score: verdict.score,\n          weight,\n          rationale: verdict.rationale,\n          evidence: verdict.evidence,\n        })\n        gradedThis = true\n        // Persist the verdict as a JudgeSpan so the query pipelines see it\n        await emitter.recordJudge({\n          judgeId: `prm:${rubric.id}`,\n          targetSpanId: step.span.spanId,\n          dimension: 'step_quality',\n          score: verdict.score,\n          rationale: verdict.rationale,\n          evidence: verdict.evidence,\n          name: `prm:${rubric.id}`,\n        })\n      }\n      if (!gradedThis) ungraded++\n    }\n\n    const totalWeight = steps.reduce((a, s) => a + s.weight, 0)\n    const aggregateScore =\n      totalWeight === 0 ? 0 : steps.reduce((a, s) => a + s.score * s.weight, 0) / totalWeight\n\n    return { runId, steps, aggregateScore, gradedCount: steps.length, ungradedCount: ungraded }\n  }\n}\n\n/** Helper: reads JudgeVerdict spans that PRM emitted so downstream pipelines\n *  can distinguish PRM verdicts from human or top-level LLM judges. */\nexport function isPrmVerdict(verdict: JudgeSpan): boolean {\n  return verdict.judgeId.startsWith('prm:')\n}\n"],"mappings":";;;;;;;;;;;;;;AAWO,SAAS,mBACd,OAAkE,CAAC,GACvD;AACZ,QAAM,MAAM,KAAK,YAAY;AAC7B,QAAM,MAAM,KAAK,YAAY;AAC7B,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,MAAM,KAAK;AACjB,YAAM,OAAO,IAAI,UAAU,IAAI;AAC/B,UAAI,QAAQ,EAAG,QAAO,EAAE,OAAO,GAAG,WAAW,eAAe;AAC5D,UAAI,MAAM;AACR,eAAO,EAAE,OAAO,KAAK,IAAI,GAAG,MAAM,GAAG,GAAG,WAAW,cAAc,GAAG,MAAM,GAAG,IAAI;AACnF,UAAI,MAAM;AACR,eAAO;AAAA,UACL,OAAO,KAAK,IAAI,GAAG,KAAK,MAAM,OAAO,GAAG;AAAA,UACxC,WAAW,cAAc,GAAG,MAAM,GAAG;AAAA,QACvC;AACF,aAAO,EAAE,OAAO,GAAG,WAAW,GAAG,GAAG,mBAAmB;AAAA,IACzD;AAAA,EACF;AACF;AAGO,SAAS,kBAAkB,OAA4B,CAAC,GAAe;AAC5E,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,MAAM;AAAA,IACd,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,OAAO,KAAK;AAClB,UAAI,KAAK,WAAW;AAClB,eAAO,EAAE,OAAO,GAAG,WAAW,UAAU,KAAK,SAAS,SAAS,GAAG;AACpE,YAAM,IAAI,KAAK;AACf,UAAI,MAAM,QAAQ,MAAM,OAAW,QAAO,EAAE,OAAO,KAAK,WAAW,eAAe;AAClF,YAAM,SAAS,OAAO,MAAM,WAAW,IAAI,KAAK,UAAU,CAAC;AAC3D,UAAI,OAAO,SAAS,EAAG,QAAO,EAAE,OAAO,KAAK,WAAW,cAAc;AACrE,aAAO,EAAE,OAAO,GAAG,WAAW,GAAG,KAAK,QAAQ,MAAM;AAAA,IACtD;AAAA,EACF;AACF;AAGO,SAAS,uBAAuB,OAA4B,CAAC,GAAe;AACjF,QAAM,SAAS,KAAK,UAAU;AAC9B,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,MAAM;AAAA,IACd;AAAA,IACA,MAAM,MAAM,EAAE,MAAM,MAAM,GAAG;AAC3B,YAAM,OAAO,KAAK;AAClB,YAAM,eAAe,MAAM,OAAO,CAAC,MAAM;AACvC,YAAI,EAAE,KAAK,SAAS,OAAQ,QAAO;AACnC,cAAM,KAAK,EAAE;AACb,eACE,GAAG,aAAa,KAAK,YAAY,gBAAgB,GAAG,IAAI,MAAM,gBAAgB,KAAK,IAAI;AAAA,MAE3F,CAAC;AACD,UAAI,aAAa,WAAW,EAAG,QAAO,EAAE,OAAO,GAAG,WAAW,aAAa;AAC1E,aAAO;AAAA,QACL,OAAO,KAAK,IAAI,GAAG,IAAI,aAAa,SAAS,GAAG;AAAA,QAChD,WAAW,GAAG,aAAa,MAAM;AAAA,MACnC;AAAA,IACF;AAAA,EACF;AACF;AAIO,SAAS,iBAAiB,OAAgD,CAAC,GAAe;AAC/F,QAAM,SAAS,KAAK,UAAU;AAC9B,QAAM,UAAU,KAAK,WAAW;AAAA,IAC9B;AAAA,IACA;AAAA,EACF;AACA,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb;AAAA,IACA,MAAM,MAAM,EAAE,KAAK,GAAG;AACpB,YAAM,MAAM,KAAK;AACjB,YAAM,MAAM,IAAI,UAAU;AAC1B,YAAM,UAAU,QAAQ,KAAK,CAAC,OAAO,GAAG,KAAK,GAAG,CAAC;AACjD,aAAO,UACH,EAAE,OAAO,GAAG,WAAW,yBAAyB,IAChD,EAAE,OAAO,GAAG,WAAW,aAAa;AAAA,IAC1C;AAAA,EACF;AACF;AAIO,SAAS,0BAA0B,OAA4B,CAAC,GAAe;AACpF,SAAO;AAAA,IACL,IAAI;AAAA,IACJ,OAAO,CAAC,KAAK;AAAA,IACb,QAAQ,KAAK,UAAU;AAAA,IACvB,MAAM,MAAM,EAAE,MAAM,KAAK,GAAG;AAC1B,YAAM,MAAM,KAAK;AACjB,YAAM,WAAW,KAAK,KAAK,CAAC,MAAM,EAAE,KAAK,SAAS,MAAM;AACxD,UAAI,CAAC,SAAU,QAAO;AACtB,YAAM,WAAY,SAAS,KAAkB;AAC7C,YAAM,OAAO,IAAI,UAAU,IAAI,YAAY;AAC3C,YAAM,YAAY,IAAI,SAAS,SAAS,YAAY,CAAC;AACrD,aAAO,YACH,EAAE,OAAO,GAAG,WAAW,cAAc,QAAQ,sBAAsB,IACnE,EAAE,OAAO,KAAK,WAAW,WAAW,QAAQ,0BAA0B;AAAA,IAC5E;AAAA,EACF;AACF;AAEA,SAAS,gBAAgB,OAAwB;AAC/C,MAAI,UAAU,QAAQ,OAAO,UAAU,SAAU,QAAO,KAAK,UAAU,KAAK;AAC5E,MAAI,MAAM,QAAQ,KAAK,EAAG,QAAO,IAAI,MAAM,IAAI,eAAe,EAAE,KAAK,GAAG,CAAC;AACzE,QAAM,OAAO,OAAO,KAAK,KAAgC,EAAE,KAAK;AAChE,SAAO,IAAI,KAAK,IAAI,CAAC,MAAM,GAAG,KAAK,UAAU,CAAC,CAAC,IAAI,gBAAiB,MAAkC,CAAC,CAAC,CAAC,EAAE,EAAE,KAAK,GAAG,CAAC;AACxH;;;AC5GA,IAAM,0BAA0B;AAOhC,eAAsB,WACpB,OACA,QACA,QACA,UAA6B,CAAC,GACN;AACxB,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,2CAA2C;AACpF,QAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,eAAe,uBAAuB;AAC9E,QAAM,SAA2B,IAAI,MAAM,OAAO,MAAM;AACxD,MAAI,SAAS;AACb,iBAAe,SAAwB;AACrC,WAAO,MAAM;AACX,YAAM,IAAI;AACV,UAAI,KAAK,OAAO,OAAQ;AACxB,aAAO,CAAC,IAAI,MAAM,OAAO,MAAM,OAAO,OAAO,CAAC,CAAE;AAAA,IAClD;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,OAAO,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAC9F,QAAM,SAAS,CAAC,GAAG,MAAM,EAAE,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAC7E,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,gBAAgB,CAAC,IAAI,OAAO;AACvE,QAAM,WAAW,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,iBAAiB,SAAS,GAAG,CAAC,IAAI,OAAO;AACzF,SAAO,EAAE,QAAQ,OAAO,CAAC,GAAI,QAAQ,QAAQ,KAAK,KAAK,QAAQ,EAAE;AACnE;AAQA,eAAsB,mBACpB,OACA,SACA,QACA,UAA6B,CAAC,GACN;AACxB,MAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC;AACjF,MAAI,OAAO,WAAW,EAAG,OAAM,IAAI,MAAM,mDAAmD;AAC5F,QAAM,cAAc,KAAK,IAAI,GAAG,QAAQ,eAAe,uBAAuB;AAO9E,QAAM,OAAc,CAAC;AACrB,WAAS,KAAK,GAAG,KAAK,QAAQ,QAAQ,MAAM;AAC1C,eAAW,SAAS,OAAQ,MAAK,KAAK,EAAE,WAAW,IAAI,MAAM,CAAC;AAAA,EAChE;AACA,QAAM,UAAkD,IAAI,MAAM,KAAK,MAAM;AAC7E,MAAI,SAAS;AACb,iBAAe,SAAwB;AACrC,WAAO,MAAM;AACX,YAAM,IAAI;AACV,UAAI,KAAK,KAAK,OAAQ;AACtB,YAAM,MAAM,KAAK,CAAC;AAClB,cAAQ,CAAC,IAAI,MAAM,QAAQ,IAAI,SAAS,EAAG,MAAM,OAAO,IAAI,KAAK,EAC9D,KAAK,CAAC,WAAiD,EAAE,QAAQ,aAAa,MAAM,EAAE,EACtF,MAAM,CAAC,YAAkD,EAAE,QAAQ,YAAY,OAAO,EAAE;AAAA,IAC7F;AAAA,EACF;AACA,QAAM,QAAQ,IAAI,MAAM,KAAK,EAAE,QAAQ,KAAK,IAAI,aAAa,KAAK,MAAM,EAAE,GAAG,MAAM,OAAO,CAAC,CAAC;AAK5F,QAAM,YAAgC,QAAQ,IAAI,MAAM,CAAC,CAAC;AAC1D,QAAM,WAAkE,CAAC;AACzE,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,MAAM,KAAK,CAAC;AAClB,UAAM,SAAS,QAAQ,CAAC;AACxB,QAAI,OAAO,WAAW,YAAa,WAAU,IAAI,SAAS,EAAG,KAAK,OAAO,KAAK;AAAA,SACzE;AACH,YAAM,QAAQ,OAAO,kBAAkB,QAAQ,OAAO,OAAO,UAAU,OAAO,OAAO,MAAM;AAC3F,eAAS,KAAK,EAAE,WAAW,IAAI,WAAW,OAAO,IAAI,OAAO,MAAM,CAAC;AAAA,IACrE;AAAA,EACF;AACA,QAAM,mBAAmB,UAAU,OAAO,CAAC,YAAY,QAAQ,SAAS,CAAC;AACzE,MAAI,iBAAiB,WAAW,GAAG;AACjC,UAAM,IAAI;AAAA,MACR,+DAA+D,SAAS,MAAM,2BAA2B,SAAS,CAAC,GAAG,SAAS,SAAS;AAAA,IAC1I;AAAA,EACF;AACA,aAAW,WAAW;AACpB,YAAQ,KAAK,CAAC,GAAG,MAAM,EAAE,iBAAiB,EAAE,cAAc;AAG5D,QAAM,cAAc,oBAAI,IAAoB;AAC5C,aAAW,WAAW,kBAAkB;AACtC,YAAQ,QAAQ,CAAC,GAAG,SAAS;AAC3B,kBAAY,IAAI,EAAE,QAAQ,YAAY,IAAI,EAAE,KAAK,KAAK,MAAM,QAAQ,SAAS,KAAK;AAAA,IACpF,CAAC;AAAA,EACH;AAKA,QAAM,QAAQ,oBAAI,IAA4B;AAC9C,aAAW,WAAW,kBAAkB;AACtC,eAAW,KAAK,QAAS,KAAI,CAAC,MAAM,IAAI,EAAE,KAAK,EAAG,OAAM,IAAI,EAAE,OAAO,CAAC;AAAA,EACxE;AACA,QAAM,SAAS,CAAC,GAAG,MAAM,OAAO,CAAC,EAAE;AAAA,IACjC,CAAC,GAAG,OAAO,YAAY,IAAI,EAAE,KAAK,KAAK,MAAM,YAAY,IAAI,EAAE,KAAK,KAAK;AAAA,EAC3E;AACA,QAAM,OAAO,OAAO,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,gBAAgB,CAAC,IAAI,OAAO;AACvE,QAAM,WAAW,OAAO,OAAO,CAAC,GAAG,MAAM,KAAK,EAAE,iBAAiB,SAAS,GAAG,CAAC,IAAI,OAAO;AACzF,SAAO,EAAE,QAAQ,OAAO,CAAC,GAAI,QAAQ,QAAQ,KAAK,KAAK,QAAQ,EAAE;AACnE;;;AC1EO,IAAM,YAAN,MAAgB;AAAA,EACrB,YAAoB,SAAuB;AAAvB;AAClB,QAAI,QAAQ,WAAW,EAAG,OAAM,IAAI,MAAM,uCAAuC;AAAA,EACnF;AAAA,EAFoB;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EASpB,MAAM,MAAM,OAAmB,OAAwC;AACrE,UAAM,aAAa,MAAM,gBAAgB,OAAO,KAAK;AACrD,UAAM,UAAU,IAAI,aAAa,OAAO,EAAE,MAAM,CAAC;AACjD,UAAM,QAAsB,CAAC;AAC7B,QAAI,WAAW;AACf,aAAS,IAAI,GAAG,IAAI,WAAW,MAAM,QAAQ,KAAK;AAChD,YAAM,OAAO,WAAW,MAAM,CAAC;AAC/B,YAAM,MAAmB;AAAA,QACvB;AAAA,QACA;AAAA,QACA,OAAO,WAAW,MAAM,MAAM,GAAG,CAAC;AAAA,QAClC,MAAM,WAAW,MAAM,MAAM,IAAI,CAAC;AAAA,MACpC;AACA,UAAI,aAAa;AACjB,iBAAW,UAAU,KAAK,SAAS;AACjC,YAAI,OAAO,SAAS,CAAC,OAAO,MAAM,SAAS,KAAK,KAAK,IAAI,EAAG;AAC5D,cAAM,UAAU,MAAM,OAAO,MAAM,GAAG;AACtC,YAAI,YAAY,KAAM;AACtB,cAAM,SAAS,OAAO,UAAU;AAChC,cAAM,KAAK;AAAA,UACT,QAAQ,KAAK,KAAK;AAAA,UAClB,UAAU,OAAO;AAAA,UACjB,OAAO,QAAQ;AAAA,UACf;AAAA,UACA,WAAW,QAAQ;AAAA,UACnB,UAAU,QAAQ;AAAA,QACpB,CAAC;AACD,qBAAa;AAEb,cAAM,QAAQ,YAAY;AAAA,UACxB,SAAS,OAAO,OAAO,EAAE;AAAA,UACzB,cAAc,KAAK,KAAK;AAAA,UACxB,WAAW;AAAA,UACX,OAAO,QAAQ;AAAA,UACf,WAAW,QAAQ;AAAA,UACnB,UAAU,QAAQ;AAAA,UAClB,MAAM,OAAO,OAAO,EAAE;AAAA,QACxB,CAAC;AAAA,MACH;AACA,UAAI,CAAC,WAAY;AAAA,IACnB;AAEA,UAAM,cAAc,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,CAAC;AAC1D,UAAM,iBACJ,gBAAgB,IAAI,IAAI,MAAM,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,IAAI;AAE9E,WAAO,EAAE,OAAO,OAAO,gBAAgB,aAAa,MAAM,QAAQ,eAAe,SAAS;AAAA,EAC5F;AACF;AAIO,SAAS,aAAa,SAA6B;AACxD,SAAO,QAAQ,QAAQ,WAAW,MAAM;AAC1C;","names":[]}

package/dist/prm/inference.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Inference-time PRM scoring — pick the best of N candidate trajectories
+ * using a trained reward model (or a rule-based PRM as a proxy).
+ *
+ * The canonical Best-of-N pattern: generate N completions, score each
+ * with a PRM, pick the winner. Here the scoring loop is framework-agnostic
+ * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.
+ */
+import type { TraceStore } from '../trace/store';
+import type { PrmGradedTrace, PrmGrader } from './rubric';
+export interface BestOfNResult {
+    winner: PrmGradedTrace;
+    ranked: PrmGradedTrace[];
+    /** Standard deviation of aggregate scores — small = candidates were homogenous. */
+    stdDev: number;
+}
+export interface PrmBestOfNOptions {
+    /** Max concurrent `grader.grade` calls. Default 4. */
+    concurrency?: number;
+}
+export declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[], options?: PrmBestOfNOptions): Promise<BestOfNResult>;
+/**
+ * Weighted vote across multiple graders — use when you want a PRM ensemble
+ * (e.g. rule-based + LLM-based + trained model). Each grader produces its
+ * own ranking; we aggregate via rank-sum (Borda count) so no single grader
+ * dominates via a different score scale.
+ */
+export declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[], options?: PrmBestOfNOptions): Promise<BestOfNResult>;
+//# sourceMappingURL=inference.d.ts.map

package/dist/prm/inference.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../src/prm/inference.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AAEzD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,cAAc,CAAA;IACtB,MAAM,EAAE,cAAc,EAAE,CAAA;IACxB,mFAAmF;IACnF,MAAM,EAAE,MAAM,CAAA;CACf;AAMD,MAAM,WAAW,iBAAiB;IAChC,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,wBAAsB,UAAU,CAC9B,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,aAAa,CAAC,CAiBxB;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,SAAS,EAAE,EACpB,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,aAAa,CAAC,CAwExB"}

package/dist/prm/inference.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=inference.test.d.ts.map

package/dist/prm/inference.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference.test.d.ts","sourceRoot":"","sources":["../../src/prm/inference.test.ts"],"names":[],"mappings":""}

package/dist/{rubric-BOfxn4ja.d.ts → prm/rubric.d.ts}

@@ -1,7 +1,3 @@
-import { S as Span, J as JudgeSpan } from './schema-m0gsnbt3.js';
-import { T as TraceStore } from './store-CKUAgsJz.js';
-import { T as Trajectory, a as TrajectoryStep } from './trajectory-GEdXJCL5.js';
-
 /**
  * Process Reward Modeling — per-step rubric grading.
  *
@@ -15,8 +11,10 @@ import { T as Trajectory, a as TrajectoryStep } from './trajectory-GEdXJCL5.js';
  * densify the signal so optimizers and RL fine-tuning can assign
  * credit per turn.
  */
-
-interface StepContext {
+import type { JudgeSpan, Span } from '../trace/schema';
+import type { TraceStore } from '../trace/store';
+import { type Trajectory, type TrajectoryStep } from '../trajectory';
+export interface StepContext {
     trajectory: Trajectory;
     step: TrajectoryStep;
     /** Steps preceding `step` in trajectory order. */
@@ -24,7 +22,7 @@ interface StepContext {
     /** Steps following `step`. */
     next: TrajectoryStep[];
 }
-interface StepRubric {
+export interface StepRubric {
     id: string;
     /** Only grade spans of these kinds (default: all). */
     kinds?: Array<Span['kind']>;
@@ -38,7 +36,7 @@ interface StepRubric {
         evidence?: string;
     } | null>;
 }
-interface GradedStep {
+export interface GradedStep {
     spanId: string;
     rubricId: string;
     score: number;
@@ -46,7 +44,7 @@ interface GradedStep {
     rationale?: string;
     evidence?: string;
 }
-interface PrmGradedTrace {
+export interface PrmGradedTrace {
     runId: string;
     steps: GradedStep[];
     /** Weighted mean of all graded steps; 0..1. */
@@ -56,7 +54,7 @@ interface PrmGradedTrace {
     /** Number of spans in the trajectory that no rubric matched. */
     ungradedCount: number;
 }
-declare class PrmGrader {
+export declare class PrmGrader {
     private rubrics;
     constructor(rubrics: StepRubric[]);
     /**
@@ -68,6 +66,5 @@ declare class PrmGrader {
 }
 /** Helper: reads JudgeVerdict spans that PRM emitted so downstream pipelines
  *  can distinguish PRM verdicts from human or top-level LLM judges. */
-declare function isPrmVerdict(verdict: JudgeSpan): boolean;
-
-export { type GradedStep as G, type PrmGradedTrace as P, type StepRubric as S, PrmGrader as a, type StepContext as b, isPrmVerdict as i };
+export declare function isPrmVerdict(verdict: JudgeSpan): boolean;
+//# sourceMappingURL=rubric.d.ts.map

package/dist/prm/rubric.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rubric.d.ts","sourceRoot":"","sources":["../../src/prm/rubric.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,EAAmB,KAAK,UAAU,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAA;AAErF,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,UAAU,CAAA;IACtB,IAAI,EAAE,cAAc,CAAA;IACpB,kDAAkD;IAClD,KAAK,EAAE,cAAc,EAAE,CAAA;IACvB,8BAA8B;IAC9B,IAAI,EAAE,cAAc,EAAE,CAAA;CACvB;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAA;IACV,sDAAsD;IACtD,KAAK,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAC3B,gDAAgD;IAChD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;4DACwD;IACxD,KAAK,EAAE,CACL,GAAG,EAAE,WAAW,KACb,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAA;CAC9E;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,+CAA+C;IAC/C,cAAc,EAAE,MAAM,CAAA;IACtB,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAA;IACnB,gEAAgE;IAChE,aAAa,EAAE,MAAM,CAAA;CACtB;AAED,qBAAa,SAAS;IACR,OAAO,CAAC,OAAO;gBAAP,OAAO,EAAE,UAAU,EAAE;IAIzC;;;;OAIG;IACG,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;CAgDvE;AAED;uEACuE;AACvE,wBAAgB,YAAY,CAAC,OAAO,EAAE,SAAS,GAAG,OAAO,CAExD"}

package/dist/prm/training-export.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Export PRM-graded traces as training data for downstream reward-model
+ * fine-tuning. Canonical format is NDJSON of
+ * `{ trajectory_text, step_index, rubric, score }` so a small model can
+ * learn to predict step rewards from step context.
+ *
+ * The framework doesn't train the model — we emit the data; callers
+ * plug it into their preferred trainer (TRL, Unsloth, custom).
+ */
+import type { Span } from '../trace/schema';
+import type { TraceStore } from '../trace/store';
+import type { PrmGradedTrace } from './rubric';
+export interface PrmTrainingSample {
+    runId: string;
+    spanId: string;
+    rubricId: string;
+    score: number;
+    /** Serialized step context — step + surrounding conversation. */
+    context: {
+        priorTurns: Array<{
+            role: string;
+            content: string;
+        }>;
+        step: {
+            kind: Span['kind'];
+            text: string;
+        };
+    };
+    /** Optional evidence + rationale for auditability. */
+    rationale?: string;
+    evidence?: string;
+}
+export declare function exportTrainingData(store: TraceStore, graded: PrmGradedTrace[], options?: {
+    contextWindow?: number;
+}): Promise<PrmTrainingSample[]>;
+/** NDJSON serialization — write to file or stream directly to a trainer. */
+export declare function toNdjson(samples: PrmTrainingSample[]): string;
+//# sourceMappingURL=training-export.d.ts.map

package/dist/prm/training-export.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"training-export.d.ts","sourceRoot":"","sources":["../../src/prm/training-export.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAW,IAAI,EAAE,MAAM,iBAAiB,CAAA;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAE9C,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,iEAAiE;IACjE,OAAO,EAAE;QACP,UAAU,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QACpD,IAAI,EAAE;YAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAC3C,CAAA;IACD,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,cAAc,EAAE,EACxB,OAAO,GAAE;IAAE,aAAa,CAAC,EAAE,MAAM,CAAA;CAAO,GACvC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CA4B9B;AAED,4EAA4E;AAC5E,wBAAgB,QAAQ,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAE7D"}

package/dist/produced-state.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Produced-state extraction — normalize a run's runtime event stream into the
+ * typed `ProducedState` the completion oracle consumes.
+ *
+ * `ProducedState` answers "what did the agent actually produce" — vault
+ * artifacts, proposals, tool calls. The runtime emits these as a stream of
+ * events; this module is the single normalization point from that stream to
+ * the shape `verifyCompletion` expects.
+ *
+ * Input is structurally typed (`RuntimeEventLike`) so this module does not
+ * depend on agent-runtime — agent-runtime's `RuntimeStreamEvent` satisfies it
+ * structurally. The `content` on `ArtifactEventLike` and the whole
+ * `proposal_created` variant are the runtime-side enrichments this contract
+ * requires; the runtime emits them, this module consumes them.
+ */
+import type { ProducedState } from './completion-verifier';
+/** A tool the agent invoked. */
+export interface ToolCallEventLike {
+    type: 'tool_call';
+    toolName: string;
+}
+/**
+ * An artifact the agent produced. `content` is the enriched field — the
+ * runtime's base `artifact` event carries only metadata; the completion
+ * oracle needs the body to verify the deliverable, so the runtime emits it.
+ */
+export interface ArtifactEventLike {
+    type: 'artifact';
+    artifactId: string;
+    name?: string;
+    mimeType?: string;
+    uri?: string;
+    content?: string;
+}
+/** A proposal / filing the agent created. */
+export interface ProposalEventLike {
+    type: 'proposal_created';
+    proposalId: string;
+    title: string;
+    status?: 'pending' | 'approved' | 'rejected';
+    content?: string;
+}
+/**
+ * The subset of runtime stream events `extractProducedState` consumes.
+ * agent-runtime's full `RuntimeStreamEvent` union satisfies this structurally;
+ * the `{ type: string }` catch-all keeps the input permissive so callers can
+ * pass the whole unfiltered telemetry stream — unrecognized events are skipped.
+ */
+export type RuntimeEventLike = ToolCallEventLike | ArtifactEventLike | ProposalEventLike | {
+    type: string;
+};
+/**
+ * Normalize a run's runtime event stream into `ProducedState`.
+ *
+ * Pure and total — unrecognized event types are skipped. `toolCalls` is
+ * deduplicated by name in first-seen order (completion cares about a tool's
+ * presence, not its call count). An artifact with neither a name nor a uri
+ * still yields an entry keyed by its `artifactId` so it is never silently
+ * dropped; an artifact with no `content` yields empty content, which the
+ * completion oracle's structural check then rejects on its own.
+ */
+export declare function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState;
+//# sourceMappingURL=produced-state.d.ts.map

package/dist/produced-state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"produced-state.d.ts","sourceRoot":"","sources":["../src/produced-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,KAAK,EAAoB,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAE5E,gCAAgC;AAChC,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,WAAW,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,UAAU,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,6CAA6C;AAC7C,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,kBAAkB,CAAA;IACxB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,SAAS,GAAG,UAAU,GAAG,UAAU,CAAA;IAI5C,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GACxB,iBAAiB,GACjB,iBAAiB,GACjB,iBAAiB,GACjB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AASpB;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,SAAS,gBAAgB,EAAE,GAAG,aAAa,CAgCvF"}

package/dist/produced-state.test.d.ts

@@ -0,0 +1,8 @@
+/**
+ * extractProducedState is the single normalization point from a run's event
+ * stream to the `ProducedState` the completion oracle consumes. These tests
+ * pin that mapping — a regression here silently corrupts every completion
+ * verdict downstream.
+ */
+export {};
+//# sourceMappingURL=produced-state.test.d.ts.map

package/dist/produced-state.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"produced-state.test.d.ts","sourceRoot":"","sources":["../src/produced-state.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/dist/profile/baselines.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Strong, generically-useful baseline ROLES — the top zone of an `AgentProfile`
+ * before any domain layer. A product composes one of these with its own
+ * environment description (its sandbox) and its domain guidance, which lives in
+ * the product repo (not here): a profile is `<baseline role> + <environment> +
+ * <domain sections>`. Domain strength is NOT generalized into the substrate —
+ * it is lifted here only once ≥2 products genuinely reuse it.
+ *
+ * Three roles cover the common shapes:
+ *   - `engineerRole`   — builds + verifies real artifacts in a sandbox. Distilled
+ *                        from agent-runtime's `coderProfile` doctrine (minimal
+ *                        correct change, run the checks, fix the root cause,
+ *                        never weaken a test or hide an error).
+ *   - `researcherRole` — gathers from real sources and grounds every claim
+ *                        (cite or mark inferred; never fabricate).
+ *   - `generalistRole` — a strong default: helpful, grounded, verify-before-assert.
+ *
+ * All three are verification-first and domain-agnostic. They describe HOW a
+ * capable IC operates, not WHAT domain it works in.
+ */
+/** Senior-IC engineer operating in a sandbox — produces real artifacts and
+ *  verifies them before declaring done. The shared operator foundation for any
+ *  file-producing, tool-using product agent. */
+export declare const engineerRole: string;
+/** Researcher who grounds every claim in real sources — gathers, cites, and
+ *  distinguishes verified fact from inference. */
+export declare const researcherRole: string;
+/** Strong general-purpose default — helpful, grounded, verifies before asserting. */
+export declare const generalistRole: string;
+/** The named baseline roles, for selection by key (e.g. from a product config). */
+export declare const BASELINE_ROLES: {
+    readonly engineer: string;
+    readonly researcher: string;
+    readonly generalist: string;
+};
+export type BaselineRoleKey = keyof typeof BASELINE_ROLES;
+//# sourceMappingURL=baselines.d.ts.map

package/dist/profile/baselines.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"baselines.d.ts","sourceRoot":"","sources":["../../src/profile/baselines.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;gDAEgD;AAChD,eAAO,MAAM,YAAY,EAAE,MASf,CAAA;AAEZ;kDACkD;AAClD,eAAO,MAAM,cAAc,EAAE,MASjB,CAAA;AAEZ,qFAAqF;AACrF,eAAO,MAAM,cAAc,EAAE,MASjB,CAAA;AAEZ,mFAAmF;AACnF,eAAO,MAAM,cAAc;;;;CAIjB,CAAA;AAEV,MAAM,MAAM,eAAe,GAAG,MAAM,OAAO,cAAc,CAAA"}