@tangle-network/agent-eval 0.94.0 → 0.95.0

package/dist/pr-review-benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pr-review-benchmark.d.ts","sourceRoot":"","sources":["../src/pr-review-benchmark.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,cAAc,GACtB,MAAM,GACN,SAAS,GACT,OAAO,GACP,OAAO,GACP,aAAa,GACb,cAAc,GACd,sBAAsB,GACtB,MAAM,GACN,UAAU,GACV,CAAC,MAAM,GAAG,EAAE,CAAC,CAAA;AAEjB,MAAM,MAAM,gBAAgB,GAAG,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,KAAK,GAAG,KAAK,CAAA;AAE7E,MAAM,MAAM,eAAe,GAAG,UAAU,GAAG,OAAO,GAAG,UAAU,GAAG,WAAW,GAAG,OAAO,GAAG,SAAS,CAAA;AAEnG,MAAM,WAAW,eAAe;IAC9B,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,cAAc,CAAA;IACtB,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,gBAAgB,CAAA;IAC3B,OAAO,CAAC,EAAE,eAAe,CAAA;IACzB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,wBAAwB;IACvC,EAAE,EAAE,MAAM,CAAA;IACV,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,gBAAgB,CAAA;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,IAAI,CAAC,EAAE,MAAM,CAAA;IACb;;;OAGG;IACH,QAAQ,CAAC,EAAE,MAAM,EAAE,CAAA;IACnB,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,gBAAgB,CAAC,EAAE,MAAM,EAAE,CAAA;IAC3B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAA;IACV,IAAI,EAAE,MAAM,CAAA;IACZ,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,KAAK,CAAC,EAAE,MAAM,CAAA;IACd,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,KAAK,CAAC,EAAE,OAAO,GAAG,YAAY,GAAG,MAAM,GAAG,SAAS,GAAG,CAAC,MAAM,GAAG,EAAE,CAAC,CAAA;IACnE,QAAQ,EAAE,eAAe,EAAE,CAAA;IAC3B,iBAAiB,EAAE,wBAAwB,EAAE,CAAA;IAC7C,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CACnC;AAED,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,aAAa,EAAE,MAAM,CAAA;IACrB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED,MAAM,WAAW,sBAAsB;IACrC,WAAW,EAAE,MAAM,CAAA;IACnB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,CAAA;CACd;AAED,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAA;IACd,MAAM,EAAE,cAAc,CAAA;IACtB,YAAY,EAAE,MAAM,CAAA;IACpB,cAAc,EAAE,MAAM,CAAA;IACtB,eAAe,EAAE,sBAAsB,EAAE,CAAA;IACzC,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,EAAE,MAAM,CAAA;IACjB,aAAa,EAAE,MAAM,CAAA;IACrB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,KAAK,EAAE,MAAM,EAAE,CAAA;CAChB;AAED,MAAM,WAAW,wBAAwB;IACvC,MAAM,EAAE,cAAc,CAAA;IACtB,SAAS,EAAE,MAAM,CAAA;IACjB,YAAY,EAAE,MAAM,CAAA;IACpB,aAAa,EAAE,MAAM,CAAA;IACrB,UAAU,EAAE,MAAM,CAAA;IAClB,aAAa,EAAE,MAAM,CAAA;IACrB,iBAAiB,EAAE,MAAM,CAAA;IACzB,uBAAuB,EAAE,MAAM,CAAA;IAC/B,YAAY,EAAE,MAAM,CAAA;CACrB;AAED,eAAO,MAAM,+BAA+B,EAAE,oBAM7C,CAAA;AAED,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE,iBAAiB,EAC5B,MAAM,EAAE,cAAc,GACrB,eAAe,EAAE,CAEnB;AAED,wBAAgB,mBAAmB,CACjC,SAAS,EAAE,iBAAiB,EAC5B,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,OAAO,CAAC,oBAAoB,CAAM,GAC1C,aAAa,CAEf;AAED,wBAAgB,qBAAqB,CACnC,SAAS,EAAE,iBAAiB,EAC5B,QAAQ,EAAE,eAAe,EAAE,EAC3B,MAAM,EAAE,cAAc,EACtB,OAAO,GAAE,OAAO,CAAC,oBAAoB,CAAM,GAC1C,aAAa,CA2Df;AAED,wBAAgB,0BAA0B,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,wBAAwB,EAAE,CAkB9F;AAED,wBAAgB,sBAAsB,CACpC,UAAU,EAAE,IAAI,CACd,aAAa,EACb,QAAQ,GAAG,WAAW,GAAG,eAAe,GAAG,qBAAqB,GAAG,UAAU,CAC9E,EACD,OAAO,GAAE,OAAO,CAAC,oBAAoB,CAAM,GAC1C,MAAM,CAYR"}

package/dist/pr-review-benchmark.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=pr-review-benchmark.test.d.ts.map

package/dist/pr-review-benchmark.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pr-review-benchmark.test.d.ts","sourceRoot":"","sources":["../src/pr-review-benchmark.test.ts"],"names":[],"mappings":""}

package/dist/pre-registration.d.ts

@@ -0,0 +1,125 @@
+/**
+ * Pre-registered hypotheses — declare what you're testing BEFORE the
+ * run, check it AFTER. Prevents p-hacking, optional stopping, and the
+ * "we ran until it looked good" failure mode.
+ *
+ * Manifest is a plain JSON-friendly object. Sign it with a content hash
+ * + timestamp; the registered record becomes immutable. Post-run,
+ * evaluate the manifest against observed results — the library refuses
+ * to let you re-interpret a different metric as the declared one.
+ */
+export interface HypothesisManifest {
+    id: string;
+    /** Human prose — goes into the audit trail. */
+    hypothesis: string;
+    /** Metric the hypothesis claims to move. */
+    metric: string;
+    /** 'increase' = candidate should score higher than baseline; 'decrease' = lower. */
+    direction: 'increase' | 'decrease';
+    /** Minimum effect size to count (same units as the metric). */
+    minEffect: number;
+    /** Alpha threshold. */
+    alpha: number;
+    /** Target statistical power at which sample size was pre-computed. */
+    power: number;
+    /** Declared N per arm before running. */
+    preRegisteredN: number;
+    /** ISO8601 timestamp the manifest was registered. */
+    registeredAt: string;
+    /** Optional identifiers to tie into the trace corpus. */
+    baselineLabel?: string;
+    candidateLabel?: string;
+}
+/**
+ * Identifier for the hashing scheme used to produce `contentHash`.
+ *
+ * `'sha256-content'` — sha256 hex over the canonicalized manifest with
+ * the `contentHash` and `algo` fields stripped. Held as a string union
+ * so future schemes can be added without breaking parsers; SignedManifest
+ * values without `algo` deserialize cleanly because the field is optional.
+ */
+export type SignedManifestAlgo = 'sha256-content';
+export interface SignedManifest extends HypothesisManifest {
+    /** sha256 hex of canonicalized manifest (everything except contentHash and algo). */
+    contentHash: string;
+    /**
+     * Algorithm string describing how `contentHash` was produced.
+     *
+     * Optional on the type so serialized manifests without it still parse,
+     * but ALWAYS populated by {@link signManifest}. Consumers that want to
+     * enforce a known algorithm should reject manifests where this field
+     * is missing or unrecognized.
+     */
+    algo?: SignedManifestAlgo;
+}
+export interface HypothesisResult {
+    manifest: SignedManifest;
+    observedN: number;
+    observedEffect: number;
+    observedPValue: number;
+    /** True iff the observed effect hits the pre-declared direction with
+     *  magnitude ≥ minEffect AND p < alpha. */
+    confirmed: boolean;
+    /** Enumerated reasons the hypothesis was rejected (each a machine-tag). */
+    rejectionReasons: Array<'wrong_direction' | 'effect_too_small' | 'not_significant' | 'undersampled'>;
+    notes?: string;
+}
+/**
+ * Deterministic JSON canonicalization — sort object keys recursively.
+ *
+ * Two semantically-equal objects produce byte-identical canonicalized output;
+ * this is what makes a content-hash stable across encoders, key insertion
+ * orders, and runtime versions. Exported for any consumer that needs the same
+ * canonicalization guarantee outside the manifest-signing path (e.g., signing
+ * an artifact bundle, hashing a dataset version, etc.).
+ */
+export declare function canonicalize(v: unknown): unknown;
+/**
+ * SHA-256 hex (full 64 chars) over the canonicalized JSON encoding of `obj`.
+ *
+ * The same primitive `signManifest` and `verifyManifest` are built on, exposed
+ * directly so consumers signing arbitrary structured content (artifact bundles,
+ * production packets, dataset manifests, etc.) don't have to re-derive
+ * canonicalize+sha256 from scratch.
+ *
+ * Stable across:
+ *   - object key insertion order (canonicalization sorts keys recursively)
+ *   - encoder choice (UTF-8 via TextEncoder, fixed)
+ *   - runtime (uses the Web Crypto subtle digest, present in Node ≥18 and browsers)
+ *
+ * Named `hashJson` to disambiguate from `prompt-registry.ts`'s `hashContent`,
+ * which takes a string input and returns a truncated 12-char prompt id.
+ * Use `hashJson` when you mean "canonicalize then hash."
+ *
+ * @example
+ *   const hash = await hashJson({ id: '1', kind: 'spec' })
+ *   // 'a3f1...' (64 hex chars)
+ */
+export declare function hashJson<T>(obj: T): Promise<string>;
+/**
+ * Sign a manifest with a SHA-256 content hash.
+ *
+ * The hash covers the canonicalized manifest with the `contentHash`
+ * and `algo` fields stripped; this lets verifiers re-sign the rest and
+ * compare. Returned manifest always carries `algo: 'sha256-content'`
+ * so downstream consumers can identify the scheme; manifests without
+ * `algo` still verify because it is stripped before hashing on both sides.
+ */
+export declare function signManifest(m: HypothesisManifest): Promise<SignedManifest>;
+/**
+ * Verify that a signed manifest has not been tampered with.
+ *
+ * Strips `contentHash` and `algo` before re-signing so manifests without
+ * `algo` verify identically to ones that carry it.
+ */
+export declare function verifyManifest(m: SignedManifest): Promise<boolean>;
+/**
+ * Evaluate a pre-registered hypothesis against observed results.
+ * Mechanical — no re-interpretation permitted.
+ */
+export declare function evaluateHypothesis(manifest: SignedManifest, observed: {
+    n: number;
+    effect: number;
+    pValue: number;
+}): Promise<HypothesisResult>;
+//# sourceMappingURL=pre-registration.d.ts.map

package/dist/pre-registration.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"pre-registration.d.ts","sourceRoot":"","sources":["../src/pre-registration.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAA;IACV,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAA;IAClB,4CAA4C;IAC5C,MAAM,EAAE,MAAM,CAAA;IACd,oFAAoF;IACpF,SAAS,EAAE,UAAU,GAAG,UAAU,CAAA;IAClC,+DAA+D;IAC/D,SAAS,EAAE,MAAM,CAAA;IACjB,uBAAuB;IACvB,KAAK,EAAE,MAAM,CAAA;IACb,sEAAsE;IACtE,KAAK,EAAE,MAAM,CAAA;IACb,yCAAyC;IACzC,cAAc,EAAE,MAAM,CAAA;IACtB,qDAAqD;IACrD,YAAY,EAAE,MAAM,CAAA;IACpB,yDAAyD;IACzD,aAAa,CAAC,EAAE,MAAM,CAAA;IACtB,cAAc,CAAC,EAAE,MAAM,CAAA;CACxB;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,kBAAkB,GAAG,gBAAgB,CAAA;AAEjD,MAAM,WAAW,cAAe,SAAQ,kBAAkB;IACxD,qFAAqF;IACrF,WAAW,EAAE,MAAM,CAAA;IACnB;;;;;;;OAOG;IACH,IAAI,CAAC,EAAE,kBAAkB,CAAA;CAC1B;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,cAAc,CAAA;IACxB,SAAS,EAAE,MAAM,CAAA;IACjB,cAAc,EAAE,MAAM,CAAA;IACtB,cAAc,EAAE,MAAM,CAAA;IACtB;+CAC2C;IAC3C,SAAS,EAAE,OAAO,CAAA;IAClB,2EAA2E;IAC3E,gBAAgB,EAAE,KAAK,CACrB,iBAAiB,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,cAAc,CAC5E,CAAA;IACD,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAED;;;;;;;;GAQG;AACH,wBAAgB,YAAY,CAAC,CAAC,EAAE,OAAO,GAAG,OAAO,CAOhD;AAED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAsB,QAAQ,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAOzD;AAED;;;;;;;;GAQG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,kBAAkB,GAAG,OAAO,CAAC,cAAc,CAAC,CAGjF;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CAAC,CAAC,EAAE,cAAc,GAAG,OAAO,CAAC,OAAO,CAAC,CAKxE;AAED;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,cAAc,EACxB,QAAQ,EAAE;IAAE,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,GACtD,OAAO,CAAC,gBAAgB,CAAC,CAkB3B"}

package/dist/prm/builtin-rubrics.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Built-in reference rubrics. Consumers combine these with domain
+ * rubrics. All are deterministic, rule-based — cheap to run + easy
+ * to unit-test. LLM-based rubrics are trivially authored by
+ * following the StepRubric contract.
+ */
+import type { StepRubric } from './rubric';
+/** Penalize very short or very long assistant outputs. */
+export declare function outputLengthRubric(args?: {
+    minChars?: number;
+    maxChars?: number;
+    weight?: number;
+}): StepRubric;
+/** Reward tool calls that succeeded (status='ok') with an informative result. */
+export declare function toolSuccessRubric(args?: {
+    weight?: number;
+}): StepRubric;
+/** Penalize tool calls that duplicate a prior call with identical args. */
+export declare function toolNonRedundantRubric(args?: {
+    weight?: number;
+}): StepRubric;
+/** Penalize LLM outputs that contain common refusal markers when a refusal
+ *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */
+export declare function nonRefusalRubric(args?: {
+    markers?: RegExp[];
+    weight?: number;
+}): StepRubric;
+/** Reward outputs that invoke the next-step tool the trajectory actually uses
+ *  (i.e. the LLM span announced "I will call X" and the following tool span IS X). */
+export declare function toolIntentAlignmentRubric(args?: {
+    weight?: number;
+}): StepRubric;
+//# sourceMappingURL=builtin-rubrics.d.ts.map

package/dist/prm/builtin-rubrics.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"builtin-rubrics.d.ts","sourceRoot":"","sources":["../../src/prm/builtin-rubrics.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,UAAU,CAAA;AAE1C,0DAA0D;AAC1D,wBAAgB,kBAAkB,CAChC,IAAI,GAAE;IAAE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GACnE,UAAU,CAqBZ;AAED,iFAAiF;AACjF,wBAAgB,iBAAiB,CAAC,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,CAgB5E;AAED,2EAA2E;AAC3E,wBAAgB,sBAAsB,CAAC,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,CAsBjF;AAED;uFACuF;AACvF,wBAAgB,gBAAgB,CAAC,IAAI,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,CAmB/F;AAED;sFACsF;AACtF,wBAAgB,yBAAyB,CAAC,IAAI,GAAE;IAAE,MAAM,CAAC,EAAE,MAAM,CAAA;CAAO,GAAG,UAAU,CAiBpF"}

package/dist/prm/index.d.ts

@@ -1,104 +1,5 @@
-import { P as PrmGradedTrace, S as StepRubric, a as PrmGrader } from '../rubric-Cc6UHvUb.js';
-export { G as GradedStep, b as StepContext, i as isPrmVerdict } from '../rubric-Cc6UHvUb.js';
-import { T as TraceStore } from '../store-BcFXE6LG.js';
-import { S as Span } from '../schema-m0gsnbt3.js';
-import '../trajectory-2TkpSEVh.js';
-
-/**
- * Export PRM-graded traces as training data for downstream reward-model
- * fine-tuning. Canonical format is NDJSON of
- * `{ trajectory_text, step_index, rubric, score }` so a small model can
- * learn to predict step rewards from step context.
- *
- * The framework doesn't train the model — we emit the data; callers
- * plug it into their preferred trainer (TRL, Unsloth, custom).
- */
-
-interface PrmTrainingSample {
-    runId: string;
-    spanId: string;
-    rubricId: string;
-    score: number;
-    /** Serialized step context — step + surrounding conversation. */
-    context: {
-        priorTurns: Array<{
-            role: string;
-            content: string;
-        }>;
-        step: {
-            kind: Span['kind'];
-            text: string;
-        };
-    };
-    /** Optional evidence + rationale for auditability. */
-    rationale?: string;
-    evidence?: string;
-}
-declare function exportTrainingData(store: TraceStore, graded: PrmGradedTrace[], options?: {
-    contextWindow?: number;
-}): Promise<PrmTrainingSample[]>;
-/** NDJSON serialization — write to file or stream directly to a trainer. */
-declare function toNdjson(samples: PrmTrainingSample[]): string;
-
-/**
- * Built-in reference rubrics. Consumers combine these with domain
- * rubrics. All are deterministic, rule-based — cheap to run + easy
- * to unit-test. LLM-based rubrics are trivially authored by
- * following the StepRubric contract.
- */
-
-/** Penalize very short or very long assistant outputs. */
-declare function outputLengthRubric(args?: {
-    minChars?: number;
-    maxChars?: number;
-    weight?: number;
-}): StepRubric;
-/** Reward tool calls that succeeded (status='ok') with an informative result. */
-declare function toolSuccessRubric(args?: {
-    weight?: number;
-}): StepRubric;
-/** Penalize tool calls that duplicate a prior call with identical args. */
-declare function toolNonRedundantRubric(args?: {
-    weight?: number;
-}): StepRubric;
-/** Penalize LLM outputs that contain common refusal markers when a refusal
- *  is NOT expected (caller inverts weight for scenarios where refusal IS expected). */
-declare function nonRefusalRubric(args?: {
-    markers?: RegExp[];
-    weight?: number;
-}): StepRubric;
-/** Reward outputs that invoke the next-step tool the trajectory actually uses
- *  (i.e. the LLM span announced "I will call X" and the following tool span IS X). */
-declare function toolIntentAlignmentRubric(args?: {
-    weight?: number;
-}): StepRubric;
-
-/**
- * Inference-time PRM scoring — pick the best of N candidate trajectories
- * using a trained reward model (or a rule-based PRM as a proxy).
- *
- * The canonical Best-of-N pattern: generate N completions, score each
- * with a PRM, pick the winner. Here the scoring loop is framework-agnostic
- * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.
- */
-
-interface BestOfNResult {
-    winner: PrmGradedTrace;
-    ranked: PrmGradedTrace[];
-    /** Standard deviation of aggregate scores — small = candidates were homogenous. */
-    stdDev: number;
-}
-interface PrmBestOfNOptions {
-    /** Max concurrent `grader.grade` calls. Default 4. */
-    concurrency?: number;
-}
-declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[], options?: PrmBestOfNOptions): Promise<BestOfNResult>;
-/**
- * Weighted vote across multiple graders — use when you want a PRM ensemble
- * (e.g. rule-based + LLM-based + trained model). Each grader produces its
- * own ranking; we aggregate via rank-sum (Borda count) so no single grader
- * dominates via a different score scale.
- */
-declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[], options?: PrmBestOfNOptions): Promise<BestOfNResult>;
-
-export { type BestOfNResult, type PrmBestOfNOptions, PrmGradedTrace, PrmGrader, type PrmTrainingSample, StepRubric, exportTrainingData, nonRefusalRubric, outputLengthRubric, prmBestOfN, prmEnsembleBestOfN, toNdjson, toolIntentAlignmentRubric, toolNonRedundantRubric, toolSuccessRubric };
+export * from './builtin-rubrics';
+export * from './inference';
+export * from './rubric';
+export * from './training-export';
+//# sourceMappingURL=index.d.ts.map

package/dist/prm/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/prm/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAA;AACjC,cAAc,aAAa,CAAA;AAC3B,cAAc,UAAU,CAAA;AACxB,cAAc,mBAAmB,CAAA"}

package/dist/prm/inference.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Inference-time PRM scoring — pick the best of N candidate trajectories
+ * using a trained reward model (or a rule-based PRM as a proxy).
+ *
+ * The canonical Best-of-N pattern: generate N completions, score each
+ * with a PRM, pick the winner. Here the scoring loop is framework-agnostic
+ * — supply a TraceStore + PrmGrader + N run IDs → get ranking + winner.
+ */
+import type { TraceStore } from '../trace/store';
+import type { PrmGradedTrace, PrmGrader } from './rubric';
+export interface BestOfNResult {
+    winner: PrmGradedTrace;
+    ranked: PrmGradedTrace[];
+    /** Standard deviation of aggregate scores — small = candidates were homogenous. */
+    stdDev: number;
+}
+export interface PrmBestOfNOptions {
+    /** Max concurrent `grader.grade` calls. Default 4. */
+    concurrency?: number;
+}
+export declare function prmBestOfN(store: TraceStore, grader: PrmGrader, runIds: string[], options?: PrmBestOfNOptions): Promise<BestOfNResult>;
+/**
+ * Weighted vote across multiple graders — use when you want a PRM ensemble
+ * (e.g. rule-based + LLM-based + trained model). Each grader produces its
+ * own ranking; we aggregate via rank-sum (Borda count) so no single grader
+ * dominates via a different score scale.
+ */
+export declare function prmEnsembleBestOfN(store: TraceStore, graders: PrmGrader[], runIds: string[], options?: PrmBestOfNOptions): Promise<BestOfNResult>;
+//# sourceMappingURL=inference.d.ts.map

package/dist/prm/inference.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference.d.ts","sourceRoot":"","sources":["../../src/prm/inference.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,KAAK,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,UAAU,CAAA;AAEzD,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,cAAc,CAAA;IACtB,MAAM,EAAE,cAAc,EAAE,CAAA;IACxB,mFAAmF;IACnF,MAAM,EAAE,MAAM,CAAA;CACf;AAMD,MAAM,WAAW,iBAAiB;IAChC,sDAAsD;IACtD,WAAW,CAAC,EAAE,MAAM,CAAA;CACrB;AAED,wBAAsB,UAAU,CAC9B,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,SAAS,EACjB,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,aAAa,CAAC,CAiBxB;AAED;;;;;GAKG;AACH,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,UAAU,EACjB,OAAO,EAAE,SAAS,EAAE,EACpB,MAAM,EAAE,MAAM,EAAE,EAChB,OAAO,GAAE,iBAAsB,GAC9B,OAAO,CAAC,aAAa,CAAC,CAwExB"}

package/dist/prm/inference.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=inference.test.d.ts.map

package/dist/prm/inference.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"inference.test.d.ts","sourceRoot":"","sources":["../../src/prm/inference.test.ts"],"names":[],"mappings":""}

package/dist/{rubric-Cc6UHvUb.d.ts → prm/rubric.d.ts}

@@ -1,7 +1,3 @@
-import { S as Span, J as JudgeSpan } from './schema-m0gsnbt3.js';
-import { T as TraceStore } from './store-BcFXE6LG.js';
-import { T as Trajectory, a as TrajectoryStep } from './trajectory-2TkpSEVh.js';
-
 /**
  * Process Reward Modeling — per-step rubric grading.
  *
@@ -15,8 +11,10 @@ import { T as Trajectory, a as TrajectoryStep } from './trajectory-2TkpSEVh.js';
  * densify the signal so optimizers and RL fine-tuning can assign
  * credit per turn.
  */
-
-interface StepContext {
+import type { JudgeSpan, Span } from '../trace/schema';
+import type { TraceStore } from '../trace/store';
+import { type Trajectory, type TrajectoryStep } from '../trajectory';
+export interface StepContext {
     trajectory: Trajectory;
     step: TrajectoryStep;
     /** Steps preceding `step` in trajectory order. */
@@ -24,7 +22,7 @@ interface StepContext {
     /** Steps following `step`. */
     next: TrajectoryStep[];
 }
-interface StepRubric {
+export interface StepRubric {
     id: string;
     /** Only grade spans of these kinds (default: all). */
     kinds?: Array<Span['kind']>;
@@ -38,7 +36,7 @@ interface StepRubric {
         evidence?: string;
     } | null>;
 }
-interface GradedStep {
+export interface GradedStep {
     spanId: string;
     rubricId: string;
     score: number;
@@ -46,7 +44,7 @@ interface GradedStep {
     rationale?: string;
     evidence?: string;
 }
-interface PrmGradedTrace {
+export interface PrmGradedTrace {
     runId: string;
     steps: GradedStep[];
     /** Weighted mean of all graded steps; 0..1. */
@@ -56,7 +54,7 @@ interface PrmGradedTrace {
     /** Number of spans in the trajectory that no rubric matched. */
     ungradedCount: number;
 }
-declare class PrmGrader {
+export declare class PrmGrader {
     private rubrics;
     constructor(rubrics: StepRubric[]);
     /**
@@ -68,6 +66,5 @@ declare class PrmGrader {
 }
 /** Helper: reads JudgeVerdict spans that PRM emitted so downstream pipelines
  *  can distinguish PRM verdicts from human or top-level LLM judges. */
-declare function isPrmVerdict(verdict: JudgeSpan): boolean;
-
-export { type GradedStep as G, type PrmGradedTrace as P, type StepRubric as S, PrmGrader as a, type StepContext as b, isPrmVerdict as i };
+export declare function isPrmVerdict(verdict: JudgeSpan): boolean;
+//# sourceMappingURL=rubric.d.ts.map

package/dist/prm/rubric.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"rubric.d.ts","sourceRoot":"","sources":["../../src/prm/rubric.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAGH,OAAO,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,iBAAiB,CAAA;AACtD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAChD,OAAO,EAAmB,KAAK,UAAU,EAAE,KAAK,cAAc,EAAE,MAAM,eAAe,CAAA;AAErF,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,UAAU,CAAA;IACtB,IAAI,EAAE,cAAc,CAAA;IACpB,kDAAkD;IAClD,KAAK,EAAE,cAAc,EAAE,CAAA;IACvB,8BAA8B;IAC9B,IAAI,EAAE,cAAc,EAAE,CAAA;CACvB;AAED,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAA;IACV,sDAAsD;IACtD,KAAK,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAA;IAC3B,gDAAgD;IAChD,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;4DACwD;IACxD,KAAK,EAAE,CACL,GAAG,EAAE,WAAW,KACb,OAAO,CAAC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,IAAI,CAAC,CAAA;CAC9E;AAED,MAAM,WAAW,UAAU;IACzB,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,UAAU,EAAE,CAAA;IACnB,+CAA+C;IAC/C,cAAc,EAAE,MAAM,CAAA;IACtB,oEAAoE;IACpE,WAAW,EAAE,MAAM,CAAA;IACnB,gEAAgE;IAChE,aAAa,EAAE,MAAM,CAAA;CACtB;AAED,qBAAa,SAAS;IACR,OAAO,CAAC,OAAO;gBAAP,OAAO,EAAE,UAAU,EAAE;IAIzC;;;;OAIG;IACG,KAAK,CAAC,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,CAAC;CAgDvE;AAED;uEACuE;AACvE,wBAAgB,YAAY,CAAC,OAAO,EAAE,SAAS,GAAG,OAAO,CAExD"}

package/dist/prm/training-export.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Export PRM-graded traces as training data for downstream reward-model
+ * fine-tuning. Canonical format is NDJSON of
+ * `{ trajectory_text, step_index, rubric, score }` so a small model can
+ * learn to predict step rewards from step context.
+ *
+ * The framework doesn't train the model — we emit the data; callers
+ * plug it into their preferred trainer (TRL, Unsloth, custom).
+ */
+import type { Span } from '../trace/schema';
+import type { TraceStore } from '../trace/store';
+import type { PrmGradedTrace } from './rubric';
+export interface PrmTrainingSample {
+    runId: string;
+    spanId: string;
+    rubricId: string;
+    score: number;
+    /** Serialized step context — step + surrounding conversation. */
+    context: {
+        priorTurns: Array<{
+            role: string;
+            content: string;
+        }>;
+        step: {
+            kind: Span['kind'];
+            text: string;
+        };
+    };
+    /** Optional evidence + rationale for auditability. */
+    rationale?: string;
+    evidence?: string;
+}
+export declare function exportTrainingData(store: TraceStore, graded: PrmGradedTrace[], options?: {
+    contextWindow?: number;
+}): Promise<PrmTrainingSample[]>;
+/** NDJSON serialization — write to file or stream directly to a trainer. */
+export declare function toNdjson(samples: PrmTrainingSample[]): string;
+//# sourceMappingURL=training-export.d.ts.map

package/dist/prm/training-export.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"training-export.d.ts","sourceRoot":"","sources":["../../src/prm/training-export.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAW,IAAI,EAAE,MAAM,iBAAiB,CAAA;AAEpD,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAA;AAEhD,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,UAAU,CAAA;AAE9C,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;IACb,iEAAiE;IACjE,OAAO,EAAE;QACP,UAAU,EAAE,KAAK,CAAC;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,CAAC,CAAA;QACpD,IAAI,EAAE;YAAE,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,CAAA;KAC3C,CAAA;IACD,sDAAsD;IACtD,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAA;CAClB;AAED,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,UAAU,EACjB,MAAM,EAAE,cAAc,EAAE,EACxB,OAAO,GAAE;IAAE,aAAa,CAAC,EAAE,MAAM,CAAA;CAAO,GACvC,OAAO,CAAC,iBAAiB,EAAE,CAAC,CA4B9B;AAED,4EAA4E;AAC5E,wBAAgB,QAAQ,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAE7D"}

package/dist/produced-state.d.ts

@@ -0,0 +1,63 @@
+/**
+ * Produced-state extraction — normalize a run's runtime event stream into the
+ * typed `ProducedState` the completion oracle consumes.
+ *
+ * `ProducedState` answers "what did the agent actually produce" — vault
+ * artifacts, proposals, tool calls. The runtime emits these as a stream of
+ * events; this module is the single normalization point from that stream to
+ * the shape `verifyCompletion` expects.
+ *
+ * Input is structurally typed (`RuntimeEventLike`) so this module does not
+ * depend on agent-runtime — agent-runtime's `RuntimeStreamEvent` satisfies it
+ * structurally. The `content` on `ArtifactEventLike` and the whole
+ * `proposal_created` variant are the runtime-side enrichments this contract
+ * requires; the runtime emits them, this module consumes them.
+ */
+import type { ProducedState } from './completion-verifier';
+/** A tool the agent invoked. */
+export interface ToolCallEventLike {
+    type: 'tool_call';
+    toolName: string;
+}
+/**
+ * An artifact the agent produced. `content` is the enriched field — the
+ * runtime's base `artifact` event carries only metadata; the completion
+ * oracle needs the body to verify the deliverable, so the runtime emits it.
+ */
+export interface ArtifactEventLike {
+    type: 'artifact';
+    artifactId: string;
+    name?: string;
+    mimeType?: string;
+    uri?: string;
+    content?: string;
+}
+/** A proposal / filing the agent created. */
+export interface ProposalEventLike {
+    type: 'proposal_created';
+    proposalId: string;
+    title: string;
+    status?: 'pending' | 'approved' | 'rejected';
+    content?: string;
+}
+/**
+ * The subset of runtime stream events `extractProducedState` consumes.
+ * agent-runtime's full `RuntimeStreamEvent` union satisfies this structurally;
+ * the `{ type: string }` catch-all keeps the input permissive so callers can
+ * pass the whole unfiltered telemetry stream — unrecognized events are skipped.
+ */
+export type RuntimeEventLike = ToolCallEventLike | ArtifactEventLike | ProposalEventLike | {
+    type: string;
+};
+/**
+ * Normalize a run's runtime event stream into `ProducedState`.
+ *
+ * Pure and total — unrecognized event types are skipped. `toolCalls` is
+ * deduplicated by name in first-seen order (completion cares about a tool's
+ * presence, not its call count). An artifact with neither a name nor a uri
+ * still yields an entry keyed by its `artifactId` so it is never silently
+ * dropped; an artifact with no `content` yields empty content, which the
+ * completion oracle's structural check then rejects on its own.
+ */
+export declare function extractProducedState(events: readonly RuntimeEventLike[]): ProducedState;
+//# sourceMappingURL=produced-state.d.ts.map

package/dist/produced-state.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"produced-state.d.ts","sourceRoot":"","sources":["../src/produced-state.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAGH,OAAO,KAAK,EAAoB,aAAa,EAAE,MAAM,uBAAuB,CAAA;AAE5E,gCAAgC;AAChC,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,WAAW,CAAA;IACjB,QAAQ,EAAE,MAAM,CAAA;CACjB;AAED;;;;GAIG;AACH,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,UAAU,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,IAAI,CAAC,EAAE,MAAM,CAAA;IACb,QAAQ,CAAC,EAAE,MAAM,CAAA;IACjB,GAAG,CAAC,EAAE,MAAM,CAAA;IACZ,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED,6CAA6C;AAC7C,MAAM,WAAW,iBAAiB;IAChC,IAAI,EAAE,kBAAkB,CAAA;IACxB,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,CAAC,EAAE,SAAS,GAAG,UAAU,GAAG,UAAU,CAAA;IAI5C,OAAO,CAAC,EAAE,MAAM,CAAA;CACjB;AAED;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GACxB,iBAAiB,GACjB,iBAAiB,GACjB,iBAAiB,GACjB;IAAE,IAAI,EAAE,MAAM,CAAA;CAAE,CAAA;AASpB;;;;;;;;;GASG;AACH,wBAAgB,oBAAoB,CAAC,MAAM,EAAE,SAAS,gBAAgB,EAAE,GAAG,aAAa,CAgCvF"}

package/dist/produced-state.test.d.ts

@@ -0,0 +1,8 @@
+/**
+ * extractProducedState is the single normalization point from a run's event
+ * stream to the `ProducedState` the completion oracle consumes. These tests
+ * pin that mapping — a regression here silently corrupts every completion
+ * verdict downstream.
+ */
+export {};
+//# sourceMappingURL=produced-state.test.d.ts.map

package/dist/produced-state.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"produced-state.test.d.ts","sourceRoot":"","sources":["../src/produced-state.test.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}

package/dist/profile/baselines.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Strong, generically-useful baseline ROLES — the top zone of an `AgentProfile`
+ * before any domain layer. A product composes one of these with its own
+ * environment description (its sandbox) and its domain guidance, which lives in
+ * the product repo (not here): a profile is `<baseline role> + <environment> +
+ * <domain sections>`. Domain strength is NOT generalized into the substrate —
+ * it is lifted here only once ≥2 products genuinely reuse it.
+ *
+ * Three roles cover the common shapes:
+ *   - `engineerRole`   — builds + verifies real artifacts in a sandbox. Distilled
+ *                        from agent-runtime's `coderProfile` doctrine (minimal
+ *                        correct change, run the checks, fix the root cause,
+ *                        never weaken a test or hide an error).
+ *   - `researcherRole` — gathers from real sources and grounds every claim
+ *                        (cite or mark inferred; never fabricate).
+ *   - `generalistRole` — a strong default: helpful, grounded, verify-before-assert.
+ *
+ * All three are verification-first and domain-agnostic. They describe HOW a
+ * capable IC operates, not WHAT domain it works in.
+ */
+/** Senior-IC engineer operating in a sandbox — produces real artifacts and
+ *  verifies them before declaring done. The shared operator foundation for any
+ *  file-producing, tool-using product agent. */
+export declare const engineerRole: string;
+/** Researcher who grounds every claim in real sources — gathers, cites, and
+ *  distinguishes verified fact from inference. */
+export declare const researcherRole: string;
+/** Strong general-purpose default — helpful, grounded, verifies before asserting. */
+export declare const generalistRole: string;
+/** The named baseline roles, for selection by key (e.g. from a product config). */
+export declare const BASELINE_ROLES: {
+    readonly engineer: string;
+    readonly researcher: string;
+    readonly generalist: string;
+};
+export type BaselineRoleKey = keyof typeof BASELINE_ROLES;
+//# sourceMappingURL=baselines.d.ts.map

package/dist/profile/baselines.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"baselines.d.ts","sourceRoot":"","sources":["../../src/profile/baselines.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH;;gDAEgD;AAChD,eAAO,MAAM,YAAY,EAAE,MASf,CAAA;AAEZ;kDACkD;AAClD,eAAO,MAAM,cAAc,EAAE,MASjB,CAAA;AAEZ,qFAAqF;AACrF,eAAO,MAAM,cAAc,EAAE,MASjB,CAAA;AAEZ,mFAAmF;AACnF,eAAO,MAAM,cAAc;;;;CAIjB,CAAA;AAEV,MAAM,MAAM,eAAe,GAAG,MAAM,OAAO,cAAc,CAAA"}