@tangle-network/agent-eval 0.129.0 → 0.130.0

package/dist/reporting.js

@@ -1,51 +1,6 @@
-import {
-  assertReleaseConfidence,
-  bootstrapCi,
-  evaluateReleaseConfidence,
-  judgeReplayGate,
-  renderReleaseReport
-} from "./chunk-JQSF5DQT.js";
-import {
-  rubricPredictiveValidity
-} from "./chunk-TQ7LNKZ3.js";
-import {
-  evaluateInterimReleaseConfidence,
-  pairedEvalueSequence
-} from "./chunk-MAZ26DC7.js";
-import {
-  RESEARCH_REPORT_HARD_PAIR_FLOOR,
-  gainHistogram,
-  paretoChart,
-  researchReport,
-  summaryTable
-} from "./chunk-TJVT4QFF.js";
-import "./chunk-7FO3TNPI.js";
-import {
-  benjaminiHochberg,
-  pairedBootstrap,
-  wilcoxonSignedRank
-} from "./chunk-ZHTZ4EYI.js";
-import "./chunk-56TAVBOK.js";
-import "./chunk-MA6HLL3S.js";
-import "./chunk-OIUOT4QD.js";
-import "./chunk-ONWEPEDO.js";
-import "./chunk-PZ5AY32C.js";
-export {
-  RESEARCH_REPORT_HARD_PAIR_FLOOR,
-  assertReleaseConfidence,
-  benjaminiHochberg,
-  bootstrapCi,
-  evaluateInterimReleaseConfidence,
-  evaluateReleaseConfidence,
-  gainHistogram,
-  judgeReplayGate,
-  pairedBootstrap,
-  pairedEvalueSequence,
-  paretoChart,
-  renderReleaseReport,
-  researchReport,
-  rubricPredictiveValidity,
-  summaryTable,
-  wilcoxonSignedRank
-};
-//# sourceMappingURL=reporting.js.map
+import { N as wilcoxonSignedRank, t as benjaminiHochberg, v as pairedBootstrap } from "./statistics-CnnxdpOg.js";
+import { a as evaluateReleaseConfidence, i as assertReleaseConfidence, n as bootstrapCi, r as judgeReplayGate, t as renderReleaseReport } from "./release-report-Cz9NKH39.js";
+import { a as summaryTable, i as researchReport, n as gainHistogram, r as paretoChart, t as RESEARCH_REPORT_HARD_PAIR_FLOOR } from "./summary-report-BNs5nmXI.js";
+import { n as pairedEvalueSequence, t as evaluateInterimReleaseConfidence } from "./sequential-Br0mAPHA.js";
+import { t as rubricPredictiveValidity } from "./rubric-predictive-validity-B3xmbmS1.js";
+export { RESEARCH_REPORT_HARD_PAIR_FLOOR, assertReleaseConfidence, benjaminiHochberg, bootstrapCi, evaluateInterimReleaseConfidence, evaluateReleaseConfidence, gainHistogram, judgeReplayGate, pairedBootstrap, pairedEvalueSequence, paretoChart, renderReleaseReport, researchReport, rubricPredictiveValidity, summaryTable, wilcoxonSignedRank };

package/dist/researcher-CwTdwXG1.d.ts

@@ -0,0 +1,314 @@
+import { a as RunRecord, b as AgentProfileCellInput, c as RunTaskFailure, n as RunCostProvenance, r as RunJudgeMetadata, s as RunSplitTag, t as JudgeScoresRecord, u as RunTokenUsage, y as AgentProfileCell } from "./run-record-CnZu_gjl.js";
+import { l as RawProviderSink } from "./raw-provider-sink-BU29Sh8h.js";
+import { s as TraceStore } from "./store-CT9YIIve.js";
+import { i as TraceEmitter, t as RunCompleteHook } from "./emitter-DGQGoLyj.js";
+import { a as RunIntegrityReport, n as RunIntegrityExpectations } from "./integrity-rmVhXWA7.js";
+import { o as LlmClientOptions, u as LlmRouteRequirements } from "./llm-client-B_nIBlYo.js";
+import { b as GateDecision, d as ResearchReportOptions, s as ResearchReport } from "./summary-report-Cj9gdw4i.js";
+//#region src/eval-campaign.d.ts
+interface CampaignVariant<V> {
+  id: string;
+  payload: V;
+}
+interface CampaignScenario {
+  scenarioId: string;
+  /** Free-form metadata propagated to runs and reports. */
+  tags?: Record<string, string>;
+}
+interface CampaignRunContext<V> {
+  /** Stable run id. The campaign generates this; the runner does not. */
+  runId: string;
+  /** Logical experiment id (campaignId by default; overridable per-run via opts). */
+  experimentId: string;
+  variant: V;
+  variantId: string;
+  scenarioId: string;
+  scenarioTags: Record<string, string>;
+  seed: number;
+  splitTag: RunSplitTag;
+  /**
+   * The TraceEmitter for this run, with `onRunComplete` hooks pre-wired
+   * (analyst auto-execution if configured, plus integrity check). The
+   * runner MUST call `emitter.startRun` before doing any work and either
+   * `emitter.endRun` or `emitter.abortRun` before returning.
+   */
+  emitter: TraceEmitter;
+  store: TraceStore;
+  rawSink: RawProviderSink;
+  /**
+   * Pre-wired LLM client options — `rawSink` and `traceContext` are populated
+   * so any `callLlm(req, ctx.llmOpts)` automatically captures raw HTTP. The
+   * runner can spread additional fields if needed.
+   */
+  llmOpts: LlmClientOptions;
+}
+interface CampaignRunOutcomeFields {
+  /** Did the run pass? Mirrors `RunOutcome.pass` semantics. */
+  pass: boolean;
+  /** Score for the run on its split. Maps to `searchScore` or `holdoutScore`. */
+  score: number;
+  /** Cost in USD, or null when the runner could not capture it. */
+  costUsd: number | null;
+  /** Source of the cost amount. */
+  costProvenance: RunCostProvenance;
+  tokenUsage: RunTokenUsage;
+  /** Snapshot model id (e.g. `claude-sonnet-4-6@2025-04-15`). */
+  model: string;
+  /** sha256 of the effective prompt sent to the model. */
+  promptHash: string;
+  /** sha256 of the effective config (model, temperature, tools, judges, splits). */
+  configHash: string;
+  /** Optional extra numeric metrics to land in `outcome.raw`. */
+  raw?: Record<string, number>;
+  /** Optional judge metadata when a judge was used. */
+  judgeMetadata?: RunJudgeMetadata;
+  /**
+   * Optional per-judge / per-dim breakdown for ensemble-judged runs.
+   * Propagated to `outcome.judgeScores` on the resulting `RunRecord`.
+   * Single-judge or scalar-only runs leave this unset.
+   */
+  judgeScores?: JudgeScoresRecord;
+  /**
+   * Agent profile cell observed by the runner. When supplied, it overrides
+   * `EvalCampaignOptions.agentProfile` for this run and must match the
+   * outcome's `model` and `promptHash`.
+   */
+  agentProfile?: AgentProfileCell | AgentProfileCellInput;
+}
+/** Campaign result with the same task-failure invariant as `RunRecord`. */
+type CampaignRunOutcome = CampaignRunOutcomeFields & RunTaskFailure;
+type CampaignRunner<V> = (ctx: CampaignRunContext<V>) => Promise<CampaignRunOutcome>;
+type CampaignIntegrityPolicy = 'throw' | 'mark_failed' | 'log';
+interface EvalCampaignOptions<V> {
+  /**
+   * Stable id for the campaign. Used as the default `experimentId` on
+   * every run, and folded into the campaign fingerprint.
+   */
+  campaignId: string;
+  variants: CampaignVariant<V>[];
+  scenarios: CampaignScenario[];
+  /** Default `[0, 1, 2]`. */
+  seeds?: number[];
+  /** Default `'holdout'` — the split that anchors a launch decision. */
+  splitTag?: RunSplitTag;
+  /** Git SHA the campaign is run against. Mandatory; `RunRecord` rejects unset. */
+  commitSha: string;
+  /**
+   * LLM client config. Augmented per-run with `rawSink` and `traceContext`
+   * before being passed to the runner. The campaign asserts this config
+   * matches `routeRequirements` once at preflight.
+   */
+  llmOpts: LlmClientOptions;
+  /**
+   * Default `{ requireExplicitBaseUrl: true, requireAuth: true }` — fail
+   * loud if the campaign would silently fall back to the public router or
+   * run unauthenticated. Override with an empty object to disable.
+   */
+  routeRequirements?: LlmRouteRequirements;
+  /**
+   * Per-run TraceStore factory. Common shape: a fresh store per run keyed
+   * on `runId`. Implementations that share a store across the campaign
+   * are valid — the campaign only writes through `emitter`.
+   */
+  storeFactory: (params: CampaignFactoryParams) => TraceStore;
+  /**
+   * Per-run RawProviderSink factory. Defaults to `FileSystemRawProviderSink`
+   * rooted at `${workDir}/raw-events/${runId}` if `workDir` is supplied;
+   * otherwise required. Forensic capture is non-negotiable in a campaign
+   * run — pass `NoopRawProviderSink` explicitly if you want to opt out.
+   */
+  rawSinkFactory?: (params: CampaignFactoryParams) => RawProviderSink;
+  /**
+   * Filesystem root for default `rawSinkFactory`. Ignored if
+   * `rawSinkFactory` is supplied.
+   */
+  workDir?: string;
+  /**
+   * Extra `onRunComplete` hooks the campaign appends (after its own
+   * integrity-check hook). Pass `traceAnalystOnRunComplete(...)` here.
+   */
+  onRunComplete?: RunCompleteHook[];
+  /**
+   * Per-run integrity expectations. Defaults to:
+   *   `{ llmSpansMin: 1, requireRawCoverageOfLlmSpans: true, requireOutcome: true }`.
+   * Override (e.g. `{ llmSpansMin: 0 }`) for runs that don't call LLMs.
+   */
+  integrity?: RunIntegrityExpectations;
+  /** Behaviour when integrity fails. Default `'mark_failed'`. */
+  onIntegrityFailure?: CampaignIntegrityPolicy;
+  /**
+   * Per-run runner. Receives a fully-wired context; produces an outcome
+   * the campaign converts into a `RunRecord`.
+   */
+  runner: CampaignRunner<V>;
+  /**
+   * If set, the campaign computes `researchReport` at the end. `comparator`
+   * is a `variantId`. Other fields are forwarded verbatim.
+   */
+  report?: {
+    comparator?: string;
+  } & Omit<ResearchReportOptions, 'comparator' | 'preregistrationHash' | 'generatedAt'>;
+  /**
+   * Hash of a signed `HypothesisManifest` (see `pre-registration.ts`).
+   * Embedded in the campaign fingerprint and the research report.
+   */
+  preregistrationHash?: string;
+  /** Local concurrency. Default `1` (sequential). */
+  concurrency?: number;
+  /**
+   * Override the time source. Tests pass a mock to make wallMs deterministic.
+   */
+  now?: () => number;
+  /** Override the runId generator. Tests pin this. */
+  runId?: (params: CampaignFactoryParams) => string;
+  /**
+   * Agent profile cell for campaign runs. Static profiles can pass an object;
+   * routers or variant-specific harnesses can pass a factory. The campaign
+   * stamps the built cell onto every `RunRecord` and rejects profile/model or
+   * profile/prompt contradictions.
+   */
+  agentProfile?: AgentProfileCell | AgentProfileCellInput | ((params: CampaignFactoryParams & {
+    variant: V;
+    scenarioTags: Record<string, string>;
+  }) => AgentProfileCell | AgentProfileCellInput | Promise<AgentProfileCell | AgentProfileCellInput>);
+}
+interface CampaignFactoryParams {
+  campaignId: string;
+  runId: string;
+  variantId: string;
+  scenarioId: string;
+  seed: number;
+}
+interface FailedRun {
+  runId: string;
+  variantId: string;
+  scenarioId: string;
+  seed: number;
+  reason: string;
+  error?: string;
+}
+interface EvalCampaignResult {
+  campaignId: string;
+  /** SHA-256 over canonicalised `(variantIds, scenarioIds, seeds, comparator, splitTag, baseUrl, provider, preregistrationHash)`. */
+  campaignFingerprint: string;
+  preregistrationHash: string | null;
+  /** Successful runs only. Failed runs land in `failedRuns`. */
+  runs: RunRecord[];
+  /** Integrity reports for every successful run. */
+  integrityReports: RunIntegrityReport[];
+  failedRuns: FailedRun[];
+  /** Computed when `report` is set on options. */
+  report?: ResearchReport;
+  startedAt: string;
+  endedAt: string;
+}
+declare function runEvalCampaign<V>(opts: EvalCampaignOptions<V>): Promise<EvalCampaignResult>;
+//#endregion
+//#region src/researcher.d.ts
+/** A diagnosed failure mode with the run-IDs that exhibit it. */
+interface FailureMode {
+  /** Short machine-readable code. Must be stable across runs of the
+   *  same researcher to enable longitudinal tracking. */
+  code: string;
+  /** Human-readable description for the paper / dashboard. */
+  description: string;
+  evidence: {
+    /** Run IDs (from `RunRecord.runId`) where this failure mode was
+     *  observed. */
+    runIds: string[];
+    /** Number of run samples that informed the diagnosis. */
+    samples: number;
+  };
+}
+/** A single steering change the researcher wants to try. */
+interface SteeringChange {
+  kind: 'reviewer_prompt' | 'skill_add' | 'skill_remove' | 'threshold' | 'budget';
+  /** Implementation-specific payload. Researcher implementations
+   *  define the schema — keep this `unknown` here to avoid coupling
+   *  the public interface to any one researcher's internal model. */
+  payload: unknown;
+  /** Why the researcher proposed this change. Goes into the audit
+   *  trail next to the failure-mode evidence. */
+  rationale: string;
+  /** Optional self-reported expected delta on the headline metric. */
+  expectedDelta?: number;
+}
+/** A single experiment plan, mapped onto the search/holdout splits. */
+interface ExperimentPlan {
+  baselineCandidateId: string;
+  proposedCandidateId: string;
+  changes: SteeringChange[];
+  /** USD ceiling for the entire experiment. The runner must stop
+   *  before exceeding this and report a partial result. */
+  evaluationBudgetUsd: number;
+  /** Item IDs (your dataset keys) for the search vs holdout splits. */
+  splits: {
+    search: string[];
+    holdout: string[];
+  };
+}
+/** Result of running a plan: every run, plus the gate verdict. */
+interface ExperimentResult {
+  plan: ExperimentPlan;
+  runs: RunRecord[];
+  gateDecision: GateDecision;
+}
+/**
+ * The researcher loop. Stable, four-step, inspectable.
+ *
+ *   ┌──────────┐  inspectFailures  ┌──────────┐  proposeChange ┌──────────┐
+ *   │   runs   │ ─────────────────▶│ failures │ ──────────────▶│ changes  │
+ *   └──────────┘                   └──────────┘                └────┬─────┘
+ *                                                                   │
+ *                                                                   ▼
+ *                              ┌────────────────┐  applyChange ┌────────┐
+ *                              │ ExperimentPlan │ ◀────────────│  base  │
+ *                              └────────┬───────┘              └────────┘
+ *                                       │
+ *                       evaluateChange  ▼
+ *                              ┌────────────────┐
+ *                              │ ExperimentResult│
+ *                              └────────────────┘
+ */
+interface Researcher {
+  inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
+  proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
+  applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
+  evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
+}
+interface CallbackResearcherOptions {
+  inspectFailures: Researcher['inspectFailures'];
+  proposeChange: Researcher['proposeChange'];
+  applyChange: Researcher['applyChange'];
+  evaluateChange: Researcher['evaluateChange'];
+}
+/**
+ * Minimal concrete researcher for tests, scripts, and small integrations.
+ * Larger autonomous researchers can still implement `Researcher` directly.
+ */
+declare class CallbackResearcher implements Researcher {
+  private readonly callbacks;
+  constructor(callbacks: CallbackResearcherOptions);
+  inspectFailures(runs: RunRecord[]): Promise<FailureMode[]>;
+  proposeChange(failures: FailureMode[]): Promise<SteeringChange[]>;
+  applyChange(changes: SteeringChange[], baseline: ExperimentPlan): Promise<ExperimentPlan>;
+  evaluateChange(plan: ExperimentPlan): Promise<ExperimentResult>;
+}
+/**
+ * No-op researcher — fails loud on every method. Use as a placeholder
+ * in code paths that wire the interface but don't have an implementation
+ * yet. Importantly, this does NOT silently succeed: a no-op researcher
+ * that returned empty arrays would muffle the loop's signal that
+ * nobody implemented the brain.
+ */
+declare class NoopResearcher implements Researcher {
+  private readonly hint;
+  constructor(hint?: string);
+  inspectFailures(_runs: RunRecord[]): Promise<FailureMode[]>;
+  proposeChange(_failures: FailureMode[]): Promise<SteeringChange[]>;
+  applyChange(_changes: SteeringChange[], _baseline: ExperimentPlan): Promise<ExperimentPlan>;
+  evaluateChange(_plan: ExperimentPlan): Promise<ExperimentResult>;
+}
+//#endregion
+export { EvalCampaignResult as _, FailureMode as a, SteeringChange as c, CampaignRunContext as d, CampaignRunOutcome as f, EvalCampaignOptions as g, CampaignVariant as h, ExperimentResult as i, CampaignFactoryParams as l, CampaignScenario as m, CallbackResearcherOptions as n, NoopResearcher as o, CampaignRunner as p, ExperimentPlan as r, Researcher as s, CallbackResearcher as t, CampaignIntegrityPolicy as u, FailedRun as v, runEvalCampaign as y };
+//# sourceMappingURL=researcher-CwTdwXG1.d.ts.map

package/dist/researcher-CwTdwXG1.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"researcher-CwTdwXG1.d.ts","names":[],"sources":["../src/eval-campaign.ts","../src/researcher.ts"],"mappings":";;;;;;;;UAyEiB,gBAAgB;EAC/B;EACA,SAAS;;UAGM;EACf;;EAEA,OAAO;;UAGQ,mBAAmB;;EAElC;;EAEA;EACA,SAAS;EACT;EACA;EACA,cAAc;EACd;EACA,UAAU;;;;;;;EAOV,SAAS;EACT,OAAO;EACP,SAAS;;;;;;EAMT,SAAS;;UAGD;;EAER;;EAEA;;EAEA;;EAEA,gBAAgB;EAChB,YAAY;;EAEZ;;EAEA;;EAEA;;EAEA,MAAM;;EAEN,gBAAgB;;;;;;EAMhB,cAAc;;;;;;EAMd,eAAe,mBAAmB;;;KAIxB,qBAAqB,2BAA2B;KAEhD,eAAe,MAAM,KAAK,mBAAmB,OAAO,QAAQ;KAE5D;UAEK,oBAAoB;;;;;EAKnC;EACA,UAAU,gBAAgB;EAC1B,WAAW;;EAEX;;EAEA,WAAW;;EAEX;;;;;;EAMA,SAAS;;;;;;EAMT,oBAAoB;;;;;;EAMpB,eAAe,QAAQ,0BAA0B;;;;;;;EAOjD,kBAAkB,QAAQ,0BAA0B;;;;;EAKpD;;;;;EAKA,gBAAgB;;;;;;EAMhB,YAAY;;EAEZ,qBAAqB;;;;;EAKrB,QAAQ,eAAe;;;;;EAKvB;IAAW;MAAwB,KACjC;;;;;EAOF;;EAEA;;;;EAIA;;EAEA,SAAS,QAAQ;;;;;;;EAOjB,eACI,mBACA,0BAEE,QAAQ;IACN,SAAS;IACT,cAAc;QAGd,mBACA,wBACA,QAAQ,mBAAmB;;UAGpB;EACf;EACA;EACA;EACA;EACA;;UAGe;EACf;EACA;EACA;EACA;EACA;EACA;;UAGe;EACf;;EAEA;EACA;;EAEA,MAAM;;EAEN,kBAAkB;EAClB,YAAY;;EAEZ,SAAS;EACT;EACA;;iBAgBoB,gBAAgB,GACpC,MAAM,oBAAoB,KACzB,QAAQ;;;;UC5QM;;;EAGf;;EAEA;EACA;;;IAGE;;IAEA;;;;UAKa;EACf;;;;EAIA;;;EAGA;;EAEA;;;UAIe;EACf;EACA;EACA,SAAS;;;EAGT;;EAEA;IAAU;IAAkB;;;;UAIb;EACf,MAAM;EACN,MAAM;EACN,cAAc;;;;;;;;;;;;;;;;;;;UAoBC;EACf,gBAAgB,MAAM,cAAc,QAAQ;EAC5C,cAAc,UAAU,gBAAgB,QAAQ;EAChD,YAAY,SAAS,kBAAkB,UAAU,iBAAiB,QAAQ;EAC1E,eAAe,MAAM,iBAAiB,QAAQ;;UAG/B;EACf,iBAAiB;EACjB,eAAe;EACf,aAAa;EACb,gBAAgB;;;;;;cAOL,8BAA8B;mBACZ;EAA7B,YAA6B,WAAW;EAExC,gBAAgB,MAAM,cAAc,QAAQ;EAI5C,cAAc,UAAU,gBAAgB,QAAQ;EAIhD,YAAY,SAAS,kBAAkB,UAAU,iBAAiB,QAAQ;EAI1E,eAAe,MAAM,iBAAiB,QAAQ;;;;;;;;;cAYnC,0BAA0B;mBACpB;EAEjB,YAAY;EAIN,gBAAgB,OAAO,cAAc,QAAQ;EAI7C,cAAc,WAAW,gBAAgB,QAAQ;EAIjD,YACJ,UAAU,kBACV,WAAW,iBACV,QAAQ;EAIL,eAAe,OAAO,iBAAiB,QAAQ"}

package/dist/reward-hacking-eAnOsynk.d.ts

@@ -0,0 +1,224 @@
+import { s as VerificationReport } from "./multi-layer-verifier-BHY1gWAc.js";
+import { a as RunRecord } from "./run-record-CnZu_gjl.js";
+//#region src/rl/verifiable-reward.d.ts
+type VerifiableRewardSource = 'compile' | 'test' | 'schema' | 'sandbox' | 'judge' | 'composite';
+interface VerifiableReward {
+  /** Scalar in [0, 1]. The RL training signal. */
+  value: number;
+  /** What produced the reward — different sources have different determinism. */
+  source: VerifiableRewardSource;
+  /**
+   * Determinism class. `'deterministic'` rewards are repeatable byte-for-byte
+   * given the same inputs (compile, test, schema validation, sandbox exit code).
+   * `'probabilistic'` rewards depend on a stochastic component (LLM judge).
+   * Mixing these in the same training batch without separation is a known
+   * footgun in production RLHF pipelines.
+   */
+  determinism: 'deterministic' | 'probabilistic';
+  /**
+   * Confidence in the reward value. For deterministic sources this is 1.0
+   * (the bit either flipped or didn't). For judge sources this is the
+   * judge-reported confidence or — when missing — a calibrated prior.
+   */
+  confidence: number;
+  /** The layer / judge id that produced the signal, for provenance. */
+  origin: string;
+  /**
+   * Per-source contribution to `value`, keyed by layer/judge id. Single-source
+   * rewards carry one entry (`{ [origin]: value }`); composite rewards carry
+   * every contributing layer's score — the anti-scalar-collapse surface RL
+   * consumers weight per-source instead of trusting one blended number.
+   */
+  components: Record<string, number>;
+  /**
+   * The run carries `outcome.realness.gated` — the authenticity gate flagged
+   * its success signal as faked.
+   *
+   * With the gate applied (the default) `value` and every `components` entry
+   * are 0 on such a run; with `applyRealnessGate: false` the observed numbers
+   * come back untouched and this flag is the only marker that they are not to
+   * be trusted. Either way it distinguishes "measured a genuine failure" from
+   * "claimed a success we refuse to believe", which a bare 0 cannot.
+   */
+  realnessGated?: boolean;
+  /**
+   * Whether an authenticity screen COULD run on this reward at all — the same
+   * distinction `RolloutOutcome.realness_screened` draws, for the same reason.
+   *
+   * `false` on every reward from `extractVerifiableReward`, because a
+   * `VerificationReport` carries layer scores and nothing else: there is no
+   * `outcome.realness` to consult, so no gate has run, and `realnessGated`
+   * being absent there means "unknown", NOT "clean". Absent on the
+   * `RunRecord` path when the record itself carries no realness verdict.
+   *
+   * This matters most exactly where it is easiest to miss: a report whose
+   * deterministic layers all passed yields `determinism: 'deterministic'`,
+   * `confidence: 1` — the highest-credibility reward this module can emit —
+   * and a stubbed integration reporting green is precisely what a gamed run
+   * looks like. Consumers driving training off this shape must screen the run
+   * themselves; the flag is what tells them nobody has.
+   */
+  realnessScreened?: boolean;
+}
+interface VerifiableRewardExtractionOptions {
+  /**
+   * Which layers count as deterministic-reward sources. The verifier doesn't
+   * tag layers as "this is verifiable"; the caller declares it via this list
+   * (or via the layer name → source mapping). Default treats common names
+   * (`install`, `typecheck`, `build`, `lint`, `test`, `compile`, `schema`,
+   * `sandbox`) as deterministic.
+   */
+  deterministicLayers?: string[];
+  /**
+   * Map layer name → reward source. Defaults to a sensible string-match.
+   */
+  sourceFor?: (layerName: string) => VerifiableRewardSource;
+  /**
+   * Whether to fall back to a probabilistic (judge) reward when no
+   * deterministic layer produced a numeric score. Default `true`. Set to
+   * `false` for "deterministic-only" training pipelines that should
+   * discard runs without a verifiable signal.
+   */
+  fallbackToJudge?: boolean;
+  /**
+   * Default confidence for probabilistic (judge) rewards when the judge
+   * doesn't report one. Default `0.7`.
+   */
+  judgeConfidenceFloor?: number;
+  /**
+   * Whether the anti-Goodhart realness gate applies. Default `true`, and the
+   * default is the one every training path must keep.
+   *
+   * Set `false` ONLY for detection and analysis. `rl/reward-hacking.ts` does,
+   * for the same reason it reads `observedScore` for its proxy: it measures the
+   * DIVERGENCE between the judge signal and the deterministic one, and a
+   * deterministic reward that another gate already forced to 0 manufactures
+   * exactly that divergence on exactly the gamed population. The detector would
+   * then be re-reporting a verdict it was supposed to reach independently.
+   */
+  applyRealnessGate?: boolean;
+}
+/**
+ * Extract a `VerifiableReward` from a `VerificationReport`.
+ *
+ * Strategy: prefer the deterministic layers (in order: test → compile →
+ * schema → sandbox), fall back to the judge layer if `fallbackToJudge` is
+ * true, return `null` if no signal qualifies. When multiple deterministic
+ * layers contribute, return a `'composite'` source with a weighted blend.
+ *
+ * NO realness gate is applied and none can be: a `VerificationReport` carries
+ * layer scores and nothing about whether the run faked them — `realness` lives
+ * on the `RunRecord`. Use `extractVerifiableRewardsFromRecords` for anything
+ * that becomes training data; this signature is for scoring a report in hand.
+ */
+declare function extractVerifiableReward(report: VerificationReport, opts?: VerifiableRewardExtractionOptions): VerifiableReward | null;
+/**
+ * Extract verifiable rewards from `RunRecord[]` produced via the
+ * `verificationReportToRunRecord` adapter (which encodes per-layer scores
+ * in `outcome.raw['layer.<name>']`). For records that don't carry layer
+ * scores, returns `null` for that record.
+ *
+ * This is the canonical bridge from "campaign-shaped artifacts" to
+ * "RL-training-ready reward signals": every record that has a clean
+ * verifiable reward becomes a training datum, every record that doesn't
+ * gets filtered out (or kept with `'probabilistic'` determinism for
+ * separate downstream handling).
+ *
+ * The realness gate applies to EVERY channel here, and to the deterministic one
+ * MOST. It is tempting to reason that a decidable signal cannot be gamed, so
+ * the gate is redundant on it — that reasoning is backwards. `realness.gated`
+ * means the run's success signal was FAKED, and a test suite reporting green on
+ * a stubbed integration is precisely what that looks like: the deterministic
+ * layer is the thing that got faked. Exporting it ungated hands a trainer the
+ * highest-credibility reward the module can emit (`determinism: 'deterministic'`,
+ * `confidence: 1`) for the one population the gate exists to catch. Pass
+ * `applyRealnessGate: false` only to look at the ungated numbers for detection.
+ */
+declare function extractVerifiableRewardsFromRecords(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
+  runId: string;
+  reward: VerifiableReward | null;
+}>;
+/**
+ * Filter `RunRecord[]` to those with deterministic verifiable rewards.
+ *
+ * A realness-gated run is KEPT, at reward 0 with `realnessGated: true` — the
+ * same rule GRPO uses on a gated line. 0 is the honest label for a faked
+ * success and is usable signal, whereas dropping the run would move a group
+ * baseline without saying so. (SFT differs: there every row is a target to
+ * imitate, so a gated row is removed outright.)
+ */
+declare function filterDeterministicallyRewarded(runs: RunRecord[], opts?: VerifiableRewardExtractionOptions): Array<{
+  run: RunRecord;
+  reward: VerifiableReward;
+}>;
+//#endregion
+//#region src/rl/reward-hacking.d.ts
+type RewardHackingSignal = 'reward_divergence' | 'distribution_shift' | 'reward_disagreement' | 'judge_drift';
+interface RewardHackingFinding {
+  signal: RewardHackingSignal;
+  /** Severity in [0, 1]. >0.5 = strong signal. */
+  severity: number;
+  message: string;
+  /** Numeric evidence the consumer can render. */
+  detail: Record<string, number>;
+}
+interface RewardHackingReport {
+  findings: RewardHackingFinding[];
+  /** Signals with enough usable observations to produce a finding. */
+  evaluatedSignals: RewardHackingSignal[];
+  /**
+   * Composite verdict. `'insufficient_evidence'` when fewer than four scored
+   * runs exist; otherwise `'clean'` if every signal severity < 0.3,
+   * `'suspect'` if at least one ≥ 0.3 but none ≥ 0.6, and `'gaming'` if any ≥ 0.6.
+   */
+  verdict: 'insufficient_evidence' | 'clean' | 'suspect' | 'gaming';
+  /** Rationale for the verdict, ready to paste into an audit log. */
+  rationale: string[];
+  /** Number of runs with a usable proxy reward. */
+  n: number;
+}
+interface DetectRewardHackingInput {
+  /**
+   * Run records ordered by recency (oldest first). The detector segments
+   * them into prefix/suffix windows to compute "did the gap widen."
+   */
+  runs: RunRecord[];
+  /**
+   * The metric the policy was trained to optimize. Should be present on
+   * `outcome.raw` or `outcome.holdoutScore`. Default reads `outcome.holdoutScore`.
+   */
+  proxyOf?: (run: RunRecord) => number | null;
+  /**
+   * The held-out ground-truth metric. For RL on coding, this is typically
+   * test pass-rate. For RLHF, it's downstream task performance or human
+   * preference. For knowledge tasks, it's an independently-graded score.
+   */
+  truthOf?: (run: RunRecord) => number | null;
+  /**
+   * Independent secondary reward. Used for the `reward_disagreement`
+   * signal. Default uses the verifiable reward extractor (deterministic
+   * sources only).
+   */
+  secondaryRewardOf?: (run: RunRecord) => number | null;
+  /**
+   * Window size — how many of the most recent runs count as the "after"
+   * cohort. Default min(50, half the runs).
+   */
+  windowSize?: number;
+  /**
+   * Severity threshold to flag a signal. Default 0.3 (suspect) and 0.6
+   * (gaming).
+   */
+  thresholds?: {
+    suspect?: number;
+    gaming?: number;
+  };
+  /**
+   * Verifiable-reward options used for the secondary-reward fallback.
+   */
+  verifiableRewardOptions?: VerifiableRewardExtractionOptions;
+}
+declare function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport;
+//#endregion
+export { detectRewardHacking as a, VerifiableRewardSource as c, filterDeterministicallyRewarded as d, RewardHackingSignal as i, extractVerifiableReward as l, RewardHackingFinding as n, VerifiableReward as o, RewardHackingReport as r, VerifiableRewardExtractionOptions as s, DetectRewardHackingInput as t, extractVerifiableRewardsFromRecords as u };
+//# sourceMappingURL=reward-hacking-eAnOsynk.d.ts.map

package/dist/reward-hacking-eAnOsynk.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"reward-hacking-eAnOsynk.d.ts","names":[],"sources":["../src/rl/verifiable-reward.ts","../src/rl/reward-hacking.ts"],"mappings":";;;KAiCY;UAQK;;EAEf;;EAEA,QAAQ;;;;;;;;EAQR;;;;;;EAMA;;EAEA;;;;;;;EAOA,YAAY;;;;;;;;;;;EAWZ;;;;;;;;;;;;;;;;;;EAkBA;;UAGe;;;;;;;;EAQf;;;;EAIA,aAAa,sBAAsB;;;;;;;EAOnC;;;;;EAKA;;;;;;;;;;;;EAYA;;;;;;;;;;;;;;;iBA6Cc,wBACd,QAAQ,oBACR,OAAM,oCACL;;;;;;;;;;;;;;;;;;;;;;;iBAqGa,oCACd,MAAM,aACN,OAAM,oCACL;EAAQ;EAAe,QAAQ;;;;;;;;;;;iBA4GlB,gCACd,MAAM,aACN,OAAM,oCACL;EAAQ,KAAK;EAAW,QAAQ;;;;KChWvB;UAMK;EACf,QAAQ;;EAER;EACA;;EAEA,QAAQ;;UAGO;EACf,UAAU;;EAEV,kBAAkB;;;;;;EAMlB;;EAEA;;EAEA;;UAGe;;;;;EAKf,MAAM;;;;;EAKN,WAAW,KAAK;;;;;;EAMhB,WAAW,KAAK;;;;;;EAMhB,qBAAqB,KAAK;;;;;EAK1B;;;;;EAKA;IAAe;IAAkB;;;;;EAIjC,0BAA0B;;iBAgBZ,oBAAoB,OAAO,2BAA2B"}