@tangle-network/agent-eval 0.86.0 → 0.90.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -1
- package/dist/adapters/http.d.ts +3 -3
- package/dist/adapters/langchain.d.ts +3 -3
- package/dist/adapters/otel.d.ts +6 -6
- package/dist/adversarial-DIVcDoI_.d.ts +88 -0
- package/dist/analyst/index.d.ts +11 -10
- package/dist/analyst/index.js +13 -8
- package/dist/analyst/index.js.map +1 -1
- package/dist/analyze-runs-DwCEkpO_.d.ts +81 -0
- package/dist/belief-state/index.d.ts +4 -4
- package/dist/belief-state/index.js +1 -1
- package/dist/benchmarks/index.d.ts +3 -3
- package/dist/campaign/index.d.ts +165 -18
- package/dist/campaign/index.js +289 -14
- package/dist/campaign/index.js.map +1 -1
- package/dist/chunk-45EEMHTC.js +35 -0
- package/dist/chunk-45EEMHTC.js.map +1 -0
- package/dist/{chunk-FZWAFVAA.js → chunk-4FBZZIYD.js} +2 -2
- package/dist/{chunk-YV7J7X5N.js → chunk-5HRORJQY.js} +22 -12
- package/dist/chunk-5HRORJQY.js.map +1 -0
- package/dist/{chunk-OTYQPHPL.js → chunk-6SOJM3VR.js} +5 -5
- package/dist/chunk-BOD4O7OF.js +40 -0
- package/dist/chunk-BOD4O7OF.js.map +1 -0
- package/dist/{chunk-Z7VFTS2J.js → chunk-CY6U5S3X.js} +2 -2
- package/dist/{chunk-VIDQF3F5.js → chunk-D3V5B42D.js} +5 -34
- package/dist/chunk-D3V5B42D.js.map +1 -0
- package/dist/{chunk-YGYXHNAQ.js → chunk-FIUKOSWI.js} +21 -8
- package/dist/chunk-FIUKOSWI.js.map +1 -0
- package/dist/{chunk-WJL2NJXN.js → chunk-GSH6QNNS.js} +2 -2
- package/dist/{chunk-RBNA5AZT.js → chunk-L3JOU6XM.js} +2 -2
- package/dist/{chunk-IDVBLYCY.js → chunk-LMZQ2Z4U.js} +56 -2
- package/dist/{chunk-IDVBLYCY.js.map → chunk-LMZQ2Z4U.js.map} +1 -1
- package/dist/{chunk-VUINJM5M.js → chunk-QAY5UIJO.js} +2 -193
- package/dist/chunk-QAY5UIJO.js.map +1 -0
- package/dist/{chunk-P2J6SOXT.js → chunk-QG2OVF2D.js} +5 -3
- package/dist/{chunk-P2J6SOXT.js.map → chunk-QG2OVF2D.js.map} +1 -1
- package/dist/chunk-REVYNR6C.js +100 -0
- package/dist/chunk-REVYNR6C.js.map +1 -0
- package/dist/chunk-STGVSCDH.js +202 -0
- package/dist/chunk-STGVSCDH.js.map +1 -0
- package/dist/{chunk-ZZ2HOPME.js → chunk-TWS7AZEY.js} +2 -2
- package/dist/chunk-UHMJT4T7.js +200 -0
- package/dist/chunk-UHMJT4T7.js.map +1 -0
- package/dist/chunk-UMMZHCPB.js +190 -0
- package/dist/chunk-UMMZHCPB.js.map +1 -0
- package/dist/chunk-VZSRQ272.js +149 -0
- package/dist/chunk-VZSRQ272.js.map +1 -0
- package/dist/{chunk-L5G7OUKD.js → chunk-XY4DDNEG.js} +8 -190
- package/dist/chunk-XY4DDNEG.js.map +1 -0
- package/dist/chunk-Y47J2LJ3.js +859 -0
- package/dist/chunk-Y47J2LJ3.js.map +1 -0
- package/dist/{chunk-BABOZOSN.js → chunk-ZFIBGEOL.js} +3 -3
- package/dist/chunk-ZFIBGEOL.js.map +1 -0
- package/dist/{code-agent-session-BRXmavYv.d.ts → code-agent-session-BO8nCnv3.d.ts} +1 -1
- package/dist/contract/index.d.ts +24 -95
- package/dist/contract/index.js +25 -764
- package/dist/contract/index.js.map +1 -1
- package/dist/{control-GeE8OhpN.d.ts → control-_Qb7skHX.d.ts} +2 -2
- package/dist/control.d.ts +5 -5
- package/dist/corpus-BoR-041R.d.ts +560 -0
- package/dist/cost-ledger-DuSqlw5B.d.ts +113 -0
- package/dist/counterfactual-Dwibr5IW.d.ts +85 -0
- package/dist/{dataset-B2kL-fSM.d.ts → dataset-BbGkaN2I.d.ts} +1 -1
- package/dist/{registry-DrEQ3Luj.d.ts → default-registry-zoGHUQEH.d.ts} +29 -2
- package/dist/diagnose.d.ts +251 -0
- package/dist/diagnose.js +381 -0
- package/dist/diagnose.js.map +1 -0
- package/dist/{errors-Dwqw-T_m.d.ts → errors-CzMUYo7b.d.ts} +1 -1
- package/dist/{feedback-trajectory-B3rErRsh.d.ts → feedback-trajectory-D9OVLrg9.d.ts} +1 -1
- package/dist/fuzz.d.ts +484 -0
- package/dist/fuzz.js +613 -0
- package/dist/fuzz.js.map +1 -0
- package/dist/governance/index.d.ts +4 -4
- package/dist/hosted/index.d.ts +6 -6
- package/dist/{index-DE3RXAXD.d.ts → index-Bx3gZ8xl.d.ts} +1 -1
- package/dist/index.d.ts +718 -455
- package/dist/index.js +1604 -793
- package/dist/index.js.map +1 -1
- package/dist/{insight-report-3ADTfClO.d.ts → insight-report-BBwvOh6x.d.ts} +2 -2
- package/dist/{integrity-CJzrpUua.d.ts → integrity-VJ9A7aST.d.ts} +1 -1
- package/dist/{judge-calibration-DilmB3Ml.d.ts → judge-calibration-0p2QcWNE.d.ts} +1 -1
- package/dist/{kind-factory-CVecZZG_.d.ts → kind-factory-5b7xXXOr.d.ts} +2 -2
- package/dist/{llm-client-CuUg2Mn3.d.ts → llm-client-BeEcAokY.d.ts} +1 -1
- package/dist/matrix/index.d.ts +2 -2
- package/dist/meta-eval/index.d.ts +177 -3
- package/dist/meta-eval/index.js +260 -1
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/{multi-layer-verifier-DlWCXuxL.d.ts → multi-layer-verifier-DUZXrPDA.d.ts} +7 -1
- package/dist/multishot/index.d.ts +25 -11
- package/dist/multishot/index.js +36 -7
- package/dist/multishot/index.js.map +1 -1
- package/dist/openapi.json +1 -1
- package/dist/perf/index.d.ts +123 -0
- package/dist/perf/index.js +18 -0
- package/dist/pipelines/index.js +2 -2
- package/dist/{agent-profile-D0PBIWlV.d.ts → pre-registration-DELOEJ8v.d.ts} +144 -4
- package/dist/{provenance-DPpNIOJD.d.ts → provenance-LnqRT0sS.d.ts} +5 -5
- package/dist/{red-team-DW9Ca_tj.d.ts → red-team-BXHil6c8.d.ts} +1 -1
- package/dist/{release-report-hlNtD12q.d.ts → release-report-euXIV_Sk.d.ts} +3 -3
- package/dist/reporting.d.ts +8 -8
- package/dist/reporting.js +3 -3
- package/dist/{researcher-BLPHBbNV.d.ts → researcher-DE6Gpnb4.d.ts} +4 -4
- package/dist/rl.d.ts +194 -656
- package/dist/rl.js +231 -149
- package/dist/rl.js.map +1 -1
- package/dist/{rubric-predictive-validity-CnEl9Jc8.d.ts → rubric-predictive-validity-Cy_W-hWZ.d.ts} +1 -1
- package/dist/{run-campaign-4Y5V5CN3.js → run-campaign-RDGAM5KJ.js} +3 -3
- package/dist/run-campaign-RDGAM5KJ.js.map +1 -0
- package/dist/{run-improvement-loop-CNqQckTj.d.ts → run-improvement-loop-5z_l5zDz.d.ts} +2 -2
- package/dist/{run-record-De9VarXR.d.ts → run-record-e7vj1uZQ.d.ts} +1 -1
- package/dist/{runtime-trajectory-BLRiaifm.d.ts → runtime-trajectory-BDgfGZSr.d.ts} +1 -1
- package/dist/{semantic-concept-judge-DIEgr_6v.d.ts → semantic-concept-judge-Dn8Z6KEG.d.ts} +5 -31
- package/dist/series-convergence-D5OWMBg6.d.ts +33 -0
- package/dist/{statistics-CnC1FMbx.d.ts → statistics-C7PozGrZ.d.ts} +71 -2
- package/dist/{summary-report-Db0dDSWP.d.ts → summary-report-DGmUucwQ.d.ts} +1 -1
- package/dist/traces.d.ts +3 -3
- package/dist/traces.js +8 -6
- package/dist/{types-Cu3u_x59.d.ts → types-2VVIL04s.d.ts} +2 -2
- package/dist/{types-D7lLRYe9.d.ts → types-BU-7W85F.d.ts} +21 -1
- package/dist/{types-CqPax19X.d.ts → types-mn5Aqk7x.d.ts} +1 -1
- package/dist/{verdict-CeEgtjyI.d.ts → verdict-C9MlYujm.d.ts} +3 -0
- package/dist/wire/index.d.ts +3 -3
- package/dist/workflow/index.d.ts +12 -11
- package/dist/workflow/index.js +1 -1
- package/package.json +16 -1
- package/dist/chunk-BABOZOSN.js.map +0 -1
- package/dist/chunk-L5G7OUKD.js.map +0 -1
- package/dist/chunk-SHTXZ4O2.js +0 -113
- package/dist/chunk-SHTXZ4O2.js.map +0 -1
- package/dist/chunk-VIDQF3F5.js.map +0 -1
- package/dist/chunk-VUINJM5M.js.map +0 -1
- package/dist/chunk-YGYXHNAQ.js.map +0 -1
- package/dist/chunk-YV7J7X5N.js.map +0 -1
- /package/dist/{chunk-FZWAFVAA.js.map → chunk-4FBZZIYD.js.map} +0 -0
- /package/dist/{chunk-OTYQPHPL.js.map → chunk-6SOJM3VR.js.map} +0 -0
- /package/dist/{chunk-Z7VFTS2J.js.map → chunk-CY6U5S3X.js.map} +0 -0
- /package/dist/{chunk-WJL2NJXN.js.map → chunk-GSH6QNNS.js.map} +0 -0
- /package/dist/{chunk-RBNA5AZT.js.map → chunk-L3JOU6XM.js.map} +0 -0
- /package/dist/{chunk-ZZ2HOPME.js.map → chunk-TWS7AZEY.js.map} +0 -0
- /package/dist/{run-campaign-4Y5V5CN3.js.map → perf/index.js.map} +0 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
// src/analyst/types.ts
|
|
2
|
+
import { createHash } from "crypto";
|
|
3
|
+
function computeFindingId(input) {
|
|
4
|
+
const basis = JSON.stringify({
|
|
5
|
+
a: input.analyst_id,
|
|
6
|
+
r: input.area,
|
|
7
|
+
s: input.subject ?? "",
|
|
8
|
+
c: normalizeClaim(input.id_basis ?? input.claim)
|
|
9
|
+
});
|
|
10
|
+
return `f_${createHash("sha256").update(basis).digest("hex").slice(0, 20)}`;
|
|
11
|
+
}
|
|
12
|
+
function normalizeClaim(c) {
|
|
13
|
+
return c.toLowerCase().replace(/\s+/g, " ").replace(/[.!?;:,]+$/g, "").trim();
|
|
14
|
+
}
|
|
15
|
+
function makeFinding(init) {
|
|
16
|
+
const { id_basis, produced_at, ...rest } = init;
|
|
17
|
+
return {
|
|
18
|
+
schema_version: "1.0.0",
|
|
19
|
+
finding_id: computeFindingId({
|
|
20
|
+
analyst_id: rest.analyst_id,
|
|
21
|
+
area: rest.area,
|
|
22
|
+
subject: rest.subject,
|
|
23
|
+
claim: rest.claim,
|
|
24
|
+
id_basis
|
|
25
|
+
}),
|
|
26
|
+
produced_at: produced_at ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
27
|
+
...rest
|
|
28
|
+
};
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
export {
|
|
32
|
+
computeFindingId,
|
|
33
|
+
makeFinding
|
|
34
|
+
};
|
|
35
|
+
//# sourceMappingURL=chunk-45EEMHTC.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/analyst/types.ts"],"sourcesContent":["/**\n * Analyst contract — the missing orchestration layer over agent-eval's\n * existing analyzers (analyzeTraces, MultiLayerVerifier, RunCritic,\n * SemanticConceptJudge, JudgeFn, ...).\n *\n * Each existing primitive returns its own output shape. The Analyst\n * contract is the single envelope every primitive lifts into, so a\n * registry can run N analysts against a run and a single renderer can\n * compose findings without knowing which analyzer produced them.\n *\n * The contract is intentionally domain-agnostic: nothing here knows\n * about code, voice, RAG, or any particular agent stack. Analysts\n * declare what INPUT KIND they need (a trace store, an artifact dir,\n * a RunRecord, a JudgeInput, or `custom`), and the registry routes\n * the matching input from `AnalystRunInputs`.\n */\n\nimport { createHash } from 'node:crypto'\nimport type { RunRecord } from '../run-record'\nimport type { TraceAnalysisStore } from '../trace-analyst/store'\nimport type { JudgeInput } from '../types'\nimport type { ChatClient } from './chat-client'\n\n/**\n * Unified envelope every analyst emits. Schema-versioned so renderers\n * and time-series diffs survive future field additions.\n */\nexport interface AnalystFinding {\n schema_version: '1.0.0'\n /**\n * Stable hash over identity-defining fields (analyst_id + canonical\n * claim + area + optional subject). Two findings from two runs that\n * \"are the same finding\" share this id — that's what `diffFindings`\n * uses to compute appeared/disappeared sets across runs.\n */\n finding_id: string\n analyst_id: string\n produced_at: string\n severity: AnalystSeverity\n /**\n * Coarse classification. Renderers group by this. Free-form so\n * domain-specific analysts can introduce categories without a\n * schema change ('agent-reasoning', 'verification', 'cost',\n * 'tool-use', 'safety', 'latency', 'data-quality', ...).\n */\n area: string\n claim: string\n rationale?: string\n evidence_refs: EvidenceRef[]\n recommended_action?: string\n validation_plan?: string\n /** 0..1 — the analyst's own confidence. Not calibrated across analysts. */\n confidence: number\n /**\n * Optional subject the finding is about — leaf id, agent id, request\n * id. Included in finding_id when present so per-subject findings\n * diff cleanly across runs.\n */\n subject?: string\n /** FIREWALL provenance (docs/learning-flywheel.md): true iff this finding was\n * lifted from a JUDGE verdict (an acceptance score), not OBSERVED from the\n * agent's behavior. A judge-derived finding must NEVER be admitted as a\n * steering input — that is the held-out judge leaking into the loop. Set at\n * the lift site (createJudgeAdapter); checked by `assertNoJudgeVerdict`.\n * Provenance, not evidence presence, is the correct discriminator: an\n * evidence-less trace-analyst observation legitimately steers, while a judge\n * verdict that happens to cite an artifact must not. */\n derived_from_judge?: boolean\n /** Analyst-private extras; renderers ignore unless they know the analyst. */\n metadata?: Record<string, unknown>\n}\n\nexport type AnalystSeverity = 'critical' | 'high' | 'medium' | 'low' | 'info'\n\nexport interface EvidenceRef {\n /**\n * Where the evidence lives. `span` and `event` refer to OTLP trace\n * elements; `artifact` to a file inside the run's artifact tree;\n * `finding` to another AnalystFinding (cross-analyst chaining);\n * `metric` to a named scalar reading the renderer knows how to read.\n */\n kind: 'span' | 'event' | 'artifact' | 'finding' | 'metric'\n uri: string\n excerpt?: string\n}\n\n// ── Analyst contract ─────────────────────────────────────────────────\n\n/**\n * The discriminator the registry uses to pass the right input.\n * `custom` is the escape hatch — analysts that need something else\n * (e.g. an embedding cache, a partner SDK handle) read it from\n * `AnalystRunInputs.custom[<analyst id>]`.\n */\nexport type AnalystInputKind =\n | 'trace-store'\n | 'artifact-dir'\n | 'run-record'\n | 'judge-input'\n | 'custom'\n\nexport interface AnalystCost {\n /** `deterministic` analysts MUST NOT call the LLM. */\n kind: 'deterministic' | 'llm'\n /** Optional declared upper bound; the registry can enforce a budget. */\n est_usd_per_run?: number\n /** Models the analyst expects to use (informational). */\n models?: string[]\n}\n\nexport interface AnalystRequirements {\n /** Min number of shots / samples the analyst needs to produce signal. */\n min_shots?: number\n /** Capabilities the runtime must supply (e.g. ['network', 'gpu']). */\n capabilities?: string[]\n}\n\n/**\n * What's passed to every analyst call. The registry resolves which\n * field the analyst's `inputKind` selects and asserts it's present.\n */\nexport interface AnalystRunInputs {\n traceStore?: TraceAnalysisStore\n artifactDir?: string\n runRecord?: RunRecord\n judgeInput?: JudgeInput\n /** Keyed by analyst id; populated by callers that registered custom analysts. */\n custom?: Record<string, unknown>\n}\n\nexport interface AnalystContext {\n runId: string\n /** Stable correlation id so logs from a single registry.run() share a tag. */\n correlationId: string\n /** Wall-clock deadline (epoch ms). Analysts SHOULD honor for graceful cancel. */\n deadlineMs?: number\n /** Per-analyst USD budget. Analysts MAY check before issuing LLM calls. */\n budgetUsd?: number\n /**\n * Shared chat client. Analysts that call an LLM go through this so\n * the operator picks transport (sandbox-sdk | router | cli-bridge |\n * direct-provider | mock) at the registry boundary without touching\n * analyst code.\n */\n chat?: ChatClient\n /**\n * Findings from a prior run the operator wants the analyst to see as\n * retrieval context. Kinds that take advantage of cross-run memory\n * (failure-mode \"I saw this cluster last run\", knowledge-gap \"the wiki\n * page I asked for is still missing\") render these into the actor's\n * working set. Filtering is the operator's job: pass the slice that\n * matches the analyst's id, or pass everything and let the kind\n * filter. Empty / absent means no cross-run context.\n */\n priorFindings?: ReadonlyArray<AnalystFinding>\n /** Free-form runtime tags (env, host, op). Findings can echo these into metadata. */\n tags?: Record<string, string>\n /** Logger callback — analysts SHOULD prefer this over console.* for testability. */\n log?: (msg: string, fields?: Record<string, unknown>) => void\n /** Optional abort signal. Analysts SHOULD pass it through to LLM calls. */\n signal?: AbortSignal\n}\n\n/**\n * The minimal contract. Concrete analysts can refine `TInput` so\n * implementations stay type-safe (e.g. a trace analyst's `TInput` is\n * `TraceAnalysisStore`); the registry passes the right field from\n * `AnalystRunInputs` based on `inputKind`.\n */\nexport interface Analyst<TInput = unknown> {\n /** Stable identifier — appears in finding_id, telemetry, and registry exclusion lists. */\n readonly id: string\n /** Human-readable. One sentence. */\n readonly description: string\n readonly inputKind: AnalystInputKind\n readonly cost: AnalystCost\n readonly requires?: AnalystRequirements\n /** Bump on breaking changes to claim wording or area so old finding_ids don't collide. */\n readonly version: string\n analyze(input: TInput, ctx: AnalystContext): Promise<AnalystFinding[]>\n}\n\n// ── finding_id stability ─────────────────────────────────────────────\n\n/**\n * Compute the stable finding_id from the identity-defining fields.\n * Default implementation hashes {analyst_id, area, subject, normalized claim}.\n * Analysts that emit findings whose claim text varies per run (timestamps,\n * counts) SHOULD either: (a) pass an explicit `id_basis` to fix the hash,\n * or (b) move the variable part into `rationale`/`metadata` and keep the\n * `claim` static.\n */\nexport function computeFindingId(input: {\n analyst_id: string\n area: string\n subject?: string\n claim: string\n /** Override the claim for hashing — use when the displayed claim has run-specific bits. */\n id_basis?: string\n}): string {\n const basis = JSON.stringify({\n a: input.analyst_id,\n r: input.area,\n s: input.subject ?? '',\n c: normalizeClaim(input.id_basis ?? input.claim),\n })\n return `f_${createHash('sha256').update(basis).digest('hex').slice(0, 20)}`\n}\n\nfunction normalizeClaim(c: string): string {\n // Lowercase, collapse whitespace, strip trailing punctuation. Goal:\n // \"Leaf X failed install\" and \"Leaf X failed install.\" hash the same.\n return c\n .toLowerCase()\n .replace(/\\s+/g, ' ')\n .replace(/[.!?;:,]+$/g, '')\n .trim()\n}\n\n/**\n * Convenience factory: produce a fully-formed AnalystFinding with the\n * id computed automatically. Analyst code stays terse.\n */\nexport function makeFinding(\n init: Omit<AnalystFinding, 'schema_version' | 'finding_id' | 'produced_at'> & {\n id_basis?: string\n produced_at?: string\n },\n): AnalystFinding {\n const { id_basis, produced_at, ...rest } = init\n return {\n schema_version: '1.0.0',\n finding_id: computeFindingId({\n analyst_id: rest.analyst_id,\n area: rest.area,\n subject: rest.subject,\n claim: rest.claim,\n id_basis,\n }),\n produced_at: produced_at ?? new Date().toISOString(),\n ...rest,\n }\n}\n\n// ── Registry result envelope ────────────────────────────────────────\n\nexport interface AnalystRunSummary {\n analyst_id: string\n status: 'ok' | 'skipped' | 'failed'\n /** Why skipped — missing input, budget exceeded, capability unmet. */\n reason?: string\n findings_count: number\n latency_ms: number\n cost_usd: number\n /** When `status='failed'`: the error class + message, never the full stack. */\n error?: { class: string; message: string }\n}\n\nexport interface AnalystRunResult {\n run_id: string\n correlation_id: string\n started_at: string\n ended_at: string\n findings: AnalystFinding[]\n per_analyst: AnalystRunSummary[]\n /** Total LLM cost in USD across all analysts in this registry.run(). */\n total_cost_usd: number\n}\n\n// ── Streaming event envelope ────────────────────────────────────────\n\n/**\n * Events emitted by `AnalystRegistry.runStream(...)` in real time as\n * the registry executes. UIs subscribe via `for await (const ev of\n * registry.runStream(...))`; `registry.run(...)` is a thin collector\n * over the same stream, so the two surfaces share their invariants.\n *\n * Per-finding events are intentionally omitted — analyzers are batch\n * operations (an Ax actor returns the full `findings:json[]` at the\n * end of the responder), so streaming inside one analyst would only\n * emit partial JSON consumers can't render. The kind-completion event\n * is the right granularity; subscribers wanting per-finding rendering\n * iterate `event.findings` themselves.\n */\nexport type AnalystRunEvent =\n | {\n type: 'run-started'\n run_id: string\n correlation_id: string\n started_at: string\n /** The ordered list of analyst ids the registry will run. */\n analyst_ids: ReadonlyArray<string>\n }\n | {\n type: 'analyst-skipped'\n summary: AnalystRunSummary\n }\n | {\n type: 'analyst-started'\n analyst_id: string\n started_at: string\n }\n | {\n type: 'analyst-completed'\n /** `summary.status` is `'ok'` for clean completion or `'failed'` for thrown analysts. */\n summary: AnalystRunSummary\n findings: ReadonlyArray<AnalystFinding>\n }\n | {\n type: 'run-completed'\n result: AnalystRunResult\n }\n"],"mappings":";AAiBA,SAAS,kBAAkB;AA+KpB,SAAS,iBAAiB,OAOtB;AACT,QAAM,QAAQ,KAAK,UAAU;AAAA,IAC3B,GAAG,MAAM;AAAA,IACT,GAAG,MAAM;AAAA,IACT,GAAG,MAAM,WAAW;AAAA,IACpB,GAAG,eAAe,MAAM,YAAY,MAAM,KAAK;AAAA,EACjD,CAAC;AACD,SAAO,KAAK,WAAW,QAAQ,EAAE,OAAO,KAAK,EAAE,OAAO,KAAK,EAAE,MAAM,GAAG,EAAE,CAAC;AAC3E;AAEA,SAAS,eAAe,GAAmB;AAGzC,SAAO,EACJ,YAAY,EACZ,QAAQ,QAAQ,GAAG,EACnB,QAAQ,eAAe,EAAE,EACzB,KAAK;AACV;AAMO,SAAS,YACd,MAIgB;AAChB,QAAM,EAAE,UAAU,aAAa,GAAG,KAAK,IAAI;AAC3C,SAAO;AAAA,IACL,gBAAgB;AAAA,IAChB,YAAY,iBAAiB;AAAA,MAC3B,YAAY,KAAK;AAAA,MACjB,MAAM,KAAK;AAAA,MACX,SAAS,KAAK;AAAA,MACd,OAAO,KAAK;AAAA,MACZ;AAAA,IACF,CAAC;AAAA,IACD,aAAa,gBAAe,oBAAI,KAAK,GAAE,YAAY;AAAA,IACnD,GAAG;AAAA,EACL;AACF;","names":[]}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
summaryTable
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-CY6U5S3X.js";
|
|
4
4
|
import {
|
|
5
5
|
VerificationError
|
|
6
6
|
} from "./chunk-3BFEG2F6.js";
|
|
@@ -574,4 +574,4 @@ export {
|
|
|
574
574
|
judgeReplayGate,
|
|
575
575
|
renderReleaseReport
|
|
576
576
|
};
|
|
577
|
-
//# sourceMappingURL=chunk-
|
|
577
|
+
//# sourceMappingURL=chunk-4FBZZIYD.js.map
|
|
@@ -31,24 +31,26 @@ function extractVerifiableReward(report, opts = {}) {
|
|
|
31
31
|
);
|
|
32
32
|
if (deterministic.length === 1) {
|
|
33
33
|
const layer = deterministic[0];
|
|
34
|
+
const value = clamp01(layer.score);
|
|
34
35
|
return {
|
|
35
|
-
value
|
|
36
|
+
value,
|
|
36
37
|
source: sourceFor(layer.layer),
|
|
37
38
|
determinism: "deterministic",
|
|
38
39
|
confidence: 1,
|
|
39
40
|
origin: layer.layer,
|
|
41
|
+
components: { [layer.layer]: value },
|
|
40
42
|
breakdown: layerBreakdown(layer)
|
|
41
43
|
};
|
|
42
44
|
}
|
|
43
45
|
if (deterministic.length > 1) {
|
|
44
46
|
let num = 0;
|
|
45
47
|
let denom = 0;
|
|
46
|
-
const
|
|
48
|
+
const components = {};
|
|
47
49
|
for (const l of deterministic) {
|
|
48
50
|
const w = l.detail?.weight ?? 1;
|
|
49
51
|
num += w * (l.score ?? 0);
|
|
50
52
|
denom += w;
|
|
51
|
-
|
|
53
|
+
components[l.layer] = l.score;
|
|
52
54
|
}
|
|
53
55
|
return {
|
|
54
56
|
value: denom === 0 ? 0 : clamp01(num / denom),
|
|
@@ -56,7 +58,8 @@ function extractVerifiableReward(report, opts = {}) {
|
|
|
56
58
|
determinism: "deterministic",
|
|
57
59
|
confidence: 1,
|
|
58
60
|
origin: deterministic.map((l) => l.layer).join("+"),
|
|
59
|
-
|
|
61
|
+
components,
|
|
62
|
+
breakdown: { ...components }
|
|
60
63
|
};
|
|
61
64
|
}
|
|
62
65
|
if (!fallbackToJudge) return null;
|
|
@@ -65,12 +68,14 @@ function extractVerifiableReward(report, opts = {}) {
|
|
|
65
68
|
) ?? report.layers.find((l) => typeof l.score === "number" && Number.isFinite(l.score));
|
|
66
69
|
if (!judge) return null;
|
|
67
70
|
const confFromDetail = judge.detail?.confidence;
|
|
71
|
+
const judgeValue = clamp01(judge.score);
|
|
68
72
|
return {
|
|
69
|
-
value:
|
|
73
|
+
value: judgeValue,
|
|
70
74
|
source: "judge",
|
|
71
75
|
determinism: "probabilistic",
|
|
72
76
|
confidence: typeof confFromDetail === "number" ? confFromDetail : judgeFloor,
|
|
73
77
|
origin: judge.layer,
|
|
78
|
+
components: { [judge.layer]: judgeValue },
|
|
74
79
|
breakdown: layerBreakdown(judge)
|
|
75
80
|
};
|
|
76
81
|
}
|
|
@@ -89,20 +94,22 @@ function extractVerifiableRewardsFromRecords(runs, opts = {}) {
|
|
|
89
94
|
const det = layerScores.filter((l) => deterministicSet.has(l.name));
|
|
90
95
|
if (det.length === 1) {
|
|
91
96
|
const layer = det[0];
|
|
97
|
+
const value = clamp01(layer.score);
|
|
92
98
|
return {
|
|
93
99
|
runId: run.runId,
|
|
94
100
|
reward: {
|
|
95
|
-
value
|
|
101
|
+
value,
|
|
96
102
|
source: sourceFor(layer.name),
|
|
97
103
|
determinism: "deterministic",
|
|
98
104
|
confidence: 1,
|
|
99
|
-
origin: layer.name
|
|
105
|
+
origin: layer.name,
|
|
106
|
+
components: { [layer.name]: value }
|
|
100
107
|
}
|
|
101
108
|
};
|
|
102
109
|
}
|
|
103
110
|
if (det.length > 1) {
|
|
104
111
|
const value = det.reduce((s, l) => s + l.score, 0) / det.length;
|
|
105
|
-
const
|
|
112
|
+
const components = Object.fromEntries(
|
|
106
113
|
det.map((l) => [l.name, l.score])
|
|
107
114
|
);
|
|
108
115
|
return {
|
|
@@ -113,7 +120,8 @@ function extractVerifiableRewardsFromRecords(runs, opts = {}) {
|
|
|
113
120
|
determinism: "deterministic",
|
|
114
121
|
confidence: 1,
|
|
115
122
|
origin: det.map((l) => l.name).join("+"),
|
|
116
|
-
|
|
123
|
+
components,
|
|
124
|
+
breakdown: { ...components }
|
|
117
125
|
}
|
|
118
126
|
};
|
|
119
127
|
}
|
|
@@ -122,14 +130,16 @@ function extractVerifiableRewardsFromRecords(runs, opts = {}) {
|
|
|
122
130
|
if (typeof primary !== "number" || !Number.isFinite(primary)) {
|
|
123
131
|
return { runId: run.runId, reward: null };
|
|
124
132
|
}
|
|
133
|
+
const primaryValue = clamp01(primary);
|
|
125
134
|
return {
|
|
126
135
|
runId: run.runId,
|
|
127
136
|
reward: {
|
|
128
|
-
value:
|
|
137
|
+
value: primaryValue,
|
|
129
138
|
source: "judge",
|
|
130
139
|
determinism: "probabilistic",
|
|
131
140
|
confidence: judgeFloor,
|
|
132
|
-
origin: "run.outcome.score"
|
|
141
|
+
origin: "run.outcome.score",
|
|
142
|
+
components: { "run.outcome.score": primaryValue }
|
|
133
143
|
}
|
|
134
144
|
};
|
|
135
145
|
});
|
|
@@ -310,4 +320,4 @@ export {
|
|
|
310
320
|
filterDeterministicallyRewarded,
|
|
311
321
|
detectRewardHacking
|
|
312
322
|
};
|
|
313
|
-
//# sourceMappingURL=chunk-
|
|
323
|
+
//# sourceMappingURL=chunk-5HRORJQY.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/rl/verifiable-reward.ts","../src/rl/reward-hacking.ts"],"sourcesContent":["/**\n * Verifiable reward channel.\n *\n * For RL on coding / math / theorem-proving / structured-output tasks, the\n * reward signal is *decidable* — a test passes or fails, a proof checks or\n * doesn't, an output validates against a schema or doesn't. These rewards\n * are dramatically more useful for RL training than LLM-judge scores\n * because they don't drift, can't be Goodhart-gamed by the policy in the\n * same way, and don't require a separate calibration loop.\n *\n * The `MultiLayerVerifier` already produces this signal — it just doesn't\n * surface it in a shape that's clean enough for RL training. This module\n * wraps the verifier output so consumers can:\n *\n * 1. Extract a clean `VerifiableReward` from a `VerificationReport`\n * 2. Distinguish *deterministic* rewards (compile, test, schema) from\n * *probabilistic* rewards (judge) so they can be weighted differently\n * in the RL training step\n * 3. Filter `RunRecord[]` to only those with a verifiable reward,\n * producing the clean training set that DeepSeek-R1-style GRPO and\n * AlphaProof-style search both depend on\n *\n * Why this matters: every credible 2025-2026 frontier RL result on coding\n * agents leans on verifiable reward (DeepSeek-R1 GRPO on test pass-rate,\n * o-series RL on math/code, AlphaProof on Lean kernel checking). Mixing\n * judge scores into the reward signal poisons the gradient. This module\n * is the seam.\n */\n\nimport type { LayerResult, VerificationReport } from '../multi-layer-verifier'\nimport type { RunRecord } from '../run-record'\n\nexport type VerifiableRewardSource =\n | 'compile' // typecheck / build / lint passed\n | 'test' // unit / integration test pass-rate\n | 'schema' // structured output validates\n | 'sandbox' // sandbox exec exit code\n | 'judge' // LLM judge — probabilistic, included for completeness\n | 'composite' // weighted blend across multiple of the above\n\nexport interface VerifiableReward {\n /** Scalar in [0, 1]. The RL training signal. */\n value: number\n /** What produced the reward — different sources have different determinism. */\n source: VerifiableRewardSource\n /**\n * Determinism class. `'deterministic'` rewards are repeatable byte-for-byte\n * given the same inputs (compile, test, schema validation, sandbox exit code).\n * `'probabilistic'` rewards depend on a stochastic component (LLM judge).\n * Mixing these in the same training batch without separation is a known\n * footgun in production RLHF pipelines.\n */\n determinism: 'deterministic' | 'probabilistic'\n /**\n * Confidence in the reward value. For deterministic sources this is 1.0\n * (the bit either flipped or didn't). For judge sources this is the\n * judge-reported confidence or — when missing — a calibrated prior.\n */\n confidence: number\n /** The layer / judge id that produced the signal, for provenance. */\n origin: string\n /**\n * Per-source contribution to `value`, keyed by layer/judge id. Single-source\n * rewards carry one entry (`{ [origin]: value }`); composite rewards carry\n * every contributing layer's score — the anti-scalar-collapse surface RL\n * consumers weight per-source instead of trusting one blended number.\n */\n components: Record<string, number>\n /**\n * @deprecated Read `components` for per-source reward values. Kept for\n * published-API compatibility: single-source rewards carry the layer's\n * diagnostics here (e.g. `{ tests_passed: 7 }`); composite rewards carry\n * the same per-layer scores `components` now holds.\n */\n breakdown?: Record<string, number>\n}\n\nexport interface VerifiableRewardExtractionOptions {\n /**\n * Which layers count as deterministic-reward sources. The verifier doesn't\n * tag layers as \"this is verifiable\"; the caller declares it via this list\n * (or via the layer name → source mapping). Default treats common names\n * (`install`, `typecheck`, `build`, `lint`, `test`, `compile`, `schema`,\n * `sandbox`) as deterministic.\n */\n deterministicLayers?: string[]\n /**\n * Map layer name → reward source. Defaults to a sensible string-match.\n */\n sourceFor?: (layerName: string) => VerifiableRewardSource\n /**\n * Whether to fall back to a probabilistic (judge) reward when no\n * deterministic layer produced a numeric score. Default `true`. Set to\n * `false` for \"deterministic-only\" training pipelines that should\n * discard runs without a verifiable signal.\n */\n fallbackToJudge?: boolean\n /**\n * Default confidence for probabilistic (judge) rewards when the judge\n * doesn't report one. Default `0.7`.\n */\n judgeConfidenceFloor?: number\n}\n\nconst DEFAULT_DETERMINISTIC_LAYERS = new Set([\n 'install',\n 'typecheck',\n 'build',\n 'lint',\n 'test',\n 'compile',\n 'schema',\n 'sandbox',\n 'unit_tests',\n 'integration_tests',\n])\n\nconst DEFAULT_SOURCE_FOR = (name: string): VerifiableRewardSource => {\n const lower = name.toLowerCase()\n if (lower.includes('test')) return 'test'\n if (\n lower.includes('compile') ||\n lower.includes('build') ||\n lower.includes('typecheck') ||\n lower.includes('lint')\n )\n return 'compile'\n if (lower.includes('schema')) return 'schema'\n if (lower.includes('sandbox')) return 'sandbox'\n if (lower.includes('judge') || lower.includes('semantic')) return 'judge'\n return 'composite'\n}\n\n/**\n * Extract a `VerifiableReward` from a `VerificationReport`.\n *\n * Strategy: prefer the deterministic layers (in order: test → compile →\n * schema → sandbox), fall back to the judge layer if `fallbackToJudge` is\n * true, return `null` if no signal qualifies. When multiple deterministic\n * layers contribute, return a `'composite'` source with a weighted blend.\n */\nexport function extractVerifiableReward(\n report: VerificationReport,\n opts: VerifiableRewardExtractionOptions = {},\n): VerifiableReward | null {\n const deterministicSet = new Set(opts.deterministicLayers ?? [...DEFAULT_DETERMINISTIC_LAYERS])\n const sourceFor = opts.sourceFor ?? DEFAULT_SOURCE_FOR\n const fallbackToJudge = opts.fallbackToJudge ?? true\n const judgeFloor = opts.judgeConfidenceFloor ?? 0.7\n\n const deterministic = report.layers.filter(\n (l) => deterministicSet.has(l.layer) && typeof l.score === 'number' && Number.isFinite(l.score),\n )\n\n if (deterministic.length === 1) {\n const layer = deterministic[0]!\n const value = clamp01(layer.score!)\n return {\n value,\n source: sourceFor(layer.layer),\n determinism: 'deterministic',\n confidence: 1,\n origin: layer.layer,\n components: { [layer.layer]: value },\n breakdown: layerBreakdown(layer),\n }\n }\n\n if (deterministic.length > 1) {\n // Composite: weighted blend by `Layer.weight` if present, else equal.\n let num = 0\n let denom = 0\n const components: Record<string, number> = {}\n for (const l of deterministic) {\n const w = (l.detail?.weight as number | undefined) ?? 1\n num += w * (l.score ?? 0)\n denom += w\n components[l.layer] = l.score!\n }\n return {\n value: denom === 0 ? 0 : clamp01(num / denom),\n source: 'composite',\n determinism: 'deterministic',\n confidence: 1,\n origin: deterministic.map((l) => l.layer).join('+'),\n components,\n breakdown: { ...components },\n }\n }\n\n if (!fallbackToJudge) return null\n\n const judge =\n report.layers.find(\n (l) =>\n typeof l.score === 'number' && Number.isFinite(l.score) && sourceFor(l.layer) === 'judge',\n ) ?? report.layers.find((l) => typeof l.score === 'number' && Number.isFinite(l.score))\n\n if (!judge) return null\n\n const confFromDetail = judge.detail?.confidence as number | undefined\n const judgeValue = clamp01(judge.score!)\n return {\n value: judgeValue,\n source: 'judge',\n determinism: 'probabilistic',\n confidence: typeof confFromDetail === 'number' ? confFromDetail : judgeFloor,\n origin: judge.layer,\n components: { [judge.layer]: judgeValue },\n breakdown: layerBreakdown(judge),\n }\n}\n\n/**\n * Extract verifiable rewards from `RunRecord[]` produced via the\n * `verificationReportToRunRecord` adapter (which encodes per-layer scores\n * in `outcome.raw['layer.<name>']`). For records that don't carry layer\n * scores, returns `null` for that record.\n *\n * This is the canonical bridge from \"campaign-shaped artifacts\" to\n * \"RL-training-ready reward signals\": every record that has a clean\n * verifiable reward becomes a training datum, every record that doesn't\n * gets filtered out (or kept with `'probabilistic'` determinism for\n * separate downstream handling).\n */\nexport function extractVerifiableRewardsFromRecords(\n runs: RunRecord[],\n opts: VerifiableRewardExtractionOptions = {},\n): Array<{ runId: string; reward: VerifiableReward | null }> {\n const sourceFor = opts.sourceFor ?? DEFAULT_SOURCE_FOR\n const deterministicSet = new Set(opts.deterministicLayers ?? [...DEFAULT_DETERMINISTIC_LAYERS])\n const fallbackToJudge = opts.fallbackToJudge ?? true\n const judgeFloor = opts.judgeConfidenceFloor ?? 0.7\n\n return runs.map((run) => {\n // Recover per-layer scores from outcome.raw['layer.<name>']\n const layerScores: Array<{ name: string; score: number }> = []\n for (const [k, v] of Object.entries(run.outcome.raw)) {\n if (\n k.startsWith('layer.') &&\n !k.includes('.', 6) &&\n typeof v === 'number' &&\n Number.isFinite(v)\n ) {\n layerScores.push({ name: k.slice('layer.'.length), score: v })\n }\n }\n const det = layerScores.filter((l) => deterministicSet.has(l.name))\n\n if (det.length === 1) {\n const layer = det[0]!\n const value = clamp01(layer.score)\n return {\n runId: run.runId,\n reward: {\n value,\n source: sourceFor(layer.name),\n determinism: 'deterministic',\n confidence: 1,\n origin: layer.name,\n components: { [layer.name]: value },\n },\n }\n }\n if (det.length > 1) {\n const value = det.reduce((s, l) => s + l.score, 0) / det.length\n const components: Record<string, number> = Object.fromEntries(\n det.map((l) => [l.name, l.score]),\n )\n return {\n runId: run.runId,\n reward: {\n value: clamp01(value),\n source: 'composite',\n determinism: 'deterministic',\n confidence: 1,\n origin: det.map((l) => l.name).join('+'),\n components,\n breakdown: { ...components },\n },\n }\n }\n if (!fallbackToJudge) return { runId: run.runId, reward: null }\n\n // Probabilistic fallback: use the run's primary score.\n const primary = run.outcome.holdoutScore ?? run.outcome.searchScore\n if (typeof primary !== 'number' || !Number.isFinite(primary)) {\n return { runId: run.runId, reward: null }\n }\n const primaryValue = clamp01(primary)\n return {\n runId: run.runId,\n reward: {\n value: primaryValue,\n source: 'judge',\n determinism: 'probabilistic',\n confidence: judgeFloor,\n origin: 'run.outcome.score',\n components: { 'run.outcome.score': primaryValue },\n },\n }\n })\n}\n\n/** Filter `RunRecord[]` to those with deterministic verifiable rewards. */\nexport function filterDeterministicallyRewarded(\n runs: RunRecord[],\n opts: VerifiableRewardExtractionOptions = {},\n): Array<{ run: RunRecord; reward: VerifiableReward }> {\n const rewarded = extractVerifiableRewardsFromRecords(runs, { ...opts, fallbackToJudge: false })\n const out: Array<{ run: RunRecord; reward: VerifiableReward }> = []\n for (let i = 0; i < runs.length; i++) {\n const r = rewarded[i]!\n if (r.reward && r.reward.determinism === 'deterministic') {\n out.push({ run: runs[i]!, reward: r.reward })\n }\n }\n return out\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\nfunction clamp01(x: number): number {\n if (!Number.isFinite(x)) return 0\n return Math.max(0, Math.min(1, x))\n}\n\nfunction layerBreakdown(l: LayerResult): Record<string, number> {\n const out: Record<string, number> = {}\n if (l.diagnostics) {\n for (const [k, v] of Object.entries(l.diagnostics)) {\n if (typeof v === 'number' && Number.isFinite(v)) out[k] = v\n }\n }\n return out\n}\n","/**\n * Reward hacking / Goodhart detection.\n *\n * Goodhart's Law says: when a measure becomes a target, it ceases to be\n * a good measure. In RLHF and agentic-RL settings this is the dominant\n * failure mode — the policy learns to produce outputs that score well on\n * the proxy reward (judge, rubric, test pass-rate) without producing\n * the underlying capability the proxy was meant to track.\n *\n * Krakovna et al. (2020, \"Specification Gaming Examples in AI\") and the\n * subsequent RLHF reward-hacking literature (Skalse et al. 2022, Kim et al.\n * 2023) converge on a few diagnostic signatures:\n *\n * 1. **Reward divergence:** the proxy reward grows while the held-out\n * ground-truth signal stagnates or drops. Predictive validity over\n * time captures this.\n * 2. **Distributional shift in outputs:** after RL, the policy produces\n * outputs that no longer match the reference distribution — usually\n * because it found a high-reward attractor that's degenerate (e.g.\n * one-token responses, repetition, formatting tricks).\n * 3. **Disagreement between independent rewards:** if you train on\n * reward A and a held-out independent reward B drops sharply, you're\n * probably hacking A.\n * 4. **Calibration drift:** the verifiable / deterministic component of\n * the reward is stable; the probabilistic / judge component drifts up\n * while the deterministic component doesn't. The judge is being\n * gamed.\n *\n * This module ships explicit detectors for all four signatures, plus a\n * combined verdict. The output is diagnostic — actionable signals,\n * not autoreject — because each signature has known false positives\n * (e.g., a policy that genuinely improves can show distributional shift).\n *\n * Differs from `rubricPredictiveValidity` (which is a *standing* check on\n * whether rubrics correlate with deployment outcomes) — this is a\n * *temporal* check on whether the reward-vs-truth gap is *widening over\n * time during a training run*.\n */\n\nimport type { RunRecord } from '../run-record'\nimport {\n filterDeterministicallyRewarded,\n type VerifiableRewardExtractionOptions,\n} from './verifiable-reward'\n\nexport type RewardHackingSignal =\n | 'reward_divergence'\n | 'distribution_shift'\n | 'reward_disagreement'\n | 'judge_drift'\n\nexport interface RewardHackingFinding {\n signal: RewardHackingSignal\n /** Severity in [0, 1]. >0.5 = strong signal. */\n severity: number\n message: string\n /** Numeric evidence the consumer can render. */\n detail: Record<string, number>\n}\n\nexport interface RewardHackingReport {\n findings: RewardHackingFinding[]\n /**\n * Composite verdict. `'clean'` if every signal severity < 0.3;\n * `'suspect'` if at least one ≥ 0.3 but none ≥ 0.6; `'gaming'` if any ≥ 0.6.\n */\n verdict: 'clean' | 'suspect' | 'gaming'\n /** Rationale for the verdict, ready to paste into an audit log. */\n rationale: string[]\n /** Number of paired (proxy, truth) data points the report saw. */\n n: number\n}\n\nexport interface DetectRewardHackingInput {\n /**\n * Run records ordered by recency (oldest first). The detector segments\n * them into prefix/suffix windows to compute \"did the gap widen.\"\n */\n runs: RunRecord[]\n /**\n * The metric the policy was trained to optimize. Should be present on\n * `outcome.raw` or `outcome.holdoutScore`. Default reads `outcome.holdoutScore`.\n */\n proxyOf?: (run: RunRecord) => number | null\n /**\n * The held-out ground-truth metric. For RL on coding, this is typically\n * test pass-rate. For RLHF, it's downstream task performance or human\n * preference. For knowledge tasks, it's an independently-graded score.\n */\n truthOf?: (run: RunRecord) => number | null\n /**\n * Independent secondary reward. Used for the `reward_disagreement`\n * signal. Default uses the verifiable reward extractor (deterministic\n * sources only).\n */\n secondaryRewardOf?: (run: RunRecord) => number | null\n /**\n * Window size — how many of the most recent runs count as the \"after\"\n * cohort. Default min(50, half the runs).\n */\n windowSize?: number\n /**\n * Severity threshold to flag a signal. Default 0.3 (suspect) and 0.6\n * (gaming).\n */\n thresholds?: { suspect?: number; gaming?: number }\n /**\n * Verifiable-reward options used for the secondary-reward fallback.\n */\n verifiableRewardOptions?: VerifiableRewardExtractionOptions\n}\n\nconst DEFAULT_PROXY = (r: RunRecord): number | null => {\n const v = r.outcome.holdoutScore ?? r.outcome.searchScore\n return typeof v === 'number' && Number.isFinite(v) ? v : null\n}\n\nexport function detectRewardHacking(input: DetectRewardHackingInput): RewardHackingReport {\n const proxyOf = input.proxyOf ?? DEFAULT_PROXY\n const truthOf = input.truthOf\n const sus = input.thresholds?.suspect ?? 0.3\n const gam = input.thresholds?.gaming ?? 0.6\n\n const runs = input.runs.filter((r) => proxyOf(r) !== null)\n const n = runs.length\n if (n < 4) {\n return {\n findings: [],\n verdict: 'clean',\n n,\n rationale: [`fewer than 4 runs with proxy reward (n=${n}); insufficient evidence`],\n }\n }\n const windowSize = Math.max(1, input.windowSize ?? Math.min(50, Math.floor(n / 2)))\n const before = runs.slice(0, n - windowSize)\n const after = runs.slice(n - windowSize)\n\n const findings: RewardHackingFinding[] = []\n\n // ── Signal 1: reward divergence (proxy ↑ while truth flat or ↓) ──────\n if (truthOf) {\n const beforeProxy = before.map(proxyOf).filter((v): v is number => typeof v === 'number')\n const afterProxy = after.map(proxyOf).filter((v): v is number => typeof v === 'number')\n const beforeTruth = before.map(truthOf).filter((v): v is number => typeof v === 'number')\n const afterTruth = after.map(truthOf).filter((v): v is number => typeof v === 'number')\n if (\n beforeProxy.length >= 2 &&\n afterProxy.length >= 2 &&\n beforeTruth.length >= 2 &&\n afterTruth.length >= 2\n ) {\n const proxyDelta = mean(afterProxy) - mean(beforeProxy)\n const truthDelta = mean(afterTruth) - mean(beforeTruth)\n // Divergence: proxy goes up while truth goes flat or down.\n // Severity = max(0, (proxyDelta - truthDelta)) — bigger gap = bigger signal.\n const gap = Math.max(0, proxyDelta - truthDelta)\n const severity = clamp01(gap * 5) // scale: 0.2 absolute gap → severity 1.0\n findings.push({\n signal: 'reward_divergence',\n severity,\n message:\n severity >= sus\n ? `proxy reward rose by ${proxyDelta.toFixed(3)} while truth changed by ${truthDelta.toFixed(3)} — potential Goodhart`\n : `proxy and truth moved together (proxy ${proxyDelta.toFixed(3)}, truth ${truthDelta.toFixed(3)})`,\n detail: {\n proxyDelta,\n truthDelta,\n gap,\n beforeN: beforeProxy.length,\n afterN: afterProxy.length,\n },\n })\n }\n }\n\n // ── Signal 2: distributional shift in outputs (KS on score distributions) ──\n {\n const beforeP = before.map(proxyOf).filter((v): v is number => typeof v === 'number')\n const afterP = after.map(proxyOf).filter((v): v is number => typeof v === 'number')\n if (beforeP.length >= 4 && afterP.length >= 4) {\n const ks = ksStatistic(beforeP, afterP)\n // KS statistic: bigger = more shift. We're agnostic about direction;\n // genuine improvement ALSO produces shift, so this signal is\n // contributory rather than load-bearing.\n const severity = clamp01(ks - 0.2)\n findings.push({\n signal: 'distribution_shift',\n severity,\n message:\n severity >= sus\n ? `KS=${ks.toFixed(3)} between before/after windows — distributional shift large`\n : `KS=${ks.toFixed(3)} between before/after windows — within-distribution drift`,\n detail: { ks, beforeN: beforeP.length, afterN: afterP.length },\n })\n }\n }\n\n // ── Signal 3: reward disagreement (proxy vs independent secondary) ────\n {\n const secondaryOf = input.secondaryRewardOf ?? defaultSecondary(input.verifiableRewardOptions)\n const aligned = runs\n .map((r) => ({ p: proxyOf(r), s: secondaryOf(r) }))\n .filter(\n (x): x is { p: number; s: number } => typeof x.p === 'number' && typeof x.s === 'number',\n )\n if (aligned.length >= 4) {\n const ps = aligned.map((x) => x.p)\n const ss = aligned.map((x) => x.s)\n const r = pearsonR(ps, ss)\n // Disagreement: low or negative correlation between primary proxy\n // reward and an independent secondary signal.\n const severity = clamp01(0.5 - Math.max(0, r))\n findings.push({\n signal: 'reward_disagreement',\n severity,\n message:\n severity >= sus\n ? `proxy and independent secondary reward correlate ρ=${r.toFixed(3)} — possibly hacking proxy`\n : `proxy and secondary reward correlate ρ=${r.toFixed(3)}`,\n detail: { pearson: r, n: aligned.length },\n })\n }\n }\n\n // ── Signal 4: judge drift (probabilistic up while deterministic flat) ─\n {\n const detRuns = filterDeterministicallyRewarded(runs, input.verifiableRewardOptions ?? {})\n if (detRuns.length >= 4) {\n const detBefore = detRuns.slice(0, Math.floor(detRuns.length / 2))\n const detAfter = detRuns.slice(Math.floor(detRuns.length / 2))\n const detDelta =\n mean(detAfter.map((r) => r.reward.value)) - mean(detBefore.map((r) => r.reward.value))\n const proxyDelta =\n mean(after.map(proxyOf).filter((v): v is number => typeof v === 'number')) -\n mean(before.map(proxyOf).filter((v): v is number => typeof v === 'number'))\n const driftGap = Math.max(0, proxyDelta - detDelta)\n const severity = clamp01(driftGap * 5)\n findings.push({\n signal: 'judge_drift',\n severity,\n message:\n severity >= sus\n ? `judge proxy +${proxyDelta.toFixed(3)} while deterministic reward +${detDelta.toFixed(3)} — judge drifting up without verifiable backing`\n : `judge and deterministic rewards move in step (judge ${proxyDelta.toFixed(3)}, det ${detDelta.toFixed(3)})`,\n detail: { proxyDelta, detDelta, driftGap, n: detRuns.length },\n })\n }\n }\n\n const maxSev = findings.reduce((m, f) => Math.max(m, f.severity), 0)\n const verdict: RewardHackingReport['verdict'] =\n maxSev >= gam ? 'gaming' : maxSev >= sus ? 'suspect' : 'clean'\n const rationale = findings\n .filter((f) => f.severity >= sus)\n .map((f) => `${f.signal}: severity ${f.severity.toFixed(2)} — ${f.message}`)\n if (rationale.length === 0) rationale.push('no signals fired above suspect threshold')\n\n return { findings, verdict, rationale, n }\n}\n\n// ── Helpers ──────────────────────────────────────────────────────────────\n\nfunction mean(xs: number[]): number {\n if (xs.length === 0) return 0\n return xs.reduce((s, x) => s + x, 0) / xs.length\n}\n\nfunction clamp01(x: number): number {\n if (!Number.isFinite(x)) return 0\n return Math.max(0, Math.min(1, x))\n}\n\nfunction pearsonR(a: number[], b: number[]): number {\n if (a.length !== b.length || a.length < 2) return 0\n const ma = mean(a)\n const mb = mean(b)\n let num = 0,\n da = 0,\n db = 0\n for (let i = 0; i < a.length; i++) {\n const xa = a[i]! - ma\n const xb = b[i]! - mb\n num += xa * xb\n da += xa * xa\n db += xb * xb\n }\n if (da === 0 || db === 0) return 0\n return num / Math.sqrt(da * db)\n}\n\nfunction ksStatistic(a: number[], b: number[]): number {\n // Two-sample Kolmogorov-Smirnov statistic.\n const sortedA = [...a].sort((x, y) => x - y)\n const sortedB = [...b].sort((x, y) => x - y)\n const all = [...new Set([...sortedA, ...sortedB])].sort((x, y) => x - y)\n let max = 0\n for (const v of all) {\n const fa = sortedA.filter((x) => x <= v).length / sortedA.length\n const fb = sortedB.filter((x) => x <= v).length / sortedB.length\n max = Math.max(max, Math.abs(fa - fb))\n }\n return max\n}\n\nfunction defaultSecondary(\n verifiableOpts?: VerifiableRewardExtractionOptions,\n): (run: RunRecord) => number | null {\n return (run: RunRecord) => {\n const filtered = filterDeterministicallyRewarded([run], verifiableOpts ?? {})\n return filtered.length === 1 ? filtered[0]!.reward.value : null\n }\n}\n"],"mappings":";AAwGA,IAAM,+BAA+B,oBAAI,IAAI;AAAA,EAC3C;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF,CAAC;AAED,IAAM,qBAAqB,CAAC,SAAyC;AACnE,QAAM,QAAQ,KAAK,YAAY;AAC/B,MAAI,MAAM,SAAS,MAAM,EAAG,QAAO;AACnC,MACE,MAAM,SAAS,SAAS,KACxB,MAAM,SAAS,OAAO,KACtB,MAAM,SAAS,WAAW,KAC1B,MAAM,SAAS,MAAM;AAErB,WAAO;AACT,MAAI,MAAM,SAAS,QAAQ,EAAG,QAAO;AACrC,MAAI,MAAM,SAAS,SAAS,EAAG,QAAO;AACtC,MAAI,MAAM,SAAS,OAAO,KAAK,MAAM,SAAS,UAAU,EAAG,QAAO;AAClE,SAAO;AACT;AAUO,SAAS,wBACd,QACA,OAA0C,CAAC,GAClB;AACzB,QAAM,mBAAmB,IAAI,IAAI,KAAK,uBAAuB,CAAC,GAAG,4BAA4B,CAAC;AAC9F,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,kBAAkB,KAAK,mBAAmB;AAChD,QAAM,aAAa,KAAK,wBAAwB;AAEhD,QAAM,gBAAgB,OAAO,OAAO;AAAA,IAClC,CAAC,MAAM,iBAAiB,IAAI,EAAE,KAAK,KAAK,OAAO,EAAE,UAAU,YAAY,OAAO,SAAS,EAAE,KAAK;AAAA,EAChG;AAEA,MAAI,cAAc,WAAW,GAAG;AAC9B,UAAM,QAAQ,cAAc,CAAC;AAC7B,UAAM,QAAQ,QAAQ,MAAM,KAAM;AAClC,WAAO;AAAA,MACL;AAAA,MACA,QAAQ,UAAU,MAAM,KAAK;AAAA,MAC7B,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ,MAAM;AAAA,MACd,YAAY,EAAE,CAAC,MAAM,KAAK,GAAG,MAAM;AAAA,MACnC,WAAW,eAAe,KAAK;AAAA,IACjC;AAAA,EACF;AAEA,MAAI,cAAc,SAAS,GAAG;AAE5B,QAAI,MAAM;AACV,QAAI,QAAQ;AACZ,UAAM,aAAqC,CAAC;AAC5C,eAAW,KAAK,eAAe;AAC7B,YAAM,IAAK,EAAE,QAAQ,UAAiC;AACtD,aAAO,KAAK,EAAE,SAAS;AACvB,eAAS;AACT,iBAAW,EAAE,KAAK,IAAI,EAAE;AAAA,IAC1B;AACA,WAAO;AAAA,MACL,OAAO,UAAU,IAAI,IAAI,QAAQ,MAAM,KAAK;AAAA,MAC5C,QAAQ;AAAA,MACR,aAAa;AAAA,MACb,YAAY;AAAA,MACZ,QAAQ,cAAc,IAAI,CAAC,MAAM,EAAE,KAAK,EAAE,KAAK,GAAG;AAAA,MAClD;AAAA,MACA,WAAW,EAAE,GAAG,WAAW;AAAA,IAC7B;AAAA,EACF;AAEA,MAAI,CAAC,gBAAiB,QAAO;AAE7B,QAAM,QACJ,OAAO,OAAO;AAAA,IACZ,CAAC,MACC,OAAO,EAAE,UAAU,YAAY,OAAO,SAAS,EAAE,KAAK,KAAK,UAAU,EAAE,KAAK,MAAM;AAAA,EACtF,KAAK,OAAO,OAAO,KAAK,CAAC,MAAM,OAAO,EAAE,UAAU,YAAY,OAAO,SAAS,EAAE,KAAK,CAAC;AAExF,MAAI,CAAC,MAAO,QAAO;AAEnB,QAAM,iBAAiB,MAAM,QAAQ;AACrC,QAAM,aAAa,QAAQ,MAAM,KAAM;AACvC,SAAO;AAAA,IACL,OAAO;AAAA,IACP,QAAQ;AAAA,IACR,aAAa;AAAA,IACb,YAAY,OAAO,mBAAmB,WAAW,iBAAiB;AAAA,IAClE,QAAQ,MAAM;AAAA,IACd,YAAY,EAAE,CAAC,MAAM,KAAK,GAAG,WAAW;AAAA,IACxC,WAAW,eAAe,KAAK;AAAA,EACjC;AACF;AAcO,SAAS,oCACd,MACA,OAA0C,CAAC,GACgB;AAC3D,QAAM,YAAY,KAAK,aAAa;AACpC,QAAM,mBAAmB,IAAI,IAAI,KAAK,uBAAuB,CAAC,GAAG,4BAA4B,CAAC;AAC9F,QAAM,kBAAkB,KAAK,mBAAmB;AAChD,QAAM,aAAa,KAAK,wBAAwB;AAEhD,SAAO,KAAK,IAAI,CAAC,QAAQ;AAEvB,UAAM,cAAsD,CAAC;AAC7D,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,IAAI,QAAQ,GAAG,GAAG;AACpD,UACE,EAAE,WAAW,QAAQ,KACrB,CAAC,EAAE,SAAS,KAAK,CAAC,KAClB,OAAO,MAAM,YACb,OAAO,SAAS,CAAC,GACjB;AACA,oBAAY,KAAK,EAAE,MAAM,EAAE,MAAM,SAAS,MAAM,GAAG,OAAO,EAAE,CAAC;AAAA,MAC/D;AAAA,IACF;AACA,UAAM,MAAM,YAAY,OAAO,CAAC,MAAM,iBAAiB,IAAI,EAAE,IAAI,CAAC;AAElE,QAAI,IAAI,WAAW,GAAG;AACpB,YAAM,QAAQ,IAAI,CAAC;AACnB,YAAM,QAAQ,QAAQ,MAAM,KAAK;AACjC,aAAO;AAAA,QACL,OAAO,IAAI;AAAA,QACX,QAAQ;AAAA,UACN;AAAA,UACA,QAAQ,UAAU,MAAM,IAAI;AAAA,UAC5B,aAAa;AAAA,UACb,YAAY;AAAA,UACZ,QAAQ,MAAM;AAAA,UACd,YAAY,EAAE,CAAC,MAAM,IAAI,GAAG,MAAM;AAAA,QACpC;AAAA,MACF;AAAA,IACF;AACA,QAAI,IAAI,SAAS,GAAG;AAClB,YAAM,QAAQ,IAAI,OAAO,CAAC,GAAG,MAAM,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI;AACzD,YAAM,aAAqC,OAAO;AAAA,QAChD,IAAI,IAAI,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,KAAK,CAAC;AAAA,MAClC;AACA,aAAO;AAAA,QACL,OAAO,IAAI;AAAA,QACX,QAAQ;AAAA,UACN,OAAO,QAAQ,KAAK;AAAA,UACpB,QAAQ;AAAA,UACR,aAAa;AAAA,UACb,YAAY;AAAA,UACZ,QAAQ,IAAI,IAAI,CAAC,MAAM,EAAE,IAAI,EAAE,KAAK,GAAG;AAAA,UACvC;AAAA,UACA,WAAW,EAAE,GAAG,WAAW;AAAA,QAC7B;AAAA,MACF;AAAA,IACF;AACA,QAAI,CAAC,gBAAiB,QAAO,EAAE,OAAO,IAAI,OAAO,QAAQ,KAAK;AAG9D,UAAM,UAAU,IAAI,QAAQ,gBAAgB,IAAI,QAAQ;AACxD,QAAI,OAAO,YAAY,YAAY,CAAC,OAAO,SAAS,OAAO,GAAG;AAC5D,aAAO,EAAE,OAAO,IAAI,OAAO,QAAQ,KAAK;AAAA,IAC1C;AACA,UAAM,eAAe,QAAQ,OAAO;AACpC,WAAO;AAAA,MACL,OAAO,IAAI;AAAA,MACX,QAAQ;AAAA,QACN,OAAO;AAAA,QACP,QAAQ;AAAA,QACR,aAAa;AAAA,QACb,YAAY;AAAA,QACZ,QAAQ;AAAA,QACR,YAAY,EAAE,qBAAqB,aAAa;AAAA,MAClD;AAAA,IACF;AAAA,EACF,CAAC;AACH;AAGO,SAAS,gCACd,MACA,OAA0C,CAAC,GACU;AACrD,QAAM,WAAW,oCAAoC,MAAM,EAAE,GAAG,MAAM,iBAAiB,MAAM,CAAC;AAC9F,QAAM,MAA2D,CAAC;AAClE,WAAS,IAAI,GAAG,IAAI,KAAK,QAAQ,KAAK;AACpC,UAAM,IAAI,SAAS,CAAC;AACpB,QAAI,EAAE,UAAU,EAAE,OAAO,gBAAgB,iBAAiB;AACxD,UAAI,KAAK,EAAE,KAAK,KAAK,CAAC,GAAI,QAAQ,EAAE,OAAO,CAAC;AAAA,IAC9C;AAAA,EACF;AACA,SAAO;AACT;AAIA,SAAS,QAAQ,GAAmB;AAClC,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AACnC;AAEA,SAAS,eAAe,GAAwC;AAC9D,QAAM,MAA8B,CAAC;AACrC,MAAI,EAAE,aAAa;AACjB,eAAW,CAAC,GAAG,CAAC,KAAK,OAAO,QAAQ,EAAE,WAAW,GAAG;AAClD,UAAI,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,EAAG,KAAI,CAAC,IAAI;AAAA,IAC5D;AAAA,EACF;AACA,SAAO;AACT;;;AC/NA,IAAM,gBAAgB,CAAC,MAAgC;AACrD,QAAM,IAAI,EAAE,QAAQ,gBAAgB,EAAE,QAAQ;AAC9C,SAAO,OAAO,MAAM,YAAY,OAAO,SAAS,CAAC,IAAI,IAAI;AAC3D;AAEO,SAAS,oBAAoB,OAAsD;AACxF,QAAM,UAAU,MAAM,WAAW;AACjC,QAAM,UAAU,MAAM;AACtB,QAAM,MAAM,MAAM,YAAY,WAAW;AACzC,QAAM,MAAM,MAAM,YAAY,UAAU;AAExC,QAAM,OAAO,MAAM,KAAK,OAAO,CAAC,MAAM,QAAQ,CAAC,MAAM,IAAI;AACzD,QAAM,IAAI,KAAK;AACf,MAAI,IAAI,GAAG;AACT,WAAO;AAAA,MACL,UAAU,CAAC;AAAA,MACX,SAAS;AAAA,MACT;AAAA,MACA,WAAW,CAAC,0CAA0C,CAAC,0BAA0B;AAAA,IACnF;AAAA,EACF;AACA,QAAM,aAAa,KAAK,IAAI,GAAG,MAAM,cAAc,KAAK,IAAI,IAAI,KAAK,MAAM,IAAI,CAAC,CAAC,CAAC;AAClF,QAAM,SAAS,KAAK,MAAM,GAAG,IAAI,UAAU;AAC3C,QAAM,QAAQ,KAAK,MAAM,IAAI,UAAU;AAEvC,QAAM,WAAmC,CAAC;AAG1C,MAAI,SAAS;AACX,UAAM,cAAc,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACxF,UAAM,aAAa,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACtF,UAAM,cAAc,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACxF,UAAM,aAAa,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACtF,QACE,YAAY,UAAU,KACtB,WAAW,UAAU,KACrB,YAAY,UAAU,KACtB,WAAW,UAAU,GACrB;AACA,YAAM,aAAa,KAAK,UAAU,IAAI,KAAK,WAAW;AACtD,YAAM,aAAa,KAAK,UAAU,IAAI,KAAK,WAAW;AAGtD,YAAM,MAAM,KAAK,IAAI,GAAG,aAAa,UAAU;AAC/C,YAAM,WAAWA,SAAQ,MAAM,CAAC;AAChC,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,wBAAwB,WAAW,QAAQ,CAAC,CAAC,2BAA2B,WAAW,QAAQ,CAAC,CAAC,+BAC7F,yCAAyC,WAAW,QAAQ,CAAC,CAAC,WAAW,WAAW,QAAQ,CAAC,CAAC;AAAA,QACpG,QAAQ;AAAA,UACN;AAAA,UACA;AAAA,UACA;AAAA,UACA,SAAS,YAAY;AAAA,UACrB,QAAQ,WAAW;AAAA,QACrB;AAAA,MACF,CAAC;AAAA,IACH;AAAA,EACF;AAGA;AACE,UAAM,UAAU,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AACpF,UAAM,SAAS,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ;AAClF,QAAI,QAAQ,UAAU,KAAK,OAAO,UAAU,GAAG;AAC7C,YAAM,KAAK,YAAY,SAAS,MAAM;AAItC,YAAM,WAAWA,SAAQ,KAAK,GAAG;AACjC,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,MAAM,GAAG,QAAQ,CAAC,CAAC,oEACnB,MAAM,GAAG,QAAQ,CAAC,CAAC;AAAA,QACzB,QAAQ,EAAE,IAAI,SAAS,QAAQ,QAAQ,QAAQ,OAAO,OAAO;AAAA,MAC/D,CAAC;AAAA,IACH;AAAA,EACF;AAGA;AACE,UAAM,cAAc,MAAM,qBAAqB,iBAAiB,MAAM,uBAAuB;AAC7F,UAAM,UAAU,KACb,IAAI,CAAC,OAAO,EAAE,GAAG,QAAQ,CAAC,GAAG,GAAG,YAAY,CAAC,EAAE,EAAE,EACjD;AAAA,MACC,CAAC,MAAqC,OAAO,EAAE,MAAM,YAAY,OAAO,EAAE,MAAM;AAAA,IAClF;AACF,QAAI,QAAQ,UAAU,GAAG;AACvB,YAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;AACjC,YAAM,KAAK,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;AACjC,YAAM,IAAI,SAAS,IAAI,EAAE;AAGzB,YAAM,WAAWA,SAAQ,MAAM,KAAK,IAAI,GAAG,CAAC,CAAC;AAC7C,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,2DAAsD,EAAE,QAAQ,CAAC,CAAC,mCAClE,+CAA0C,EAAE,QAAQ,CAAC,CAAC;AAAA,QAC5D,QAAQ,EAAE,SAAS,GAAG,GAAG,QAAQ,OAAO;AAAA,MAC1C,CAAC;AAAA,IACH;AAAA,EACF;AAGA;AACE,UAAM,UAAU,gCAAgC,MAAM,MAAM,2BAA2B,CAAC,CAAC;AACzF,QAAI,QAAQ,UAAU,GAAG;AACvB,YAAM,YAAY,QAAQ,MAAM,GAAG,KAAK,MAAM,QAAQ,SAAS,CAAC,CAAC;AACjE,YAAM,WAAW,QAAQ,MAAM,KAAK,MAAM,QAAQ,SAAS,CAAC,CAAC;AAC7D,YAAM,WACJ,KAAK,SAAS,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,CAAC,MAAM,EAAE,OAAO,KAAK,CAAC;AACvF,YAAM,aACJ,KAAK,MAAM,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,CAAC,IACzE,KAAK,OAAO,IAAI,OAAO,EAAE,OAAO,CAAC,MAAmB,OAAO,MAAM,QAAQ,CAAC;AAC5E,YAAM,WAAW,KAAK,IAAI,GAAG,aAAa,QAAQ;AAClD,YAAM,WAAWA,SAAQ,WAAW,CAAC;AACrC,eAAS,KAAK;AAAA,QACZ,QAAQ;AAAA,QACR;AAAA,QACA,SACE,YAAY,MACR,gBAAgB,WAAW,QAAQ,CAAC,CAAC,gCAAgC,SAAS,QAAQ,CAAC,CAAC,yDACxF,uDAAuD,WAAW,QAAQ,CAAC,CAAC,SAAS,SAAS,QAAQ,CAAC,CAAC;AAAA,QAC9G,QAAQ,EAAE,YAAY,UAAU,UAAU,GAAG,QAAQ,OAAO;AAAA,MAC9D,CAAC;AAAA,IACH;AAAA,EACF;AAEA,QAAM,SAAS,SAAS,OAAO,CAAC,GAAG,MAAM,KAAK,IAAI,GAAG,EAAE,QAAQ,GAAG,CAAC;AACnE,QAAM,UACJ,UAAU,MAAM,WAAW,UAAU,MAAM,YAAY;AACzD,QAAM,YAAY,SACf,OAAO,CAAC,MAAM,EAAE,YAAY,GAAG,EAC/B,IAAI,CAAC,MAAM,GAAG,EAAE,MAAM,cAAc,EAAE,SAAS,QAAQ,CAAC,CAAC,WAAM,EAAE,OAAO,EAAE;AAC7E,MAAI,UAAU,WAAW,EAAG,WAAU,KAAK,0CAA0C;AAErF,SAAO,EAAE,UAAU,SAAS,WAAW,EAAE;AAC3C;AAIA,SAAS,KAAK,IAAsB;AAClC,MAAI,GAAG,WAAW,EAAG,QAAO;AAC5B,SAAO,GAAG,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,GAAG;AAC5C;AAEA,SAASA,SAAQ,GAAmB;AAClC,MAAI,CAAC,OAAO,SAAS,CAAC,EAAG,QAAO;AAChC,SAAO,KAAK,IAAI,GAAG,KAAK,IAAI,GAAG,CAAC,CAAC;AACnC;AAEA,SAAS,SAAS,GAAa,GAAqB;AAClD,MAAI,EAAE,WAAW,EAAE,UAAU,EAAE,SAAS,EAAG,QAAO;AAClD,QAAM,KAAK,KAAK,CAAC;AACjB,QAAM,KAAK,KAAK,CAAC;AACjB,MAAI,MAAM,GACR,KAAK,GACL,KAAK;AACP,WAAS,IAAI,GAAG,IAAI,EAAE,QAAQ,KAAK;AACjC,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,UAAM,KAAK,EAAE,CAAC,IAAK;AACnB,WAAO,KAAK;AACZ,UAAM,KAAK;AACX,UAAM,KAAK;AAAA,EACb;AACA,MAAI,OAAO,KAAK,OAAO,EAAG,QAAO;AACjC,SAAO,MAAM,KAAK,KAAK,KAAK,EAAE;AAChC;AAEA,SAAS,YAAY,GAAa,GAAqB;AAErD,QAAM,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3C,QAAM,UAAU,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AAC3C,QAAM,MAAM,CAAC,GAAG,oBAAI,IAAI,CAAC,GAAG,SAAS,GAAG,OAAO,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,MAAM,IAAI,CAAC;AACvE,MAAI,MAAM;AACV,aAAW,KAAK,KAAK;AACnB,UAAM,KAAK,QAAQ,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,SAAS,QAAQ;AAC1D,UAAM,KAAK,QAAQ,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,SAAS,QAAQ;AAC1D,UAAM,KAAK,IAAI,KAAK,KAAK,IAAI,KAAK,EAAE,CAAC;AAAA,EACvC;AACA,SAAO;AACT;AAEA,SAAS,iBACP,gBACmC;AACnC,SAAO,CAAC,QAAmB;AACzB,UAAM,WAAW,gCAAgC,CAAC,GAAG,GAAG,kBAAkB,CAAC,CAAC;AAC5E,WAAO,SAAS,WAAW,IAAI,SAAS,CAAC,EAAG,OAAO,QAAQ;AAAA,EAC7D;AACF;","names":["clamp01"]}
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
import {
|
|
2
2
|
runCanaries,
|
|
3
3
|
scoreRedTeamOutput
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-ZFIBGEOL.js";
|
|
5
5
|
import {
|
|
6
6
|
runCampaign
|
|
7
|
-
} from "./chunk-
|
|
7
|
+
} from "./chunk-TWS7AZEY.js";
|
|
8
8
|
import {
|
|
9
9
|
detectRewardHacking
|
|
10
|
-
} from "./chunk-
|
|
10
|
+
} from "./chunk-5HRORJQY.js";
|
|
11
11
|
import {
|
|
12
12
|
pairedBootstrap
|
|
13
|
-
} from "./chunk-
|
|
13
|
+
} from "./chunk-LMZQ2Z4U.js";
|
|
14
14
|
|
|
15
15
|
// src/campaign/drivers/evolutionary.ts
|
|
16
16
|
function evolutionaryDriver(opts) {
|
|
@@ -432,4 +432,4 @@ export {
|
|
|
432
432
|
paretoSignificanceGate,
|
|
433
433
|
runEval
|
|
434
434
|
};
|
|
435
|
-
//# sourceMappingURL=chunk-
|
|
435
|
+
//# sourceMappingURL=chunk-6SOJM3VR.js.map
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// src/series-convergence.ts
|
|
2
|
+
function analyzeSeries(values, options = {}) {
|
|
3
|
+
const window = options.window ?? 5;
|
|
4
|
+
const stableCv = options.stableCv ?? 0.05;
|
|
5
|
+
const driftRun = options.driftRun ?? 3;
|
|
6
|
+
if (values.length < Math.max(2, Math.min(window, 3))) {
|
|
7
|
+
return { state: "insufficient-data", windowMean: 0, windowCv: 0, tailRun: 0, stable: false };
|
|
8
|
+
}
|
|
9
|
+
const tail = values.slice(-window);
|
|
10
|
+
const mean = tail.reduce((a, b) => a + b, 0) / tail.length;
|
|
11
|
+
const variance = tail.reduce((acc, v) => acc + (v - mean) ** 2, 0) / tail.length;
|
|
12
|
+
const stdDev = Math.sqrt(variance);
|
|
13
|
+
const refMean = Math.abs(mean) > 1e-9 ? Math.abs(mean) : 1;
|
|
14
|
+
const cv = stdDev / refMean;
|
|
15
|
+
const stable = tail.length >= window && cv <= stableCv;
|
|
16
|
+
let tailRun = 0;
|
|
17
|
+
let direction = 0;
|
|
18
|
+
for (let i = values.length - 1; i > 0; i--) {
|
|
19
|
+
const delta = values[i] - values[i - 1];
|
|
20
|
+
if (delta === 0) break;
|
|
21
|
+
const dir = delta > 0 ? 1 : -1;
|
|
22
|
+
if (direction === 0) direction = dir;
|
|
23
|
+
if (dir !== direction) break;
|
|
24
|
+
tailRun += dir;
|
|
25
|
+
}
|
|
26
|
+
let state;
|
|
27
|
+
if (stable) {
|
|
28
|
+
state = "stabilized";
|
|
29
|
+
} else if (Math.abs(tailRun) >= driftRun) {
|
|
30
|
+
state = tailRun > 0 ? "drifting-up" : "drifting-down";
|
|
31
|
+
} else {
|
|
32
|
+
state = "noisy";
|
|
33
|
+
}
|
|
34
|
+
return { state, windowMean: mean, windowCv: cv, tailRun, stable };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export {
|
|
38
|
+
analyzeSeries
|
|
39
|
+
};
|
|
40
|
+
//# sourceMappingURL=chunk-BOD4O7OF.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/series-convergence.ts"],"sourcesContent":["/**\n * Series convergence — detects whether a sequence of scalar measurements\n * is stabilizing, drifting, or noisy.\n *\n * Lifted from ADC convergence.ts. The per-turn `ConvergenceTracker` is\n * about progress *within* a single run; this module is about drift\n * *across* runs (e.g. \"are my nightly eval scores stabilizing?\").\n *\n * Three signals:\n * - stabilized: last K values have low variance (< epsilon) — done\n * - drifting: recent trend is monotonic and beyond noise — regressing or improving\n * - noisy: neither — keep iterating, but flag as untrustworthy for gating\n */\n\nexport interface SeriesConvergenceOptions {\n /** Window size for \"recent\" analysis (default 5). */\n window?: number\n /** Coefficient-of-variation threshold below which the window is stabilized (default 0.05 = 5%). */\n stableCv?: number\n /** Minimum monotone run length to call drift (default 3). */\n driftRun?: number\n}\n\nexport interface SeriesConvergenceResult {\n state: 'stabilized' | 'drifting-up' | 'drifting-down' | 'noisy' | 'insufficient-data'\n windowMean: number\n windowCv: number\n /** Longest monotonic run at the tail of the series (positive for up, negative for down). */\n tailRun: number\n /** True when n ≥ window AND windowCv ≤ stableCv. */\n stable: boolean\n}\n\nexport function analyzeSeries(\n values: number[],\n options: SeriesConvergenceOptions = {},\n): SeriesConvergenceResult {\n const window = options.window ?? 5\n const stableCv = options.stableCv ?? 0.05\n const driftRun = options.driftRun ?? 3\n\n if (values.length < Math.max(2, Math.min(window, 3))) {\n return { state: 'insufficient-data', windowMean: 0, windowCv: 0, tailRun: 0, stable: false }\n }\n\n const tail = values.slice(-window)\n const mean = tail.reduce((a, b) => a + b, 0) / tail.length\n const variance = tail.reduce((acc, v) => acc + (v - mean) ** 2, 0) / tail.length\n const stdDev = Math.sqrt(variance)\n const refMean = Math.abs(mean) > 1e-9 ? Math.abs(mean) : 1\n const cv = stdDev / refMean\n const stable = tail.length >= window && cv <= stableCv\n\n // Tail monotonic run: count how many consecutive strictly-increasing (or decreasing)\n // steps end at the final value.\n let tailRun = 0\n let direction: 1 | -1 | 0 = 0\n for (let i = values.length - 1; i > 0; i--) {\n const delta = values[i]! - values[i - 1]!\n if (delta === 0) break\n const dir = delta > 0 ? 1 : -1\n if (direction === 0) direction = dir\n if (dir !== direction) break\n tailRun += dir\n }\n\n let state: SeriesConvergenceResult['state']\n if (stable) {\n state = 'stabilized'\n } else if (Math.abs(tailRun) >= driftRun) {\n state = tailRun > 0 ? 'drifting-up' : 'drifting-down'\n } else {\n state = 'noisy'\n }\n\n return { state, windowMean: mean, windowCv: cv, tailRun, stable }\n}\n"],"mappings":";AAiCO,SAAS,cACd,QACA,UAAoC,CAAC,GACZ;AACzB,QAAM,SAAS,QAAQ,UAAU;AACjC,QAAM,WAAW,QAAQ,YAAY;AACrC,QAAM,WAAW,QAAQ,YAAY;AAErC,MAAI,OAAO,SAAS,KAAK,IAAI,GAAG,KAAK,IAAI,QAAQ,CAAC,CAAC,GAAG;AACpD,WAAO,EAAE,OAAO,qBAAqB,YAAY,GAAG,UAAU,GAAG,SAAS,GAAG,QAAQ,MAAM;AAAA,EAC7F;AAEA,QAAM,OAAO,OAAO,MAAM,CAAC,MAAM;AACjC,QAAM,OAAO,KAAK,OAAO,CAAC,GAAG,MAAM,IAAI,GAAG,CAAC,IAAI,KAAK;AACpD,QAAM,WAAW,KAAK,OAAO,CAAC,KAAK,MAAM,OAAO,IAAI,SAAS,GAAG,CAAC,IAAI,KAAK;AAC1E,QAAM,SAAS,KAAK,KAAK,QAAQ;AACjC,QAAM,UAAU,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,IAAI,IAAI,IAAI;AACzD,QAAM,KAAK,SAAS;AACpB,QAAM,SAAS,KAAK,UAAU,UAAU,MAAM;AAI9C,MAAI,UAAU;AACd,MAAI,YAAwB;AAC5B,WAAS,IAAI,OAAO,SAAS,GAAG,IAAI,GAAG,KAAK;AAC1C,UAAM,QAAQ,OAAO,CAAC,IAAK,OAAO,IAAI,CAAC;AACvC,QAAI,UAAU,EAAG;AACjB,UAAM,MAAM,QAAQ,IAAI,IAAI;AAC5B,QAAI,cAAc,EAAG,aAAY;AACjC,QAAI,QAAQ,UAAW;AACvB,eAAW;AAAA,EACb;AAEA,MAAI;AACJ,MAAI,QAAQ;AACV,YAAQ;AAAA,EACV,WAAW,KAAK,IAAI,OAAO,KAAK,UAAU;AACxC,YAAQ,UAAU,IAAI,gBAAgB;AAAA,EACxC,OAAO;AACL,YAAQ;AAAA,EACV;AAEA,SAAO,EAAE,OAAO,YAAY,MAAM,UAAU,IAAI,SAAS,OAAO;AAClE;","names":[]}
|
|
@@ -5,7 +5,7 @@ import {
|
|
|
5
5
|
pairedBootstrap,
|
|
6
6
|
pairedMde,
|
|
7
7
|
wilcoxonSignedRank
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-LMZQ2Z4U.js";
|
|
9
9
|
import {
|
|
10
10
|
canonicalize,
|
|
11
11
|
hashJson
|
|
@@ -877,4 +877,4 @@ export {
|
|
|
877
877
|
RESEARCH_REPORT_HARD_PAIR_FLOOR,
|
|
878
878
|
researchReport
|
|
879
879
|
};
|
|
880
|
-
//# sourceMappingURL=chunk-
|
|
880
|
+
//# sourceMappingURL=chunk-CY6U5S3X.js.map
|
|
@@ -1,40 +1,13 @@
|
|
|
1
1
|
import {
|
|
2
2
|
callLlm
|
|
3
3
|
} from "./chunk-CVVHBFGN.js";
|
|
4
|
+
import {
|
|
5
|
+
makeFinding
|
|
6
|
+
} from "./chunk-45EEMHTC.js";
|
|
4
7
|
import {
|
|
5
8
|
TraceFileMissingError,
|
|
6
9
|
buildTraceAnalystTools
|
|
7
|
-
} from "./chunk-
|
|
8
|
-
|
|
9
|
-
// src/analyst/types.ts
|
|
10
|
-
import { createHash } from "crypto";
|
|
11
|
-
function computeFindingId(input) {
|
|
12
|
-
const basis = JSON.stringify({
|
|
13
|
-
a: input.analyst_id,
|
|
14
|
-
r: input.area,
|
|
15
|
-
s: input.subject ?? "",
|
|
16
|
-
c: normalizeClaim(input.id_basis ?? input.claim)
|
|
17
|
-
});
|
|
18
|
-
return `f_${createHash("sha256").update(basis).digest("hex").slice(0, 20)}`;
|
|
19
|
-
}
|
|
20
|
-
function normalizeClaim(c) {
|
|
21
|
-
return c.toLowerCase().replace(/\s+/g, " ").replace(/[.!?;:,]+$/g, "").trim();
|
|
22
|
-
}
|
|
23
|
-
function makeFinding(init) {
|
|
24
|
-
const { id_basis, produced_at, ...rest } = init;
|
|
25
|
-
return {
|
|
26
|
-
schema_version: "1.0.0",
|
|
27
|
-
finding_id: computeFindingId({
|
|
28
|
-
analyst_id: rest.analyst_id,
|
|
29
|
-
area: rest.area,
|
|
30
|
-
subject: rest.subject,
|
|
31
|
-
claim: rest.claim,
|
|
32
|
-
id_basis
|
|
33
|
-
}),
|
|
34
|
-
produced_at: produced_at ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
35
|
-
...rest
|
|
36
|
-
};
|
|
37
|
-
}
|
|
10
|
+
} from "./chunk-QAY5UIJO.js";
|
|
38
11
|
|
|
39
12
|
// src/analyst/finding-subject.ts
|
|
40
13
|
import { z } from "zod";
|
|
@@ -1015,8 +988,6 @@ function selectPriorFindings(source, analystId) {
|
|
|
1015
988
|
}
|
|
1016
989
|
|
|
1017
990
|
export {
|
|
1018
|
-
computeFindingId,
|
|
1019
|
-
makeFinding,
|
|
1020
991
|
FINDING_SUBJECT_KINDS,
|
|
1021
992
|
parseFindingSubject,
|
|
1022
993
|
renderFindingSubject,
|
|
@@ -1041,4 +1012,4 @@ export {
|
|
|
1041
1012
|
DEFAULT_TRACE_ANALYST_KINDS,
|
|
1042
1013
|
AnalystRegistry
|
|
1043
1014
|
};
|
|
1044
|
-
//# sourceMappingURL=chunk-
|
|
1015
|
+
//# sourceMappingURL=chunk-D3V5B42D.js.map
|