@tangle-network/agent-eval 0.117.1 → 0.118.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +23 -0
- package/dist/analyst/index.d.ts +2772 -21
- package/dist/analyst/index.js +7 -6
- package/dist/analyst/index.js.map +1 -1
- package/dist/belief-state/index.d.ts +706 -10
- package/dist/belief-state/index.js +2 -1
- package/dist/belief-state/index.js.map +1 -1
- package/dist/benchmarks/index.d.ts +958 -14
- package/dist/benchmarks/index.js +12 -10
- package/dist/builder-eval/index.d.ts +449 -4
- package/dist/builder-eval/index.js +4 -3
- package/dist/builder-eval/index.js.map +1 -1
- package/dist/campaign/index.d.ts +4275 -73
- package/dist/campaign/index.js +12 -10
- package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js} +6 -6
- package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js} +2 -2
- package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js} +2 -2
- package/dist/{chunk-YZPO4UHR.js → chunk-DP3BBMYJ.js} +100 -143
- package/dist/chunk-DP3BBMYJ.js.map +1 -0
- package/dist/{chunk-E4BUPP7Z.js → chunk-HKUCJ437.js} +38 -66
- package/dist/chunk-HKUCJ437.js.map +1 -0
- package/dist/{chunk-HZHNRYHK.js → chunk-K6N6XJJX.js} +2 -2
- package/dist/chunk-KSDQVPLR.js +286 -0
- package/dist/chunk-KSDQVPLR.js.map +1 -0
- package/dist/chunk-MA6HLL3S.js +65 -0
- package/dist/chunk-MA6HLL3S.js.map +1 -0
- package/dist/{chunk-MGEHEHSN.js → chunk-OIIMMLRB.js} +11 -11
- package/dist/{chunk-DXZRATT5.js → chunk-OYZAPX5G.js} +3 -3
- package/dist/chunk-PXE2VKMX.js +140 -0
- package/dist/chunk-PXE2VKMX.js.map +1 -0
- package/dist/{chunk-JSJZ4PJ6.js → chunk-Q442S5AS.js} +17 -17
- package/dist/{chunk-ODVOOEWQ.js → chunk-QBRSJK47.js} +2 -2
- package/dist/{chunk-S2F4J57L.js → chunk-QKEGNI5B.js} +77 -32
- package/dist/chunk-QKEGNI5B.js.map +1 -0
- package/dist/{chunk-5UF54T55.js → chunk-S3UZOQ5Y.js} +34 -7
- package/dist/chunk-S3UZOQ5Y.js.map +1 -0
- package/dist/{chunk-FQNLDL4D.js → chunk-SVH2ANFD.js} +136 -4
- package/dist/chunk-SVH2ANFD.js.map +1 -0
- package/dist/{chunk-GQCZRZ7L.js → chunk-U5CHZ5M3.js} +9 -9
- package/dist/{chunk-TVVP3ZZQ.js → chunk-VQMK5FMP.js} +2 -1
- package/dist/{chunk-TVVP3ZZQ.js.map → chunk-VQMK5FMP.js.map} +1 -1
- package/dist/chunk-WDHBCA3M.js +31 -0
- package/dist/chunk-WDHBCA3M.js.map +1 -0
- package/dist/{chunk-HQPHZGL6.js → chunk-YLMUS4MM.js} +9 -9
- package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js} +15 -65
- package/dist/chunk-ZET2UAYW.js.map +1 -0
- package/dist/contract/index.d.ts +4012 -38
- package/dist/contract/index.js +56 -29
- package/dist/contract/index.js.map +1 -1
- package/dist/control.d.ts +1013 -9
- package/dist/control.js +4 -3
- package/dist/fuzz.d.ts +194 -4
- package/dist/fuzz.js +3 -3
- package/dist/hosted/index.d.ts +498 -17
- package/dist/index.d.ts +10982 -1286
- package/dist/index.js +97 -80
- package/dist/index.js.map +1 -1
- package/dist/matrix/index.d.ts +139 -4
- package/dist/meta-eval/index.d.ts +862 -15
- package/dist/meta-eval/index.js +2 -1
- package/dist/meta-eval/index.js.map +1 -1
- package/dist/multishot/index.d.ts +214 -14
- package/dist/openapi.json +1 -1
- package/dist/pipelines/index.d.ts +392 -7
- package/dist/pipelines/index.js +5 -3
- package/dist/pipelines/index.js.map +1 -1
- package/dist/reporting.d.ts +1277 -17
- package/dist/rl.d.ts +2359 -28
- package/dist/rl.js +6 -5
- package/dist/rl.js.map +1 -1
- package/dist/storyboard/index.d.ts +86 -1
- package/dist/trace-attributes.d.ts +16 -0
- package/dist/trace-attributes.js +32 -0
- package/dist/trace-attributes.js.map +1 -0
- package/dist/traces.d.ts +1970 -697
- package/dist/traces.js +49 -32
- package/dist/wire/index.d.ts +655 -9
- package/docs/insight-report.md +44 -0
- package/package.json +9 -3
- package/dist/adversarial-B7loGVVX.d.ts +0 -19
- package/dist/analyst-C8HHvfJp.d.ts +0 -88
- package/dist/analyze-runs--2x39HZ7.d.ts +0 -81
- package/dist/baseline-DKq3gJpP.d.ts +0 -141
- package/dist/calibration-C8MTS7cw.d.ts +0 -101
- package/dist/chunk-5UF54T55.js.map +0 -1
- package/dist/chunk-E4BUPP7Z.js.map +0 -1
- package/dist/chunk-FQNLDL4D.js.map +0 -1
- package/dist/chunk-LQUTGLOZ.js.map +0 -1
- package/dist/chunk-S2F4J57L.js.map +0 -1
- package/dist/chunk-YZPO4UHR.js.map +0 -1
- package/dist/code-agent-session-CjZsVd19.d.ts +0 -87
- package/dist/control-6vuGfmDH.d.ts +0 -258
- package/dist/cost-ledger-DWy3XdJc.d.ts +0 -183
- package/dist/dataset-NENEzRgk.d.ts +0 -115
- package/dist/default-registry-DaK8b3fv.d.ts +0 -155
- package/dist/emitter-CjD7vUwv.d.ts +0 -122
- package/dist/errors-oeQrLqXC.d.ts +0 -74
- package/dist/failure-cluster-DOAcSJ87.d.ts +0 -76
- package/dist/feedback-trajectory-BUnM58xL.d.ts +0 -348
- package/dist/gepa-eESocoDi.d.ts +0 -642
- package/dist/index-PdX4VnPA.d.ts +0 -423
- package/dist/insight-report-DY4nDW9Q.d.ts +0 -310
- package/dist/integrity-DqlBiLyK.d.ts +0 -81
- package/dist/judge-calibration-7C-IDmKr.d.ts +0 -145
- package/dist/kind-factory-ClZmO25A.d.ts +0 -171
- package/dist/llm-client-qoDd18Qz.d.ts +0 -289
- package/dist/multi-layer-verifier-BsqKuLyN.d.ts +0 -150
- package/dist/off-policy-DiwuKKg7.d.ts +0 -132
- package/dist/outcome-store-rnXLEqSn.d.ts +0 -63
- package/dist/policy-edit-wG9uFEFm.d.ts +0 -455
- package/dist/pre-registration-BWQhJ3vz.d.ts +0 -761
- package/dist/provenance-DpjwyseI.d.ts +0 -541
- package/dist/query-CF7PG61p.d.ts +0 -35
- package/dist/raw-provider-sink-C46HDghv.d.ts +0 -132
- package/dist/release-report-C8G2i5Xi.d.ts +0 -236
- package/dist/researcher-C8XyxQsu.d.ts +0 -387
- package/dist/rubric-predictive-validity-p49lLVrE.d.ts +0 -105
- package/dist/run-record-BDH49H2E.d.ts +0 -360
- package/dist/runtime-trajectory-DGBIUt4B.d.ts +0 -49
- package/dist/schema-B3Q3l9Z_.d.ts +0 -201
- package/dist/semantic-concept-judge-CXnPEJbf.d.ts +0 -723
- package/dist/sequential-5iSVfzl2.d.ts +0 -139
- package/dist/series-convergence-D5OWMBg6.d.ts +0 -33
- package/dist/statistics-KUnG73jH.d.ts +0 -494
- package/dist/storage-DrX3v_5B.d.ts +0 -50
- package/dist/store-C1YxJDEK.d.ts +0 -248
- package/dist/store-DGqD0Pyo.d.ts +0 -116
- package/dist/summary-report-C5bKFfm-.d.ts +0 -445
- package/dist/test-graded-scenario-B0ybnPY7.d.ts +0 -166
- package/dist/types-BSw1rOUB.d.ts +0 -634
- package/dist/types-BUxNaJ8c.d.ts +0 -108
- package/dist/types-BkfcQnxV.d.ts +0 -313
- package/dist/verdict-C9MlYujm.d.ts +0 -35
- /package/dist/{chunk-VF3XSYTI.js.map → chunk-33JA4TFA.js.map} +0 -0
- /package/dist/{chunk-4JLWXDYA.js.map → chunk-3EHHMC6E.js.map} +0 -0
- /package/dist/{chunk-CCZIVI3F.js.map → chunk-BFW56GTT.js.map} +0 -0
- /package/dist/{chunk-HZHNRYHK.js.map → chunk-K6N6XJJX.js.map} +0 -0
- /package/dist/{chunk-MGEHEHSN.js.map → chunk-OIIMMLRB.js.map} +0 -0
- /package/dist/{chunk-DXZRATT5.js.map → chunk-OYZAPX5G.js.map} +0 -0
- /package/dist/{chunk-JSJZ4PJ6.js.map → chunk-Q442S5AS.js.map} +0 -0
- /package/dist/{chunk-ODVOOEWQ.js.map → chunk-QBRSJK47.js.map} +0 -0
- /package/dist/{chunk-GQCZRZ7L.js.map → chunk-U5CHZ5M3.js.map} +0 -0
- /package/dist/{chunk-HQPHZGL6.js.map → chunk-YLMUS4MM.js.map} +0 -0
package/dist/types-BSw1rOUB.d.ts
DELETED
|
@@ -1,634 +0,0 @@
|
|
|
1
|
-
import { P as PolicyEditCandidateRecord } from './policy-edit-wG9uFEFm.js';
|
|
2
|
-
import { R as RunPaidCallInput, a as CostChannel, P as PaidCallResult, C as CostLedger, b as CostLedgerSummary } from './cost-ledger-DWy3XdJc.js';
|
|
3
|
-
import { L as LlmCallMetadata } from './llm-client-qoDd18Qz.js';
|
|
4
|
-
import { c as RunTokenUsage } from './run-record-BDH49H2E.js';
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Pass A substrate types — `runCampaign` is the one primitive every
|
|
8
|
-
* eval flow composes from. Three contracts in this file:
|
|
9
|
-
*
|
|
10
|
-
* - `Scenario` input set
|
|
11
|
-
* - `DispatchFn` how to run one scenario → artifact
|
|
12
|
-
* - `CampaignResult` defined output schema (the contract downstream tools depend on)
|
|
13
|
-
*
|
|
14
|
-
* Three more lifted from earlier substrate work (re-exported):
|
|
15
|
-
*
|
|
16
|
-
* - `JudgeConfig` pluggable dimensional scorer (0.38)
|
|
17
|
-
* - `Mutator` optimization-loop surface mutator
|
|
18
|
-
* - `Gate` promotion gate (`HeldOutGate` and friends adapt to this)
|
|
19
|
-
*
|
|
20
|
-
* No new architecture vs 0.38 — Pass A formalizes the shapes so consumers
|
|
21
|
-
* can build dashboards / CI gates / regression diffs against a stable schema.
|
|
22
|
-
*/
|
|
23
|
-
|
|
24
|
-
/** Stable identifier + kind tag for any scenario. Consumers
|
|
25
|
-
* extend with their per-domain payload (persona, task, requirement, ...). */
|
|
26
|
-
interface Scenario {
|
|
27
|
-
id: string;
|
|
28
|
-
kind: string;
|
|
29
|
-
tags?: string[];
|
|
30
|
-
}
|
|
31
|
-
/** Context handed to every dispatch invocation. Scoped — every
|
|
32
|
-
* trace/span carries the cellId, every artifact write lands under the cell's
|
|
33
|
-
* artifact root, the cost meter accumulates per cell. */
|
|
34
|
-
interface DispatchContext {
|
|
35
|
-
cellId: string;
|
|
36
|
-
rep: number;
|
|
37
|
-
generation?: number;
|
|
38
|
-
seed: number;
|
|
39
|
-
signal: AbortSignal;
|
|
40
|
-
trace: CampaignTraceWriter;
|
|
41
|
-
artifacts: CampaignArtifactWriter;
|
|
42
|
-
cost: CampaignCostMeter;
|
|
43
|
-
/** Populated when this run is part of a multi-cycle improvement loop. */
|
|
44
|
-
cycleId?: string;
|
|
45
|
-
/** Populated when the substrate resumed from a prior cache hit. */
|
|
46
|
-
resumedFrom?: string;
|
|
47
|
-
/**
|
|
48
|
-
* Opaque placement key supplied by `RunCampaignOptions.cellPlacement`.
|
|
49
|
-
* The substrate forwards it through unchanged; placement-aware Dispatch
|
|
50
|
-
* implementations (e.g. `httpDispatch` from `/adapters/http`) read it to
|
|
51
|
-
* route the cell to the right worker / region / sandbox. `undefined`
|
|
52
|
-
* when no placement strategy is configured.
|
|
53
|
-
*/
|
|
54
|
-
placement?: string;
|
|
55
|
-
}
|
|
56
|
-
/** One function: scenario + ctx → artifact. Dispatcher chooses
|
|
57
|
-
* whether to call `runMultishot`, `runLoop`, raw `streamPrompt`, anything. */
|
|
58
|
-
type DispatchFn<TScenario extends Scenario, TArtifact> = (scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
|
|
59
|
-
/** One session within a multi-session journey. Dispatch is
|
|
60
|
-
* invoked once per session in order; state from prior session's artifact
|
|
61
|
-
* is exposed via `ctx.priorSessionArtifact`. */
|
|
62
|
-
interface SessionScript<TScenario, TArtifact> {
|
|
63
|
-
id: string;
|
|
64
|
-
intent: string;
|
|
65
|
-
maxTurns?: number;
|
|
66
|
-
/** When true, knowledge accumulated this session persists to next. */
|
|
67
|
-
affectsKnowledge?: boolean;
|
|
68
|
-
/** Optional per-session persona evolution — called after the session
|
|
69
|
-
* resolves. Returns the persona shape used by the NEXT session. */
|
|
70
|
-
evolveAfterSession?: (artifact: TArtifact, sessionIndex: number, scenario: TScenario) => TScenario;
|
|
71
|
-
}
|
|
72
|
-
interface JudgeDimension {
|
|
73
|
-
/** JSON field name + score key. */
|
|
74
|
-
key: string;
|
|
75
|
-
/** Description shown in the judge's user prompt. */
|
|
76
|
-
description: string;
|
|
77
|
-
}
|
|
78
|
-
/** Pluggable dimensional scorer. `score` is the contract:
|
|
79
|
-
* given an artifact + scenario, return a `JudgeScore`. This is deliberately a
|
|
80
|
-
* function, not a fixed LLM-prompt shape — real consumers judge with
|
|
81
|
-
* ensembles, deterministic checks, or a single LLM call, and the substrate
|
|
82
|
-
* must not constrain that. The `llmJudge()` helper builds a `score` that does
|
|
83
|
-
* one LLM call for the common case. `appliesTo` lets a judge run only on
|
|
84
|
-
* scenarios that match (e.g. a legal-citation judge only on legal scenarios). */
|
|
85
|
-
interface JudgeConfig<TArtifact, TScenario extends Scenario = Scenario> {
|
|
86
|
-
name: string;
|
|
87
|
-
dimensions: JudgeDimension[];
|
|
88
|
-
/** Stable scoring revision used by campaign resume and verdict caches.
|
|
89
|
-
* Built-in judges derive this from their prompt, model, and rubric. Custom
|
|
90
|
-
* judges should set it when closure state can change without changing code. */
|
|
91
|
-
judgeVersion?: string;
|
|
92
|
-
/** Score one artifact. Throw on failure — a thrown judge is recorded as a
|
|
93
|
-
* failed cell, never silently folded into a zero. */
|
|
94
|
-
score(input: {
|
|
95
|
-
artifact: TArtifact;
|
|
96
|
-
scenario: TScenario;
|
|
97
|
-
signal: AbortSignal;
|
|
98
|
-
/** Shared run spend account and receipt attribution phase. */
|
|
99
|
-
costLedger?: CostLedger;
|
|
100
|
-
costPhase?: string;
|
|
101
|
-
costTags?: Record<string, string>;
|
|
102
|
-
}): JudgeScore | Promise<JudgeScore>;
|
|
103
|
-
appliesTo?: (scenario: TScenario) => boolean;
|
|
104
|
-
}
|
|
105
|
-
/** The canonical judge verdict shape — one declaration, shared by campaign
|
|
106
|
-
* judges and the multishot judge runner (which re-exports this type).
|
|
107
|
-
*
|
|
108
|
-
* Scale is PRODUCER-DEFINED: campaign convention is [0,1]; the legacy
|
|
109
|
-
* multishot runner emits 0-10. Cross-scale comparison must go through
|
|
110
|
-
* `detectScale` (src/campaign/gates/statistical-heldout.ts, used by
|
|
111
|
-
* promotion-policy) — never renormalize a producer's values in place, as
|
|
112
|
-
* downstream thresholds (`composite >= 5` in multishot/matrix.ts, live-soak
|
|
113
|
-
* `>= 7` gates) key on the producer's native scale. */
|
|
114
|
-
interface JudgeScore {
|
|
115
|
-
dimensions: Record<string, number>;
|
|
116
|
-
composite: number;
|
|
117
|
-
notes: string;
|
|
118
|
-
/** Provider metadata for display and diagnostics; accounting uses CostLedger receipts. */
|
|
119
|
-
llmCall?: LlmCallMetadata;
|
|
120
|
-
/** Set when the judge itself failed (call error, unparseable output).
|
|
121
|
-
* `composite`/`dimensions` carry no signal — aggregators MUST exclude
|
|
122
|
-
* failed scores from means instead of folding them into zeros. */
|
|
123
|
-
failed?: true;
|
|
124
|
-
/** Ensemble extras (populated by `ensembleJudge`): max per-dimension
|
|
125
|
-
* spread across surviving judges — the inter-rater signal. */
|
|
126
|
-
maxDisagreement?: number;
|
|
127
|
-
/** Ensemble extras: judge identities whose verdict failed. */
|
|
128
|
-
failedJudges?: string[];
|
|
129
|
-
/** Ensemble extras: each surviving judge's per-dimension scores. */
|
|
130
|
-
perJudge?: Record<string, Record<string, number>>;
|
|
131
|
-
}
|
|
132
|
-
/** A tier-4 code surface — a finalized candidate change to the agent's
|
|
133
|
-
* IMPLEMENTATION, not its prompt. Produced by autoresearch (reads codebase +
|
|
134
|
-
* trace findings → opens a worktree). `worktreeRef` locates the candidate;
|
|
135
|
-
* the exact commits, tree, and binary-patch digest identify it. See the
|
|
136
|
-
* improvement-tier table in `docs/design/loop-taxonomy.md`. */
|
|
137
|
-
interface CodeSurface {
|
|
138
|
-
readonly kind: 'code';
|
|
139
|
-
/** Worktree path or git ref holding the candidate code change. This is a
|
|
140
|
-
* mutable locator and is deliberately excluded from content hashes. */
|
|
141
|
-
readonly worktreeRef: string;
|
|
142
|
-
/** Human-readable ref the worktree was forked from. Not identity-bearing. */
|
|
143
|
-
readonly baseRef: string;
|
|
144
|
-
/** Exact commit the candidate was forked from. */
|
|
145
|
-
readonly baseCommit: string;
|
|
146
|
-
/** Exact tree object for `baseCommit`. */
|
|
147
|
-
readonly baseTree: string;
|
|
148
|
-
/** Exact finalized candidate commit. */
|
|
149
|
-
readonly candidateCommit: string;
|
|
150
|
-
/** Exact tree object for `candidateCommit`. */
|
|
151
|
-
readonly candidateTree: string;
|
|
152
|
-
/** Identity of the exact patch artifact. The deployable candidate bundle
|
|
153
|
-
* carries the same descriptor plus its base64-encoded content. */
|
|
154
|
-
readonly patch: {
|
|
155
|
-
readonly format: 'git-diff-binary';
|
|
156
|
-
readonly sha256: `sha256:${string}`;
|
|
157
|
-
readonly byteLength: number;
|
|
158
|
-
};
|
|
159
|
-
/** Human summary of what changed — rendered into the auto-PR body. */
|
|
160
|
-
readonly summary?: string;
|
|
161
|
-
}
|
|
162
|
-
/** The mutable surface a proposer changes. Tiers (see
|
|
163
|
-
* `docs/design/loop-taxonomy.md`):
|
|
164
|
-
* - `string` — tiers 1-2: system-prompt addendum / serialized tool
|
|
165
|
-
* config. Cheap, reversible, text-diffable.
|
|
166
|
-
* - `CodeSurface` — tier 4: an implementation change behind a worktree ref.
|
|
167
|
-
* Tier 3 (knowledge) is owned by agent-knowledge and rides its own adapter,
|
|
168
|
-
* not this type. */
|
|
169
|
-
type MutableSurface = string | CodeSurface;
|
|
170
|
-
/** A proposer output carrying the surface AND the WHY behind
|
|
171
|
-
* it. Reflective proposers (`gepaProposer`) parse a `{label, rationale, payload}`
|
|
172
|
-
* from the model; without this wrapper the loop keeps only `payload` and the
|
|
173
|
-
* rationale that motivated the change is lost — the candidate becomes
|
|
174
|
-
* unattributable. `propose()` may return either bare `MutableSurface`s (cheap
|
|
175
|
-
* blind mutators) or these (reflective proposers); the loop normalizes both. */
|
|
176
|
-
interface ProposedCandidate {
|
|
177
|
-
surface: MutableSurface;
|
|
178
|
-
/** Short human label for the change (≤ 40 chars typical). */
|
|
179
|
-
label: string;
|
|
180
|
-
/** Why this change was proposed — which failure it targets, which
|
|
181
|
-
* primitive it used. Survives to `GenerationCandidate.rationale` and the
|
|
182
|
-
* emitted provenance record. */
|
|
183
|
-
rationale: string;
|
|
184
|
-
/** Structured, JSON-safe cause for this exact candidate when the proposer
|
|
185
|
-
* can provide one. Policy edits retain the full validated edit here. */
|
|
186
|
-
candidateRecord?: PolicyEditCandidateRecord;
|
|
187
|
-
}
|
|
188
|
-
/** Type guard: a proposal carrying its rationale vs a bare
|
|
189
|
-
* surface. The loop branches on this to populate `GenerationCandidate`. */
|
|
190
|
-
declare function isProposedCandidate(value: MutableSurface | ProposedCandidate): value is ProposedCandidate;
|
|
191
|
-
/** A non-dominated parent on the GEPA Pareto frontier — a
|
|
192
|
-
* surface that, across the per-scenario objective vectors, no other tried
|
|
193
|
-
* surface beats on every scenario. A candidate worse on the mean composite
|
|
194
|
-
* but uniquely best on one hard scenario is non-dominated and survives here;
|
|
195
|
-
* the composite-best ranking would discard the lesson it carries. The loop
|
|
196
|
-
* computes the frontier across ALL generations and hands it to the proposer so
|
|
197
|
-
* a reflective proposer can combine complementary lessons (GEPA, Agrawal et
|
|
198
|
-
* al., arXiv:2507.19457). See `pareto.ts` (`paretoFrontier`). */
|
|
199
|
-
interface ParetoParent {
|
|
200
|
-
surface: MutableSurface;
|
|
201
|
-
surfaceHash: string;
|
|
202
|
-
/** The objective vector: per-scenario composite (higher is better). The
|
|
203
|
-
* axes the frontier is computed over. */
|
|
204
|
-
objectives: Record<string, number>;
|
|
205
|
-
/** Mean composite across the objective scenarios — the scalar summary used
|
|
206
|
-
* for ordering + display, NOT for dominance. */
|
|
207
|
-
composite: number;
|
|
208
|
-
/** Generation that produced this surface (`-1` for the baseline). */
|
|
209
|
-
generation: number;
|
|
210
|
-
label?: string;
|
|
211
|
-
rationale?: string;
|
|
212
|
-
}
|
|
213
|
-
/** Exact measured state for the surface an optimizer is learning from.
|
|
214
|
-
* Unlike a model-authored expected gain, every value here comes from a
|
|
215
|
-
* completed campaign over the designed denominator. */
|
|
216
|
-
interface ScoredSurfaceOutcome {
|
|
217
|
-
/** Optimization/search evidence only. Held-out results must never flow back
|
|
218
|
-
* into a proposer through this type. */
|
|
219
|
-
split: 'search';
|
|
220
|
-
/** Generation that actually measured this surface (`-1` for the baseline). */
|
|
221
|
-
generation: number;
|
|
222
|
-
surfaceHash: string;
|
|
223
|
-
composite: number;
|
|
224
|
-
dimensions: Record<string, number>;
|
|
225
|
-
scenarios: Array<{
|
|
226
|
-
scenarioId: string;
|
|
227
|
-
composite: number;
|
|
228
|
-
notes?: string;
|
|
229
|
-
}>;
|
|
230
|
-
coverage: {
|
|
231
|
-
expectedCells: number;
|
|
232
|
-
scorableCells: number;
|
|
233
|
-
};
|
|
234
|
-
}
|
|
235
|
-
/** Stateless surface mutation — given findings + current
|
|
236
|
-
* surface, return N candidate surfaces. Pure transform, no generation
|
|
237
|
-
* awareness. Reflective-mutation and `AxGEPA` mutators conform. Wrapped by
|
|
238
|
-
* `evolutionaryProposer` to become a `SurfaceProposer`. */
|
|
239
|
-
interface Mutator<TFindings = unknown> {
|
|
240
|
-
kind: string;
|
|
241
|
-
mutate(args: {
|
|
242
|
-
findings: TFindings[];
|
|
243
|
-
currentSurface: MutableSurface;
|
|
244
|
-
populationSize: number;
|
|
245
|
-
signal: AbortSignal;
|
|
246
|
-
}): Promise<Array<MutableSurface | ProposedCandidate>>;
|
|
247
|
-
}
|
|
248
|
-
/** Everything a proposer may read to plan the next
|
|
249
|
-
* batch of candidates. The first six fields are always present; the rest are
|
|
250
|
-
* optional context the loop supplies when available, so cheap proposers
|
|
251
|
-
* (`evolutionaryProposer`) can ignore them while a code-tier agentic generator
|
|
252
|
-
* consumes the report + dataset to drive a coding harness.
|
|
253
|
-
* See `docs/campaign-proposers.md`. */
|
|
254
|
-
interface ProposeContext<TFindings = unknown> {
|
|
255
|
-
currentSurface: MutableSurface;
|
|
256
|
-
history: GenerationRecord[];
|
|
257
|
-
findings: TFindings[];
|
|
258
|
-
/** BREADTH: how many candidate surfaces to return this generation. */
|
|
259
|
-
populationSize: number;
|
|
260
|
-
generation: number;
|
|
261
|
-
signal: AbortSignal;
|
|
262
|
-
/** Measured baseline for this optimization run. `runOptimization` always
|
|
263
|
-
* supplies it; optional for standalone proposer callers. */
|
|
264
|
-
baselineOutcome?: ScoredSurfaceOutcome;
|
|
265
|
-
/** Measured result for `currentSurface`, the complete global incumbent every
|
|
266
|
-
* new candidate mutates. `runOptimization` always supplies it. */
|
|
267
|
-
incumbentOutcome?: ScoredSurfaceOutcome;
|
|
268
|
-
/** Optional analysis report produced before proposal. Opaque to the substrate:
|
|
269
|
-
* the proposer that consumes it owns the shape. */
|
|
270
|
-
report?: unknown;
|
|
271
|
-
/** Handle to all captured data — the proposer samples traces / artifacts /
|
|
272
|
-
* rewards here to ground its proposals. */
|
|
273
|
-
dataset?: LabeledScenarioStore;
|
|
274
|
-
/** DEPTH: max iterations the agentic generator may take per candidate.
|
|
275
|
-
* 1 = single-shot; >1 = it may iterate on its own change before handing it
|
|
276
|
-
* back to be measured. */
|
|
277
|
-
maxImprovementShots?: number;
|
|
278
|
-
/** GEPA Pareto frontier across ALL generations so far — the non-dominated
|
|
279
|
-
* surfaces by per-scenario objective vector. Empty/absent on generation 0
|
|
280
|
-
* (only the baseline is scored). A reflective proposer combines the
|
|
281
|
-
* complementary lessons of these parents (each excels on different
|
|
282
|
-
* scenarios) into a merged candidate. Proposers doing pure single-parent
|
|
283
|
-
* reflection may ignore it. See {@link ParetoParent}. */
|
|
284
|
-
paretoParents?: ParetoParent[];
|
|
285
|
-
/** Shared run spend account and receipt attribution phase. */
|
|
286
|
-
costLedger?: CostLedger;
|
|
287
|
-
costPhase?: string;
|
|
288
|
-
/** FIREWALL (non-negotiable): the held-out judge is write-only — its verdicts
|
|
289
|
-
* score the chosen output and gate promotion, and are NEVER an input to
|
|
290
|
-
* proposal/steering (else the optimizer games the acceptance axis = an
|
|
291
|
-
* oracle). This `never`-typed field makes that a compile-time tripwire: a
|
|
292
|
-
* proposer that tries to thread judge verdicts into the proposal will not type.
|
|
293
|
-
* Steering may consume TRACE-OBSERVABLE signals (what the agent did) via
|
|
294
|
-
* `findings`/`report`; it may NOT consume the judge's held-out verdict. */
|
|
295
|
-
judgeScores?: never;
|
|
296
|
-
}
|
|
297
|
-
/** A surface-improvement strategy. Given the current best
|
|
298
|
-
* surface, the history of what's been tried + scored, and any external
|
|
299
|
-
* findings, propose the next batch of candidate surfaces to measure.
|
|
300
|
-
* Optionally decide to stop early.
|
|
301
|
-
*
|
|
302
|
-
* The evolutionary mutator (`evolutionaryProposer`, here) and agent-runtime's
|
|
303
|
-
* reflective / agentic generators both conform. They are proposers for the
|
|
304
|
-
* SAME loop, not separate loops. The loop body (`runOptimization`) and the
|
|
305
|
-
* gated promotion shell (`runImprovementLoop`) are proposer-agnostic.
|
|
306
|
-
*
|
|
307
|
-
* This is THE optimization proposer — every optimizer is a factory
|
|
308
|
-
* `xProposer(opts): SurfaceProposer` (`evolutionaryProposer`, `aceProposer`,
|
|
309
|
-
* `gepaProposer`, `skillOptProposer`, `traceAnalystProposer`, `haloProposer`,
|
|
310
|
-
* `memoryCurationProposer`, `fapoProposer`), all exported from `/campaign` and
|
|
311
|
-
* drivable by `selfImprove({ proposer })`. Not to be confused with the
|
|
312
|
-
* behavior-fuzzing `MutationProposer` (`fuzz/types`), a scenario generator for
|
|
313
|
-
* a different loop.
|
|
314
|
-
*/
|
|
315
|
-
interface SurfaceProposer<TFindings = unknown> {
|
|
316
|
-
kind: string;
|
|
317
|
-
/** Plan: propose N candidate surfaces for the next generation. A proposer
|
|
318
|
-
* may return bare `MutableSurface`s or `ProposedCandidate`s that carry the
|
|
319
|
-
* `{label, rationale}` motivating the change — the loop threads the
|
|
320
|
-
* rationale into `GenerationCandidate` and the emitted provenance. */
|
|
321
|
-
propose(ctx: ProposeContext<TFindings>): Promise<Array<MutableSurface | ProposedCandidate>>;
|
|
322
|
-
/** Decide: stop early when the proposer judges the search converged or
|
|
323
|
-
* exhausted. Default (omitted) runs all `maxGenerations`. */
|
|
324
|
-
decide?(args: {
|
|
325
|
-
history: GenerationRecord[];
|
|
326
|
-
}): {
|
|
327
|
-
stop: boolean;
|
|
328
|
-
reason?: string;
|
|
329
|
-
};
|
|
330
|
-
}
|
|
331
|
-
/** Optional vocabulary alias. The loop is the optimizer; this object is the
|
|
332
|
-
* proposer inside that loop. */
|
|
333
|
-
type OptimizationProposer<TFindings = unknown> = SurfaceProposer<TFindings>;
|
|
334
|
-
interface OptimizerConfigBase {
|
|
335
|
-
populationSize: number;
|
|
336
|
-
maxGenerations: number;
|
|
337
|
-
surfaceExtractor: (profile: unknown) => MutableSurface;
|
|
338
|
-
}
|
|
339
|
-
interface OptimizerConfig extends OptimizerConfigBase {
|
|
340
|
-
proposer: SurfaceProposer;
|
|
341
|
-
}
|
|
342
|
-
/** Five-valued verdict taxonomy (MOSS-paper alignment). */
|
|
343
|
-
type GateDecision = 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
|
|
344
|
-
interface GateContext<TArtifact, TScenario extends Scenario> {
|
|
345
|
-
candidateArtifacts: Map<string, TArtifact>;
|
|
346
|
-
baselineArtifacts?: Map<string, TArtifact>;
|
|
347
|
-
/** Candidate (winner) judge scores, keyed by cellId. */
|
|
348
|
-
judgeScores: Map<string, Record<string, JudgeScore>>;
|
|
349
|
-
/** Baseline judge scores, keyed by cellId. SEPARATE from `judgeScores` —
|
|
350
|
-
* baseline + candidate share cellIds (same scenarios), so a single map
|
|
351
|
-
* cannot represent both. A gate computing a holdout delta MUST read
|
|
352
|
-
* candidate from `judgeScores` and baseline from here. */
|
|
353
|
-
baselineJudgeScores?: Map<string, Record<string, JudgeScore>>;
|
|
354
|
-
/** Neutralized-arm judge scores, keyed by cellId — the winner surface with its
|
|
355
|
-
* content footprint-matched-blanked (via a `neutralize` fn). Same scenarios as
|
|
356
|
-
* `judgeScores`. Present ONLY when `runImprovementLoop` was given a `neutralize`
|
|
357
|
-
* function. A placebo gate (`neutralizationGate`) compares this arm's lift
|
|
358
|
-
* against the candidate's to reject decorative wins (lift from footprint, not
|
|
359
|
-
* content). Undefined otherwise. */
|
|
360
|
-
neutralizedJudgeScores?: Map<string, Record<string, JudgeScore>>;
|
|
361
|
-
/** Neutralized-arm artifacts, keyed by cellId. Present alongside
|
|
362
|
-
* `neutralizedJudgeScores`. */
|
|
363
|
-
neutralizedArtifacts?: Map<string, TArtifact>;
|
|
364
|
-
scenarios: TScenario[];
|
|
365
|
-
cost: {
|
|
366
|
-
candidate: number;
|
|
367
|
-
baseline: number;
|
|
368
|
-
};
|
|
369
|
-
/** Shared run spend account and receipt attribution phase. */
|
|
370
|
-
costLedger?: CostLedger;
|
|
371
|
-
costPhase?: string;
|
|
372
|
-
signal: AbortSignal;
|
|
373
|
-
}
|
|
374
|
-
interface GateResult {
|
|
375
|
-
decision: GateDecision;
|
|
376
|
-
reasons: string[];
|
|
377
|
-
contributingGates: Array<{
|
|
378
|
-
name: string;
|
|
379
|
-
passed: boolean;
|
|
380
|
-
detail: unknown;
|
|
381
|
-
}>;
|
|
382
|
-
delta?: number;
|
|
383
|
-
}
|
|
384
|
-
/** Composable promotion gate. */
|
|
385
|
-
interface Gate<TArtifact = unknown, TScenario extends Scenario = Scenario> {
|
|
386
|
-
name: string;
|
|
387
|
-
decide(ctx: GateContext<TArtifact, TScenario>): Promise<GateResult>;
|
|
388
|
-
}
|
|
389
|
-
/** Scoped trace writer handed to each dispatch — every span
|
|
390
|
-
* auto-tagged with the cellId so traces filter cleanly. */
|
|
391
|
-
interface CampaignTraceWriter {
|
|
392
|
-
span(name: string, attributes?: Record<string, unknown>): TraceSpan;
|
|
393
|
-
flush(): Promise<void>;
|
|
394
|
-
}
|
|
395
|
-
interface TraceSpan {
|
|
396
|
-
end(attributes?: Record<string, unknown>): void;
|
|
397
|
-
setAttribute(key: string, value: unknown): void;
|
|
398
|
-
}
|
|
399
|
-
/** Scoped artifact writer — `write(path, content)` lands under
|
|
400
|
-
* `<runDir>/<cellId>/<path>`. */
|
|
401
|
-
interface CampaignArtifactWriter {
|
|
402
|
-
write(path: string, content: string | Uint8Array): Promise<string>;
|
|
403
|
-
writeJson(path: string, value: unknown): Promise<string>;
|
|
404
|
-
}
|
|
405
|
-
/** Token usage accumulated for a cell. Aliased to the canonical `RunTokenUsage`
|
|
406
|
-
* (run-record.ts, same package) so a cell maps onto a `RunRecord` for the
|
|
407
|
-
* backend-integrity guard with ONE source of truth — a field added to
|
|
408
|
-
* `RunTokenUsage` is a compile error here, not a silent drift. */
|
|
409
|
-
type CampaignTokenUsage = RunTokenUsage;
|
|
410
|
-
/** Cell-scoped paid-call entry point. The dispatch places every paid operation
|
|
411
|
-
* inside `runPaidCall`; the returned provider result supplies one receipt with
|
|
412
|
-
* cost, tokens, and resolved model. Calls made outside this method are not
|
|
413
|
-
* admitted or captured. */
|
|
414
|
-
interface CampaignCostMeter {
|
|
415
|
-
/** The only paid-call path. Returns a typed result; callers must inspect it. */
|
|
416
|
-
runPaidCall<T>(input: Omit<RunPaidCallInput<T>, 'channel' | 'phase' | 'tags'> & {
|
|
417
|
-
channel?: CostChannel;
|
|
418
|
-
}): Promise<PaidCallResult<T>>;
|
|
419
|
-
}
|
|
420
|
-
/** Source tag — required on every store write. Used by the
|
|
421
|
-
* default training-source filter (production-trace samples NOT used as
|
|
422
|
-
* training scenarios unless explicitly opted in). */
|
|
423
|
-
type LabeledScenarioSource = 'production-trace' | 'eval-run' | 'manual' | 'red-team' | 'synthetic';
|
|
424
|
-
type RedactionStatus = 'raw' | 'redacted-pii' | 'redacted-secrets' | 'fully-redacted';
|
|
425
|
-
/** How much a label can be trusted to evaluate against — the gold-admission
|
|
426
|
-
* gate. Strictly ordered: a record qualifies for a `minTrust` filter when its
|
|
427
|
-
* trust rank is >= the requested rank.
|
|
428
|
-
*
|
|
429
|
-
* - `unverified` — label is a heuristic (e.g. raw outcome success/fail).
|
|
430
|
-
* Fine as corpus; MUST NOT enter a gold set that lift
|
|
431
|
-
* numbers are computed against.
|
|
432
|
-
* - `verified-signal` — an external signal confirmed the outcome (PR merged,
|
|
433
|
-
* tests green, user did not retry, downstream check).
|
|
434
|
-
* - `human-rated` — a human explicitly rated or corrected the artifact.
|
|
435
|
-
*
|
|
436
|
-
* Absent on a write ⇒ treated as `unverified` (fail-closed: a writer must
|
|
437
|
-
* explicitly assert trust to make a record gold-eligible — it never happens
|
|
438
|
-
* by accident). */
|
|
439
|
-
type LabelTrust = 'unverified' | 'verified-signal' | 'human-rated';
|
|
440
|
-
/** Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). */
|
|
441
|
-
declare function labelTrustRank(trust: LabelTrust | undefined): number;
|
|
442
|
-
/** Required-provenance write. The store rejects writes that
|
|
443
|
-
* lack provenance — a default-on flywheel without provenance is the
|
|
444
|
-
* data-poisoning vector flagged in the alignment review. */
|
|
445
|
-
interface LabeledScenarioWrite<TScenario extends Scenario = Scenario, TArtifact = unknown> {
|
|
446
|
-
scenario: TScenario;
|
|
447
|
-
artifact: TArtifact;
|
|
448
|
-
judgeScores: Record<string, JudgeScore>;
|
|
449
|
-
source: LabeledScenarioSource;
|
|
450
|
-
sourceVersionHash: string;
|
|
451
|
-
capturedAt: string;
|
|
452
|
-
redactionStatus: RedactionStatus;
|
|
453
|
-
/** Gold-admission trust tier. Absent ⇒ `unverified` (fail-closed): the
|
|
454
|
-
* record is corpus, never gold. A writer must explicitly assert
|
|
455
|
-
* `verified-signal` or `human-rated` to make it eligible for a gold
|
|
456
|
-
* sample. See {@link LabelTrust}. */
|
|
457
|
-
labelTrust?: LabelTrust;
|
|
458
|
-
/** Optional per-source rate-limit bucket key (e.g., the tenant id). */
|
|
459
|
-
rateLimitBucket?: string;
|
|
460
|
-
}
|
|
461
|
-
interface LabeledScenarioRecord<TScenario extends Scenario = Scenario, TArtifact = unknown> extends LabeledScenarioWrite<TScenario, TArtifact> {
|
|
462
|
-
/** Stable hash of (scenario.id, source, capturedAt, sourceVersionHash). */
|
|
463
|
-
recordHash: string;
|
|
464
|
-
/** Substrate-assigned split — train if captured before the campaign's
|
|
465
|
-
* `temporalCutoff`, test if after. Explicit override allowed via filter. */
|
|
466
|
-
split: 'train' | 'test';
|
|
467
|
-
}
|
|
468
|
-
interface LabeledScenarioSampleArgs {
|
|
469
|
-
count: number;
|
|
470
|
-
/** REQUIRED — substrate refuses to sample without an explicit split. */
|
|
471
|
-
split: 'train' | 'test';
|
|
472
|
-
/** REQUIRED — only records captured before this timestamp are returned.
|
|
473
|
-
* Enforces temporal split discipline (test scenarios captured AFTER train
|
|
474
|
-
* cannot enter the training pool). */
|
|
475
|
-
capturedBefore: string;
|
|
476
|
-
filter?: {
|
|
477
|
-
kind?: string;
|
|
478
|
-
source?: LabeledScenarioSource | LabeledScenarioSource[];
|
|
479
|
-
minComposite?: number;
|
|
480
|
-
maxComposite?: number;
|
|
481
|
-
/** Gold gate: only records whose trust rank is >= this tier are
|
|
482
|
-
* returned. `sample({ split: 'test', minTrust: 'verified-signal' })` is
|
|
483
|
-
* the canonical "give me the gold set" call. Absent ⇒ no trust gate
|
|
484
|
-
* (corpus-level read). */
|
|
485
|
-
minTrust?: LabelTrust;
|
|
486
|
-
};
|
|
487
|
-
}
|
|
488
|
-
interface LabeledScenarioStore {
|
|
489
|
-
observe(write: LabeledScenarioWrite): Promise<void>;
|
|
490
|
-
sample(args: LabeledScenarioSampleArgs): Promise<LabeledScenarioRecord[]>;
|
|
491
|
-
size(): Promise<{
|
|
492
|
-
train: number;
|
|
493
|
-
test: number;
|
|
494
|
-
bySource: Record<string, number>;
|
|
495
|
-
/** Count by trust tier — tells the flywheel how much gold it has
|
|
496
|
-
* accumulated vs. raw corpus. */
|
|
497
|
-
byTrust: Record<LabelTrust, number>;
|
|
498
|
-
}>;
|
|
499
|
-
}
|
|
500
|
-
interface CampaignCellResult<TArtifact> {
|
|
501
|
-
/** Manifest that produced this cell. Resumability refuses to reuse a cell
|
|
502
|
-
* whose manifest differs from the current run. */
|
|
503
|
-
manifestHash?: string;
|
|
504
|
-
cellId: string;
|
|
505
|
-
scenarioId: string;
|
|
506
|
-
rep: number;
|
|
507
|
-
generation?: number;
|
|
508
|
-
artifact: TArtifact;
|
|
509
|
-
judgeScores: Record<string, JudgeScore>;
|
|
510
|
-
costUsd: number;
|
|
511
|
-
/** True when at least one priced receipt used the model table instead of a provider bill. */
|
|
512
|
-
costEstimated?: boolean;
|
|
513
|
-
/** Exact durable receipts required to reuse this cached result. */
|
|
514
|
-
costCallIds?: string[];
|
|
515
|
-
/** Agent-call token usage committed by `ctx.cost.runPaidCall`.
|
|
516
|
-
* `{ input: 0, output: 0 }` when no paid agent call was recorded. */
|
|
517
|
-
tokenUsage: CampaignTokenUsage;
|
|
518
|
-
/** Concrete model from the latest committed agent receipt. Consumed by
|
|
519
|
-
* `buildRunRecord` to pin the model when the declared profile uses a
|
|
520
|
-
* runtime-resolved sentinel. */
|
|
521
|
-
resolvedModel?: string;
|
|
522
|
-
durationMs: number;
|
|
523
|
-
seed: number;
|
|
524
|
-
cached: boolean;
|
|
525
|
-
error?: string;
|
|
526
|
-
}
|
|
527
|
-
interface JudgeAggregate {
|
|
528
|
-
mean: number;
|
|
529
|
-
stdev: number;
|
|
530
|
-
ci95: [number, number];
|
|
531
|
-
n: number;
|
|
532
|
-
}
|
|
533
|
-
interface ScenarioAggregate {
|
|
534
|
-
meanComposite: number;
|
|
535
|
-
ci95: [number, number];
|
|
536
|
-
n: number;
|
|
537
|
-
}
|
|
538
|
-
interface GenerationRecord {
|
|
539
|
-
generationIndex: number;
|
|
540
|
-
candidates: GenerationCandidate[];
|
|
541
|
-
promoted: string[];
|
|
542
|
-
}
|
|
543
|
-
/** One scored candidate surface in a generation. `dimensions` + `scenarios`
|
|
544
|
-
* let a reflective proposer ground its next proposal on WHICH
|
|
545
|
-
* dimensions the candidate is weakest on and WHICH scenarios it best/worst
|
|
546
|
-
* handled — the evidence a blind `Mutator` cannot see. */
|
|
547
|
-
interface GenerationCandidate {
|
|
548
|
-
surfaceHash: string;
|
|
549
|
-
composite: number;
|
|
550
|
-
ci95: [number, number];
|
|
551
|
-
/** Exact surface this candidate mutated. */
|
|
552
|
-
parentSurfaceHash?: string;
|
|
553
|
-
/** Measured search-split composite of the exact parent surface. */
|
|
554
|
-
parentComposite?: number;
|
|
555
|
-
/** Candidate composite minus its parent's composite. Present only when the
|
|
556
|
-
* candidate completed the designed denominator. */
|
|
557
|
-
observedDeltaFromParent?: number;
|
|
558
|
-
/** Whether this candidate had a scorable result for every designed campaign
|
|
559
|
-
* cell and was therefore eligible for ranking, promotion, and Pareto
|
|
560
|
-
* selection. Older externally-authored records may omit this field; loop
|
|
561
|
-
* records always populate it. */
|
|
562
|
-
eligibleForPromotion?: boolean;
|
|
563
|
-
/** Exact denominator receipt for selection eligibility. Scores stay
|
|
564
|
-
* descriptive: an incomplete candidate is retained with its observed score
|
|
565
|
-
* and errors instead of receiving an invented penalty. */
|
|
566
|
-
coverage?: {
|
|
567
|
-
expectedCells: number;
|
|
568
|
-
scorableCells: number;
|
|
569
|
-
unscorableCells: Array<{
|
|
570
|
-
cellId: string;
|
|
571
|
-
reason: string;
|
|
572
|
-
}>;
|
|
573
|
-
};
|
|
574
|
-
/** Mean score per judge dimension across all cells (scenarios × reps ×
|
|
575
|
-
* judges that reported the dimension). */
|
|
576
|
-
dimensions: Record<string, number>;
|
|
577
|
-
/** Per-scenario composite (mean over reps + judges), plus the judge's
|
|
578
|
-
* free-form `notes` for that scenario — the "why it scored low" evidence a
|
|
579
|
-
* reflective proposer grounds its next edit on. Keep `notes` GENERALIZABLE
|
|
580
|
-
* (which checks/lines/dimensions failed and how), NOT case-specific ground
|
|
581
|
-
* truth: leaking expected answers into the prompt is memorization, and the
|
|
582
|
-
* held-out gate would reject it anyway. */
|
|
583
|
-
scenarios: Array<{
|
|
584
|
-
scenarioId: string;
|
|
585
|
-
composite: number;
|
|
586
|
-
notes?: string;
|
|
587
|
-
}>;
|
|
588
|
-
/** Proposer-supplied short label for the change. Present when the proposer
|
|
589
|
-
* returned a `ProposedCandidate`; absent for bare-surface mutators. */
|
|
590
|
-
label?: string;
|
|
591
|
-
/** Proposer-supplied rationale — WHY this candidate was proposed. The
|
|
592
|
-
* "because rationale Z" the audit requires to survive to the result.
|
|
593
|
-
* Present when the proposer returned a `ProposedCandidate`. */
|
|
594
|
-
rationale?: string;
|
|
595
|
-
/** Exact structured cause threaded from the proposer, when available. */
|
|
596
|
-
candidateRecord?: PolicyEditCandidateRecord;
|
|
597
|
-
}
|
|
598
|
-
interface CampaignAggregates {
|
|
599
|
-
byJudge: Record<string, JudgeAggregate>;
|
|
600
|
-
byScenario: Record<string, ScenarioAggregate>;
|
|
601
|
-
/** Canonical campaign accounting, including worker and judge calls. */
|
|
602
|
-
cost: CostLedgerSummary;
|
|
603
|
-
/** Compatibility alias of `cost.totalCostUsd`. */
|
|
604
|
-
totalCostUsd: number;
|
|
605
|
-
cellsExecuted: number;
|
|
606
|
-
cellsSkipped: number;
|
|
607
|
-
cellsCached: number;
|
|
608
|
-
cellsFailed: number;
|
|
609
|
-
}
|
|
610
|
-
interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scenario> {
|
|
611
|
-
/** sha256(scenarios, judges, dispatch source ref, optimizer config, seed). Stable identity for reruns. */
|
|
612
|
-
manifestHash: string;
|
|
613
|
-
seed: number;
|
|
614
|
-
startedAt: string;
|
|
615
|
-
endedAt: string;
|
|
616
|
-
durationMs: number;
|
|
617
|
-
cells: Array<CampaignCellResult<TArtifact>>;
|
|
618
|
-
aggregates: CampaignAggregates;
|
|
619
|
-
optimization?: {
|
|
620
|
-
generations: GenerationRecord[];
|
|
621
|
-
winnerSurfaceHash?: string;
|
|
622
|
-
};
|
|
623
|
-
gate?: GateResult;
|
|
624
|
-
prUrl?: string;
|
|
625
|
-
runDir: string;
|
|
626
|
-
artifactsByPath: Record<string, string>;
|
|
627
|
-
/** Substrate strips the input scenarios to id+kind for the result manifest;
|
|
628
|
-
* consumers needing full payload look it up via the original input. The
|
|
629
|
-
* type parameter `TScenario` is propagated for downstream consumers that
|
|
630
|
-
* want narrowed types when extending `CampaignResult`. */
|
|
631
|
-
scenarios: Array<Pick<TScenario, 'id' | 'kind'>>;
|
|
632
|
-
}
|
|
633
|
-
|
|
634
|
-
export { type JudgeAggregate as A, type ScenarioAggregate as B, type CampaignResult as C, type DispatchContext as D, type ScoredSurfaceOutcome as E, isProposedCandidate as F, type Gate as G, labelTrustRank as H, type JudgeScore as J, type LabeledScenarioStore as L, type MutableSurface as M, type OptimizationProposer as O, type ParetoParent as P, type RedactionStatus as R, type Scenario as S, type TraceSpan as T, type JudgeDimension as a, type JudgeConfig as b, type SurfaceProposer as c, type GateDecision as d, type CampaignAggregates as e, type CampaignArtifactWriter as f, type CampaignCellResult as g, type CampaignCostMeter as h, type CampaignTraceWriter as i, type CodeSurface as j, type DispatchFn as k, type GateContext as l, type GateResult as m, type GenerationCandidate as n, type GenerationRecord as o, type Mutator as p, type OptimizerConfig as q, type SessionScript as r, type LabeledScenarioWrite as s, type LabeledScenarioSampleArgs as t, type LabeledScenarioRecord as u, type LabelTrust as v, type ProposedCandidate as w, type ProposeContext as x, type LabeledScenarioSource as y, type CampaignTokenUsage as z };
|