@tangle-network/agent-eval 0.117.0 → 0.118.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (143) hide show
  1. package/CHANGELOG.md +30 -0
  2. package/dist/analyst/index.d.ts +2772 -21
  3. package/dist/analyst/index.js +7 -6
  4. package/dist/analyst/index.js.map +1 -1
  5. package/dist/belief-state/index.d.ts +706 -10
  6. package/dist/belief-state/index.js +2 -1
  7. package/dist/belief-state/index.js.map +1 -1
  8. package/dist/benchmarks/index.d.ts +958 -14
  9. package/dist/benchmarks/index.js +12 -10
  10. package/dist/builder-eval/index.d.ts +449 -4
  11. package/dist/builder-eval/index.js +4 -3
  12. package/dist/builder-eval/index.js.map +1 -1
  13. package/dist/campaign/index.d.ts +4276 -74
  14. package/dist/campaign/index.js +12 -10
  15. package/dist/{chunk-VF3XSYTI.js → chunk-33JA4TFA.js} +6 -6
  16. package/dist/{chunk-4JLWXDYA.js → chunk-3EHHMC6E.js} +2 -2
  17. package/dist/{chunk-CCZIVI3F.js → chunk-BFW56GTT.js} +2 -2
  18. package/dist/{chunk-YZPO4UHR.js → chunk-DP3BBMYJ.js} +100 -143
  19. package/dist/chunk-DP3BBMYJ.js.map +1 -0
  20. package/dist/{chunk-E4BUPP7Z.js → chunk-HKUCJ437.js} +38 -66
  21. package/dist/chunk-HKUCJ437.js.map +1 -0
  22. package/dist/{chunk-HZHNRYHK.js → chunk-K6N6XJJX.js} +2 -2
  23. package/dist/chunk-KSDQVPLR.js +286 -0
  24. package/dist/chunk-KSDQVPLR.js.map +1 -0
  25. package/dist/chunk-MA6HLL3S.js +65 -0
  26. package/dist/chunk-MA6HLL3S.js.map +1 -0
  27. package/dist/{chunk-MGEHEHSN.js → chunk-OIIMMLRB.js} +11 -11
  28. package/dist/{chunk-DXZRATT5.js → chunk-OYZAPX5G.js} +3 -3
  29. package/dist/chunk-PXE2VKMX.js +140 -0
  30. package/dist/chunk-PXE2VKMX.js.map +1 -0
  31. package/dist/{chunk-ZUXV7UWZ.js → chunk-Q442S5AS.js} +73 -19
  32. package/dist/{chunk-ZUXV7UWZ.js.map → chunk-Q442S5AS.js.map} +1 -1
  33. package/dist/{chunk-ODVOOEWQ.js → chunk-QBRSJK47.js} +2 -2
  34. package/dist/{chunk-S2F4J57L.js → chunk-QKEGNI5B.js} +77 -32
  35. package/dist/chunk-QKEGNI5B.js.map +1 -0
  36. package/dist/{chunk-5UF54T55.js → chunk-S3UZOQ5Y.js} +34 -7
  37. package/dist/chunk-S3UZOQ5Y.js.map +1 -0
  38. package/dist/{chunk-FQNLDL4D.js → chunk-SVH2ANFD.js} +136 -4
  39. package/dist/chunk-SVH2ANFD.js.map +1 -0
  40. package/dist/{chunk-GQCZRZ7L.js → chunk-U5CHZ5M3.js} +9 -9
  41. package/dist/{chunk-TVVP3ZZQ.js → chunk-VQMK5FMP.js} +2 -1
  42. package/dist/{chunk-TVVP3ZZQ.js.map → chunk-VQMK5FMP.js.map} +1 -1
  43. package/dist/chunk-WDHBCA3M.js +31 -0
  44. package/dist/chunk-WDHBCA3M.js.map +1 -0
  45. package/dist/{chunk-HQPHZGL6.js → chunk-YLMUS4MM.js} +9 -9
  46. package/dist/{chunk-LQUTGLOZ.js → chunk-ZET2UAYW.js} +15 -65
  47. package/dist/chunk-ZET2UAYW.js.map +1 -0
  48. package/dist/contract/index.d.ts +4012 -38
  49. package/dist/contract/index.js +56 -29
  50. package/dist/contract/index.js.map +1 -1
  51. package/dist/control.d.ts +1013 -9
  52. package/dist/control.js +4 -3
  53. package/dist/fuzz.d.ts +194 -4
  54. package/dist/fuzz.js +3 -3
  55. package/dist/hosted/index.d.ts +498 -17
  56. package/dist/index.d.ts +10982 -1286
  57. package/dist/index.js +97 -80
  58. package/dist/index.js.map +1 -1
  59. package/dist/matrix/index.d.ts +139 -4
  60. package/dist/meta-eval/index.d.ts +862 -15
  61. package/dist/meta-eval/index.js +2 -1
  62. package/dist/meta-eval/index.js.map +1 -1
  63. package/dist/multishot/index.d.ts +214 -14
  64. package/dist/openapi.json +1 -1
  65. package/dist/pipelines/index.d.ts +392 -7
  66. package/dist/pipelines/index.js +5 -3
  67. package/dist/pipelines/index.js.map +1 -1
  68. package/dist/reporting.d.ts +1277 -17
  69. package/dist/rl.d.ts +2359 -28
  70. package/dist/rl.js +6 -5
  71. package/dist/rl.js.map +1 -1
  72. package/dist/storyboard/index.d.ts +86 -1
  73. package/dist/trace-attributes.d.ts +16 -0
  74. package/dist/trace-attributes.js +32 -0
  75. package/dist/trace-attributes.js.map +1 -0
  76. package/dist/traces.d.ts +1970 -697
  77. package/dist/traces.js +49 -32
  78. package/dist/wire/index.d.ts +655 -9
  79. package/docs/insight-report.md +44 -0
  80. package/package.json +9 -3
  81. package/dist/adversarial-B7loGVVX.d.ts +0 -19
  82. package/dist/analyst-C8HHvfJp.d.ts +0 -88
  83. package/dist/analyze-runs--2x39HZ7.d.ts +0 -81
  84. package/dist/baseline-DKq3gJpP.d.ts +0 -141
  85. package/dist/calibration-C8MTS7cw.d.ts +0 -101
  86. package/dist/chunk-5UF54T55.js.map +0 -1
  87. package/dist/chunk-E4BUPP7Z.js.map +0 -1
  88. package/dist/chunk-FQNLDL4D.js.map +0 -1
  89. package/dist/chunk-LQUTGLOZ.js.map +0 -1
  90. package/dist/chunk-S2F4J57L.js.map +0 -1
  91. package/dist/chunk-YZPO4UHR.js.map +0 -1
  92. package/dist/code-agent-session-CjZsVd19.d.ts +0 -87
  93. package/dist/control-6vuGfmDH.d.ts +0 -258
  94. package/dist/cost-ledger-DWy3XdJc.d.ts +0 -183
  95. package/dist/dataset-NENEzRgk.d.ts +0 -115
  96. package/dist/default-registry-DaK8b3fv.d.ts +0 -155
  97. package/dist/emitter-CjD7vUwv.d.ts +0 -122
  98. package/dist/errors-oeQrLqXC.d.ts +0 -74
  99. package/dist/failure-cluster-DOAcSJ87.d.ts +0 -76
  100. package/dist/feedback-trajectory-BUnM58xL.d.ts +0 -348
  101. package/dist/gepa-eESocoDi.d.ts +0 -642
  102. package/dist/index-PdX4VnPA.d.ts +0 -423
  103. package/dist/insight-report-DY4nDW9Q.d.ts +0 -310
  104. package/dist/integrity-DqlBiLyK.d.ts +0 -81
  105. package/dist/judge-calibration-7C-IDmKr.d.ts +0 -145
  106. package/dist/kind-factory-ClZmO25A.d.ts +0 -171
  107. package/dist/llm-client-qoDd18Qz.d.ts +0 -289
  108. package/dist/multi-layer-verifier-BsqKuLyN.d.ts +0 -150
  109. package/dist/off-policy-DiwuKKg7.d.ts +0 -132
  110. package/dist/outcome-store-rnXLEqSn.d.ts +0 -63
  111. package/dist/policy-edit-wG9uFEFm.d.ts +0 -455
  112. package/dist/pre-registration-BWQhJ3vz.d.ts +0 -761
  113. package/dist/provenance-DpjwyseI.d.ts +0 -541
  114. package/dist/query-CF7PG61p.d.ts +0 -35
  115. package/dist/raw-provider-sink-C46HDghv.d.ts +0 -132
  116. package/dist/release-report-C8G2i5Xi.d.ts +0 -236
  117. package/dist/researcher-C8XyxQsu.d.ts +0 -387
  118. package/dist/rubric-predictive-validity-p49lLVrE.d.ts +0 -105
  119. package/dist/run-record-BDH49H2E.d.ts +0 -360
  120. package/dist/runtime-trajectory-DGBIUt4B.d.ts +0 -49
  121. package/dist/schema-B3Q3l9Z_.d.ts +0 -201
  122. package/dist/semantic-concept-judge-CXnPEJbf.d.ts +0 -723
  123. package/dist/sequential-5iSVfzl2.d.ts +0 -139
  124. package/dist/series-convergence-D5OWMBg6.d.ts +0 -33
  125. package/dist/statistics-KUnG73jH.d.ts +0 -494
  126. package/dist/storage-DrX3v_5B.d.ts +0 -50
  127. package/dist/store-C1YxJDEK.d.ts +0 -248
  128. package/dist/store-DGqD0Pyo.d.ts +0 -116
  129. package/dist/summary-report-C5bKFfm-.d.ts +0 -445
  130. package/dist/test-graded-scenario-B0ybnPY7.d.ts +0 -166
  131. package/dist/types-BSw1rOUB.d.ts +0 -634
  132. package/dist/types-BUxNaJ8c.d.ts +0 -108
  133. package/dist/types-BkfcQnxV.d.ts +0 -313
  134. package/dist/verdict-C9MlYujm.d.ts +0 -35
  135. /package/dist/{chunk-VF3XSYTI.js.map → chunk-33JA4TFA.js.map} +0 -0
  136. /package/dist/{chunk-4JLWXDYA.js.map → chunk-3EHHMC6E.js.map} +0 -0
  137. /package/dist/{chunk-CCZIVI3F.js.map → chunk-BFW56GTT.js.map} +0 -0
  138. /package/dist/{chunk-HZHNRYHK.js.map → chunk-K6N6XJJX.js.map} +0 -0
  139. /package/dist/{chunk-MGEHEHSN.js.map → chunk-OIIMMLRB.js.map} +0 -0
  140. /package/dist/{chunk-DXZRATT5.js.map → chunk-OYZAPX5G.js.map} +0 -0
  141. /package/dist/{chunk-ODVOOEWQ.js.map → chunk-QBRSJK47.js.map} +0 -0
  142. /package/dist/{chunk-GQCZRZ7L.js.map → chunk-U5CHZ5M3.js.map} +0 -0
  143. /package/dist/{chunk-HQPHZGL6.js.map → chunk-YLMUS4MM.js.map} +0 -0
@@ -1,634 +0,0 @@
1
- import { P as PolicyEditCandidateRecord } from './policy-edit-wG9uFEFm.js';
2
- import { R as RunPaidCallInput, a as CostChannel, P as PaidCallResult, C as CostLedger, b as CostLedgerSummary } from './cost-ledger-DWy3XdJc.js';
3
- import { L as LlmCallMetadata } from './llm-client-qoDd18Qz.js';
4
- import { c as RunTokenUsage } from './run-record-BDH49H2E.js';
5
-
6
- /**
7
- * Pass A substrate types — `runCampaign` is the one primitive every
8
- * eval flow composes from. Three contracts in this file:
9
- *
10
- * - `Scenario` input set
11
- * - `DispatchFn` how to run one scenario → artifact
12
- * - `CampaignResult` defined output schema (the contract downstream tools depend on)
13
- *
14
- * Three more lifted from earlier substrate work (re-exported):
15
- *
16
- * - `JudgeConfig` pluggable dimensional scorer (0.38)
17
- * - `Mutator` optimization-loop surface mutator
18
- * - `Gate` promotion gate (`HeldOutGate` and friends adapt to this)
19
- *
20
- * No new architecture vs 0.38 — Pass A formalizes the shapes so consumers
21
- * can build dashboards / CI gates / regression diffs against a stable schema.
22
- */
23
-
24
- /** Stable identifier + kind tag for any scenario. Consumers
25
- * extend with their per-domain payload (persona, task, requirement, ...). */
26
- interface Scenario {
27
- id: string;
28
- kind: string;
29
- tags?: string[];
30
- }
31
- /** Context handed to every dispatch invocation. Scoped — every
32
- * trace/span carries the cellId, every artifact write lands under the cell's
33
- * artifact root, the cost meter accumulates per cell. */
34
- interface DispatchContext {
35
- cellId: string;
36
- rep: number;
37
- generation?: number;
38
- seed: number;
39
- signal: AbortSignal;
40
- trace: CampaignTraceWriter;
41
- artifacts: CampaignArtifactWriter;
42
- cost: CampaignCostMeter;
43
- /** Populated when this run is part of a multi-cycle improvement loop. */
44
- cycleId?: string;
45
- /** Populated when the substrate resumed from a prior cache hit. */
46
- resumedFrom?: string;
47
- /**
48
- * Opaque placement key supplied by `RunCampaignOptions.cellPlacement`.
49
- * The substrate forwards it through unchanged; placement-aware Dispatch
50
- * implementations (e.g. `httpDispatch` from `/adapters/http`) read it to
51
- * route the cell to the right worker / region / sandbox. `undefined`
52
- * when no placement strategy is configured.
53
- */
54
- placement?: string;
55
- }
56
- /** One function: scenario + ctx → artifact. Dispatcher chooses
57
- * whether to call `runMultishot`, `runLoop`, raw `streamPrompt`, anything. */
58
- type DispatchFn<TScenario extends Scenario, TArtifact> = (scenario: TScenario, ctx: DispatchContext) => Promise<TArtifact>;
59
- /** One session within a multi-session journey. Dispatch is
60
- * invoked once per session in order; state from prior session's artifact
61
- * is exposed via `ctx.priorSessionArtifact`. */
62
- interface SessionScript<TScenario, TArtifact> {
63
- id: string;
64
- intent: string;
65
- maxTurns?: number;
66
- /** When true, knowledge accumulated this session persists to next. */
67
- affectsKnowledge?: boolean;
68
- /** Optional per-session persona evolution — called after the session
69
- * resolves. Returns the persona shape used by the NEXT session. */
70
- evolveAfterSession?: (artifact: TArtifact, sessionIndex: number, scenario: TScenario) => TScenario;
71
- }
72
- interface JudgeDimension {
73
- /** JSON field name + score key. */
74
- key: string;
75
- /** Description shown in the judge's user prompt. */
76
- description: string;
77
- }
78
- /** Pluggable dimensional scorer. `score` is the contract:
79
- * given an artifact + scenario, return a `JudgeScore`. This is deliberately a
80
- * function, not a fixed LLM-prompt shape — real consumers judge with
81
- * ensembles, deterministic checks, or a single LLM call, and the substrate
82
- * must not constrain that. The `llmJudge()` helper builds a `score` that does
83
- * one LLM call for the common case. `appliesTo` lets a judge run only on
84
- * scenarios that match (e.g. a legal-citation judge only on legal scenarios). */
85
- interface JudgeConfig<TArtifact, TScenario extends Scenario = Scenario> {
86
- name: string;
87
- dimensions: JudgeDimension[];
88
- /** Stable scoring revision used by campaign resume and verdict caches.
89
- * Built-in judges derive this from their prompt, model, and rubric. Custom
90
- * judges should set it when closure state can change without changing code. */
91
- judgeVersion?: string;
92
- /** Score one artifact. Throw on failure — a thrown judge is recorded as a
93
- * failed cell, never silently folded into a zero. */
94
- score(input: {
95
- artifact: TArtifact;
96
- scenario: TScenario;
97
- signal: AbortSignal;
98
- /** Shared run spend account and receipt attribution phase. */
99
- costLedger?: CostLedger;
100
- costPhase?: string;
101
- costTags?: Record<string, string>;
102
- }): JudgeScore | Promise<JudgeScore>;
103
- appliesTo?: (scenario: TScenario) => boolean;
104
- }
105
- /** The canonical judge verdict shape — one declaration, shared by campaign
106
- * judges and the multishot judge runner (which re-exports this type).
107
- *
108
- * Scale is PRODUCER-DEFINED: campaign convention is [0,1]; the legacy
109
- * multishot runner emits 0-10. Cross-scale comparison must go through
110
- * `detectScale` (src/campaign/gates/statistical-heldout.ts, used by
111
- * promotion-policy) — never renormalize a producer's values in place, as
112
- * downstream thresholds (`composite >= 5` in multishot/matrix.ts, live-soak
113
- * `>= 7` gates) key on the producer's native scale. */
114
- interface JudgeScore {
115
- dimensions: Record<string, number>;
116
- composite: number;
117
- notes: string;
118
- /** Provider metadata for display and diagnostics; accounting uses CostLedger receipts. */
119
- llmCall?: LlmCallMetadata;
120
- /** Set when the judge itself failed (call error, unparseable output).
121
- * `composite`/`dimensions` carry no signal — aggregators MUST exclude
122
- * failed scores from means instead of folding them into zeros. */
123
- failed?: true;
124
- /** Ensemble extras (populated by `ensembleJudge`): max per-dimension
125
- * spread across surviving judges — the inter-rater signal. */
126
- maxDisagreement?: number;
127
- /** Ensemble extras: judge identities whose verdict failed. */
128
- failedJudges?: string[];
129
- /** Ensemble extras: each surviving judge's per-dimension scores. */
130
- perJudge?: Record<string, Record<string, number>>;
131
- }
132
- /** A tier-4 code surface — a finalized candidate change to the agent's
133
- * IMPLEMENTATION, not its prompt. Produced by autoresearch (reads codebase +
134
- * trace findings → opens a worktree). `worktreeRef` locates the candidate;
135
- * the exact commits, tree, and binary-patch digest identify it. See the
136
- * improvement-tier table in `docs/design/loop-taxonomy.md`. */
137
- interface CodeSurface {
138
- readonly kind: 'code';
139
- /** Worktree path or git ref holding the candidate code change. This is a
140
- * mutable locator and is deliberately excluded from content hashes. */
141
- readonly worktreeRef: string;
142
- /** Human-readable ref the worktree was forked from. Not identity-bearing. */
143
- readonly baseRef: string;
144
- /** Exact commit the candidate was forked from. */
145
- readonly baseCommit: string;
146
- /** Exact tree object for `baseCommit`. */
147
- readonly baseTree: string;
148
- /** Exact finalized candidate commit. */
149
- readonly candidateCommit: string;
150
- /** Exact tree object for `candidateCommit`. */
151
- readonly candidateTree: string;
152
- /** Identity of the exact patch artifact. The deployable candidate bundle
153
- * carries the same descriptor plus its base64-encoded content. */
154
- readonly patch: {
155
- readonly format: 'git-diff-binary';
156
- readonly sha256: `sha256:${string}`;
157
- readonly byteLength: number;
158
- };
159
- /** Human summary of what changed — rendered into the auto-PR body. */
160
- readonly summary?: string;
161
- }
162
- /** The mutable surface a proposer changes. Tiers (see
163
- * `docs/design/loop-taxonomy.md`):
164
- * - `string` — tiers 1-2: system-prompt addendum / serialized tool
165
- * config. Cheap, reversible, text-diffable.
166
- * - `CodeSurface` — tier 4: an implementation change behind a worktree ref.
167
- * Tier 3 (knowledge) is owned by agent-knowledge and rides its own adapter,
168
- * not this type. */
169
- type MutableSurface = string | CodeSurface;
170
- /** A proposer output carrying the surface AND the WHY behind
171
- * it. Reflective proposers (`gepaProposer`) parse a `{label, rationale, payload}`
172
- * from the model; without this wrapper the loop keeps only `payload` and the
173
- * rationale that motivated the change is lost — the candidate becomes
174
- * unattributable. `propose()` may return either bare `MutableSurface`s (cheap
175
- * blind mutators) or these (reflective proposers); the loop normalizes both. */
176
- interface ProposedCandidate {
177
- surface: MutableSurface;
178
- /** Short human label for the change (≤ 40 chars typical). */
179
- label: string;
180
- /** Why this change was proposed — which failure it targets, which
181
- * primitive it used. Survives to `GenerationCandidate.rationale` and the
182
- * emitted provenance record. */
183
- rationale: string;
184
- /** Structured, JSON-safe cause for this exact candidate when the proposer
185
- * can provide one. Policy edits retain the full validated edit here. */
186
- candidateRecord?: PolicyEditCandidateRecord;
187
- }
188
- /** Type guard: a proposal carrying its rationale vs a bare
189
- * surface. The loop branches on this to populate `GenerationCandidate`. */
190
- declare function isProposedCandidate(value: MutableSurface | ProposedCandidate): value is ProposedCandidate;
191
- /** A non-dominated parent on the GEPA Pareto frontier — a
192
- * surface that, across the per-scenario objective vectors, no other tried
193
- * surface beats on every scenario. A candidate worse on the mean composite
194
- * but uniquely best on one hard scenario is non-dominated and survives here;
195
- * the composite-best ranking would discard the lesson it carries. The loop
196
- * computes the frontier across ALL generations and hands it to the proposer so
197
- * a reflective proposer can combine complementary lessons (GEPA, Agrawal et
198
- * al., arXiv:2507.19457). See `pareto.ts` (`paretoFrontier`). */
199
- interface ParetoParent {
200
- surface: MutableSurface;
201
- surfaceHash: string;
202
- /** The objective vector: per-scenario composite (higher is better). The
203
- * axes the frontier is computed over. */
204
- objectives: Record<string, number>;
205
- /** Mean composite across the objective scenarios — the scalar summary used
206
- * for ordering + display, NOT for dominance. */
207
- composite: number;
208
- /** Generation that produced this surface (`-1` for the baseline). */
209
- generation: number;
210
- label?: string;
211
- rationale?: string;
212
- }
213
- /** Exact measured state for the surface an optimizer is learning from.
214
- * Unlike a model-authored expected gain, every value here comes from a
215
- * completed campaign over the designed denominator. */
216
- interface ScoredSurfaceOutcome {
217
- /** Optimization/search evidence only. Held-out results must never flow back
218
- * into a proposer through this type. */
219
- split: 'search';
220
- /** Generation that actually measured this surface (`-1` for the baseline). */
221
- generation: number;
222
- surfaceHash: string;
223
- composite: number;
224
- dimensions: Record<string, number>;
225
- scenarios: Array<{
226
- scenarioId: string;
227
- composite: number;
228
- notes?: string;
229
- }>;
230
- coverage: {
231
- expectedCells: number;
232
- scorableCells: number;
233
- };
234
- }
235
- /** Stateless surface mutation — given findings + current
236
- * surface, return N candidate surfaces. Pure transform, no generation
237
- * awareness. Reflective-mutation and `AxGEPA` mutators conform. Wrapped by
238
- * `evolutionaryProposer` to become a `SurfaceProposer`. */
239
- interface Mutator<TFindings = unknown> {
240
- kind: string;
241
- mutate(args: {
242
- findings: TFindings[];
243
- currentSurface: MutableSurface;
244
- populationSize: number;
245
- signal: AbortSignal;
246
- }): Promise<Array<MutableSurface | ProposedCandidate>>;
247
- }
248
- /** Everything a proposer may read to plan the next
249
- * batch of candidates. The first six fields are always present; the rest are
250
- * optional context the loop supplies when available, so cheap proposers
251
- * (`evolutionaryProposer`) can ignore them while a code-tier agentic generator
252
- * consumes the report + dataset to drive a coding harness.
253
- * See `docs/campaign-proposers.md`. */
254
- interface ProposeContext<TFindings = unknown> {
255
- currentSurface: MutableSurface;
256
- history: GenerationRecord[];
257
- findings: TFindings[];
258
- /** BREADTH: how many candidate surfaces to return this generation. */
259
- populationSize: number;
260
- generation: number;
261
- signal: AbortSignal;
262
- /** Measured baseline for this optimization run. `runOptimization` always
263
- * supplies it; optional for standalone proposer callers. */
264
- baselineOutcome?: ScoredSurfaceOutcome;
265
- /** Measured result for `currentSurface`, the complete global incumbent every
266
- * new candidate mutates. `runOptimization` always supplies it. */
267
- incumbentOutcome?: ScoredSurfaceOutcome;
268
- /** Optional analysis report produced before proposal. Opaque to the substrate:
269
- * the proposer that consumes it owns the shape. */
270
- report?: unknown;
271
- /** Handle to all captured data — the proposer samples traces / artifacts /
272
- * rewards here to ground its proposals. */
273
- dataset?: LabeledScenarioStore;
274
- /** DEPTH: max iterations the agentic generator may take per candidate.
275
- * 1 = single-shot; >1 = it may iterate on its own change before handing it
276
- * back to be measured. */
277
- maxImprovementShots?: number;
278
- /** GEPA Pareto frontier across ALL generations so far — the non-dominated
279
- * surfaces by per-scenario objective vector. Empty/absent on generation 0
280
- * (only the baseline is scored). A reflective proposer combines the
281
- * complementary lessons of these parents (each excels on different
282
- * scenarios) into a merged candidate. Proposers doing pure single-parent
283
- * reflection may ignore it. See {@link ParetoParent}. */
284
- paretoParents?: ParetoParent[];
285
- /** Shared run spend account and receipt attribution phase. */
286
- costLedger?: CostLedger;
287
- costPhase?: string;
288
- /** FIREWALL (non-negotiable): the held-out judge is write-only — its verdicts
289
- * score the chosen output and gate promotion, and are NEVER an input to
290
- * proposal/steering (else the optimizer games the acceptance axis = an
291
- * oracle). This `never`-typed field makes that a compile-time tripwire: a
292
- * proposer that tries to thread judge verdicts into the proposal will not type.
293
- * Steering may consume TRACE-OBSERVABLE signals (what the agent did) via
294
- * `findings`/`report`; it may NOT consume the judge's held-out verdict. */
295
- judgeScores?: never;
296
- }
297
- /** A surface-improvement strategy. Given the current best
298
- * surface, the history of what's been tried + scored, and any external
299
- * findings, propose the next batch of candidate surfaces to measure.
300
- * Optionally decide to stop early.
301
- *
302
- * The evolutionary mutator (`evolutionaryProposer`, here) and agent-runtime's
303
- * reflective / agentic generators both conform. They are proposers for the
304
- * SAME loop, not separate loops. The loop body (`runOptimization`) and the
305
- * gated promotion shell (`runImprovementLoop`) are proposer-agnostic.
306
- *
307
- * This is THE optimization proposer — every optimizer is a factory
308
- * `xProposer(opts): SurfaceProposer` (`evolutionaryProposer`, `aceProposer`,
309
- * `gepaProposer`, `skillOptProposer`, `traceAnalystProposer`, `haloProposer`,
310
- * `memoryCurationProposer`, `fapoProposer`), all exported from `/campaign` and
311
- * drivable by `selfImprove({ proposer })`. Not to be confused with the
312
- * behavior-fuzzing `MutationProposer` (`fuzz/types`), a scenario generator for
313
- * a different loop.
314
- */
315
- interface SurfaceProposer<TFindings = unknown> {
316
- kind: string;
317
- /** Plan: propose N candidate surfaces for the next generation. A proposer
318
- * may return bare `MutableSurface`s or `ProposedCandidate`s that carry the
319
- * `{label, rationale}` motivating the change — the loop threads the
320
- * rationale into `GenerationCandidate` and the emitted provenance. */
321
- propose(ctx: ProposeContext<TFindings>): Promise<Array<MutableSurface | ProposedCandidate>>;
322
- /** Decide: stop early when the proposer judges the search converged or
323
- * exhausted. Default (omitted) runs all `maxGenerations`. */
324
- decide?(args: {
325
- history: GenerationRecord[];
326
- }): {
327
- stop: boolean;
328
- reason?: string;
329
- };
330
- }
331
- /** Optional vocabulary alias. The loop is the optimizer; this object is the
332
- * proposer inside that loop. */
333
- type OptimizationProposer<TFindings = unknown> = SurfaceProposer<TFindings>;
334
- interface OptimizerConfigBase {
335
- populationSize: number;
336
- maxGenerations: number;
337
- surfaceExtractor: (profile: unknown) => MutableSurface;
338
- }
339
- interface OptimizerConfig extends OptimizerConfigBase {
340
- proposer: SurfaceProposer;
341
- }
342
- /** Five-valued verdict taxonomy (MOSS-paper alignment). */
343
- type GateDecision = 'ship' | 'hold' | 'need_more_work' | 'model_ceiling' | 'arch_ceiling';
344
- interface GateContext<TArtifact, TScenario extends Scenario> {
345
- candidateArtifacts: Map<string, TArtifact>;
346
- baselineArtifacts?: Map<string, TArtifact>;
347
- /** Candidate (winner) judge scores, keyed by cellId. */
348
- judgeScores: Map<string, Record<string, JudgeScore>>;
349
- /** Baseline judge scores, keyed by cellId. SEPARATE from `judgeScores` —
350
- * baseline + candidate share cellIds (same scenarios), so a single map
351
- * cannot represent both. A gate computing a holdout delta MUST read
352
- * candidate from `judgeScores` and baseline from here. */
353
- baselineJudgeScores?: Map<string, Record<string, JudgeScore>>;
354
- /** Neutralized-arm judge scores, keyed by cellId — the winner surface with its
355
- * content footprint-matched-blanked (via a `neutralize` fn). Same scenarios as
356
- * `judgeScores`. Present ONLY when `runImprovementLoop` was given a `neutralize`
357
- * function. A placebo gate (`neutralizationGate`) compares this arm's lift
358
- * against the candidate's to reject decorative wins (lift from footprint, not
359
- * content). Undefined otherwise. */
360
- neutralizedJudgeScores?: Map<string, Record<string, JudgeScore>>;
361
- /** Neutralized-arm artifacts, keyed by cellId. Present alongside
362
- * `neutralizedJudgeScores`. */
363
- neutralizedArtifacts?: Map<string, TArtifact>;
364
- scenarios: TScenario[];
365
- cost: {
366
- candidate: number;
367
- baseline: number;
368
- };
369
- /** Shared run spend account and receipt attribution phase. */
370
- costLedger?: CostLedger;
371
- costPhase?: string;
372
- signal: AbortSignal;
373
- }
374
- interface GateResult {
375
- decision: GateDecision;
376
- reasons: string[];
377
- contributingGates: Array<{
378
- name: string;
379
- passed: boolean;
380
- detail: unknown;
381
- }>;
382
- delta?: number;
383
- }
384
- /** Composable promotion gate. */
385
- interface Gate<TArtifact = unknown, TScenario extends Scenario = Scenario> {
386
- name: string;
387
- decide(ctx: GateContext<TArtifact, TScenario>): Promise<GateResult>;
388
- }
389
- /** Scoped trace writer handed to each dispatch — every span
390
- * auto-tagged with the cellId so traces filter cleanly. */
391
- interface CampaignTraceWriter {
392
- span(name: string, attributes?: Record<string, unknown>): TraceSpan;
393
- flush(): Promise<void>;
394
- }
395
- interface TraceSpan {
396
- end(attributes?: Record<string, unknown>): void;
397
- setAttribute(key: string, value: unknown): void;
398
- }
399
- /** Scoped artifact writer — `write(path, content)` lands under
400
- * `<runDir>/<cellId>/<path>`. */
401
- interface CampaignArtifactWriter {
402
- write(path: string, content: string | Uint8Array): Promise<string>;
403
- writeJson(path: string, value: unknown): Promise<string>;
404
- }
405
- /** Token usage accumulated for a cell. Aliased to the canonical `RunTokenUsage`
406
- * (run-record.ts, same package) so a cell maps onto a `RunRecord` for the
407
- * backend-integrity guard with ONE source of truth — a field added to
408
- * `RunTokenUsage` is a compile error here, not a silent drift. */
409
- type CampaignTokenUsage = RunTokenUsage;
410
- /** Cell-scoped paid-call entry point. The dispatch places every paid operation
411
- * inside `runPaidCall`; the returned provider result supplies one receipt with
412
- * cost, tokens, and resolved model. Calls made outside this method are not
413
- * admitted or captured. */
414
- interface CampaignCostMeter {
415
- /** The only paid-call path. Returns a typed result; callers must inspect it. */
416
- runPaidCall<T>(input: Omit<RunPaidCallInput<T>, 'channel' | 'phase' | 'tags'> & {
417
- channel?: CostChannel;
418
- }): Promise<PaidCallResult<T>>;
419
- }
420
- /** Source tag — required on every store write. Used by the
421
- * default training-source filter (production-trace samples NOT used as
422
- * training scenarios unless explicitly opted in). */
423
- type LabeledScenarioSource = 'production-trace' | 'eval-run' | 'manual' | 'red-team' | 'synthetic';
424
- type RedactionStatus = 'raw' | 'redacted-pii' | 'redacted-secrets' | 'fully-redacted';
425
- /** How much a label can be trusted to evaluate against — the gold-admission
426
- * gate. Strictly ordered: a record qualifies for a `minTrust` filter when its
427
- * trust rank is >= the requested rank.
428
- *
429
- * - `unverified` — label is a heuristic (e.g. raw outcome success/fail).
430
- * Fine as corpus; MUST NOT enter a gold set that lift
431
- * numbers are computed against.
432
- * - `verified-signal` — an external signal confirmed the outcome (PR merged,
433
- * tests green, user did not retry, downstream check).
434
- * - `human-rated` — a human explicitly rated or corrected the artifact.
435
- *
436
- * Absent on a write ⇒ treated as `unverified` (fail-closed: a writer must
437
- * explicitly assert trust to make a record gold-eligible — it never happens
438
- * by accident). */
439
- type LabelTrust = 'unverified' | 'verified-signal' | 'human-rated';
440
- /** Ordinal rank for a label-trust tier; absent ⇒ `unverified` (rank 0). */
441
- declare function labelTrustRank(trust: LabelTrust | undefined): number;
442
- /** Required-provenance write. The store rejects writes that
443
- * lack provenance — a default-on flywheel without provenance is the
444
- * data-poisoning vector flagged in the alignment review. */
445
- interface LabeledScenarioWrite<TScenario extends Scenario = Scenario, TArtifact = unknown> {
446
- scenario: TScenario;
447
- artifact: TArtifact;
448
- judgeScores: Record<string, JudgeScore>;
449
- source: LabeledScenarioSource;
450
- sourceVersionHash: string;
451
- capturedAt: string;
452
- redactionStatus: RedactionStatus;
453
- /** Gold-admission trust tier. Absent ⇒ `unverified` (fail-closed): the
454
- * record is corpus, never gold. A writer must explicitly assert
455
- * `verified-signal` or `human-rated` to make it eligible for a gold
456
- * sample. See {@link LabelTrust}. */
457
- labelTrust?: LabelTrust;
458
- /** Optional per-source rate-limit bucket key (e.g., the tenant id). */
459
- rateLimitBucket?: string;
460
- }
461
- interface LabeledScenarioRecord<TScenario extends Scenario = Scenario, TArtifact = unknown> extends LabeledScenarioWrite<TScenario, TArtifact> {
462
- /** Stable hash of (scenario.id, source, capturedAt, sourceVersionHash). */
463
- recordHash: string;
464
- /** Substrate-assigned split — train if captured before the campaign's
465
- * `temporalCutoff`, test if after. Explicit override allowed via filter. */
466
- split: 'train' | 'test';
467
- }
468
- interface LabeledScenarioSampleArgs {
469
- count: number;
470
- /** REQUIRED — substrate refuses to sample without an explicit split. */
471
- split: 'train' | 'test';
472
- /** REQUIRED — only records captured before this timestamp are returned.
473
- * Enforces temporal split discipline (test scenarios captured AFTER train
474
- * cannot enter the training pool). */
475
- capturedBefore: string;
476
- filter?: {
477
- kind?: string;
478
- source?: LabeledScenarioSource | LabeledScenarioSource[];
479
- minComposite?: number;
480
- maxComposite?: number;
481
- /** Gold gate: only records whose trust rank is >= this tier are
482
- * returned. `sample({ split: 'test', minTrust: 'verified-signal' })` is
483
- * the canonical "give me the gold set" call. Absent ⇒ no trust gate
484
- * (corpus-level read). */
485
- minTrust?: LabelTrust;
486
- };
487
- }
488
- interface LabeledScenarioStore {
489
- observe(write: LabeledScenarioWrite): Promise<void>;
490
- sample(args: LabeledScenarioSampleArgs): Promise<LabeledScenarioRecord[]>;
491
- size(): Promise<{
492
- train: number;
493
- test: number;
494
- bySource: Record<string, number>;
495
- /** Count by trust tier — tells the flywheel how much gold it has
496
- * accumulated vs. raw corpus. */
497
- byTrust: Record<LabelTrust, number>;
498
- }>;
499
- }
500
- interface CampaignCellResult<TArtifact> {
501
- /** Manifest that produced this cell. Resumability refuses to reuse a cell
502
- * whose manifest differs from the current run. */
503
- manifestHash?: string;
504
- cellId: string;
505
- scenarioId: string;
506
- rep: number;
507
- generation?: number;
508
- artifact: TArtifact;
509
- judgeScores: Record<string, JudgeScore>;
510
- costUsd: number;
511
- /** True when at least one priced receipt used the model table instead of a provider bill. */
512
- costEstimated?: boolean;
513
- /** Exact durable receipts required to reuse this cached result. */
514
- costCallIds?: string[];
515
- /** Agent-call token usage committed by `ctx.cost.runPaidCall`.
516
- * `{ input: 0, output: 0 }` when no paid agent call was recorded. */
517
- tokenUsage: CampaignTokenUsage;
518
- /** Concrete model from the latest committed agent receipt. Consumed by
519
- * `buildRunRecord` to pin the model when the declared profile uses a
520
- * runtime-resolved sentinel. */
521
- resolvedModel?: string;
522
- durationMs: number;
523
- seed: number;
524
- cached: boolean;
525
- error?: string;
526
- }
527
- interface JudgeAggregate {
528
- mean: number;
529
- stdev: number;
530
- ci95: [number, number];
531
- n: number;
532
- }
533
- interface ScenarioAggregate {
534
- meanComposite: number;
535
- ci95: [number, number];
536
- n: number;
537
- }
538
- interface GenerationRecord {
539
- generationIndex: number;
540
- candidates: GenerationCandidate[];
541
- promoted: string[];
542
- }
543
- /** One scored candidate surface in a generation. `dimensions` + `scenarios`
544
- * let a reflective proposer ground its next proposal on WHICH
545
- * dimensions the candidate is weakest on and WHICH scenarios it best/worst
546
- * handled — the evidence a blind `Mutator` cannot see. */
547
- interface GenerationCandidate {
548
- surfaceHash: string;
549
- composite: number;
550
- ci95: [number, number];
551
- /** Exact surface this candidate mutated. */
552
- parentSurfaceHash?: string;
553
- /** Measured search-split composite of the exact parent surface. */
554
- parentComposite?: number;
555
- /** Candidate composite minus its parent's composite. Present only when the
556
- * candidate completed the designed denominator. */
557
- observedDeltaFromParent?: number;
558
- /** Whether this candidate had a scorable result for every designed campaign
559
- * cell and was therefore eligible for ranking, promotion, and Pareto
560
- * selection. Older externally-authored records may omit this field; loop
561
- * records always populate it. */
562
- eligibleForPromotion?: boolean;
563
- /** Exact denominator receipt for selection eligibility. Scores stay
564
- * descriptive: an incomplete candidate is retained with its observed score
565
- * and errors instead of receiving an invented penalty. */
566
- coverage?: {
567
- expectedCells: number;
568
- scorableCells: number;
569
- unscorableCells: Array<{
570
- cellId: string;
571
- reason: string;
572
- }>;
573
- };
574
- /** Mean score per judge dimension across all cells (scenarios × reps ×
575
- * judges that reported the dimension). */
576
- dimensions: Record<string, number>;
577
- /** Per-scenario composite (mean over reps + judges), plus the judge's
578
- * free-form `notes` for that scenario — the "why it scored low" evidence a
579
- * reflective proposer grounds its next edit on. Keep `notes` GENERALIZABLE
580
- * (which checks/lines/dimensions failed and how), NOT case-specific ground
581
- * truth: leaking expected answers into the prompt is memorization, and the
582
- * held-out gate would reject it anyway. */
583
- scenarios: Array<{
584
- scenarioId: string;
585
- composite: number;
586
- notes?: string;
587
- }>;
588
- /** Proposer-supplied short label for the change. Present when the proposer
589
- * returned a `ProposedCandidate`; absent for bare-surface mutators. */
590
- label?: string;
591
- /** Proposer-supplied rationale — WHY this candidate was proposed. The
592
- * "because rationale Z" the audit requires to survive to the result.
593
- * Present when the proposer returned a `ProposedCandidate`. */
594
- rationale?: string;
595
- /** Exact structured cause threaded from the proposer, when available. */
596
- candidateRecord?: PolicyEditCandidateRecord;
597
- }
598
- interface CampaignAggregates {
599
- byJudge: Record<string, JudgeAggregate>;
600
- byScenario: Record<string, ScenarioAggregate>;
601
- /** Canonical campaign accounting, including worker and judge calls. */
602
- cost: CostLedgerSummary;
603
- /** Compatibility alias of `cost.totalCostUsd`. */
604
- totalCostUsd: number;
605
- cellsExecuted: number;
606
- cellsSkipped: number;
607
- cellsCached: number;
608
- cellsFailed: number;
609
- }
610
- interface CampaignResult<TArtifact = unknown, TScenario extends Scenario = Scenario> {
611
- /** sha256(scenarios, judges, dispatch source ref, optimizer config, seed). Stable identity for reruns. */
612
- manifestHash: string;
613
- seed: number;
614
- startedAt: string;
615
- endedAt: string;
616
- durationMs: number;
617
- cells: Array<CampaignCellResult<TArtifact>>;
618
- aggregates: CampaignAggregates;
619
- optimization?: {
620
- generations: GenerationRecord[];
621
- winnerSurfaceHash?: string;
622
- };
623
- gate?: GateResult;
624
- prUrl?: string;
625
- runDir: string;
626
- artifactsByPath: Record<string, string>;
627
- /** Substrate strips the input scenarios to id+kind for the result manifest;
628
- * consumers needing full payload look it up via the original input. The
629
- * type parameter `TScenario` is propagated for downstream consumers that
630
- * want narrowed types when extending `CampaignResult`. */
631
- scenarios: Array<Pick<TScenario, 'id' | 'kind'>>;
632
- }
633
-
634
- export { type JudgeAggregate as A, type ScenarioAggregate as B, type CampaignResult as C, type DispatchContext as D, type ScoredSurfaceOutcome as E, isProposedCandidate as F, type Gate as G, labelTrustRank as H, type JudgeScore as J, type LabeledScenarioStore as L, type MutableSurface as M, type OptimizationProposer as O, type ParetoParent as P, type RedactionStatus as R, type Scenario as S, type TraceSpan as T, type JudgeDimension as a, type JudgeConfig as b, type SurfaceProposer as c, type GateDecision as d, type CampaignAggregates as e, type CampaignArtifactWriter as f, type CampaignCellResult as g, type CampaignCostMeter as h, type CampaignTraceWriter as i, type CodeSurface as j, type DispatchFn as k, type GateContext as l, type GateResult as m, type GenerationCandidate as n, type GenerationRecord as o, type Mutator as p, type OptimizerConfig as q, type SessionScript as r, type LabeledScenarioWrite as s, type LabeledScenarioSampleArgs as t, type LabeledScenarioRecord as u, type LabelTrust as v, type ProposedCandidate as w, type ProposeContext as x, type LabeledScenarioSource as y, type CampaignTokenUsage as z };