@tangle-network/agent-eval 0.120.1 → 0.120.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (171) hide show
  1. package/CHANGELOG.md +6 -0
  2. package/package.json +1 -1
  3. package/dist/analyst/index.d.ts +0 -3111
  4. package/dist/analyst/index.js +0 -403
  5. package/dist/analyst/index.js.map +0 -1
  6. package/dist/authenticity/index.d.ts +0 -161
  7. package/dist/authenticity/index.js +0 -215
  8. package/dist/authenticity/index.js.map +0 -1
  9. package/dist/belief-state/index.d.ts +0 -1301
  10. package/dist/belief-state/index.js +0 -2152
  11. package/dist/belief-state/index.js.map +0 -1
  12. package/dist/benchmarks/index.d.ts +0 -974
  13. package/dist/benchmarks/index.js +0 -60
  14. package/dist/benchmarks/index.js.map +0 -1
  15. package/dist/builder-eval/index.d.ts +0 -695
  16. package/dist/builder-eval/index.js +0 -366
  17. package/dist/builder-eval/index.js.map +0 -1
  18. package/dist/campaign/index.d.ts +0 -7454
  19. package/dist/campaign/index.js +0 -272
  20. package/dist/campaign/index.js.map +0 -1
  21. package/dist/chunk-32BZXMSO.js +0 -3878
  22. package/dist/chunk-32BZXMSO.js.map +0 -1
  23. package/dist/chunk-3A246TSA.js +0 -998
  24. package/dist/chunk-3A246TSA.js.map +0 -1
  25. package/dist/chunk-3RF76KTD.js +0 -84
  26. package/dist/chunk-3RF76KTD.js.map +0 -1
  27. package/dist/chunk-3YYRZDON.js +0 -45
  28. package/dist/chunk-3YYRZDON.js.map +0 -1
  29. package/dist/chunk-4I2E3LLO.js +0 -1030
  30. package/dist/chunk-4I2E3LLO.js.map +0 -1
  31. package/dist/chunk-ARU2PZFM.js +0 -312
  32. package/dist/chunk-ARU2PZFM.js.map +0 -1
  33. package/dist/chunk-BOD4O7OF.js +0 -40
  34. package/dist/chunk-BOD4O7OF.js.map +0 -1
  35. package/dist/chunk-DPZAEKA6.js +0 -880
  36. package/dist/chunk-DPZAEKA6.js.map +0 -1
  37. package/dist/chunk-DTJ6QUQB.js +0 -131
  38. package/dist/chunk-DTJ6QUQB.js.map +0 -1
  39. package/dist/chunk-GGE4NNQT.js +0 -65
  40. package/dist/chunk-GGE4NNQT.js.map +0 -1
  41. package/dist/chunk-H5UD2323.js +0 -286
  42. package/dist/chunk-H5UD2323.js.map +0 -1
  43. package/dist/chunk-HHWE3POT.js +0 -94
  44. package/dist/chunk-HHWE3POT.js.map +0 -1
  45. package/dist/chunk-HKUCJ437.js +0 -787
  46. package/dist/chunk-HKUCJ437.js.map +0 -1
  47. package/dist/chunk-JHCHEVET.js +0 -274
  48. package/dist/chunk-JHCHEVET.js.map +0 -1
  49. package/dist/chunk-JHOJHHU7.js +0 -867
  50. package/dist/chunk-JHOJHHU7.js.map +0 -1
  51. package/dist/chunk-JM2SKQMS.js +0 -750
  52. package/dist/chunk-JM2SKQMS.js.map +0 -1
  53. package/dist/chunk-JN2FCO5W.js +0 -7958
  54. package/dist/chunk-JN2FCO5W.js.map +0 -1
  55. package/dist/chunk-K4DBDHLK.js +0 -158
  56. package/dist/chunk-K4DBDHLK.js.map +0 -1
  57. package/dist/chunk-K6N6XJJX.js +0 -306
  58. package/dist/chunk-K6N6XJJX.js.map +0 -1
  59. package/dist/chunk-MA6HLL3S.js +0 -65
  60. package/dist/chunk-MA6HLL3S.js.map +0 -1
  61. package/dist/chunk-MAZ26DC7.js +0 -99
  62. package/dist/chunk-MAZ26DC7.js.map +0 -1
  63. package/dist/chunk-MOXWMGPC.js +0 -577
  64. package/dist/chunk-MOXWMGPC.js.map +0 -1
  65. package/dist/chunk-NJC7U437.js +0 -626
  66. package/dist/chunk-NJC7U437.js.map +0 -1
  67. package/dist/chunk-NPCTHQIO.js +0 -91
  68. package/dist/chunk-NPCTHQIO.js.map +0 -1
  69. package/dist/chunk-ONWEPEDO.js +0 -57
  70. package/dist/chunk-ONWEPEDO.js.map +0 -1
  71. package/dist/chunk-OYZAPX5G.js +0 -1526
  72. package/dist/chunk-OYZAPX5G.js.map +0 -1
  73. package/dist/chunk-PC4UYEBM.js +0 -166
  74. package/dist/chunk-PC4UYEBM.js.map +0 -1
  75. package/dist/chunk-PICTDURQ.js +0 -766
  76. package/dist/chunk-PICTDURQ.js.map +0 -1
  77. package/dist/chunk-PJQFMIOX.js +0 -1182
  78. package/dist/chunk-PJQFMIOX.js.map +0 -1
  79. package/dist/chunk-PXD6ZFNY.js +0 -1107
  80. package/dist/chunk-PXD6ZFNY.js.map +0 -1
  81. package/dist/chunk-PXE2VKMX.js +0 -140
  82. package/dist/chunk-PXE2VKMX.js.map +0 -1
  83. package/dist/chunk-PZ5AY32C.js +0 -10
  84. package/dist/chunk-PZ5AY32C.js.map +0 -1
  85. package/dist/chunk-QBRSJK47.js +0 -622
  86. package/dist/chunk-QBRSJK47.js.map +0 -1
  87. package/dist/chunk-QWMPPZ3X.js +0 -550
  88. package/dist/chunk-QWMPPZ3X.js.map +0 -1
  89. package/dist/chunk-S3UZOQ5Y.js +0 -328
  90. package/dist/chunk-S3UZOQ5Y.js.map +0 -1
  91. package/dist/chunk-S5TT5R3L.js +0 -2668
  92. package/dist/chunk-S5TT5R3L.js.map +0 -1
  93. package/dist/chunk-T4SQEITX.js +0 -95
  94. package/dist/chunk-T4SQEITX.js.map +0 -1
  95. package/dist/chunk-TT4KNT67.js +0 -124
  96. package/dist/chunk-TT4KNT67.js.map +0 -1
  97. package/dist/chunk-U5CHZ5M3.js +0 -357
  98. package/dist/chunk-U5CHZ5M3.js.map +0 -1
  99. package/dist/chunk-ULOKLHIQ.js +0 -1937
  100. package/dist/chunk-ULOKLHIQ.js.map +0 -1
  101. package/dist/chunk-VI2UW6B6.js +0 -162
  102. package/dist/chunk-VI2UW6B6.js.map +0 -1
  103. package/dist/chunk-VQMK5FMP.js +0 -247
  104. package/dist/chunk-VQMK5FMP.js.map +0 -1
  105. package/dist/chunk-VSMTAMNK.js +0 -53
  106. package/dist/chunk-VSMTAMNK.js.map +0 -1
  107. package/dist/chunk-VZSRQ272.js +0 -149
  108. package/dist/chunk-VZSRQ272.js.map +0 -1
  109. package/dist/chunk-WW2A73HW.js +0 -159
  110. package/dist/chunk-WW2A73HW.js.map +0 -1
  111. package/dist/chunk-X4UCIOTZ.js +0 -136
  112. package/dist/chunk-X4UCIOTZ.js.map +0 -1
  113. package/dist/chunk-XDIRG3TO.js +0 -1266
  114. package/dist/chunk-XDIRG3TO.js.map +0 -1
  115. package/dist/chunk-XJYR7XFV.js +0 -317
  116. package/dist/chunk-XJYR7XFV.js.map +0 -1
  117. package/dist/chunk-ZET2UAYW.js +0 -89
  118. package/dist/chunk-ZET2UAYW.js.map +0 -1
  119. package/dist/chunk-ZZUXHH3R.js +0 -99
  120. package/dist/chunk-ZZUXHH3R.js.map +0 -1
  121. package/dist/cli.d.ts +0 -1
  122. package/dist/cli.js +0 -112
  123. package/dist/cli.js.map +0 -1
  124. package/dist/contract/index.d.ts +0 -4972
  125. package/dist/contract/index.js +0 -1654
  126. package/dist/contract/index.js.map +0 -1
  127. package/dist/control.d.ts +0 -1013
  128. package/dist/control.js +0 -34
  129. package/dist/control.js.map +0 -1
  130. package/dist/fuzz.d.ts +0 -759
  131. package/dist/fuzz.js +0 -714
  132. package/dist/fuzz.js.map +0 -1
  133. package/dist/hosted/index.d.ts +0 -730
  134. package/dist/hosted/index.js +0 -14
  135. package/dist/hosted/index.js.map +0 -1
  136. package/dist/index.d.ts +0 -16780
  137. package/dist/index.js +0 -12168
  138. package/dist/index.js.map +0 -1
  139. package/dist/matrix/index.d.ts +0 -155
  140. package/dist/matrix/index.js +0 -8
  141. package/dist/matrix/index.js.map +0 -1
  142. package/dist/meta-eval/index.d.ts +0 -1030
  143. package/dist/meta-eval/index.js +0 -417
  144. package/dist/meta-eval/index.js.map +0 -1
  145. package/dist/multishot/index.d.ts +0 -579
  146. package/dist/multishot/index.js +0 -589
  147. package/dist/multishot/index.js.map +0 -1
  148. package/dist/openapi.json +0 -992
  149. package/dist/pipelines/index.d.ts +0 -567
  150. package/dist/pipelines/index.js +0 -515
  151. package/dist/pipelines/index.js.map +0 -1
  152. package/dist/reporting.d.ts +0 -1277
  153. package/dist/reporting.js +0 -48
  154. package/dist/reporting.js.map +0 -1
  155. package/dist/rl.d.ts +0 -4092
  156. package/dist/rl.js +0 -1724
  157. package/dist/rl.js.map +0 -1
  158. package/dist/run-campaign-HNFPJET4.js +0 -14
  159. package/dist/run-campaign-HNFPJET4.js.map +0 -1
  160. package/dist/storyboard/index.d.ts +0 -279
  161. package/dist/storyboard/index.js +0 -767
  162. package/dist/storyboard/index.js.map +0 -1
  163. package/dist/trace-attributes.d.ts +0 -52
  164. package/dist/trace-attributes.js +0 -62
  165. package/dist/trace-attributes.js.map +0 -1
  166. package/dist/traces.d.ts +0 -2343
  167. package/dist/traces.js +0 -249
  168. package/dist/traces.js.map +0 -1
  169. package/dist/wire/index.d.ts +0 -1252
  170. package/dist/wire/index.js +0 -81
  171. package/dist/wire/index.js.map +0 -1
package/dist/fuzz.d.ts DELETED
@@ -1,759 +0,0 @@
1
- type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
2
- interface CostUsage {
3
- inputTokens: number;
4
- /** Includes reasoning tokens when the provider bills them as output. */
5
- outputTokens: number;
6
- /** Reasoning-token subset of outputTokens, when reported. */
7
- reasoningTokens?: number;
8
- /** Prompt tokens served from a provider cache. */
9
- cachedTokens?: number;
10
- /** Prompt tokens written into a provider cache. */
11
- cacheWriteTokens?: number;
12
- }
13
- interface CostCallBase {
14
- callId: string;
15
- channel: CostChannel;
16
- phase: string;
17
- actor: string;
18
- model: string;
19
- maximumCostUsd?: number;
20
- tags?: Record<string, string>;
21
- timestamp: number;
22
- }
23
- interface CostReceipt extends CostCallBase, CostUsage {
24
- status: 'settled';
25
- costUsd: number;
26
- costUnknown: boolean;
27
- usageUnknown?: boolean;
28
- pricing?: {
29
- inputUsdPerThousand: number;
30
- outputUsdPerThousand: number;
31
- };
32
- actualCostUsd?: number;
33
- error?: string;
34
- }
35
- interface CostReceiptInput extends CostUsage {
36
- model: string;
37
- actualCostUsd?: number;
38
- costUnknown?: boolean;
39
- usageUnknown?: boolean;
40
- }
41
- type MaximumCharge = {
42
- externallyEnforcedMaximumUsd: number;
43
- } | ({
44
- model: string;
45
- } & CostUsage);
46
- interface RunPaidCallInput<T> {
47
- callId?: string;
48
- channel: CostChannel;
49
- phase: string;
50
- actor: string;
51
- /** Used before a provider receipt exists and on failures without one. */
52
- model?: string;
53
- tags?: Record<string, string>;
54
- signal?: AbortSignal;
55
- /** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
56
- maximumCharge?: MaximumCharge;
57
- /** `callId` can be forwarded as the provider's idempotency key. */
58
- execute(signal: AbortSignal, callId: string): Promise<T>;
59
- receipt(value: T): CostReceiptInput;
60
- receiptFromError?(error: Error): CostReceiptInput | undefined;
61
- }
62
- type PaidCallResult<T> = {
63
- succeeded: true;
64
- callId: string;
65
- value: T;
66
- receipt: CostReceipt;
67
- } | {
68
- succeeded: false;
69
- callId?: string;
70
- error: Error;
71
- receipt?: CostReceipt;
72
- };
73
- interface ChannelRollup {
74
- channel: CostChannel;
75
- calls: number;
76
- inputTokens: number;
77
- outputTokens: number;
78
- reasoningTokens?: number;
79
- cachedTokens: number;
80
- cacheWriteTokens?: number;
81
- costUsd: number;
82
- unpricedCalls: number;
83
- unknownUsageCalls: number;
84
- }
85
- interface CostLedgerSummary {
86
- totalCalls: number;
87
- pendingCalls: number;
88
- unresolvedCalls: number;
89
- reservedCostUsd: number;
90
- inputTokens: number;
91
- outputTokens: number;
92
- reasoningTokens?: number;
93
- cachedTokens: number;
94
- cacheWriteTokens?: number;
95
- totalCostUsd: number;
96
- byChannel: ChannelRollup[];
97
- unpricedModels: string[];
98
- fullyPriced: boolean;
99
- usageComplete: boolean;
100
- accountingComplete: boolean;
101
- incompleteReasons: string[];
102
- }
103
- interface CostLedgerFilter {
104
- channel?: CostChannel;
105
- phase?: string;
106
- tags?: Record<string, string>;
107
- }
108
- interface CostLedgerWaitOptions {
109
- /** Maximum time to wait for active provider calls. Default 5 seconds. */
110
- timeoutMs?: number;
111
- }
112
- /** Append-only storage. `append` must atomically reject stale revisions. */
113
- interface CostLedgerPersistence {
114
- read(): {
115
- revision: string;
116
- events: string;
117
- };
118
- append(expectedRevision: string, event: string): string | undefined;
119
- }
120
- interface CostLedgerOptions {
121
- costCeilingUsd?: number;
122
- persistence?: CostLedgerPersistence;
123
- /** Import already-settled receipts without admitting new paid work. */
124
- receipts?: readonly CostReceipt[];
125
- }
126
- /** Run-wide paid-call admission, durable call state, receipts, and summaries. */
127
- declare class CostLedger {
128
- private readonly records;
129
- private readonly activeCallIds;
130
- private readonly lateCallIds;
131
- private readonly idleWaiters;
132
- private completedTasks;
133
- private revision;
134
- private costLimitPersisted;
135
- readonly costCeilingUsd?: number;
136
- private readonly persistence?;
137
- constructor(input?: number | CostLedgerOptions);
138
- runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
139
- /** Wait until every call started by this ledger has produced a durable outcome. */
140
- waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
141
- /** Settle a call left pending by a crashed process after reconciling with the provider. */
142
- reconcile(callId: string, observed: CostReceiptInput, options?: {
143
- error?: string;
144
- }): CostReceipt;
145
- list(filter?: CostLedgerFilter): CostReceipt[];
146
- summary(filter?: CostLedgerFilter): CostLedgerSummary;
147
- markCompleted(count?: number): void;
148
- costPerCompletedTask(): number | null;
149
- private execute;
150
- private captureLateOutcome;
151
- private releaseActiveCall;
152
- private commitOutcome;
153
- private captureFailure;
154
- private commitReceipt;
155
- private resolveMaximum;
156
- private hasIncompleteSettledCall;
157
- private appendRecord;
158
- private ensureCostLimitPersisted;
159
- private appendEvent;
160
- }
161
- /** Public callback surface for a shared cost ledger.
162
- *
163
- * Declaration bundles may expose this type through multiple package subpaths.
164
- * Keeping callback contracts structural lets those subpaths compose while the
165
- * concrete {@link CostLedger} retains its private durable state.
166
- */
167
- type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
168
-
169
- /**
170
- * Adversarial mutation contract.
171
- *
172
- * `AdversarialMutation<S>` is the scenario-mutation strategy the fuzz harness
173
- * (`fuzzAgent`, src/fuzz) drives: paraphrase, edge-case substitution, or
174
- * compositional combination of a scenario the policy currently passes, looking
175
- * for the tail inputs that break it. The harness supplies the loop; consumers
176
- * supply the mutations and the failure detector.
177
- */
178
- interface AdversarialMutation<S> {
179
- id: string;
180
- /**
181
- * Mutate one scenario. Return null to skip; return one or more new
182
- * scenarios. The harness deduplicates by `mutateScenarioId(scenario)`.
183
- */
184
- mutate(parent: S, rng: () => number): Promise<S[]> | S[];
185
- }
186
-
187
- /**
188
- * Validator-output verdict — substrate primitive for "did this output pass,
189
- * and how well?"
190
- *
191
- * Used by:
192
- * - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.
193
- * - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.
194
- * Runtime keeps `Validator` because it's coupled to runtime-shaped
195
- * `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE
196
- * itself is a substrate concept and lives here.
197
- *
198
- * Repo layering: agent-eval is the substrate (no upward deps). Both
199
- * agent-runtime and agent-knowledge consume this type FROM agent-eval —
200
- * never the other way around. See CLAUDE.md "Repo layering" for the rule.
201
- */
202
- /**
203
- * Minimal verdict shape — `valid` + `score` are required; `scores` +
204
- * `notes` are optional surface. Validators that need richer shapes
205
- * parameterise `Validator<Output, MyVerdict>` with their own type.
206
- *
207
- * Need structured extras? Extend DefaultVerdict with typed fields — never
208
- * serialize extras into `notes`.
209
- */
210
- interface DefaultVerdict {
211
- /** Whether the output meets the validator's pass criteria. */
212
- valid: boolean;
213
- /** Aggregate score in [0, 1]. Drivers use this for winner selection. */
214
- score: number;
215
- /** Per-dimension scores. Free-form; weighted into `score` by the validator. */
216
- scores?: Record<string, number>;
217
- /** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */
218
- notes?: string;
219
- }
220
-
221
- /**
222
- * Behavior-space exploration — types.
223
- *
224
- * One engine searches a space of inputs against a target, scores each run with a
225
- * multi-objective verdict, keeps a quality-diversity archive, and admits only
226
- * findings that pass the validity gates. Adversarial fuzzing is the headline
227
- * preset (`fuzzAgent`); swapping the `Objective` re-points the same engine at
228
- * novelty search, curriculum growth, or user-simulation.
229
- *
230
- * Two kinds of coordinates, deliberately distinct:
231
- * - INPUT axes (`space.axes`) are the stratification plan — enumerable up front,
232
- * so allocation and the coverage denominator (planned vs covered) are honest.
233
- * - MEASURED descriptors (`descriptor(scenario, ev)`) are read off the rollout —
234
- * they bin the archive by what the agent DID, and never inflate the coverage
235
- * denominator (a behavior you haven't seen yet is not a planned cell).
236
- *
237
- * An `Evaluation` IS a `DefaultVerdict` — same spine as judges and verifiers,
238
- * never a parallel score shape.
239
- */
240
-
241
- /** One input axis of the stratification plan (e.g. matterType, difficulty, personaRigor). */
242
- interface SpaceAxis {
243
- name: string;
244
- values: string[];
245
- }
246
- /** The input space to stratify. Cells are the cartesian product of the axes. */
247
- interface BehaviorSpace {
248
- axes: SpaceAxis[];
249
- }
250
- /** One input cell: a coordinate in the stratification plan. */
251
- interface Cell {
252
- /** Stable id, e.g. `matterType=nda|difficulty=hard`. */
253
- id: string;
254
- coords: Record<string, string>;
255
- }
256
- /**
257
- * The outcome of running the target against one scenario: a `DefaultVerdict`
258
- * (`valid`, headline `score` in [0,1], per-dimension `scores`, `notes`) plus the
259
- * fields exploration needs. Keep `scores` populated — the coverage map surfaces
260
- * WHICH dimension is weak only when evaluations carry it.
261
- */
262
- interface Evaluation extends DefaultVerdict {
263
- /** Measured behavior coordinates (e.g. `{ outcome: 'refused' }`). Bins the archive. */
264
- descriptor?: Record<string, string>;
265
- /** Surfaced output — drives exemplars + minimization. */
266
- output?: string;
267
- /** RunRecord id when the target persisted a trace. */
268
- runId?: string;
269
- /** Structured labels, e.g. failure classes (`hallucination`, `refusal`). */
270
- labels?: string[];
271
- /** Wall-clock for the evaluation, when the consumer measures it more precisely
272
- * than the engine can (e.g. excluding judge time). Engine-measured otherwise. */
273
- latencyMs?: number;
274
- }
275
- /** Run the target against one scenario in a cell. */
276
- type Evaluator<S> = (scenario: S, cell: Cell) => Promise<Evaluation>;
277
- /** Context a proposer sees — prior elites + findings let a skill-backed proposer steer. */
278
- interface ProposeContext<S> {
279
- cell: Cell;
280
- seeds: S[];
281
- /** Current archive elites whose input cell matches. */
282
- elites: S[];
283
- /** Verified findings so far (read-only) — probe new gaps, not re-found ones. */
284
- findings: ReadonlyArray<Finding<S>>;
285
- /** How many candidates to propose. */
286
- count: number;
287
- rng: () => number;
288
- }
289
- /**
290
- * Produces candidate scenarios for a cell. A plain function — `mutationProposer`
291
- * builds one from mutation operators; an agent running a generator skill IS one
292
- * (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed.
293
- *
294
- * Distinct from the optimization `SurfaceProposer` (`campaign/types`): that one
295
- * is the proposer in the surface-optimization loop (`runOptimization`); this one
296
- * is the scenario generator in the behavior-fuzzing loop. `SurfaceProposer` is
297
- * THE optimization proposer.
298
- */
299
- type MutationProposer<S> = (ctx: ProposeContext<S>) => Promise<S[]> | S[];
300
- /**
301
- * What "interesting" means. `interest` in [0,1]; a candidate is notable (gate-
302
- * checked, reported) when `interest >= threshold`. `adversarialObjective` (low
303
- * score is interesting) and `noveltyObjective` (far from the archive) ship.
304
- */
305
- interface Objective {
306
- kind: string;
307
- interest(ev: Evaluation, ctx: ObjectiveContext): number;
308
- /** Default 0.5. */
309
- threshold?: number;
310
- }
311
- interface ObjectiveContext {
312
- archiveScores: number[];
313
- archiveDescriptors: Array<Record<string, string> | undefined>;
314
- }
315
- /**
316
- * Validity gates — the moat. A notable candidate is admitted ONLY when it is a
317
- * fair, answerable task (`isValid`) AND reproduces under a meaning-preserving
318
- * rephrase (`isUncontaminated`). Pass-through by default (testable without an
319
- * LLM); live wiring supplies real gates.
320
- */
321
- interface ValidityGates<S> {
322
- isValid?: (scenario: S, ev: Evaluation, cell: Cell) => boolean | Promise<boolean>;
323
- isUncontaminated?: (scenario: S, ev: Evaluation, cell: Cell) => boolean | Promise<boolean>;
324
- }
325
- /** A gate-verified, minimized finding — the unit the capsule reports. */
326
- interface Finding<S> {
327
- id: string;
328
- cell: Cell;
329
- scenario: S;
330
- /** The minimized trigger (== `scenario` when no minimizer is supplied). */
331
- minimized: S;
332
- /** Legible text of the minimized trigger, when `scenarioText` is supplied. */
333
- text?: string;
334
- /** The full multi-objective verdict. */
335
- evaluation: Evaluation;
336
- /** The objective's interest score that flagged it. */
337
- interest: number;
338
- /** Which objective flagged it. */
339
- objective: string;
340
- }
341
- /** An archive elite — the most interesting scenario seen for one bin. */
342
- interface ArchiveEntry<S> {
343
- /** Input cell + measured descriptor coords combined, e.g. `difficulty=hard|outcome=refused`. */
344
- binId: string;
345
- cell: Cell;
346
- scenario: S;
347
- evaluation: Evaluation;
348
- interest: number;
349
- }
350
- /** Summary of a sample — every aggregate carries its spread, never a bare mean. */
351
- interface Distribution {
352
- mean: number;
353
- median: number;
354
- p90: number;
355
- min: number;
356
- max: number;
357
- n: number;
358
- }
359
- /** Per-INPUT-cell coverage — the planned-vs-covered map. */
360
- interface CoverageCell {
361
- cell: Cell;
362
- runs: number;
363
- /** Headline score distribution in [0,1]; `null` when the cell was never run
364
- * (honestly uncovered — never a fabricated zero). */
365
- score: Distribution | null;
366
- /** Fraction of runs the objective flagged as notable. */
367
- findingRate: number;
368
- /** Per-dimension score distributions — surfaces WHICH dimension is weak and
369
- * how consistently. */
370
- dimensions: Record<string, Distribution>;
371
- /** Evaluation wall-clock per run; engine-measured unless the evaluation
372
- * carried its own `latencyMs`. `null` when the cell was never run. */
373
- latencyMs: Distribution | null;
374
- /** Known dollars spent in this cell — present only when cost tracking was
375
- * wired; runs with unknown cost are counted apart, never folded in as $0. */
376
- costUsd?: number;
377
- costUnknownRuns?: number;
378
- }
379
- /** The artifact every exploration produces. */
380
- interface CapsuleData<S> {
381
- target: string;
382
- objective: string;
383
- /** Stamped by the caller — the engine stays clock-free and deterministic. */
384
- generatedAt?: string;
385
- coverage: CoverageCell[];
386
- /** Verified findings, sorted by descending interest. */
387
- findings: Finding<S>[];
388
- /** QD archive elites (binned by input × measured coords). */
389
- archive: ArchiveEntry<S>[];
390
- /** Post-harden lift, filled by a second pass after an improvement. */
391
- lift?: {
392
- before: number;
393
- after: number;
394
- verdict: string;
395
- };
396
- stats: {
397
- totalRuns: number;
398
- /** Input-cell denominator — the stratification plan. */
399
- cellsTotal: number;
400
- cellsCovered: number;
401
- /** Distinct measured-descriptor bins observed (never part of the denominator). */
402
- behaviorBinsObserved: number;
403
- candidateFindings: number;
404
- verifiedFindings: number;
405
- /** Distribution of per-cell mean scores across covered cells (cells weigh
406
- * equally — variance steering sends more runs to weak cells, so a
407
- * run-weighted average would bias low). `null` when nothing ran. */
408
- robustness: Distribution | null;
409
- /** Evaluation wall-clock across all runs. `null` when nothing ran. */
410
- latencyMs: Distribution | null;
411
- /** Known dollars spent on this exploration's runs. Present only when cost
412
- * tracking was wired (`costOf`) — absent means "not tracked", never $0. */
413
- costUsd?: number;
414
- /** Runs whose cost was unknown (`costOf` returned null) — counted apart,
415
- * never folded into `costUsd` as a fabricated $0. */
416
- costUnknownRuns?: number;
417
- /** Evaluations that threw (transport/backend failures). They consumed no
418
- * run budget and scored nothing — an infra axis, never folded into
419
- * robustness or reported as findings. */
420
- evalErrors: number;
421
- /** Present when the run stopped before its budget because consecutive
422
- * eval errors tripped the circuit breaker (a dead backend must not burn
423
- * the remaining budget). The capsule-so-far is complete and honest. */
424
- stoppedEarly?: {
425
- reason: 'eval-errors';
426
- detail: string;
427
- };
428
- };
429
- }
430
- /**
431
- * Known cost of one evaluated run. `model` attributes the spend in the ledger's
432
- * per-model rollup; absent, the entry is labeled `unattributed` (the dollars are
433
- * real either way — recorded as `actualCostUsd`, never an estimate).
434
- */
435
- interface RunCost {
436
- usd: number;
437
- model?: string;
438
- }
439
- type ExploreEvent<S> = {
440
- type: 'cell-allocated';
441
- cell: Cell;
442
- count: number;
443
- } | {
444
- type: 'evaluated';
445
- cell: Cell;
446
- scenario: S;
447
- evaluation: Evaluation;
448
- } | {
449
- type: 'finding';
450
- finding: Finding<S>;
451
- } | {
452
- type: 'eval-error';
453
- cell: Cell;
454
- scenarioId: string;
455
- message: string;
456
- } | {
457
- type: 'round';
458
- runsUsed: number;
459
- budget: number;
460
- };
461
- interface ExploreOptions<S> {
462
- /** Name of the target under exploration — labels the capsule. */
463
- target: string;
464
- /** The input stratification plan. */
465
- space: BehaviorSpace;
466
- /** Candidate generator. */
467
- proposer: MutationProposer<S>;
468
- /** Runs the target → multi-objective `Evaluation`. */
469
- evaluate: Evaluator<S>;
470
- /** Seed corpus per cell. */
471
- seedsFor: (cell: Cell) => S[] | Promise<S[]>;
472
- /** Stable id for a scenario (dedup + lineage). */
473
- scenarioId: (scenario: S) => string;
474
- /** Human-legible text — drives capsule exemplars. */
475
- scenarioText?: (scenario: S) => string;
476
- /** Measured behavior coords appended to the archive bin. Default: input cell only. */
477
- descriptor?: (scenario: S, ev: Evaluation) => Record<string, string>;
478
- /** What "interesting" means. Default: `adversarialObjective(0.5)`. */
479
- objective?: Objective;
480
- /** Validity gates. Default pass-through. */
481
- gates?: ValidityGates<S>;
482
- /** Budget steering across input cells. `variance` chases uncertainty; `uniform` is the unsteered ablation baseline. Default `variance`. */
483
- allocation?: 'variance' | 'uniform';
484
- /** Total target evaluations. */
485
- budget: number;
486
- /** Minimum evaluations per input cell before steering. Default 2. */
487
- floorPerCell?: number;
488
- /** Shrink a notable scenario to its minimal trigger. Default: identity. */
489
- minimize?: (scenario: S, evaluate: Evaluator<S>, cell: Cell) => Promise<S> | S;
490
- /** Max concurrent `evaluate` calls. Default 1. */
491
- concurrency?: number;
492
- /** Stop the run after this many CONSECUTIVE eval errors (a dead backend must
493
- * not burn the remaining budget). Successes reset the streak. Default 5. */
494
- maxConsecutiveEvalErrors?: number;
495
- /** Cooperative cancellation. */
496
- signal?: AbortSignal;
497
- /** Progress stream. */
498
- onProgress?: (event: ExploreEvent<S>) => void;
499
- /** Deterministic seed. Default 1. */
500
- seed?: number;
501
- /**
502
- * Cost of one evaluated run — consumer-supplied; the explorer cannot know
503
- * token usage. Return null when the cost is unknown: the run is COUNTED in
504
- * `stats.costUnknownRuns`, never folded into the total as $0. Required by
505
- * every other cost option (`costBudgetUsd` / `ledger` / `onCost`).
506
- */
507
- costOf?: (scenario: S, cell: Cell, ev: Evaluation) => RunCost | null;
508
- /** Pre-call hard maximum. Required whenever the explorer uses a capped ledger. */
509
- maximumChargeOf?: (scenario: S, cell: Cell) => MaximumCharge;
510
- /**
511
- * Hard dollar cap. Each evaluation reserves `maximumChargeOf` before it starts;
512
- * calls that do not fit are rejected. Unknown totals stop further paid work.
513
- */
514
- costBudgetUsd?: number;
515
- /**
516
- * Sink for per-run cost entries — each known `costOf` result is recorded
517
- * with channel 'agent' and `actualCostUsd` (token axes are zero: the
518
- * explorer only sees dollars). Pass the program's shared `CostLedger` so
519
- * `costReport` stamps fuzz spend alongside judge/analyst spend.
520
- */
521
- ledger?: CostLedgerHandle;
522
- /** Observer fired for every known-cost run recorded. */
523
- onCost?: (entry: {
524
- usd: number;
525
- channel: CostChannel;
526
- }) => void;
527
- }
528
-
529
- /**
530
- * Input-space tiling + coverage projection.
531
- *
532
- * Cells are the cartesian product of the input axes — the stratification plan,
533
- * enumerable up front so the planned-vs-covered denominator is honest. Coverage
534
- * is projected from the evaluation log: per cell, the full DISTRIBUTION of the
535
- * headline score, of each scored dimension, and of evaluation latency — a bare
536
- * mean hides outliers, so every aggregate carries its spread. Per-cell cost is
537
- * split known-dollars vs unknown-runs, never folded into a fabricated $0.
538
- */
539
-
540
- /** One recorded evaluation — the unit coverage and the capsule are built from. */
541
- interface EvalRecord {
542
- cell: Cell;
543
- ev: Evaluation;
544
- /** The objective's interest score for this evaluation. */
545
- interest: number;
546
- /** Evaluation wall-clock — engine-measured unless `ev.latencyMs` overrode it. */
547
- latencyMs: number;
548
- /** Known dollars for this run. `null` = cost tracking was wired but this
549
- * run's cost was unknowable (counted apart). Absent = not tracked at all. */
550
- costUsd?: number | null;
551
- }
552
- /** Enumerate every input cell (cartesian product of the axes), in stable order. */
553
- declare function enumerateCells(space: BehaviorSpace): Cell[];
554
- /** Deterministic id for a coordinate map, e.g. `matterType=nda|difficulty=hard`. */
555
- declare function cellId(space: BehaviorSpace, coords: Record<string, string>): string;
556
- /**
557
- * Project the evaluation log into the per-input-cell coverage map. A cell with
558
- * no evaluations reports `score: null` (honestly uncovered), never zeros.
559
- */
560
- declare function buildCoverage(cells: Cell[], log: EvalRecord[], threshold: number): CoverageCell[];
561
-
562
- /**
563
- * The capsule — the artifact every exploration produces.
564
- *
565
- * `buildCapsule` assembles coverage + verified findings + the QD archive into a
566
- * pure `CapsuleData` (no clock, no I/O — deterministic and snapshot-testable).
567
- * `renderCapsuleHtml` turns it into a standalone page: the input-cell heat-map
568
- * (planned vs covered), per-dimension weakness chips, and the minimized finding
569
- * exemplars. One artifact — the hardening map and the shareable proof object.
570
- */
571
-
572
- interface BuildCapsuleInput<S> {
573
- target: string;
574
- objective: string;
575
- cells: Cell[];
576
- log: EvalRecord[];
577
- /** The objective's notable threshold — drives findingRate. */
578
- threshold: number;
579
- archive: ArchiveEntry<S>[];
580
- findings: Finding<S>[];
581
- candidateFindings: number;
582
- runsUsed: number;
583
- /** Known-dollar / unknown-run split — present only when cost tracking was
584
- * wired; the capsule never fabricates a $0 total. */
585
- cost?: {
586
- costUsd: number;
587
- costUnknownRuns: number;
588
- };
589
- /** Evaluations that threw — infra outcomes, never folded into robustness. */
590
- evalErrors: number;
591
- /** Set when the consecutive-error circuit breaker stopped the run early. */
592
- stoppedEarly?: {
593
- reason: 'eval-errors';
594
- detail: string;
595
- };
596
- }
597
- declare function buildCapsule<S>(input: BuildCapsuleInput<S>): CapsuleData<S>;
598
- interface RenderCapsuleOptions {
599
- /** Max finding exemplars to show. Default 8. */
600
- maxFindings?: number;
601
- /** ISO timestamp to stamp into the page (keeps the pure capsule clock-free). */
602
- generatedAt?: string;
603
- }
604
- /** Render a self-contained HTML capsule — heat-map + per-dimension chips + verified findings. */
605
- declare function renderCapsuleHtml<S>(capsule: CapsuleData<S>, opts?: RenderCapsuleOptions): string;
606
-
607
- /**
608
- * The exploration engine — a stateful session over a behavior space.
609
- *
610
- * Each `step()`: allocate budget across INPUT cells (floor first, then variance
611
- * steering toward the least-certain cells), propose candidates (the proposer
612
- * reads current elites + findings, so the search deepens generationally),
613
- * evaluate with bounded concurrency, archive the most interesting scenario per
614
- * input×measured bin, and admit notable candidates that pass the validity gates.
615
- * `run()` loops to budget. `coverage()`/`findings()`/`capsule()` read live state —
616
- * the surface `makeExploreTools` exposes so an agent can drive the session.
617
- *
618
- * One evaluation log (`EvalRecord[]`) is the source of truth; allocation
619
- * observations and coverage are projections of it.
620
- */
621
-
622
- declare class BehaviorExplorer<S> {
623
- private readonly opts;
624
- private readonly cells;
625
- private readonly cellById;
626
- private readonly objective;
627
- private readonly threshold;
628
- private readonly floorPerCell;
629
- private readonly perRoundBudget;
630
- /** The single evaluation log — coverage + allocation are projections of it. */
631
- private readonly log;
632
- /** binId (input × measured coords) → the most interesting entry seen. */
633
- private readonly archiveByBin;
634
- private readonly _findings;
635
- private runsUsed;
636
- private candidateFindings;
637
- private evalErrors;
638
- private consecutiveEvalErrors;
639
- private stoppedEarly;
640
- private rngState;
641
- private readonly costLedger?;
642
- private readonly costPhase;
643
- constructor(opts: ExploreOptions<S>);
644
- private rng;
645
- private binId;
646
- private allocate;
647
- private objectiveContext;
648
- /** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
649
- private elitesFor;
650
- /** One allocate → propose → evaluate → gate → archive round. */
651
- step(): Promise<{
652
- runs: number;
653
- findings: Finding<S>[];
654
- }>;
655
- /** Loop `step()` until the run or dollar budget is spent, the signal aborts,
656
- * or no progress is made. */
657
- run(): Promise<CapsuleData<S>>;
658
- coverage(): CoverageCell[];
659
- findings(): Finding<S>[];
660
- capsule(): CapsuleData<S>;
661
- }
662
-
663
- /**
664
- * `fuzzAgent` — the adversarial batch preset over `BehaviorExplorer`.
665
- *
666
- * One call: explore the space to budget with the adversarial objective and
667
- * return the capsule. For agent-driven, incremental, or multi-objective use,
668
- * construct a `BehaviorExplorer` and drive it via `makeExploreTools`.
669
- */
670
-
671
- type FuzzAgentOptions<S> = Omit<ExploreOptions<S>, 'objective'> & {
672
- /** Score strictly below this is a candidate failure. Default 0.5. */
673
- failureThreshold?: number;
674
- };
675
- declare function fuzzAgent<S>(opts: FuzzAgentOptions<S>): Promise<{
676
- capsule: CapsuleData<S>;
677
- }>;
678
-
679
- /**
680
- * Validity gates — what separates a fuzzer from a slop generator.
681
- *
682
- * A notable candidate is admitted only when it is fair and reproducible. None of
683
- * these are on by default: the live wiring opts in, so reported findings carry
684
- * their proof.
685
- */
686
-
687
- /** Combine gate sets; a candidate must pass every gate in every set. */
688
- declare function composeGates<S>(...sets: Array<ValidityGates<S> | undefined>): ValidityGates<S>;
689
- /**
690
- * Reproducibility gate. Re-run the target on a meaning-preserving rephrase of the
691
- * flagged scenario; keep the finding only when the rephrase ALSO scores below the
692
- * threshold. A finding that flips under a cosmetic rewrite was keyed to surface
693
- * form, not the task — a false signal we must not report. Costs one extra
694
- * evaluation per candidate (candidates are rare, so cheap).
695
- */
696
- declare function perturbationStabilityGate<S>(opts: {
697
- evaluate: Evaluator<S>;
698
- /** Produce a semantic-preserving rephrase. Return null to skip (treated as pass). */
699
- perturb: (scenario: S) => S | null;
700
- /** Score strictly below this still counts as failing. Default 0.5. */
701
- failureThreshold?: number;
702
- }): ValidityGates<S>;
703
- /**
704
- * Severity-floor gate. Reject borderline candidates whose score sits in a band
705
- * just under the threshold — judge noise, not a real defect.
706
- */
707
- declare function severityFloorGate<S>(opts: {
708
- failureThreshold?: number;
709
- margin?: number;
710
- }): ValidityGates<S>;
711
-
712
- /**
713
- * Shipped policies for the exploration engine.
714
- *
715
- * `MutationProposer` is a plain function type — an agent running a generator skill
716
- * IS a proposer (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed. `mutationProposer`
717
- * builds the deterministic, LLM-free one from mutation operators. Objectives are
718
- * interfaces because the engine reads `kind` + `threshold` off them.
719
- */
720
-
721
- /**
722
- * Perturbation-based search: apply the cell's mutation operators to the current
723
- * elites + seeds, deduping by id. Elites first — mutating the most interesting
724
- * scenario found so far is what makes the search deepen across rounds.
725
- */
726
- declare function mutationProposer<S>(opts: {
727
- mutationsFor: (cell: Cell) => AdversarialMutation<S>[];
728
- scenarioId: (s: S) => string;
729
- }): MutationProposer<S>;
730
- /** Adversarial: a low headline score is interesting — find where the agent fails. */
731
- declare function adversarialObjective(threshold?: number): Objective;
732
- /**
733
- * Novelty: interesting when far from the archive in score AND measured behavior
734
- * descriptor — quality-diversity's diversity pressure; drives corpus growth
735
- * rather than re-finding the same hole.
736
- */
737
- declare function noveltyObjective(threshold?: number): Objective;
738
-
739
- /**
740
- * Agent-drivable surface over a live exploration session.
741
- *
742
- * Framework-neutral tool defs ({name, description, parameters: JSON Schema,
743
- * handler}) so the on-demand agent — not a batch script — drives the search:
744
- * step it, read coverage, inspect findings, render the capsule. Transport
745
- * encodings (OpenAI function shape, MCP) are one-line mappings the host owns.
746
- */
747
-
748
- interface ExploreToolDef {
749
- name: string;
750
- description: string;
751
- /** JSON Schema (draft-07+) for the arguments. */
752
- parameters: Record<string, unknown>;
753
- handler: (args: unknown, ctx?: {
754
- signal?: AbortSignal;
755
- }) => Promise<unknown>;
756
- }
757
- declare function makeExploreTools<S>(explorer: BehaviorExplorer<S>): ExploreToolDef[];
758
-
759
- export { type ArchiveEntry, BehaviorExplorer, type BehaviorSpace, type BuildCapsuleInput, type CapsuleData, type Cell, type CoverageCell, type EvalRecord, type Evaluation, type Evaluator, type ExploreEvent, type ExploreOptions, type ExploreToolDef, type Finding, type FuzzAgentOptions, type MutationProposer, type Objective, type ObjectiveContext, type ProposeContext, type RenderCapsuleOptions, type RunCost, type SpaceAxis, type ValidityGates, adversarialObjective, buildCapsule, buildCoverage, cellId, composeGates, enumerateCells, fuzzAgent, makeExploreTools, mutationProposer, noveltyObjective, perturbationStabilityGate, renderCapsuleHtml, severityFloorGate };