@tangle-network/agent-eval 0.120.1 → 0.120.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +6 -0
- package/package.json +1 -1
- package/dist/analyst/index.d.ts +0 -3111
- package/dist/analyst/index.js +0 -403
- package/dist/analyst/index.js.map +0 -1
- package/dist/authenticity/index.d.ts +0 -161
- package/dist/authenticity/index.js +0 -215
- package/dist/authenticity/index.js.map +0 -1
- package/dist/belief-state/index.d.ts +0 -1301
- package/dist/belief-state/index.js +0 -2152
- package/dist/belief-state/index.js.map +0 -1
- package/dist/benchmarks/index.d.ts +0 -974
- package/dist/benchmarks/index.js +0 -60
- package/dist/benchmarks/index.js.map +0 -1
- package/dist/builder-eval/index.d.ts +0 -695
- package/dist/builder-eval/index.js +0 -366
- package/dist/builder-eval/index.js.map +0 -1
- package/dist/campaign/index.d.ts +0 -7454
- package/dist/campaign/index.js +0 -272
- package/dist/campaign/index.js.map +0 -1
- package/dist/chunk-32BZXMSO.js +0 -3878
- package/dist/chunk-32BZXMSO.js.map +0 -1
- package/dist/chunk-3A246TSA.js +0 -998
- package/dist/chunk-3A246TSA.js.map +0 -1
- package/dist/chunk-3RF76KTD.js +0 -84
- package/dist/chunk-3RF76KTD.js.map +0 -1
- package/dist/chunk-3YYRZDON.js +0 -45
- package/dist/chunk-3YYRZDON.js.map +0 -1
- package/dist/chunk-4I2E3LLO.js +0 -1030
- package/dist/chunk-4I2E3LLO.js.map +0 -1
- package/dist/chunk-ARU2PZFM.js +0 -312
- package/dist/chunk-ARU2PZFM.js.map +0 -1
- package/dist/chunk-BOD4O7OF.js +0 -40
- package/dist/chunk-BOD4O7OF.js.map +0 -1
- package/dist/chunk-DPZAEKA6.js +0 -880
- package/dist/chunk-DPZAEKA6.js.map +0 -1
- package/dist/chunk-DTJ6QUQB.js +0 -131
- package/dist/chunk-DTJ6QUQB.js.map +0 -1
- package/dist/chunk-GGE4NNQT.js +0 -65
- package/dist/chunk-GGE4NNQT.js.map +0 -1
- package/dist/chunk-H5UD2323.js +0 -286
- package/dist/chunk-H5UD2323.js.map +0 -1
- package/dist/chunk-HHWE3POT.js +0 -94
- package/dist/chunk-HHWE3POT.js.map +0 -1
- package/dist/chunk-HKUCJ437.js +0 -787
- package/dist/chunk-HKUCJ437.js.map +0 -1
- package/dist/chunk-JHCHEVET.js +0 -274
- package/dist/chunk-JHCHEVET.js.map +0 -1
- package/dist/chunk-JHOJHHU7.js +0 -867
- package/dist/chunk-JHOJHHU7.js.map +0 -1
- package/dist/chunk-JM2SKQMS.js +0 -750
- package/dist/chunk-JM2SKQMS.js.map +0 -1
- package/dist/chunk-JN2FCO5W.js +0 -7958
- package/dist/chunk-JN2FCO5W.js.map +0 -1
- package/dist/chunk-K4DBDHLK.js +0 -158
- package/dist/chunk-K4DBDHLK.js.map +0 -1
- package/dist/chunk-K6N6XJJX.js +0 -306
- package/dist/chunk-K6N6XJJX.js.map +0 -1
- package/dist/chunk-MA6HLL3S.js +0 -65
- package/dist/chunk-MA6HLL3S.js.map +0 -1
- package/dist/chunk-MAZ26DC7.js +0 -99
- package/dist/chunk-MAZ26DC7.js.map +0 -1
- package/dist/chunk-MOXWMGPC.js +0 -577
- package/dist/chunk-MOXWMGPC.js.map +0 -1
- package/dist/chunk-NJC7U437.js +0 -626
- package/dist/chunk-NJC7U437.js.map +0 -1
- package/dist/chunk-NPCTHQIO.js +0 -91
- package/dist/chunk-NPCTHQIO.js.map +0 -1
- package/dist/chunk-ONWEPEDO.js +0 -57
- package/dist/chunk-ONWEPEDO.js.map +0 -1
- package/dist/chunk-OYZAPX5G.js +0 -1526
- package/dist/chunk-OYZAPX5G.js.map +0 -1
- package/dist/chunk-PC4UYEBM.js +0 -166
- package/dist/chunk-PC4UYEBM.js.map +0 -1
- package/dist/chunk-PICTDURQ.js +0 -766
- package/dist/chunk-PICTDURQ.js.map +0 -1
- package/dist/chunk-PJQFMIOX.js +0 -1182
- package/dist/chunk-PJQFMIOX.js.map +0 -1
- package/dist/chunk-PXD6ZFNY.js +0 -1107
- package/dist/chunk-PXD6ZFNY.js.map +0 -1
- package/dist/chunk-PXE2VKMX.js +0 -140
- package/dist/chunk-PXE2VKMX.js.map +0 -1
- package/dist/chunk-PZ5AY32C.js +0 -10
- package/dist/chunk-PZ5AY32C.js.map +0 -1
- package/dist/chunk-QBRSJK47.js +0 -622
- package/dist/chunk-QBRSJK47.js.map +0 -1
- package/dist/chunk-QWMPPZ3X.js +0 -550
- package/dist/chunk-QWMPPZ3X.js.map +0 -1
- package/dist/chunk-S3UZOQ5Y.js +0 -328
- package/dist/chunk-S3UZOQ5Y.js.map +0 -1
- package/dist/chunk-S5TT5R3L.js +0 -2668
- package/dist/chunk-S5TT5R3L.js.map +0 -1
- package/dist/chunk-T4SQEITX.js +0 -95
- package/dist/chunk-T4SQEITX.js.map +0 -1
- package/dist/chunk-TT4KNT67.js +0 -124
- package/dist/chunk-TT4KNT67.js.map +0 -1
- package/dist/chunk-U5CHZ5M3.js +0 -357
- package/dist/chunk-U5CHZ5M3.js.map +0 -1
- package/dist/chunk-ULOKLHIQ.js +0 -1937
- package/dist/chunk-ULOKLHIQ.js.map +0 -1
- package/dist/chunk-VI2UW6B6.js +0 -162
- package/dist/chunk-VI2UW6B6.js.map +0 -1
- package/dist/chunk-VQMK5FMP.js +0 -247
- package/dist/chunk-VQMK5FMP.js.map +0 -1
- package/dist/chunk-VSMTAMNK.js +0 -53
- package/dist/chunk-VSMTAMNK.js.map +0 -1
- package/dist/chunk-VZSRQ272.js +0 -149
- package/dist/chunk-VZSRQ272.js.map +0 -1
- package/dist/chunk-WW2A73HW.js +0 -159
- package/dist/chunk-WW2A73HW.js.map +0 -1
- package/dist/chunk-X4UCIOTZ.js +0 -136
- package/dist/chunk-X4UCIOTZ.js.map +0 -1
- package/dist/chunk-XDIRG3TO.js +0 -1266
- package/dist/chunk-XDIRG3TO.js.map +0 -1
- package/dist/chunk-XJYR7XFV.js +0 -317
- package/dist/chunk-XJYR7XFV.js.map +0 -1
- package/dist/chunk-ZET2UAYW.js +0 -89
- package/dist/chunk-ZET2UAYW.js.map +0 -1
- package/dist/chunk-ZZUXHH3R.js +0 -99
- package/dist/chunk-ZZUXHH3R.js.map +0 -1
- package/dist/cli.d.ts +0 -1
- package/dist/cli.js +0 -112
- package/dist/cli.js.map +0 -1
- package/dist/contract/index.d.ts +0 -4972
- package/dist/contract/index.js +0 -1654
- package/dist/contract/index.js.map +0 -1
- package/dist/control.d.ts +0 -1013
- package/dist/control.js +0 -34
- package/dist/control.js.map +0 -1
- package/dist/fuzz.d.ts +0 -759
- package/dist/fuzz.js +0 -714
- package/dist/fuzz.js.map +0 -1
- package/dist/hosted/index.d.ts +0 -730
- package/dist/hosted/index.js +0 -14
- package/dist/hosted/index.js.map +0 -1
- package/dist/index.d.ts +0 -16780
- package/dist/index.js +0 -12168
- package/dist/index.js.map +0 -1
- package/dist/matrix/index.d.ts +0 -155
- package/dist/matrix/index.js +0 -8
- package/dist/matrix/index.js.map +0 -1
- package/dist/meta-eval/index.d.ts +0 -1030
- package/dist/meta-eval/index.js +0 -417
- package/dist/meta-eval/index.js.map +0 -1
- package/dist/multishot/index.d.ts +0 -579
- package/dist/multishot/index.js +0 -589
- package/dist/multishot/index.js.map +0 -1
- package/dist/openapi.json +0 -992
- package/dist/pipelines/index.d.ts +0 -567
- package/dist/pipelines/index.js +0 -515
- package/dist/pipelines/index.js.map +0 -1
- package/dist/reporting.d.ts +0 -1277
- package/dist/reporting.js +0 -48
- package/dist/reporting.js.map +0 -1
- package/dist/rl.d.ts +0 -4092
- package/dist/rl.js +0 -1724
- package/dist/rl.js.map +0 -1
- package/dist/run-campaign-HNFPJET4.js +0 -14
- package/dist/run-campaign-HNFPJET4.js.map +0 -1
- package/dist/storyboard/index.d.ts +0 -279
- package/dist/storyboard/index.js +0 -767
- package/dist/storyboard/index.js.map +0 -1
- package/dist/trace-attributes.d.ts +0 -52
- package/dist/trace-attributes.js +0 -62
- package/dist/trace-attributes.js.map +0 -1
- package/dist/traces.d.ts +0 -2343
- package/dist/traces.js +0 -249
- package/dist/traces.js.map +0 -1
- package/dist/wire/index.d.ts +0 -1252
- package/dist/wire/index.js +0 -81
- package/dist/wire/index.js.map +0 -1
package/dist/fuzz.d.ts
DELETED
|
@@ -1,759 +0,0 @@
|
|
|
1
|
-
type CostChannel = 'agent' | 'judge' | 'verifier' | 'analyst' | 'driver' | (string & {});
|
|
2
|
-
interface CostUsage {
|
|
3
|
-
inputTokens: number;
|
|
4
|
-
/** Includes reasoning tokens when the provider bills them as output. */
|
|
5
|
-
outputTokens: number;
|
|
6
|
-
/** Reasoning-token subset of outputTokens, when reported. */
|
|
7
|
-
reasoningTokens?: number;
|
|
8
|
-
/** Prompt tokens served from a provider cache. */
|
|
9
|
-
cachedTokens?: number;
|
|
10
|
-
/** Prompt tokens written into a provider cache. */
|
|
11
|
-
cacheWriteTokens?: number;
|
|
12
|
-
}
|
|
13
|
-
interface CostCallBase {
|
|
14
|
-
callId: string;
|
|
15
|
-
channel: CostChannel;
|
|
16
|
-
phase: string;
|
|
17
|
-
actor: string;
|
|
18
|
-
model: string;
|
|
19
|
-
maximumCostUsd?: number;
|
|
20
|
-
tags?: Record<string, string>;
|
|
21
|
-
timestamp: number;
|
|
22
|
-
}
|
|
23
|
-
interface CostReceipt extends CostCallBase, CostUsage {
|
|
24
|
-
status: 'settled';
|
|
25
|
-
costUsd: number;
|
|
26
|
-
costUnknown: boolean;
|
|
27
|
-
usageUnknown?: boolean;
|
|
28
|
-
pricing?: {
|
|
29
|
-
inputUsdPerThousand: number;
|
|
30
|
-
outputUsdPerThousand: number;
|
|
31
|
-
};
|
|
32
|
-
actualCostUsd?: number;
|
|
33
|
-
error?: string;
|
|
34
|
-
}
|
|
35
|
-
interface CostReceiptInput extends CostUsage {
|
|
36
|
-
model: string;
|
|
37
|
-
actualCostUsd?: number;
|
|
38
|
-
costUnknown?: boolean;
|
|
39
|
-
usageUnknown?: boolean;
|
|
40
|
-
}
|
|
41
|
-
type MaximumCharge = {
|
|
42
|
-
externallyEnforcedMaximumUsd: number;
|
|
43
|
-
} | ({
|
|
44
|
-
model: string;
|
|
45
|
-
} & CostUsage);
|
|
46
|
-
interface RunPaidCallInput<T> {
|
|
47
|
-
callId?: string;
|
|
48
|
-
channel: CostChannel;
|
|
49
|
-
phase: string;
|
|
50
|
-
actor: string;
|
|
51
|
-
/** Used before a provider receipt exists and on failures without one. */
|
|
52
|
-
model?: string;
|
|
53
|
-
tags?: Record<string, string>;
|
|
54
|
-
signal?: AbortSignal;
|
|
55
|
-
/** Provider-enforced dollar maximum, or maximum priced token usage. Required when capped. */
|
|
56
|
-
maximumCharge?: MaximumCharge;
|
|
57
|
-
/** `callId` can be forwarded as the provider's idempotency key. */
|
|
58
|
-
execute(signal: AbortSignal, callId: string): Promise<T>;
|
|
59
|
-
receipt(value: T): CostReceiptInput;
|
|
60
|
-
receiptFromError?(error: Error): CostReceiptInput | undefined;
|
|
61
|
-
}
|
|
62
|
-
type PaidCallResult<T> = {
|
|
63
|
-
succeeded: true;
|
|
64
|
-
callId: string;
|
|
65
|
-
value: T;
|
|
66
|
-
receipt: CostReceipt;
|
|
67
|
-
} | {
|
|
68
|
-
succeeded: false;
|
|
69
|
-
callId?: string;
|
|
70
|
-
error: Error;
|
|
71
|
-
receipt?: CostReceipt;
|
|
72
|
-
};
|
|
73
|
-
interface ChannelRollup {
|
|
74
|
-
channel: CostChannel;
|
|
75
|
-
calls: number;
|
|
76
|
-
inputTokens: number;
|
|
77
|
-
outputTokens: number;
|
|
78
|
-
reasoningTokens?: number;
|
|
79
|
-
cachedTokens: number;
|
|
80
|
-
cacheWriteTokens?: number;
|
|
81
|
-
costUsd: number;
|
|
82
|
-
unpricedCalls: number;
|
|
83
|
-
unknownUsageCalls: number;
|
|
84
|
-
}
|
|
85
|
-
interface CostLedgerSummary {
|
|
86
|
-
totalCalls: number;
|
|
87
|
-
pendingCalls: number;
|
|
88
|
-
unresolvedCalls: number;
|
|
89
|
-
reservedCostUsd: number;
|
|
90
|
-
inputTokens: number;
|
|
91
|
-
outputTokens: number;
|
|
92
|
-
reasoningTokens?: number;
|
|
93
|
-
cachedTokens: number;
|
|
94
|
-
cacheWriteTokens?: number;
|
|
95
|
-
totalCostUsd: number;
|
|
96
|
-
byChannel: ChannelRollup[];
|
|
97
|
-
unpricedModels: string[];
|
|
98
|
-
fullyPriced: boolean;
|
|
99
|
-
usageComplete: boolean;
|
|
100
|
-
accountingComplete: boolean;
|
|
101
|
-
incompleteReasons: string[];
|
|
102
|
-
}
|
|
103
|
-
interface CostLedgerFilter {
|
|
104
|
-
channel?: CostChannel;
|
|
105
|
-
phase?: string;
|
|
106
|
-
tags?: Record<string, string>;
|
|
107
|
-
}
|
|
108
|
-
interface CostLedgerWaitOptions {
|
|
109
|
-
/** Maximum time to wait for active provider calls. Default 5 seconds. */
|
|
110
|
-
timeoutMs?: number;
|
|
111
|
-
}
|
|
112
|
-
/** Append-only storage. `append` must atomically reject stale revisions. */
|
|
113
|
-
interface CostLedgerPersistence {
|
|
114
|
-
read(): {
|
|
115
|
-
revision: string;
|
|
116
|
-
events: string;
|
|
117
|
-
};
|
|
118
|
-
append(expectedRevision: string, event: string): string | undefined;
|
|
119
|
-
}
|
|
120
|
-
interface CostLedgerOptions {
|
|
121
|
-
costCeilingUsd?: number;
|
|
122
|
-
persistence?: CostLedgerPersistence;
|
|
123
|
-
/** Import already-settled receipts without admitting new paid work. */
|
|
124
|
-
receipts?: readonly CostReceipt[];
|
|
125
|
-
}
|
|
126
|
-
/** Run-wide paid-call admission, durable call state, receipts, and summaries. */
|
|
127
|
-
declare class CostLedger {
|
|
128
|
-
private readonly records;
|
|
129
|
-
private readonly activeCallIds;
|
|
130
|
-
private readonly lateCallIds;
|
|
131
|
-
private readonly idleWaiters;
|
|
132
|
-
private completedTasks;
|
|
133
|
-
private revision;
|
|
134
|
-
private costLimitPersisted;
|
|
135
|
-
readonly costCeilingUsd?: number;
|
|
136
|
-
private readonly persistence?;
|
|
137
|
-
constructor(input?: number | CostLedgerOptions);
|
|
138
|
-
runPaidCall<T>(input: RunPaidCallInput<T>): Promise<PaidCallResult<T>>;
|
|
139
|
-
/** Wait until every call started by this ledger has produced a durable outcome. */
|
|
140
|
-
waitForIdle(options?: CostLedgerWaitOptions): Promise<boolean>;
|
|
141
|
-
/** Settle a call left pending by a crashed process after reconciling with the provider. */
|
|
142
|
-
reconcile(callId: string, observed: CostReceiptInput, options?: {
|
|
143
|
-
error?: string;
|
|
144
|
-
}): CostReceipt;
|
|
145
|
-
list(filter?: CostLedgerFilter): CostReceipt[];
|
|
146
|
-
summary(filter?: CostLedgerFilter): CostLedgerSummary;
|
|
147
|
-
markCompleted(count?: number): void;
|
|
148
|
-
costPerCompletedTask(): number | null;
|
|
149
|
-
private execute;
|
|
150
|
-
private captureLateOutcome;
|
|
151
|
-
private releaseActiveCall;
|
|
152
|
-
private commitOutcome;
|
|
153
|
-
private captureFailure;
|
|
154
|
-
private commitReceipt;
|
|
155
|
-
private resolveMaximum;
|
|
156
|
-
private hasIncompleteSettledCall;
|
|
157
|
-
private appendRecord;
|
|
158
|
-
private ensureCostLimitPersisted;
|
|
159
|
-
private appendEvent;
|
|
160
|
-
}
|
|
161
|
-
/** Public callback surface for a shared cost ledger.
|
|
162
|
-
*
|
|
163
|
-
* Declaration bundles may expose this type through multiple package subpaths.
|
|
164
|
-
* Keeping callback contracts structural lets those subpaths compose while the
|
|
165
|
-
* concrete {@link CostLedger} retains its private durable state.
|
|
166
|
-
*/
|
|
167
|
-
type CostLedgerHandle = Pick<CostLedger, Exclude<keyof CostLedger, 'waitForIdle'>> & Partial<Pick<CostLedger, 'waitForIdle'>>;
|
|
168
|
-
|
|
169
|
-
/**
|
|
170
|
-
* Adversarial mutation contract.
|
|
171
|
-
*
|
|
172
|
-
* `AdversarialMutation<S>` is the scenario-mutation strategy the fuzz harness
|
|
173
|
-
* (`fuzzAgent`, src/fuzz) drives: paraphrase, edge-case substitution, or
|
|
174
|
-
* compositional combination of a scenario the policy currently passes, looking
|
|
175
|
-
* for the tail inputs that break it. The harness supplies the loop; consumers
|
|
176
|
-
* supply the mutations and the failure detector.
|
|
177
|
-
*/
|
|
178
|
-
interface AdversarialMutation<S> {
|
|
179
|
-
id: string;
|
|
180
|
-
/**
|
|
181
|
-
* Mutate one scenario. Return null to skip; return one or more new
|
|
182
|
-
* scenarios. The harness deduplicates by `mutateScenarioId(scenario)`.
|
|
183
|
-
*/
|
|
184
|
-
mutate(parent: S, rng: () => number): Promise<S[]> | S[];
|
|
185
|
-
}
|
|
186
|
-
|
|
187
|
-
/**
|
|
188
|
-
* Validator-output verdict — substrate primitive for "did this output pass,
|
|
189
|
-
* and how well?"
|
|
190
|
-
*
|
|
191
|
-
* Used by:
|
|
192
|
-
* - `@tangle-network/agent-eval/matrix` — verdict per cell in the cartesian.
|
|
193
|
-
* - `@tangle-network/agent-runtime` — Validator<Output, Verdict = DefaultVerdict>.
|
|
194
|
-
* Runtime keeps `Validator` because it's coupled to runtime-shaped
|
|
195
|
-
* `ValidationCtx` (iteration, signal, traceEmitter); the verdict TYPE
|
|
196
|
-
* itself is a substrate concept and lives here.
|
|
197
|
-
*
|
|
198
|
-
* Repo layering: agent-eval is the substrate (no upward deps). Both
|
|
199
|
-
* agent-runtime and agent-knowledge consume this type FROM agent-eval —
|
|
200
|
-
* never the other way around. See CLAUDE.md "Repo layering" for the rule.
|
|
201
|
-
*/
|
|
202
|
-
/**
|
|
203
|
-
* Minimal verdict shape — `valid` + `score` are required; `scores` +
|
|
204
|
-
* `notes` are optional surface. Validators that need richer shapes
|
|
205
|
-
* parameterise `Validator<Output, MyVerdict>` with their own type.
|
|
206
|
-
*
|
|
207
|
-
* Need structured extras? Extend DefaultVerdict with typed fields — never
|
|
208
|
-
* serialize extras into `notes`.
|
|
209
|
-
*/
|
|
210
|
-
interface DefaultVerdict {
|
|
211
|
-
/** Whether the output meets the validator's pass criteria. */
|
|
212
|
-
valid: boolean;
|
|
213
|
-
/** Aggregate score in [0, 1]. Drivers use this for winner selection. */
|
|
214
|
-
score: number;
|
|
215
|
-
/** Per-dimension scores. Free-form; weighted into `score` by the validator. */
|
|
216
|
-
scores?: Record<string, number>;
|
|
217
|
-
/** Human-readable rationale; surfaces in trace + final-result `winner.verdict`. */
|
|
218
|
-
notes?: string;
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
/**
|
|
222
|
-
* Behavior-space exploration — types.
|
|
223
|
-
*
|
|
224
|
-
* One engine searches a space of inputs against a target, scores each run with a
|
|
225
|
-
* multi-objective verdict, keeps a quality-diversity archive, and admits only
|
|
226
|
-
* findings that pass the validity gates. Adversarial fuzzing is the headline
|
|
227
|
-
* preset (`fuzzAgent`); swapping the `Objective` re-points the same engine at
|
|
228
|
-
* novelty search, curriculum growth, or user-simulation.
|
|
229
|
-
*
|
|
230
|
-
* Two kinds of coordinates, deliberately distinct:
|
|
231
|
-
* - INPUT axes (`space.axes`) are the stratification plan — enumerable up front,
|
|
232
|
-
* so allocation and the coverage denominator (planned vs covered) are honest.
|
|
233
|
-
* - MEASURED descriptors (`descriptor(scenario, ev)`) are read off the rollout —
|
|
234
|
-
* they bin the archive by what the agent DID, and never inflate the coverage
|
|
235
|
-
* denominator (a behavior you haven't seen yet is not a planned cell).
|
|
236
|
-
*
|
|
237
|
-
* An `Evaluation` IS a `DefaultVerdict` — same spine as judges and verifiers,
|
|
238
|
-
* never a parallel score shape.
|
|
239
|
-
*/
|
|
240
|
-
|
|
241
|
-
/** One input axis of the stratification plan (e.g. matterType, difficulty, personaRigor). */
|
|
242
|
-
interface SpaceAxis {
|
|
243
|
-
name: string;
|
|
244
|
-
values: string[];
|
|
245
|
-
}
|
|
246
|
-
/** The input space to stratify. Cells are the cartesian product of the axes. */
|
|
247
|
-
interface BehaviorSpace {
|
|
248
|
-
axes: SpaceAxis[];
|
|
249
|
-
}
|
|
250
|
-
/** One input cell: a coordinate in the stratification plan. */
|
|
251
|
-
interface Cell {
|
|
252
|
-
/** Stable id, e.g. `matterType=nda|difficulty=hard`. */
|
|
253
|
-
id: string;
|
|
254
|
-
coords: Record<string, string>;
|
|
255
|
-
}
|
|
256
|
-
/**
|
|
257
|
-
* The outcome of running the target against one scenario: a `DefaultVerdict`
|
|
258
|
-
* (`valid`, headline `score` in [0,1], per-dimension `scores`, `notes`) plus the
|
|
259
|
-
* fields exploration needs. Keep `scores` populated — the coverage map surfaces
|
|
260
|
-
* WHICH dimension is weak only when evaluations carry it.
|
|
261
|
-
*/
|
|
262
|
-
interface Evaluation extends DefaultVerdict {
|
|
263
|
-
/** Measured behavior coordinates (e.g. `{ outcome: 'refused' }`). Bins the archive. */
|
|
264
|
-
descriptor?: Record<string, string>;
|
|
265
|
-
/** Surfaced output — drives exemplars + minimization. */
|
|
266
|
-
output?: string;
|
|
267
|
-
/** RunRecord id when the target persisted a trace. */
|
|
268
|
-
runId?: string;
|
|
269
|
-
/** Structured labels, e.g. failure classes (`hallucination`, `refusal`). */
|
|
270
|
-
labels?: string[];
|
|
271
|
-
/** Wall-clock for the evaluation, when the consumer measures it more precisely
|
|
272
|
-
* than the engine can (e.g. excluding judge time). Engine-measured otherwise. */
|
|
273
|
-
latencyMs?: number;
|
|
274
|
-
}
|
|
275
|
-
/** Run the target against one scenario in a cell. */
|
|
276
|
-
type Evaluator<S> = (scenario: S, cell: Cell) => Promise<Evaluation>;
|
|
277
|
-
/** Context a proposer sees — prior elites + findings let a skill-backed proposer steer. */
|
|
278
|
-
interface ProposeContext<S> {
|
|
279
|
-
cell: Cell;
|
|
280
|
-
seeds: S[];
|
|
281
|
-
/** Current archive elites whose input cell matches. */
|
|
282
|
-
elites: S[];
|
|
283
|
-
/** Verified findings so far (read-only) — probe new gaps, not re-found ones. */
|
|
284
|
-
findings: ReadonlyArray<Finding<S>>;
|
|
285
|
-
/** How many candidates to propose. */
|
|
286
|
-
count: number;
|
|
287
|
-
rng: () => number;
|
|
288
|
-
}
|
|
289
|
-
/**
|
|
290
|
-
* Produces candidate scenarios for a cell. A plain function — `mutationProposer`
|
|
291
|
-
* builds one from mutation operators; an agent running a generator skill IS one
|
|
292
|
-
* (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed.
|
|
293
|
-
*
|
|
294
|
-
* Distinct from the optimization `SurfaceProposer` (`campaign/types`): that one
|
|
295
|
-
* is the proposer in the surface-optimization loop (`runOptimization`); this one
|
|
296
|
-
* is the scenario generator in the behavior-fuzzing loop. `SurfaceProposer` is
|
|
297
|
-
* THE optimization proposer.
|
|
298
|
-
*/
|
|
299
|
-
type MutationProposer<S> = (ctx: ProposeContext<S>) => Promise<S[]> | S[];
|
|
300
|
-
/**
|
|
301
|
-
* What "interesting" means. `interest` in [0,1]; a candidate is notable (gate-
|
|
302
|
-
* checked, reported) when `interest >= threshold`. `adversarialObjective` (low
|
|
303
|
-
* score is interesting) and `noveltyObjective` (far from the archive) ship.
|
|
304
|
-
*/
|
|
305
|
-
interface Objective {
|
|
306
|
-
kind: string;
|
|
307
|
-
interest(ev: Evaluation, ctx: ObjectiveContext): number;
|
|
308
|
-
/** Default 0.5. */
|
|
309
|
-
threshold?: number;
|
|
310
|
-
}
|
|
311
|
-
interface ObjectiveContext {
|
|
312
|
-
archiveScores: number[];
|
|
313
|
-
archiveDescriptors: Array<Record<string, string> | undefined>;
|
|
314
|
-
}
|
|
315
|
-
/**
|
|
316
|
-
* Validity gates — the moat. A notable candidate is admitted ONLY when it is a
|
|
317
|
-
* fair, answerable task (`isValid`) AND reproduces under a meaning-preserving
|
|
318
|
-
* rephrase (`isUncontaminated`). Pass-through by default (testable without an
|
|
319
|
-
* LLM); live wiring supplies real gates.
|
|
320
|
-
*/
|
|
321
|
-
interface ValidityGates<S> {
|
|
322
|
-
isValid?: (scenario: S, ev: Evaluation, cell: Cell) => boolean | Promise<boolean>;
|
|
323
|
-
isUncontaminated?: (scenario: S, ev: Evaluation, cell: Cell) => boolean | Promise<boolean>;
|
|
324
|
-
}
|
|
325
|
-
/** A gate-verified, minimized finding — the unit the capsule reports. */
|
|
326
|
-
interface Finding<S> {
|
|
327
|
-
id: string;
|
|
328
|
-
cell: Cell;
|
|
329
|
-
scenario: S;
|
|
330
|
-
/** The minimized trigger (== `scenario` when no minimizer is supplied). */
|
|
331
|
-
minimized: S;
|
|
332
|
-
/** Legible text of the minimized trigger, when `scenarioText` is supplied. */
|
|
333
|
-
text?: string;
|
|
334
|
-
/** The full multi-objective verdict. */
|
|
335
|
-
evaluation: Evaluation;
|
|
336
|
-
/** The objective's interest score that flagged it. */
|
|
337
|
-
interest: number;
|
|
338
|
-
/** Which objective flagged it. */
|
|
339
|
-
objective: string;
|
|
340
|
-
}
|
|
341
|
-
/** An archive elite — the most interesting scenario seen for one bin. */
|
|
342
|
-
interface ArchiveEntry<S> {
|
|
343
|
-
/** Input cell + measured descriptor coords combined, e.g. `difficulty=hard|outcome=refused`. */
|
|
344
|
-
binId: string;
|
|
345
|
-
cell: Cell;
|
|
346
|
-
scenario: S;
|
|
347
|
-
evaluation: Evaluation;
|
|
348
|
-
interest: number;
|
|
349
|
-
}
|
|
350
|
-
/** Summary of a sample — every aggregate carries its spread, never a bare mean. */
|
|
351
|
-
interface Distribution {
|
|
352
|
-
mean: number;
|
|
353
|
-
median: number;
|
|
354
|
-
p90: number;
|
|
355
|
-
min: number;
|
|
356
|
-
max: number;
|
|
357
|
-
n: number;
|
|
358
|
-
}
|
|
359
|
-
/** Per-INPUT-cell coverage — the planned-vs-covered map. */
|
|
360
|
-
interface CoverageCell {
|
|
361
|
-
cell: Cell;
|
|
362
|
-
runs: number;
|
|
363
|
-
/** Headline score distribution in [0,1]; `null` when the cell was never run
|
|
364
|
-
* (honestly uncovered — never a fabricated zero). */
|
|
365
|
-
score: Distribution | null;
|
|
366
|
-
/** Fraction of runs the objective flagged as notable. */
|
|
367
|
-
findingRate: number;
|
|
368
|
-
/** Per-dimension score distributions — surfaces WHICH dimension is weak and
|
|
369
|
-
* how consistently. */
|
|
370
|
-
dimensions: Record<string, Distribution>;
|
|
371
|
-
/** Evaluation wall-clock per run; engine-measured unless the evaluation
|
|
372
|
-
* carried its own `latencyMs`. `null` when the cell was never run. */
|
|
373
|
-
latencyMs: Distribution | null;
|
|
374
|
-
/** Known dollars spent in this cell — present only when cost tracking was
|
|
375
|
-
* wired; runs with unknown cost are counted apart, never folded in as $0. */
|
|
376
|
-
costUsd?: number;
|
|
377
|
-
costUnknownRuns?: number;
|
|
378
|
-
}
|
|
379
|
-
/** The artifact every exploration produces. */
|
|
380
|
-
interface CapsuleData<S> {
|
|
381
|
-
target: string;
|
|
382
|
-
objective: string;
|
|
383
|
-
/** Stamped by the caller — the engine stays clock-free and deterministic. */
|
|
384
|
-
generatedAt?: string;
|
|
385
|
-
coverage: CoverageCell[];
|
|
386
|
-
/** Verified findings, sorted by descending interest. */
|
|
387
|
-
findings: Finding<S>[];
|
|
388
|
-
/** QD archive elites (binned by input × measured coords). */
|
|
389
|
-
archive: ArchiveEntry<S>[];
|
|
390
|
-
/** Post-harden lift, filled by a second pass after an improvement. */
|
|
391
|
-
lift?: {
|
|
392
|
-
before: number;
|
|
393
|
-
after: number;
|
|
394
|
-
verdict: string;
|
|
395
|
-
};
|
|
396
|
-
stats: {
|
|
397
|
-
totalRuns: number;
|
|
398
|
-
/** Input-cell denominator — the stratification plan. */
|
|
399
|
-
cellsTotal: number;
|
|
400
|
-
cellsCovered: number;
|
|
401
|
-
/** Distinct measured-descriptor bins observed (never part of the denominator). */
|
|
402
|
-
behaviorBinsObserved: number;
|
|
403
|
-
candidateFindings: number;
|
|
404
|
-
verifiedFindings: number;
|
|
405
|
-
/** Distribution of per-cell mean scores across covered cells (cells weigh
|
|
406
|
-
* equally — variance steering sends more runs to weak cells, so a
|
|
407
|
-
* run-weighted average would bias low). `null` when nothing ran. */
|
|
408
|
-
robustness: Distribution | null;
|
|
409
|
-
/** Evaluation wall-clock across all runs. `null` when nothing ran. */
|
|
410
|
-
latencyMs: Distribution | null;
|
|
411
|
-
/** Known dollars spent on this exploration's runs. Present only when cost
|
|
412
|
-
* tracking was wired (`costOf`) — absent means "not tracked", never $0. */
|
|
413
|
-
costUsd?: number;
|
|
414
|
-
/** Runs whose cost was unknown (`costOf` returned null) — counted apart,
|
|
415
|
-
* never folded into `costUsd` as a fabricated $0. */
|
|
416
|
-
costUnknownRuns?: number;
|
|
417
|
-
/** Evaluations that threw (transport/backend failures). They consumed no
|
|
418
|
-
* run budget and scored nothing — an infra axis, never folded into
|
|
419
|
-
* robustness or reported as findings. */
|
|
420
|
-
evalErrors: number;
|
|
421
|
-
/** Present when the run stopped before its budget because consecutive
|
|
422
|
-
* eval errors tripped the circuit breaker (a dead backend must not burn
|
|
423
|
-
* the remaining budget). The capsule-so-far is complete and honest. */
|
|
424
|
-
stoppedEarly?: {
|
|
425
|
-
reason: 'eval-errors';
|
|
426
|
-
detail: string;
|
|
427
|
-
};
|
|
428
|
-
};
|
|
429
|
-
}
|
|
430
|
-
/**
|
|
431
|
-
* Known cost of one evaluated run. `model` attributes the spend in the ledger's
|
|
432
|
-
* per-model rollup; absent, the entry is labeled `unattributed` (the dollars are
|
|
433
|
-
* real either way — recorded as `actualCostUsd`, never an estimate).
|
|
434
|
-
*/
|
|
435
|
-
interface RunCost {
|
|
436
|
-
usd: number;
|
|
437
|
-
model?: string;
|
|
438
|
-
}
|
|
439
|
-
type ExploreEvent<S> = {
|
|
440
|
-
type: 'cell-allocated';
|
|
441
|
-
cell: Cell;
|
|
442
|
-
count: number;
|
|
443
|
-
} | {
|
|
444
|
-
type: 'evaluated';
|
|
445
|
-
cell: Cell;
|
|
446
|
-
scenario: S;
|
|
447
|
-
evaluation: Evaluation;
|
|
448
|
-
} | {
|
|
449
|
-
type: 'finding';
|
|
450
|
-
finding: Finding<S>;
|
|
451
|
-
} | {
|
|
452
|
-
type: 'eval-error';
|
|
453
|
-
cell: Cell;
|
|
454
|
-
scenarioId: string;
|
|
455
|
-
message: string;
|
|
456
|
-
} | {
|
|
457
|
-
type: 'round';
|
|
458
|
-
runsUsed: number;
|
|
459
|
-
budget: number;
|
|
460
|
-
};
|
|
461
|
-
interface ExploreOptions<S> {
|
|
462
|
-
/** Name of the target under exploration — labels the capsule. */
|
|
463
|
-
target: string;
|
|
464
|
-
/** The input stratification plan. */
|
|
465
|
-
space: BehaviorSpace;
|
|
466
|
-
/** Candidate generator. */
|
|
467
|
-
proposer: MutationProposer<S>;
|
|
468
|
-
/** Runs the target → multi-objective `Evaluation`. */
|
|
469
|
-
evaluate: Evaluator<S>;
|
|
470
|
-
/** Seed corpus per cell. */
|
|
471
|
-
seedsFor: (cell: Cell) => S[] | Promise<S[]>;
|
|
472
|
-
/** Stable id for a scenario (dedup + lineage). */
|
|
473
|
-
scenarioId: (scenario: S) => string;
|
|
474
|
-
/** Human-legible text — drives capsule exemplars. */
|
|
475
|
-
scenarioText?: (scenario: S) => string;
|
|
476
|
-
/** Measured behavior coords appended to the archive bin. Default: input cell only. */
|
|
477
|
-
descriptor?: (scenario: S, ev: Evaluation) => Record<string, string>;
|
|
478
|
-
/** What "interesting" means. Default: `adversarialObjective(0.5)`. */
|
|
479
|
-
objective?: Objective;
|
|
480
|
-
/** Validity gates. Default pass-through. */
|
|
481
|
-
gates?: ValidityGates<S>;
|
|
482
|
-
/** Budget steering across input cells. `variance` chases uncertainty; `uniform` is the unsteered ablation baseline. Default `variance`. */
|
|
483
|
-
allocation?: 'variance' | 'uniform';
|
|
484
|
-
/** Total target evaluations. */
|
|
485
|
-
budget: number;
|
|
486
|
-
/** Minimum evaluations per input cell before steering. Default 2. */
|
|
487
|
-
floorPerCell?: number;
|
|
488
|
-
/** Shrink a notable scenario to its minimal trigger. Default: identity. */
|
|
489
|
-
minimize?: (scenario: S, evaluate: Evaluator<S>, cell: Cell) => Promise<S> | S;
|
|
490
|
-
/** Max concurrent `evaluate` calls. Default 1. */
|
|
491
|
-
concurrency?: number;
|
|
492
|
-
/** Stop the run after this many CONSECUTIVE eval errors (a dead backend must
|
|
493
|
-
* not burn the remaining budget). Successes reset the streak. Default 5. */
|
|
494
|
-
maxConsecutiveEvalErrors?: number;
|
|
495
|
-
/** Cooperative cancellation. */
|
|
496
|
-
signal?: AbortSignal;
|
|
497
|
-
/** Progress stream. */
|
|
498
|
-
onProgress?: (event: ExploreEvent<S>) => void;
|
|
499
|
-
/** Deterministic seed. Default 1. */
|
|
500
|
-
seed?: number;
|
|
501
|
-
/**
|
|
502
|
-
* Cost of one evaluated run — consumer-supplied; the explorer cannot know
|
|
503
|
-
* token usage. Return null when the cost is unknown: the run is COUNTED in
|
|
504
|
-
* `stats.costUnknownRuns`, never folded into the total as $0. Required by
|
|
505
|
-
* every other cost option (`costBudgetUsd` / `ledger` / `onCost`).
|
|
506
|
-
*/
|
|
507
|
-
costOf?: (scenario: S, cell: Cell, ev: Evaluation) => RunCost | null;
|
|
508
|
-
/** Pre-call hard maximum. Required whenever the explorer uses a capped ledger. */
|
|
509
|
-
maximumChargeOf?: (scenario: S, cell: Cell) => MaximumCharge;
|
|
510
|
-
/**
|
|
511
|
-
* Hard dollar cap. Each evaluation reserves `maximumChargeOf` before it starts;
|
|
512
|
-
* calls that do not fit are rejected. Unknown totals stop further paid work.
|
|
513
|
-
*/
|
|
514
|
-
costBudgetUsd?: number;
|
|
515
|
-
/**
|
|
516
|
-
* Sink for per-run cost entries — each known `costOf` result is recorded
|
|
517
|
-
* with channel 'agent' and `actualCostUsd` (token axes are zero: the
|
|
518
|
-
* explorer only sees dollars). Pass the program's shared `CostLedger` so
|
|
519
|
-
* `costReport` stamps fuzz spend alongside judge/analyst spend.
|
|
520
|
-
*/
|
|
521
|
-
ledger?: CostLedgerHandle;
|
|
522
|
-
/** Observer fired for every known-cost run recorded. */
|
|
523
|
-
onCost?: (entry: {
|
|
524
|
-
usd: number;
|
|
525
|
-
channel: CostChannel;
|
|
526
|
-
}) => void;
|
|
527
|
-
}
|
|
528
|
-
|
|
529
|
-
/**
|
|
530
|
-
* Input-space tiling + coverage projection.
|
|
531
|
-
*
|
|
532
|
-
* Cells are the cartesian product of the input axes — the stratification plan,
|
|
533
|
-
* enumerable up front so the planned-vs-covered denominator is honest. Coverage
|
|
534
|
-
* is projected from the evaluation log: per cell, the full DISTRIBUTION of the
|
|
535
|
-
* headline score, of each scored dimension, and of evaluation latency — a bare
|
|
536
|
-
* mean hides outliers, so every aggregate carries its spread. Per-cell cost is
|
|
537
|
-
* split known-dollars vs unknown-runs, never folded into a fabricated $0.
|
|
538
|
-
*/
|
|
539
|
-
|
|
540
|
-
/** One recorded evaluation — the unit coverage and the capsule are built from. */
|
|
541
|
-
interface EvalRecord {
|
|
542
|
-
cell: Cell;
|
|
543
|
-
ev: Evaluation;
|
|
544
|
-
/** The objective's interest score for this evaluation. */
|
|
545
|
-
interest: number;
|
|
546
|
-
/** Evaluation wall-clock — engine-measured unless `ev.latencyMs` overrode it. */
|
|
547
|
-
latencyMs: number;
|
|
548
|
-
/** Known dollars for this run. `null` = cost tracking was wired but this
|
|
549
|
-
* run's cost was unknowable (counted apart). Absent = not tracked at all. */
|
|
550
|
-
costUsd?: number | null;
|
|
551
|
-
}
|
|
552
|
-
/** Enumerate every input cell (cartesian product of the axes), in stable order. */
|
|
553
|
-
declare function enumerateCells(space: BehaviorSpace): Cell[];
|
|
554
|
-
/** Deterministic id for a coordinate map, e.g. `matterType=nda|difficulty=hard`. */
|
|
555
|
-
declare function cellId(space: BehaviorSpace, coords: Record<string, string>): string;
|
|
556
|
-
/**
|
|
557
|
-
* Project the evaluation log into the per-input-cell coverage map. A cell with
|
|
558
|
-
* no evaluations reports `score: null` (honestly uncovered), never zeros.
|
|
559
|
-
*/
|
|
560
|
-
declare function buildCoverage(cells: Cell[], log: EvalRecord[], threshold: number): CoverageCell[];
|
|
561
|
-
|
|
562
|
-
/**
|
|
563
|
-
* The capsule — the artifact every exploration produces.
|
|
564
|
-
*
|
|
565
|
-
* `buildCapsule` assembles coverage + verified findings + the QD archive into a
|
|
566
|
-
* pure `CapsuleData` (no clock, no I/O — deterministic and snapshot-testable).
|
|
567
|
-
* `renderCapsuleHtml` turns it into a standalone page: the input-cell heat-map
|
|
568
|
-
* (planned vs covered), per-dimension weakness chips, and the minimized finding
|
|
569
|
-
* exemplars. One artifact — the hardening map and the shareable proof object.
|
|
570
|
-
*/
|
|
571
|
-
|
|
572
|
-
interface BuildCapsuleInput<S> {
|
|
573
|
-
target: string;
|
|
574
|
-
objective: string;
|
|
575
|
-
cells: Cell[];
|
|
576
|
-
log: EvalRecord[];
|
|
577
|
-
/** The objective's notable threshold — drives findingRate. */
|
|
578
|
-
threshold: number;
|
|
579
|
-
archive: ArchiveEntry<S>[];
|
|
580
|
-
findings: Finding<S>[];
|
|
581
|
-
candidateFindings: number;
|
|
582
|
-
runsUsed: number;
|
|
583
|
-
/** Known-dollar / unknown-run split — present only when cost tracking was
|
|
584
|
-
* wired; the capsule never fabricates a $0 total. */
|
|
585
|
-
cost?: {
|
|
586
|
-
costUsd: number;
|
|
587
|
-
costUnknownRuns: number;
|
|
588
|
-
};
|
|
589
|
-
/** Evaluations that threw — infra outcomes, never folded into robustness. */
|
|
590
|
-
evalErrors: number;
|
|
591
|
-
/** Set when the consecutive-error circuit breaker stopped the run early. */
|
|
592
|
-
stoppedEarly?: {
|
|
593
|
-
reason: 'eval-errors';
|
|
594
|
-
detail: string;
|
|
595
|
-
};
|
|
596
|
-
}
|
|
597
|
-
declare function buildCapsule<S>(input: BuildCapsuleInput<S>): CapsuleData<S>;
|
|
598
|
-
interface RenderCapsuleOptions {
|
|
599
|
-
/** Max finding exemplars to show. Default 8. */
|
|
600
|
-
maxFindings?: number;
|
|
601
|
-
/** ISO timestamp to stamp into the page (keeps the pure capsule clock-free). */
|
|
602
|
-
generatedAt?: string;
|
|
603
|
-
}
|
|
604
|
-
/** Render a self-contained HTML capsule — heat-map + per-dimension chips + verified findings. */
|
|
605
|
-
declare function renderCapsuleHtml<S>(capsule: CapsuleData<S>, opts?: RenderCapsuleOptions): string;
|
|
606
|
-
|
|
607
|
-
/**
|
|
608
|
-
* The exploration engine — a stateful session over a behavior space.
|
|
609
|
-
*
|
|
610
|
-
* Each `step()`: allocate budget across INPUT cells (floor first, then variance
|
|
611
|
-
* steering toward the least-certain cells), propose candidates (the proposer
|
|
612
|
-
* reads current elites + findings, so the search deepens generationally),
|
|
613
|
-
* evaluate with bounded concurrency, archive the most interesting scenario per
|
|
614
|
-
* input×measured bin, and admit notable candidates that pass the validity gates.
|
|
615
|
-
* `run()` loops to budget. `coverage()`/`findings()`/`capsule()` read live state —
|
|
616
|
-
* the surface `makeExploreTools` exposes so an agent can drive the session.
|
|
617
|
-
*
|
|
618
|
-
* One evaluation log (`EvalRecord[]`) is the source of truth; allocation
|
|
619
|
-
* observations and coverage are projections of it.
|
|
620
|
-
*/
|
|
621
|
-
|
|
622
|
-
declare class BehaviorExplorer<S> {
|
|
623
|
-
private readonly opts;
|
|
624
|
-
private readonly cells;
|
|
625
|
-
private readonly cellById;
|
|
626
|
-
private readonly objective;
|
|
627
|
-
private readonly threshold;
|
|
628
|
-
private readonly floorPerCell;
|
|
629
|
-
private readonly perRoundBudget;
|
|
630
|
-
/** The single evaluation log — coverage + allocation are projections of it. */
|
|
631
|
-
private readonly log;
|
|
632
|
-
/** binId (input × measured coords) → the most interesting entry seen. */
|
|
633
|
-
private readonly archiveByBin;
|
|
634
|
-
private readonly _findings;
|
|
635
|
-
private runsUsed;
|
|
636
|
-
private candidateFindings;
|
|
637
|
-
private evalErrors;
|
|
638
|
-
private consecutiveEvalErrors;
|
|
639
|
-
private stoppedEarly;
|
|
640
|
-
private rngState;
|
|
641
|
-
private readonly costLedger?;
|
|
642
|
-
private readonly costPhase;
|
|
643
|
-
constructor(opts: ExploreOptions<S>);
|
|
644
|
-
private rng;
|
|
645
|
-
private binId;
|
|
646
|
-
private allocate;
|
|
647
|
-
private objectiveContext;
|
|
648
|
-
/** Elites whose INPUT cell matches — what the proposer mutates/deepens from. */
|
|
649
|
-
private elitesFor;
|
|
650
|
-
/** One allocate → propose → evaluate → gate → archive round. */
|
|
651
|
-
step(): Promise<{
|
|
652
|
-
runs: number;
|
|
653
|
-
findings: Finding<S>[];
|
|
654
|
-
}>;
|
|
655
|
-
/** Loop `step()` until the run or dollar budget is spent, the signal aborts,
|
|
656
|
-
* or no progress is made. */
|
|
657
|
-
run(): Promise<CapsuleData<S>>;
|
|
658
|
-
coverage(): CoverageCell[];
|
|
659
|
-
findings(): Finding<S>[];
|
|
660
|
-
capsule(): CapsuleData<S>;
|
|
661
|
-
}
|
|
662
|
-
|
|
663
|
-
/**
|
|
664
|
-
* `fuzzAgent` — the adversarial batch preset over `BehaviorExplorer`.
|
|
665
|
-
*
|
|
666
|
-
* One call: explore the space to budget with the adversarial objective and
|
|
667
|
-
* return the capsule. For agent-driven, incremental, or multi-objective use,
|
|
668
|
-
* construct a `BehaviorExplorer` and drive it via `makeExploreTools`.
|
|
669
|
-
*/
|
|
670
|
-
|
|
671
|
-
type FuzzAgentOptions<S> = Omit<ExploreOptions<S>, 'objective'> & {
|
|
672
|
-
/** Score strictly below this is a candidate failure. Default 0.5. */
|
|
673
|
-
failureThreshold?: number;
|
|
674
|
-
};
|
|
675
|
-
declare function fuzzAgent<S>(opts: FuzzAgentOptions<S>): Promise<{
|
|
676
|
-
capsule: CapsuleData<S>;
|
|
677
|
-
}>;
|
|
678
|
-
|
|
679
|
-
/**
|
|
680
|
-
* Validity gates — what separates a fuzzer from a slop generator.
|
|
681
|
-
*
|
|
682
|
-
* A notable candidate is admitted only when it is fair and reproducible. None of
|
|
683
|
-
* these are on by default: the live wiring opts in, so reported findings carry
|
|
684
|
-
* their proof.
|
|
685
|
-
*/
|
|
686
|
-
|
|
687
|
-
/** Combine gate sets; a candidate must pass every gate in every set. */
|
|
688
|
-
declare function composeGates<S>(...sets: Array<ValidityGates<S> | undefined>): ValidityGates<S>;
|
|
689
|
-
/**
|
|
690
|
-
* Reproducibility gate. Re-run the target on a meaning-preserving rephrase of the
|
|
691
|
-
* flagged scenario; keep the finding only when the rephrase ALSO scores below the
|
|
692
|
-
* threshold. A finding that flips under a cosmetic rewrite was keyed to surface
|
|
693
|
-
* form, not the task — a false signal we must not report. Costs one extra
|
|
694
|
-
* evaluation per candidate (candidates are rare, so cheap).
|
|
695
|
-
*/
|
|
696
|
-
declare function perturbationStabilityGate<S>(opts: {
|
|
697
|
-
evaluate: Evaluator<S>;
|
|
698
|
-
/** Produce a semantic-preserving rephrase. Return null to skip (treated as pass). */
|
|
699
|
-
perturb: (scenario: S) => S | null;
|
|
700
|
-
/** Score strictly below this still counts as failing. Default 0.5. */
|
|
701
|
-
failureThreshold?: number;
|
|
702
|
-
}): ValidityGates<S>;
|
|
703
|
-
/**
|
|
704
|
-
* Severity-floor gate. Reject borderline candidates whose score sits in a band
|
|
705
|
-
* just under the threshold — judge noise, not a real defect.
|
|
706
|
-
*/
|
|
707
|
-
declare function severityFloorGate<S>(opts: {
|
|
708
|
-
failureThreshold?: number;
|
|
709
|
-
margin?: number;
|
|
710
|
-
}): ValidityGates<S>;
|
|
711
|
-
|
|
712
|
-
/**
|
|
713
|
-
* Shipped policies for the exploration engine.
|
|
714
|
-
*
|
|
715
|
-
* `MutationProposer` is a plain function type — an agent running a generator skill
|
|
716
|
-
* IS a proposer (`(ctx) => dispatchToSkill(ctx)`), no wrapper needed. `mutationProposer`
|
|
717
|
-
* builds the deterministic, LLM-free one from mutation operators. Objectives are
|
|
718
|
-
* interfaces because the engine reads `kind` + `threshold` off them.
|
|
719
|
-
*/
|
|
720
|
-
|
|
721
|
-
/**
|
|
722
|
-
* Perturbation-based search: apply the cell's mutation operators to the current
|
|
723
|
-
* elites + seeds, deduping by id. Elites first — mutating the most interesting
|
|
724
|
-
* scenario found so far is what makes the search deepen across rounds.
|
|
725
|
-
*/
|
|
726
|
-
declare function mutationProposer<S>(opts: {
|
|
727
|
-
mutationsFor: (cell: Cell) => AdversarialMutation<S>[];
|
|
728
|
-
scenarioId: (s: S) => string;
|
|
729
|
-
}): MutationProposer<S>;
|
|
730
|
-
/** Adversarial: a low headline score is interesting — find where the agent fails. */
|
|
731
|
-
declare function adversarialObjective(threshold?: number): Objective;
|
|
732
|
-
/**
|
|
733
|
-
* Novelty: interesting when far from the archive in score AND measured behavior
|
|
734
|
-
* descriptor — quality-diversity's diversity pressure; drives corpus growth
|
|
735
|
-
* rather than re-finding the same hole.
|
|
736
|
-
*/
|
|
737
|
-
declare function noveltyObjective(threshold?: number): Objective;
|
|
738
|
-
|
|
739
|
-
/**
|
|
740
|
-
* Agent-drivable surface over a live exploration session.
|
|
741
|
-
*
|
|
742
|
-
* Framework-neutral tool defs ({name, description, parameters: JSON Schema,
|
|
743
|
-
* handler}) so the on-demand agent — not a batch script — drives the search:
|
|
744
|
-
* step it, read coverage, inspect findings, render the capsule. Transport
|
|
745
|
-
* encodings (OpenAI function shape, MCP) are one-line mappings the host owns.
|
|
746
|
-
*/
|
|
747
|
-
|
|
748
|
-
interface ExploreToolDef {
|
|
749
|
-
name: string;
|
|
750
|
-
description: string;
|
|
751
|
-
/** JSON Schema (draft-07+) for the arguments. */
|
|
752
|
-
parameters: Record<string, unknown>;
|
|
753
|
-
handler: (args: unknown, ctx?: {
|
|
754
|
-
signal?: AbortSignal;
|
|
755
|
-
}) => Promise<unknown>;
|
|
756
|
-
}
|
|
757
|
-
declare function makeExploreTools<S>(explorer: BehaviorExplorer<S>): ExploreToolDef[];
|
|
758
|
-
|
|
759
|
-
export { type ArchiveEntry, BehaviorExplorer, type BehaviorSpace, type BuildCapsuleInput, type CapsuleData, type Cell, type CoverageCell, type EvalRecord, type Evaluation, type Evaluator, type ExploreEvent, type ExploreOptions, type ExploreToolDef, type Finding, type FuzzAgentOptions, type MutationProposer, type Objective, type ObjectiveContext, type ProposeContext, type RenderCapsuleOptions, type RunCost, type SpaceAxis, type ValidityGates, adversarialObjective, buildCapsule, buildCoverage, cellId, composeGates, enumerateCells, fuzzAgent, makeExploreTools, mutationProposer, noveltyObjective, perturbationStabilityGate, renderCapsuleHtml, severityFloorGate };
|