@vostride/agent-qa-eval-local 0.0.0-stage → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +105 -0
- package/NOTICE.md +11 -0
- package/dist/index.cjs +7893 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +2930 -0
- package/dist/index.d.ts +2930 -0
- package/dist/index.js +7903 -0
- package/dist/index.js.map +1 -0
- package/dist/reader.cjs +753 -0
- package/dist/reader.cjs.map +1 -0
- package/dist/reader.d.cts +74 -0
- package/dist/reader.d.ts +74 -0
- package/dist/reader.js +744 -0
- package/dist/reader.js.map +1 -0
- package/package.json +69 -4
- package/README.md +0 -3
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,2930 @@
|
|
|
1
|
+
import * as _vostride_agent_qa_eval from '@vostride/agent-qa-eval';
|
|
2
|
+
import { AdversarialCapSource, AdversarialMutator, ProbeGeneratorKind, GeneratorKind, ProbeRefusal, ProbeInapplicability, ProbeAuthorPort, AuthoringModel, NoProbeModelGap, AdversarialSeedSkip, LeafCassette, StandaloneLabelRecord, AdversarialCorpusEntry, JudgePanelPort, JudgePort, JudgeCalibrationEvidence, JudgeCalibrationDiagnostics, JudgeTargetedCalibrationMember, BaselineProfileKeyInput, JudgeIdentityBlock, JudgeCalibrationMinimum, UsageAggregator, JudgeTargetedCalibrationDisclosure, LabelCalibrationReport, CommittedCorpusSnapshot, ExtractedBlob, SeedingPlan, ExtractedCassette, PairSkip, SilverExclusion, PartitionManifest, RunExclusionRecord, HandoffSource, ArtifactSource, Cassette, ReplayRefusal, CassetteScore, DecisionDiff, ScoringContext, ReplayCassetteOutcome, DeclaredBriefDelta, ReplayClock, ReplayEscalationPredicate, SeedPolicy, ScoringJudgeRequest, BASELINE_POINTER_SCHEMA_VERSION, CorpusEntry, EvalBatchReport, DecisionVectorsDocument, NoiseFloorArtefact, ReplaySample, ScoredBatchEntry, FloorRefusal, DecisionCorrectnessVector, ClassifierRepeatability, GateVerdict, BaselinePointer, BaselineComparatorRefusal, CorpusCensus, JudgeCapabilityInput, CanaryEvidenceSummary, GatePosture, LabelCalibrationInput, CalibrationObservation } from '@vostride/agent-qa-eval';
|
|
3
|
+
export { ArtifactSource, HandoffSource } from '@vostride/agent-qa-eval';
|
|
4
|
+
import * as _vostride_agent_qa_schema from '@vostride/agent-qa-schema';
|
|
5
|
+
import { MaskFn, CompletedRunHandoff, MarkReceipt, MarkAddress, PerTestOutcome, CandidateInjection, CrossCheck, Gate1Receipt, Gate1KSource, Benefit, ScopeKey } from '@vostride/agent-qa-schema';
|
|
6
|
+
import { z } from 'zod';
|
|
7
|
+
import { ModelProfile, ResolvedModelAuth, ModelProviderOverrides, ModelCallObservation } from '@vostride/agent-qa-model';
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Durable refresh state lives BESIDE the staging tree, never inside it: a staged batch
|
|
11
|
+
* directory is reviewable data an operator may delete after admitting or rejecting it, and the
|
|
12
|
+
* record of what was already decided must outlive that.
|
|
13
|
+
*/
|
|
14
|
+
declare const ADVERSARIAL_STATE_SUBDIR = "eval/adversarial-state";
|
|
15
|
+
declare const ADVERSARIAL_STATE_SCHEMA_VERSION = 2;
|
|
16
|
+
/**
|
|
17
|
+
* What a completed `(seed, mutator)` tuple resolved to.
|
|
18
|
+
*
|
|
19
|
+
* Both are COMPLETED work. `derived` produced reviewable bytes; `inapplicable` is the
|
|
20
|
+
* generator's own typed statement that the family cannot act on this seed. A tuple that is
|
|
21
|
+
* merely DEFERRED (`cap-exceeded`) or that was already settled earlier (`already-derived`) is not
|
|
22
|
+
* an outcome and never lands here — recording either would settle work nobody did.
|
|
23
|
+
*/
|
|
24
|
+
type AdversarialTupleOutcome = 'derived' | 'inapplicable';
|
|
25
|
+
interface AdversarialRefreshTuple {
|
|
26
|
+
seedCassetteId: string;
|
|
27
|
+
mutator: AdversarialMutator;
|
|
28
|
+
/** The derived id this tuple addresses — stable across outcomes, because it is a function of the generation, not of the result (D-08). */
|
|
29
|
+
cassetteId: string;
|
|
30
|
+
outcome: AdversarialTupleOutcome;
|
|
31
|
+
/** The inapplicability reason for `inapplicable`; absent for `derived`. */
|
|
32
|
+
detail?: string;
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* The resolved cap and where the operator set it — recorded so a refresh can explain itself.
|
|
36
|
+
*
|
|
37
|
+
* 2026-09-12, 422-22: `source` names the NEUTRAL vocabulary rather than re-spelling it. This
|
|
38
|
+
* record is persisted, and a storage-side copy of a persisted vocabulary is a second list to keep
|
|
39
|
+
* in step with the first — which is how one of them ends up admitting an origin the other refuses.
|
|
40
|
+
*/
|
|
41
|
+
interface AdversarialSeedCap {
|
|
42
|
+
value: number;
|
|
43
|
+
source: AdversarialCapSource;
|
|
44
|
+
}
|
|
45
|
+
interface AdversarialRefreshRecord {
|
|
46
|
+
schemaVersion: typeof ADVERSARIAL_STATE_SCHEMA_VERSION;
|
|
47
|
+
product: string;
|
|
48
|
+
/** The frozen table generation these outcomes were decided under. */
|
|
49
|
+
tableVersion: number;
|
|
50
|
+
/** The committed corpus digest this refresh read its seeds from. */
|
|
51
|
+
sourceDigest: string;
|
|
52
|
+
cap: AdversarialSeedCap;
|
|
53
|
+
/** Novel eligible seeds counted BEFORE the cap was applied. */
|
|
54
|
+
eligibleSeeds: number;
|
|
55
|
+
/** Novel eligible seeds the cap deferred to a later refresh. */
|
|
56
|
+
excessSeeds: number;
|
|
57
|
+
/** Immutable batch membership: the seed ids this refresh selected, ascending. */
|
|
58
|
+
seedIds: readonly string[];
|
|
59
|
+
tuples: readonly AdversarialRefreshTuple[];
|
|
60
|
+
}
|
|
61
|
+
/** The record's own content address — its file name, and the whole of its identity. */
|
|
62
|
+
declare function adversarialRefreshRecordDigest(record: AdversarialRefreshRecord): string;
|
|
63
|
+
/** The settled-tuple key. Public so a caller can build the same key it will be checked against. */
|
|
64
|
+
declare function adversarialTupleKey(seedCassetteId: string, mutator: AdversarialMutator): string;
|
|
65
|
+
/**
|
|
66
|
+
* Resolve the per-refresh seed cap (D-04).
|
|
67
|
+
*
|
|
68
|
+
* Every rejected boundary is rejected LOUDLY: zero, negative, fractional, nonfinite and
|
|
69
|
+
* beyond-safe-integer caps are a TypeError rather than a silent fallback to the default. A cap
|
|
70
|
+
* that quietly becomes 8 because the operator typed `0` is a cap the operator does not have.
|
|
71
|
+
*/
|
|
72
|
+
declare function resolveAdversarialSeedCap(input: {
|
|
73
|
+
value?: number;
|
|
74
|
+
source?: Exclude<AdversarialCapSource, 'default'>;
|
|
75
|
+
}): AdversarialSeedCap;
|
|
76
|
+
/**
|
|
77
|
+
* Every published refresh record for one product, in file-name order.
|
|
78
|
+
*
|
|
79
|
+
* An absent tree is an empty history, not a failure — the first refresh on a host has no state
|
|
80
|
+
* and must not be told the store is broken. A record that fails the strict schema (one written
|
|
81
|
+
* before Phase 441 carries a policy digest) is absent too: D-20 migrates nothing (441-64, CR-C02).
|
|
82
|
+
* Unparseable bytes, and a record that parses but whose digest name or product does not match,
|
|
83
|
+
* are still failures.
|
|
84
|
+
*/
|
|
85
|
+
declare function readAdversarialRefreshRecords(input: {
|
|
86
|
+
agentQaRoot: string;
|
|
87
|
+
product: string;
|
|
88
|
+
}): AdversarialRefreshRecord[];
|
|
89
|
+
/**
|
|
90
|
+
* Publish one refresh record write-once, under its own content address.
|
|
91
|
+
*
|
|
92
|
+
* `unchanged` means the identical record was already published — the concurrent-refresh case,
|
|
93
|
+
* and the re-run-over-an-unchanged-corpus case. Anything else under that name is a conflict.
|
|
94
|
+
*/
|
|
95
|
+
declare function publishAdversarialRefreshRecord(input: {
|
|
96
|
+
agentQaRoot: string;
|
|
97
|
+
record: AdversarialRefreshRecord;
|
|
98
|
+
}): 'published' | 'unchanged';
|
|
99
|
+
/**
|
|
100
|
+
* The `(seed, mutator)` tuples already decided under THIS generation.
|
|
101
|
+
*
|
|
102
|
+
* A refresh skips these, which is what lets its cap reach the seeds behind them. Scoping by the
|
|
103
|
+
* generation is the whole contract: a new generation does not inherit a decision that was not
|
|
104
|
+
* made about it.
|
|
105
|
+
*/
|
|
106
|
+
declare function settledAdversarialTuples(records: readonly AdversarialRefreshRecord[], scope: {
|
|
107
|
+
tableVersion: number;
|
|
108
|
+
}): ReadonlySet<string>;
|
|
109
|
+
/**
|
|
110
|
+
* Every id this product has EVER derived, across all generations.
|
|
111
|
+
*
|
|
112
|
+
* Unscoped on purpose (D-08): a generation bump creates new ids beside the old, and the old
|
|
113
|
+
* ones are committed history. Re-deriving one would address bytes that already exist, so this
|
|
114
|
+
* set only ever grows.
|
|
115
|
+
*/
|
|
116
|
+
declare function derivedAdversarialCassetteIds(records: readonly AdversarialRefreshRecord[]): ReadonlySet<string>;
|
|
117
|
+
/**
|
|
118
|
+
* Probe reservations live beside the refresh-state tree, never inside a staged batch: a batch is
|
|
119
|
+
* reviewable data an operator may delete, and the record of what was already PAID FOR must
|
|
120
|
+
* outlive that even more emphatically than the record of what was already decided.
|
|
121
|
+
*/
|
|
122
|
+
declare const ADVERSARIAL_PROBE_SUBDIR = "eval/adversarial-probes";
|
|
123
|
+
declare const ADVERSARIAL_PROBE_SCHEMA_VERSION = 2;
|
|
124
|
+
/** D-11's idempotence key, as durable fields. */
|
|
125
|
+
interface ProbeReservationKey {
|
|
126
|
+
product: string;
|
|
127
|
+
seedCassetteId: string;
|
|
128
|
+
kind: ProbeGeneratorKind;
|
|
129
|
+
promptVersion: number;
|
|
130
|
+
}
|
|
131
|
+
/** The row written BEFORE a provider is contacted. Its existence is the charge. */
|
|
132
|
+
interface ProbeReservation extends ProbeReservationKey {
|
|
133
|
+
schemaVersion: typeof ADVERSARIAL_PROBE_SCHEMA_VERSION;
|
|
134
|
+
/** The frozen prompt hash the request was rendered from. */
|
|
135
|
+
promptHash: string;
|
|
136
|
+
/** sha256 of the configured model id — never the id, which can embed a credential. */
|
|
137
|
+
authorDigest: string;
|
|
138
|
+
/** The resolved provider family, which cannot. */
|
|
139
|
+
authorFamily: string;
|
|
140
|
+
}
|
|
141
|
+
/**
|
|
142
|
+
* What a provider call resolved to.
|
|
143
|
+
*
|
|
144
|
+
* All four are ANSWERS — the port returned control. `derived` carries the authored text so a
|
|
145
|
+
* later refresh can re-derive the same case for free; `refused` is a policy block recorded
|
|
146
|
+
* before 441-42 and only read back now, `invalid` an
|
|
147
|
+
* answer the pure contract rejected, and `error` a call that failed. None of the last three
|
|
148
|
+
* carries text: a refused surface is data, not a diagnostic.
|
|
149
|
+
*/
|
|
150
|
+
type ProbeCompletionOutcome = 'derived' | 'refused' | 'invalid' | 'error';
|
|
151
|
+
interface ProbeCompletion {
|
|
152
|
+
schemaVersion: typeof ADVERSARIAL_PROBE_SCHEMA_VERSION;
|
|
153
|
+
outcome: ProbeCompletionOutcome;
|
|
154
|
+
/** Present only on `derived`: the authored text and the id it produced. */
|
|
155
|
+
text?: string;
|
|
156
|
+
cassetteId?: string;
|
|
157
|
+
/** Reported spend, present only when the provider reported it. ABSENT is not zero. */
|
|
158
|
+
promptTokens?: number;
|
|
159
|
+
completionTokens?: number;
|
|
160
|
+
/** The firing detector or the refusal reason; never the refused text. */
|
|
161
|
+
detail?: string;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* The store's answer about one tuple.
|
|
165
|
+
*
|
|
166
|
+
* `usageKnown` is on every arm and is never inferred from a zero: an interrupted attempt and a
|
|
167
|
+
* failed one both have unknown tokens, and reporting them as `0` would make a real charge read
|
|
168
|
+
* as a free one.
|
|
169
|
+
*/
|
|
170
|
+
type ProbeAttemptState = {
|
|
171
|
+
kind: 'unattempted';
|
|
172
|
+
usageKnown: false;
|
|
173
|
+
} | {
|
|
174
|
+
kind: 'interrupted';
|
|
175
|
+
reservation: ProbeReservation;
|
|
176
|
+
usageKnown: false;
|
|
177
|
+
} | {
|
|
178
|
+
kind: 'completed';
|
|
179
|
+
reservation: ProbeReservation;
|
|
180
|
+
completion: ProbeCompletion;
|
|
181
|
+
usageKnown: boolean;
|
|
182
|
+
};
|
|
183
|
+
/** Build the durable reservation for one tuple. Pure; the author is reduced to a digest here. */
|
|
184
|
+
declare function probeReservationOf(key: ProbeReservationKey, input: {
|
|
185
|
+
promptHash: string;
|
|
186
|
+
author: {
|
|
187
|
+
id: string;
|
|
188
|
+
family: string;
|
|
189
|
+
};
|
|
190
|
+
}): ProbeReservation;
|
|
191
|
+
/** What the durable store says about ONE tuple. An absent tree is an empty history, not a failure. */
|
|
192
|
+
declare function readProbeAttempt(input: {
|
|
193
|
+
agentQaRoot: string;
|
|
194
|
+
key: ProbeReservationKey;
|
|
195
|
+
}): ProbeAttemptState;
|
|
196
|
+
/**
|
|
197
|
+
* Every attempt this product has on disk, in tuple-digest order.
|
|
198
|
+
*
|
|
199
|
+
* The disclosure surface: a report that wants to say "these calls were made and this one is
|
|
200
|
+
* still unanswered" reads it here. A PRESENT but malformed row is a failure, never silently
|
|
201
|
+
* dropped — this is the evidence a later refresh declines to spend on.
|
|
202
|
+
*/
|
|
203
|
+
declare function readProbeAttempts(input: {
|
|
204
|
+
agentQaRoot: string;
|
|
205
|
+
product: string;
|
|
206
|
+
}): ProbeAttemptState[];
|
|
207
|
+
/**
|
|
208
|
+
* Claim the allowance for ONE tuple, exclusively, BEFORE anything reaches a provider.
|
|
209
|
+
*
|
|
210
|
+
* `already-claimed` is not an error and is not a retry: it is the answer a concurrent owner —
|
|
211
|
+
* or a later run over the same disk — must get, together with the reservation that actually
|
|
212
|
+
* exists. Unlike the refresh record above this one is NOT content-addressed, so differing bytes
|
|
213
|
+
* under one name are not a conflict: two hosts with two authors asking the same question is
|
|
214
|
+
* exactly the race this arbitrates, and the first claim wins outright.
|
|
215
|
+
*/
|
|
216
|
+
declare function reserveProbeCall(input: {
|
|
217
|
+
agentQaRoot: string;
|
|
218
|
+
reservation: ProbeReservation;
|
|
219
|
+
}): {
|
|
220
|
+
kind: 'claimed' | 'already-claimed';
|
|
221
|
+
state: ProbeAttemptState;
|
|
222
|
+
};
|
|
223
|
+
/**
|
|
224
|
+
* Record what ONE reserved call resolved to, write-once.
|
|
225
|
+
*
|
|
226
|
+
* Two refusals, both deliberate. A completion for a tuple nothing reserved is refused, because
|
|
227
|
+
* it would show an answer nobody was ever charged for. A SECOND, different completion for one
|
|
228
|
+
* tuple is refused, because the completion is the reason a tuple is closed and overwriting it
|
|
229
|
+
* would erase why — the same law the refresh record keeps, for the same reason. An identical
|
|
230
|
+
* re-write is `unchanged`: a retry that recomputes the same answer is a no-op, not a conflict.
|
|
231
|
+
*/
|
|
232
|
+
declare function completeProbeCall(input: {
|
|
233
|
+
agentQaRoot: string;
|
|
234
|
+
key: ProbeReservationKey;
|
|
235
|
+
completion: ProbeCompletion;
|
|
236
|
+
}): 'published' | 'unchanged';
|
|
237
|
+
|
|
238
|
+
/** Local staging root for reviewable adversarial batches (D-01, beside `eval/feedback`). */
|
|
239
|
+
declare const ADVERSARIAL_STAGING_SUBDIR = "eval/adversarial";
|
|
240
|
+
/** The reviewed manifest's file name inside a staged batch directory. */
|
|
241
|
+
declare const ADVERSARIAL_MANIFEST_FILE = "manifest.json";
|
|
242
|
+
/**
|
|
243
|
+
* The SEED's label identity as it stood when the batch was reviewed (D-06).
|
|
244
|
+
*
|
|
245
|
+
* This is the pin that a recomputed manifest cannot launder. Every other pin here describes
|
|
246
|
+
* STAGED bytes, so an operator who edits a staged document and regenerates the manifest gets
|
|
247
|
+
* a self-consistent manifest — with a different digest, which is already refused. This one is
|
|
248
|
+
* checked against the seed's COMMITTED label at admission, so a derived record whose
|
|
249
|
+
* provenance, channel or tier was rewritten to something the seed never had is refused even
|
|
250
|
+
* if every staged pin agrees with itself.
|
|
251
|
+
*/
|
|
252
|
+
interface AdversarialSeedProvenancePin {
|
|
253
|
+
runId: string;
|
|
254
|
+
capturedAtOrdinal: number;
|
|
255
|
+
sourceChannel: string;
|
|
256
|
+
tier: string;
|
|
257
|
+
}
|
|
258
|
+
/**
|
|
259
|
+
* One reviewed case, pinned by content address (D-02).
|
|
260
|
+
*
|
|
261
|
+
* TWO DERIVATION PATHS, TWO PROVENANCE SHAPES (D-05/D-11). A table-drawn variant names
|
|
262
|
+
* `mutator` + `tableEntryId`; an LLM-authored probe names `promptVersion` and neither of the
|
|
263
|
+
* other two, because no frozen table drew its text. The pair is mutually exclusive and
|
|
264
|
+
* `listStagedAdversarial` refuses a case that carries both or neither — a manifest case whose
|
|
265
|
+
* provenance cannot be true is exactly what a refusal is for.
|
|
266
|
+
*
|
|
267
|
+
* The author's model id is deliberately NOT pinned here. It is already on the corpus entry,
|
|
268
|
+
* whose bytes the manifest pins.
|
|
269
|
+
*/
|
|
270
|
+
interface AdversarialBatchManifestCase {
|
|
271
|
+
cassetteId: string;
|
|
272
|
+
seedCassetteId: string;
|
|
273
|
+
/** Table path only: the mutation family. Absent on a probe. */
|
|
274
|
+
mutator?: AdversarialMutator;
|
|
275
|
+
kind: GeneratorKind;
|
|
276
|
+
/** Table path only. Disclosure of the drawn table entry; not part of the derived identity (D-08). */
|
|
277
|
+
tableEntryId?: string;
|
|
278
|
+
/** Probe path only: the frozen prompt generation the text was authored under (D-11). */
|
|
279
|
+
promptVersion?: number;
|
|
280
|
+
cassetteTextHash: string;
|
|
281
|
+
labelTextHash: string;
|
|
282
|
+
entryTextHash: string;
|
|
283
|
+
/** The ORIGINAL label identity the clone must still carry (D-06). */
|
|
284
|
+
seedProvenance: AdversarialSeedProvenancePin;
|
|
285
|
+
/** The derived cassette's blob closure, validated against the seed's committed bytes. */
|
|
286
|
+
blobShas: readonly string[];
|
|
287
|
+
}
|
|
288
|
+
/** The reviewed batch: what an operator admits with `--batch <digest>` (D-01/D-02). */
|
|
289
|
+
interface AdversarialBatchManifest {
|
|
290
|
+
schemaVersion: 2;
|
|
291
|
+
product: string;
|
|
292
|
+
/** sha256 over the pinned case list; equals the staging directory name. */
|
|
293
|
+
batchDigest: string;
|
|
294
|
+
tableVersion: number;
|
|
295
|
+
tableHash: string;
|
|
296
|
+
/** Disclosure only — deliberately OUTSIDE the digest so re-staging is idempotent. */
|
|
297
|
+
stagedAt: string;
|
|
298
|
+
cases: readonly AdversarialBatchManifestCase[];
|
|
299
|
+
}
|
|
300
|
+
/** A per-detector refusal row. Nothing produces one since 441-42 (D-15); the field stays empty. */
|
|
301
|
+
interface AdversarialRefusal {
|
|
302
|
+
cassetteId: string;
|
|
303
|
+
seedCassetteId: string;
|
|
304
|
+
generator: GeneratorKind;
|
|
305
|
+
detector: string;
|
|
306
|
+
action: 'block' | 'redact';
|
|
307
|
+
}
|
|
308
|
+
/**
|
|
309
|
+
* A refusal a previous refresh recorded. Nothing produces one since 441-53 (D-15): the refresh
|
|
310
|
+
* record no longer holds a refused tuple, so the field stays empty.
|
|
311
|
+
*/
|
|
312
|
+
interface AdversarialSettledRefusal {
|
|
313
|
+
seedCassetteId: string;
|
|
314
|
+
mutator: AdversarialMutator;
|
|
315
|
+
detector?: string;
|
|
316
|
+
}
|
|
317
|
+
/** Everything that stopped a derived case short of publication for a non-policy reason. */
|
|
318
|
+
interface AdversarialStagingSkip {
|
|
319
|
+
cassetteId: string;
|
|
320
|
+
seedCassetteId: string;
|
|
321
|
+
reason: 'blob-unresolvable' | 'blob-hash-mismatch' | 'text-hash-mismatch' | 'manifest-digest-mismatch' | 'path-identity-mismatch' | 'seed-label-unavailable' | 'seed-provenance-mismatch'
|
|
322
|
+
/** This id is already committed with DIFFERENT bytes. History wins; nothing is overwritten. */
|
|
323
|
+
| 'committed-conflict'
|
|
324
|
+
/** 441 GAP3-D1: a reviewed string still holds a declared value raw; the writer refused it. */
|
|
325
|
+
| 'unmasked-declared-value';
|
|
326
|
+
}
|
|
327
|
+
/**
|
|
328
|
+
* Recorded default cap on PROVIDER CALLS one refresh may make (D-04/D-10).
|
|
329
|
+
*
|
|
330
|
+
* The rationale, stated rather than tuned: a refresh authors at most one probe per
|
|
331
|
+
* `(seed, kind, promptVersion)` tuple, and the two frozen probe kinds over two seeds is four
|
|
332
|
+
* requested generations. It is deliberately INDEPENDENT of
|
|
333
|
+
* `ADVERSARIAL_DEFAULT_MAX_SEEDS_PER_REFRESH` — eight deterministic seeds cost nothing but CPU,
|
|
334
|
+
* while four authored probes cost four provider calls that no later run can recover. Two
|
|
335
|
+
* budgets, because the two halves are bounded for two different reasons.
|
|
336
|
+
*
|
|
337
|
+
* It counts CALLS and not tokens. A provider that reports no usage yields zero tokens
|
|
338
|
+
* (422-05), which UNDER-states spend; an unreported token must never be readable as "free", so
|
|
339
|
+
* the budget is denominated in the one quantity this layer always knows exactly.
|
|
340
|
+
*/
|
|
341
|
+
declare const ADVERSARIAL_DEFAULT_MAX_PROBE_CALLS = 4;
|
|
342
|
+
/**
|
|
343
|
+
* The resolved probe-call cap and where the operator set it (the seed-cap idiom).
|
|
344
|
+
*
|
|
345
|
+
* 2026-09-12, 422-22: `source` names the NEUTRAL `AdversarialCapSource` for the same reason the
|
|
346
|
+
* seed cap does — this value is recorded beside every refresh, and a second storage-side spelling
|
|
347
|
+
* of a persisted vocabulary is a second list to keep in step with the first.
|
|
348
|
+
*/
|
|
349
|
+
interface AdversarialProbeCallCap {
|
|
350
|
+
value: number;
|
|
351
|
+
source: AdversarialCapSource;
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* What a refresh SPENT at a provider.
|
|
355
|
+
*
|
|
356
|
+
* `modelCalls` is the authoritative number and is counted at RESERVATION time, not at answer
|
|
357
|
+
* time: a call that was announced and never answered still cost whatever it cost, and counting
|
|
358
|
+
* only answers would let an interrupted refresh under-report its bill. Token counts are carried
|
|
359
|
+
* exactly as the provider reported them and are never inferred.
|
|
360
|
+
*/
|
|
361
|
+
interface AdversarialSpend {
|
|
362
|
+
modelCalls: number;
|
|
363
|
+
promptTokens: number;
|
|
364
|
+
completionTokens: number;
|
|
365
|
+
/**
|
|
366
|
+
* Whether every call counted above reported its token usage.
|
|
367
|
+
*
|
|
368
|
+
* `false` is a statement, not a warning: a call that was announced and never answered, or one
|
|
369
|
+
* that failed, has UNKNOWN tokens, and the alternative to saying so is reporting `0` — which
|
|
370
|
+
* makes a real charge indistinguishable from a free one. It is exactly the reason the budget
|
|
371
|
+
* is denominated in calls rather than in tokens.
|
|
372
|
+
*/
|
|
373
|
+
usageKnown: boolean;
|
|
374
|
+
}
|
|
375
|
+
/**
|
|
376
|
+
* Why a `(seed, kind, promptVersion)` tuple yielded no authored probe.
|
|
377
|
+
*
|
|
378
|
+
* The first members are the pure contract's own seed law, re-reported verbatim so a staging
|
|
379
|
+
* report speaks ONE vocabulary about table families and probes alike. The last two are this
|
|
380
|
+
* layer's: a tuple already decided by a previous refresh, and a tuple this refresh's call
|
|
381
|
+
* budget deferred to the next one.
|
|
382
|
+
*/
|
|
383
|
+
type AdversarialProbeSkipReason = ProbeInapplicability | 'tuple-already-decided'
|
|
384
|
+
/**
|
|
385
|
+
* A durable reservation exists with no completion: a previous run announced this call and
|
|
386
|
+
* never recorded an answer. It is CHARGED and it is NOT re-authored — re-asking would buy the
|
|
387
|
+
* same question twice — and it is its own reason precisely because "asked and lost" is a
|
|
388
|
+
* different fact from "asked and answered" and from "never asked".
|
|
389
|
+
*/
|
|
390
|
+
| 'tuple-interrupted' | 'probe-cap-exceeded'
|
|
391
|
+
/** Paid output belongs to a different author or prompt; never implicitly re-author it. */
|
|
392
|
+
| 'probe-reservation-mismatch'
|
|
393
|
+
/** Re-derivation must recover the identity already recorded with the paid response. */
|
|
394
|
+
| 'probe-completion-identity-mismatch';
|
|
395
|
+
/** An honest row for every probe tuple that produced no case. */
|
|
396
|
+
interface AdversarialProbeSkip {
|
|
397
|
+
seedCassetteId: string;
|
|
398
|
+
kind: ProbeGeneratorKind;
|
|
399
|
+
promptVersion: number;
|
|
400
|
+
reason: AdversarialProbeSkipReason;
|
|
401
|
+
}
|
|
402
|
+
/**
|
|
403
|
+
* An authored RESPONSE the pure contract refused.
|
|
404
|
+
*
|
|
405
|
+
* It says the model's answer was not a usable probe — too long, not a name, folded back onto the
|
|
406
|
+
* recorded identity. It names no text either, for the same reason.
|
|
407
|
+
*/
|
|
408
|
+
interface AdversarialProbeRefusal {
|
|
409
|
+
seedCassetteId: string;
|
|
410
|
+
kind: ProbeGeneratorKind;
|
|
411
|
+
promptVersion: number;
|
|
412
|
+
reason: ProbeRefusal;
|
|
413
|
+
}
|
|
414
|
+
interface StageAdversarialProposalInput {
|
|
415
|
+
agentQaRoot: string;
|
|
416
|
+
committedRoot: string;
|
|
417
|
+
product: string;
|
|
418
|
+
mutators?: readonly AdversarialMutator[];
|
|
419
|
+
/** D-04: the per-refresh NOVEL-seed cap; absent resolves to the recorded default. */
|
|
420
|
+
maxSeeds?: number;
|
|
421
|
+
/** Where `maxSeeds` came from, recorded beside the refresh. Absent with a value means 'flag'. */
|
|
422
|
+
maxSeedsSource?: Exclude<AdversarialCapSource, 'default'>;
|
|
423
|
+
/**
|
|
424
|
+
* D-10: the bound author, or nothing.
|
|
425
|
+
*
|
|
426
|
+
* A PORT and never a model: this package constructs no provider client and sees no profile,
|
|
427
|
+
* credential or provider name. Absent means this refresh authors nothing at all — the frozen
|
|
428
|
+
* families still derive, because a host with no eligible author is not a host with no
|
|
429
|
+
* adversarial corpus.
|
|
430
|
+
*/
|
|
431
|
+
probeAuthor?: ProbeAuthorPort;
|
|
432
|
+
/** D-11: the identity every authored entry stamps. Required with `probeAuthor`. */
|
|
433
|
+
authoringModel?: AuthoringModel;
|
|
434
|
+
/** D-09: the host's capability gap, reported on the batch when there is no author. */
|
|
435
|
+
noProbeModel?: NoProbeModelGap;
|
|
436
|
+
/** D-04/D-10: the per-refresh provider-CALL cap; absent resolves to the recorded default. */
|
|
437
|
+
maxProbeCalls?: number;
|
|
438
|
+
/** Where `maxProbeCalls` came from. Absent with a value means 'flag'. */
|
|
439
|
+
maxProbeCallsSource?: Exclude<AdversarialCapSource, 'default'>;
|
|
440
|
+
/** Derive, publish nothing — neither staged bytes nor a settle record. */
|
|
441
|
+
dryRun?: boolean;
|
|
442
|
+
}
|
|
443
|
+
interface StagedAdversarialCase {
|
|
444
|
+
cassetteId: string;
|
|
445
|
+
seedCassetteId: string;
|
|
446
|
+
/** Table path only. */
|
|
447
|
+
mutator?: AdversarialMutator;
|
|
448
|
+
kind: GeneratorKind;
|
|
449
|
+
/** Table path only. */
|
|
450
|
+
tableEntryId?: string;
|
|
451
|
+
/** Probe path only: the frozen prompt generation this text was authored under (D-11). */
|
|
452
|
+
promptVersion?: number;
|
|
453
|
+
}
|
|
454
|
+
interface StageAdversarialProposalResult {
|
|
455
|
+
kind: 'staged' | 'unchanged' | 'dry-run' | 'nothing-derived';
|
|
456
|
+
product: string;
|
|
457
|
+
batchDigest?: string;
|
|
458
|
+
dir?: string;
|
|
459
|
+
staged: readonly StagedAdversarialCase[];
|
|
460
|
+
refusals: readonly AdversarialRefusal[];
|
|
461
|
+
skips: readonly AdversarialSeedSkip[];
|
|
462
|
+
stagingSkips: readonly AdversarialStagingSkip[];
|
|
463
|
+
seedsConsidered: number;
|
|
464
|
+
seedsDeferred: number;
|
|
465
|
+
tableVersion: number;
|
|
466
|
+
tableHash: string;
|
|
467
|
+
/**
|
|
468
|
+
* D-04: the committed root's GATE posture, not the reader's length.
|
|
469
|
+
*
|
|
470
|
+
* `openCorpusReader` answers an absent, unmarked or unreadable root with an empty list —
|
|
471
|
+
* byte-identical to a healthy root that genuinely holds no visible entry. Only the gate's own
|
|
472
|
+
* digest read establishes the root, and a refresh that cannot establish it settles nothing:
|
|
473
|
+
* a settle record written against silence is evidence manufactured from an outage.
|
|
474
|
+
*/
|
|
475
|
+
rootPosture: 'available' | 'unavailable';
|
|
476
|
+
/** The corpus digest this refresh read its seeds from; absent when the posture is unavailable. */
|
|
477
|
+
sourceDigest?: string;
|
|
478
|
+
/** The resolved cap and its source, recorded beside every refresh. */
|
|
479
|
+
cap: AdversarialSeedCap;
|
|
480
|
+
/** NOVEL eligible seeds, counted BEFORE the cap; `seedsDeferred` is the overflow. */
|
|
481
|
+
eligibleSeeds: number;
|
|
482
|
+
/** The durable refresh record this refresh published, if it published one. */
|
|
483
|
+
stateRecordDigest?: string;
|
|
484
|
+
/** Always empty since 441-53 (D-15). */
|
|
485
|
+
settledRefusals: readonly AdversarialSettledRefusal[];
|
|
486
|
+
/**
|
|
487
|
+
* D-09: why this host authored no probes.
|
|
488
|
+
*
|
|
489
|
+
* ABSENT means the capability was never consulted (an admission, or a caller that declined to
|
|
490
|
+
* resolve credentials) — which is a different fact from "consulted, and there is no author".
|
|
491
|
+
* Collapsing the two would let a host that never asked read as a host that has nothing.
|
|
492
|
+
*/
|
|
493
|
+
noProbeModel?: NoProbeModelGap;
|
|
494
|
+
/** The identity that authored this refresh's probes; absent when none were authored. */
|
|
495
|
+
authoringModel?: AuthoringModel;
|
|
496
|
+
/** The resolved provider-call cap and its source, recorded beside every refresh. */
|
|
497
|
+
probeCap: AdversarialProbeCallCap;
|
|
498
|
+
/** What this refresh actually spent at a provider. */
|
|
499
|
+
spend: AdversarialSpend;
|
|
500
|
+
/** Every probe tuple that produced no case, and why. */
|
|
501
|
+
probeSkips: readonly AdversarialProbeSkip[];
|
|
502
|
+
/** Authored responses the pure contract refused. */
|
|
503
|
+
probeRefusals: readonly AdversarialProbeRefusal[];
|
|
504
|
+
/**
|
|
505
|
+
* Calls a PREVIOUS run announced and never recorded an answer for, still charged against
|
|
506
|
+
* their tuples. Reported separately from `spend.modelCalls`, which counts only what THIS
|
|
507
|
+
* refresh asked: folding them together would claim this run made a call it did not.
|
|
508
|
+
*/
|
|
509
|
+
interruptedProbeCalls: number;
|
|
510
|
+
}
|
|
511
|
+
/** A staged batch loaded back from disk with its bytes verified against the manifest pins. */
|
|
512
|
+
interface StagedAdversarialBatch {
|
|
513
|
+
dir: string;
|
|
514
|
+
manifest: AdversarialBatchManifest;
|
|
515
|
+
retirement?: AdversarialRetirementRecord;
|
|
516
|
+
cases: readonly {
|
|
517
|
+
manifest: AdversarialBatchManifestCase;
|
|
518
|
+
caseDir: string;
|
|
519
|
+
cassette: LeafCassette;
|
|
520
|
+
labelRecord: StandaloneLabelRecord;
|
|
521
|
+
corpusEntry: AdversarialCorpusEntry;
|
|
522
|
+
/** The EXACT reviewed bytes; admission never re-serializes from the parsed objects. */
|
|
523
|
+
texts: {
|
|
524
|
+
cassette: string;
|
|
525
|
+
label: string;
|
|
526
|
+
entry: string;
|
|
527
|
+
};
|
|
528
|
+
}[];
|
|
529
|
+
}
|
|
530
|
+
interface RefreshAdversarialSplitInput {
|
|
531
|
+
agentQaRoot: string;
|
|
532
|
+
committedRoot: string;
|
|
533
|
+
product: string;
|
|
534
|
+
/** The reviewed batch digest the operator named; exact, never "the latest". */
|
|
535
|
+
batchDigest: string;
|
|
536
|
+
dryRun?: boolean;
|
|
537
|
+
/** Snapshot host receipts only after validation, under lifecycle ownership, before writes. */
|
|
538
|
+
beforeWrites?: () => void;
|
|
539
|
+
/**
|
|
540
|
+
* Runs after ONE case's bytes have landed and been byte-verified.
|
|
541
|
+
*
|
|
542
|
+
* An observation and fault-injection seam. Admission is the only place in this composition
|
|
543
|
+
* where an interruption can leave the committed root part-way through a reviewed batch, and
|
|
544
|
+
* "a case that landed stays landed" is not a claim anything could check without a way to
|
|
545
|
+
* interrupt it between cases. A throw here is treated exactly like a real IO failure.
|
|
546
|
+
*/
|
|
547
|
+
afterCaseCommitted?: (cassetteId: string) => void;
|
|
548
|
+
/** 441 GAP3-D1: the workspace's declared-secret mask, passed to the seeding writer. Required. */
|
|
549
|
+
mask: MaskFn;
|
|
550
|
+
}
|
|
551
|
+
/**
|
|
552
|
+
* The outcome of ONE operator admission (D-01..D-04).
|
|
553
|
+
*
|
|
554
|
+
* Four counts, deliberately not three. `admitted` are cases whose bytes this run committed;
|
|
555
|
+
* `unchanged` are reviewed cases already present BYTE-IDENTICALLY, which move no digest and
|
|
556
|
+
* must not be double-counted as admissions; `refusals` is always empty (441 D-15); `remaining` are
|
|
557
|
+
* reviewed members that have not landed — the number that makes a partial admission legible
|
|
558
|
+
* as one. Collapsing any pair of them is how an interrupted run comes to read as a complete one.
|
|
559
|
+
*/
|
|
560
|
+
interface AdversarialAdmissionResult {
|
|
561
|
+
kind: 'admitted' | 'partial' | 'dry-run' | 'refused';
|
|
562
|
+
reason?: 'batch-not-found' | 'staging-integrity' | 'corpus-unavailable' | 'admission-refused' | 'admission-interrupted';
|
|
563
|
+
product: string;
|
|
564
|
+
batchDigest: string;
|
|
565
|
+
/** Cases whose bytes THIS run committed. */
|
|
566
|
+
admitted: readonly StagedAdversarialCase[];
|
|
567
|
+
/** Reviewed cases already committed byte-identically; the digest does not move for these. */
|
|
568
|
+
unchanged: readonly StagedAdversarialCase[];
|
|
569
|
+
refusals: readonly AdversarialRefusal[];
|
|
570
|
+
/** Reviewed members that have not landed — zero on a complete admission. */
|
|
571
|
+
remaining: readonly StagedAdversarialCase[];
|
|
572
|
+
stagingSkips: readonly AdversarialStagingSkip[];
|
|
573
|
+
previousCorpusDigest?: string;
|
|
574
|
+
nextCorpusDigest?: string;
|
|
575
|
+
writesStarted: boolean;
|
|
576
|
+
retirement?: AdversarialRetirementResult;
|
|
577
|
+
lifecycleBusy?: {
|
|
578
|
+
lockPath: string;
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
declare const retirementSchema: z.ZodObject<{
|
|
582
|
+
schemaVersion: z.ZodLiteral<2>;
|
|
583
|
+
product: z.ZodString;
|
|
584
|
+
batchDigest: z.ZodString;
|
|
585
|
+
manifestSha256: z.ZodString;
|
|
586
|
+
reviewedCaseIds: z.ZodArray<z.ZodString>;
|
|
587
|
+
alreadyCommittedCaseIds: z.ZodArray<z.ZodString>;
|
|
588
|
+
}, z.core.$strict>;
|
|
589
|
+
type AdversarialRetirementRecord = z.infer<typeof retirementSchema>;
|
|
590
|
+
interface RetireStagedAdversarialBatchInput {
|
|
591
|
+
agentQaRoot: string;
|
|
592
|
+
committedRoot: string;
|
|
593
|
+
product: string;
|
|
594
|
+
batchDigest: string;
|
|
595
|
+
}
|
|
596
|
+
/** A decision over retained reviewed bytes, never an admission outcome. */
|
|
597
|
+
interface AdversarialRetirementResult {
|
|
598
|
+
retired: true;
|
|
599
|
+
alreadyRetired: boolean;
|
|
600
|
+
product: string;
|
|
601
|
+
batchDigest: string;
|
|
602
|
+
manifestSha256: string;
|
|
603
|
+
retainedBatchDir: string;
|
|
604
|
+
retirementRecordPath: string;
|
|
605
|
+
reviewedCaseIds: readonly string[];
|
|
606
|
+
alreadyCommittedCaseIds: readonly string[];
|
|
607
|
+
remainingCaseIds: readonly string[];
|
|
608
|
+
}
|
|
609
|
+
declare class AdversarialLifecycleBusyError extends Error {
|
|
610
|
+
readonly lockPath: string;
|
|
611
|
+
constructor(lockPath: string);
|
|
612
|
+
}
|
|
613
|
+
/** Retain the reviewed batch and publish its operator decision once, under admission's lock. */
|
|
614
|
+
declare function retireStagedAdversarialBatch(input: RetireStagedAdversarialBatchInput): AdversarialRetirementResult;
|
|
615
|
+
/** 422-01's spelling of the same result, retained for its existing callers. */
|
|
616
|
+
type RefreshAdversarialSplitResult = AdversarialAdmissionResult;
|
|
617
|
+
/**
|
|
618
|
+
* The batch digest (D-02). `stagedAt` is deliberately absent: two refreshes that derive the
|
|
619
|
+
* same cases must address the same directory, so re-staging is an idempotent no-op rather
|
|
620
|
+
* than an unbounded pile of near-identical batches.
|
|
621
|
+
*/
|
|
622
|
+
declare function computeAdversarialBatchDigest(product: string, cases: readonly AdversarialBatchManifestCase[]): string;
|
|
623
|
+
/**
|
|
624
|
+
* Resolve the per-refresh provider-call cap (D-04/D-10).
|
|
625
|
+
*
|
|
626
|
+
* Every rejected boundary is rejected LOUDLY, exactly as the seed cap is: zero, negative,
|
|
627
|
+
* fractional, nonfinite and beyond-safe-integer budgets are a TypeError rather than a silent
|
|
628
|
+
* fallback. A budget that quietly becomes four because the operator typed `0` is a budget the
|
|
629
|
+
* operator does not have — and here the silent version would buy provider calls they refused.
|
|
630
|
+
*/
|
|
631
|
+
declare function resolveAdversarialProbeCallCap(input: {
|
|
632
|
+
value?: number;
|
|
633
|
+
source?: Exclude<AdversarialCapSource, 'default'>;
|
|
634
|
+
}): AdversarialProbeCallCap;
|
|
635
|
+
/**
|
|
636
|
+
* D-11's idempotence key: `(seed, kind, promptVersion)` and NOTHING else.
|
|
637
|
+
*
|
|
638
|
+
* Not the authored text — that is part of the derived IDENTITY, so keying on it would make
|
|
639
|
+
* every answer a new tuple and no refresh could ever recognise that it had already asked. Not
|
|
640
|
+
* the author either: a second host with a second model must not buy a second copy of a probe
|
|
641
|
+
* this corpus already holds. A future `promptVersion` IS a new tuple, deliberately — different
|
|
642
|
+
* instructions are a different case, not a duplicate.
|
|
643
|
+
*/
|
|
644
|
+
declare function adversarialProbeTupleKey(seedCassetteId: string, kind: ProbeGeneratorKind, promptVersion: number): string;
|
|
645
|
+
/**
|
|
646
|
+
* Derive and stage one reviewable adversarial batch for a product.
|
|
647
|
+
*
|
|
648
|
+
* Seeds come from a FRESH `/reader` snapshot of the committed root, so a private held-out
|
|
649
|
+
* entry is structurally unreachable — the proposer's door is the same door this uses.
|
|
650
|
+
* Nothing under the committed root is written, read-modified, or re-digested.
|
|
651
|
+
*/
|
|
652
|
+
declare function stageAdversarialProposal(input: StageAdversarialProposalInput): Promise<StageAdversarialProposalResult>;
|
|
653
|
+
/**
|
|
654
|
+
* Load staged batches, verifying identity at every level: the directory name IS the manifest
|
|
655
|
+
* digest, the digest recomputed from the manifest's own case list equals it, and each staged
|
|
656
|
+
* document re-hashes to its pinned value. A structural violation throws — a tampered batch is
|
|
657
|
+
* never handed back as if it were reviewable.
|
|
658
|
+
*/
|
|
659
|
+
declare function listStagedAdversarial(input: {
|
|
660
|
+
agentQaRoot: string;
|
|
661
|
+
product?: string;
|
|
662
|
+
batchDigest?: string;
|
|
663
|
+
includeRetired?: boolean;
|
|
664
|
+
}): StagedAdversarialBatch[];
|
|
665
|
+
/**
|
|
666
|
+
* Commit ONE reviewed batch, named by its digest.
|
|
667
|
+
*
|
|
668
|
+
* The order is deliberate: integrity first (nothing is trusted because it is on disk), then
|
|
669
|
+
* blob resolution, then the one sanctioned seeding writer, then a byte-verification of what
|
|
670
|
+
* actually landed. Only a case whose committed bytes equal the reviewed bytes counts as
|
|
671
|
+
* admitted; the reviewed bytes are committed as authored (441 D-15), and the writer refuses a
|
|
672
|
+
* case that still holds a declared value raw under the required mask (441 GAP3-D1), which this
|
|
673
|
+
* reports as an `unmasked-declared-value` staging skip. No other content check runs.
|
|
674
|
+
*/
|
|
675
|
+
declare function refreshAdversarialSplit(input: RefreshAdversarialSplitInput): RefreshAdversarialSplitResult;
|
|
676
|
+
|
|
677
|
+
interface JudgeBindingInput {
|
|
678
|
+
/** The judge model profile — a SECOND profile, never the replay planner's (D-12 disjointness). */
|
|
679
|
+
profile: ModelProfile;
|
|
680
|
+
/** Resolved provider auth for the judge model. */
|
|
681
|
+
auth: ResolvedModelAuth;
|
|
682
|
+
/** Optional provider overrides — the captured-boundary stub seam for tests (createModel passthrough). */
|
|
683
|
+
modelOverrides?: ModelProviderOverrides;
|
|
684
|
+
modelCallObservation?: ModelCallObservation;
|
|
685
|
+
mask: MaskFn;
|
|
686
|
+
}
|
|
687
|
+
/** N existing bindings in declaration order, each retaining bindJudgePort's fresh copy. */
|
|
688
|
+
declare function bindJudgePanel(inputs: readonly JudgeBindingInput[]): JudgePanelPort;
|
|
689
|
+
/**
|
|
690
|
+
* Bind the tier-4 JudgePort over the production createModel recipe: a fresh ModelProfile copy
|
|
691
|
+
* (createModel mutates `profile.capabilities` — RESEARCH Pitfall 8), ONE handle built at bind
|
|
692
|
+
* time and shared across calls, model identity from the caller profile only.
|
|
693
|
+
*/
|
|
694
|
+
declare function bindJudgePort(input: JudgeBindingInput): JudgePort;
|
|
695
|
+
|
|
696
|
+
interface LoadJudgeCalibrationEvidenceInput {
|
|
697
|
+
committedRoot: string;
|
|
698
|
+
agentQaRoot: string;
|
|
699
|
+
profile: BaselineProfileKeyInput;
|
|
700
|
+
/** Explicit operator selection when supplied; never choose the newest matching directory. */
|
|
701
|
+
replayRunRoots?: readonly string[];
|
|
702
|
+
}
|
|
703
|
+
interface JudgeCalibrationEvidenceRead {
|
|
704
|
+
/** Executable selected gold before saved-replay admission; not a sum of skip diagnostics. */
|
|
705
|
+
selected: number;
|
|
706
|
+
evidence: JudgeCalibrationEvidence[];
|
|
707
|
+
skippedCounts: JudgeCalibrationDiagnostics;
|
|
708
|
+
corpusDigest: string;
|
|
709
|
+
goldSetDigest: string;
|
|
710
|
+
replayEvidenceDigest: string;
|
|
711
|
+
evidenceDigest: string;
|
|
712
|
+
profileKey: string;
|
|
713
|
+
selectedInputs: {
|
|
714
|
+
decisionId: string;
|
|
715
|
+
goldCassetteId: string;
|
|
716
|
+
replayFile: string;
|
|
717
|
+
reportFile: string;
|
|
718
|
+
replayDigest: string;
|
|
719
|
+
scoreDigest: string;
|
|
720
|
+
reportDigest: string;
|
|
721
|
+
}[];
|
|
722
|
+
/**
|
|
723
|
+
* 422 D-12: the committed ADVERSARIAL population — the bait disclosure's source, never gold.
|
|
724
|
+
*
|
|
725
|
+
* ABSENT means this reader could not open the committed corpus at all, which is a different
|
|
726
|
+
* fact from a corpus that holds no adversarial member; an empty array standing in for the
|
|
727
|
+
* former would report a clean zero nobody measured.
|
|
728
|
+
*/
|
|
729
|
+
adversarialMembers?: readonly JudgeTargetedCalibrationMember[];
|
|
730
|
+
}
|
|
731
|
+
/** Operator-only IO. Every action/truth derivation stays in the neutral package. */
|
|
732
|
+
declare function loadJudgeCalibrationEvidence(input: LoadJudgeCalibrationEvidenceInput): JudgeCalibrationEvidenceRead;
|
|
733
|
+
|
|
734
|
+
declare const JUDGE_CALIBRATION_SCHEMA_VERSION = 2;
|
|
735
|
+
declare const JUDGE_CALIBRATION_SUBDIR = "judge-calibration";
|
|
736
|
+
declare const schema: z.ZodUnion<readonly [z.ZodObject<{
|
|
737
|
+
status: z.ZodLiteral<"capability-gap">;
|
|
738
|
+
reason: z.ZodEnum<{
|
|
739
|
+
"no-credential": "no-credential";
|
|
740
|
+
"single-family-host": "single-family-host";
|
|
741
|
+
}>;
|
|
742
|
+
n: z.ZodLiteral<0>;
|
|
743
|
+
schemaVersion: z.ZodLiteral<2>;
|
|
744
|
+
generationId: z.ZodString;
|
|
745
|
+
judgeTargeted: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
746
|
+
basis: z.ZodLiteral<"raw">;
|
|
747
|
+
source: z.ZodLiteral<"unavailable">;
|
|
748
|
+
reason: z.ZodEnum<{
|
|
749
|
+
"not-read": "not-read";
|
|
750
|
+
"corpus-unreadable": "corpus-unreadable";
|
|
751
|
+
}>;
|
|
752
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
753
|
+
basis: z.ZodLiteral<"raw">;
|
|
754
|
+
source: z.ZodLiteral<"resolved">;
|
|
755
|
+
total: z.ZodNumber;
|
|
756
|
+
metadataUnavailable: z.ZodNumber;
|
|
757
|
+
byProvenance: z.ZodRecord<z.ZodEnum<{
|
|
758
|
+
absent: "absent";
|
|
759
|
+
table: "table";
|
|
760
|
+
probe: "probe";
|
|
761
|
+
malformed: "malformed";
|
|
762
|
+
}>, z.ZodNumber>;
|
|
763
|
+
byKind: z.ZodRecord<z.ZodEnum<{
|
|
764
|
+
"fuzz-selector": "fuzz-selector";
|
|
765
|
+
"fuzz-label": "fuzz-label";
|
|
766
|
+
"fuzz-error": "fuzz-error";
|
|
767
|
+
"injection-name": "injection-name";
|
|
768
|
+
"optimized-trigger": "optimized-trigger";
|
|
769
|
+
"judge-targeted": "judge-targeted";
|
|
770
|
+
}>, z.ZodNumber>;
|
|
771
|
+
judgeTargeted: z.ZodNumber;
|
|
772
|
+
selfAuthored: z.ZodNumber;
|
|
773
|
+
byAuthoringModel: z.ZodArray<z.ZodObject<{
|
|
774
|
+
authoringModel: z.ZodString;
|
|
775
|
+
authoringFamily: z.ZodString;
|
|
776
|
+
panelMember: z.ZodBoolean;
|
|
777
|
+
probeAuthored: z.ZodNumber;
|
|
778
|
+
judgeTargeted: z.ZodNumber;
|
|
779
|
+
}, z.core.$strict>>;
|
|
780
|
+
observedInCalibration: z.ZodNumber;
|
|
781
|
+
}, z.core.$strict>]>>;
|
|
782
|
+
profileKey: z.ZodString;
|
|
783
|
+
profile: z.ZodObject<{
|
|
784
|
+
plannerModelId: z.ZodString;
|
|
785
|
+
fingerprint: z.ZodString;
|
|
786
|
+
matcherVersion: z.ZodNumber;
|
|
787
|
+
judgeModelId: z.ZodString;
|
|
788
|
+
judgeVersion: z.ZodNumber;
|
|
789
|
+
}, z.core.$strict>;
|
|
790
|
+
identity: z.ZodObject<{
|
|
791
|
+
judgeModelId: z.ZodString;
|
|
792
|
+
judgeFamily: z.ZodString;
|
|
793
|
+
judgeVersion: z.ZodNumber;
|
|
794
|
+
confidenceThreshold: z.ZodLiteral<"high">;
|
|
795
|
+
panelSize: z.ZodNumber;
|
|
796
|
+
members: z.ZodArray<z.ZodObject<{
|
|
797
|
+
judgeModelId: z.ZodString;
|
|
798
|
+
judgeFamily: z.ZodString;
|
|
799
|
+
}, z.core.$strict>>;
|
|
800
|
+
}, z.core.$strict>;
|
|
801
|
+
constitutionDigest: z.ZodString;
|
|
802
|
+
corpusDigest: z.ZodString;
|
|
803
|
+
goldSetDigest: z.ZodString;
|
|
804
|
+
replayEvidenceDigest: z.ZodString;
|
|
805
|
+
evidenceDigest: z.ZodString;
|
|
806
|
+
minN: z.ZodNumber;
|
|
807
|
+
minNSource: z.ZodEnum<{
|
|
808
|
+
default: "default";
|
|
809
|
+
flag: "flag";
|
|
810
|
+
}>;
|
|
811
|
+
selected: z.ZodNumber;
|
|
812
|
+
admitted: z.ZodNumber;
|
|
813
|
+
cost: z.ZodObject<{
|
|
814
|
+
callBound: z.ZodNumber;
|
|
815
|
+
usage: z.ZodObject<{
|
|
816
|
+
calls: z.ZodNumber;
|
|
817
|
+
inputTokens: z.ZodOptional<z.ZodNumber>;
|
|
818
|
+
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
819
|
+
}, z.core.$strict>;
|
|
820
|
+
}, z.core.$strict>;
|
|
821
|
+
skippedCounts: z.ZodOptional<z.ZodObject<{
|
|
822
|
+
'missing-executable-correction': z.ZodNumber;
|
|
823
|
+
}, z.core.$strict>>;
|
|
824
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
825
|
+
status: z.ZodEnum<{
|
|
826
|
+
uncalibrated: "uncalibrated";
|
|
827
|
+
calibrated: "calibrated";
|
|
828
|
+
}>;
|
|
829
|
+
n: z.ZodNumber;
|
|
830
|
+
counts: z.ZodObject<{
|
|
831
|
+
selected: z.ZodNumber;
|
|
832
|
+
admitted: z.ZodNumber;
|
|
833
|
+
eligible: z.ZodNumber;
|
|
834
|
+
attempted: z.ZodNumber;
|
|
835
|
+
eligibilityRefused: z.ZodNumber;
|
|
836
|
+
evidenceSkipped: z.ZodNumber;
|
|
837
|
+
truthComparable: z.ZodNumber;
|
|
838
|
+
unresolvedTruth: z.ZodNumber;
|
|
839
|
+
}, z.core.$strict>;
|
|
840
|
+
pooled: z.ZodObject<{
|
|
841
|
+
settled: z.ZodNumber;
|
|
842
|
+
abstained: z.ZodNumber;
|
|
843
|
+
errors: z.ZodNumber;
|
|
844
|
+
n: z.ZodNumber;
|
|
845
|
+
numerator: z.ZodNumber;
|
|
846
|
+
denominator: z.ZodNumber;
|
|
847
|
+
agreement: z.ZodOptional<z.ZodNumber>;
|
|
848
|
+
wilson95: z.ZodOptional<z.ZodObject<{
|
|
849
|
+
lower: z.ZodNumber;
|
|
850
|
+
upper: z.ZodNumber;
|
|
851
|
+
halfWidth: z.ZodNumber;
|
|
852
|
+
}, z.core.$strict>>;
|
|
853
|
+
}, z.core.$strict>;
|
|
854
|
+
perMember: z.ZodArray<z.ZodObject<{
|
|
855
|
+
settled: z.ZodNumber;
|
|
856
|
+
abstained: z.ZodNumber;
|
|
857
|
+
errors: z.ZodNumber;
|
|
858
|
+
n: z.ZodNumber;
|
|
859
|
+
numerator: z.ZodNumber;
|
|
860
|
+
denominator: z.ZodNumber;
|
|
861
|
+
agreement: z.ZodOptional<z.ZodNumber>;
|
|
862
|
+
wilson95: z.ZodOptional<z.ZodObject<{
|
|
863
|
+
lower: z.ZodNumber;
|
|
864
|
+
upper: z.ZodNumber;
|
|
865
|
+
halfWidth: z.ZodNumber;
|
|
866
|
+
}, z.core.$strict>>;
|
|
867
|
+
judgeModelId: z.ZodString;
|
|
868
|
+
}, z.core.$strict>>;
|
|
869
|
+
pooledAgreement: z.ZodOptional<z.ZodNumber>;
|
|
870
|
+
wilson95: z.ZodOptional<z.ZodObject<{
|
|
871
|
+
lower: z.ZodNumber;
|
|
872
|
+
upper: z.ZodNumber;
|
|
873
|
+
halfWidth: z.ZodNumber;
|
|
874
|
+
}, z.core.$strict>>;
|
|
875
|
+
abstention: z.ZodObject<{
|
|
876
|
+
numerator: z.ZodNumber;
|
|
877
|
+
denominator: z.ZodNumber;
|
|
878
|
+
rate: z.ZodOptional<z.ZodNumber>;
|
|
879
|
+
}, z.core.$strict>;
|
|
880
|
+
disagreement: z.ZodObject<{
|
|
881
|
+
pairs: z.ZodArray<z.ZodObject<{
|
|
882
|
+
numerator: z.ZodNumber;
|
|
883
|
+
denominator: z.ZodNumber;
|
|
884
|
+
rate: z.ZodOptional<z.ZodNumber>;
|
|
885
|
+
memberModelIds: z.ZodTuple<[z.ZodString, z.ZodString], null>;
|
|
886
|
+
}, z.core.$strict>>;
|
|
887
|
+
numerator: z.ZodNumber;
|
|
888
|
+
denominator: z.ZodNumber;
|
|
889
|
+
rate: z.ZodOptional<z.ZodNumber>;
|
|
890
|
+
}, z.core.$strict>;
|
|
891
|
+
schemaVersion: z.ZodLiteral<2>;
|
|
892
|
+
generationId: z.ZodString;
|
|
893
|
+
judgeTargeted: z.ZodOptional<z.ZodUnion<readonly [z.ZodObject<{
|
|
894
|
+
basis: z.ZodLiteral<"raw">;
|
|
895
|
+
source: z.ZodLiteral<"unavailable">;
|
|
896
|
+
reason: z.ZodEnum<{
|
|
897
|
+
"not-read": "not-read";
|
|
898
|
+
"corpus-unreadable": "corpus-unreadable";
|
|
899
|
+
}>;
|
|
900
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
901
|
+
basis: z.ZodLiteral<"raw">;
|
|
902
|
+
source: z.ZodLiteral<"resolved">;
|
|
903
|
+
total: z.ZodNumber;
|
|
904
|
+
metadataUnavailable: z.ZodNumber;
|
|
905
|
+
byProvenance: z.ZodRecord<z.ZodEnum<{
|
|
906
|
+
absent: "absent";
|
|
907
|
+
table: "table";
|
|
908
|
+
probe: "probe";
|
|
909
|
+
malformed: "malformed";
|
|
910
|
+
}>, z.ZodNumber>;
|
|
911
|
+
byKind: z.ZodRecord<z.ZodEnum<{
|
|
912
|
+
"fuzz-selector": "fuzz-selector";
|
|
913
|
+
"fuzz-label": "fuzz-label";
|
|
914
|
+
"fuzz-error": "fuzz-error";
|
|
915
|
+
"injection-name": "injection-name";
|
|
916
|
+
"optimized-trigger": "optimized-trigger";
|
|
917
|
+
"judge-targeted": "judge-targeted";
|
|
918
|
+
}>, z.ZodNumber>;
|
|
919
|
+
judgeTargeted: z.ZodNumber;
|
|
920
|
+
selfAuthored: z.ZodNumber;
|
|
921
|
+
byAuthoringModel: z.ZodArray<z.ZodObject<{
|
|
922
|
+
authoringModel: z.ZodString;
|
|
923
|
+
authoringFamily: z.ZodString;
|
|
924
|
+
panelMember: z.ZodBoolean;
|
|
925
|
+
probeAuthored: z.ZodNumber;
|
|
926
|
+
judgeTargeted: z.ZodNumber;
|
|
927
|
+
}, z.core.$strict>>;
|
|
928
|
+
observedInCalibration: z.ZodNumber;
|
|
929
|
+
}, z.core.$strict>]>>;
|
|
930
|
+
profileKey: z.ZodString;
|
|
931
|
+
profile: z.ZodObject<{
|
|
932
|
+
plannerModelId: z.ZodString;
|
|
933
|
+
fingerprint: z.ZodString;
|
|
934
|
+
matcherVersion: z.ZodNumber;
|
|
935
|
+
judgeModelId: z.ZodString;
|
|
936
|
+
judgeVersion: z.ZodNumber;
|
|
937
|
+
}, z.core.$strict>;
|
|
938
|
+
identity: z.ZodObject<{
|
|
939
|
+
judgeModelId: z.ZodString;
|
|
940
|
+
judgeFamily: z.ZodString;
|
|
941
|
+
judgeVersion: z.ZodNumber;
|
|
942
|
+
confidenceThreshold: z.ZodLiteral<"high">;
|
|
943
|
+
panelSize: z.ZodNumber;
|
|
944
|
+
members: z.ZodArray<z.ZodObject<{
|
|
945
|
+
judgeModelId: z.ZodString;
|
|
946
|
+
judgeFamily: z.ZodString;
|
|
947
|
+
}, z.core.$strict>>;
|
|
948
|
+
}, z.core.$strict>;
|
|
949
|
+
constitutionDigest: z.ZodString;
|
|
950
|
+
corpusDigest: z.ZodString;
|
|
951
|
+
goldSetDigest: z.ZodString;
|
|
952
|
+
replayEvidenceDigest: z.ZodString;
|
|
953
|
+
evidenceDigest: z.ZodString;
|
|
954
|
+
minN: z.ZodNumber;
|
|
955
|
+
minNSource: z.ZodEnum<{
|
|
956
|
+
default: "default";
|
|
957
|
+
flag: "flag";
|
|
958
|
+
}>;
|
|
959
|
+
selected: z.ZodNumber;
|
|
960
|
+
admitted: z.ZodNumber;
|
|
961
|
+
cost: z.ZodObject<{
|
|
962
|
+
callBound: z.ZodNumber;
|
|
963
|
+
usage: z.ZodObject<{
|
|
964
|
+
calls: z.ZodNumber;
|
|
965
|
+
inputTokens: z.ZodOptional<z.ZodNumber>;
|
|
966
|
+
outputTokens: z.ZodOptional<z.ZodNumber>;
|
|
967
|
+
}, z.core.$strict>;
|
|
968
|
+
}, z.core.$strict>;
|
|
969
|
+
skippedCounts: z.ZodOptional<z.ZodObject<{
|
|
970
|
+
'missing-executable-correction': z.ZodNumber;
|
|
971
|
+
}, z.core.$strict>>;
|
|
972
|
+
}, z.core.$strict>]>;
|
|
973
|
+
type JudgeCalibrationArtifact = z.infer<typeof schema>;
|
|
974
|
+
interface JudgeCalibrationReadInput extends LoadJudgeCalibrationEvidenceInput {
|
|
975
|
+
identity: JudgeIdentityBlock;
|
|
976
|
+
minN: JudgeCalibrationMinimum;
|
|
977
|
+
}
|
|
978
|
+
type JudgeCalibrationRead = {
|
|
979
|
+
kind: 'absent';
|
|
980
|
+
} | {
|
|
981
|
+
kind: 'loaded';
|
|
982
|
+
artifact: JudgeCalibrationArtifact;
|
|
983
|
+
} | {
|
|
984
|
+
kind: 'refused';
|
|
985
|
+
reason: string;
|
|
986
|
+
};
|
|
987
|
+
type JudgeCalibrationPublication = {
|
|
988
|
+
kind: 'committed' | 'unchanged';
|
|
989
|
+
artifact: JudgeCalibrationArtifact;
|
|
990
|
+
path: string;
|
|
991
|
+
} | {
|
|
992
|
+
kind: 'refused' | 'busy' | 'input-drift' | 'publication-failed';
|
|
993
|
+
reason: string;
|
|
994
|
+
} | {
|
|
995
|
+
kind: 'write-conflict';
|
|
996
|
+
path: string;
|
|
997
|
+
};
|
|
998
|
+
/** Locator only. Admission always checks the complete key and profile separately. */
|
|
999
|
+
declare function judgeCalibrationPath(profileKey: string): string;
|
|
1000
|
+
declare function judgeCalibrationGenerationId(body: unknown): string;
|
|
1001
|
+
/** Strict syntax and arithmetic, including independently recomputed Wilson bounds. */
|
|
1002
|
+
declare function guardJudgeCalibration(value: unknown): JudgeCalibrationArtifact | undefined;
|
|
1003
|
+
/** Project only measured summary data; raw rows, credentials and audit paths cannot cross. */
|
|
1004
|
+
declare function createJudgeCalibrationArtifact(input: JudgeCalibrationReadInput, result: {
|
|
1005
|
+
status: 'capability-gap';
|
|
1006
|
+
reason: 'no-credential' | 'single-family-host';
|
|
1007
|
+
} | ReturnType<typeof _vostride_agent_qa_eval.summarizeJudgeCalibration>): JudgeCalibrationArtifact;
|
|
1008
|
+
declare function readJudgeCalibration(input: Omit<JudgeCalibrationReadInput, 'minN'> & {
|
|
1009
|
+
/** Operators may require an exact minimum; report consumers retain the recorded policy. */
|
|
1010
|
+
minN?: JudgeCalibrationMinimum;
|
|
1011
|
+
}): JudgeCalibrationRead;
|
|
1012
|
+
/** One checked file, with immutable create or explicit atomic replace under the operator lock. */
|
|
1013
|
+
declare function publishJudgeCalibration(input: JudgeCalibrationReadInput & {
|
|
1014
|
+
artifact: JudgeCalibrationArtifact;
|
|
1015
|
+
replace?: boolean;
|
|
1016
|
+
onBoundary?: (stage: string) => void;
|
|
1017
|
+
}): Promise<JudgeCalibrationPublication>;
|
|
1018
|
+
|
|
1019
|
+
type RunJudgeCalibrationResult = Awaited<ReturnType<typeof runJudgePanelCalibration>> | JudgeCalibrationPublication | {
|
|
1020
|
+
kind: 'capability-gap';
|
|
1021
|
+
status: 'capability-gap';
|
|
1022
|
+
reason: 'panel-not-configured' | 'no-credential' | 'single-family-host';
|
|
1023
|
+
minN: number;
|
|
1024
|
+
minNSource: JudgeCalibrationMinimum['source'];
|
|
1025
|
+
n: 0;
|
|
1026
|
+
identity?: JudgeCalibrationReadInput['identity'];
|
|
1027
|
+
cost: {
|
|
1028
|
+
callBound: number;
|
|
1029
|
+
usage: {
|
|
1030
|
+
calls: number;
|
|
1031
|
+
};
|
|
1032
|
+
};
|
|
1033
|
+
artifactDisposition: 'not-created';
|
|
1034
|
+
judgeTargeted: JudgeTargetedCalibrationDisclosure;
|
|
1035
|
+
};
|
|
1036
|
+
declare function maskJudgeCalibrationText<T>(value: T, mask: MaskFn): T;
|
|
1037
|
+
/** Publication routing is separate from the forced probe, so a missing composite does no IO. */
|
|
1038
|
+
declare function runJudgeCalibration(input: {
|
|
1039
|
+
commit?: boolean;
|
|
1040
|
+
replace?: boolean;
|
|
1041
|
+
} & ({
|
|
1042
|
+
kind: 'capability-gap';
|
|
1043
|
+
reason: 'panel-not-configured' | 'no-credential' | 'single-family-host';
|
|
1044
|
+
minN: JudgeCalibrationMinimum;
|
|
1045
|
+
publication?: JudgeCalibrationReadInput;
|
|
1046
|
+
} | {
|
|
1047
|
+
kind: 'probe';
|
|
1048
|
+
probe: RunJudgePanelCalibrationInput;
|
|
1049
|
+
})): Promise<RunJudgeCalibrationResult>;
|
|
1050
|
+
interface RunJudgePanelCalibrationInput {
|
|
1051
|
+
committedRoot: string;
|
|
1052
|
+
agentQaRoot: string;
|
|
1053
|
+
profile: BaselineProfileKeyInput;
|
|
1054
|
+
replayProfile: ModelProfile;
|
|
1055
|
+
members: readonly Omit<JudgeBindingInput, 'mask'>[];
|
|
1056
|
+
minN?: JudgeCalibrationMinimum;
|
|
1057
|
+
replayRunRoots?: readonly string[];
|
|
1058
|
+
mask: MaskFn;
|
|
1059
|
+
}
|
|
1060
|
+
/** Operator-only forced probe. Credentials and source action content never enter its DTO. */
|
|
1061
|
+
declare function runJudgePanelCalibration(input: RunJudgePanelCalibrationInput): Promise<{
|
|
1062
|
+
identity: _vostride_agent_qa_eval.JudgeIdentityBlock;
|
|
1063
|
+
profileKey: string;
|
|
1064
|
+
corpusDigest: string;
|
|
1065
|
+
goldSetDigest: string;
|
|
1066
|
+
replayEvidenceDigest: string;
|
|
1067
|
+
evidenceDigest: string;
|
|
1068
|
+
skippedCounts: Partial<Record<_vostride_agent_qa_eval.JudgeCalibrationSkipReason, number>>;
|
|
1069
|
+
rows: ({
|
|
1070
|
+
axis: "value" | "element-identity";
|
|
1071
|
+
outcome: _vostride_agent_qa_eval.JudgePairwiseOutcome;
|
|
1072
|
+
members: _vostride_agent_qa_eval.JudgeMemberOutcome[] | undefined;
|
|
1073
|
+
usage: {
|
|
1074
|
+
outputTokens?: number | undefined;
|
|
1075
|
+
inputTokens?: number | undefined;
|
|
1076
|
+
calls: number;
|
|
1077
|
+
};
|
|
1078
|
+
kind: "skipped" | "not-invoked" | "attempted";
|
|
1079
|
+
eligibility: _vostride_agent_qa_eval.JudgeEligibility;
|
|
1080
|
+
truth: _vostride_agent_qa_eval.JudgeCalibrationTruth;
|
|
1081
|
+
recordedModelId?: string | undefined;
|
|
1082
|
+
decisionId: string;
|
|
1083
|
+
replayedModelId: string;
|
|
1084
|
+
} | {
|
|
1085
|
+
reason: "no-eligible-judge" | "missing-evidence";
|
|
1086
|
+
kind: "skipped" | "not-invoked" | "attempted";
|
|
1087
|
+
eligibility: _vostride_agent_qa_eval.JudgeEligibility;
|
|
1088
|
+
truth: _vostride_agent_qa_eval.JudgeCalibrationTruth;
|
|
1089
|
+
recordedModelId?: string | undefined;
|
|
1090
|
+
decisionId: string;
|
|
1091
|
+
replayedModelId: string;
|
|
1092
|
+
})[];
|
|
1093
|
+
abstention: {
|
|
1094
|
+
rate?: number | undefined;
|
|
1095
|
+
numerator: number;
|
|
1096
|
+
denominator: number;
|
|
1097
|
+
};
|
|
1098
|
+
disagreement: {
|
|
1099
|
+
pairs: {
|
|
1100
|
+
rate?: number | undefined;
|
|
1101
|
+
numerator: number;
|
|
1102
|
+
denominator: number;
|
|
1103
|
+
memberModelIds: readonly [string, string];
|
|
1104
|
+
}[];
|
|
1105
|
+
rate?: number | undefined;
|
|
1106
|
+
numerator: number;
|
|
1107
|
+
denominator: number;
|
|
1108
|
+
};
|
|
1109
|
+
cost: {
|
|
1110
|
+
callBound: number;
|
|
1111
|
+
usage: Omit<UsageAggregator, "members">;
|
|
1112
|
+
};
|
|
1113
|
+
pooledAgreement?: number | undefined;
|
|
1114
|
+
wilson95?: {
|
|
1115
|
+
lower: number;
|
|
1116
|
+
upper: number;
|
|
1117
|
+
halfWidth: number;
|
|
1118
|
+
} | undefined;
|
|
1119
|
+
status: "calibrated" | "uncalibrated";
|
|
1120
|
+
n: number;
|
|
1121
|
+
minN: number;
|
|
1122
|
+
minNSource: "default" | "flag";
|
|
1123
|
+
counts: {
|
|
1124
|
+
selected: number;
|
|
1125
|
+
admitted: number;
|
|
1126
|
+
eligible: number;
|
|
1127
|
+
attempted: number;
|
|
1128
|
+
eligibilityRefused: number;
|
|
1129
|
+
evidenceSkipped: number;
|
|
1130
|
+
truthComparable: number;
|
|
1131
|
+
unresolvedTruth: number;
|
|
1132
|
+
};
|
|
1133
|
+
perMember: {
|
|
1134
|
+
agreement?: number | undefined;
|
|
1135
|
+
wilson95?: {
|
|
1136
|
+
lower: number;
|
|
1137
|
+
upper: number;
|
|
1138
|
+
halfWidth: number;
|
|
1139
|
+
} | undefined;
|
|
1140
|
+
settled: number;
|
|
1141
|
+
abstained: number;
|
|
1142
|
+
errors: number;
|
|
1143
|
+
n: number;
|
|
1144
|
+
numerator: number;
|
|
1145
|
+
denominator: number;
|
|
1146
|
+
judgeModelId: string;
|
|
1147
|
+
}[];
|
|
1148
|
+
pooled: {
|
|
1149
|
+
agreement?: number | undefined;
|
|
1150
|
+
wilson95?: {
|
|
1151
|
+
lower: number;
|
|
1152
|
+
upper: number;
|
|
1153
|
+
halfWidth: number;
|
|
1154
|
+
} | undefined;
|
|
1155
|
+
settled: number;
|
|
1156
|
+
abstained: number;
|
|
1157
|
+
errors: number;
|
|
1158
|
+
n: number;
|
|
1159
|
+
numerator: number;
|
|
1160
|
+
denominator: number;
|
|
1161
|
+
};
|
|
1162
|
+
judgeTargeted: JudgeTargetedCalibrationDisclosure;
|
|
1163
|
+
kind: "dry-run";
|
|
1164
|
+
}>;
|
|
1165
|
+
|
|
1166
|
+
declare const LABEL_CALIBRATION_PATH = "labels/calibration.json";
|
|
1167
|
+
declare const LABEL_CALIBRATION_SCHEMA_VERSION = 2;
|
|
1168
|
+
interface LabelCalibrationArtifact extends Omit<LabelCalibrationReport, 'minGold'> {
|
|
1169
|
+
schemaVersion: typeof LABEL_CALIBRATION_SCHEMA_VERSION;
|
|
1170
|
+
minGold: {
|
|
1171
|
+
value: number;
|
|
1172
|
+
source: 'default' | 'cli';
|
|
1173
|
+
};
|
|
1174
|
+
evidenceDigest: string;
|
|
1175
|
+
generationId: string;
|
|
1176
|
+
targetRevisions: readonly {
|
|
1177
|
+
cassetteId: string;
|
|
1178
|
+
revisionId: string;
|
|
1179
|
+
}[];
|
|
1180
|
+
}
|
|
1181
|
+
type LabelCalibrationRead = {
|
|
1182
|
+
kind: 'absent';
|
|
1183
|
+
} | {
|
|
1184
|
+
kind: 'refused';
|
|
1185
|
+
reason: string;
|
|
1186
|
+
targetCassetteIds?: readonly string[];
|
|
1187
|
+
} | {
|
|
1188
|
+
kind: 'loaded';
|
|
1189
|
+
artifact: LabelCalibrationArtifact;
|
|
1190
|
+
authorizedRevisionIds: ReadonlySet<string>;
|
|
1191
|
+
};
|
|
1192
|
+
/** Independent marker loading: never calls the corpus reader and never grants a whole generation. */
|
|
1193
|
+
declare function readLabelCalibration(committedRoot: string): LabelCalibrationRead;
|
|
1194
|
+
|
|
1195
|
+
interface RunLabelCalibrationInput {
|
|
1196
|
+
committedRoot: string;
|
|
1197
|
+
agentQaRoot: string;
|
|
1198
|
+
commit?: boolean;
|
|
1199
|
+
replace?: boolean;
|
|
1200
|
+
minGold?: {
|
|
1201
|
+
value: number;
|
|
1202
|
+
source: 'default' | 'cli';
|
|
1203
|
+
};
|
|
1204
|
+
/** Synchronous failure injection at actual publication boundaries; never supplied by learners. */
|
|
1205
|
+
onBoundary?: (stage: string) => void;
|
|
1206
|
+
}
|
|
1207
|
+
type RunLabelCalibrationResult = {
|
|
1208
|
+
kind: 'dry-run' | 'committed' | 'unchanged';
|
|
1209
|
+
artifact: LabelCalibrationArtifact;
|
|
1210
|
+
refreshedCount: number;
|
|
1211
|
+
} | {
|
|
1212
|
+
kind: 'refused' | 'busy' | 'input-drift' | 'publication-failed';
|
|
1213
|
+
reason: string;
|
|
1214
|
+
} | {
|
|
1215
|
+
kind: 'write-conflict';
|
|
1216
|
+
path: string;
|
|
1217
|
+
};
|
|
1218
|
+
/** The only explicit operator producer. Immutable revisions precede the one atomic marker swap. */
|
|
1219
|
+
declare function runLabelCalibration(input: RunLabelCalibrationInput): Promise<RunLabelCalibrationResult>;
|
|
1220
|
+
|
|
1221
|
+
interface LoadLabelEvidenceInput {
|
|
1222
|
+
snapshot: CommittedCorpusSnapshot;
|
|
1223
|
+
agentQaRoot: string;
|
|
1224
|
+
incoming?: CompletedRunHandoff;
|
|
1225
|
+
}
|
|
1226
|
+
interface LabelEvidenceRead {
|
|
1227
|
+
handoffs: CompletedRunHandoff[];
|
|
1228
|
+
marks: MarkReceipt[];
|
|
1229
|
+
refusals: {
|
|
1230
|
+
runId: string;
|
|
1231
|
+
reason: 'invalid-run-link' | 'handoff-unavailable' | 'run-identity-mismatch' | 'product-mismatch' | 'path-outside-root' | 'conflicting-handoff' | 'marks-unavailable';
|
|
1232
|
+
}[];
|
|
1233
|
+
}
|
|
1234
|
+
/** Load only positively linked runs. A slug locates bytes; the guarded run identity authorizes them. */
|
|
1235
|
+
declare function loadLabelEvidence(input: LoadLabelEvidenceInput): LabelEvidenceRead;
|
|
1236
|
+
|
|
1237
|
+
declare const COMMITTED_CASSETTES_DIR = "cassettes";
|
|
1238
|
+
declare const COMMITTED_CORPUS_DIR = "corpus";
|
|
1239
|
+
declare const COMMITTED_LABELS_DIR = "labels";
|
|
1240
|
+
declare const COMMITTED_CONTRACTS_DIR = "contracts";
|
|
1241
|
+
declare const COMMITTED_REPORTS_DIR = "reports";
|
|
1242
|
+
/** 414 D-02: one committed file per excluded run, product-partitioned. */
|
|
1243
|
+
declare const COMMITTED_EXCLUSIONS_DIR = "exclusions";
|
|
1244
|
+
/** 414 D-05: the file-name prefix that makes a partition manifest findable under contracts/. */
|
|
1245
|
+
declare const PARTITION_MANIFEST_FILE_PREFIX = "partition-";
|
|
1246
|
+
/** 413 D-19: baseline pointers live one level under reports/, beside the batchKey dirs. */
|
|
1247
|
+
declare const COMMITTED_BASELINES_SUBDIR = "baselines";
|
|
1248
|
+
/** 413 D-15: the corpus census lives one level under reports/, beside the batchKey dirs. */
|
|
1249
|
+
declare const COMMITTED_CENSUS_SUBDIR = "census";
|
|
1250
|
+
/** 415 D-04: one floor artefact per profile, beside the baseline-pointer set. */
|
|
1251
|
+
declare const COMMITTED_NOISE_FLOOR_SUBDIR = "noise-floor";
|
|
1252
|
+
declare const COMMITTED_ROOT_MARKER_FILE = "eval-root.json";
|
|
1253
|
+
declare const COMMITTED_ROOT_SCHEMA_VERSION = 2;
|
|
1254
|
+
/** Scaffolded ignore content: one rule covering every blobs/ dir at any depth (D-15). */
|
|
1255
|
+
declare const COMMITTED_GITIGNORE_TEXT = "# agent-qa committed eval root: binary blob bytes stay untracked (D-15/L12).\n**/blobs/\n";
|
|
1256
|
+
interface CommittedRootStoreConfig {
|
|
1257
|
+
/**
|
|
1258
|
+
* Explicit absolute committed-root path — `<configDir>/agent-qa-eval` for the Phase 372
|
|
1259
|
+
* CLI. Deliberately NO default: nothing may silently resolve into a workdir (OQ-2).
|
|
1260
|
+
*/
|
|
1261
|
+
committedRoot: string;
|
|
1262
|
+
}
|
|
1263
|
+
type CommittedWriteOutcome = {
|
|
1264
|
+
kind: 'written';
|
|
1265
|
+
} | {
|
|
1266
|
+
kind: 'unchanged';
|
|
1267
|
+
} | {
|
|
1268
|
+
kind: 'write-conflict';
|
|
1269
|
+
path: string;
|
|
1270
|
+
};
|
|
1271
|
+
declare function buildRootMarkerValue(): {
|
|
1272
|
+
schemaVersion: number;
|
|
1273
|
+
};
|
|
1274
|
+
declare function committedCassettePath(product: string, cassetteId: string): string;
|
|
1275
|
+
declare function committedBlobPath(product: string, cassetteId: string, sha: string): string;
|
|
1276
|
+
declare function committedLabelPath(product: string, cassetteId: string): string;
|
|
1277
|
+
/** Preserve validated cassette path segments; suffix only the JSON leaf, never slug the whole ID. */
|
|
1278
|
+
declare function committedLabelRevisionPath(product: string, cassetteId: string, revisionId: string): string;
|
|
1279
|
+
declare function committedCorpusPath(product: string, cassetteId: string): string;
|
|
1280
|
+
declare function committedContractPath(fileName: string): string;
|
|
1281
|
+
/**
|
|
1282
|
+
* Phase 372 D-13: the baseline report commits beside the corpus/contracts that produced it —
|
|
1283
|
+
* one dir per batchKey, so a changed corpus (⇒ changed key) gets a fresh dir under write-once
|
|
1284
|
+
* while an identical regen byte-compares 'unchanged'. Every segment slugged (T-368-10).
|
|
1285
|
+
*/
|
|
1286
|
+
declare function committedReportPath(batchKey: string, fileName: string): string;
|
|
1287
|
+
/**
|
|
1288
|
+
* Phase 413 D-15: the census commits one dir per CORPUS DIGEST, the reports/<batchKey>/ shape
|
|
1289
|
+
* applied to the population rather than to a replay batch — a grown corpus hashes to a fresh
|
|
1290
|
+
* digest and therefore gets a fresh dir under write-once, while re-running the census over an
|
|
1291
|
+
* unchanged corpus byte-compares 'unchanged' and is an idempotent no-op. Every segment slugged,
|
|
1292
|
+
* so a crafted digest cannot escape the committed root (T-368-10 / T-413-18).
|
|
1293
|
+
*/
|
|
1294
|
+
declare function committedCensusPath(corpusDigest: string, fileName: string): string;
|
|
1295
|
+
/**
|
|
1296
|
+
* Phase 413 D-19: baseline designation is a committed pointer, one file per profile key, so
|
|
1297
|
+
* moving a baseline is a reviewed git edit of that file rather than a silent overwrite. The
|
|
1298
|
+
* key is a single path segment and every segment is slugged (T-368-10).
|
|
1299
|
+
*/
|
|
1300
|
+
declare function committedBaselinePointerPath(profileKey: string): string;
|
|
1301
|
+
/** 415 D-04: the entire profile key is one slugged segment, never a batch report dir. */
|
|
1302
|
+
declare function committedNoiseFloorPath(profileKey: string): string;
|
|
1303
|
+
/**
|
|
1304
|
+
* Phase 414 D-05: one partition manifest per product, under contracts/ rather than beside the
|
|
1305
|
+
* entries it governs. Two reasons. (1) The recursive corpus/ walk parses every *.json it finds,
|
|
1306
|
+
* so a manifest under corpus/ would be handed to the corpus-entry guard and refused as a corpus
|
|
1307
|
+
* entry — putting it under contracts/ means the walk never sees it (T-414-20). (2) contracts/ is
|
|
1308
|
+
* already where this root keeps its reviewed constitution documents, and a manifest IS one: a
|
|
1309
|
+
* change to its salt or fractions re-partitions a committed corpus, so it lands as a reviewed git
|
|
1310
|
+
* edit and a named migration, never a silent rewrite. Every segment slugged (T-368-10).
|
|
1311
|
+
*/
|
|
1312
|
+
declare function committedPartitionManifestPath(product: string): string;
|
|
1313
|
+
/**
|
|
1314
|
+
* Phase 414 D-02: one committed file per excluded run, so corpus membership becomes a function of
|
|
1315
|
+
* a committed list rather than a fact about whoever ran the growth campaign. 413 dropped excluded
|
|
1316
|
+
* runs before anything was committed and they left no trace; a record per run lets the census
|
|
1317
|
+
* count what never arrived and 422's curator ask why a run is missing. Every segment slugged, so a
|
|
1318
|
+
* crafted product or runSlug cannot escape the committed root (T-368-10 / T-414-16).
|
|
1319
|
+
*/
|
|
1320
|
+
declare function committedExclusionPath(product: string, runSlug: string): string;
|
|
1321
|
+
declare class FsCommittedRootStore {
|
|
1322
|
+
private readonly committedRoot;
|
|
1323
|
+
constructor(config: CommittedRootStoreConfig);
|
|
1324
|
+
/**
|
|
1325
|
+
* Scaffold the committed root deterministically: the blobs-covering .gitignore plus the
|
|
1326
|
+
* schemaVersion root marker (D-15). Both texts arrive as ALREADY-CHECKED strings from
|
|
1327
|
+
* the seeding composition helper — the store never imports the guardrail. Re-scaffolding
|
|
1328
|
+
* an intact root yields two 'unchanged' outcomes and mutates nothing.
|
|
1329
|
+
*/
|
|
1330
|
+
scaffold(checkedGitignoreText: string, checkedRootMarkerText: string): {
|
|
1331
|
+
gitignore: CommittedWriteOutcome;
|
|
1332
|
+
rootMarker: CommittedWriteOutcome;
|
|
1333
|
+
};
|
|
1334
|
+
/** The write-once comparison WITHOUT any mkdir or write — pure read + compare. */
|
|
1335
|
+
previewText(relPath: string, text: string): 'absent' | 'identical' | 'conflict';
|
|
1336
|
+
/** Write-once text: read-compare-refuse, never overwrite (D-17). */
|
|
1337
|
+
writeText(relPath: string, text: string): CommittedWriteOutcome;
|
|
1338
|
+
/**
|
|
1339
|
+
* Write-once blob bytes, confined to ignore-covered blobs/ dirs. The sha must match the
|
|
1340
|
+
* lowercase 64-hex content-address format, the byte hash must equal it, and the target
|
|
1341
|
+
* path must be `.../blobs/<sha>` — any verification failure is a typed refusal, so
|
|
1342
|
+
* corrupt bytes never land under an untrue content address (T-371-11).
|
|
1343
|
+
*/
|
|
1344
|
+
writeBlob(relBlobPath: string, blob: ExtractedBlob): CommittedWriteOutcome;
|
|
1345
|
+
}
|
|
1346
|
+
|
|
1347
|
+
interface SeedCommittedCorpusInput {
|
|
1348
|
+
/** The pure eval-core plan — the ONLY decision-maker; this helper executes it verbatim (D-16). */
|
|
1349
|
+
plan: SeedingPlan;
|
|
1350
|
+
/** The extracted cassettes the plan's cases reference (cassette bytes + verified blobs). */
|
|
1351
|
+
cassettes: readonly ExtractedCassette[];
|
|
1352
|
+
/** Explicit absolute committed-root path — `<configDir>/agent-qa-eval` for the 372 CLI. */
|
|
1353
|
+
committedRoot: string;
|
|
1354
|
+
/** 441 GAP3-D1: the workspace's declared-secret mask. Required; a missing one throws first. */
|
|
1355
|
+
mask: MaskFn;
|
|
1356
|
+
}
|
|
1357
|
+
type SeedingRefusal = {
|
|
1358
|
+
cassetteId: string;
|
|
1359
|
+
kind: 'write-conflict';
|
|
1360
|
+
path: string;
|
|
1361
|
+
} | {
|
|
1362
|
+
cassetteId: string;
|
|
1363
|
+
kind: 'missing-cassette';
|
|
1364
|
+
}
|
|
1365
|
+
/** A committed string of the case still holds a declared value raw; nothing of it is written. */
|
|
1366
|
+
| {
|
|
1367
|
+
cassetteId: string;
|
|
1368
|
+
kind: 'unmasked-declared-value';
|
|
1369
|
+
};
|
|
1370
|
+
interface SeedingReport {
|
|
1371
|
+
supersededOriginalCount: number;
|
|
1372
|
+
retirementCounts: NonNullable<SeedingPlan['retirementCounts']>;
|
|
1373
|
+
softEvidence?: SeedingPlan['softEvidence'];
|
|
1374
|
+
product: string;
|
|
1375
|
+
runId: string;
|
|
1376
|
+
committedRoot: string;
|
|
1377
|
+
/** Cases with at least one newly written file. */
|
|
1378
|
+
seededCount: number;
|
|
1379
|
+
/** Cases whose every file was byte-identical (D-17 idempotent no-op). */
|
|
1380
|
+
unchangedCount: number;
|
|
1381
|
+
/** Typed per-case refusals — write conflicts and missing cassettes. Never silent. */
|
|
1382
|
+
refusals: SeedingRefusal[];
|
|
1383
|
+
/** Matcher skips, carried verbatim from the plan. */
|
|
1384
|
+
pairSkips: PairSkip[];
|
|
1385
|
+
/** Sampler exclusions, carried verbatim from the plan. */
|
|
1386
|
+
silverExclusions: SilverExclusion[];
|
|
1387
|
+
/** Both pair classes counted, carried verbatim from the plan (D-07). */
|
|
1388
|
+
pairCounts: {
|
|
1389
|
+
total: number;
|
|
1390
|
+
sameState: number;
|
|
1391
|
+
differingState: number;
|
|
1392
|
+
};
|
|
1393
|
+
/** Count of 'procedure-granularity' exclusions, carried verbatim from the plan. */
|
|
1394
|
+
proceduresExcludedCount: number;
|
|
1395
|
+
/** Per-contract-file outcome for the D-04 constitution projections. */
|
|
1396
|
+
contracts: {
|
|
1397
|
+
fileName: string;
|
|
1398
|
+
outcome: CommittedWriteOutcome['kind'];
|
|
1399
|
+
}[];
|
|
1400
|
+
/** D-15 scaffold outcomes for the blobs-covering gitignore and the schemaVersion marker. */
|
|
1401
|
+
scaffold: {
|
|
1402
|
+
gitignore: CommittedWriteOutcome['kind'];
|
|
1403
|
+
marker: CommittedWriteOutcome['kind'];
|
|
1404
|
+
};
|
|
1405
|
+
}
|
|
1406
|
+
/**
|
|
1407
|
+
* Execute a SeedingPlan against the committed root: scaffold (D-15), contracts projections
|
|
1408
|
+
* (D-04), then each case as an atomic preview-then-write unit (D-14/D-17). Conflicted cases
|
|
1409
|
+
* refuse atomically; the batch always runs to completion.
|
|
1410
|
+
*/
|
|
1411
|
+
/** 2026-09-08, 418 D-21: one reviewed invocation may contain several actual product/run plans.
|
|
1412
|
+
* The existing per-case sequential writer remains the unit; no whole-root transaction is implied.
|
|
1413
|
+
*/
|
|
1414
|
+
interface SeedCommittedCorpusBatchInput extends Omit<SeedCommittedCorpusInput, 'plan' | 'cassettes'> {
|
|
1415
|
+
batches: readonly Pick<SeedCommittedCorpusInput, 'plan' | 'cassettes'>[];
|
|
1416
|
+
}
|
|
1417
|
+
interface SeedingBatchReport {
|
|
1418
|
+
reports: SeedingReport[];
|
|
1419
|
+
}
|
|
1420
|
+
declare function seedCommittedCorpus(input: SeedCommittedCorpusBatchInput): SeedingBatchReport;
|
|
1421
|
+
declare function seedCommittedCorpus(input: SeedCommittedCorpusInput): SeedingReport;
|
|
1422
|
+
/** Both writers answer with the same outcomes. */
|
|
1423
|
+
type CommittedDocumentOutcome = {
|
|
1424
|
+
kind: 'written' | 'unchanged';
|
|
1425
|
+
path: string;
|
|
1426
|
+
} | {
|
|
1427
|
+
kind: 'write-conflict';
|
|
1428
|
+
path: string;
|
|
1429
|
+
};
|
|
1430
|
+
interface WritePartitionManifestInput {
|
|
1431
|
+
/** An already-guarded manifest; its `product` names the committed file. */
|
|
1432
|
+
manifest: PartitionManifest;
|
|
1433
|
+
/** Explicit absolute committed-root path — `<configDir>/agent-qa-eval` for the CLI. */
|
|
1434
|
+
committedRoot: string;
|
|
1435
|
+
}
|
|
1436
|
+
type WritePartitionManifestOutcome = CommittedDocumentOutcome;
|
|
1437
|
+
interface WriteRunExclusionInput {
|
|
1438
|
+
/** An already-guarded exclusion record; its product + runSlug name the file. */
|
|
1439
|
+
record: RunExclusionRecord;
|
|
1440
|
+
committedRoot: string;
|
|
1441
|
+
}
|
|
1442
|
+
type WriteRunExclusionOutcome = CommittedDocumentOutcome;
|
|
1443
|
+
/**
|
|
1444
|
+
* Commit one product's partition manifest (414 D-05).
|
|
1445
|
+
*
|
|
1446
|
+
* Write-once is the entire point. The salt and the two fractions decide which split every
|
|
1447
|
+
* cassette in the product lands in, so a second manifest whose flags disagree with the committed
|
|
1448
|
+
* one is not an update — it silently re-partitions a corpus that has already been gated against.
|
|
1449
|
+
* That is refused as a `write-conflict` with the committed bytes untouched; changing a live
|
|
1450
|
+
* partition is a reviewed git edit plus a named migration, never a rewrite through this function.
|
|
1451
|
+
*/
|
|
1452
|
+
declare function writePartitionManifest(input: WritePartitionManifestInput): WritePartitionManifestOutcome;
|
|
1453
|
+
/**
|
|
1454
|
+
* Commit one excluded run's record (414 D-02).
|
|
1455
|
+
*
|
|
1456
|
+
* Identical-regen is decided on `runExclusionCore` — the record WITHOUT its `backfill` block —
|
|
1457
|
+
* rather than on the bytes. `backfill` records a LATER act (entries deleted from the root after
|
|
1458
|
+
* the fact); the rest of the record is the original exclusion decision. Comparing bytes would
|
|
1459
|
+
* make re-recording an unchanged decision beside a newly-added backfill look like a conflicting
|
|
1460
|
+
* exclusion, and would equally refuse the mirror case. Anything that differs in the CORE — a
|
|
1461
|
+
* different verdict class, a different analysisDigest — still falls through to the write-once
|
|
1462
|
+
* preview and refuses.
|
|
1463
|
+
*/
|
|
1464
|
+
declare function writeRunExclusion(input: WriteRunExclusionInput): WriteRunExclusionOutcome;
|
|
1465
|
+
|
|
1466
|
+
interface FreezeGoldenTrajectoryInput {
|
|
1467
|
+
agentQaRoot: string;
|
|
1468
|
+
committedRoot: string;
|
|
1469
|
+
product: string;
|
|
1470
|
+
mark: MarkReceipt;
|
|
1471
|
+
handoff: CompletedRunHandoff;
|
|
1472
|
+
supersedes?: string;
|
|
1473
|
+
priorGold?: MarkReceipt['gold'];
|
|
1474
|
+
/** 441 GAP3-D1: the declared mask the run and the mark's correction are frozen under. */
|
|
1475
|
+
mask: MaskFn;
|
|
1476
|
+
}
|
|
1477
|
+
type FreezeOutcome = {
|
|
1478
|
+
kind: 'ambiguous-correction';
|
|
1479
|
+
decisionCount: number;
|
|
1480
|
+
} | {
|
|
1481
|
+
kind: 'frozen';
|
|
1482
|
+
cases: MarkReceipt['gold'];
|
|
1483
|
+
census: 'committed' | 'unchanged' | 'write-conflict';
|
|
1484
|
+
} | {
|
|
1485
|
+
kind: 'refused';
|
|
1486
|
+
refusals: SeedingRefusal[];
|
|
1487
|
+
} | {
|
|
1488
|
+
kind: 'step-not-found';
|
|
1489
|
+
stepCount: number;
|
|
1490
|
+
} | {
|
|
1491
|
+
kind: 'no-leaf-cassette';
|
|
1492
|
+
};
|
|
1493
|
+
declare function freezeGoldenTrajectory(input: FreezeGoldenTrajectoryInput): FreezeOutcome;
|
|
1494
|
+
|
|
1495
|
+
declare const MARKS_LEDGER_FILE = "marks.jsonl";
|
|
1496
|
+
declare const MARKS_LEDGER_LOCK_FILE = "marks.jsonl.lock";
|
|
1497
|
+
/** A run's one unfinished mark, beside its ledger. */
|
|
1498
|
+
declare const MARK_INTENT_FILE = "mark-intent.json";
|
|
1499
|
+
declare const intentSchema: z.ZodObject<{
|
|
1500
|
+
schemaVersion: z.ZodLiteral<2>;
|
|
1501
|
+
mark: z.ZodObject<{
|
|
1502
|
+
runSlug: z.ZodString;
|
|
1503
|
+
kind: z.ZodEnum<{
|
|
1504
|
+
"false-fail": "false-fail";
|
|
1505
|
+
"false-pass": "false-pass";
|
|
1506
|
+
"confirmed-bug": "confirmed-bug";
|
|
1507
|
+
}>;
|
|
1508
|
+
origin: z.ZodLiteral<"user">;
|
|
1509
|
+
ordinal: z.ZodNumber;
|
|
1510
|
+
markedAt: z.ZodNumber;
|
|
1511
|
+
note: z.ZodOptional<z.ZodString>;
|
|
1512
|
+
correctedExpectation: z.ZodOptional<z.ZodString>;
|
|
1513
|
+
correctedAction: z.ZodOptional<z.ZodType<_vostride_agent_qa_schema.Action, unknown, z.core.$ZodTypeInternals<_vostride_agent_qa_schema.Action, unknown>>>;
|
|
1514
|
+
rootCause: z.ZodOptional<z.ZodObject<{
|
|
1515
|
+
class: z.ZodEnum<{
|
|
1516
|
+
unknown: "unknown";
|
|
1517
|
+
environment: "environment";
|
|
1518
|
+
timeout: "timeout";
|
|
1519
|
+
test_bug: "test_bug";
|
|
1520
|
+
app_bug: "app_bug";
|
|
1521
|
+
flake: "flake";
|
|
1522
|
+
config_error: "config_error";
|
|
1523
|
+
llm_error: "llm_error";
|
|
1524
|
+
}>;
|
|
1525
|
+
producer: z.ZodString;
|
|
1526
|
+
}, z.core.$strict>>;
|
|
1527
|
+
isRerun: z.ZodOptional<z.ZodLiteral<true>>;
|
|
1528
|
+
disposition: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
1529
|
+
kind: z.ZodLiteral<"quarantined">;
|
|
1530
|
+
class: z.ZodEnum<{
|
|
1531
|
+
environment: "environment";
|
|
1532
|
+
timeout: "timeout";
|
|
1533
|
+
flake: "flake";
|
|
1534
|
+
}>;
|
|
1535
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1536
|
+
kind: z.ZodLiteral<"applied">;
|
|
1537
|
+
fix: z.ZodEnum<{
|
|
1538
|
+
fact: "fact";
|
|
1539
|
+
caution: "caution";
|
|
1540
|
+
"procedure-hint": "procedure-hint";
|
|
1541
|
+
"test-edit": "test-edit";
|
|
1542
|
+
}>;
|
|
1543
|
+
change: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
1544
|
+
kind: z.ZodLiteral<"knowledge">;
|
|
1545
|
+
changeId: z.ZodString;
|
|
1546
|
+
recordIds: z.ZodArray<z.ZodString>;
|
|
1547
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1548
|
+
kind: z.ZodLiteral<"test-file">;
|
|
1549
|
+
filePath: z.ZodString;
|
|
1550
|
+
contentDigest: z.ZodString;
|
|
1551
|
+
}, z.core.$strict>], "kind">;
|
|
1552
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1553
|
+
kind: z.ZodLiteral<"no-draftable-fix">;
|
|
1554
|
+
checked: z.ZodObject<{
|
|
1555
|
+
servedRecords: z.ZodNumber;
|
|
1556
|
+
procedureReads: z.ZodNumber;
|
|
1557
|
+
assertionDrift: z.ZodBoolean;
|
|
1558
|
+
rootCause: z.ZodOptional<z.ZodEnum<{
|
|
1559
|
+
unknown: "unknown";
|
|
1560
|
+
environment: "environment";
|
|
1561
|
+
timeout: "timeout";
|
|
1562
|
+
test_bug: "test_bug";
|
|
1563
|
+
app_bug: "app_bug";
|
|
1564
|
+
flake: "flake";
|
|
1565
|
+
config_error: "config_error";
|
|
1566
|
+
llm_error: "llm_error";
|
|
1567
|
+
}>>;
|
|
1568
|
+
}, z.core.$strict>;
|
|
1569
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1570
|
+
kind: z.ZodLiteral<"confirmed-bug">;
|
|
1571
|
+
confirmedRecords: z.ZodNumber;
|
|
1572
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1573
|
+
kind: z.ZodLiteral<"not-applied">;
|
|
1574
|
+
fix: z.ZodEnum<{
|
|
1575
|
+
fact: "fact";
|
|
1576
|
+
caution: "caution";
|
|
1577
|
+
"procedure-hint": "procedure-hint";
|
|
1578
|
+
"test-edit": "test-edit";
|
|
1579
|
+
}>;
|
|
1580
|
+
reason: z.ZodEnum<{
|
|
1581
|
+
"policy-materialize-refused": "policy-materialize-refused";
|
|
1582
|
+
"target-not-found": "target-not-found";
|
|
1583
|
+
"outside-workspace": "outside-workspace";
|
|
1584
|
+
}>;
|
|
1585
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1586
|
+
kind: z.ZodLiteral<"drafted">;
|
|
1587
|
+
fix: z.ZodEnum<{
|
|
1588
|
+
fact: "fact";
|
|
1589
|
+
caution: "caution";
|
|
1590
|
+
"procedure-hint": "procedure-hint";
|
|
1591
|
+
"test-edit": "test-edit";
|
|
1592
|
+
}>;
|
|
1593
|
+
proposalId: z.ZodString;
|
|
1594
|
+
proposalKind: z.ZodEnum<{
|
|
1595
|
+
knowledge: "knowledge";
|
|
1596
|
+
"test-draft": "test-draft";
|
|
1597
|
+
}>;
|
|
1598
|
+
}, z.core.$strict>], "kind">;
|
|
1599
|
+
gold: z.ZodArray<z.ZodObject<{
|
|
1600
|
+
cassetteId: z.ZodString;
|
|
1601
|
+
sourceCassetteId: z.ZodString;
|
|
1602
|
+
channel: z.ZodEnum<{
|
|
1603
|
+
"user-mark": "user-mark";
|
|
1604
|
+
"overturned-verdict": "overturned-verdict";
|
|
1605
|
+
}>;
|
|
1606
|
+
}, z.core.$strict>>;
|
|
1607
|
+
evidence: z.ZodObject<{
|
|
1608
|
+
confirmed: z.ZodNumber;
|
|
1609
|
+
overturned: z.ZodNumber;
|
|
1610
|
+
}, z.core.$strict>;
|
|
1611
|
+
supersedes: z.ZodOptional<z.ZodString>;
|
|
1612
|
+
runId: z.ZodType<_vostride_agent_qa_schema.RunId, unknown, z.core.$ZodTypeInternals<_vostride_agent_qa_schema.RunId, unknown>>;
|
|
1613
|
+
testIndex: z.ZodNumber;
|
|
1614
|
+
testId: z.ZodOptional<z.ZodString>;
|
|
1615
|
+
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
1616
|
+
schemaVersion: z.ZodLiteral<2>;
|
|
1617
|
+
markId: z.ZodString;
|
|
1618
|
+
}, z.core.$strict>;
|
|
1619
|
+
stage: z.ZodEnum<{
|
|
1620
|
+
prepared: "prepared";
|
|
1621
|
+
evidence: "evidence";
|
|
1622
|
+
applied: "applied";
|
|
1623
|
+
}>;
|
|
1624
|
+
operationId: z.ZodUUID;
|
|
1625
|
+
plan: z.ZodOptional<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
1626
|
+
fix: z.ZodEnum<{
|
|
1627
|
+
fact: "fact";
|
|
1628
|
+
caution: "caution";
|
|
1629
|
+
"procedure-hint": "procedure-hint";
|
|
1630
|
+
}>;
|
|
1631
|
+
target: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
1632
|
+
grain: z.ZodLiteral<"record">;
|
|
1633
|
+
recordId: z.ZodString;
|
|
1634
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1635
|
+
grain: z.ZodLiteral<"bundle">;
|
|
1636
|
+
shortId: z.ZodString;
|
|
1637
|
+
}, z.core.$strict>], "grain">;
|
|
1638
|
+
deltas: z.ZodArray<z.ZodType<_vostride_agent_qa_schema.Delta, unknown, z.core.$ZodTypeInternals<_vostride_agent_qa_schema.Delta, unknown>>>;
|
|
1639
|
+
wording: z.ZodEnum<{
|
|
1640
|
+
floor: "floor";
|
|
1641
|
+
model: "model";
|
|
1642
|
+
}>;
|
|
1643
|
+
schemaVersion: z.ZodLiteral<2>;
|
|
1644
|
+
markId: z.ZodString;
|
|
1645
|
+
feedbackSource: z.ZodObject<{
|
|
1646
|
+
runId: z.ZodType<_vostride_agent_qa_schema.RunId, unknown, z.core.$ZodTypeInternals<_vostride_agent_qa_schema.RunId, unknown>>;
|
|
1647
|
+
testIndex: z.ZodNumber;
|
|
1648
|
+
testId: z.ZodOptional<z.ZodString>;
|
|
1649
|
+
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
1650
|
+
kind: z.ZodEnum<{
|
|
1651
|
+
"false-fail": "false-fail";
|
|
1652
|
+
"false-pass": "false-pass";
|
|
1653
|
+
"confirmed-bug": "confirmed-bug";
|
|
1654
|
+
}>;
|
|
1655
|
+
}, z.core.$strict>;
|
|
1656
|
+
runTraceRef: z.ZodObject<{
|
|
1657
|
+
runId: z.ZodType<_vostride_agent_qa_schema.RunId, unknown, z.core.$ZodTypeInternals<_vostride_agent_qa_schema.RunId, unknown>>;
|
|
1658
|
+
runSlug: z.ZodString;
|
|
1659
|
+
}, z.core.$strict>;
|
|
1660
|
+
kind: z.ZodLiteral<"knowledge">;
|
|
1661
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1662
|
+
fix: z.ZodLiteral<"test-edit">;
|
|
1663
|
+
testRef: z.ZodString;
|
|
1664
|
+
intent: z.ZodString;
|
|
1665
|
+
wording: z.ZodEnum<{
|
|
1666
|
+
floor: "floor";
|
|
1667
|
+
model: "model";
|
|
1668
|
+
}>;
|
|
1669
|
+
schemaVersion: z.ZodLiteral<2>;
|
|
1670
|
+
markId: z.ZodString;
|
|
1671
|
+
feedbackSource: z.ZodObject<{
|
|
1672
|
+
runId: z.ZodType<_vostride_agent_qa_schema.RunId, unknown, z.core.$ZodTypeInternals<_vostride_agent_qa_schema.RunId, unknown>>;
|
|
1673
|
+
testIndex: z.ZodNumber;
|
|
1674
|
+
testId: z.ZodOptional<z.ZodString>;
|
|
1675
|
+
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
1676
|
+
kind: z.ZodEnum<{
|
|
1677
|
+
"false-fail": "false-fail";
|
|
1678
|
+
"false-pass": "false-pass";
|
|
1679
|
+
"confirmed-bug": "confirmed-bug";
|
|
1680
|
+
}>;
|
|
1681
|
+
}, z.core.$strict>;
|
|
1682
|
+
runTraceRef: z.ZodObject<{
|
|
1683
|
+
runId: z.ZodType<_vostride_agent_qa_schema.RunId, unknown, z.core.$ZodTypeInternals<_vostride_agent_qa_schema.RunId, unknown>>;
|
|
1684
|
+
runSlug: z.ZodString;
|
|
1685
|
+
}, z.core.$strict>;
|
|
1686
|
+
kind: z.ZodLiteral<"test-edit">;
|
|
1687
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
1688
|
+
checked: z.ZodObject<{
|
|
1689
|
+
servedRecords: z.ZodNumber;
|
|
1690
|
+
procedureReads: z.ZodNumber;
|
|
1691
|
+
assertionDrift: z.ZodBoolean;
|
|
1692
|
+
rootCause: z.ZodOptional<z.ZodEnum<{
|
|
1693
|
+
unknown: "unknown";
|
|
1694
|
+
environment: "environment";
|
|
1695
|
+
timeout: "timeout";
|
|
1696
|
+
test_bug: "test_bug";
|
|
1697
|
+
app_bug: "app_bug";
|
|
1698
|
+
flake: "flake";
|
|
1699
|
+
config_error: "config_error";
|
|
1700
|
+
llm_error: "llm_error";
|
|
1701
|
+
}>>;
|
|
1702
|
+
}, z.core.$strict>;
|
|
1703
|
+
schemaVersion: z.ZodLiteral<2>;
|
|
1704
|
+
markId: z.ZodString;
|
|
1705
|
+
feedbackSource: z.ZodObject<{
|
|
1706
|
+
runId: z.ZodType<_vostride_agent_qa_schema.RunId, unknown, z.core.$ZodTypeInternals<_vostride_agent_qa_schema.RunId, unknown>>;
|
|
1707
|
+
testIndex: z.ZodNumber;
|
|
1708
|
+
testId: z.ZodOptional<z.ZodString>;
|
|
1709
|
+
stepIndex: z.ZodOptional<z.ZodNumber>;
|
|
1710
|
+
kind: z.ZodEnum<{
|
|
1711
|
+
"false-fail": "false-fail";
|
|
1712
|
+
"false-pass": "false-pass";
|
|
1713
|
+
"confirmed-bug": "confirmed-bug";
|
|
1714
|
+
}>;
|
|
1715
|
+
}, z.core.$strict>;
|
|
1716
|
+
runTraceRef: z.ZodObject<{
|
|
1717
|
+
runId: z.ZodType<_vostride_agent_qa_schema.RunId, unknown, z.core.$ZodTypeInternals<_vostride_agent_qa_schema.RunId, unknown>>;
|
|
1718
|
+
runSlug: z.ZodString;
|
|
1719
|
+
}, z.core.$strict>;
|
|
1720
|
+
kind: z.ZodLiteral<"no-draftable-fix">;
|
|
1721
|
+
}, z.core.$strict>], "kind">>;
|
|
1722
|
+
}, z.core.$strict>;
|
|
1723
|
+
type MarkIntent = z.infer<typeof intentSchema>;
|
|
1724
|
+
declare class MarkLedgerBusyError extends Error {
|
|
1725
|
+
constructor();
|
|
1726
|
+
}
|
|
1727
|
+
declare class MarkLedgerCorruptError extends Error {
|
|
1728
|
+
readonly line: number;
|
|
1729
|
+
constructor(line: number);
|
|
1730
|
+
}
|
|
1731
|
+
/**
|
|
1732
|
+
* 2026-10-03 (quick 261003-lgd, Phase 432 review IN-23): the run's unfinished-mark record cannot be
|
|
1733
|
+
* resumed, refused by the record's own name. It used to surface as `MarkLedgerCorruptError` at the
|
|
1734
|
+
* NEXT ordinal, a line `marks.jsonl` does not have. D-02: nothing is migrated or read further; the
|
|
1735
|
+
* message names the file to remove, after which the run can be marked again.
|
|
1736
|
+
*/
|
|
1737
|
+
declare class MarkIntentUnusableError extends Error {
|
|
1738
|
+
readonly intentPath: string;
|
|
1739
|
+
readonly reason: 'earlier-version' | 'unreadable' | 'mismatched';
|
|
1740
|
+
constructor(intentPath: string, reason: 'earlier-version' | 'unreadable' | 'mismatched');
|
|
1741
|
+
}
|
|
1742
|
+
/** The one latest-per-address rule, over receipts a caller has already read once. */
|
|
1743
|
+
declare function latestMarkReceiptsPerAddress(receipts: readonly MarkReceipt[]): MarkReceipt[];
|
|
1744
|
+
declare function latestMarkReceiptFor(receipts: readonly MarkReceipt[], runId: string, address: MarkAddress): MarkReceipt | undefined;
|
|
1745
|
+
declare class MarkLedgerNoRunError extends Error {
|
|
1746
|
+
constructor();
|
|
1747
|
+
}
|
|
1748
|
+
/** Gate-side runtime ledger. The proposer reader entry never exports this store. */
|
|
1749
|
+
declare class FsMarkLedger {
|
|
1750
|
+
private readonly dir;
|
|
1751
|
+
private readonly file;
|
|
1752
|
+
private readonly lock;
|
|
1753
|
+
private readonly runId;
|
|
1754
|
+
private readonly runSlug;
|
|
1755
|
+
private inTransaction;
|
|
1756
|
+
private intentValue;
|
|
1757
|
+
constructor(config: {
|
|
1758
|
+
root: string;
|
|
1759
|
+
runId: string;
|
|
1760
|
+
});
|
|
1761
|
+
read(): MarkReceipt[];
|
|
1762
|
+
latestPerAddress(): MarkReceipt[];
|
|
1763
|
+
latestFor(address: MarkAddress): MarkReceipt | undefined;
|
|
1764
|
+
/** Private resumable state is admitted only while this run's existing lock is held. */
|
|
1765
|
+
prepareIntent(mark: MarkReceipt, operationId: string): MarkIntent;
|
|
1766
|
+
checkpointIntent(value: MarkIntent): MarkIntent;
|
|
1767
|
+
transact(fn: (nextOrdinal: number) => Promise<Omit<MarkReceipt, 'ordinal'> | undefined>): Promise<MarkReceipt | undefined>;
|
|
1768
|
+
append(record: Omit<MarkReceipt, 'ordinal'>): Promise<MarkReceipt | undefined>;
|
|
1769
|
+
}
|
|
1770
|
+
|
|
1771
|
+
interface FsHandoffSourceConfig {
|
|
1772
|
+
/** Runtime root (for example, the workdir's `.agent-qa` directory). */
|
|
1773
|
+
root: string;
|
|
1774
|
+
/** The run to load; runner-local sanitizes it before path use. */
|
|
1775
|
+
runId: string;
|
|
1776
|
+
}
|
|
1777
|
+
declare class FsHandoffSource implements HandoffSource {
|
|
1778
|
+
private readonly file;
|
|
1779
|
+
constructor(config: FsHandoffSourceConfig);
|
|
1780
|
+
load(): CompletedRunHandoff;
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1783
|
+
declare const absentArtifactSource: ArtifactSource;
|
|
1784
|
+
interface DirArtifactSourceConfig {
|
|
1785
|
+
/** Directory containing one file per lowercase 64-hex content sha. */
|
|
1786
|
+
dir: string;
|
|
1787
|
+
}
|
|
1788
|
+
declare class DirArtifactSource implements ArtifactSource {
|
|
1789
|
+
private readonly dir;
|
|
1790
|
+
constructor(config: DirArtifactSourceConfig);
|
|
1791
|
+
getBlob(sha: string): Uint8Array | undefined;
|
|
1792
|
+
}
|
|
1793
|
+
interface FsRunArtifactSourceConfig {
|
|
1794
|
+
/** Runtime root the production run store persisted under (e.g. `.agent-qa`). */
|
|
1795
|
+
root: string;
|
|
1796
|
+
/** The run whose sha-keyed screenshot blobs this source reads. */
|
|
1797
|
+
runId: string;
|
|
1798
|
+
}
|
|
1799
|
+
declare class FsRunArtifactSource implements ArtifactSource {
|
|
1800
|
+
private readonly store;
|
|
1801
|
+
constructor(config: FsRunArtifactSourceConfig);
|
|
1802
|
+
getBlob(sha: string): Uint8Array | undefined;
|
|
1803
|
+
}
|
|
1804
|
+
|
|
1805
|
+
declare const CASSETTE_FILE = "cassette.json";
|
|
1806
|
+
declare const BLOBS_SUBDIR = "blobs";
|
|
1807
|
+
declare const DEFAULT_EVAL_CASSETTES_SUBDIR = "eval/cassettes";
|
|
1808
|
+
interface FsCassetteStoreConfig {
|
|
1809
|
+
/** Runtime root (normally the workdir's `.agent-qa` directory). */
|
|
1810
|
+
root: string;
|
|
1811
|
+
/** Optional retargeted root used by later committed-corpus composition. */
|
|
1812
|
+
outputDir?: string;
|
|
1813
|
+
}
|
|
1814
|
+
interface WrittenCassette {
|
|
1815
|
+
id: string;
|
|
1816
|
+
dir: string;
|
|
1817
|
+
writtenBlobShas: string[];
|
|
1818
|
+
corruptBlobShas: string[];
|
|
1819
|
+
}
|
|
1820
|
+
declare class FsCassetteStore {
|
|
1821
|
+
private readonly outputRoot;
|
|
1822
|
+
constructor(config: FsCassetteStoreConfig);
|
|
1823
|
+
write(extracted: ExtractedCassette): WrittenCassette;
|
|
1824
|
+
}
|
|
1825
|
+
|
|
1826
|
+
/** The discriminated bundle-load outcome: a guarded cassette + its blob source, or a typed refusal. */
|
|
1827
|
+
type CassetteBundleLoad = {
|
|
1828
|
+
kind: 'loaded';
|
|
1829
|
+
cassette: Cassette;
|
|
1830
|
+
artifacts: ArtifactSource;
|
|
1831
|
+
} | {
|
|
1832
|
+
kind: 'refused';
|
|
1833
|
+
refusal: ReplayRefusal;
|
|
1834
|
+
};
|
|
1835
|
+
/**
|
|
1836
|
+
* Load a cassette bundle directory into a guarded Cassette + DirArtifactSource over its blobs.
|
|
1837
|
+
* Never throws for a malformed bundle — read/parse failures map to a 'parse-failed' refusal and
|
|
1838
|
+
* guard rejections (version or structural) map to 'guard-failed', each carrying the underlying
|
|
1839
|
+
* message in `detail` (D-08).
|
|
1840
|
+
*/
|
|
1841
|
+
declare function loadCassetteBundle(bundleDir: string): CassetteBundleLoad;
|
|
1842
|
+
|
|
1843
|
+
declare const SCORE_FILE = "score.json";
|
|
1844
|
+
/** The persisted per-cassette score document — computed entirely in eval core (F9). */
|
|
1845
|
+
interface ScoreRecord {
|
|
1846
|
+
cassetteId: string;
|
|
1847
|
+
/** The matcher constitution that minted the score (attribution stamp). */
|
|
1848
|
+
matcherVersion: number;
|
|
1849
|
+
score: CassetteScore;
|
|
1850
|
+
diffs: readonly DecisionDiff[];
|
|
1851
|
+
/**
|
|
1852
|
+
* D-16 score-side reproducibility receipt: the full identity of the judge that produced any
|
|
1853
|
+
* stamp on this score — judge model id, resolved family, judgeVersion, threshold. Computed
|
|
1854
|
+
* entirely by eval-core adjudication and serialized verbatim here (F9: eval-local never
|
|
1855
|
+
* computes, only writes). Typed-absent when no judge walk ran, keeping 369 records
|
|
1856
|
+
* byte-compatible. The cassette's reserved tier4JudgeId is Phase 371 label territory —
|
|
1857
|
+
* the scorer never writes back into the measurement.
|
|
1858
|
+
*/
|
|
1859
|
+
judge?: JudgeIdentityBlock;
|
|
1860
|
+
scoringContext?: ScoringContext;
|
|
1861
|
+
}
|
|
1862
|
+
interface FsScoreStoreConfig {
|
|
1863
|
+
/** Runtime root (normally the workdir's `.agent-qa` directory). */
|
|
1864
|
+
root: string;
|
|
1865
|
+
/** Optional retargeted root used by later committed-corpus composition. */
|
|
1866
|
+
outputDir?: string;
|
|
1867
|
+
}
|
|
1868
|
+
interface WrittenScoreRecord {
|
|
1869
|
+
cassetteId: string;
|
|
1870
|
+
dir: string;
|
|
1871
|
+
file: string;
|
|
1872
|
+
}
|
|
1873
|
+
declare class FsScoreStore {
|
|
1874
|
+
private readonly outputRoot;
|
|
1875
|
+
constructor(config: FsScoreStoreConfig);
|
|
1876
|
+
save(record: ScoreRecord): WrittenScoreRecord;
|
|
1877
|
+
}
|
|
1878
|
+
|
|
1879
|
+
/** The discriminated replay-record read outcome: a persisted outcome, or a typed refusal. */
|
|
1880
|
+
type ReplayRecordLoad = {
|
|
1881
|
+
kind: 'loaded';
|
|
1882
|
+
outcome: ReplayCassetteOutcome;
|
|
1883
|
+
} | {
|
|
1884
|
+
kind: 'refused';
|
|
1885
|
+
refusal: ReplayRefusal;
|
|
1886
|
+
};
|
|
1887
|
+
/** The score read outcome: a persisted record, typed absence (never scored), or a typed refusal. */
|
|
1888
|
+
type ScoreRecordLoad = {
|
|
1889
|
+
kind: 'loaded';
|
|
1890
|
+
record: ScoreRecord;
|
|
1891
|
+
} | {
|
|
1892
|
+
kind: 'absent';
|
|
1893
|
+
} | {
|
|
1894
|
+
kind: 'refused';
|
|
1895
|
+
refusal: ReplayRefusal;
|
|
1896
|
+
};
|
|
1897
|
+
/**
|
|
1898
|
+
* Read a persisted ReplayCassetteOutcome back from `<dir>/replay.json`. A replay dir without its
|
|
1899
|
+
* marker file is corrupt, not empty — a missing or unreadable file refuses like a parse failure.
|
|
1900
|
+
* Never throws; the store serialized the outcome verbatim so the parsed document is it.
|
|
1901
|
+
*/
|
|
1902
|
+
declare function loadReplayRecord(dir: string): ReplayRecordLoad;
|
|
1903
|
+
/**
|
|
1904
|
+
* Read a persisted ScoreRecord back from `<dir>/score.json`. Absence is typed and legitimate
|
|
1905
|
+
* (refusal outcomes are never scored); an existing-but-unreadable/unparseable file refuses.
|
|
1906
|
+
* Never throws.
|
|
1907
|
+
*/
|
|
1908
|
+
declare function loadScoreRecord(dir: string): ScoreRecordLoad;
|
|
1909
|
+
/** Enumerate every staged replay-record dir (contains replay.json) under the replays root, sorted. */
|
|
1910
|
+
declare function listReplayRecordDirs(replaysRoot: string): string[];
|
|
1911
|
+
/** Enumerate every staged cassette-bundle dir (contains cassette.json) under the cassettes root, sorted. */
|
|
1912
|
+
declare function listCassetteBundleDirs(cassettesRoot: string): string[];
|
|
1913
|
+
|
|
1914
|
+
declare const DEFAULT_EVAL_REPLAYS_SUBDIR = "eval/replays";
|
|
1915
|
+
declare const REPLAY_RECORD_FILE = "replay.json";
|
|
1916
|
+
interface FsReplayRecordStoreConfig {
|
|
1917
|
+
/** Runtime root (normally the workdir's `.agent-qa` directory). */
|
|
1918
|
+
root: string;
|
|
1919
|
+
/** Optional retargeted root used by later committed-corpus composition. */
|
|
1920
|
+
outputDir?: string;
|
|
1921
|
+
/** Actual producer seed; embedded in the bytes bound by the run publication. */
|
|
1922
|
+
executionSeed?: number;
|
|
1923
|
+
}
|
|
1924
|
+
interface WrittenReplayRecord {
|
|
1925
|
+
cassetteId: string;
|
|
1926
|
+
dir: string;
|
|
1927
|
+
file: string;
|
|
1928
|
+
}
|
|
1929
|
+
declare class FsReplayRecordStore {
|
|
1930
|
+
private readonly outputRoot;
|
|
1931
|
+
private readonly executionSeed?;
|
|
1932
|
+
constructor(config: FsReplayRecordStoreConfig);
|
|
1933
|
+
save(outcome: ReplayCassetteOutcome): WrittenReplayRecord;
|
|
1934
|
+
}
|
|
1935
|
+
|
|
1936
|
+
type JudgeDeclaration = Omit<JudgeBindingInput, 'mask'>;
|
|
1937
|
+
interface RunCassetteReplayInput {
|
|
1938
|
+
/** A loaded, guarded cassette (leaf or procedure) — normally from loadCassetteBundle. */
|
|
1939
|
+
cassette: Cassette;
|
|
1940
|
+
/** Authoritative score metadata; never included in replayCassette's planner inputs. */
|
|
1941
|
+
labelRecord?: StandaloneLabelRecord;
|
|
1942
|
+
/** 417 D-01 / 415 D-05: already applied by the caller; the engine reverses it only before hashing. */
|
|
1943
|
+
declaredBriefDelta?: DeclaredBriefDelta;
|
|
1944
|
+
/** Blob source for full-capture screenshot reconstruction (normally a DirArtifactSource). */
|
|
1945
|
+
artifacts: ArtifactSource;
|
|
1946
|
+
/** The currently-configured model profile — the SOLE source of model selection (D-03). */
|
|
1947
|
+
profile: ModelProfile;
|
|
1948
|
+
/** Resolved provider auth for the configured model. */
|
|
1949
|
+
auth: ResolvedModelAuth;
|
|
1950
|
+
/** Optional provider overrides — the captured-boundary stub seam for tests (createModel passthrough). */
|
|
1951
|
+
modelOverrides?: ModelProviderOverrides;
|
|
1952
|
+
/** Whole-run counter shared by planner and judge, including provider retries. */
|
|
1953
|
+
modelCallObservation?: ModelCallObservation;
|
|
1954
|
+
/** Optional deterministic timing port for repeatable measurement fixtures. */
|
|
1955
|
+
clock?: ReplayClock;
|
|
1956
|
+
/** Sample budget (D-10). Defaults to the engine's k=1. */
|
|
1957
|
+
k?: number;
|
|
1958
|
+
/** Optional replay seed applied through the 366 sampling seam when the provider supports it. */
|
|
1959
|
+
seed?: number;
|
|
1960
|
+
/** k-sample escalation seam (D-10). Defaults to the imported neverEscalate — never re-minted. */
|
|
1961
|
+
escalationPredicate?: ReplayEscalationPredicate;
|
|
1962
|
+
/** Optional persistence: when supplied, the outcome is saved after the replay returns (D-12). */
|
|
1963
|
+
recordStore?: FsReplayRecordStore;
|
|
1964
|
+
/**
|
|
1965
|
+
* Optional D-13 score persistence: when supplied and the outcome is a ReplayResult (refusals are
|
|
1966
|
+
* never scored — reports count them instead), the score + per-decision diffs are computed in
|
|
1967
|
+
* eval core and saved AFTER recordStore.save (record-then-score ordering, OQ-2).
|
|
1968
|
+
*/
|
|
1969
|
+
scoreStore?: FsScoreStore;
|
|
1970
|
+
/**
|
|
1971
|
+
* Optional D-06 tier-4 judge bag: an EXPLICIT second ModelProfile plus its auth and stub seam
|
|
1972
|
+
* (372's CLI surfaces it as config). No auto-pick from configured models, no fallback scanning —
|
|
1973
|
+
* absence is a first-class configuration, in which case every unresolved tie is stamped
|
|
1974
|
+
* not-invoked{no-judge-configured} whenever the score leg runs (D-05/D-10).
|
|
1975
|
+
* allowSameFamily (owner override of D-10, 2026-07-12): explicit default-off opt-in threaded
|
|
1976
|
+
* verbatim into assessJudgeEligibility — it lifts ONLY known-same-family refusal and stamps
|
|
1977
|
+
* the D-16 identity block with familyOverride; unknown families stay forbidden.
|
|
1978
|
+
*/
|
|
1979
|
+
judge?: (JudgeDeclaration & {
|
|
1980
|
+
allowSameFamily?: boolean;
|
|
1981
|
+
}) | {
|
|
1982
|
+
members: readonly JudgeDeclaration[];
|
|
1983
|
+
allowSameFamily?: boolean;
|
|
1984
|
+
};
|
|
1985
|
+
mask: MaskFn;
|
|
1986
|
+
}
|
|
1987
|
+
/** Persist the declared identity, including a refused panel; singletons retain old bytes. */
|
|
1988
|
+
declare function replayJudgeRequest(judge: RunCassetteReplayInput['judge']): ScoringJudgeRequest;
|
|
1989
|
+
/**
|
|
1990
|
+
* Name the resolved temperature-zero replay profile without auth resolution or model creation.
|
|
1991
|
+
* D-11: keep these inputs aligned with the single createModel site below. Override names are
|
|
1992
|
+
* the injected provider identities; seed values and provider disclosures never enter the key.
|
|
1993
|
+
*/
|
|
1994
|
+
declare function computeReplayProfileFingerprintFor(input: {
|
|
1995
|
+
profile: ModelProfile;
|
|
1996
|
+
seedPolicy: SeedPolicy;
|
|
1997
|
+
modelOverrides?: ModelProviderOverrides;
|
|
1998
|
+
}): string;
|
|
1999
|
+
/**
|
|
2000
|
+
* Run one loaded cassette through the pure replay engine bound to the production planner/model/
|
|
2001
|
+
* safety stack, optionally persisting the typed outcome. The one sanctioned model+runner
|
|
2002
|
+
* composition in eval-local (D-04).
|
|
2003
|
+
*/
|
|
2004
|
+
declare function runCassetteReplay(input: RunCassetteReplayInput): Promise<ReplayCassetteOutcome>;
|
|
2005
|
+
|
|
2006
|
+
declare function listBaselineRunRoots(agentQaRoot: string, profileKey: string): {
|
|
2007
|
+
root: string;
|
|
2008
|
+
kind: 'gate' | 'floor';
|
|
2009
|
+
seed?: number;
|
|
2010
|
+
}[];
|
|
2011
|
+
type BaselineCacheRow = {
|
|
2012
|
+
cassetteId: string;
|
|
2013
|
+
kind: 'cached';
|
|
2014
|
+
root: string;
|
|
2015
|
+
replay: ReplayCassetteOutcome;
|
|
2016
|
+
score: ScoreRecord;
|
|
2017
|
+
seed: number;
|
|
2018
|
+
seedSource: string;
|
|
2019
|
+
seedHonored: boolean;
|
|
2020
|
+
} | {
|
|
2021
|
+
cassetteId: string;
|
|
2022
|
+
kind: 'refused';
|
|
2023
|
+
root: string;
|
|
2024
|
+
reason: string;
|
|
2025
|
+
} | {
|
|
2026
|
+
cassetteId: string;
|
|
2027
|
+
kind: 'uncached';
|
|
2028
|
+
reason: string;
|
|
2029
|
+
};
|
|
2030
|
+
interface BaselineCacheResolution {
|
|
2031
|
+
rows: BaselineCacheRow[];
|
|
2032
|
+
roots: string[];
|
|
2033
|
+
}
|
|
2034
|
+
declare function resolveBaselineCache(input: {
|
|
2035
|
+
agentQaRoot: string;
|
|
2036
|
+
profileKey: string;
|
|
2037
|
+
fingerprint: string;
|
|
2038
|
+
matcherVersion: number;
|
|
2039
|
+
plannerModelId: string;
|
|
2040
|
+
judgeIdentity?: {
|
|
2041
|
+
judgeModelId: string;
|
|
2042
|
+
judgeVersion: number;
|
|
2043
|
+
};
|
|
2044
|
+
gateRootSeed: number;
|
|
2045
|
+
gateRootSeedSource: string;
|
|
2046
|
+
cassettes: readonly {
|
|
2047
|
+
id: string;
|
|
2048
|
+
promptInputHash: string;
|
|
2049
|
+
}[];
|
|
2050
|
+
}): BaselineCacheResolution;
|
|
2051
|
+
|
|
2052
|
+
/** Gate-side effective view; raw committed history and the proposer reader remain unchanged. */
|
|
2053
|
+
declare function readActiveCommittedCorpus(committedRoot: string, agentQaRoot: string, calibrationRead?: LabelCalibrationRead): CommittedCorpusRead;
|
|
2054
|
+
/** One unreadable or unadmittable committed file, attributable by path (D-08). */
|
|
2055
|
+
interface CommittedReadRefusal {
|
|
2056
|
+
path: string;
|
|
2057
|
+
reason: string;
|
|
2058
|
+
}
|
|
2059
|
+
/** The loaded committed root: eval core's snapshot plus everything the read learned about it. */
|
|
2060
|
+
interface CommittedCorpusRead {
|
|
2061
|
+
/** Guarded ownership including scoring-excluded history, for incurred-cost reports. */
|
|
2062
|
+
corpusEntriesById?: ReadonlyMap<string, CorpusEntry>;
|
|
2063
|
+
calibration?: EvalBatchReport['calibration'];
|
|
2064
|
+
retiredMarkedCassetteIds?: ReadonlySet<string>;
|
|
2065
|
+
snapshot: CommittedCorpusSnapshot;
|
|
2066
|
+
productsSeen: readonly string[];
|
|
2067
|
+
notices: string[];
|
|
2068
|
+
refusals: readonly CommittedReadRefusal[];
|
|
2069
|
+
/**
|
|
2070
|
+
* How many committed files declared each schemaVersion, keyed by the declared version as a
|
|
2071
|
+
* string. An absent version key means zero files declared it.
|
|
2072
|
+
*/
|
|
2073
|
+
schemaVersionsSeen: Record<string, number>;
|
|
2074
|
+
}
|
|
2075
|
+
/** Every committed partition manifest under contracts/, and what the read refused (D-05). */
|
|
2076
|
+
interface PartitionManifestRead {
|
|
2077
|
+
manifests: readonly PartitionManifest[];
|
|
2078
|
+
notices: string[];
|
|
2079
|
+
refusals: readonly CommittedReadRefusal[];
|
|
2080
|
+
}
|
|
2081
|
+
/** Every committed run-exclusion record under exclusions/, and what the read refused (D-02). */
|
|
2082
|
+
interface RunExclusionRead {
|
|
2083
|
+
exclusions: readonly RunExclusionRecord[];
|
|
2084
|
+
refusals: readonly CommittedReadRefusal[];
|
|
2085
|
+
}
|
|
2086
|
+
/** One committed batch report keyed by its own dir name. */
|
|
2087
|
+
interface CommittedReportEntry {
|
|
2088
|
+
batchKey: string;
|
|
2089
|
+
report: EvalBatchReport;
|
|
2090
|
+
path: string;
|
|
2091
|
+
}
|
|
2092
|
+
/** One committed per-profile baseline designation (D-19). */
|
|
2093
|
+
interface BaselinePointerEntry {
|
|
2094
|
+
schemaVersion: typeof BASELINE_POINTER_SCHEMA_VERSION;
|
|
2095
|
+
profileKey: string;
|
|
2096
|
+
batchKey: string;
|
|
2097
|
+
fingerprint: string;
|
|
2098
|
+
corpusDigest: string;
|
|
2099
|
+
path: string;
|
|
2100
|
+
}
|
|
2101
|
+
interface NoiseFloorArtefactEntry {
|
|
2102
|
+
profileKey: string;
|
|
2103
|
+
artefact: NoiseFloorArtefact;
|
|
2104
|
+
path: string;
|
|
2105
|
+
}
|
|
2106
|
+
type CommittedDecisionVectorsLoad = {
|
|
2107
|
+
kind: 'loaded';
|
|
2108
|
+
document: DecisionVectorsDocument;
|
|
2109
|
+
path: string;
|
|
2110
|
+
} | {
|
|
2111
|
+
kind: 'absent';
|
|
2112
|
+
path: string;
|
|
2113
|
+
} | {
|
|
2114
|
+
kind: 'refused';
|
|
2115
|
+
refusal: CommittedReadRefusal;
|
|
2116
|
+
};
|
|
2117
|
+
type ResolveMinimumGateSetOutcome = {
|
|
2118
|
+
kind: 'resolved';
|
|
2119
|
+
minimumGateSet: number;
|
|
2120
|
+
path: string;
|
|
2121
|
+
} | {
|
|
2122
|
+
kind: 'no-floor';
|
|
2123
|
+
} | {
|
|
2124
|
+
kind: 'no-value';
|
|
2125
|
+
path: string;
|
|
2126
|
+
};
|
|
2127
|
+
/** Read only the per-profile files; the former literal reports/noise-floor.json is invalid. */
|
|
2128
|
+
declare function listNoiseFloorArtefacts(committedRoot: string): {
|
|
2129
|
+
artefacts: NoiseFloorArtefactEntry[];
|
|
2130
|
+
refusals: CommittedReadRefusal[];
|
|
2131
|
+
};
|
|
2132
|
+
declare function readCommittedDecisionVectors(committedRoot: string, batchKey: string): CommittedDecisionVectorsLoad;
|
|
2133
|
+
/** 414 D-15 → 417 handoff: resolve the measured integer without changing the pure posture port. */
|
|
2134
|
+
declare function resolveMinimumGateSet(input: {
|
|
2135
|
+
committedRoot: string;
|
|
2136
|
+
profileKey: string;
|
|
2137
|
+
}): ResolveMinimumGateSetOutcome;
|
|
2138
|
+
/**
|
|
2139
|
+
* Load the committed root at `committedRoot` into the snapshot eval core's `deriveCensus` folds.
|
|
2140
|
+
* Never throws for content: every parse or guard failure lands in `refusals`, and a missing root
|
|
2141
|
+
* yields an empty snapshot with a notice naming it.
|
|
2142
|
+
*/
|
|
2143
|
+
declare function readCommittedCorpus(committedRoot: string, calibrationRead?: LabelCalibrationRead): CommittedCorpusRead;
|
|
2144
|
+
/**
|
|
2145
|
+
* Load every committed partition manifest under `contracts/` (414 D-05).
|
|
2146
|
+
*
|
|
2147
|
+
* Manifests are found by file-name prefix, not by parsing every contract: the dir also holds the
|
|
2148
|
+
* D-04 matcher and judge constitution projections, and asking the manifest guard about those would
|
|
2149
|
+
* manufacture refusals for documents that are not manifests. An absent `contracts/` is an empty
|
|
2150
|
+
* list with no refusal — a root that has never been partitioned is a legitimate root.
|
|
2151
|
+
*/
|
|
2152
|
+
declare function readPartitionManifests(committedRoot: string): PartitionManifestRead;
|
|
2153
|
+
/**
|
|
2154
|
+
* Load every committed run-exclusion record under `exclusions/<product>/` (414 D-02).
|
|
2155
|
+
*
|
|
2156
|
+
* Same posture as the corpus walk: a committed root is hand-editable, so an unparseable or
|
|
2157
|
+
* unadmittable record is a refusal row naming its path, never a throw. An absent `exclusions/`
|
|
2158
|
+
* dir is an empty list with no refusal — a root with nothing excluded is the common case.
|
|
2159
|
+
*/
|
|
2160
|
+
declare function readRunExclusions(committedRoot: string): RunExclusionRead;
|
|
2161
|
+
/**
|
|
2162
|
+
* List every committed batch report under `reports/`, one per batchKey dir. The reserved `census/`
|
|
2163
|
+
* and `baselines/`, `noise-floor/`, `judge-calibration/` dirs are not batch reports.
|
|
2164
|
+
*/
|
|
2165
|
+
declare function listCommittedReports(committedRoot: string): {
|
|
2166
|
+
reports: CommittedReportEntry[];
|
|
2167
|
+
refusals: CommittedReadRefusal[];
|
|
2168
|
+
};
|
|
2169
|
+
/** List every committed per-profile baseline pointer under `reports/baselines/`, sorted (D-19). */
|
|
2170
|
+
declare function listBaselinePointers(committedRoot: string): {
|
|
2171
|
+
pointers: BaselinePointerEntry[];
|
|
2172
|
+
refusals: CommittedReadRefusal[];
|
|
2173
|
+
};
|
|
2174
|
+
|
|
2175
|
+
declare const GATE1_AUDIT_SUBDIR = "eval/gate1";
|
|
2176
|
+
/** A slug locates storage; only the loaded run/step/test establishes target identity. */
|
|
2177
|
+
declare function locateTargetCassettes(input: {
|
|
2178
|
+
committedRoot: string;
|
|
2179
|
+
agentQaRoot: string;
|
|
2180
|
+
gate1Dir: string;
|
|
2181
|
+
corpusEntries: readonly CorpusEntry[];
|
|
2182
|
+
target: NonNullable<CandidateInjection['target']>;
|
|
2183
|
+
/** 418 D-02: a live ablation must not substitute a committed/corrected gold record. */
|
|
2184
|
+
source?: 'handoff';
|
|
2185
|
+
}): Promise<{
|
|
2186
|
+
kind: 'committed' | 'extracted';
|
|
2187
|
+
cassettes: LeafBundle[];
|
|
2188
|
+
detail: TargetLocationDetail;
|
|
2189
|
+
} | {
|
|
2190
|
+
kind: 'unavailable';
|
|
2191
|
+
detail: TargetLocationDetail;
|
|
2192
|
+
}>;
|
|
2193
|
+
interface TargetLocationDetail {
|
|
2194
|
+
runId: string;
|
|
2195
|
+
stepIndex: number;
|
|
2196
|
+
corpusEntriesForRun: number;
|
|
2197
|
+
handoffPresent: boolean;
|
|
2198
|
+
reason?: string;
|
|
2199
|
+
}
|
|
2200
|
+
/** The CLI's JSON output uses the same schema-validated DTO as persisted receipts. */
|
|
2201
|
+
declare function serializeReplayGateReceipt(receipt: unknown): string;
|
|
2202
|
+
type Planner = {
|
|
2203
|
+
profile: ModelProfile;
|
|
2204
|
+
modelOverrides?: ModelProviderOverrides;
|
|
2205
|
+
resolveAuth: () => Promise<ResolvedModelAuth>;
|
|
2206
|
+
};
|
|
2207
|
+
interface RunReplayGateInput {
|
|
2208
|
+
injection: CandidateInjection;
|
|
2209
|
+
committedRoot: string;
|
|
2210
|
+
agentQaRoot: string;
|
|
2211
|
+
identity: {
|
|
2212
|
+
plannerModelId: string;
|
|
2213
|
+
matcherVersion: number;
|
|
2214
|
+
judgeModelId?: string;
|
|
2215
|
+
judgeVersion?: number;
|
|
2216
|
+
judgeMemberModelIds?: readonly string[];
|
|
2217
|
+
fingerprint: string;
|
|
2218
|
+
profileKey: string;
|
|
2219
|
+
};
|
|
2220
|
+
planner: Planner;
|
|
2221
|
+
judge?: Planner;
|
|
2222
|
+
/** Complete explicit lazy declaration; Gate 1 evaluates the raw members as a unit. */
|
|
2223
|
+
judgeMembers?: readonly Planner[];
|
|
2224
|
+
judgeOverrideConfigured: boolean;
|
|
2225
|
+
k: number;
|
|
2226
|
+
kSource: Gate1KSource;
|
|
2227
|
+
maxScreens: number;
|
|
2228
|
+
baselineSeed: {
|
|
2229
|
+
seed?: number;
|
|
2230
|
+
source: 'flag' | 'config';
|
|
2231
|
+
} | {
|
|
2232
|
+
floorSeeds: readonly number[];
|
|
2233
|
+
};
|
|
2234
|
+
/** Effective CLI selection for a copyable, spend-bearing cache recovery command. */
|
|
2235
|
+
baselineWarmup?: {
|
|
2236
|
+
configPath: string;
|
|
2237
|
+
judge?: string;
|
|
2238
|
+
judges?: readonly string[];
|
|
2239
|
+
};
|
|
2240
|
+
mask: RunCassetteReplayInput['mask'];
|
|
2241
|
+
producedBy: {
|
|
2242
|
+
opId: string;
|
|
2243
|
+
};
|
|
2244
|
+
clock?: () => number;
|
|
2245
|
+
onProgress?: (line: string) => void;
|
|
2246
|
+
}
|
|
2247
|
+
type RunReplayGateResult = {
|
|
2248
|
+
kind: 'receipt';
|
|
2249
|
+
receipt: Gate1Receipt;
|
|
2250
|
+
lines: string[];
|
|
2251
|
+
notices: string[];
|
|
2252
|
+
refusals: readonly CommittedReadRefusal[];
|
|
2253
|
+
modelCalls: number;
|
|
2254
|
+
wallClockMs: number;
|
|
2255
|
+
} | {
|
|
2256
|
+
kind: 'refused';
|
|
2257
|
+
refusal: CommittedReadRefusal;
|
|
2258
|
+
notices: string[];
|
|
2259
|
+
};
|
|
2260
|
+
type LeafBundle = {
|
|
2261
|
+
cassette: LeafCassette;
|
|
2262
|
+
artifacts: ArtifactSource;
|
|
2263
|
+
};
|
|
2264
|
+
type ReplayBag = Pick<RunCassetteReplayInput, 'profile' | 'auth' | 'modelOverrides' | 'judge' | 'mask' | 'modelCallObservation' | 'clock'>;
|
|
2265
|
+
interface RemovalFlipsOutcomeInput {
|
|
2266
|
+
recordId: string;
|
|
2267
|
+
cassettes: readonly (LeafBundle & {
|
|
2268
|
+
labelRecord?: StandaloneLabelRecord;
|
|
2269
|
+
existing?: {
|
|
2270
|
+
withRecord: ReplaySample;
|
|
2271
|
+
withoutRecord: ReplaySample;
|
|
2272
|
+
};
|
|
2273
|
+
baseSeed?: number;
|
|
2274
|
+
seedHonored?: boolean;
|
|
2275
|
+
})[];
|
|
2276
|
+
mode: {
|
|
2277
|
+
kind: 'injected';
|
|
2278
|
+
injection: CandidateInjection;
|
|
2279
|
+
} | {
|
|
2280
|
+
kind: 'active-record';
|
|
2281
|
+
servedCanary?: true;
|
|
2282
|
+
};
|
|
2283
|
+
k: number;
|
|
2284
|
+
baseSeed: number;
|
|
2285
|
+
existing?: {
|
|
2286
|
+
withRecord: ReplaySample;
|
|
2287
|
+
withoutRecord: ReplaySample;
|
|
2288
|
+
};
|
|
2289
|
+
replay: ReplayBag;
|
|
2290
|
+
gate1Dir: string;
|
|
2291
|
+
/**
|
|
2292
|
+
* 2026-09-11, 422 D-18: the ONE way an ORDINARY candidate — served on channel 1 without a
|
|
2293
|
+
* canary flag — becomes ablatable. Deliberately a sibling of `mode`, not a third mode arm:
|
|
2294
|
+
* the served-canary rule above is an opt-in FLAG, and this is positive PROOF. Absent, the
|
|
2295
|
+
* arbiter removes nothing that the active and served-canary rules did not already admit.
|
|
2296
|
+
*/
|
|
2297
|
+
failingPathCandidate?: FailingPathCandidateAuthorization;
|
|
2298
|
+
}
|
|
2299
|
+
interface RemovalFlipsOutcomeResult {
|
|
2300
|
+
crossCheck: Exclude<CrossCheck, 'not-run'>;
|
|
2301
|
+
perTarget: {
|
|
2302
|
+
cassetteId: string;
|
|
2303
|
+
crossCheck: Exclude<CrossCheck, 'not-run'>;
|
|
2304
|
+
}[];
|
|
2305
|
+
seeds: number[];
|
|
2306
|
+
seedHonored: boolean;
|
|
2307
|
+
detail: NonNullable<Gate1Receipt['samples']['detail']>;
|
|
2308
|
+
modelCalls: number;
|
|
2309
|
+
screens: number;
|
|
2310
|
+
promptTokens: number;
|
|
2311
|
+
completionTokens: number;
|
|
2312
|
+
notices: string[];
|
|
2313
|
+
}
|
|
2314
|
+
/**
|
|
2315
|
+
* 2026-09-11, 422 D-18: the arbiter's authorization to ablate an ORDINARY candidate.
|
|
2316
|
+
*
|
|
2317
|
+
* Ordinary candidate memory is the user's, and this is the one path in the phase that deletes
|
|
2318
|
+
* it from a replayed input. So the authorization carries the run's OWN recorded bytes and
|
|
2319
|
+
* DECLARES nothing that code can recompute: which step failed is re-derived with the single
|
|
2320
|
+
* exported `failingStepOf` rule, and whether the record was on that step's path is re-read
|
|
2321
|
+
* from that step's own served brief with `deriveMemoryOnPathSites`. A caller can therefore
|
|
2322
|
+
* name a record and hand over evidence, but it cannot assert either fact.
|
|
2323
|
+
*/
|
|
2324
|
+
interface FailingPathCandidateAuthorization {
|
|
2325
|
+
/** The record to ablate. Must be the record `removalFlipsOutcome` was asked to remove. */
|
|
2326
|
+
readonly recordId: string;
|
|
2327
|
+
/** The run that recorded {@link perTest}. Must be the target cassette's own run. */
|
|
2328
|
+
readonly runId: string;
|
|
2329
|
+
/** The run's own recorded member outcome — the source of the two recomputed facts above. */
|
|
2330
|
+
readonly perTest: PerTestOutcome;
|
|
2331
|
+
}
|
|
2332
|
+
/**
|
|
2333
|
+
* Three outcomes, never two. `absent` (no authorization was offered) and `refused` (one was,
|
|
2334
|
+
* and it did not establish the removal) both leave memory intact, but they are different
|
|
2335
|
+
* facts about the caller and the gate reports them apart.
|
|
2336
|
+
*/
|
|
2337
|
+
type FailingPathCandidateAdmission = {
|
|
2338
|
+
kind: 'authorized';
|
|
2339
|
+
stepIndex: number;
|
|
2340
|
+
} | {
|
|
2341
|
+
kind: 'absent';
|
|
2342
|
+
} | {
|
|
2343
|
+
kind: 'refused';
|
|
2344
|
+
reason: string;
|
|
2345
|
+
};
|
|
2346
|
+
/**
|
|
2347
|
+
* Evaluate a D-18 authorization against the decision it claims to authorize. Every refusal
|
|
2348
|
+
* names its own cause, because "removed nothing" for six different reasons is one fact a
|
|
2349
|
+
* reader cannot act on.
|
|
2350
|
+
*/
|
|
2351
|
+
declare function resolveFailingPathCandidate(input: {
|
|
2352
|
+
cassette: LeafCassette;
|
|
2353
|
+
recordId: string;
|
|
2354
|
+
authorization?: FailingPathCandidateAuthorization;
|
|
2355
|
+
}): FailingPathCandidateAdmission;
|
|
2356
|
+
/** Existing samples count as sample one; every additional sample gets a distinct derived seed. */
|
|
2357
|
+
declare function removalFlipsOutcome(input: RemovalFlipsOutcomeInput): Promise<RemovalFlipsOutcomeResult>;
|
|
2358
|
+
declare function runReplayGate(input: RunReplayGateInput): Promise<RunReplayGateResult>;
|
|
2359
|
+
|
|
2360
|
+
declare const GATE2_AUDIT_SUBDIR = "eval/gate2";
|
|
2361
|
+
declare const GATE2_TARGET_SUBDIR = "target";
|
|
2362
|
+
declare const FEEDBACK_STAGING_SUBDIR = "eval/feedback";
|
|
2363
|
+
declare const feedbackSchema: z.ZodObject<{
|
|
2364
|
+
schemaVersion: z.ZodLiteral<2>;
|
|
2365
|
+
product: z.ZodString;
|
|
2366
|
+
stateDigest: z.ZodString;
|
|
2367
|
+
attributedObservationId: z.ZodString;
|
|
2368
|
+
benefit: z.ZodEnum<{
|
|
2369
|
+
"exercised-benefited": "exercised-benefited";
|
|
2370
|
+
"rode-along": "rode-along";
|
|
2371
|
+
"exercised-harmed": "exercised-harmed";
|
|
2372
|
+
}>;
|
|
2373
|
+
candidateId: z.ZodString;
|
|
2374
|
+
runId: z.ZodString;
|
|
2375
|
+
testId: z.ZodString;
|
|
2376
|
+
stepIndex: z.ZodNumber;
|
|
2377
|
+
stagedAt: z.ZodISODateTime;
|
|
2378
|
+
}, z.core.$strict>;
|
|
2379
|
+
type FeedbackMetadata = z.infer<typeof feedbackSchema>;
|
|
2380
|
+
interface StagedFeedbackEntry extends FeedbackMetadata {
|
|
2381
|
+
dir: string;
|
|
2382
|
+
extracted: ExtractedCassette & {
|
|
2383
|
+
cassette: LeafCassette;
|
|
2384
|
+
};
|
|
2385
|
+
}
|
|
2386
|
+
interface FeedbackStateInput {
|
|
2387
|
+
agentQaRoot: string;
|
|
2388
|
+
committedRoot: string;
|
|
2389
|
+
product: string;
|
|
2390
|
+
stateDigest: string;
|
|
2391
|
+
}
|
|
2392
|
+
interface StageFeedbackEntryInput extends FeedbackStateInput {
|
|
2393
|
+
target: {
|
|
2394
|
+
runId: string;
|
|
2395
|
+
testId: string;
|
|
2396
|
+
stepIndex: number;
|
|
2397
|
+
};
|
|
2398
|
+
attributedObservationId: string;
|
|
2399
|
+
benefit: Benefit;
|
|
2400
|
+
candidateId: string;
|
|
2401
|
+
/** 441 GAP3-D1: required. The staged cassette is extracted from the masked run. */
|
|
2402
|
+
mask: MaskFn;
|
|
2403
|
+
}
|
|
2404
|
+
type StageFeedbackEntryResult = {
|
|
2405
|
+
kind: 'staged';
|
|
2406
|
+
dir: string;
|
|
2407
|
+
} | {
|
|
2408
|
+
kind: 'already-staged' | 'not-novel';
|
|
2409
|
+
} | {
|
|
2410
|
+
kind: 'target-unavailable' | 'write-conflict';
|
|
2411
|
+
};
|
|
2412
|
+
declare function isNovelState(input: FeedbackStateInput): boolean;
|
|
2413
|
+
declare function stageFeedbackEntry(input: StageFeedbackEntryInput): Promise<StageFeedbackEntryResult>;
|
|
2414
|
+
declare function listStagedFeedback(input: {
|
|
2415
|
+
agentQaRoot: string;
|
|
2416
|
+
product?: string;
|
|
2417
|
+
}): StagedFeedbackEntry[];
|
|
2418
|
+
interface AdmitStagedFeedbackResult {
|
|
2419
|
+
entries: StagedFeedbackEntry[];
|
|
2420
|
+
admitted: StagedFeedbackEntry[];
|
|
2421
|
+
refusals: (SeedingRefusal & {
|
|
2422
|
+
product: string;
|
|
2423
|
+
runId: string;
|
|
2424
|
+
})[];
|
|
2425
|
+
previousCorpusDigest: string;
|
|
2426
|
+
nextCorpusDigest?: string;
|
|
2427
|
+
writesStarted: boolean;
|
|
2428
|
+
failures: {
|
|
2429
|
+
phase: 'write' | 'cleanup' | 'final-digest';
|
|
2430
|
+
reason: 'seeding-unavailable' | 'staging-cleanup-failed' | 'corpus-unavailable';
|
|
2431
|
+
cassetteId?: string;
|
|
2432
|
+
product?: string;
|
|
2433
|
+
runId?: string;
|
|
2434
|
+
}[];
|
|
2435
|
+
}
|
|
2436
|
+
declare function admitStagedFeedback(input: {
|
|
2437
|
+
agentQaRoot: string;
|
|
2438
|
+
committedRoot: string;
|
|
2439
|
+
product?: string;
|
|
2440
|
+
dryRun: boolean;
|
|
2441
|
+
/** 441 GAP3-D1: required. A staged case still holding a declared value is refused by the writer. */
|
|
2442
|
+
mask: MaskFn;
|
|
2443
|
+
}): Promise<AdmitStagedFeedbackResult>;
|
|
2444
|
+
interface AblateCanaryStepInput {
|
|
2445
|
+
agentQaRoot: string;
|
|
2446
|
+
committedRoot: string;
|
|
2447
|
+
recordId: string;
|
|
2448
|
+
target: {
|
|
2449
|
+
runId: string;
|
|
2450
|
+
testId: string;
|
|
2451
|
+
stepIndex: number;
|
|
2452
|
+
};
|
|
2453
|
+
/** The gate-2 audit sub-tree this ablation writes under. Slugged before it becomes a path. */
|
|
2454
|
+
attributedObservationId: string;
|
|
2455
|
+
k: number;
|
|
2456
|
+
baseSeed: number;
|
|
2457
|
+
replay: ReplayBag;
|
|
2458
|
+
/**
|
|
2459
|
+
* 2026-09-12, 422 D-18: WHICH removal rule authorizes this ablation.
|
|
2460
|
+
*
|
|
2461
|
+
* ABSENT keeps 418's opt-in served-canary flag, which is what every canary caller means and
|
|
2462
|
+
* what this function did unconditionally before — so no existing caller changes behaviour. The
|
|
2463
|
+
* pre-analysis circularity arbiter passes a D-18 authorization INSTEAD, because its subject is
|
|
2464
|
+
* an ORDINARY candidate that was never flagged a canary, and `resolveFailingPathCandidate`
|
|
2465
|
+
* recomputes the failing step, the served membership and the brief identity from the run's own
|
|
2466
|
+
* bytes before admitting it. The two are a sibling pair rather than one switch: the canary flag
|
|
2467
|
+
* is an opt-in and the authorization is positive proof.
|
|
2468
|
+
*/
|
|
2469
|
+
admission?: {
|
|
2470
|
+
servedCanary?: true;
|
|
2471
|
+
failingPathCandidate?: FailingPathCandidateAuthorization;
|
|
2472
|
+
};
|
|
2473
|
+
}
|
|
2474
|
+
type CommittedCorpusDigestRead = {
|
|
2475
|
+
kind: 'digest';
|
|
2476
|
+
corpusDigest: string;
|
|
2477
|
+
} | {
|
|
2478
|
+
kind: 'unavailable';
|
|
2479
|
+
reason: 'corpus-unreadable';
|
|
2480
|
+
detail: string;
|
|
2481
|
+
};
|
|
2482
|
+
type AblateCanaryStepResult = {
|
|
2483
|
+
kind: 'ablated';
|
|
2484
|
+
crossCheck: CrossCheck;
|
|
2485
|
+
corpusDigest: string;
|
|
2486
|
+
k: number;
|
|
2487
|
+
baseSeed: number;
|
|
2488
|
+
seeds: number[];
|
|
2489
|
+
seedHonored: boolean;
|
|
2490
|
+
spend: {
|
|
2491
|
+
modelCalls: number;
|
|
2492
|
+
screens: number;
|
|
2493
|
+
promptTokens: number;
|
|
2494
|
+
completionTokens: number;
|
|
2495
|
+
};
|
|
2496
|
+
gate2Dir: string;
|
|
2497
|
+
notices: string[];
|
|
2498
|
+
} | {
|
|
2499
|
+
kind: 'unavailable';
|
|
2500
|
+
reason: 'target-unavailable' | 'brief-line-not-found' | 'no-live-decision' | 'corpus-unreadable';
|
|
2501
|
+
detail: string;
|
|
2502
|
+
};
|
|
2503
|
+
/** D-07: the same committed corpus definition the posture resolver cites, including negatives. */
|
|
2504
|
+
declare function readCommittedCorpusDigest(input: {
|
|
2505
|
+
committedRoot: string;
|
|
2506
|
+
}): CommittedCorpusDigestRead;
|
|
2507
|
+
declare function ablateCanaryStep(input: AblateCanaryStepInput): Promise<AblateCanaryStepResult>;
|
|
2508
|
+
|
|
2509
|
+
type LoadedCassetteBundle = Extract<CassetteBundleLoad, {
|
|
2510
|
+
kind: 'loaded';
|
|
2511
|
+
}>;
|
|
2512
|
+
/** Enumerate every bundle once, retaining load refusals as batch counters. */
|
|
2513
|
+
declare function loadCassetteBundles(cassettesRoot: string): {
|
|
2514
|
+
swept: number;
|
|
2515
|
+
loaded: LoadedCassetteBundle[];
|
|
2516
|
+
loadRefusals: Record<string, number>;
|
|
2517
|
+
};
|
|
2518
|
+
interface SweepCassetteReplaysInput {
|
|
2519
|
+
/** Caller-selected order; whole-corpus callers sort by cassette id before sweeping. */
|
|
2520
|
+
selected: readonly LoadedCassetteBundle[];
|
|
2521
|
+
labelRecords?: ReadonlyMap<string, StandaloneLabelRecord>;
|
|
2522
|
+
profile: ModelProfile;
|
|
2523
|
+
auth: ResolvedModelAuth;
|
|
2524
|
+
modelOverrides?: ModelProviderOverrides;
|
|
2525
|
+
modelCallObservation?: RunCassetteReplayInput['modelCallObservation'];
|
|
2526
|
+
clock?: RunCassetteReplayInput['clock'];
|
|
2527
|
+
k?: number;
|
|
2528
|
+
seed?: number;
|
|
2529
|
+
judge?: RunCassetteReplayInput['judge'];
|
|
2530
|
+
mask: RunCassetteReplayInput['mask'];
|
|
2531
|
+
recordStore: FsReplayRecordStore;
|
|
2532
|
+
scoreStore: FsScoreStore;
|
|
2533
|
+
/** The explicit output root shared by the caller's record and score stores. */
|
|
2534
|
+
replaysRoot: string;
|
|
2535
|
+
onProgress?: (cassetteId: string) => void;
|
|
2536
|
+
}
|
|
2537
|
+
interface SweepCassetteReplaysResult {
|
|
2538
|
+
replayed: number;
|
|
2539
|
+
replayRefusals: Record<string, number>;
|
|
2540
|
+
replayedDirs: string[];
|
|
2541
|
+
/** Only invocations that returned and persisted this time, including typed refusals. */
|
|
2542
|
+
recordedDirs: string[];
|
|
2543
|
+
notices: string[];
|
|
2544
|
+
}
|
|
2545
|
+
/**
|
|
2546
|
+
* Run the selected batch through the one sanctioned replay composition. Persistence roots
|
|
2547
|
+
* come from the caller so repeat runs can retain independent records. Refusals and thrown
|
|
2548
|
+
* harness errors remain data; one broken cassette never aborts the rest of the batch.
|
|
2549
|
+
*/
|
|
2550
|
+
declare function sweepCassetteReplays(input: SweepCassetteReplaysInput): Promise<SweepCassetteReplaysResult>;
|
|
2551
|
+
|
|
2552
|
+
declare const DEFAULT_EVAL_REPORTS_SUBDIR = "eval/reports";
|
|
2553
|
+
interface FsReportStoreConfig {
|
|
2554
|
+
/** Runtime root (normally the workdir's `.agent-qa` directory). */
|
|
2555
|
+
root: string;
|
|
2556
|
+
/** Optional retargeted root used by later committed-corpus composition. */
|
|
2557
|
+
outputDir?: string;
|
|
2558
|
+
}
|
|
2559
|
+
interface WrittenReport {
|
|
2560
|
+
batchKey: string;
|
|
2561
|
+
file: string;
|
|
2562
|
+
decisionsFile?: string;
|
|
2563
|
+
}
|
|
2564
|
+
type StagedReportLoad = {
|
|
2565
|
+
kind: 'loaded';
|
|
2566
|
+
report: EvalBatchReport;
|
|
2567
|
+
decisions?: DecisionVectorsDocument;
|
|
2568
|
+
file: string;
|
|
2569
|
+
replayGeneration?: ReplayGeneration;
|
|
2570
|
+
} | {
|
|
2571
|
+
kind: 'absent';
|
|
2572
|
+
file: string;
|
|
2573
|
+
} | {
|
|
2574
|
+
kind: 'refused';
|
|
2575
|
+
file: string;
|
|
2576
|
+
reason: string;
|
|
2577
|
+
};
|
|
2578
|
+
declare const replayGenerationSchema: z.ZodObject<{
|
|
2579
|
+
seed: z.ZodNumber;
|
|
2580
|
+
records: z.ZodArray<z.ZodObject<{
|
|
2581
|
+
cassetteId: z.ZodString;
|
|
2582
|
+
replayDigest: z.ZodString;
|
|
2583
|
+
scoreDigest: z.ZodNullable<z.ZodString>;
|
|
2584
|
+
}, z.core.$strict>>;
|
|
2585
|
+
}, z.core.$strict>;
|
|
2586
|
+
type ReplayGeneration = z.infer<typeof replayGenerationSchema>;
|
|
2587
|
+
/** Probe exactly one directory; callers own the gate-root-first search order. */
|
|
2588
|
+
declare function loadStagedReport(root: string, batchKey: string, outputDir?: string): StagedReportLoad;
|
|
2589
|
+
declare class FsReportStore {
|
|
2590
|
+
private readonly outputRoot;
|
|
2591
|
+
constructor(config: FsReportStoreConfig);
|
|
2592
|
+
save(report: EvalBatchReport, decisions?: DecisionVectorsDocument, replayGeneration?: ReplayGeneration): WrittenReport;
|
|
2593
|
+
}
|
|
2594
|
+
|
|
2595
|
+
/**
|
|
2596
|
+
* Resolve the product partition from the recorded served-brief scope. Procedures read their
|
|
2597
|
+
* first decision record; a cassette with no decision maps to the typed 'default' partition.
|
|
2598
|
+
*/
|
|
2599
|
+
declare function cassetteProductPartition(cassette: Cassette): string;
|
|
2600
|
+
/**
|
|
2601
|
+
* Assemble report inputs from the caller's replay root, joining labels and product partitions
|
|
2602
|
+
* from cassette bundles. Scores and judge blocks come only from persisted records; every
|
|
2603
|
+
* replay remains an entry, including a record that can only be read as a typed refusal.
|
|
2604
|
+
*/
|
|
2605
|
+
declare function assembleScoredBatch(input: {
|
|
2606
|
+
replaysRoot: string;
|
|
2607
|
+
cassettesRoot: string;
|
|
2608
|
+
source: string;
|
|
2609
|
+
/** Fresh-run callers supply only completed writes; absence retains the generic disk sweep. */
|
|
2610
|
+
recordDirs?: readonly string[];
|
|
2611
|
+
/** Committed-corpus ownership is authoritative; generic callers retain scope semantics. */
|
|
2612
|
+
productByCassetteId?: ReadonlyMap<string, string>;
|
|
2613
|
+
/** Latest committed mark generations only; historical bundles remain on disk. */
|
|
2614
|
+
eligibleCassetteIds?: ReadonlySet<string>;
|
|
2615
|
+
labelRecords?: ReadonlyMap<string, StandaloneLabelRecord>;
|
|
2616
|
+
corpusEntriesById?: ReadonlyMap<string, CorpusEntry>;
|
|
2617
|
+
excludedLabelIds?: ReadonlySet<string>;
|
|
2618
|
+
calibration?: EvalBatchReport['calibration'];
|
|
2619
|
+
expectedJudgeRequest?: ScoringJudgeRequest;
|
|
2620
|
+
}): {
|
|
2621
|
+
entries: ScoredBatchEntry[];
|
|
2622
|
+
judgeBlocks: JudgeIdentityBlock[];
|
|
2623
|
+
partitionsById: Map<string, string>;
|
|
2624
|
+
missingBundleCount: number;
|
|
2625
|
+
notices: string[];
|
|
2626
|
+
calibration?: EvalBatchReport['calibration'];
|
|
2627
|
+
};
|
|
2628
|
+
/** Attach disclosure only after the caller has compared persisted score identities. */
|
|
2629
|
+
declare function copyJudgeCalibrationToReport(report: EvalBatchReport, read?: JudgeCalibrationRead): void;
|
|
2630
|
+
/** Existing producer publication must bind the full profile to these exact saved inputs. */
|
|
2631
|
+
declare function readReportProfileProof(input: {
|
|
2632
|
+
agentQaRoot: string;
|
|
2633
|
+
replaysRoot: string;
|
|
2634
|
+
report: EvalBatchReport;
|
|
2635
|
+
entries: readonly ScoredBatchEntry[];
|
|
2636
|
+
fingerprint: string;
|
|
2637
|
+
corpusDigest: string;
|
|
2638
|
+
seed?: number;
|
|
2639
|
+
}): {
|
|
2640
|
+
runIdentity: NonNullable<EvalBatchReport['runIdentity']>;
|
|
2641
|
+
replayGeneration: ReplayGeneration;
|
|
2642
|
+
} | undefined;
|
|
2643
|
+
|
|
2644
|
+
interface WriteNoiseFloorArtefactInput {
|
|
2645
|
+
artefact: NoiseFloorArtefact;
|
|
2646
|
+
committedRoot: string;
|
|
2647
|
+
}
|
|
2648
|
+
type WriteNoiseFloorArtefactOutcome = {
|
|
2649
|
+
kind: 'written' | 'unchanged';
|
|
2650
|
+
path: string;
|
|
2651
|
+
} | {
|
|
2652
|
+
kind: 'refused';
|
|
2653
|
+
refusal: FloorRefusal;
|
|
2654
|
+
} | {
|
|
2655
|
+
kind: 'write-conflict';
|
|
2656
|
+
path: string;
|
|
2657
|
+
};
|
|
2658
|
+
declare function writeNoiseFloorArtefact(input: WriteNoiseFloorArtefactInput): WriteNoiseFloorArtefactOutcome;
|
|
2659
|
+
|
|
2660
|
+
interface ReplayCorpusOnceInput {
|
|
2661
|
+
committedRoot?: string;
|
|
2662
|
+
selected: readonly LoadedCassetteBundle[];
|
|
2663
|
+
/** Bundle-load refusals excluded from selected; never include returned replay refusals. */
|
|
2664
|
+
loadRefusals?: Readonly<Partial<EvalBatchReport['honesty']['refusalCounts']>>;
|
|
2665
|
+
cassettesRoot: string;
|
|
2666
|
+
agentQaRoot: string;
|
|
2667
|
+
runRoot: string;
|
|
2668
|
+
profile: ModelProfile;
|
|
2669
|
+
auth: ResolvedModelAuth;
|
|
2670
|
+
modelOverrides?: ModelProviderOverrides;
|
|
2671
|
+
judge?: RunCassetteReplayInput['judge'];
|
|
2672
|
+
mask: RunCassetteReplayInput['mask'];
|
|
2673
|
+
seed: number;
|
|
2674
|
+
k?: number;
|
|
2675
|
+
source: string;
|
|
2676
|
+
/** Digest of the committed snapshot supplying this run, when the caller read one. */
|
|
2677
|
+
corpusDigest?: string;
|
|
2678
|
+
productByCassetteId?: ReadonlyMap<string, string>;
|
|
2679
|
+
labelRecords?: ReadonlyMap<string, StandaloneLabelRecord>;
|
|
2680
|
+
corpusEntriesById?: ReadonlyMap<string, CorpusEntry>;
|
|
2681
|
+
excludedLabelIds?: ReadonlySet<string>;
|
|
2682
|
+
calibration?: EvalBatchReport['calibration'];
|
|
2683
|
+
clock?: () => number;
|
|
2684
|
+
onProgress?: (cassetteId: string) => void;
|
|
2685
|
+
}
|
|
2686
|
+
interface RunNoiseFloorMeasurementInput {
|
|
2687
|
+
committedRoot: string;
|
|
2688
|
+
cassettesRoot: string;
|
|
2689
|
+
agentQaRoot: string;
|
|
2690
|
+
profile: ModelProfile;
|
|
2691
|
+
auth?: ResolvedModelAuth;
|
|
2692
|
+
/** Explicit read-only recovery from complete saved runs and the original cost log. */
|
|
2693
|
+
fromMeasured?: string;
|
|
2694
|
+
modelOverrides?: ModelProviderOverrides;
|
|
2695
|
+
judge?: RunCassetteReplayInput['judge'];
|
|
2696
|
+
mask: RunCassetteReplayInput['mask'];
|
|
2697
|
+
profileKeyInputs: {
|
|
2698
|
+
plannerModelId: string;
|
|
2699
|
+
matcherVersion: number;
|
|
2700
|
+
judgeModelId?: string;
|
|
2701
|
+
judgeVersion?: number;
|
|
2702
|
+
};
|
|
2703
|
+
seeds: readonly number[];
|
|
2704
|
+
repeats: number;
|
|
2705
|
+
bootstrapSeed: number;
|
|
2706
|
+
classifierRepeatability?: ClassifierRepeatability;
|
|
2707
|
+
explanation?: string;
|
|
2708
|
+
commit: boolean;
|
|
2709
|
+
clock?: () => number;
|
|
2710
|
+
onProgress?: (line: string) => void;
|
|
2711
|
+
}
|
|
2712
|
+
interface RunNoiseFloorMeasurementResult {
|
|
2713
|
+
artefact: NoiseFloorArtefact;
|
|
2714
|
+
profileKey: string;
|
|
2715
|
+
fingerprint: string;
|
|
2716
|
+
runRoots: readonly string[];
|
|
2717
|
+
committed?: WriteNoiseFloorArtefactOutcome;
|
|
2718
|
+
notices: string[];
|
|
2719
|
+
refusals: readonly CommittedReadRefusal[];
|
|
2720
|
+
/** Calls in this invocation, separate from the original observations retained in artefact.cost. */
|
|
2721
|
+
recovery?: {
|
|
2722
|
+
costLog: string;
|
|
2723
|
+
modelCalls: 0;
|
|
2724
|
+
};
|
|
2725
|
+
}
|
|
2726
|
+
declare class FloorProtocolError extends Error {
|
|
2727
|
+
readonly code = "FLOOR_PROTOCOL_MINIMUM";
|
|
2728
|
+
constructor();
|
|
2729
|
+
}
|
|
2730
|
+
/** Whole-corpus repetitions, then the pure floor fold and the existing write-once checkpoint. */
|
|
2731
|
+
declare function runNoiseFloorMeasurement(input: RunNoiseFloorMeasurementInput): Promise<RunNoiseFloorMeasurementResult>;
|
|
2732
|
+
interface ReplayCorpusOnceResult {
|
|
2733
|
+
report: EvalBatchReport;
|
|
2734
|
+
vectors: DecisionCorrectnessVector[];
|
|
2735
|
+
replaysRoot: string;
|
|
2736
|
+
reportFile: string;
|
|
2737
|
+
decisionsFile: string;
|
|
2738
|
+
sweep: SweepCassetteReplaysResult;
|
|
2739
|
+
wallClockMs: number;
|
|
2740
|
+
modelCalls: number;
|
|
2741
|
+
notices: string[];
|
|
2742
|
+
}
|
|
2743
|
+
declare function replayCorpusOnce(input: ReplayCorpusOnceInput): Promise<ReplayCorpusOnceResult>;
|
|
2744
|
+
|
|
2745
|
+
type ReleaseGateMode = {
|
|
2746
|
+
kind: 'fresh';
|
|
2747
|
+
profile: ModelProfile;
|
|
2748
|
+
auth: ResolvedModelAuth;
|
|
2749
|
+
modelOverrides?: ModelProviderOverrides;
|
|
2750
|
+
judge?: RunCassetteReplayInput['judge'];
|
|
2751
|
+
mask: RunCassetteReplayInput['mask'];
|
|
2752
|
+
seed: number;
|
|
2753
|
+
cassettesRoot: string;
|
|
2754
|
+
onProgress?: (line: string) => void;
|
|
2755
|
+
} | {
|
|
2756
|
+
kind: 'report';
|
|
2757
|
+
batchKey: string;
|
|
2758
|
+
/** Exactly one report directory; the CLI owns any search order. */
|
|
2759
|
+
stagedOutputDir?: string;
|
|
2760
|
+
profile: ModelProfile;
|
|
2761
|
+
modelOverrides?: ModelProviderOverrides;
|
|
2762
|
+
};
|
|
2763
|
+
interface RunReleaseGateInput {
|
|
2764
|
+
committedRoot: string;
|
|
2765
|
+
agentQaRoot: string;
|
|
2766
|
+
profileKeyInputs: {
|
|
2767
|
+
plannerModelId: string;
|
|
2768
|
+
matcherVersion: number;
|
|
2769
|
+
judgeModelId?: string;
|
|
2770
|
+
judgeVersion?: number;
|
|
2771
|
+
};
|
|
2772
|
+
mode: ReleaseGateMode;
|
|
2773
|
+
clock?: () => number;
|
|
2774
|
+
}
|
|
2775
|
+
type RunReleaseGateResult = {
|
|
2776
|
+
kind: 'verdict';
|
|
2777
|
+
verdict: GateVerdict;
|
|
2778
|
+
lines: string[];
|
|
2779
|
+
mode: 'fresh' | 'report';
|
|
2780
|
+
profileKey: string;
|
|
2781
|
+
fingerprint: string;
|
|
2782
|
+
currentReport: EvalBatchReport;
|
|
2783
|
+
currentReportFile: string;
|
|
2784
|
+
/** Actual calls in this invocation, including retries and judge calls; offline is zero. */
|
|
2785
|
+
modelCalls: number;
|
|
2786
|
+
/** Fresh helper replay duration, or offline read/compare duration; never historical cost. */
|
|
2787
|
+
wallClockMs: number;
|
|
2788
|
+
notices: string[];
|
|
2789
|
+
refusals: readonly CommittedReadRefusal[];
|
|
2790
|
+
} | {
|
|
2791
|
+
kind: 'refused';
|
|
2792
|
+
refusal: CommittedReadRefusal;
|
|
2793
|
+
notices: string[];
|
|
2794
|
+
};
|
|
2795
|
+
/** Compose guarded disk evidence with eval core's single comparison law. */
|
|
2796
|
+
declare function runReleaseGate(input: RunReleaseGateInput): Promise<RunReleaseGateResult>;
|
|
2797
|
+
|
|
2798
|
+
declare const COMMITTED_REPORT_JSON_FILE = "report.json";
|
|
2799
|
+
declare const COMMITTED_REPORT_MARKDOWN_FILE = "report.md";
|
|
2800
|
+
declare const COMMITTED_REPORT_DECISIONS_FILE = "decisions.json";
|
|
2801
|
+
interface CommitEvalReportInput {
|
|
2802
|
+
/** The pure eval-core report DTO — batchKey names the committed dir. */
|
|
2803
|
+
report: EvalBatchReport;
|
|
2804
|
+
/** Optional pairing evidence for this same batch and matcher, never a separate channel. */
|
|
2805
|
+
decisions?: DecisionVectorsDocument;
|
|
2806
|
+
/** The rendered markdown projection; byte-determinism is the renderer's contract. */
|
|
2807
|
+
markdown: string;
|
|
2808
|
+
/** Explicit absolute committed-root path — `<configDir>/agent-qa-eval` for the 372 CLI. */
|
|
2809
|
+
committedRoot: string;
|
|
2810
|
+
}
|
|
2811
|
+
type CommitEvalReportOutcome = {
|
|
2812
|
+
kind: 'committed';
|
|
2813
|
+
json: 'written' | 'unchanged';
|
|
2814
|
+
markdown: 'written' | 'unchanged';
|
|
2815
|
+
decisions?: 'written' | 'unchanged';
|
|
2816
|
+
reportDir: string;
|
|
2817
|
+
} | {
|
|
2818
|
+
kind: 'write-conflict';
|
|
2819
|
+
path: string;
|
|
2820
|
+
};
|
|
2821
|
+
/**
|
|
2822
|
+
* Commit the baseline report beside the corpus/contracts that produced it: preview ALL paths,
|
|
2823
|
+
* then write — one atomic unit (D-13/D-17).
|
|
2824
|
+
*/
|
|
2825
|
+
declare function commitEvalReport(input: CommitEvalReportInput): CommitEvalReportOutcome;
|
|
2826
|
+
|
|
2827
|
+
interface WriteBaselinePointerInput {
|
|
2828
|
+
/** The report being designated — its judge provenance decides comparator eligibility. */
|
|
2829
|
+
report: EvalBatchReport;
|
|
2830
|
+
/** The v2 pointer, including fingerprint and corpusDigest; batchKey names the same report. */
|
|
2831
|
+
pointer: BaselinePointer;
|
|
2832
|
+
/** Explicit absolute committed-root path — `<configDir>/agent-qa-eval` for the CLI. */
|
|
2833
|
+
committedRoot: string;
|
|
2834
|
+
}
|
|
2835
|
+
type WriteBaselinePointerOutcome = {
|
|
2836
|
+
kind: 'written' | 'unchanged';
|
|
2837
|
+
path: string;
|
|
2838
|
+
} | {
|
|
2839
|
+
kind: 'refused';
|
|
2840
|
+
refusal: BaselineComparatorRefusal;
|
|
2841
|
+
} | {
|
|
2842
|
+
kind: 'write-conflict';
|
|
2843
|
+
path: string;
|
|
2844
|
+
};
|
|
2845
|
+
/**
|
|
2846
|
+
* Designate a per-profile baseline: refuse-then-preview-then-write.
|
|
2847
|
+
*
|
|
2848
|
+
* A pointer whose batchKey does not name the designated report is a CALLER bug, not data — the
|
|
2849
|
+
* outcome union stays closed for the three honest answers and a mismatch raises instead.
|
|
2850
|
+
*/
|
|
2851
|
+
declare function writeBaselinePointer(input: WriteBaselinePointerInput): WriteBaselinePointerOutcome;
|
|
2852
|
+
|
|
2853
|
+
declare const COMMITTED_CENSUS_JSON_FILE = "census.json";
|
|
2854
|
+
declare const COMMITTED_CENSUS_MARKDOWN_FILE = "census.md";
|
|
2855
|
+
interface CommitCensusInput {
|
|
2856
|
+
/** The pure eval-core census DTO — its corpusDigest names the committed dir. */
|
|
2857
|
+
census: CorpusCensus;
|
|
2858
|
+
/** The rendered markdown projection; byte-determinism is the renderer's contract. */
|
|
2859
|
+
markdown: string;
|
|
2860
|
+
/** Explicit absolute committed-root path — `<configDir>/agent-qa-eval` for the CLI. */
|
|
2861
|
+
committedRoot: string;
|
|
2862
|
+
}
|
|
2863
|
+
type CommitCensusOutcome = {
|
|
2864
|
+
kind: 'committed';
|
|
2865
|
+
json: 'written' | 'unchanged';
|
|
2866
|
+
markdown: 'written' | 'unchanged';
|
|
2867
|
+
censusDir: string;
|
|
2868
|
+
} | {
|
|
2869
|
+
kind: 'write-conflict';
|
|
2870
|
+
path: string;
|
|
2871
|
+
};
|
|
2872
|
+
/**
|
|
2873
|
+
* Commit the census beside the corpus it measures: preview BOTH paths, then write — one
|
|
2874
|
+
* atomic unit.
|
|
2875
|
+
*/
|
|
2876
|
+
declare function commitCensus(input: CommitCensusInput): CommitCensusOutcome;
|
|
2877
|
+
|
|
2878
|
+
interface ReadGatePostureForInput {
|
|
2879
|
+
/** Explicit absolute committed-root path — `<configDir>/agent-qa-eval` for the CLI. */
|
|
2880
|
+
committedRoot: string;
|
|
2881
|
+
agentQaRoot?: string;
|
|
2882
|
+
/** The scope being asked about; `resolveProductPartition` is the ONE mapping (D-16). */
|
|
2883
|
+
scope: ScopeKey;
|
|
2884
|
+
/** REQUIRED integer floor, >= 1. No default exists anywhere (D-12). */
|
|
2885
|
+
minimumGateSet: number;
|
|
2886
|
+
/** Complete raw judge declaration and recorded identity; passed intact to the pure fold. */
|
|
2887
|
+
judge: JudgeCapabilityInput;
|
|
2888
|
+
/** Canary evidence as DATA; eval never imports knowledge-local. */
|
|
2889
|
+
canary: CanaryEvidenceSummary;
|
|
2890
|
+
}
|
|
2891
|
+
interface ReadGatePostureForResult {
|
|
2892
|
+
posture: GatePosture;
|
|
2893
|
+
/** Every notice both reads produced, in read order: corpus first, then manifests. */
|
|
2894
|
+
notices: string[];
|
|
2895
|
+
/** Every refusal both reads produced — never dropped, never merged into a count. */
|
|
2896
|
+
refusals: readonly CommittedReadRefusal[];
|
|
2897
|
+
}
|
|
2898
|
+
/**
|
|
2899
|
+
* Read the committed root and fold it into the gate posture for one scope.
|
|
2900
|
+
*
|
|
2901
|
+
* Throws only what `readGatePosture` throws: a `minimumGateSet` that is not a positive integer is
|
|
2902
|
+
* a caller bug, not data. Everything the DISK got wrong is a refusal row.
|
|
2903
|
+
*/
|
|
2904
|
+
declare function readGatePostureFor(input: ReadGatePostureForInput): ReadGatePostureForResult;
|
|
2905
|
+
|
|
2906
|
+
type CalibrationEvidenceMissingReason = 'corpus-refused' | 'invalid-run-link' | 'path-refused' | 'marks-missing' | 'marks-unreadable' | 'handoff-missing' | 'handoff-unreadable' | 'handoff-refused' | 'source-refused' | 'deterministic-missing' | 'deterministic-refused' | 'analysis-missing' | 'analysis-unreadable' | 'analysis-refused' | 'analyze-run-missing';
|
|
2907
|
+
interface CalibrationEvidenceRead extends LabelCalibrationInput {
|
|
2908
|
+
marks: MarkReceipt[];
|
|
2909
|
+
observations: CalibrationObservation[];
|
|
2910
|
+
missingEvidenceCounts: Partial<Record<CalibrationEvidenceMissingReason, number>>;
|
|
2911
|
+
}
|
|
2912
|
+
/** Guard persisted observations independently; a winning merged verdict loses producer evidence. */
|
|
2913
|
+
declare function loadCalibrationEvidence(input: {
|
|
2914
|
+
committedRoot: string;
|
|
2915
|
+
agentQaRoot: string;
|
|
2916
|
+
calibrationRead?: LabelCalibrationRead;
|
|
2917
|
+
}): CalibrationEvidenceRead;
|
|
2918
|
+
/** Guarded diagnostic read using the caller's already validated calibration generation. */
|
|
2919
|
+
declare function readCalibrationGoldIdentity(input: {
|
|
2920
|
+
committedRoot: string;
|
|
2921
|
+
agentQaRoot: string;
|
|
2922
|
+
calibrationRead: LabelCalibrationRead;
|
|
2923
|
+
}): {
|
|
2924
|
+
kind: 'loaded';
|
|
2925
|
+
goldSetDigest: string;
|
|
2926
|
+
} | {
|
|
2927
|
+
kind: 'unavailable';
|
|
2928
|
+
};
|
|
2929
|
+
|
|
2930
|
+
export { ADVERSARIAL_DEFAULT_MAX_PROBE_CALLS, ADVERSARIAL_MANIFEST_FILE, ADVERSARIAL_PROBE_SCHEMA_VERSION, ADVERSARIAL_PROBE_SUBDIR, ADVERSARIAL_STAGING_SUBDIR, ADVERSARIAL_STATE_SCHEMA_VERSION, ADVERSARIAL_STATE_SUBDIR, type AblateCanaryStepInput, type AblateCanaryStepResult, type AdmitStagedFeedbackResult, type AdversarialBatchManifest, type AdversarialBatchManifestCase, AdversarialLifecycleBusyError, type AdversarialProbeCallCap, type AdversarialProbeRefusal, type AdversarialProbeSkip, type AdversarialProbeSkipReason, type AdversarialRefreshRecord, type AdversarialRefreshTuple, type AdversarialRefusal, type AdversarialRetirementResult, type AdversarialSeedCap, type AdversarialSpend, type AdversarialStagingSkip, type AdversarialTupleOutcome, BLOBS_SUBDIR, type BaselineCacheResolution, type BaselineCacheRow, type BaselinePointerEntry, CASSETTE_FILE, COMMITTED_BASELINES_SUBDIR, COMMITTED_CASSETTES_DIR, COMMITTED_CENSUS_JSON_FILE, COMMITTED_CENSUS_MARKDOWN_FILE, COMMITTED_CENSUS_SUBDIR, COMMITTED_CONTRACTS_DIR, COMMITTED_CORPUS_DIR, COMMITTED_EXCLUSIONS_DIR, COMMITTED_GITIGNORE_TEXT, COMMITTED_LABELS_DIR, COMMITTED_NOISE_FLOOR_SUBDIR, COMMITTED_REPORTS_DIR, COMMITTED_REPORT_DECISIONS_FILE, COMMITTED_REPORT_JSON_FILE, COMMITTED_REPORT_MARKDOWN_FILE, COMMITTED_ROOT_MARKER_FILE, COMMITTED_ROOT_SCHEMA_VERSION, type CalibrationEvidenceMissingReason, type CalibrationEvidenceRead, type CassetteBundleLoad, type CommitCensusInput, type CommitCensusOutcome, type CommitEvalReportInput, type CommitEvalReportOutcome, type CommittedCorpusDigestRead, type CommittedCorpusRead, type CommittedDecisionVectorsLoad, type CommittedReadRefusal, type CommittedReportEntry, type CommittedRootStoreConfig, type CommittedWriteOutcome, DEFAULT_EVAL_CASSETTES_SUBDIR, DEFAULT_EVAL_REPLAYS_SUBDIR, DEFAULT_EVAL_REPORTS_SUBDIR, DirArtifactSource, type DirArtifactSourceConfig, FEEDBACK_STAGING_SUBDIR, type FailingPathCandidateAdmission, type FailingPathCandidateAuthorization, type FeedbackStateInput, FloorProtocolError, type FreezeGoldenTrajectoryInput, type FreezeOutcome, FsCassetteStore, type FsCassetteStoreConfig, FsCommittedRootStore, FsHandoffSource, type FsHandoffSourceConfig, FsMarkLedger, FsReplayRecordStore, type FsReplayRecordStoreConfig, FsReportStore, type FsReportStoreConfig, FsRunArtifactSource, type FsRunArtifactSourceConfig, FsScoreStore, type FsScoreStoreConfig, GATE1_AUDIT_SUBDIR, GATE2_AUDIT_SUBDIR, GATE2_TARGET_SUBDIR, JUDGE_CALIBRATION_SCHEMA_VERSION, JUDGE_CALIBRATION_SUBDIR, type JudgeBindingInput, type JudgeCalibrationArtifact, type JudgeCalibrationEvidenceRead, type JudgeCalibrationPublication, type JudgeCalibrationRead, type JudgeCalibrationReadInput, LABEL_CALIBRATION_PATH, LABEL_CALIBRATION_SCHEMA_VERSION, type LabelCalibrationArtifact, type LabelCalibrationRead, type LabelEvidenceRead, type LoadJudgeCalibrationEvidenceInput, type LoadLabelEvidenceInput, type LoadedCassetteBundle, MARKS_LEDGER_FILE, MARKS_LEDGER_LOCK_FILE, MARK_INTENT_FILE, MarkIntentUnusableError, MarkLedgerBusyError, MarkLedgerCorruptError, MarkLedgerNoRunError, type NoiseFloorArtefactEntry, PARTITION_MANIFEST_FILE_PREFIX, type PartitionManifestRead, type ProbeAttemptState, type ProbeCompletion, type ProbeCompletionOutcome, type ProbeReservation, type ProbeReservationKey, REPLAY_RECORD_FILE, type ReadGatePostureForInput, type ReadGatePostureForResult, type RefreshAdversarialSplitInput, type RefreshAdversarialSplitResult, type ReleaseGateMode, type RemovalFlipsOutcomeInput, type RemovalFlipsOutcomeResult, type ReplayCorpusOnceInput, type ReplayCorpusOnceResult, type ReplayRecordLoad, type ResolveMinimumGateSetOutcome, type RetireStagedAdversarialBatchInput, type RunCassetteReplayInput, type RunExclusionRead, type RunJudgeCalibrationResult, type RunJudgePanelCalibrationInput, type RunLabelCalibrationInput, type RunLabelCalibrationResult, type RunNoiseFloorMeasurementInput, type RunNoiseFloorMeasurementResult, type RunReleaseGateInput, type RunReleaseGateResult, type RunReplayGateInput, type RunReplayGateResult, SCORE_FILE, type ScoreRecord, type ScoreRecordLoad, type SeedCommittedCorpusInput, type SeedingRefusal, type SeedingReport, type StageAdversarialProposalInput, type StageAdversarialProposalResult, type StageFeedbackEntryInput, type StageFeedbackEntryResult, type StagedAdversarialBatch, type StagedAdversarialCase, type StagedFeedbackEntry, type StagedReportLoad, type SweepCassetteReplaysInput, type SweepCassetteReplaysResult, type WriteBaselinePointerInput, type WriteBaselinePointerOutcome, type WriteNoiseFloorArtefactInput, type WriteNoiseFloorArtefactOutcome, type WritePartitionManifestInput, type WritePartitionManifestOutcome, type WriteRunExclusionInput, type WriteRunExclusionOutcome, type WrittenCassette, type WrittenReplayRecord, type WrittenReport, type WrittenScoreRecord, ablateCanaryStep, absentArtifactSource, admitStagedFeedback, adversarialProbeTupleKey, adversarialRefreshRecordDigest, adversarialTupleKey, assembleScoredBatch, bindJudgePanel, bindJudgePort, buildRootMarkerValue, cassetteProductPartition, commitCensus, commitEvalReport, committedBaselinePointerPath, committedBlobPath, committedCassettePath, committedCensusPath, committedContractPath, committedCorpusPath, committedExclusionPath, committedLabelPath, committedLabelRevisionPath, committedNoiseFloorPath, committedPartitionManifestPath, committedReportPath, completeProbeCall, computeAdversarialBatchDigest, computeReplayProfileFingerprintFor, copyJudgeCalibrationToReport, createJudgeCalibrationArtifact, derivedAdversarialCassetteIds, freezeGoldenTrajectory, guardJudgeCalibration, isNovelState, judgeCalibrationGenerationId, judgeCalibrationPath, latestMarkReceiptFor, latestMarkReceiptsPerAddress, listBaselinePointers, listBaselineRunRoots, listCassetteBundleDirs, listCommittedReports, listNoiseFloorArtefacts, listReplayRecordDirs, listStagedAdversarial, listStagedFeedback, loadCalibrationEvidence, loadCassetteBundle, loadCassetteBundles, loadJudgeCalibrationEvidence, loadLabelEvidence, loadReplayRecord, loadScoreRecord, loadStagedReport, locateTargetCassettes, maskJudgeCalibrationText, probeReservationOf, publishAdversarialRefreshRecord, publishJudgeCalibration, readActiveCommittedCorpus, readAdversarialRefreshRecords, readCalibrationGoldIdentity, readCommittedCorpus, readCommittedCorpusDigest, readCommittedDecisionVectors, readGatePostureFor, readJudgeCalibration, readLabelCalibration, readPartitionManifests, readProbeAttempt, readProbeAttempts, readReportProfileProof, readRunExclusions, refreshAdversarialSplit, removalFlipsOutcome, replayCorpusOnce, replayJudgeRequest, reserveProbeCall, resolveAdversarialProbeCallCap, resolveAdversarialSeedCap, resolveBaselineCache, resolveFailingPathCandidate, resolveMinimumGateSet, retireStagedAdversarialBatch, runCassetteReplay, runJudgeCalibration, runJudgePanelCalibration, runLabelCalibration, runNoiseFloorMeasurement, runReleaseGate, runReplayGate, seedCommittedCorpus, serializeReplayGateReceipt, settledAdversarialTuples, stageAdversarialProposal, stageFeedbackEntry, sweepCassetteReplays, writeBaselinePointer, writeNoiseFloorArtefact, writePartitionManifest, writeRunExclusion };
|