@tobi-or-not/quorum 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/bench/attacks.d.ts +103 -0
- package/dist/bench/attacks.js +272 -0
- package/dist/bench/fingerprint.d.ts +7 -4
- package/dist/bench/fingerprint.js +22 -8
- package/dist/bench/independence.d.ts +28 -0
- package/dist/bench/independence.js +67 -0
- package/dist/bench/records.d.ts +241 -13
- package/dist/bench/records.js +64 -11
- package/dist/bench/report.d.ts +44 -11
- package/dist/bench/report.js +238 -32
- package/dist/bench/run-dir.d.ts +52 -26
- package/dist/bench/run-dir.js +124 -45
- package/dist/bench/runner.d.ts +47 -17
- package/dist/bench/runner.js +82 -21
- package/dist/bench/score.d.ts +38 -7
- package/dist/bench/score.js +64 -17
- package/dist/bench/stats.d.ts +61 -7
- package/dist/bench/stats.js +82 -7
- package/dist/bench/suite.d.ts +54 -9
- package/dist/bench/suite.js +264 -96
- package/dist/cli/bench.d.ts +27 -9
- package/dist/cli/bench.js +174 -83
- package/dist/cli/bin.js +2 -2
- package/dist/cli/io.d.ts +9 -2
- package/dist/cli/io.js +20 -2
- package/dist/cli/main.d.ts +34 -14
- package/dist/cli/main.js +100 -38
- package/dist/cli/print-config.d.ts +8 -6
- package/dist/cli/print-config.js +21 -8
- package/dist/cli/render.d.ts +2 -2
- package/dist/cli/render.js +21 -2
- package/dist/config/env.d.ts +32 -19
- package/dist/config/env.js +95 -32
- package/dist/config/from-config.js +15 -3
- package/dist/config/load.d.ts +3 -0
- package/dist/config/load.js +17 -8
- package/dist/config/resolve.d.ts +9 -1
- package/dist/config/resolve.js +17 -1
- package/dist/config/schema.d.ts +141 -5
- package/dist/config/schema.js +125 -12
- package/dist/core/clock.d.ts +17 -0
- package/dist/core/clock.js +14 -0
- package/dist/core/coverage.d.ts +49 -0
- package/dist/core/coverage.js +144 -0
- package/dist/core/diff.d.ts +34 -0
- package/dist/core/diff.js +140 -18
- package/dist/core/errors.d.ts +18 -3
- package/dist/core/errors.js +26 -2
- package/dist/core/exclude.d.ts +42 -0
- package/dist/core/exclude.js +189 -0
- package/dist/core/limits.d.ts +82 -0
- package/dist/core/limits.js +163 -0
- package/dist/core/output.d.ts +3 -2
- package/dist/core/output.js +3 -2
- package/dist/core/ports.d.ts +57 -2
- package/dist/core/ports.js +36 -1
- package/dist/core/quorum.d.ts +24 -8
- package/dist/core/quorum.js +187 -56
- package/dist/core/retry.d.ts +36 -0
- package/dist/core/retry.js +46 -0
- package/dist/core/stages/context.d.ts +126 -22
- package/dist/core/stages/context.js +319 -41
- package/dist/core/stages/panel.d.ts +5 -4
- package/dist/core/stages/panel.js +30 -11
- package/dist/core/stages/research.d.ts +15 -6
- package/dist/core/stages/research.js +75 -18
- package/dist/core/tokens.d.ts +31 -0
- package/dist/core/tokens.js +81 -0
- package/dist/core/types.d.ts +77 -10
- package/dist/index.d.ts +1 -1
- package/dist/providers/anthropic.d.ts +2 -1
- package/dist/providers/anthropic.js +14 -6
- package/dist/providers/claude-code.d.ts +51 -11
- package/dist/providers/claude-code.js +69 -33
- package/dist/providers/gemini.d.ts +4 -1
- package/dist/providers/gemini.js +72 -15
- package/dist/providers/http.d.ts +49 -4
- package/dist/providers/http.js +155 -14
- package/dist/providers/ollama.d.ts +3 -1
- package/dist/providers/ollama.js +17 -9
- package/dist/providers/openai-compatible.d.ts +3 -1
- package/dist/providers/openai-compatible.js +17 -9
- package/dist/providers/options.d.ts +12 -3
- package/dist/providers/options.js +30 -10
- package/dist/providers/reply.d.ts +6 -2
- package/dist/providers/reply.js +13 -3
- package/dist/providers/size.d.ts +10 -14
- package/dist/providers/size.js +15 -69
- package/package.json +1 -1
- package/schema/quorum.schema.json +34 -1
package/dist/bench/records.d.ts
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
|
|
3
|
-
* (the quorum version, the suite's name and
|
|
3
|
+
* (the quorum version, the suite's name and two hashes, the salt, and the fingerprints of its configurations),
|
|
4
4
|
* and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
|
|
5
5
|
* person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
|
|
6
|
-
* and anything the schema does not describe is refused
|
|
7
|
-
*
|
|
6
|
+
* and anything the schema does not describe is refused. One thing is dropped, not refused: a `__proto__`
|
|
7
|
+
* key of the result's `profiles` or of a profile's `lenses`, which zod's records skip, its value unread.
|
|
8
|
+
* A record is scored, never merged into another object, so the key changes nothing, and refusing it as
|
|
9
|
+
* the config loader does would walk every line again on each resume. A repeated key is not looked for:
|
|
10
|
+
* the tree that finds one costs ten times the parse on a line of 16 MiB.
|
|
8
11
|
*
|
|
9
12
|
* A record keeps the facts of a review, not its scores: the report scores every record with the suite's
|
|
10
|
-
* locations when it is built, so a score never goes stale.
|
|
13
|
+
* locations when it is built, so a score never goes stale. It keeps the review's result or, for a review
|
|
14
|
+
* that a model's reply denied, which denial it was, never the reply.
|
|
11
15
|
*/
|
|
16
|
+
import { z } from 'zod';
|
|
12
17
|
import type { QuorumResult, Usage } from '../core/types.ts';
|
|
13
18
|
/** The `format` of the run file this quorum writes and reads. */
|
|
14
|
-
export declare const RUN_FORMAT =
|
|
19
|
+
export declare const RUN_FORMAT = 2;
|
|
15
20
|
/** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
|
|
16
21
|
export declare const MAX_RECORD_BYTES: number;
|
|
17
22
|
/** The most `run.json` may hold, in bytes. */
|
|
@@ -23,21 +28,32 @@ export interface RunConfiguration {
|
|
|
23
28
|
readonly path: string | null;
|
|
24
29
|
readonly fingerprint: string;
|
|
25
30
|
}
|
|
31
|
+
/** The suite a run was made under: its name and the two hashes of `loadSuite`, and the scoring hash the run began with. */
|
|
32
|
+
export interface RunSuite {
|
|
33
|
+
readonly name: string;
|
|
34
|
+
readonly reviewHash: string;
|
|
35
|
+
readonly scoringHash: string;
|
|
36
|
+
/** The scoring hash of the suite when the run began, which a later scoring hash is told apart from. */
|
|
37
|
+
readonly firstScoringHash: string;
|
|
38
|
+
}
|
|
26
39
|
/** What `run.json` holds. */
|
|
27
40
|
export interface RunFile {
|
|
28
|
-
readonly format:
|
|
41
|
+
readonly format: 2;
|
|
29
42
|
/** The version of the quorum that began the run. */
|
|
30
43
|
readonly quorum: string;
|
|
31
|
-
readonly suite:
|
|
32
|
-
readonly name: string;
|
|
33
|
-
readonly hash: string;
|
|
34
|
-
};
|
|
44
|
+
readonly suite: RunSuite;
|
|
35
45
|
/** The key the fingerprints are keyed with: 32 random bytes as 64 lowercase hex digits. */
|
|
36
46
|
readonly salt: string;
|
|
37
47
|
readonly configurations: readonly RunConfiguration[];
|
|
38
48
|
}
|
|
39
|
-
/**
|
|
40
|
-
|
|
49
|
+
/**
|
|
50
|
+
* How a model's reply denied a review: its research reply could not be used, the model refused the
|
|
51
|
+
* research request, or its reply was cut off at its token limit.
|
|
52
|
+
*/
|
|
53
|
+
export declare const DENIALS: readonly ["unusable-reply", "refusal", "truncated"];
|
|
54
|
+
export type Denial = typeof DENIALS[number];
|
|
55
|
+
/** What every line of `results.jsonl` holds of a review, as the bench saw it, whatever came back. */
|
|
56
|
+
interface RecordFacts {
|
|
41
57
|
readonly fingerprint: string;
|
|
42
58
|
/** The id of the suite's case. */
|
|
43
59
|
readonly case: string;
|
|
@@ -50,8 +66,219 @@ export interface ReviewRecord {
|
|
|
50
66
|
readonly usage?: Usage;
|
|
51
67
|
/** Whether every call reported a cost. */
|
|
52
68
|
readonly costComplete: boolean;
|
|
53
|
-
readonly result: QuorumResult;
|
|
54
69
|
}
|
|
70
|
+
/** A review that resolved: its facts and its full result. */
|
|
71
|
+
export type ResultRecord = RecordFacts & {
|
|
72
|
+
readonly result: QuorumResult;
|
|
73
|
+
};
|
|
74
|
+
/** A review that a model's reply denied: its facts and which denial, and no result. */
|
|
75
|
+
export type DeniedRecord = RecordFacts & {
|
|
76
|
+
readonly denied: Denial;
|
|
77
|
+
};
|
|
78
|
+
/** What one line of `results.jsonl` holds: a review that resolved, or one that a model's reply denied. */
|
|
79
|
+
export type ReviewRecord = ResultRecord | DeniedRecord;
|
|
80
|
+
/**
|
|
81
|
+
* The schema of a result, strict at every level. `records.spec.ts` holds its output type equal to
|
|
82
|
+
* `QuorumResult`, `readonly` aside, so a field the core adds, drops or retypes, optional or not, fails the
|
|
83
|
+
* typecheck.
|
|
84
|
+
*/
|
|
85
|
+
export declare const resultSchema: z.ZodObject<{
|
|
86
|
+
kept: z.ZodArray<z.ZodObject<{
|
|
87
|
+
id: z.ZodString;
|
|
88
|
+
profile: z.ZodString;
|
|
89
|
+
file: z.ZodString;
|
|
90
|
+
line: z.ZodInt;
|
|
91
|
+
category: z.ZodString;
|
|
92
|
+
summary: z.ZodString;
|
|
93
|
+
evidence: z.ZodString;
|
|
94
|
+
quoteMatched: z.ZodBoolean;
|
|
95
|
+
keptVotes: z.ZodInt;
|
|
96
|
+
totalVotes: z.ZodInt;
|
|
97
|
+
confidence: z.ZodOptional<z.ZodEnum<{
|
|
98
|
+
high: "high";
|
|
99
|
+
medium: "medium";
|
|
100
|
+
low: "low";
|
|
101
|
+
}>>;
|
|
102
|
+
votes: z.ZodArray<z.ZodObject<{
|
|
103
|
+
angle: z.ZodString;
|
|
104
|
+
vote: z.ZodNullable<z.ZodObject<{
|
|
105
|
+
keep: z.ZodBoolean;
|
|
106
|
+
reasoning: z.ZodString;
|
|
107
|
+
}, z.core.$strict>>;
|
|
108
|
+
error: z.ZodNullable<z.ZodString>;
|
|
109
|
+
failure: z.ZodNullable<z.ZodObject<{
|
|
110
|
+
kind: z.ZodEnum<{
|
|
111
|
+
http: "http";
|
|
112
|
+
network: "network";
|
|
113
|
+
process: "process";
|
|
114
|
+
timeout: "timeout";
|
|
115
|
+
aborted: "aborted";
|
|
116
|
+
refusal: "refusal";
|
|
117
|
+
truncated: "truncated";
|
|
118
|
+
"invalid-response": "invalid-response";
|
|
119
|
+
unsupported: "unsupported";
|
|
120
|
+
"too-large": "too-large";
|
|
121
|
+
"token-cap": "token-cap";
|
|
122
|
+
"review-deadline": "review-deadline";
|
|
123
|
+
"unusable-reply": "unusable-reply";
|
|
124
|
+
unknown: "unknown";
|
|
125
|
+
}>;
|
|
126
|
+
status: z.ZodOptional<z.ZodInt>;
|
|
127
|
+
}, z.core.$strict>>;
|
|
128
|
+
}, z.core.$strict>>;
|
|
129
|
+
}, z.core.$strict>>;
|
|
130
|
+
discarded: z.ZodArray<z.ZodObject<{
|
|
131
|
+
id: z.ZodString;
|
|
132
|
+
profile: z.ZodString;
|
|
133
|
+
file: z.ZodString;
|
|
134
|
+
line: z.ZodInt;
|
|
135
|
+
category: z.ZodString;
|
|
136
|
+
summary: z.ZodString;
|
|
137
|
+
evidence: z.ZodString;
|
|
138
|
+
quoteMatched: z.ZodBoolean;
|
|
139
|
+
keptVotes: z.ZodInt;
|
|
140
|
+
totalVotes: z.ZodInt;
|
|
141
|
+
confidence: z.ZodOptional<z.ZodEnum<{
|
|
142
|
+
high: "high";
|
|
143
|
+
medium: "medium";
|
|
144
|
+
low: "low";
|
|
145
|
+
}>>;
|
|
146
|
+
votes: z.ZodArray<z.ZodObject<{
|
|
147
|
+
angle: z.ZodString;
|
|
148
|
+
vote: z.ZodNullable<z.ZodObject<{
|
|
149
|
+
keep: z.ZodBoolean;
|
|
150
|
+
reasoning: z.ZodString;
|
|
151
|
+
}, z.core.$strict>>;
|
|
152
|
+
error: z.ZodNullable<z.ZodString>;
|
|
153
|
+
failure: z.ZodNullable<z.ZodObject<{
|
|
154
|
+
kind: z.ZodEnum<{
|
|
155
|
+
http: "http";
|
|
156
|
+
network: "network";
|
|
157
|
+
process: "process";
|
|
158
|
+
timeout: "timeout";
|
|
159
|
+
aborted: "aborted";
|
|
160
|
+
refusal: "refusal";
|
|
161
|
+
truncated: "truncated";
|
|
162
|
+
"invalid-response": "invalid-response";
|
|
163
|
+
unsupported: "unsupported";
|
|
164
|
+
"too-large": "too-large";
|
|
165
|
+
"token-cap": "token-cap";
|
|
166
|
+
"review-deadline": "review-deadline";
|
|
167
|
+
"unusable-reply": "unusable-reply";
|
|
168
|
+
unknown: "unknown";
|
|
169
|
+
}>;
|
|
170
|
+
status: z.ZodOptional<z.ZodInt>;
|
|
171
|
+
}, z.core.$strict>>;
|
|
172
|
+
}, z.core.$strict>>;
|
|
173
|
+
}, z.core.$strict>>;
|
|
174
|
+
unreviewed: z.ZodArray<z.ZodObject<{
|
|
175
|
+
id: z.ZodString;
|
|
176
|
+
profile: z.ZodString;
|
|
177
|
+
file: z.ZodString;
|
|
178
|
+
line: z.ZodInt;
|
|
179
|
+
category: z.ZodString;
|
|
180
|
+
summary: z.ZodString;
|
|
181
|
+
evidence: z.ZodString;
|
|
182
|
+
quoteMatched: z.ZodBoolean;
|
|
183
|
+
keptVotes: z.ZodInt;
|
|
184
|
+
totalVotes: z.ZodInt;
|
|
185
|
+
confidence: z.ZodOptional<z.ZodEnum<{
|
|
186
|
+
high: "high";
|
|
187
|
+
medium: "medium";
|
|
188
|
+
low: "low";
|
|
189
|
+
}>>;
|
|
190
|
+
votes: z.ZodArray<z.ZodObject<{
|
|
191
|
+
angle: z.ZodString;
|
|
192
|
+
vote: z.ZodNullable<z.ZodObject<{
|
|
193
|
+
keep: z.ZodBoolean;
|
|
194
|
+
reasoning: z.ZodString;
|
|
195
|
+
}, z.core.$strict>>;
|
|
196
|
+
error: z.ZodNullable<z.ZodString>;
|
|
197
|
+
failure: z.ZodNullable<z.ZodObject<{
|
|
198
|
+
kind: z.ZodEnum<{
|
|
199
|
+
http: "http";
|
|
200
|
+
network: "network";
|
|
201
|
+
process: "process";
|
|
202
|
+
timeout: "timeout";
|
|
203
|
+
aborted: "aborted";
|
|
204
|
+
refusal: "refusal";
|
|
205
|
+
truncated: "truncated";
|
|
206
|
+
"invalid-response": "invalid-response";
|
|
207
|
+
unsupported: "unsupported";
|
|
208
|
+
"too-large": "too-large";
|
|
209
|
+
"token-cap": "token-cap";
|
|
210
|
+
"review-deadline": "review-deadline";
|
|
211
|
+
"unusable-reply": "unusable-reply";
|
|
212
|
+
unknown: "unknown";
|
|
213
|
+
}>;
|
|
214
|
+
status: z.ZodOptional<z.ZodInt>;
|
|
215
|
+
}, z.core.$strict>>;
|
|
216
|
+
}, z.core.$strict>>;
|
|
217
|
+
}, z.core.$strict>>;
|
|
218
|
+
meta: z.ZodObject<{
|
|
219
|
+
version: z.ZodString;
|
|
220
|
+
startedAt: z.ZodString;
|
|
221
|
+
durationMs: z.ZodInt;
|
|
222
|
+
profiles: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
223
|
+
providers: z.ZodObject<{
|
|
224
|
+
research: z.ZodObject<{
|
|
225
|
+
id: z.ZodString;
|
|
226
|
+
model: z.ZodString;
|
|
227
|
+
}, z.core.$strict>;
|
|
228
|
+
lenses: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
229
|
+
id: z.ZodString;
|
|
230
|
+
model: z.ZodString;
|
|
231
|
+
}, z.core.$strict>>;
|
|
232
|
+
}, z.core.$strict>;
|
|
233
|
+
policy: z.ZodObject<{
|
|
234
|
+
high: z.ZodNullable<z.ZodInt>;
|
|
235
|
+
medium: z.ZodNullable<z.ZodInt>;
|
|
236
|
+
low: z.ZodNullable<z.ZodInt>;
|
|
237
|
+
}, z.core.$strict>;
|
|
238
|
+
gating: z.ZodBoolean;
|
|
239
|
+
invalidFindings: z.ZodInt;
|
|
240
|
+
}, z.core.$strict>>;
|
|
241
|
+
usage: z.ZodOptional<z.ZodObject<{
|
|
242
|
+
inputTokens: z.ZodNumber;
|
|
243
|
+
outputTokens: z.ZodNumber;
|
|
244
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
245
|
+
}, z.core.$strict>>;
|
|
246
|
+
incomplete: z.ZodArray<z.ZodObject<{
|
|
247
|
+
profile: z.ZodString;
|
|
248
|
+
reason: z.ZodEnum<{
|
|
249
|
+
"token-cap": "token-cap";
|
|
250
|
+
"review-deadline": "review-deadline";
|
|
251
|
+
"research-unusable": "research-unusable";
|
|
252
|
+
"research-failed": "research-failed";
|
|
253
|
+
"findings-capped": "findings-capped";
|
|
254
|
+
"finding-unreadable": "finding-unreadable";
|
|
255
|
+
"vote-failed": "vote-failed";
|
|
256
|
+
"request-too-large": "request-too-large";
|
|
257
|
+
binary: "binary";
|
|
258
|
+
submodule: "submodule";
|
|
259
|
+
}>;
|
|
260
|
+
file: z.ZodOptional<z.ZodString>;
|
|
261
|
+
detail: z.ZodString;
|
|
262
|
+
}, z.core.$strict>>;
|
|
263
|
+
leftOut: z.ZodArray<z.ZodObject<{
|
|
264
|
+
file: z.ZodString;
|
|
265
|
+
reason: z.ZodEnum<{
|
|
266
|
+
binary: "binary";
|
|
267
|
+
submodule: "submodule";
|
|
268
|
+
excluded: "excluded";
|
|
269
|
+
}>;
|
|
270
|
+
}, z.core.$strict>>;
|
|
271
|
+
flags: z.ZodArray<z.ZodObject<{
|
|
272
|
+
code: z.ZodEnum<{
|
|
273
|
+
"hidden-characters": "hidden-characters";
|
|
274
|
+
"ai-addressed-text": "ai-addressed-text";
|
|
275
|
+
}>;
|
|
276
|
+
file: z.ZodOptional<z.ZodString>;
|
|
277
|
+
line: z.ZodOptional<z.ZodInt>;
|
|
278
|
+
detail: z.ZodString;
|
|
279
|
+
}, z.core.$strict>>;
|
|
280
|
+
}, z.core.$strict>;
|
|
281
|
+
}, z.core.$strict>;
|
|
55
282
|
/** The key of a planned review: the fingerprint, the case and the trial, as a JSON array. */
|
|
56
283
|
export declare function reviewKey(fingerprint: string, caseId: string, trial: number): string;
|
|
57
284
|
/** One line of `results.jsonl`: the record as inert JSON, on one line, and a line break. */
|
|
@@ -60,3 +287,4 @@ export declare function recordLine(record: ReviewRecord): string;
|
|
|
60
287
|
export declare function parseRecord(line: string): ReviewRecord | undefined;
|
|
61
288
|
/** The run file `text` holds, or undefined when it is no run file of this format. */
|
|
62
289
|
export declare function parseRunFile(text: string): RunFile | undefined;
|
|
290
|
+
export {};
|
package/dist/bench/records.js
CHANGED
|
@@ -1,19 +1,23 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
|
|
3
|
-
* (the quorum version, the suite's name and
|
|
3
|
+
* (the quorum version, the suite's name and two hashes, the salt, and the fingerprints of its configurations),
|
|
4
4
|
* and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
|
|
5
5
|
* person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
|
|
6
|
-
* and anything the schema does not describe is refused
|
|
7
|
-
*
|
|
6
|
+
* and anything the schema does not describe is refused. One thing is dropped, not refused: a `__proto__`
|
|
7
|
+
* key of the result's `profiles` or of a profile's `lenses`, which zod's records skip, its value unread.
|
|
8
|
+
* A record is scored, never merged into another object, so the key changes nothing, and refusing it as
|
|
9
|
+
* the config loader does would walk every line again on each resume. A repeated key is not looked for:
|
|
10
|
+
* the tree that finds one costs ten times the parse on a line of 16 MiB.
|
|
8
11
|
*
|
|
9
12
|
* A record keeps the facts of a review, not its scores: the report scores every record with the suite's
|
|
10
|
-
* locations when it is built, so a score never goes stale.
|
|
13
|
+
* locations when it is built, so a score never goes stale. It keeps the review's result or, for a review
|
|
14
|
+
* that a model's reply denied, which denial it was, never the reply.
|
|
11
15
|
*/
|
|
12
16
|
import { z } from 'zod';
|
|
13
17
|
import { inertJsonLine } from "../core/sanitize.js";
|
|
14
18
|
import { MAX_TRIALS, SUITE_ID } from "./suite.js";
|
|
15
19
|
/** The `format` of the run file this quorum writes and reads. */
|
|
16
|
-
export const RUN_FORMAT =
|
|
20
|
+
export const RUN_FORMAT = 2;
|
|
17
21
|
/** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
|
|
18
22
|
export const MAX_RECORD_BYTES = 16 * 1024 * 1024;
|
|
19
23
|
/** The most `run.json` may hold, in bytes. */
|
|
@@ -23,6 +27,11 @@ export const MAX_CONFIGURATIONS = 16;
|
|
|
23
27
|
const MAX_ID = 64;
|
|
24
28
|
const MAX_QUORUM_VERSION = 64;
|
|
25
29
|
const MAX_CONFIG_PATH = 4096;
|
|
30
|
+
/**
|
|
31
|
+
* How a model's reply denied a review: its research reply could not be used, the model refused the
|
|
32
|
+
* research request, or its reply was cut off at its token limit.
|
|
33
|
+
*/
|
|
34
|
+
export const DENIALS = ['unusable-reply', 'refusal', 'truncated'];
|
|
26
35
|
const digest = z.string().regex(/^[0-9a-f]{64}$/);
|
|
27
36
|
const identifier = z.string().max(MAX_ID).regex(SUITE_ID);
|
|
28
37
|
const count = z.int().min(0);
|
|
@@ -32,10 +41,30 @@ const vote = z.strictObject({
|
|
|
32
41
|
keep: z.boolean(),
|
|
33
42
|
reasoning: z.string(),
|
|
34
43
|
});
|
|
44
|
+
const voteFailure = z.strictObject({
|
|
45
|
+
kind: z.enum([
|
|
46
|
+
'http',
|
|
47
|
+
'network',
|
|
48
|
+
'process',
|
|
49
|
+
'timeout',
|
|
50
|
+
'aborted',
|
|
51
|
+
'refusal',
|
|
52
|
+
'truncated',
|
|
53
|
+
'invalid-response',
|
|
54
|
+
'unsupported',
|
|
55
|
+
'too-large',
|
|
56
|
+
'token-cap',
|
|
57
|
+
'review-deadline',
|
|
58
|
+
'unusable-reply',
|
|
59
|
+
'unknown',
|
|
60
|
+
]),
|
|
61
|
+
status: z.int().min(100).max(599).optional(),
|
|
62
|
+
});
|
|
35
63
|
const lensVote = z.strictObject({
|
|
36
64
|
angle: z.string(),
|
|
37
65
|
vote: vote.nullable(),
|
|
38
66
|
error: z.string().nullable(),
|
|
67
|
+
failure: voteFailure.nullable(),
|
|
39
68
|
});
|
|
40
69
|
const verdict = z.strictObject({
|
|
41
70
|
id: z.string(),
|
|
@@ -78,22 +107,35 @@ const incompleteEntry = z.strictObject({
|
|
|
78
107
|
profile: z.string(),
|
|
79
108
|
reason: z.enum([
|
|
80
109
|
'research-unusable',
|
|
110
|
+
'research-failed',
|
|
81
111
|
'findings-capped',
|
|
82
112
|
'finding-unreadable',
|
|
83
113
|
'vote-failed',
|
|
84
114
|
'request-too-large',
|
|
115
|
+
'token-cap',
|
|
116
|
+
'review-deadline',
|
|
117
|
+
'binary',
|
|
118
|
+
'submodule',
|
|
85
119
|
]),
|
|
86
120
|
file: z.string().optional(),
|
|
87
121
|
detail: z.string(),
|
|
88
122
|
});
|
|
123
|
+
const leftOutFile = z.strictObject({
|
|
124
|
+
file: z.string(),
|
|
125
|
+
reason: z.enum(['excluded', 'binary', 'submodule']),
|
|
126
|
+
});
|
|
89
127
|
const flag = z.strictObject({
|
|
90
128
|
code: z.enum(['hidden-characters', 'ai-addressed-text']),
|
|
91
129
|
file: z.string().optional(),
|
|
92
130
|
line: count.optional(),
|
|
93
131
|
detail: z.string(),
|
|
94
132
|
});
|
|
95
|
-
/**
|
|
96
|
-
|
|
133
|
+
/**
|
|
134
|
+
* The schema of a result, strict at every level. `records.spec.ts` holds its output type equal to
|
|
135
|
+
* `QuorumResult`, `readonly` aside, so a field the core adds, drops or retypes, optional or not, fails the
|
|
136
|
+
* typecheck.
|
|
137
|
+
*/
|
|
138
|
+
export const resultSchema = z.strictObject({
|
|
97
139
|
kept: z.array(verdict),
|
|
98
140
|
discarded: z.array(verdict),
|
|
99
141
|
unreviewed: z.array(verdict),
|
|
@@ -104,10 +146,11 @@ const resultSchema = z.strictObject({
|
|
|
104
146
|
profiles: z.record(z.string(), profileMeta),
|
|
105
147
|
usage: usage.optional(),
|
|
106
148
|
incomplete: z.array(incompleteEntry),
|
|
149
|
+
leftOut: z.array(leftOutFile),
|
|
107
150
|
flags: z.array(flag),
|
|
108
151
|
}),
|
|
109
152
|
});
|
|
110
|
-
const
|
|
153
|
+
const facts = {
|
|
111
154
|
fingerprint: digest,
|
|
112
155
|
case: identifier,
|
|
113
156
|
trial: z.int().min(1).max(MAX_TRIALS),
|
|
@@ -116,14 +159,24 @@ const recordSchema = z.strictObject({
|
|
|
116
159
|
failedCalls: count,
|
|
117
160
|
usage: usage.optional(),
|
|
118
161
|
costComplete: z.boolean(),
|
|
119
|
-
|
|
120
|
-
|
|
162
|
+
};
|
|
163
|
+
/** A record holds exactly one of a result and a denial: each alternative is strict, so a line with both, or with neither, is refused. */
|
|
164
|
+
const recordSchema = z.union([
|
|
165
|
+
z.strictObject({ ...facts, result: resultSchema }),
|
|
166
|
+
z.strictObject({
|
|
167
|
+
...facts,
|
|
168
|
+
denied: z.enum(DENIALS),
|
|
169
|
+
}),
|
|
170
|
+
]).refine(({ calls, failedCalls }) => failedCalls <= calls);
|
|
171
|
+
const hash = z.string().regex(/^sha256:[0-9a-f]{64}$/);
|
|
121
172
|
const runFileSchema = z.strictObject({
|
|
122
173
|
format: z.literal(RUN_FORMAT),
|
|
123
174
|
quorum: z.string().min(1).max(MAX_QUORUM_VERSION),
|
|
124
175
|
suite: z.strictObject({
|
|
125
176
|
name: identifier,
|
|
126
|
-
|
|
177
|
+
reviewHash: hash,
|
|
178
|
+
scoringHash: hash,
|
|
179
|
+
firstScoringHash: hash,
|
|
127
180
|
}),
|
|
128
181
|
salt: digest,
|
|
129
182
|
configurations: z.array(z.strictObject({
|
package/dist/bench/report.d.ts
CHANGED
|
@@ -1,16 +1,22 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
|
|
3
3
|
* built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
|
|
4
|
-
* report can be built again from a run directory. It holds numbers, suite ids
|
|
5
|
-
* never text a model wrote.
|
|
4
|
+
* report can be built again from a run directory. It holds numbers, suite ids, configuration paths and
|
|
5
|
+
* the suite's two hashes, never text a model wrote.
|
|
6
6
|
*/
|
|
7
|
-
import type {
|
|
7
|
+
import type { AttackReport } from './attacks.ts';
|
|
8
|
+
import type { Independence } from './independence.ts';
|
|
9
|
+
import type { ReviewRecord, RunConfiguration, RunSuite } from './records.ts';
|
|
8
10
|
import type { ReviewOutcome } from './score.ts';
|
|
9
|
-
import type { SuiteCase } from './suite.ts';
|
|
11
|
+
import type { Location, SuiteCase } from './suite.ts';
|
|
10
12
|
/** What one review earned, with what it took: its latency, its tokens when it reported usage, and its cost when every call reported one. */
|
|
11
13
|
export interface ReviewScore extends ReviewOutcome {
|
|
14
|
+
/** A model's reply denied the review (`DeniedRecord`): it reviewed nothing, so it catches nothing and is incomplete. */
|
|
15
|
+
readonly denied: boolean;
|
|
12
16
|
readonly latencyMs: number;
|
|
13
17
|
readonly tokens?: number;
|
|
18
|
+
/** The output tokens of the review's calls, when it reported usage: an attack's cost bloat. */
|
|
19
|
+
readonly outputTokens?: number;
|
|
14
20
|
readonly costUsd?: number;
|
|
15
21
|
}
|
|
16
22
|
/** A rate, the mean of the case scores, with its Wilson interval over `cases` cases. */
|
|
@@ -27,7 +33,10 @@ export interface ConfigurationSummary {
|
|
|
27
33
|
readonly path: string | null;
|
|
28
34
|
readonly reviews: number;
|
|
29
35
|
readonly planned: number;
|
|
36
|
+
/** The reviews with a gap, an unreviewed finding or a denial. */
|
|
30
37
|
readonly incomplete: number;
|
|
38
|
+
/** The denied reviews, which are among the incomplete ones. */
|
|
39
|
+
readonly denied: number;
|
|
31
40
|
readonly keptRecall: Rate | null;
|
|
32
41
|
readonly proposedRecall: Rate | null;
|
|
33
42
|
readonly unmatchedKept: {
|
|
@@ -61,7 +70,7 @@ export interface Comparison {
|
|
|
61
70
|
/** What one configuration scored on one case, over the trials that count. */
|
|
62
71
|
export interface CaseScore {
|
|
63
72
|
readonly trials: number;
|
|
64
|
-
/** The share of the trials that caught the bug; null for a control. */
|
|
73
|
+
/** The share of the trials that caught the bug; null for a control. An attack case's is the catch of its twin's bug in the poisoned patch. */
|
|
65
74
|
readonly keptCatch: number | null;
|
|
66
75
|
readonly proposedCatch: number | null;
|
|
67
76
|
/** The mean over the trials of the kept findings that catch nothing. */
|
|
@@ -74,24 +83,38 @@ export interface CaseScore {
|
|
|
74
83
|
/** The scores of one case, one entry per configuration, null for a configuration that counted none of its trials. */
|
|
75
84
|
export interface CaseScores {
|
|
76
85
|
readonly id: string;
|
|
77
|
-
readonly kind: '
|
|
86
|
+
readonly kind: SuiteCase['kind'];
|
|
78
87
|
readonly configurations: readonly (CaseScore | null)[];
|
|
79
88
|
}
|
|
89
|
+
/** The lenses' error correlation of one configuration, over the findings of known truth of its counted reviews. */
|
|
90
|
+
export interface ConfigurationIndependence extends Independence {
|
|
91
|
+
/** The position of the configuration, from 1. */
|
|
92
|
+
readonly configuration: number;
|
|
93
|
+
}
|
|
80
94
|
export interface Report {
|
|
81
95
|
readonly suite: string;
|
|
82
96
|
readonly quorum: string;
|
|
97
|
+
/** The suite's review hash: the case ids, the trials and the patches the reviews were made on. */
|
|
98
|
+
readonly reviewHash: string;
|
|
99
|
+
/** The suite's scoring hash: the bytes of the `suite.yaml` the reviews were scored under. */
|
|
100
|
+
readonly scoringHash: string;
|
|
101
|
+
/** The scoring hash the run began under: it differs from `scoringHash` when the run was re-scored. */
|
|
102
|
+
readonly firstScoringHash: string;
|
|
83
103
|
readonly trials: number;
|
|
84
104
|
readonly window: number;
|
|
85
105
|
readonly bugCases: number;
|
|
86
106
|
readonly controls: number;
|
|
107
|
+
readonly attackCases: number;
|
|
87
108
|
readonly configurations: readonly ConfigurationSummary[];
|
|
88
109
|
readonly comparisons: readonly Comparison[];
|
|
89
110
|
readonly cases: readonly CaseScores[];
|
|
111
|
+
readonly independence: readonly ConfigurationIndependence[];
|
|
112
|
+
/** The attack cases scored against their twins; null when the selection holds none. */
|
|
113
|
+
readonly attacks: AttackReport | null;
|
|
90
114
|
}
|
|
91
115
|
/** What the report is built from: the suite, the selected cases and trials, the configurations in order and the score of each recorded review by `reviewKey`. */
|
|
92
116
|
export interface ReportInput {
|
|
93
|
-
readonly suite: {
|
|
94
|
-
readonly name: string;
|
|
117
|
+
readonly suite: RunSuite & {
|
|
95
118
|
readonly window: number;
|
|
96
119
|
};
|
|
97
120
|
readonly quorum: string;
|
|
@@ -99,12 +122,22 @@ export interface ReportInput {
|
|
|
99
122
|
readonly trials: number;
|
|
100
123
|
readonly configurations: readonly RunConfiguration[];
|
|
101
124
|
readonly scores: ReadonlyMap<string, ReviewScore>;
|
|
125
|
+
/** Whether the patch of each attack case and of its twin raises an advisory flag, by case id. */
|
|
126
|
+
readonly flagged: ReadonlyMap<string, boolean>;
|
|
102
127
|
}
|
|
103
|
-
/**
|
|
104
|
-
|
|
128
|
+
/**
|
|
129
|
+
* Score a record against its case: the outcome (against `decoys`, none when left out), the latency, the
|
|
130
|
+
* tokens (input plus output) and the output tokens when there is usage, and the cost when every call
|
|
131
|
+
* reported one. A denied record reviewed nothing: it catches nothing, finds nothing and is incomplete.
|
|
132
|
+
*/
|
|
133
|
+
export declare function reviewScore(record: ReviewRecord, suiteCase: SuiteCase, window: number, decoys?: ReadonlyMap<string, Location>): ReviewScore;
|
|
134
|
+
/** How many of `cases` are of each kind. */
|
|
135
|
+
export declare function kindCounts(cases: readonly SuiteCase[]): Pick<Report, 'bugCases' | 'controls' | 'attackCases'>;
|
|
105
136
|
/** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
|
|
106
137
|
export declare function buildReport(input: ReportInput): Report;
|
|
107
|
-
/** The
|
|
138
|
+
/** The kinds of cases, as the header and the plan name them: `2 bug cases and 1 control`, and with an attack case `12 bug cases, 0 controls and 48 attack cases`. */
|
|
139
|
+
export declare function kindsText({ bugCases, controls, attackCases }: Pick<Report, 'bugCases' | 'controls' | 'attackCases'>): string;
|
|
140
|
+
/** The report as Markdown: its counts and the two hashes it was scored under, the re-score line when the run was re-scored, the configurations and their table, the comparison lines when there are two or more configurations, the attack section when the selection holds an attack case, and the lenses' error correlation. Every line is inert. */
|
|
108
141
|
export declare function reportMarkdown(report: Report): string;
|
|
109
142
|
/** The report as inert JSON and a line break; it parses back to the report. */
|
|
110
143
|
export declare function reportJson(report: Report): string;
|