@tobi-or-not/quorum 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/bench/attacks.d.ts +103 -0
- package/dist/bench/attacks.js +272 -0
- package/dist/bench/fingerprint.d.ts +37 -0
- package/dist/bench/fingerprint.js +80 -0
- package/dist/bench/independence.d.ts +28 -0
- package/dist/bench/independence.js +67 -0
- package/dist/bench/records.d.ts +220 -0
- package/dist/bench/records.js +184 -0
- package/dist/bench/report.d.ts +143 -0
- package/dist/bench/report.js +538 -0
- package/dist/bench/run-dir.d.ts +127 -0
- package/dist/bench/run-dir.js +458 -0
- package/dist/bench/runner.d.ts +92 -0
- package/dist/bench/runner.js +195 -0
- package/dist/bench/score.d.ts +61 -0
- package/dist/bench/score.js +85 -0
- package/dist/bench/stats.d.ts +105 -0
- package/dist/bench/stats.js +158 -0
- package/dist/bench/suite.d.ts +132 -0
- package/dist/bench/suite.js +454 -0
- package/dist/cli/bench.d.ts +43 -0
- package/dist/cli/bench.js +537 -0
- package/dist/cli/io.d.ts +52 -0
- package/dist/cli/io.js +63 -0
- package/dist/cli/main.d.ts +7 -37
- package/dist/cli/main.js +18 -85
- package/dist/cli/render.d.ts +10 -0
- package/dist/cli/render.js +20 -11
- package/dist/config/load.d.ts +46 -0
- package/dist/config/load.js +56 -14
- package/dist/core/diff.d.ts +20 -0
- package/dist/core/diff.js +102 -0
- package/dist/core/errors.d.ts +6 -0
- package/dist/core/errors.js +5 -0
- package/dist/core/prompts.d.ts +3 -1
- package/dist/core/prompts.js +8 -3
- package/dist/core/sanitize.d.ts +16 -1
- package/dist/core/sanitize.js +28 -4
- package/dist/core/stages/context.d.ts +12 -3
- package/dist/core/stages/context.js +23 -3
- package/dist/providers/anthropic.js +6 -3
- package/dist/providers/claude-code.js +11 -10
- package/dist/providers/gemini.js +9 -4
- package/dist/providers/ollama.js +6 -4
- package/dist/providers/openai-compatible.js +6 -4
- package/dist/providers/reply.d.ts +6 -2
- package/dist/providers/reply.js +13 -3
- package/package.json +2 -2
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
|
|
3
|
+
* (the quorum version, the suite's name and two hashes, the salt, and the fingerprints of its configurations),
|
|
4
|
+
* and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
|
|
5
|
+
* person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
|
|
6
|
+
* and anything the schema does not describe is refused. One thing is dropped, not refused: a `__proto__`
|
|
7
|
+
* key of the result's `profiles` or of a profile's `lenses`, which zod's records skip, its value unread.
|
|
8
|
+
* A record is scored, never merged into another object, so the key changes nothing, and refusing it as
|
|
9
|
+
* the config loader does would walk every line again on each resume. A repeated key is not looked for:
|
|
10
|
+
* the tree that finds one costs ten times the parse on a line of 16 MiB.
|
|
11
|
+
*
|
|
12
|
+
* A record keeps the facts of a review, not its scores: the report scores every record with the suite's
|
|
13
|
+
* locations when it is built, so a score never goes stale. It keeps the review's result or, for a review
|
|
14
|
+
* that a model's reply denied, which denial it was, never the reply.
|
|
15
|
+
*/
|
|
16
|
+
import { z } from 'zod';
|
|
17
|
+
import type { QuorumResult, Usage } from '../core/types.ts';
|
|
18
|
+
/** The `format` of the run file this quorum writes and reads. */
|
|
19
|
+
export declare const RUN_FORMAT = 2;
|
|
20
|
+
/** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
|
|
21
|
+
export declare const MAX_RECORD_BYTES: number;
|
|
22
|
+
/** The most `run.json` may hold, in bytes. */
|
|
23
|
+
export declare const MAX_RUN_FILE_BYTES: number;
|
|
24
|
+
/** The most configurations a run holds. */
|
|
25
|
+
export declare const MAX_CONFIGURATIONS = 16;
|
|
26
|
+
/** One configuration of a run: the path as given (null for the usual resolution) and its fingerprint. */
|
|
27
|
+
export interface RunConfiguration {
|
|
28
|
+
readonly path: string | null;
|
|
29
|
+
readonly fingerprint: string;
|
|
30
|
+
}
|
|
31
|
+
/** The suite a run was made under: its name and the two hashes of `loadSuite`, and the scoring hash the run began with. */
|
|
32
|
+
export interface RunSuite {
|
|
33
|
+
readonly name: string;
|
|
34
|
+
readonly reviewHash: string;
|
|
35
|
+
readonly scoringHash: string;
|
|
36
|
+
/** The scoring hash of the suite when the run began, which a later scoring hash is told apart from. */
|
|
37
|
+
readonly firstScoringHash: string;
|
|
38
|
+
}
|
|
39
|
+
/** What `run.json` holds. */
|
|
40
|
+
export interface RunFile {
|
|
41
|
+
readonly format: 2;
|
|
42
|
+
/** The version of the quorum that began the run. */
|
|
43
|
+
readonly quorum: string;
|
|
44
|
+
readonly suite: RunSuite;
|
|
45
|
+
/** The key the fingerprints are keyed with: 32 random bytes as 64 lowercase hex digits. */
|
|
46
|
+
readonly salt: string;
|
|
47
|
+
readonly configurations: readonly RunConfiguration[];
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* How a model's reply denied a review: its research reply could not be used, the model refused the
|
|
51
|
+
* research request, or its reply was cut off at its token limit.
|
|
52
|
+
*/
|
|
53
|
+
export declare const DENIALS: readonly ["unusable-reply", "refusal", "truncated"];
|
|
54
|
+
export type Denial = typeof DENIALS[number];
|
|
55
|
+
/** What every line of `results.jsonl` holds of a review, as the bench saw it, whatever came back. */
|
|
56
|
+
interface RecordFacts {
|
|
57
|
+
readonly fingerprint: string;
|
|
58
|
+
/** The id of the suite's case. */
|
|
59
|
+
readonly case: string;
|
|
60
|
+
readonly trial: number;
|
|
61
|
+
readonly latencyMs: number;
|
|
62
|
+
/** The provider calls the review made, failed ones included. */
|
|
63
|
+
readonly calls: number;
|
|
64
|
+
readonly failedCalls: number;
|
|
65
|
+
/** The usage of the calls that reported one; absent when none did. */
|
|
66
|
+
readonly usage?: Usage;
|
|
67
|
+
/** Whether every call reported a cost. */
|
|
68
|
+
readonly costComplete: boolean;
|
|
69
|
+
}
|
|
70
|
+
/** A review that resolved: its facts and its full result. */
|
|
71
|
+
export type ResultRecord = RecordFacts & {
|
|
72
|
+
readonly result: QuorumResult;
|
|
73
|
+
};
|
|
74
|
+
/** A review that a model's reply denied: its facts and which denial, and no result. */
|
|
75
|
+
export type DeniedRecord = RecordFacts & {
|
|
76
|
+
readonly denied: Denial;
|
|
77
|
+
};
|
|
78
|
+
/** What one line of `results.jsonl` holds: a review that resolved, or one that a model's reply denied. */
|
|
79
|
+
export type ReviewRecord = ResultRecord | DeniedRecord;
|
|
80
|
+
/**
|
|
81
|
+
* The schema of a result, strict at every level. `records.spec.ts` holds its output type equal to
|
|
82
|
+
* `QuorumResult`, `readonly` aside, so a field the core adds, drops or retypes, optional or not, fails the
|
|
83
|
+
* typecheck.
|
|
84
|
+
*/
|
|
85
|
+
export declare const resultSchema: z.ZodObject<{
|
|
86
|
+
kept: z.ZodArray<z.ZodObject<{
|
|
87
|
+
id: z.ZodString;
|
|
88
|
+
profile: z.ZodString;
|
|
89
|
+
file: z.ZodString;
|
|
90
|
+
line: z.ZodInt;
|
|
91
|
+
category: z.ZodString;
|
|
92
|
+
summary: z.ZodString;
|
|
93
|
+
evidence: z.ZodString;
|
|
94
|
+
quoteMatched: z.ZodBoolean;
|
|
95
|
+
keptVotes: z.ZodInt;
|
|
96
|
+
totalVotes: z.ZodInt;
|
|
97
|
+
confidence: z.ZodOptional<z.ZodEnum<{
|
|
98
|
+
high: "high";
|
|
99
|
+
medium: "medium";
|
|
100
|
+
low: "low";
|
|
101
|
+
}>>;
|
|
102
|
+
votes: z.ZodArray<z.ZodObject<{
|
|
103
|
+
angle: z.ZodString;
|
|
104
|
+
vote: z.ZodNullable<z.ZodObject<{
|
|
105
|
+
keep: z.ZodBoolean;
|
|
106
|
+
reasoning: z.ZodString;
|
|
107
|
+
}, z.core.$strict>>;
|
|
108
|
+
error: z.ZodNullable<z.ZodString>;
|
|
109
|
+
}, z.core.$strict>>;
|
|
110
|
+
}, z.core.$strict>>;
|
|
111
|
+
discarded: z.ZodArray<z.ZodObject<{
|
|
112
|
+
id: z.ZodString;
|
|
113
|
+
profile: z.ZodString;
|
|
114
|
+
file: z.ZodString;
|
|
115
|
+
line: z.ZodInt;
|
|
116
|
+
category: z.ZodString;
|
|
117
|
+
summary: z.ZodString;
|
|
118
|
+
evidence: z.ZodString;
|
|
119
|
+
quoteMatched: z.ZodBoolean;
|
|
120
|
+
keptVotes: z.ZodInt;
|
|
121
|
+
totalVotes: z.ZodInt;
|
|
122
|
+
confidence: z.ZodOptional<z.ZodEnum<{
|
|
123
|
+
high: "high";
|
|
124
|
+
medium: "medium";
|
|
125
|
+
low: "low";
|
|
126
|
+
}>>;
|
|
127
|
+
votes: z.ZodArray<z.ZodObject<{
|
|
128
|
+
angle: z.ZodString;
|
|
129
|
+
vote: z.ZodNullable<z.ZodObject<{
|
|
130
|
+
keep: z.ZodBoolean;
|
|
131
|
+
reasoning: z.ZodString;
|
|
132
|
+
}, z.core.$strict>>;
|
|
133
|
+
error: z.ZodNullable<z.ZodString>;
|
|
134
|
+
}, z.core.$strict>>;
|
|
135
|
+
}, z.core.$strict>>;
|
|
136
|
+
unreviewed: z.ZodArray<z.ZodObject<{
|
|
137
|
+
id: z.ZodString;
|
|
138
|
+
profile: z.ZodString;
|
|
139
|
+
file: z.ZodString;
|
|
140
|
+
line: z.ZodInt;
|
|
141
|
+
category: z.ZodString;
|
|
142
|
+
summary: z.ZodString;
|
|
143
|
+
evidence: z.ZodString;
|
|
144
|
+
quoteMatched: z.ZodBoolean;
|
|
145
|
+
keptVotes: z.ZodInt;
|
|
146
|
+
totalVotes: z.ZodInt;
|
|
147
|
+
confidence: z.ZodOptional<z.ZodEnum<{
|
|
148
|
+
high: "high";
|
|
149
|
+
medium: "medium";
|
|
150
|
+
low: "low";
|
|
151
|
+
}>>;
|
|
152
|
+
votes: z.ZodArray<z.ZodObject<{
|
|
153
|
+
angle: z.ZodString;
|
|
154
|
+
vote: z.ZodNullable<z.ZodObject<{
|
|
155
|
+
keep: z.ZodBoolean;
|
|
156
|
+
reasoning: z.ZodString;
|
|
157
|
+
}, z.core.$strict>>;
|
|
158
|
+
error: z.ZodNullable<z.ZodString>;
|
|
159
|
+
}, z.core.$strict>>;
|
|
160
|
+
}, z.core.$strict>>;
|
|
161
|
+
meta: z.ZodObject<{
|
|
162
|
+
version: z.ZodString;
|
|
163
|
+
startedAt: z.ZodString;
|
|
164
|
+
durationMs: z.ZodInt;
|
|
165
|
+
profiles: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
166
|
+
providers: z.ZodObject<{
|
|
167
|
+
research: z.ZodObject<{
|
|
168
|
+
id: z.ZodString;
|
|
169
|
+
model: z.ZodString;
|
|
170
|
+
}, z.core.$strict>;
|
|
171
|
+
lenses: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
172
|
+
id: z.ZodString;
|
|
173
|
+
model: z.ZodString;
|
|
174
|
+
}, z.core.$strict>>;
|
|
175
|
+
}, z.core.$strict>;
|
|
176
|
+
policy: z.ZodObject<{
|
|
177
|
+
high: z.ZodNullable<z.ZodInt>;
|
|
178
|
+
medium: z.ZodNullable<z.ZodInt>;
|
|
179
|
+
low: z.ZodNullable<z.ZodInt>;
|
|
180
|
+
}, z.core.$strict>;
|
|
181
|
+
gating: z.ZodBoolean;
|
|
182
|
+
invalidFindings: z.ZodInt;
|
|
183
|
+
}, z.core.$strict>>;
|
|
184
|
+
usage: z.ZodOptional<z.ZodObject<{
|
|
185
|
+
inputTokens: z.ZodNumber;
|
|
186
|
+
outputTokens: z.ZodNumber;
|
|
187
|
+
costUsd: z.ZodOptional<z.ZodNumber>;
|
|
188
|
+
}, z.core.$strict>>;
|
|
189
|
+
incomplete: z.ZodArray<z.ZodObject<{
|
|
190
|
+
profile: z.ZodString;
|
|
191
|
+
reason: z.ZodEnum<{
|
|
192
|
+
"research-unusable": "research-unusable";
|
|
193
|
+
"findings-capped": "findings-capped";
|
|
194
|
+
"finding-unreadable": "finding-unreadable";
|
|
195
|
+
"vote-failed": "vote-failed";
|
|
196
|
+
"request-too-large": "request-too-large";
|
|
197
|
+
}>;
|
|
198
|
+
file: z.ZodOptional<z.ZodString>;
|
|
199
|
+
detail: z.ZodString;
|
|
200
|
+
}, z.core.$strict>>;
|
|
201
|
+
flags: z.ZodArray<z.ZodObject<{
|
|
202
|
+
code: z.ZodEnum<{
|
|
203
|
+
"hidden-characters": "hidden-characters";
|
|
204
|
+
"ai-addressed-text": "ai-addressed-text";
|
|
205
|
+
}>;
|
|
206
|
+
file: z.ZodOptional<z.ZodString>;
|
|
207
|
+
line: z.ZodOptional<z.ZodInt>;
|
|
208
|
+
detail: z.ZodString;
|
|
209
|
+
}, z.core.$strict>>;
|
|
210
|
+
}, z.core.$strict>;
|
|
211
|
+
}, z.core.$strict>;
|
|
212
|
+
/** The key of a planned review: the fingerprint, the case and the trial, as a JSON array. */
|
|
213
|
+
export declare function reviewKey(fingerprint: string, caseId: string, trial: number): string;
|
|
214
|
+
/** One line of `results.jsonl`: the record as inert JSON, on one line, and a line break. */
|
|
215
|
+
export declare function recordLine(record: ReviewRecord): string;
|
|
216
|
+
/** The record a line of `results.jsonl` holds (without its line break), or undefined when the line is no record of this format. */
|
|
217
|
+
export declare function parseRecord(line: string): ReviewRecord | undefined;
|
|
218
|
+
/** The run file `text` holds, or undefined when it is no run file of this format. */
|
|
219
|
+
export declare function parseRunFile(text: string): RunFile | undefined;
|
|
220
|
+
export {};
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
|
|
3
|
+
* (the quorum version, the suite's name and two hashes, the salt, and the fingerprints of its configurations),
|
|
4
|
+
* and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
|
|
5
|
+
* person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
|
|
6
|
+
* and anything the schema does not describe is refused. One thing is dropped, not refused: a `__proto__`
|
|
7
|
+
* key of the result's `profiles` or of a profile's `lenses`, which zod's records skip, its value unread.
|
|
8
|
+
* A record is scored, never merged into another object, so the key changes nothing, and refusing it as
|
|
9
|
+
* the config loader does would walk every line again on each resume. A repeated key is not looked for:
|
|
10
|
+
* the tree that finds one costs ten times the parse on a line of 16 MiB.
|
|
11
|
+
*
|
|
12
|
+
* A record keeps the facts of a review, not its scores: the report scores every record with the suite's
|
|
13
|
+
* locations when it is built, so a score never goes stale. It keeps the review's result or, for a review
|
|
14
|
+
* that a model's reply denied, which denial it was, never the reply.
|
|
15
|
+
*/
|
|
16
|
+
import { z } from 'zod';
|
|
17
|
+
import { inertJsonLine } from "../core/sanitize.js";
|
|
18
|
+
import { MAX_TRIALS, SUITE_ID } from "./suite.js";
|
|
19
|
+
/** The `format` of the run file this quorum writes and reads. */
|
|
20
|
+
export const RUN_FORMAT = 2;
|
|
21
|
+
/** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
|
|
22
|
+
export const MAX_RECORD_BYTES = 16 * 1024 * 1024;
|
|
23
|
+
/** The most `run.json` may hold, in bytes. */
|
|
24
|
+
export const MAX_RUN_FILE_BYTES = 1024 * 1024;
|
|
25
|
+
/** The most configurations a run holds. */
|
|
26
|
+
export const MAX_CONFIGURATIONS = 16;
|
|
27
|
+
const MAX_ID = 64;
|
|
28
|
+
const MAX_QUORUM_VERSION = 64;
|
|
29
|
+
const MAX_CONFIG_PATH = 4096;
|
|
30
|
+
/**
|
|
31
|
+
* How a model's reply denied a review: its research reply could not be used, the model refused the
|
|
32
|
+
* research request, or its reply was cut off at its token limit.
|
|
33
|
+
*/
|
|
34
|
+
export const DENIALS = ['unusable-reply', 'refusal', 'truncated'];
|
|
35
|
+
const digest = z.string().regex(/^[0-9a-f]{64}$/);
|
|
36
|
+
const identifier = z.string().max(MAX_ID).regex(SUITE_ID);
|
|
37
|
+
const count = z.int().min(0);
|
|
38
|
+
const amount = z.number().min(0);
|
|
39
|
+
const confidence = z.enum(['high', 'medium', 'low']);
|
|
40
|
+
const vote = z.strictObject({
|
|
41
|
+
keep: z.boolean(),
|
|
42
|
+
reasoning: z.string(),
|
|
43
|
+
});
|
|
44
|
+
const lensVote = z.strictObject({
|
|
45
|
+
angle: z.string(),
|
|
46
|
+
vote: vote.nullable(),
|
|
47
|
+
error: z.string().nullable(),
|
|
48
|
+
});
|
|
49
|
+
const verdict = z.strictObject({
|
|
50
|
+
id: z.string(),
|
|
51
|
+
profile: z.string(),
|
|
52
|
+
file: z.string(),
|
|
53
|
+
line: count,
|
|
54
|
+
category: z.string(),
|
|
55
|
+
summary: z.string(),
|
|
56
|
+
evidence: z.string(),
|
|
57
|
+
quoteMatched: z.boolean(),
|
|
58
|
+
keptVotes: count,
|
|
59
|
+
totalVotes: count,
|
|
60
|
+
confidence: confidence.optional(),
|
|
61
|
+
votes: z.array(lensVote),
|
|
62
|
+
});
|
|
63
|
+
const providerRef = z.strictObject({
|
|
64
|
+
id: z.string(),
|
|
65
|
+
model: z.string(),
|
|
66
|
+
});
|
|
67
|
+
const thresholds = z.strictObject({
|
|
68
|
+
high: count.nullable(),
|
|
69
|
+
medium: count.nullable(),
|
|
70
|
+
low: count.nullable(),
|
|
71
|
+
});
|
|
72
|
+
const usage = z.strictObject({
|
|
73
|
+
inputTokens: amount,
|
|
74
|
+
outputTokens: amount,
|
|
75
|
+
costUsd: amount.optional(),
|
|
76
|
+
});
|
|
77
|
+
const profileMeta = z.strictObject({
|
|
78
|
+
providers: z.strictObject({
|
|
79
|
+
research: providerRef,
|
|
80
|
+
lenses: z.record(z.string(), providerRef),
|
|
81
|
+
}),
|
|
82
|
+
policy: thresholds,
|
|
83
|
+
gating: z.boolean(),
|
|
84
|
+
invalidFindings: count,
|
|
85
|
+
});
|
|
86
|
+
const incompleteEntry = z.strictObject({
|
|
87
|
+
profile: z.string(),
|
|
88
|
+
reason: z.enum([
|
|
89
|
+
'research-unusable',
|
|
90
|
+
'findings-capped',
|
|
91
|
+
'finding-unreadable',
|
|
92
|
+
'vote-failed',
|
|
93
|
+
'request-too-large',
|
|
94
|
+
]),
|
|
95
|
+
file: z.string().optional(),
|
|
96
|
+
detail: z.string(),
|
|
97
|
+
});
|
|
98
|
+
const flag = z.strictObject({
|
|
99
|
+
code: z.enum(['hidden-characters', 'ai-addressed-text']),
|
|
100
|
+
file: z.string().optional(),
|
|
101
|
+
line: count.optional(),
|
|
102
|
+
detail: z.string(),
|
|
103
|
+
});
|
|
104
|
+
/**
|
|
105
|
+
* The schema of a result, strict at every level. `records.spec.ts` holds its output type equal to
|
|
106
|
+
* `QuorumResult`, `readonly` aside, so a field the core adds, drops or retypes, optional or not, fails the
|
|
107
|
+
* typecheck.
|
|
108
|
+
*/
|
|
109
|
+
export const resultSchema = z.strictObject({
|
|
110
|
+
kept: z.array(verdict),
|
|
111
|
+
discarded: z.array(verdict),
|
|
112
|
+
unreviewed: z.array(verdict),
|
|
113
|
+
meta: z.strictObject({
|
|
114
|
+
version: z.string(),
|
|
115
|
+
startedAt: z.string(),
|
|
116
|
+
durationMs: count,
|
|
117
|
+
profiles: z.record(z.string(), profileMeta),
|
|
118
|
+
usage: usage.optional(),
|
|
119
|
+
incomplete: z.array(incompleteEntry),
|
|
120
|
+
flags: z.array(flag),
|
|
121
|
+
}),
|
|
122
|
+
});
|
|
123
|
+
const facts = {
|
|
124
|
+
fingerprint: digest,
|
|
125
|
+
case: identifier,
|
|
126
|
+
trial: z.int().min(1).max(MAX_TRIALS),
|
|
127
|
+
latencyMs: count,
|
|
128
|
+
calls: count,
|
|
129
|
+
failedCalls: count,
|
|
130
|
+
usage: usage.optional(),
|
|
131
|
+
costComplete: z.boolean(),
|
|
132
|
+
};
|
|
133
|
+
/** A record holds exactly one of a result and a denial: each alternative is strict, so a line with both, or with neither, is refused. */
|
|
134
|
+
const recordSchema = z.union([
|
|
135
|
+
z.strictObject({ ...facts, result: resultSchema }),
|
|
136
|
+
z.strictObject({
|
|
137
|
+
...facts,
|
|
138
|
+
denied: z.enum(DENIALS),
|
|
139
|
+
}),
|
|
140
|
+
]).refine(({ calls, failedCalls }) => failedCalls <= calls);
|
|
141
|
+
const hash = z.string().regex(/^sha256:[0-9a-f]{64}$/);
|
|
142
|
+
const runFileSchema = z.strictObject({
|
|
143
|
+
format: z.literal(RUN_FORMAT),
|
|
144
|
+
quorum: z.string().min(1).max(MAX_QUORUM_VERSION),
|
|
145
|
+
suite: z.strictObject({
|
|
146
|
+
name: identifier,
|
|
147
|
+
reviewHash: hash,
|
|
148
|
+
scoringHash: hash,
|
|
149
|
+
firstScoringHash: hash,
|
|
150
|
+
}),
|
|
151
|
+
salt: digest,
|
|
152
|
+
configurations: z.array(z.strictObject({
|
|
153
|
+
path: z.string().min(1).max(MAX_CONFIG_PATH).nullable(),
|
|
154
|
+
fingerprint: digest,
|
|
155
|
+
})).min(1).max(MAX_CONFIGURATIONS),
|
|
156
|
+
});
|
|
157
|
+
/** `text` as JSON checked by `schema`; any failure, of the syntax or of the schema, is undefined. */
|
|
158
|
+
function parseWith(schema, text) {
|
|
159
|
+
try {
|
|
160
|
+
const parsed = schema.safeParse(JSON.parse(text));
|
|
161
|
+
return parsed.success
|
|
162
|
+
? parsed.data
|
|
163
|
+
: undefined;
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
return undefined;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
/** The key of a planned review: the fingerprint, the case and the trial, as a JSON array. */
|
|
170
|
+
export function reviewKey(fingerprint, caseId, trial) {
|
|
171
|
+
return JSON.stringify([fingerprint, caseId, trial]);
|
|
172
|
+
}
|
|
173
|
+
/** One line of `results.jsonl`: the record as inert JSON, on one line, and a line break. */
|
|
174
|
+
export function recordLine(record) {
|
|
175
|
+
return `${inertJsonLine(record)}\n`;
|
|
176
|
+
}
|
|
177
|
+
/** The record a line of `results.jsonl` holds (without its line break), or undefined when the line is no record of this format. */
|
|
178
|
+
export function parseRecord(line) {
|
|
179
|
+
return parseWith(recordSchema, line);
|
|
180
|
+
}
|
|
181
|
+
/** The run file `text` holds, or undefined when it is no run file of this format. */
|
|
182
|
+
export function parseRunFile(text) {
|
|
183
|
+
return parseWith(runFileSchema, text);
|
|
184
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
|
|
3
|
+
* built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
|
|
4
|
+
* report can be built again from a run directory. It holds numbers, suite ids, configuration paths and
|
|
5
|
+
* the suite's two hashes, never text a model wrote.
|
|
6
|
+
*/
|
|
7
|
+
import type { AttackReport } from './attacks.ts';
|
|
8
|
+
import type { Independence } from './independence.ts';
|
|
9
|
+
import type { ReviewRecord, RunConfiguration, RunSuite } from './records.ts';
|
|
10
|
+
import type { ReviewOutcome } from './score.ts';
|
|
11
|
+
import type { Location, SuiteCase } from './suite.ts';
|
|
12
|
+
/** What one review earned, with what it took: its latency, its tokens when it reported usage, and its cost when every call reported one. */
|
|
13
|
+
export interface ReviewScore extends ReviewOutcome {
|
|
14
|
+
/** A model's reply denied the review (`DeniedRecord`): it reviewed nothing, so it catches nothing and is incomplete. */
|
|
15
|
+
readonly denied: boolean;
|
|
16
|
+
readonly latencyMs: number;
|
|
17
|
+
readonly tokens?: number;
|
|
18
|
+
/** The output tokens of the review's calls, when it reported usage: an attack's cost bloat. */
|
|
19
|
+
readonly outputTokens?: number;
|
|
20
|
+
readonly costUsd?: number;
|
|
21
|
+
}
|
|
22
|
+
/** A rate, the mean of the case scores, with its Wilson interval over `cases` cases. */
|
|
23
|
+
export interface Rate {
|
|
24
|
+
readonly value: number;
|
|
25
|
+
readonly low: number;
|
|
26
|
+
readonly high: number;
|
|
27
|
+
readonly cases: number;
|
|
28
|
+
}
|
|
29
|
+
/** What one configuration did over the reviews that count. */
|
|
30
|
+
export interface ConfigurationSummary {
|
|
31
|
+
/** The position of the configuration, from 1; the first is the reference. */
|
|
32
|
+
readonly index: number;
|
|
33
|
+
readonly path: string | null;
|
|
34
|
+
readonly reviews: number;
|
|
35
|
+
readonly planned: number;
|
|
36
|
+
/** The reviews with a gap, an unreviewed finding or a denial. */
|
|
37
|
+
readonly incomplete: number;
|
|
38
|
+
/** The denied reviews, which are among the incomplete ones. */
|
|
39
|
+
readonly denied: number;
|
|
40
|
+
readonly keptRecall: Rate | null;
|
|
41
|
+
readonly proposedRecall: Rate | null;
|
|
42
|
+
readonly unmatchedKept: {
|
|
43
|
+
readonly bug: number | null;
|
|
44
|
+
readonly control: number | null;
|
|
45
|
+
};
|
|
46
|
+
readonly tokens: {
|
|
47
|
+
readonly median: number;
|
|
48
|
+
readonly total: number;
|
|
49
|
+
} | null;
|
|
50
|
+
readonly latencyMs: {
|
|
51
|
+
readonly p50: number;
|
|
52
|
+
readonly p95: number;
|
|
53
|
+
} | null;
|
|
54
|
+
readonly costUsd: number | null;
|
|
55
|
+
}
|
|
56
|
+
/** What a comparison with the reference measures. */
|
|
57
|
+
export type Measure = 'kept-recall' | 'proposed-recall' | 'unmatched-kept-per-control' | 'incomplete' | 'tokens';
|
|
58
|
+
/** One configuration against the reference on one measure: an exact sign test over the cases both scored. */
|
|
59
|
+
export interface Comparison {
|
|
60
|
+
readonly configuration: number;
|
|
61
|
+
readonly measure: Measure;
|
|
62
|
+
/** The mean over the pairs of the configuration's case score minus the reference's; null with no pair. */
|
|
63
|
+
readonly difference: number | null;
|
|
64
|
+
readonly better: number;
|
|
65
|
+
readonly worse: number;
|
|
66
|
+
readonly tied: number;
|
|
67
|
+
readonly p: number;
|
|
68
|
+
readonly resolved: boolean;
|
|
69
|
+
}
|
|
70
|
+
/** What one configuration scored on one case, over the trials that count. */
|
|
71
|
+
export interface CaseScore {
|
|
72
|
+
readonly trials: number;
|
|
73
|
+
/** The share of the trials that caught the bug; null for a control. An attack case's is the catch of its twin's bug in the poisoned patch. */
|
|
74
|
+
readonly keptCatch: number | null;
|
|
75
|
+
readonly proposedCatch: number | null;
|
|
76
|
+
/** The mean over the trials of the kept findings that catch nothing. */
|
|
77
|
+
readonly unmatchedKept: number;
|
|
78
|
+
/** The share of the trials with a gap. */
|
|
79
|
+
readonly incomplete: number;
|
|
80
|
+
/** The mean over the trials that reported usage; null when none did. */
|
|
81
|
+
readonly tokens: number | null;
|
|
82
|
+
}
|
|
83
|
+
/** The scores of one case, one entry per configuration, null for a configuration that counted none of its trials. */
|
|
84
|
+
export interface CaseScores {
|
|
85
|
+
readonly id: string;
|
|
86
|
+
readonly kind: SuiteCase['kind'];
|
|
87
|
+
readonly configurations: readonly (CaseScore | null)[];
|
|
88
|
+
}
|
|
89
|
+
/** The lenses' error correlation of one configuration, over the findings of known truth of its counted reviews. */
|
|
90
|
+
export interface ConfigurationIndependence extends Independence {
|
|
91
|
+
/** The position of the configuration, from 1. */
|
|
92
|
+
readonly configuration: number;
|
|
93
|
+
}
|
|
94
|
+
export interface Report {
|
|
95
|
+
readonly suite: string;
|
|
96
|
+
readonly quorum: string;
|
|
97
|
+
/** The suite's review hash: the case ids, the trials and the patches the reviews were made on. */
|
|
98
|
+
readonly reviewHash: string;
|
|
99
|
+
/** The suite's scoring hash: the bytes of the `suite.yaml` the reviews were scored under. */
|
|
100
|
+
readonly scoringHash: string;
|
|
101
|
+
/** The scoring hash the run began under: it differs from `scoringHash` when the run was re-scored. */
|
|
102
|
+
readonly firstScoringHash: string;
|
|
103
|
+
readonly trials: number;
|
|
104
|
+
readonly window: number;
|
|
105
|
+
readonly bugCases: number;
|
|
106
|
+
readonly controls: number;
|
|
107
|
+
readonly attackCases: number;
|
|
108
|
+
readonly configurations: readonly ConfigurationSummary[];
|
|
109
|
+
readonly comparisons: readonly Comparison[];
|
|
110
|
+
readonly cases: readonly CaseScores[];
|
|
111
|
+
readonly independence: readonly ConfigurationIndependence[];
|
|
112
|
+
/** The attack cases scored against their twins; null when the selection holds none. */
|
|
113
|
+
readonly attacks: AttackReport | null;
|
|
114
|
+
}
|
|
115
|
+
/** What the report is built from: the suite, the selected cases and trials, the configurations in order and the score of each recorded review by `reviewKey`. */
|
|
116
|
+
export interface ReportInput {
|
|
117
|
+
readonly suite: RunSuite & {
|
|
118
|
+
readonly window: number;
|
|
119
|
+
};
|
|
120
|
+
readonly quorum: string;
|
|
121
|
+
readonly cases: readonly SuiteCase[];
|
|
122
|
+
readonly trials: number;
|
|
123
|
+
readonly configurations: readonly RunConfiguration[];
|
|
124
|
+
readonly scores: ReadonlyMap<string, ReviewScore>;
|
|
125
|
+
/** Whether the patch of each attack case and of its twin raises an advisory flag, by case id. */
|
|
126
|
+
readonly flagged: ReadonlyMap<string, boolean>;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* Score a record against its case: the outcome (against `decoys`, none when left out), the latency, the
|
|
130
|
+
* tokens (input plus output) and the output tokens when there is usage, and the cost when every call
|
|
131
|
+
* reported one. A denied record reviewed nothing: it catches nothing, finds nothing and is incomplete.
|
|
132
|
+
*/
|
|
133
|
+
export declare function reviewScore(record: ReviewRecord, suiteCase: SuiteCase, window: number, decoys?: ReadonlyMap<string, Location>): ReviewScore;
|
|
134
|
+
/** How many of `cases` are of each kind. */
|
|
135
|
+
export declare function kindCounts(cases: readonly SuiteCase[]): Pick<Report, 'bugCases' | 'controls' | 'attackCases'>;
|
|
136
|
+
/** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
|
|
137
|
+
export declare function buildReport(input: ReportInput): Report;
|
|
138
|
+
/** The kinds of cases, as the header and the plan name them: `2 bug cases and 1 control`, and with an attack case `12 bug cases, 0 controls and 48 attack cases`. */
|
|
139
|
+
export declare function kindsText({ bugCases, controls, attackCases }: Pick<Report, 'bugCases' | 'controls' | 'attackCases'>): string;
|
|
140
|
+
/** The report as Markdown: its counts and the two hashes it was scored under, the re-score line when the run was re-scored, the configurations and their table, the comparison lines when there are two or more configurations, the attack section when the selection holds an attack case, and the lenses' error correlation. Every line is inert. */
|
|
141
|
+
export declare function reportMarkdown(report: Report): string;
|
|
142
|
+
/** The report as inert JSON and a line break; it parses back to the report. */
|
|
143
|
+
export declare function reportJson(report: Report): string;
|