@tobi-or-not/quorum 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -77,6 +77,7 @@ The JSON result goes to stdout, and the exit code says what to do with it: `0` f
77
77
  - [Profiles](./docs/20-using/40-profiles.md) *- The security profile, writing your own, several in one review, gating and guidelines*
78
78
  - [Lenses and policy](./docs/20-using/50-lenses-and-policy.md) *- The panel's questions, the rules that turn kept votes into a confidence, failed votes, and what a level means*
79
79
  - [Prompts](./docs/20-using/60-prompts.md) *- The layout of every request, the templates and their variables, the notices, and what is not configurable*
80
+ - [Benchmarks](./docs/20-using/70-bench.md) *- quorum bench: measure configurations on a suite of real bugs, write a suite, resume a run, and read the report and its intervals*
80
81
 
81
82
  *Security*
82
83
  - [Threat model](./docs/30-security/10-threat-model.md) *- The attacker, the assets, the residual risk stated plainly, the controls in place, and what the panel's independence is worth*
@@ -0,0 +1,36 @@
1
+ /**
2
+ * The fingerprint of a configuration: the HMAC-SHA-256, under a salt the run keeps, of a canonical form
3
+ * of the resolved settings that names no provider. The bench keys its records by it, and refuses two
4
+ * configurations that resolve alike.
5
+ *
6
+ * The form holds every setting a review uses but the key: the profiles that run, in run order, each with
7
+ * its id, prompts, lenses, policy and gating, and its research provider and each lens's provider written
8
+ * out in place of their names; every field of a provider defined as data but its key, the values of
9
+ * `headers`, `query` and `extraBody` and the whole base URL included; a provider object as its config key
10
+ * and its model; `concurrency` and `maxFindings` as the review applies them, defaults filled in. It holds
11
+ * no file path, no warning and no provider name, so the same settings under other names are one
12
+ * configuration, and a rotated key keeps a run.
13
+ *
14
+ * The form can hold a secret (a credential kept in a header, a query parameter or the base path), so it
15
+ * never leaves this module: not written, not logged, not put in an error message. Only the HMAC does.
16
+ *
17
+ * What it cannot see: a provider object's code (a changed implementation under the same key and model
18
+ * needs a new run directory), and the environment a `claude-code` provider inherits.
19
+ */
20
+ import type { ResolvedConfig } from '../config/resolve.ts';
21
+ /** A fresh salt: 32 random bytes as 64 lowercase hex digits. */
22
+ export declare function newSalt(): string;
23
+ /**
24
+ * The canonical text of a configuration's settings: JSON with the keys of every object sorted, arrays in
25
+ * their order. It can hold a secret: hash it with `fingerprint`, and never write, log or print it.
26
+ *
27
+ * @throws {RangeError} when a profile names a provider the config does not hold.
28
+ */
29
+ export declare function canonicalSettings(config: ResolvedConfig): string;
30
+ /**
31
+ * The fingerprint of a configuration under a run's salt: the HMAC-SHA-256 of its canonical settings, as 64
32
+ * lowercase hex digits.
33
+ *
34
+ * @throws {RangeError} when `salt` is not 64 lowercase hex digits (the message never holds it), or a profile names a provider the config does not hold.
35
+ */
36
+ export declare function fingerprint(config: ResolvedConfig, salt: string): string;
@@ -0,0 +1,79 @@
1
+ /**
2
+ * The fingerprint of a configuration: the HMAC-SHA-256, under a salt the run keeps, of a canonical form
3
+ * of the resolved settings that names no provider. The bench keys its records by it, and refuses two
4
+ * configurations that resolve alike.
5
+ *
6
+ * The form holds every setting a review uses but the key: the profiles that run, in run order, each with
7
+ * its id, prompts, lenses, policy and gating, and its research provider and each lens's provider written
8
+ * out in place of their names; every field of a provider defined as data but its key, the values of
9
+ * `headers`, `query` and `extraBody` and the whole base URL included; a provider object as its config key
10
+ * and its model; `concurrency` and `maxFindings` as the review applies them, defaults filled in. It holds
11
+ * no file path, no warning and no provider name, so the same settings under other names are one
12
+ * configuration, and a rotated key keeps a run.
13
+ *
14
+ * The form can hold a secret (a credential kept in a header, a query parameter or the base path), so it
15
+ * never leaves this module: not written, not logged, not put in an error message. Only the HMAC does.
16
+ *
17
+ * What it cannot see: a provider object's code (a changed implementation under the same key and model
18
+ * needs a new run directory), and the environment a `claude-code` provider inherits.
19
+ */
20
+ import { createHmac, randomBytes } from 'node:crypto';
21
+ import { isPlainObject } from "../config/load.js";
22
+ import { isProvider } from "../config/schema.js";
23
+ import { DEFAULT_CONCURRENCY } from "../core/concurrency.js";
24
+ import { DEFAULT_MAX_FINDINGS } from "../core/stages/research.js";
25
+ const SALT = /^[0-9a-f]{64}$/;
26
+ /** A fresh salt: 32 random bytes as 64 lowercase hex digits. */
27
+ export function newSalt() {
28
+ return randomBytes(32).toString('hex');
29
+ }
30
+ /** A provider as the form holds it: a provider object by its config key and model, a spec by its fields less the key. */
31
+ function providerForm(provider) {
32
+ if (isProvider(provider))
33
+ return { object: provider.id, model: provider.model };
34
+ return Object.fromEntries(Object.entries(provider).filter(([field]) => field !== 'apiKey'));
35
+ }
36
+ /** The form of the provider a profile binds by `name`; a name the config holds no provider under is a RangeError, never a silent gap. */
37
+ function boundForm(config, name) {
38
+ if (!Object.hasOwn(config.providers, name))
39
+ throw new RangeError('a profile names a provider the config does not hold');
40
+ return providerForm(config.providers[name]);
41
+ }
42
+ /** A plain object with its keys in code-unit order, so the text does not depend on the order the config wrote. */
43
+ function sortKeys(_key, value) {
44
+ return isPlainObject(value)
45
+ ? Object.fromEntries(Object.keys(value).sort().map(key => [key, value[key]]))
46
+ : value;
47
+ }
48
+ /**
49
+ * The canonical text of a configuration's settings: JSON with the keys of every object sorted, arrays in
50
+ * their order. It can hold a secret: hash it with `fingerprint`, and never write, log or print it.
51
+ *
52
+ * @throws {RangeError} when a profile names a provider the config does not hold.
53
+ */
54
+ export function canonicalSettings(config) {
55
+ const profiles = config.profiles.map(({ profile, research, voters }) => ({
56
+ profile,
57
+ research: boundForm(config, research),
58
+ voters: Object.fromEntries(Object.entries(voters).map(([lens, name]) => [
59
+ lens,
60
+ boundForm(config, name),
61
+ ])),
62
+ }));
63
+ return JSON.stringify({
64
+ concurrency: config.concurrency ?? DEFAULT_CONCURRENCY,
65
+ maxFindings: config.maxFindings ?? DEFAULT_MAX_FINDINGS,
66
+ profiles,
67
+ }, sortKeys);
68
+ }
69
+ /**
70
+ * The fingerprint of a configuration under a run's salt: the HMAC-SHA-256 of its canonical settings, as 64
71
+ * lowercase hex digits.
72
+ *
73
+ * @throws {RangeError} when `salt` is not 64 lowercase hex digits (the message never holds it), or a profile names a provider the config does not hold.
74
+ */
75
+ export function fingerprint(config, salt) {
76
+ if (!SALT.test(salt))
77
+ throw new RangeError('the salt must be 64 lowercase hex digits');
78
+ return createHmac('sha256', Buffer.from(salt, 'hex')).update(canonicalSettings(config)).digest('hex');
79
+ }
@@ -0,0 +1,62 @@
1
+ /**
2
+ * The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
3
+ * (the quorum version, the suite's name and hash, the salt, and the fingerprints of its configurations),
4
+ * and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
5
+ * person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
6
+ * and anything the schema does not describe is refused, never repaired. A repeated key is not looked
7
+ * for: the tree that finds one costs ten times the parse on a line of 16 MiB.
8
+ *
9
+ * A record keeps the facts of a review, not its scores: the report scores every record with the suite's
10
+ * locations when it is built, so a score never goes stale.
11
+ */
12
+ import type { QuorumResult, Usage } from '../core/types.ts';
13
+ /** The `format` of the run file this quorum writes and reads. */
14
+ export declare const RUN_FORMAT = 1;
15
+ /** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
16
+ export declare const MAX_RECORD_BYTES: number;
17
+ /** The most `run.json` may hold, in bytes. */
18
+ export declare const MAX_RUN_FILE_BYTES: number;
19
+ /** The most configurations a run holds. */
20
+ export declare const MAX_CONFIGURATIONS = 16;
21
+ /** One configuration of a run: the path as given (null for the usual resolution) and its fingerprint. */
22
+ export interface RunConfiguration {
23
+ readonly path: string | null;
24
+ readonly fingerprint: string;
25
+ }
26
+ /** What `run.json` holds. */
27
+ export interface RunFile {
28
+ readonly format: 1;
29
+ /** The version of the quorum that began the run. */
30
+ readonly quorum: string;
31
+ readonly suite: {
32
+ readonly name: string;
33
+ readonly hash: string;
34
+ };
35
+ /** The key the fingerprints are keyed with: 32 random bytes as 64 lowercase hex digits. */
36
+ readonly salt: string;
37
+ readonly configurations: readonly RunConfiguration[];
38
+ }
39
+ /** What one line of `results.jsonl` holds: a review that resolved, as the bench saw it. */
40
+ export interface ReviewRecord {
41
+ readonly fingerprint: string;
42
+ /** The id of the suite's case. */
43
+ readonly case: string;
44
+ readonly trial: number;
45
+ readonly latencyMs: number;
46
+ /** The provider calls the review made, failed ones included. */
47
+ readonly calls: number;
48
+ readonly failedCalls: number;
49
+ /** The usage of the calls that reported one; absent when none did. */
50
+ readonly usage?: Usage;
51
+ /** Whether every call reported a cost. */
52
+ readonly costComplete: boolean;
53
+ readonly result: QuorumResult;
54
+ }
55
+ /** The key of a planned review: the fingerprint, the case and the trial, as a JSON array. */
56
+ export declare function reviewKey(fingerprint: string, caseId: string, trial: number): string;
57
+ /** One line of `results.jsonl`: the record as inert JSON, on one line, and a line break. */
58
+ export declare function recordLine(record: ReviewRecord): string;
59
+ /** The record a line of `results.jsonl` holds (without its line break), or undefined when the line is no record of this format. */
60
+ export declare function parseRecord(line: string): ReviewRecord | undefined;
61
+ /** The run file `text` holds, or undefined when it is no run file of this format. */
62
+ export declare function parseRunFile(text: string): RunFile | undefined;
@@ -0,0 +1,161 @@
1
+ /**
2
+ * The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
3
+ * (the quorum version, the suite's name and hash, the salt, and the fingerprints of its configurations),
4
+ * and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
5
+ * person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
6
+ * and anything the schema does not describe is refused, never repaired. A repeated key is not looked
7
+ * for: the tree that finds one costs ten times the parse on a line of 16 MiB.
8
+ *
9
+ * A record keeps the facts of a review, not its scores: the report scores every record with the suite's
10
+ * locations when it is built, so a score never goes stale.
11
+ */
12
+ import { z } from 'zod';
13
+ import { inertJsonLine } from "../core/sanitize.js";
14
+ import { MAX_TRIALS, SUITE_ID } from "./suite.js";
15
+ /** The `format` of the run file this quorum writes and reads. */
16
+ export const RUN_FORMAT = 1;
17
+ /** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
18
+ export const MAX_RECORD_BYTES = 16 * 1024 * 1024;
19
+ /** The most `run.json` may hold, in bytes. */
20
+ export const MAX_RUN_FILE_BYTES = 1024 * 1024;
21
+ /** The most configurations a run holds. */
22
+ export const MAX_CONFIGURATIONS = 16;
23
+ const MAX_ID = 64;
24
+ const MAX_QUORUM_VERSION = 64;
25
+ const MAX_CONFIG_PATH = 4096;
26
+ const digest = z.string().regex(/^[0-9a-f]{64}$/);
27
+ const identifier = z.string().max(MAX_ID).regex(SUITE_ID);
28
+ const count = z.int().min(0);
29
+ const amount = z.number().min(0);
30
+ const confidence = z.enum(['high', 'medium', 'low']);
31
+ const vote = z.strictObject({
32
+ keep: z.boolean(),
33
+ reasoning: z.string(),
34
+ });
35
+ const lensVote = z.strictObject({
36
+ angle: z.string(),
37
+ vote: vote.nullable(),
38
+ error: z.string().nullable(),
39
+ });
40
+ const verdict = z.strictObject({
41
+ id: z.string(),
42
+ profile: z.string(),
43
+ file: z.string(),
44
+ line: count,
45
+ category: z.string(),
46
+ summary: z.string(),
47
+ evidence: z.string(),
48
+ quoteMatched: z.boolean(),
49
+ keptVotes: count,
50
+ totalVotes: count,
51
+ confidence: confidence.optional(),
52
+ votes: z.array(lensVote),
53
+ });
54
+ const providerRef = z.strictObject({
55
+ id: z.string(),
56
+ model: z.string(),
57
+ });
58
+ const thresholds = z.strictObject({
59
+ high: count.nullable(),
60
+ medium: count.nullable(),
61
+ low: count.nullable(),
62
+ });
63
+ const usage = z.strictObject({
64
+ inputTokens: amount,
65
+ outputTokens: amount,
66
+ costUsd: amount.optional(),
67
+ });
68
+ const profileMeta = z.strictObject({
69
+ providers: z.strictObject({
70
+ research: providerRef,
71
+ lenses: z.record(z.string(), providerRef),
72
+ }),
73
+ policy: thresholds,
74
+ gating: z.boolean(),
75
+ invalidFindings: count,
76
+ });
77
+ const incompleteEntry = z.strictObject({
78
+ profile: z.string(),
79
+ reason: z.enum([
80
+ 'research-unusable',
81
+ 'findings-capped',
82
+ 'finding-unreadable',
83
+ 'vote-failed',
84
+ 'request-too-large',
85
+ ]),
86
+ file: z.string().optional(),
87
+ detail: z.string(),
88
+ });
89
+ const flag = z.strictObject({
90
+ code: z.enum(['hidden-characters', 'ai-addressed-text']),
91
+ file: z.string().optional(),
92
+ line: count.optional(),
93
+ detail: z.string(),
94
+ });
95
+ /** Declared as `QuorumResult`'s own type, so a required field the core adds and this schema lacks fails the typecheck. */
96
+ const resultSchema = z.strictObject({
97
+ kept: z.array(verdict),
98
+ discarded: z.array(verdict),
99
+ unreviewed: z.array(verdict),
100
+ meta: z.strictObject({
101
+ version: z.string(),
102
+ startedAt: z.string(),
103
+ durationMs: count,
104
+ profiles: z.record(z.string(), profileMeta),
105
+ usage: usage.optional(),
106
+ incomplete: z.array(incompleteEntry),
107
+ flags: z.array(flag),
108
+ }),
109
+ });
110
+ const recordSchema = z.strictObject({
111
+ fingerprint: digest,
112
+ case: identifier,
113
+ trial: z.int().min(1).max(MAX_TRIALS),
114
+ latencyMs: count,
115
+ calls: count,
116
+ failedCalls: count,
117
+ usage: usage.optional(),
118
+ costComplete: z.boolean(),
119
+ result: resultSchema,
120
+ }).refine(({ calls, failedCalls }) => failedCalls <= calls);
121
+ const runFileSchema = z.strictObject({
122
+ format: z.literal(RUN_FORMAT),
123
+ quorum: z.string().min(1).max(MAX_QUORUM_VERSION),
124
+ suite: z.strictObject({
125
+ name: identifier,
126
+ hash: z.string().regex(/^sha256:[0-9a-f]{64}$/),
127
+ }),
128
+ salt: digest,
129
+ configurations: z.array(z.strictObject({
130
+ path: z.string().min(1).max(MAX_CONFIG_PATH).nullable(),
131
+ fingerprint: digest,
132
+ })).min(1).max(MAX_CONFIGURATIONS),
133
+ });
134
+ /** `text` as JSON checked by `schema`; any failure, of the syntax or of the schema, is undefined. */
135
+ function parseWith(schema, text) {
136
+ try {
137
+ const parsed = schema.safeParse(JSON.parse(text));
138
+ return parsed.success
139
+ ? parsed.data
140
+ : undefined;
141
+ }
142
+ catch {
143
+ return undefined;
144
+ }
145
+ }
146
+ /** The key of a planned review: the fingerprint, the case and the trial, as a JSON array. */
147
+ export function reviewKey(fingerprint, caseId, trial) {
148
+ return JSON.stringify([fingerprint, caseId, trial]);
149
+ }
150
+ /** One line of `results.jsonl`: the record as inert JSON, on one line, and a line break. */
151
+ export function recordLine(record) {
152
+ return `${inertJsonLine(record)}\n`;
153
+ }
154
+ /** The record a line of `results.jsonl` holds (without its line break), or undefined when the line is no record of this format. */
155
+ export function parseRecord(line) {
156
+ return parseWith(recordSchema, line);
157
+ }
158
+ /** The run file `text` holds, or undefined when it is no run file of this format. */
159
+ export function parseRunFile(text) {
160
+ return parseWith(runFileSchema, text);
161
+ }
@@ -0,0 +1,110 @@
1
+ /**
2
+ * The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
3
+ * built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
4
+ * report can be built again from a run directory. It holds numbers, suite ids and configuration paths,
5
+ * never text a model wrote.
6
+ */
7
+ import type { ReviewRecord, RunConfiguration } from './records.ts';
8
+ import type { ReviewOutcome } from './score.ts';
9
+ import type { SuiteCase } from './suite.ts';
10
+ /** What one review earned, with what it took: its latency, its tokens when it reported usage, and its cost when every call reported one. */
11
+ export interface ReviewScore extends ReviewOutcome {
12
+ readonly latencyMs: number;
13
+ readonly tokens?: number;
14
+ readonly costUsd?: number;
15
+ }
16
+ /** A rate, the mean of the case scores, with its Wilson interval over `cases` cases. */
17
+ export interface Rate {
18
+ readonly value: number;
19
+ readonly low: number;
20
+ readonly high: number;
21
+ readonly cases: number;
22
+ }
23
+ /** What one configuration did over the reviews that count. */
24
+ export interface ConfigurationSummary {
25
+ /** The position of the configuration, from 1; the first is the reference. */
26
+ readonly index: number;
27
+ readonly path: string | null;
28
+ readonly reviews: number;
29
+ readonly planned: number;
30
+ readonly incomplete: number;
31
+ readonly keptRecall: Rate | null;
32
+ readonly proposedRecall: Rate | null;
33
+ readonly unmatchedKept: {
34
+ readonly bug: number | null;
35
+ readonly control: number | null;
36
+ };
37
+ readonly tokens: {
38
+ readonly median: number;
39
+ readonly total: number;
40
+ } | null;
41
+ readonly latencyMs: {
42
+ readonly p50: number;
43
+ readonly p95: number;
44
+ } | null;
45
+ readonly costUsd: number | null;
46
+ }
47
+ /** What a comparison with the reference measures. */
48
+ export type Measure = 'kept-recall' | 'proposed-recall' | 'unmatched-kept-per-control' | 'incomplete' | 'tokens';
49
+ /** One configuration against the reference on one measure: an exact sign test over the cases both scored. */
50
+ export interface Comparison {
51
+ readonly configuration: number;
52
+ readonly measure: Measure;
53
+ /** The mean over the pairs of the configuration's case score minus the reference's; null with no pair. */
54
+ readonly difference: number | null;
55
+ readonly better: number;
56
+ readonly worse: number;
57
+ readonly tied: number;
58
+ readonly p: number;
59
+ readonly resolved: boolean;
60
+ }
61
+ /** What one configuration scored on one case, over the trials that count. */
62
+ export interface CaseScore {
63
+ readonly trials: number;
64
+ /** The share of the trials that caught the bug; null for a control. */
65
+ readonly keptCatch: number | null;
66
+ readonly proposedCatch: number | null;
67
+ /** The mean over the trials of the kept findings that catch nothing. */
68
+ readonly unmatchedKept: number;
69
+ /** The share of the trials with a gap. */
70
+ readonly incomplete: number;
71
+ /** The mean over the trials that reported usage; null when none did. */
72
+ readonly tokens: number | null;
73
+ }
74
+ /** The scores of one case, one entry per configuration, null for a configuration that counted none of its trials. */
75
+ export interface CaseScores {
76
+ readonly id: string;
77
+ readonly kind: 'bug' | 'control';
78
+ readonly configurations: readonly (CaseScore | null)[];
79
+ }
80
+ export interface Report {
81
+ readonly suite: string;
82
+ readonly quorum: string;
83
+ readonly trials: number;
84
+ readonly window: number;
85
+ readonly bugCases: number;
86
+ readonly controls: number;
87
+ readonly configurations: readonly ConfigurationSummary[];
88
+ readonly comparisons: readonly Comparison[];
89
+ readonly cases: readonly CaseScores[];
90
+ }
91
+ /** What the report is built from: the suite, the selected cases and trials, the configurations in order and the score of each recorded review by `reviewKey`. */
92
+ export interface ReportInput {
93
+ readonly suite: {
94
+ readonly name: string;
95
+ readonly window: number;
96
+ };
97
+ readonly quorum: string;
98
+ readonly cases: readonly SuiteCase[];
99
+ readonly trials: number;
100
+ readonly configurations: readonly RunConfiguration[];
101
+ readonly scores: ReadonlyMap<string, ReviewScore>;
102
+ }
103
+ /** Score a record against its case: the outcome, the latency, the tokens (input plus output) when there is usage, and the cost when every call reported one. */
104
+ export declare function reviewScore(record: ReviewRecord, suiteCase: SuiteCase, window: number): ReviewScore;
105
+ /** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
106
+ export declare function buildReport(input: ReportInput): Report;
107
+ /** The report as Markdown: its counts, the configurations and their table, and the comparison lines when there are two or more configurations. Every line is inert. */
108
+ export declare function reportMarkdown(report: Report): string;
109
+ /** The report as inert JSON and a line break; it parses back to the report. */
110
+ export declare function reportJson(report: Report): string;