@tobi-or-not/quorum 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +5 -0
  3. package/dist/bench/fingerprint.d.ts +36 -0
  4. package/dist/bench/fingerprint.js +79 -0
  5. package/dist/bench/records.d.ts +62 -0
  6. package/dist/bench/records.js +161 -0
  7. package/dist/bench/report.d.ts +110 -0
  8. package/dist/bench/report.js +332 -0
  9. package/dist/bench/run-dir.d.ts +101 -0
  10. package/dist/bench/run-dir.js +379 -0
  11. package/dist/bench/runner.d.ts +80 -0
  12. package/dist/bench/runner.js +160 -0
  13. package/dist/bench/score.d.ts +30 -0
  14. package/dist/bench/score.js +38 -0
  15. package/dist/bench/stats.d.ts +51 -0
  16. package/dist/bench/stats.js +83 -0
  17. package/dist/bench/suite.d.ts +87 -0
  18. package/dist/bench/suite.js +286 -0
  19. package/dist/cli/bench.d.ts +32 -0
  20. package/dist/cli/bench.js +463 -0
  21. package/dist/cli/bin.js +9 -1
  22. package/dist/cli/io.d.ts +45 -0
  23. package/dist/cli/io.js +45 -0
  24. package/dist/cli/main.d.ts +7 -37
  25. package/dist/cli/main.js +47 -60
  26. package/dist/cli/print-config.js +38 -9
  27. package/dist/cli/render.d.ts +10 -0
  28. package/dist/cli/render.js +37 -16
  29. package/dist/config/claude-code.js +42 -11
  30. package/dist/config/env.js +135 -32
  31. package/dist/config/from-config.js +43 -9
  32. package/dist/config/load.d.ts +43 -0
  33. package/dist/config/load.js +205 -46
  34. package/dist/config/profiles.js +87 -20
  35. package/dist/config/resolve.js +194 -47
  36. package/dist/config/schema.js +98 -23
  37. package/dist/core/diff.d.ts +10 -0
  38. package/dist/core/diff.js +90 -17
  39. package/dist/core/errors.js +14 -5
  40. package/dist/core/flags.js +30 -6
  41. package/dist/core/output.js +49 -13
  42. package/dist/core/policy.js +6 -2
  43. package/dist/core/profile.js +42 -15
  44. package/dist/core/prompts.d.ts +3 -1
  45. package/dist/core/prompts.js +28 -8
  46. package/dist/core/quorum.js +128 -33
  47. package/dist/core/sanitize.d.ts +16 -1
  48. package/dist/core/sanitize.js +31 -5
  49. package/dist/core/stages/context.js +30 -5
  50. package/dist/core/stages/panel.js +55 -9
  51. package/dist/core/stages/research.js +13 -3
  52. package/dist/core/verify.js +6 -2
  53. package/dist/index.js +5 -1
  54. package/dist/providers/anthropic.js +28 -5
  55. package/dist/providers/build.js +57 -16
  56. package/dist/providers/claude-code.js +206 -51
  57. package/dist/providers/gemini.js +61 -14
  58. package/dist/providers/http.js +32 -9
  59. package/dist/providers/messages.js +14 -2
  60. package/dist/providers/ollama.js +31 -8
  61. package/dist/providers/openai-compatible.js +51 -15
  62. package/dist/providers/options.js +120 -40
  63. package/dist/providers/presets.js +143 -16
  64. package/dist/providers/reply.js +15 -5
  65. package/dist/providers/size.js +11 -3
  66. package/package.json +6 -5
package/LICENSE ADDED
@@ -0,0 +1,202 @@
1
+
2
+ Apache License
3
+ Version 2.0, January 2004
4
+ http://www.apache.org/licenses/
5
+
6
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
7
+
8
+ 1. Definitions.
9
+
10
+ "License" shall mean the terms and conditions for use, reproduction,
11
+ and distribution as defined by Sections 1 through 9 of this document.
12
+
13
+ "Licensor" shall mean the copyright owner or entity authorized by
14
+ the copyright owner that is granting the License.
15
+
16
+ "Legal Entity" shall mean the union of the acting entity and all
17
+ other entities that control, are controlled by, or are under common
18
+ control with that entity. For the purposes of this definition,
19
+ "control" means (i) the power, direct or indirect, to cause the
20
+ direction or management of such entity, whether by contract or
21
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
22
+ outstanding shares, or (iii) beneficial ownership of such entity.
23
+
24
+ "You" (or "Your") shall mean an individual or Legal Entity
25
+ exercising permissions granted by this License.
26
+
27
+ "Source" form shall mean the preferred form for making modifications,
28
+ including but not limited to software source code, documentation
29
+ source, and configuration files.
30
+
31
+ "Object" form shall mean any form resulting from mechanical
32
+ transformation or translation of a Source form, including but
33
+ not limited to compiled object code, generated documentation,
34
+ and conversions to other media types.
35
+
36
+ "Work" shall mean the work of authorship, whether in Source or
37
+ Object form, made available under the License, as indicated by a
38
+ copyright notice that is included in or attached to the work
39
+ (an example is provided in the Appendix below).
40
+
41
+ "Derivative Works" shall mean any work, whether in Source or Object
42
+ form, that is based on (or derived from) the Work and for which the
43
+ editorial revisions, annotations, elaborations, or other modifications
44
+ represent, as a whole, an original work of authorship. For the purposes
45
+ of this License, Derivative Works shall not include works that remain
46
+ separable from, or merely link (or bind by name) to the interfaces of,
47
+ the Work and Derivative Works thereof.
48
+
49
+ "Contribution" shall mean any work of authorship, including
50
+ the original version of the Work and any modifications or additions
51
+ to that Work or Derivative Works thereof, that is intentionally
52
+ submitted to Licensor for inclusion in the Work by the copyright owner
53
+ or by an individual or Legal Entity authorized to submit on behalf of
54
+ the copyright owner. For the purposes of this definition, "submitted"
55
+ means any form of electronic, verbal, or written communication sent
56
+ to the Licensor or its representatives, including but not limited to
57
+ communication on electronic mailing lists, source code control systems,
58
+ and issue tracking systems that are managed by, or on behalf of, the
59
+ Licensor for the purpose of discussing and improving the Work, but
60
+ excluding communication that is conspicuously marked or otherwise
61
+ designated in writing by the copyright owner as "Not a Contribution."
62
+
63
+ "Contributor" shall mean Licensor and any individual or Legal Entity
64
+ on behalf of whom a Contribution has been received by Licensor and
65
+ subsequently incorporated within the Work.
66
+
67
+ 2. Grant of Copyright License. Subject to the terms and conditions of
68
+ this License, each Contributor hereby grants to You a perpetual,
69
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
70
+ copyright license to reproduce, prepare Derivative Works of,
71
+ publicly display, publicly perform, sublicense, and distribute the
72
+ Work and such Derivative Works in Source or Object form.
73
+
74
+ 3. Grant of Patent License. Subject to the terms and conditions of
75
+ this License, each Contributor hereby grants to You a perpetual,
76
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
77
+ (except as stated in this section) patent license to make, have made,
78
+ use, offer to sell, sell, import, and otherwise transfer the Work,
79
+ where such license applies only to those patent claims licensable
80
+ by such Contributor that are necessarily infringed by their
81
+ Contribution(s) alone or by combination of their Contribution(s)
82
+ with the Work to which such Contribution(s) was submitted. If You
83
+ institute patent litigation against any entity (including a
84
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
85
+ or a Contribution incorporated within the Work constitutes direct
86
+ or contributory patent infringement, then any patent licenses
87
+ granted to You under this License for that Work shall terminate
88
+ as of the date such litigation is filed.
89
+
90
+ 4. Redistribution. You may reproduce and distribute copies of the
91
+ Work or Derivative Works thereof in any medium, with or without
92
+ modifications, and in Source or Object form, provided that You
93
+ meet the following conditions:
94
+
95
+ (a) You must give any other recipients of the Work or
96
+ Derivative Works a copy of this License; and
97
+
98
+ (b) You must cause any modified files to carry prominent notices
99
+ stating that You changed the files; and
100
+
101
+ (c) You must retain, in the Source form of any Derivative Works
102
+ that You distribute, all copyright, patent, trademark, and
103
+ attribution notices from the Source form of the Work,
104
+ excluding those notices that do not pertain to any part of
105
+ the Derivative Works; and
106
+
107
+ (d) If the Work includes a "NOTICE" text file as part of its
108
+ distribution, then any Derivative Works that You distribute must
109
+ include a readable copy of the attribution notices contained
110
+ within such NOTICE file, excluding those notices that do not
111
+ pertain to any part of the Derivative Works, in at least one
112
+ of the following places: within a NOTICE text file distributed
113
+ as part of the Derivative Works; within the Source form or
114
+ documentation, if provided along with the Derivative Works; or,
115
+ within a display generated by the Derivative Works, if and
116
+ wherever such third-party notices normally appear. The contents
117
+ of the NOTICE file are for informational purposes only and
118
+ do not modify the License. You may add Your own attribution
119
+ notices within Derivative Works that You distribute, alongside
120
+ or as an addendum to the NOTICE text from the Work, provided
121
+ that such additional attribution notices cannot be construed
122
+ as modifying the License.
123
+
124
+ You may add Your own copyright statement to Your modifications and
125
+ may provide additional or different license terms and conditions
126
+ for use, reproduction, or distribution of Your modifications, or
127
+ for any such Derivative Works as a whole, provided Your use,
128
+ reproduction, and distribution of the Work otherwise complies with
129
+ the conditions stated in this License.
130
+
131
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
132
+ any Contribution intentionally submitted for inclusion in the Work
133
+ by You to the Licensor shall be under the terms and conditions of
134
+ this License, without any additional terms or conditions.
135
+ Notwithstanding the above, nothing herein shall supersede or modify
136
+ the terms of any separate license agreement you may have executed
137
+ with Licensor regarding such Contributions.
138
+
139
+ 6. Trademarks. This License does not grant permission to use the trade
140
+ names, trademarks, service marks, or product names of the Licensor,
141
+ except as required for reasonable and customary use in describing the
142
+ origin of the Work and reproducing the content of the NOTICE file.
143
+
144
+ 7. Disclaimer of Warranty. Unless required by applicable law or
145
+ agreed to in writing, Licensor provides the Work (and each
146
+ Contributor provides its Contributions) on an "AS IS" BASIS,
147
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
148
+ implied, including, without limitation, any warranties or conditions
149
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
150
+ PARTICULAR PURPOSE. You are solely responsible for determining the
151
+ appropriateness of using or redistributing the Work and assume any
152
+ risks associated with Your exercise of permissions under this License.
153
+
154
+ 8. Limitation of Liability. In no event and under no legal theory,
155
+ whether in tort (including negligence), contract, or otherwise,
156
+ unless required by applicable law (such as deliberate and grossly
157
+ negligent acts) or agreed to in writing, shall any Contributor be
158
+ liable to You for damages, including any direct, indirect, special,
159
+ incidental, or consequential damages of any character arising as a
160
+ result of this License or out of the use or inability to use the
161
+ Work (including but not limited to damages for loss of goodwill,
162
+ work stoppage, computer failure or malfunction, or any and all
163
+ other commercial damages or losses), even if such Contributor
164
+ has been advised of the possibility of such damages.
165
+
166
+ 9. Accepting Warranty or Additional Liability. While redistributing
167
+ the Work or Derivative Works thereof, You may choose to offer,
168
+ and charge a fee for, acceptance of support, warranty, indemnity,
169
+ or other liability obligations and/or rights consistent with this
170
+ License. However, in accepting such obligations, You may act only
171
+ on Your own behalf and on Your sole responsibility, not on behalf
172
+ of any other Contributor, and only if You agree to indemnify,
173
+ defend, and hold each Contributor harmless for any liability
174
+ incurred by, or claims asserted against, such Contributor by reason
175
+ of your accepting any such warranty or additional liability.
176
+
177
+ END OF TERMS AND CONDITIONS
178
+
179
+ APPENDIX: How to apply the Apache License to your work.
180
+
181
+ To apply the Apache License to your work, attach the following
182
+ boilerplate notice, with the fields enclosed by brackets "[]"
183
+ replaced with your own identifying information. (Don't include
184
+ the brackets!) The text should be enclosed in the appropriate
185
+ comment syntax for the file format. We also recommend that a
186
+ file or class name and description of purpose be included on the
187
+ same "printed page" as the copyright notice for easier
188
+ identification within third-party archives.
189
+
190
+ Copyright [yyyy] [name of copyright owner]
191
+
192
+ Licensed under the Apache License, Version 2.0 (the "License");
193
+ you may not use this file except in compliance with the License.
194
+ You may obtain a copy of the License at
195
+
196
+ http://www.apache.org/licenses/LICENSE-2.0
197
+
198
+ Unless required by applicable law or agreed to in writing, software
199
+ distributed under the License is distributed on an "AS IS" BASIS,
200
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
201
+ See the License for the specific language governing permissions and
202
+ limitations under the License.
package/README.md CHANGED
@@ -77,6 +77,7 @@ The JSON result goes to stdout, and the exit code says what to do with it: `0` f
77
77
  - [Profiles](./docs/20-using/40-profiles.md) *- The security profile, writing your own, several in one review, gating and guidelines*
78
78
  - [Lenses and policy](./docs/20-using/50-lenses-and-policy.md) *- The panel's questions, the rules that turn kept votes into a confidence, failed votes, and what a level means*
79
79
  - [Prompts](./docs/20-using/60-prompts.md) *- The layout of every request, the templates and their variables, the notices, and what is not configurable*
80
+ - [Benchmarks](./docs/20-using/70-bench.md) *- quorum bench: measure configurations on a suite of real bugs, write a suite, resume a run, and read the report and its intervals*
80
81
 
81
82
  *Security*
82
83
  - [Threat model](./docs/30-security/10-threat-model.md) *- The attacker, the assets, the residual risk stated plainly, the controls in place, and what the panel's independence is worth*
@@ -90,3 +91,7 @@ The JSON result goes to stdout, and the exit code says what to do with it: `0` f
90
91
  - [The JSON result](./docs/95-reference/20-json-result.md) *- Every field of a result in the order it is written, the incomplete reasons, and an example*
91
92
  - [Library API](./docs/95-reference/30-library-api.md) *- `createQuorum`, its ports, the events, abort, the errors and every export*
92
93
  - [Architecture](./docs/95-reference/40-architecture.md) *- The modules, the dependency rule, the stages of a review, and how the suite runs*
94
+
95
+ ## License
96
+
97
+ quorum is released under the [Apache License 2.0](./LICENSE).
@@ -0,0 +1,36 @@
1
+ /**
2
+ * The fingerprint of a configuration: the HMAC-SHA-256, under a salt the run keeps, of a canonical form
3
+ * of the resolved settings that names no provider. The bench keys its records by it, and refuses two
4
+ * configurations that resolve alike.
5
+ *
6
+ * The form holds every setting a review uses but the key: the profiles that run, in run order, each with
7
+ * its id, prompts, lenses, policy and gating, and its research provider and each lens's provider written
8
+ * out in place of their names; every field of a provider defined as data but its key, the values of
9
+ * `headers`, `query` and `extraBody` and the whole base URL included; a provider object as its config key
10
+ * and its model; `concurrency` and `maxFindings` as the review applies them, defaults filled in. It holds
11
+ * no file path, no warning and no provider name, so the same settings under other names are one
12
+ * configuration, and a rotated key keeps a run.
13
+ *
14
+ * The form can hold a secret (a credential kept in a header, a query parameter or the base path), so it
15
+ * never leaves this module: not written, not logged, not put in an error message. Only the HMAC does.
16
+ *
17
+ * What it cannot see: a provider object's code (a changed implementation under the same key and model
18
+ * needs a new run directory), and the environment a `claude-code` provider inherits.
19
+ */
20
+ import type { ResolvedConfig } from '../config/resolve.ts';
21
+ /** A fresh salt: 32 random bytes as 64 lowercase hex digits. */
22
+ export declare function newSalt(): string;
23
+ /**
24
+ * The canonical text of a configuration's settings: JSON with the keys of every object sorted, arrays in
25
+ * their order. It can hold a secret: hash it with `fingerprint`, and never write, log or print it.
26
+ *
27
+ * @throws {RangeError} when a profile names a provider the config does not hold.
28
+ */
29
+ export declare function canonicalSettings(config: ResolvedConfig): string;
30
+ /**
31
+ * The fingerprint of a configuration under a run's salt: the HMAC-SHA-256 of its canonical settings, as 64
32
+ * lowercase hex digits.
33
+ *
34
+ * @throws {RangeError} when `salt` is not 64 lowercase hex digits (the message never holds it), or a profile names a provider the config does not hold.
35
+ */
36
+ export declare function fingerprint(config: ResolvedConfig, salt: string): string;
@@ -0,0 +1,79 @@
1
+ /**
2
+ * The fingerprint of a configuration: the HMAC-SHA-256, under a salt the run keeps, of a canonical form
3
+ * of the resolved settings that names no provider. The bench keys its records by it, and refuses two
4
+ * configurations that resolve alike.
5
+ *
6
+ * The form holds every setting a review uses but the key: the profiles that run, in run order, each with
7
+ * its id, prompts, lenses, policy and gating, and its research provider and each lens's provider written
8
+ * out in place of their names; every field of a provider defined as data but its key, the values of
9
+ * `headers`, `query` and `extraBody` and the whole base URL included; a provider object as its config key
10
+ * and its model; `concurrency` and `maxFindings` as the review applies them, defaults filled in. It holds
11
+ * no file path, no warning and no provider name, so the same settings under other names are one
12
+ * configuration, and a rotated key keeps a run.
13
+ *
14
+ * The form can hold a secret (a credential kept in a header, a query parameter or the base path), so it
15
+ * never leaves this module: not written, not logged, not put in an error message. Only the HMAC does.
16
+ *
17
+ * What it cannot see: a provider object's code (a changed implementation under the same key and model
18
+ * needs a new run directory), and the environment a `claude-code` provider inherits.
19
+ */
20
+ import { createHmac, randomBytes } from 'node:crypto';
21
+ import { isPlainObject } from "../config/load.js";
22
+ import { isProvider } from "../config/schema.js";
23
+ import { DEFAULT_CONCURRENCY } from "../core/concurrency.js";
24
+ import { DEFAULT_MAX_FINDINGS } from "../core/stages/research.js";
25
+ const SALT = /^[0-9a-f]{64}$/;
26
+ /** A fresh salt: 32 random bytes as 64 lowercase hex digits. */
27
+ export function newSalt() {
28
+ return randomBytes(32).toString('hex');
29
+ }
30
+ /** A provider as the form holds it: a provider object by its config key and model, a spec by its fields less the key. */
31
+ function providerForm(provider) {
32
+ if (isProvider(provider))
33
+ return { object: provider.id, model: provider.model };
34
+ return Object.fromEntries(Object.entries(provider).filter(([field]) => field !== 'apiKey'));
35
+ }
36
+ /** The form of the provider a profile binds by `name`; a name the config holds no provider under is a RangeError, never a silent gap. */
37
+ function boundForm(config, name) {
38
+ if (!Object.hasOwn(config.providers, name))
39
+ throw new RangeError('a profile names a provider the config does not hold');
40
+ return providerForm(config.providers[name]);
41
+ }
42
+ /** A plain object with its keys in code-unit order, so the text does not depend on the order the config wrote. */
43
+ function sortKeys(_key, value) {
44
+ return isPlainObject(value)
45
+ ? Object.fromEntries(Object.keys(value).sort().map(key => [key, value[key]]))
46
+ : value;
47
+ }
48
+ /**
49
+ * The canonical text of a configuration's settings: JSON with the keys of every object sorted, arrays in
50
+ * their order. It can hold a secret: hash it with `fingerprint`, and never write, log or print it.
51
+ *
52
+ * @throws {RangeError} when a profile names a provider the config does not hold.
53
+ */
54
+ export function canonicalSettings(config) {
55
+ const profiles = config.profiles.map(({ profile, research, voters }) => ({
56
+ profile,
57
+ research: boundForm(config, research),
58
+ voters: Object.fromEntries(Object.entries(voters).map(([lens, name]) => [
59
+ lens,
60
+ boundForm(config, name),
61
+ ])),
62
+ }));
63
+ return JSON.stringify({
64
+ concurrency: config.concurrency ?? DEFAULT_CONCURRENCY,
65
+ maxFindings: config.maxFindings ?? DEFAULT_MAX_FINDINGS,
66
+ profiles,
67
+ }, sortKeys);
68
+ }
69
+ /**
70
+ * The fingerprint of a configuration under a run's salt: the HMAC-SHA-256 of its canonical settings, as 64
71
+ * lowercase hex digits.
72
+ *
73
+ * @throws {RangeError} when `salt` is not 64 lowercase hex digits (the message never holds it), or a profile names a provider the config does not hold.
74
+ */
75
+ export function fingerprint(config, salt) {
76
+ if (!SALT.test(salt))
77
+ throw new RangeError('the salt must be 64 lowercase hex digits');
78
+ return createHmac('sha256', Buffer.from(salt, 'hex')).update(canonicalSettings(config)).digest('hex');
79
+ }
@@ -0,0 +1,62 @@
1
+ /**
2
+ * The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
3
+ * (the quorum version, the suite's name and hash, the salt, and the fingerprints of its configurations),
4
+ * and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
5
+ * person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
6
+ * and anything the schema does not describe is refused, never repaired. A repeated key is not looked
7
+ * for: the tree that finds one costs ten times the parse on a line of 16 MiB.
8
+ *
9
+ * A record keeps the facts of a review, not its scores: the report scores every record with the suite's
10
+ * locations when it is built, so a score never goes stale.
11
+ */
12
+ import type { QuorumResult, Usage } from '../core/types.ts';
13
+ /** The `format` of the run file this quorum writes and reads. */
14
+ export declare const RUN_FORMAT = 1;
15
+ /** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
16
+ export declare const MAX_RECORD_BYTES: number;
17
+ /** The most `run.json` may hold, in bytes. */
18
+ export declare const MAX_RUN_FILE_BYTES: number;
19
+ /** The most configurations a run holds. */
20
+ export declare const MAX_CONFIGURATIONS = 16;
21
+ /** One configuration of a run: the path as given (null for the usual resolution) and its fingerprint. */
22
+ export interface RunConfiguration {
23
+ readonly path: string | null;
24
+ readonly fingerprint: string;
25
+ }
26
+ /** What `run.json` holds. */
27
+ export interface RunFile {
28
+ readonly format: 1;
29
+ /** The version of the quorum that began the run. */
30
+ readonly quorum: string;
31
+ readonly suite: {
32
+ readonly name: string;
33
+ readonly hash: string;
34
+ };
35
+ /** The key the fingerprints are keyed with: 32 random bytes as 64 lowercase hex digits. */
36
+ readonly salt: string;
37
+ readonly configurations: readonly RunConfiguration[];
38
+ }
39
+ /** What one line of `results.jsonl` holds: a review that resolved, as the bench saw it. */
40
+ export interface ReviewRecord {
41
+ readonly fingerprint: string;
42
+ /** The id of the suite's case. */
43
+ readonly case: string;
44
+ readonly trial: number;
45
+ readonly latencyMs: number;
46
+ /** The provider calls the review made, failed ones included. */
47
+ readonly calls: number;
48
+ readonly failedCalls: number;
49
+ /** The usage of the calls that reported one; absent when none did. */
50
+ readonly usage?: Usage;
51
+ /** Whether every call reported a cost. */
52
+ readonly costComplete: boolean;
53
+ readonly result: QuorumResult;
54
+ }
55
+ /** The key of a planned review: the fingerprint, the case and the trial, as a JSON array. */
56
+ export declare function reviewKey(fingerprint: string, caseId: string, trial: number): string;
57
+ /** One line of `results.jsonl`: the record as inert JSON, on one line, and a line break. */
58
+ export declare function recordLine(record: ReviewRecord): string;
59
+ /** The record a line of `results.jsonl` holds (without its line break), or undefined when the line is no record of this format. */
60
+ export declare function parseRecord(line: string): ReviewRecord | undefined;
61
+ /** The run file `text` holds, or undefined when it is no run file of this format. */
62
+ export declare function parseRunFile(text: string): RunFile | undefined;
@@ -0,0 +1,161 @@
1
+ /**
2
+ * The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
3
+ * (the quorum version, the suite's name and hash, the salt, and the fingerprints of its configurations),
4
+ * and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
5
+ * person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
6
+ * and anything the schema does not describe is refused, never repaired. A repeated key is not looked
7
+ * for: the tree that finds one costs ten times the parse on a line of 16 MiB.
8
+ *
9
+ * A record keeps the facts of a review, not its scores: the report scores every record with the suite's
10
+ * locations when it is built, so a score never goes stale.
11
+ */
12
+ import { z } from 'zod';
13
+ import { inertJsonLine } from "../core/sanitize.js";
14
+ import { MAX_TRIALS, SUITE_ID } from "./suite.js";
15
+ /** The `format` of the run file this quorum writes and reads. */
16
+ export const RUN_FORMAT = 1;
17
+ /** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
18
+ export const MAX_RECORD_BYTES = 16 * 1024 * 1024;
19
+ /** The most `run.json` may hold, in bytes. */
20
+ export const MAX_RUN_FILE_BYTES = 1024 * 1024;
21
+ /** The most configurations a run holds. */
22
+ export const MAX_CONFIGURATIONS = 16;
23
+ const MAX_ID = 64;
24
+ const MAX_QUORUM_VERSION = 64;
25
+ const MAX_CONFIG_PATH = 4096;
26
+ const digest = z.string().regex(/^[0-9a-f]{64}$/);
27
+ const identifier = z.string().max(MAX_ID).regex(SUITE_ID);
28
+ const count = z.int().min(0);
29
+ const amount = z.number().min(0);
30
+ const confidence = z.enum(['high', 'medium', 'low']);
31
+ const vote = z.strictObject({
32
+ keep: z.boolean(),
33
+ reasoning: z.string(),
34
+ });
35
+ const lensVote = z.strictObject({
36
+ angle: z.string(),
37
+ vote: vote.nullable(),
38
+ error: z.string().nullable(),
39
+ });
40
+ const verdict = z.strictObject({
41
+ id: z.string(),
42
+ profile: z.string(),
43
+ file: z.string(),
44
+ line: count,
45
+ category: z.string(),
46
+ summary: z.string(),
47
+ evidence: z.string(),
48
+ quoteMatched: z.boolean(),
49
+ keptVotes: count,
50
+ totalVotes: count,
51
+ confidence: confidence.optional(),
52
+ votes: z.array(lensVote),
53
+ });
54
+ const providerRef = z.strictObject({
55
+ id: z.string(),
56
+ model: z.string(),
57
+ });
58
+ const thresholds = z.strictObject({
59
+ high: count.nullable(),
60
+ medium: count.nullable(),
61
+ low: count.nullable(),
62
+ });
63
+ const usage = z.strictObject({
64
+ inputTokens: amount,
65
+ outputTokens: amount,
66
+ costUsd: amount.optional(),
67
+ });
68
+ const profileMeta = z.strictObject({
69
+ providers: z.strictObject({
70
+ research: providerRef,
71
+ lenses: z.record(z.string(), providerRef),
72
+ }),
73
+ policy: thresholds,
74
+ gating: z.boolean(),
75
+ invalidFindings: count,
76
+ });
77
+ const incompleteEntry = z.strictObject({
78
+ profile: z.string(),
79
+ reason: z.enum([
80
+ 'research-unusable',
81
+ 'findings-capped',
82
+ 'finding-unreadable',
83
+ 'vote-failed',
84
+ 'request-too-large',
85
+ ]),
86
+ file: z.string().optional(),
87
+ detail: z.string(),
88
+ });
89
+ const flag = z.strictObject({
90
+ code: z.enum(['hidden-characters', 'ai-addressed-text']),
91
+ file: z.string().optional(),
92
+ line: count.optional(),
93
+ detail: z.string(),
94
+ });
95
+ /** Declared as `QuorumResult`'s own type, so a required field the core adds and this schema lacks fails the typecheck. */
96
+ const resultSchema = z.strictObject({
97
+ kept: z.array(verdict),
98
+ discarded: z.array(verdict),
99
+ unreviewed: z.array(verdict),
100
+ meta: z.strictObject({
101
+ version: z.string(),
102
+ startedAt: z.string(),
103
+ durationMs: count,
104
+ profiles: z.record(z.string(), profileMeta),
105
+ usage: usage.optional(),
106
+ incomplete: z.array(incompleteEntry),
107
+ flags: z.array(flag),
108
+ }),
109
+ });
110
+ const recordSchema = z.strictObject({
111
+ fingerprint: digest,
112
+ case: identifier,
113
+ trial: z.int().min(1).max(MAX_TRIALS),
114
+ latencyMs: count,
115
+ calls: count,
116
+ failedCalls: count,
117
+ usage: usage.optional(),
118
+ costComplete: z.boolean(),
119
+ result: resultSchema,
120
+ }).refine(({ calls, failedCalls }) => failedCalls <= calls);
121
+ const runFileSchema = z.strictObject({
122
+ format: z.literal(RUN_FORMAT),
123
+ quorum: z.string().min(1).max(MAX_QUORUM_VERSION),
124
+ suite: z.strictObject({
125
+ name: identifier,
126
+ hash: z.string().regex(/^sha256:[0-9a-f]{64}$/),
127
+ }),
128
+ salt: digest,
129
+ configurations: z.array(z.strictObject({
130
+ path: z.string().min(1).max(MAX_CONFIG_PATH).nullable(),
131
+ fingerprint: digest,
132
+ })).min(1).max(MAX_CONFIGURATIONS),
133
+ });
134
+ /** `text` as JSON checked by `schema`; any failure, of the syntax or of the schema, is undefined. */
135
+ function parseWith(schema, text) {
136
+ try {
137
+ const parsed = schema.safeParse(JSON.parse(text));
138
+ return parsed.success
139
+ ? parsed.data
140
+ : undefined;
141
+ }
142
+ catch {
143
+ return undefined;
144
+ }
145
+ }
146
+ /** The key of a planned review: the fingerprint, the case and the trial, as a JSON array. */
147
+ export function reviewKey(fingerprint, caseId, trial) {
148
+ return JSON.stringify([fingerprint, caseId, trial]);
149
+ }
150
+ /** One line of `results.jsonl`: the record as inert JSON, on one line, and a line break. */
151
+ export function recordLine(record) {
152
+ return `${inertJsonLine(record)}\n`;
153
+ }
154
+ /** The record a line of `results.jsonl` holds (without its line break), or undefined when the line is no record of this format. */
155
+ export function parseRecord(line) {
156
+ return parseWith(recordSchema, line);
157
+ }
158
+ /** The run file `text` holds, or undefined when it is no run file of this format. */
159
+ export function parseRunFile(text) {
160
+ return parseWith(runFileSchema, text);
161
+ }
@@ -0,0 +1,110 @@
1
+ /**
2
+ * The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
3
+ * built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
4
+ * report can be built again from a run directory. It holds numbers, suite ids and configuration paths,
5
+ * never text a model wrote.
6
+ */
7
+ import type { ReviewRecord, RunConfiguration } from './records.ts';
8
+ import type { ReviewOutcome } from './score.ts';
9
+ import type { SuiteCase } from './suite.ts';
10
+ /** What one review earned, with what it took: its latency, its tokens when it reported usage, and its cost when every call reported one. */
11
+ export interface ReviewScore extends ReviewOutcome {
12
+ readonly latencyMs: number;
13
+ readonly tokens?: number;
14
+ readonly costUsd?: number;
15
+ }
16
+ /** A rate, the mean of the case scores, with its Wilson interval over `cases` cases. */
17
+ export interface Rate {
18
+ readonly value: number;
19
+ readonly low: number;
20
+ readonly high: number;
21
+ readonly cases: number;
22
+ }
23
+ /** What one configuration did over the reviews that count. */
24
+ export interface ConfigurationSummary {
25
+ /** The position of the configuration, from 1; the first is the reference. */
26
+ readonly index: number;
27
+ readonly path: string | null;
28
+ readonly reviews: number;
29
+ readonly planned: number;
30
+ readonly incomplete: number;
31
+ readonly keptRecall: Rate | null;
32
+ readonly proposedRecall: Rate | null;
33
+ readonly unmatchedKept: {
34
+ readonly bug: number | null;
35
+ readonly control: number | null;
36
+ };
37
+ readonly tokens: {
38
+ readonly median: number;
39
+ readonly total: number;
40
+ } | null;
41
+ readonly latencyMs: {
42
+ readonly p50: number;
43
+ readonly p95: number;
44
+ } | null;
45
+ readonly costUsd: number | null;
46
+ }
47
+ /** What a comparison with the reference measures. */
48
+ export type Measure = 'kept-recall' | 'proposed-recall' | 'unmatched-kept-per-control' | 'incomplete' | 'tokens';
49
+ /** One configuration against the reference on one measure: an exact sign test over the cases both scored. */
50
+ export interface Comparison {
51
+ readonly configuration: number;
52
+ readonly measure: Measure;
53
+ /** The mean over the pairs of the configuration's case score minus the reference's; null with no pair. */
54
+ readonly difference: number | null;
55
+ readonly better: number;
56
+ readonly worse: number;
57
+ readonly tied: number;
58
+ readonly p: number;
59
+ readonly resolved: boolean;
60
+ }
61
+ /** What one configuration scored on one case, over the trials that count. */
62
+ export interface CaseScore {
63
+ readonly trials: number;
64
+ /** The share of the trials that caught the bug; null for a control. */
65
+ readonly keptCatch: number | null;
66
+ readonly proposedCatch: number | null;
67
+ /** The mean over the trials of the kept findings that catch nothing. */
68
+ readonly unmatchedKept: number;
69
+ /** The share of the trials with a gap. */
70
+ readonly incomplete: number;
71
+ /** The mean over the trials that reported usage; null when none did. */
72
+ readonly tokens: number | null;
73
+ }
74
+ /** The scores of one case, one entry per configuration, null for a configuration that counted none of its trials. */
75
+ export interface CaseScores {
76
+ readonly id: string;
77
+ readonly kind: 'bug' | 'control';
78
+ readonly configurations: readonly (CaseScore | null)[];
79
+ }
80
+ export interface Report {
81
+ readonly suite: string;
82
+ readonly quorum: string;
83
+ readonly trials: number;
84
+ readonly window: number;
85
+ readonly bugCases: number;
86
+ readonly controls: number;
87
+ readonly configurations: readonly ConfigurationSummary[];
88
+ readonly comparisons: readonly Comparison[];
89
+ readonly cases: readonly CaseScores[];
90
+ }
91
+ /** What the report is built from: the suite, the selected cases and trials, the configurations in order and the score of each recorded review by `reviewKey`. */
92
+ export interface ReportInput {
93
+ readonly suite: {
94
+ readonly name: string;
95
+ readonly window: number;
96
+ };
97
+ readonly quorum: string;
98
+ readonly cases: readonly SuiteCase[];
99
+ readonly trials: number;
100
+ readonly configurations: readonly RunConfiguration[];
101
+ readonly scores: ReadonlyMap<string, ReviewScore>;
102
+ }
103
+ /** Score a record against its case: the outcome, the latency, the tokens (input plus output) when there is usage, and the cost when every call reported one. */
104
+ export declare function reviewScore(record: ReviewRecord, suiteCase: SuiteCase, window: number): ReviewScore;
105
+ /** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
106
+ export declare function buildReport(input: ReportInput): Report;
107
+ /** The report as Markdown: its counts, the configurations and their table, and the comparison lines when there are two or more configurations. Every line is inert. */
108
+ export declare function reportMarkdown(report: Report): string;
109
+ /** The report as inert JSON and a line break; it parses back to the report. */
110
+ export declare function reportJson(report: Report): string;