@tobi-or-not/quorum 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +5 -0
- package/dist/bench/fingerprint.d.ts +36 -0
- package/dist/bench/fingerprint.js +79 -0
- package/dist/bench/records.d.ts +62 -0
- package/dist/bench/records.js +161 -0
- package/dist/bench/report.d.ts +110 -0
- package/dist/bench/report.js +332 -0
- package/dist/bench/run-dir.d.ts +101 -0
- package/dist/bench/run-dir.js +379 -0
- package/dist/bench/runner.d.ts +80 -0
- package/dist/bench/runner.js +160 -0
- package/dist/bench/score.d.ts +30 -0
- package/dist/bench/score.js +38 -0
- package/dist/bench/stats.d.ts +51 -0
- package/dist/bench/stats.js +83 -0
- package/dist/bench/suite.d.ts +87 -0
- package/dist/bench/suite.js +286 -0
- package/dist/cli/bench.d.ts +32 -0
- package/dist/cli/bench.js +463 -0
- package/dist/cli/bin.js +9 -1
- package/dist/cli/io.d.ts +45 -0
- package/dist/cli/io.js +45 -0
- package/dist/cli/main.d.ts +7 -37
- package/dist/cli/main.js +47 -60
- package/dist/cli/print-config.js +38 -9
- package/dist/cli/render.d.ts +10 -0
- package/dist/cli/render.js +37 -16
- package/dist/config/claude-code.js +42 -11
- package/dist/config/env.js +135 -32
- package/dist/config/from-config.js +43 -9
- package/dist/config/load.d.ts +43 -0
- package/dist/config/load.js +205 -46
- package/dist/config/profiles.js +87 -20
- package/dist/config/resolve.js +194 -47
- package/dist/config/schema.js +98 -23
- package/dist/core/diff.d.ts +10 -0
- package/dist/core/diff.js +90 -17
- package/dist/core/errors.js +14 -5
- package/dist/core/flags.js +30 -6
- package/dist/core/output.js +49 -13
- package/dist/core/policy.js +6 -2
- package/dist/core/profile.js +42 -15
- package/dist/core/prompts.d.ts +3 -1
- package/dist/core/prompts.js +28 -8
- package/dist/core/quorum.js +128 -33
- package/dist/core/sanitize.d.ts +16 -1
- package/dist/core/sanitize.js +31 -5
- package/dist/core/stages/context.js +30 -5
- package/dist/core/stages/panel.js +55 -9
- package/dist/core/stages/research.js +13 -3
- package/dist/core/verify.js +6 -2
- package/dist/index.js +5 -1
- package/dist/providers/anthropic.js +28 -5
- package/dist/providers/build.js +57 -16
- package/dist/providers/claude-code.js +206 -51
- package/dist/providers/gemini.js +61 -14
- package/dist/providers/http.js +32 -9
- package/dist/providers/messages.js +14 -2
- package/dist/providers/ollama.js +31 -8
- package/dist/providers/openai-compatible.js +51 -15
- package/dist/providers/options.js +120 -40
- package/dist/providers/presets.js +143 -16
- package/dist/providers/reply.js +15 -5
- package/dist/providers/size.js +11 -3
- package/package.json +6 -5
package/LICENSE
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
|
|
2
|
+
Apache License
|
|
3
|
+
Version 2.0, January 2004
|
|
4
|
+
http://www.apache.org/licenses/
|
|
5
|
+
|
|
6
|
+
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
|
7
|
+
|
|
8
|
+
1. Definitions.
|
|
9
|
+
|
|
10
|
+
"License" shall mean the terms and conditions for use, reproduction,
|
|
11
|
+
and distribution as defined by Sections 1 through 9 of this document.
|
|
12
|
+
|
|
13
|
+
"Licensor" shall mean the copyright owner or entity authorized by
|
|
14
|
+
the copyright owner that is granting the License.
|
|
15
|
+
|
|
16
|
+
"Legal Entity" shall mean the union of the acting entity and all
|
|
17
|
+
other entities that control, are controlled by, or are under common
|
|
18
|
+
control with that entity. For the purposes of this definition,
|
|
19
|
+
"control" means (i) the power, direct or indirect, to cause the
|
|
20
|
+
direction or management of such entity, whether by contract or
|
|
21
|
+
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
|
22
|
+
outstanding shares, or (iii) beneficial ownership of such entity.
|
|
23
|
+
|
|
24
|
+
"You" (or "Your") shall mean an individual or Legal Entity
|
|
25
|
+
exercising permissions granted by this License.
|
|
26
|
+
|
|
27
|
+
"Source" form shall mean the preferred form for making modifications,
|
|
28
|
+
including but not limited to software source code, documentation
|
|
29
|
+
source, and configuration files.
|
|
30
|
+
|
|
31
|
+
"Object" form shall mean any form resulting from mechanical
|
|
32
|
+
transformation or translation of a Source form, including but
|
|
33
|
+
not limited to compiled object code, generated documentation,
|
|
34
|
+
and conversions to other media types.
|
|
35
|
+
|
|
36
|
+
"Work" shall mean the work of authorship, whether in Source or
|
|
37
|
+
Object form, made available under the License, as indicated by a
|
|
38
|
+
copyright notice that is included in or attached to the work
|
|
39
|
+
(an example is provided in the Appendix below).
|
|
40
|
+
|
|
41
|
+
"Derivative Works" shall mean any work, whether in Source or Object
|
|
42
|
+
form, that is based on (or derived from) the Work and for which the
|
|
43
|
+
editorial revisions, annotations, elaborations, or other modifications
|
|
44
|
+
represent, as a whole, an original work of authorship. For the purposes
|
|
45
|
+
of this License, Derivative Works shall not include works that remain
|
|
46
|
+
separable from, or merely link (or bind by name) to the interfaces of,
|
|
47
|
+
the Work and Derivative Works thereof.
|
|
48
|
+
|
|
49
|
+
"Contribution" shall mean any work of authorship, including
|
|
50
|
+
the original version of the Work and any modifications or additions
|
|
51
|
+
to that Work or Derivative Works thereof, that is intentionally
|
|
52
|
+
submitted to Licensor for inclusion in the Work by the copyright owner
|
|
53
|
+
or by an individual or Legal Entity authorized to submit on behalf of
|
|
54
|
+
the copyright owner. For the purposes of this definition, "submitted"
|
|
55
|
+
means any form of electronic, verbal, or written communication sent
|
|
56
|
+
to the Licensor or its representatives, including but not limited to
|
|
57
|
+
communication on electronic mailing lists, source code control systems,
|
|
58
|
+
and issue tracking systems that are managed by, or on behalf of, the
|
|
59
|
+
Licensor for the purpose of discussing and improving the Work, but
|
|
60
|
+
excluding communication that is conspicuously marked or otherwise
|
|
61
|
+
designated in writing by the copyright owner as "Not a Contribution."
|
|
62
|
+
|
|
63
|
+
"Contributor" shall mean Licensor and any individual or Legal Entity
|
|
64
|
+
on behalf of whom a Contribution has been received by Licensor and
|
|
65
|
+
subsequently incorporated within the Work.
|
|
66
|
+
|
|
67
|
+
2. Grant of Copyright License. Subject to the terms and conditions of
|
|
68
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
69
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
70
|
+
copyright license to reproduce, prepare Derivative Works of,
|
|
71
|
+
publicly display, publicly perform, sublicense, and distribute the
|
|
72
|
+
Work and such Derivative Works in Source or Object form.
|
|
73
|
+
|
|
74
|
+
3. Grant of Patent License. Subject to the terms and conditions of
|
|
75
|
+
this License, each Contributor hereby grants to You a perpetual,
|
|
76
|
+
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
|
77
|
+
(except as stated in this section) patent license to make, have made,
|
|
78
|
+
use, offer to sell, sell, import, and otherwise transfer the Work,
|
|
79
|
+
where such license applies only to those patent claims licensable
|
|
80
|
+
by such Contributor that are necessarily infringed by their
|
|
81
|
+
Contribution(s) alone or by combination of their Contribution(s)
|
|
82
|
+
with the Work to which such Contribution(s) was submitted. If You
|
|
83
|
+
institute patent litigation against any entity (including a
|
|
84
|
+
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
|
85
|
+
or a Contribution incorporated within the Work constitutes direct
|
|
86
|
+
or contributory patent infringement, then any patent licenses
|
|
87
|
+
granted to You under this License for that Work shall terminate
|
|
88
|
+
as of the date such litigation is filed.
|
|
89
|
+
|
|
90
|
+
4. Redistribution. You may reproduce and distribute copies of the
|
|
91
|
+
Work or Derivative Works thereof in any medium, with or without
|
|
92
|
+
modifications, and in Source or Object form, provided that You
|
|
93
|
+
meet the following conditions:
|
|
94
|
+
|
|
95
|
+
(a) You must give any other recipients of the Work or
|
|
96
|
+
Derivative Works a copy of this License; and
|
|
97
|
+
|
|
98
|
+
(b) You must cause any modified files to carry prominent notices
|
|
99
|
+
stating that You changed the files; and
|
|
100
|
+
|
|
101
|
+
(c) You must retain, in the Source form of any Derivative Works
|
|
102
|
+
that You distribute, all copyright, patent, trademark, and
|
|
103
|
+
attribution notices from the Source form of the Work,
|
|
104
|
+
excluding those notices that do not pertain to any part of
|
|
105
|
+
the Derivative Works; and
|
|
106
|
+
|
|
107
|
+
(d) If the Work includes a "NOTICE" text file as part of its
|
|
108
|
+
distribution, then any Derivative Works that You distribute must
|
|
109
|
+
include a readable copy of the attribution notices contained
|
|
110
|
+
within such NOTICE file, excluding those notices that do not
|
|
111
|
+
pertain to any part of the Derivative Works, in at least one
|
|
112
|
+
of the following places: within a NOTICE text file distributed
|
|
113
|
+
as part of the Derivative Works; within the Source form or
|
|
114
|
+
documentation, if provided along with the Derivative Works; or,
|
|
115
|
+
within a display generated by the Derivative Works, if and
|
|
116
|
+
wherever such third-party notices normally appear. The contents
|
|
117
|
+
of the NOTICE file are for informational purposes only and
|
|
118
|
+
do not modify the License. You may add Your own attribution
|
|
119
|
+
notices within Derivative Works that You distribute, alongside
|
|
120
|
+
or as an addendum to the NOTICE text from the Work, provided
|
|
121
|
+
that such additional attribution notices cannot be construed
|
|
122
|
+
as modifying the License.
|
|
123
|
+
|
|
124
|
+
You may add Your own copyright statement to Your modifications and
|
|
125
|
+
may provide additional or different license terms and conditions
|
|
126
|
+
for use, reproduction, or distribution of Your modifications, or
|
|
127
|
+
for any such Derivative Works as a whole, provided Your use,
|
|
128
|
+
reproduction, and distribution of the Work otherwise complies with
|
|
129
|
+
the conditions stated in this License.
|
|
130
|
+
|
|
131
|
+
5. Submission of Contributions. Unless You explicitly state otherwise,
|
|
132
|
+
any Contribution intentionally submitted for inclusion in the Work
|
|
133
|
+
by You to the Licensor shall be under the terms and conditions of
|
|
134
|
+
this License, without any additional terms or conditions.
|
|
135
|
+
Notwithstanding the above, nothing herein shall supersede or modify
|
|
136
|
+
the terms of any separate license agreement you may have executed
|
|
137
|
+
with Licensor regarding such Contributions.
|
|
138
|
+
|
|
139
|
+
6. Trademarks. This License does not grant permission to use the trade
|
|
140
|
+
names, trademarks, service marks, or product names of the Licensor,
|
|
141
|
+
except as required for reasonable and customary use in describing the
|
|
142
|
+
origin of the Work and reproducing the content of the NOTICE file.
|
|
143
|
+
|
|
144
|
+
7. Disclaimer of Warranty. Unless required by applicable law or
|
|
145
|
+
agreed to in writing, Licensor provides the Work (and each
|
|
146
|
+
Contributor provides its Contributions) on an "AS IS" BASIS,
|
|
147
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
|
148
|
+
implied, including, without limitation, any warranties or conditions
|
|
149
|
+
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
|
150
|
+
PARTICULAR PURPOSE. You are solely responsible for determining the
|
|
151
|
+
appropriateness of using or redistributing the Work and assume any
|
|
152
|
+
risks associated with Your exercise of permissions under this License.
|
|
153
|
+
|
|
154
|
+
8. Limitation of Liability. In no event and under no legal theory,
|
|
155
|
+
whether in tort (including negligence), contract, or otherwise,
|
|
156
|
+
unless required by applicable law (such as deliberate and grossly
|
|
157
|
+
negligent acts) or agreed to in writing, shall any Contributor be
|
|
158
|
+
liable to You for damages, including any direct, indirect, special,
|
|
159
|
+
incidental, or consequential damages of any character arising as a
|
|
160
|
+
result of this License or out of the use or inability to use the
|
|
161
|
+
Work (including but not limited to damages for loss of goodwill,
|
|
162
|
+
work stoppage, computer failure or malfunction, or any and all
|
|
163
|
+
other commercial damages or losses), even if such Contributor
|
|
164
|
+
has been advised of the possibility of such damages.
|
|
165
|
+
|
|
166
|
+
9. Accepting Warranty or Additional Liability. While redistributing
|
|
167
|
+
the Work or Derivative Works thereof, You may choose to offer,
|
|
168
|
+
and charge a fee for, acceptance of support, warranty, indemnity,
|
|
169
|
+
or other liability obligations and/or rights consistent with this
|
|
170
|
+
License. However, in accepting such obligations, You may act only
|
|
171
|
+
on Your own behalf and on Your sole responsibility, not on behalf
|
|
172
|
+
of any other Contributor, and only if You agree to indemnify,
|
|
173
|
+
defend, and hold each Contributor harmless for any liability
|
|
174
|
+
incurred by, or claims asserted against, such Contributor by reason
|
|
175
|
+
of your accepting any such warranty or additional liability.
|
|
176
|
+
|
|
177
|
+
END OF TERMS AND CONDITIONS
|
|
178
|
+
|
|
179
|
+
APPENDIX: How to apply the Apache License to your work.
|
|
180
|
+
|
|
181
|
+
To apply the Apache License to your work, attach the following
|
|
182
|
+
boilerplate notice, with the fields enclosed by brackets "[]"
|
|
183
|
+
replaced with your own identifying information. (Don't include
|
|
184
|
+
the brackets!) The text should be enclosed in the appropriate
|
|
185
|
+
comment syntax for the file format. We also recommend that a
|
|
186
|
+
file or class name and description of purpose be included on the
|
|
187
|
+
same "printed page" as the copyright notice for easier
|
|
188
|
+
identification within third-party archives.
|
|
189
|
+
|
|
190
|
+
Copyright [yyyy] [name of copyright owner]
|
|
191
|
+
|
|
192
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
193
|
+
you may not use this file except in compliance with the License.
|
|
194
|
+
You may obtain a copy of the License at
|
|
195
|
+
|
|
196
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
197
|
+
|
|
198
|
+
Unless required by applicable law or agreed to in writing, software
|
|
199
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
200
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
201
|
+
See the License for the specific language governing permissions and
|
|
202
|
+
limitations under the License.
|
package/README.md
CHANGED
|
@@ -77,6 +77,7 @@ The JSON result goes to stdout, and the exit code says what to do with it: `0` f
|
|
|
77
77
|
- [Profiles](./docs/20-using/40-profiles.md) *- The security profile, writing your own, several in one review, gating and guidelines*
|
|
78
78
|
- [Lenses and policy](./docs/20-using/50-lenses-and-policy.md) *- The panel's questions, the rules that turn kept votes into a confidence, failed votes, and what a level means*
|
|
79
79
|
- [Prompts](./docs/20-using/60-prompts.md) *- The layout of every request, the templates and their variables, the notices, and what is not configurable*
|
|
80
|
+
- [Benchmarks](./docs/20-using/70-bench.md) *- quorum bench: measure configurations on a suite of real bugs, write a suite, resume a run, and read the report and its intervals*
|
|
80
81
|
|
|
81
82
|
*Security*
|
|
82
83
|
- [Threat model](./docs/30-security/10-threat-model.md) *- The attacker, the assets, the residual risk stated plainly, the controls in place, and what the panel's independence is worth*
|
|
@@ -90,3 +91,7 @@ The JSON result goes to stdout, and the exit code says what to do with it: `0` f
|
|
|
90
91
|
- [The JSON result](./docs/95-reference/20-json-result.md) *- Every field of a result in the order it is written, the incomplete reasons, and an example*
|
|
91
92
|
- [Library API](./docs/95-reference/30-library-api.md) *- `createQuorum`, its ports, the events, abort, the errors and every export*
|
|
92
93
|
- [Architecture](./docs/95-reference/40-architecture.md) *- The modules, the dependency rule, the stages of a review, and how the suite runs*
|
|
94
|
+
|
|
95
|
+
## License
|
|
96
|
+
|
|
97
|
+
quorum is released under the [Apache License 2.0](./LICENSE).
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fingerprint of a configuration: the HMAC-SHA-256, under a salt the run keeps, of a canonical form
|
|
3
|
+
* of the resolved settings that names no provider. The bench keys its records by it, and refuses two
|
|
4
|
+
* configurations that resolve alike.
|
|
5
|
+
*
|
|
6
|
+
* The form holds every setting a review uses but the key: the profiles that run, in run order, each with
|
|
7
|
+
* its id, prompts, lenses, policy and gating, and its research provider and each lens's provider written
|
|
8
|
+
* out in place of their names; every field of a provider defined as data but its key, the values of
|
|
9
|
+
* `headers`, `query` and `extraBody` and the whole base URL included; a provider object as its config key
|
|
10
|
+
* and its model; `concurrency` and `maxFindings` as the review applies them, defaults filled in. It holds
|
|
11
|
+
* no file path, no warning and no provider name, so the same settings under other names are one
|
|
12
|
+
* configuration, and a rotated key keeps a run.
|
|
13
|
+
*
|
|
14
|
+
* The form can hold a secret (a credential kept in a header, a query parameter or the base path), so it
|
|
15
|
+
* never leaves this module: not written, not logged, not put in an error message. Only the HMAC does.
|
|
16
|
+
*
|
|
17
|
+
* What it cannot see: a provider object's code (a changed implementation under the same key and model
|
|
18
|
+
* needs a new run directory), and the environment a `claude-code` provider inherits.
|
|
19
|
+
*/
|
|
20
|
+
import type { ResolvedConfig } from '../config/resolve.ts';
|
|
21
|
+
/** A fresh salt: 32 random bytes as 64 lowercase hex digits. */
|
|
22
|
+
export declare function newSalt(): string;
|
|
23
|
+
/**
|
|
24
|
+
* The canonical text of a configuration's settings: JSON with the keys of every object sorted, arrays in
|
|
25
|
+
* their order. It can hold a secret: hash it with `fingerprint`, and never write, log or print it.
|
|
26
|
+
*
|
|
27
|
+
* @throws {RangeError} when a profile names a provider the config does not hold.
|
|
28
|
+
*/
|
|
29
|
+
export declare function canonicalSettings(config: ResolvedConfig): string;
|
|
30
|
+
/**
|
|
31
|
+
* The fingerprint of a configuration under a run's salt: the HMAC-SHA-256 of its canonical settings, as 64
|
|
32
|
+
* lowercase hex digits.
|
|
33
|
+
*
|
|
34
|
+
* @throws {RangeError} when `salt` is not 64 lowercase hex digits (the message never holds it), or a profile names a provider the config does not hold.
|
|
35
|
+
*/
|
|
36
|
+
export declare function fingerprint(config: ResolvedConfig, salt: string): string;
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The fingerprint of a configuration: the HMAC-SHA-256, under a salt the run keeps, of a canonical form
|
|
3
|
+
* of the resolved settings that names no provider. The bench keys its records by it, and refuses two
|
|
4
|
+
* configurations that resolve alike.
|
|
5
|
+
*
|
|
6
|
+
* The form holds every setting a review uses but the key: the profiles that run, in run order, each with
|
|
7
|
+
* its id, prompts, lenses, policy and gating, and its research provider and each lens's provider written
|
|
8
|
+
* out in place of their names; every field of a provider defined as data but its key, the values of
|
|
9
|
+
* `headers`, `query` and `extraBody` and the whole base URL included; a provider object as its config key
|
|
10
|
+
* and its model; `concurrency` and `maxFindings` as the review applies them, defaults filled in. It holds
|
|
11
|
+
* no file path, no warning and no provider name, so the same settings under other names are one
|
|
12
|
+
* configuration, and a rotated key keeps a run.
|
|
13
|
+
*
|
|
14
|
+
* The form can hold a secret (a credential kept in a header, a query parameter or the base path), so it
|
|
15
|
+
* never leaves this module: not written, not logged, not put in an error message. Only the HMAC does.
|
|
16
|
+
*
|
|
17
|
+
* What it cannot see: a provider object's code (a changed implementation under the same key and model
|
|
18
|
+
* needs a new run directory), and the environment a `claude-code` provider inherits.
|
|
19
|
+
*/
|
|
20
|
+
import { createHmac, randomBytes } from 'node:crypto';
|
|
21
|
+
import { isPlainObject } from "../config/load.js";
|
|
22
|
+
import { isProvider } from "../config/schema.js";
|
|
23
|
+
import { DEFAULT_CONCURRENCY } from "../core/concurrency.js";
|
|
24
|
+
import { DEFAULT_MAX_FINDINGS } from "../core/stages/research.js";
|
|
25
|
+
const SALT = /^[0-9a-f]{64}$/;
|
|
26
|
+
/** A fresh salt: 32 random bytes as 64 lowercase hex digits. */
|
|
27
|
+
export function newSalt() {
|
|
28
|
+
return randomBytes(32).toString('hex');
|
|
29
|
+
}
|
|
30
|
+
/** A provider as the form holds it: a provider object by its config key and model, a spec by its fields less the key. */
|
|
31
|
+
function providerForm(provider) {
|
|
32
|
+
if (isProvider(provider))
|
|
33
|
+
return { object: provider.id, model: provider.model };
|
|
34
|
+
return Object.fromEntries(Object.entries(provider).filter(([field]) => field !== 'apiKey'));
|
|
35
|
+
}
|
|
36
|
+
/** The form of the provider a profile binds by `name`; a name the config holds no provider under is a RangeError, never a silent gap. */
|
|
37
|
+
function boundForm(config, name) {
|
|
38
|
+
if (!Object.hasOwn(config.providers, name))
|
|
39
|
+
throw new RangeError('a profile names a provider the config does not hold');
|
|
40
|
+
return providerForm(config.providers[name]);
|
|
41
|
+
}
|
|
42
|
+
/** A plain object with its keys in code-unit order, so the text does not depend on the order the config wrote. */
|
|
43
|
+
function sortKeys(_key, value) {
|
|
44
|
+
return isPlainObject(value)
|
|
45
|
+
? Object.fromEntries(Object.keys(value).sort().map(key => [key, value[key]]))
|
|
46
|
+
: value;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The canonical text of a configuration's settings: JSON with the keys of every object sorted, arrays in
|
|
50
|
+
* their order. It can hold a secret: hash it with `fingerprint`, and never write, log or print it.
|
|
51
|
+
*
|
|
52
|
+
* @throws {RangeError} when a profile names a provider the config does not hold.
|
|
53
|
+
*/
|
|
54
|
+
export function canonicalSettings(config) {
|
|
55
|
+
const profiles = config.profiles.map(({ profile, research, voters }) => ({
|
|
56
|
+
profile,
|
|
57
|
+
research: boundForm(config, research),
|
|
58
|
+
voters: Object.fromEntries(Object.entries(voters).map(([lens, name]) => [
|
|
59
|
+
lens,
|
|
60
|
+
boundForm(config, name),
|
|
61
|
+
])),
|
|
62
|
+
}));
|
|
63
|
+
return JSON.stringify({
|
|
64
|
+
concurrency: config.concurrency ?? DEFAULT_CONCURRENCY,
|
|
65
|
+
maxFindings: config.maxFindings ?? DEFAULT_MAX_FINDINGS,
|
|
66
|
+
profiles,
|
|
67
|
+
}, sortKeys);
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* The fingerprint of a configuration under a run's salt: the HMAC-SHA-256 of its canonical settings, as 64
|
|
71
|
+
* lowercase hex digits.
|
|
72
|
+
*
|
|
73
|
+
* @throws {RangeError} when `salt` is not 64 lowercase hex digits (the message never holds it), or a profile names a provider the config does not hold.
|
|
74
|
+
*/
|
|
75
|
+
export function fingerprint(config, salt) {
|
|
76
|
+
if (!SALT.test(salt))
|
|
77
|
+
throw new RangeError('the salt must be 64 lowercase hex digits');
|
|
78
|
+
return createHmac('sha256', Buffer.from(salt, 'hex')).update(canonicalSettings(config)).digest('hex');
|
|
79
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
|
|
3
|
+
* (the quorum version, the suite's name and hash, the salt, and the fingerprints of its configurations),
|
|
4
|
+
* and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
|
|
5
|
+
* person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
|
|
6
|
+
* and anything the schema does not describe is refused, never repaired. A repeated key is not looked
|
|
7
|
+
* for: the tree that finds one costs ten times the parse on a line of 16 MiB.
|
|
8
|
+
*
|
|
9
|
+
* A record keeps the facts of a review, not its scores: the report scores every record with the suite's
|
|
10
|
+
* locations when it is built, so a score never goes stale.
|
|
11
|
+
*/
|
|
12
|
+
import type { QuorumResult, Usage } from '../core/types.ts';
|
|
13
|
+
/** The `format` of the run file this quorum writes and reads. */
|
|
14
|
+
export declare const RUN_FORMAT = 1;
|
|
15
|
+
/** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
|
|
16
|
+
export declare const MAX_RECORD_BYTES: number;
|
|
17
|
+
/** The most `run.json` may hold, in bytes. */
|
|
18
|
+
export declare const MAX_RUN_FILE_BYTES: number;
|
|
19
|
+
/** The most configurations a run holds. */
|
|
20
|
+
export declare const MAX_CONFIGURATIONS = 16;
|
|
21
|
+
/** One configuration of a run: the path as given (null for the usual resolution) and its fingerprint. */
|
|
22
|
+
export interface RunConfiguration {
|
|
23
|
+
readonly path: string | null;
|
|
24
|
+
readonly fingerprint: string;
|
|
25
|
+
}
|
|
26
|
+
/** What `run.json` holds. */
|
|
27
|
+
export interface RunFile {
|
|
28
|
+
readonly format: 1;
|
|
29
|
+
/** The version of the quorum that began the run. */
|
|
30
|
+
readonly quorum: string;
|
|
31
|
+
readonly suite: {
|
|
32
|
+
readonly name: string;
|
|
33
|
+
readonly hash: string;
|
|
34
|
+
};
|
|
35
|
+
/** The key the fingerprints are keyed with: 32 random bytes as 64 lowercase hex digits. */
|
|
36
|
+
readonly salt: string;
|
|
37
|
+
readonly configurations: readonly RunConfiguration[];
|
|
38
|
+
}
|
|
39
|
+
/** What one line of `results.jsonl` holds: a review that resolved, as the bench saw it. */
|
|
40
|
+
export interface ReviewRecord {
|
|
41
|
+
readonly fingerprint: string;
|
|
42
|
+
/** The id of the suite's case. */
|
|
43
|
+
readonly case: string;
|
|
44
|
+
readonly trial: number;
|
|
45
|
+
readonly latencyMs: number;
|
|
46
|
+
/** The provider calls the review made, failed ones included. */
|
|
47
|
+
readonly calls: number;
|
|
48
|
+
readonly failedCalls: number;
|
|
49
|
+
/** The usage of the calls that reported one; absent when none did. */
|
|
50
|
+
readonly usage?: Usage;
|
|
51
|
+
/** Whether every call reported a cost. */
|
|
52
|
+
readonly costComplete: boolean;
|
|
53
|
+
readonly result: QuorumResult;
|
|
54
|
+
}
|
|
55
|
+
/** The key of a planned review: the fingerprint, the case and the trial, as a JSON array. */
|
|
56
|
+
export declare function reviewKey(fingerprint: string, caseId: string, trial: number): string;
|
|
57
|
+
/** One line of `results.jsonl`: the record as inert JSON, on one line, and a line break. */
|
|
58
|
+
export declare function recordLine(record: ReviewRecord): string;
|
|
59
|
+
/** The record a line of `results.jsonl` holds (without its line break), or undefined when the line is no record of this format. */
|
|
60
|
+
export declare function parseRecord(line: string): ReviewRecord | undefined;
|
|
61
|
+
/** The run file `text` holds, or undefined when it is no run file of this format. */
|
|
62
|
+
export declare function parseRunFile(text: string): RunFile | undefined;
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The two files of a bench run directory, as data: `run.json`, which holds what makes the run one run
|
|
3
|
+
* (the quorum version, the suite's name and hash, the salt, and the fingerprints of its configurations),
|
|
4
|
+
* and `results.jsonl`, which holds one line per review. Both are read as untrusted: another run or a
|
|
5
|
+
* person may have written them, so each is parsed with `JSON.parse` and a strict schema at every level,
|
|
6
|
+
* and anything the schema does not describe is refused, never repaired. A repeated key is not looked
|
|
7
|
+
* for: the tree that finds one costs ten times the parse on a line of 16 MiB.
|
|
8
|
+
*
|
|
9
|
+
* A record keeps the facts of a review, not its scores: the report scores every record with the suite's
|
|
10
|
+
* locations when it is built, so a score never goes stale.
|
|
11
|
+
*/
|
|
12
|
+
import { z } from 'zod';
|
|
13
|
+
import { inertJsonLine } from "../core/sanitize.js";
|
|
14
|
+
import { MAX_TRIALS, SUITE_ID } from "./suite.js";
|
|
15
|
+
/** The `format` of the run file this quorum writes and reads. */
|
|
16
|
+
export const RUN_FORMAT = 1;
|
|
17
|
+
/** The most one line of `results.jsonl` may hold, its line break not counted, in bytes. */
|
|
18
|
+
export const MAX_RECORD_BYTES = 16 * 1024 * 1024;
|
|
19
|
+
/** The most `run.json` may hold, in bytes. */
|
|
20
|
+
export const MAX_RUN_FILE_BYTES = 1024 * 1024;
|
|
21
|
+
/** The most configurations a run holds. */
|
|
22
|
+
export const MAX_CONFIGURATIONS = 16;
|
|
23
|
+
const MAX_ID = 64;
|
|
24
|
+
const MAX_QUORUM_VERSION = 64;
|
|
25
|
+
const MAX_CONFIG_PATH = 4096;
|
|
26
|
+
const digest = z.string().regex(/^[0-9a-f]{64}$/);
|
|
27
|
+
const identifier = z.string().max(MAX_ID).regex(SUITE_ID);
|
|
28
|
+
const count = z.int().min(0);
|
|
29
|
+
const amount = z.number().min(0);
|
|
30
|
+
const confidence = z.enum(['high', 'medium', 'low']);
|
|
31
|
+
const vote = z.strictObject({
|
|
32
|
+
keep: z.boolean(),
|
|
33
|
+
reasoning: z.string(),
|
|
34
|
+
});
|
|
35
|
+
const lensVote = z.strictObject({
|
|
36
|
+
angle: z.string(),
|
|
37
|
+
vote: vote.nullable(),
|
|
38
|
+
error: z.string().nullable(),
|
|
39
|
+
});
|
|
40
|
+
const verdict = z.strictObject({
|
|
41
|
+
id: z.string(),
|
|
42
|
+
profile: z.string(),
|
|
43
|
+
file: z.string(),
|
|
44
|
+
line: count,
|
|
45
|
+
category: z.string(),
|
|
46
|
+
summary: z.string(),
|
|
47
|
+
evidence: z.string(),
|
|
48
|
+
quoteMatched: z.boolean(),
|
|
49
|
+
keptVotes: count,
|
|
50
|
+
totalVotes: count,
|
|
51
|
+
confidence: confidence.optional(),
|
|
52
|
+
votes: z.array(lensVote),
|
|
53
|
+
});
|
|
54
|
+
const providerRef = z.strictObject({
|
|
55
|
+
id: z.string(),
|
|
56
|
+
model: z.string(),
|
|
57
|
+
});
|
|
58
|
+
const thresholds = z.strictObject({
|
|
59
|
+
high: count.nullable(),
|
|
60
|
+
medium: count.nullable(),
|
|
61
|
+
low: count.nullable(),
|
|
62
|
+
});
|
|
63
|
+
const usage = z.strictObject({
|
|
64
|
+
inputTokens: amount,
|
|
65
|
+
outputTokens: amount,
|
|
66
|
+
costUsd: amount.optional(),
|
|
67
|
+
});
|
|
68
|
+
const profileMeta = z.strictObject({
|
|
69
|
+
providers: z.strictObject({
|
|
70
|
+
research: providerRef,
|
|
71
|
+
lenses: z.record(z.string(), providerRef),
|
|
72
|
+
}),
|
|
73
|
+
policy: thresholds,
|
|
74
|
+
gating: z.boolean(),
|
|
75
|
+
invalidFindings: count,
|
|
76
|
+
});
|
|
77
|
+
const incompleteEntry = z.strictObject({
|
|
78
|
+
profile: z.string(),
|
|
79
|
+
reason: z.enum([
|
|
80
|
+
'research-unusable',
|
|
81
|
+
'findings-capped',
|
|
82
|
+
'finding-unreadable',
|
|
83
|
+
'vote-failed',
|
|
84
|
+
'request-too-large',
|
|
85
|
+
]),
|
|
86
|
+
file: z.string().optional(),
|
|
87
|
+
detail: z.string(),
|
|
88
|
+
});
|
|
89
|
+
const flag = z.strictObject({
|
|
90
|
+
code: z.enum(['hidden-characters', 'ai-addressed-text']),
|
|
91
|
+
file: z.string().optional(),
|
|
92
|
+
line: count.optional(),
|
|
93
|
+
detail: z.string(),
|
|
94
|
+
});
|
|
95
|
+
/** Declared as `QuorumResult`'s own type, so a required field the core adds and this schema lacks fails the typecheck. */
|
|
96
|
+
const resultSchema = z.strictObject({
|
|
97
|
+
kept: z.array(verdict),
|
|
98
|
+
discarded: z.array(verdict),
|
|
99
|
+
unreviewed: z.array(verdict),
|
|
100
|
+
meta: z.strictObject({
|
|
101
|
+
version: z.string(),
|
|
102
|
+
startedAt: z.string(),
|
|
103
|
+
durationMs: count,
|
|
104
|
+
profiles: z.record(z.string(), profileMeta),
|
|
105
|
+
usage: usage.optional(),
|
|
106
|
+
incomplete: z.array(incompleteEntry),
|
|
107
|
+
flags: z.array(flag),
|
|
108
|
+
}),
|
|
109
|
+
});
|
|
110
|
+
const recordSchema = z.strictObject({
|
|
111
|
+
fingerprint: digest,
|
|
112
|
+
case: identifier,
|
|
113
|
+
trial: z.int().min(1).max(MAX_TRIALS),
|
|
114
|
+
latencyMs: count,
|
|
115
|
+
calls: count,
|
|
116
|
+
failedCalls: count,
|
|
117
|
+
usage: usage.optional(),
|
|
118
|
+
costComplete: z.boolean(),
|
|
119
|
+
result: resultSchema,
|
|
120
|
+
}).refine(({ calls, failedCalls }) => failedCalls <= calls);
|
|
121
|
+
const runFileSchema = z.strictObject({
|
|
122
|
+
format: z.literal(RUN_FORMAT),
|
|
123
|
+
quorum: z.string().min(1).max(MAX_QUORUM_VERSION),
|
|
124
|
+
suite: z.strictObject({
|
|
125
|
+
name: identifier,
|
|
126
|
+
hash: z.string().regex(/^sha256:[0-9a-f]{64}$/),
|
|
127
|
+
}),
|
|
128
|
+
salt: digest,
|
|
129
|
+
configurations: z.array(z.strictObject({
|
|
130
|
+
path: z.string().min(1).max(MAX_CONFIG_PATH).nullable(),
|
|
131
|
+
fingerprint: digest,
|
|
132
|
+
})).min(1).max(MAX_CONFIGURATIONS),
|
|
133
|
+
});
|
|
134
|
+
/** `text` as JSON checked by `schema`; any failure, of the syntax or of the schema, is undefined. */
|
|
135
|
+
function parseWith(schema, text) {
|
|
136
|
+
try {
|
|
137
|
+
const parsed = schema.safeParse(JSON.parse(text));
|
|
138
|
+
return parsed.success
|
|
139
|
+
? parsed.data
|
|
140
|
+
: undefined;
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
return undefined;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
/** The key of a planned review: the fingerprint, the case and the trial, as a JSON array. */
|
|
147
|
+
export function reviewKey(fingerprint, caseId, trial) {
|
|
148
|
+
return JSON.stringify([fingerprint, caseId, trial]);
|
|
149
|
+
}
|
|
150
|
+
/** One line of `results.jsonl`: the record as inert JSON, on one line, and a line break. */
|
|
151
|
+
export function recordLine(record) {
|
|
152
|
+
return `${inertJsonLine(record)}\n`;
|
|
153
|
+
}
|
|
154
|
+
/** The record a line of `results.jsonl` holds (without its line break), or undefined when the line is no record of this format. */
|
|
155
|
+
export function parseRecord(line) {
|
|
156
|
+
return parseWith(recordSchema, line);
|
|
157
|
+
}
|
|
158
|
+
/** The run file `text` holds, or undefined when it is no run file of this format. */
|
|
159
|
+
export function parseRunFile(text) {
|
|
160
|
+
return parseWith(runFileSchema, text);
|
|
161
|
+
}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The report of a bench run: what each configuration caught, what it cost and how sure the numbers are,
|
|
3
|
+
* built from the scored reviews and rendered as Markdown and as JSON. Pure code over scores, so the
|
|
4
|
+
* report can be built again from a run directory. It holds numbers, suite ids and configuration paths,
|
|
5
|
+
* never text a model wrote.
|
|
6
|
+
*/
|
|
7
|
+
import type { ReviewRecord, RunConfiguration } from './records.ts';
|
|
8
|
+
import type { ReviewOutcome } from './score.ts';
|
|
9
|
+
import type { SuiteCase } from './suite.ts';
|
|
10
|
+
/** What one review earned, with what it took: its latency, its tokens when it reported usage, and its cost when every call reported one. */
|
|
11
|
+
export interface ReviewScore extends ReviewOutcome {
|
|
12
|
+
readonly latencyMs: number;
|
|
13
|
+
readonly tokens?: number;
|
|
14
|
+
readonly costUsd?: number;
|
|
15
|
+
}
|
|
16
|
+
/** A rate, the mean of the case scores, with its Wilson interval over `cases` cases. */
|
|
17
|
+
export interface Rate {
|
|
18
|
+
readonly value: number;
|
|
19
|
+
readonly low: number;
|
|
20
|
+
readonly high: number;
|
|
21
|
+
readonly cases: number;
|
|
22
|
+
}
|
|
23
|
+
/** What one configuration did over the reviews that count. */
|
|
24
|
+
export interface ConfigurationSummary {
|
|
25
|
+
/** The position of the configuration, from 1; the first is the reference. */
|
|
26
|
+
readonly index: number;
|
|
27
|
+
readonly path: string | null;
|
|
28
|
+
readonly reviews: number;
|
|
29
|
+
readonly planned: number;
|
|
30
|
+
readonly incomplete: number;
|
|
31
|
+
readonly keptRecall: Rate | null;
|
|
32
|
+
readonly proposedRecall: Rate | null;
|
|
33
|
+
readonly unmatchedKept: {
|
|
34
|
+
readonly bug: number | null;
|
|
35
|
+
readonly control: number | null;
|
|
36
|
+
};
|
|
37
|
+
readonly tokens: {
|
|
38
|
+
readonly median: number;
|
|
39
|
+
readonly total: number;
|
|
40
|
+
} | null;
|
|
41
|
+
readonly latencyMs: {
|
|
42
|
+
readonly p50: number;
|
|
43
|
+
readonly p95: number;
|
|
44
|
+
} | null;
|
|
45
|
+
readonly costUsd: number | null;
|
|
46
|
+
}
|
|
47
|
+
/** What a comparison with the reference measures. */
|
|
48
|
+
export type Measure = 'kept-recall' | 'proposed-recall' | 'unmatched-kept-per-control' | 'incomplete' | 'tokens';
|
|
49
|
+
/** One configuration against the reference on one measure: an exact sign test over the cases both scored. */
|
|
50
|
+
export interface Comparison {
|
|
51
|
+
readonly configuration: number;
|
|
52
|
+
readonly measure: Measure;
|
|
53
|
+
/** The mean over the pairs of the configuration's case score minus the reference's; null with no pair. */
|
|
54
|
+
readonly difference: number | null;
|
|
55
|
+
readonly better: number;
|
|
56
|
+
readonly worse: number;
|
|
57
|
+
readonly tied: number;
|
|
58
|
+
readonly p: number;
|
|
59
|
+
readonly resolved: boolean;
|
|
60
|
+
}
|
|
61
|
+
/** What one configuration scored on one case, over the trials that count. */
|
|
62
|
+
export interface CaseScore {
|
|
63
|
+
readonly trials: number;
|
|
64
|
+
/** The share of the trials that caught the bug; null for a control. */
|
|
65
|
+
readonly keptCatch: number | null;
|
|
66
|
+
readonly proposedCatch: number | null;
|
|
67
|
+
/** The mean over the trials of the kept findings that catch nothing. */
|
|
68
|
+
readonly unmatchedKept: number;
|
|
69
|
+
/** The share of the trials with a gap. */
|
|
70
|
+
readonly incomplete: number;
|
|
71
|
+
/** The mean over the trials that reported usage; null when none did. */
|
|
72
|
+
readonly tokens: number | null;
|
|
73
|
+
}
|
|
74
|
+
/** The scores of one case, one entry per configuration, null for a configuration that counted none of its trials. */
|
|
75
|
+
export interface CaseScores {
|
|
76
|
+
readonly id: string;
|
|
77
|
+
readonly kind: 'bug' | 'control';
|
|
78
|
+
readonly configurations: readonly (CaseScore | null)[];
|
|
79
|
+
}
|
|
80
|
+
export interface Report {
|
|
81
|
+
readonly suite: string;
|
|
82
|
+
readonly quorum: string;
|
|
83
|
+
readonly trials: number;
|
|
84
|
+
readonly window: number;
|
|
85
|
+
readonly bugCases: number;
|
|
86
|
+
readonly controls: number;
|
|
87
|
+
readonly configurations: readonly ConfigurationSummary[];
|
|
88
|
+
readonly comparisons: readonly Comparison[];
|
|
89
|
+
readonly cases: readonly CaseScores[];
|
|
90
|
+
}
|
|
91
|
+
/** What the report is built from: the suite, the selected cases and trials, the configurations in order and the score of each recorded review by `reviewKey`. */
|
|
92
|
+
export interface ReportInput {
|
|
93
|
+
readonly suite: {
|
|
94
|
+
readonly name: string;
|
|
95
|
+
readonly window: number;
|
|
96
|
+
};
|
|
97
|
+
readonly quorum: string;
|
|
98
|
+
readonly cases: readonly SuiteCase[];
|
|
99
|
+
readonly trials: number;
|
|
100
|
+
readonly configurations: readonly RunConfiguration[];
|
|
101
|
+
readonly scores: ReadonlyMap<string, ReviewScore>;
|
|
102
|
+
}
|
|
103
|
+
/** Score a record against its case: the outcome, the latency, the tokens (input plus output) when there is usage, and the cost when every call reported one. */
|
|
104
|
+
export declare function reviewScore(record: ReviewRecord, suiteCase: SuiteCase, window: number): ReviewScore;
|
|
105
|
+
/** Build the report: count the recorded reviews of the selected cases and trials, score each case, summarize each configuration and compare each with the first. */
|
|
106
|
+
export declare function buildReport(input: ReportInput): Report;
|
|
107
|
+
/** The report as Markdown: its counts, the configurations and their table, and the comparison lines when there are two or more configurations. Every line is inert. */
|
|
108
|
+
export declare function reportMarkdown(report: Report): string;
|
|
109
|
+
/** The report as inert JSON and a line break; it parses back to the report. */
|
|
110
|
+
export declare function reportJson(report: Report): string;
|