@tobi-or-not/quorum 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/bench/fingerprint.d.ts +8 -5
- package/dist/bench/fingerprint.js +28 -9
- package/dist/bench/records.d.ts +168 -0
- package/dist/bench/records.js +106 -1
- package/dist/bench/runner.d.ts +18 -0
- package/dist/bench/runner.js +30 -4
- package/dist/cli/bench.d.ts +11 -4
- package/dist/cli/bench.js +34 -17
- package/dist/cli/bin.js +2 -2
- package/dist/cli/io.d.ts +5 -0
- package/dist/cli/main.d.ts +67 -24
- package/dist/cli/main.js +288 -59
- package/dist/cli/print-config.d.ts +8 -6
- package/dist/cli/print-config.js +26 -8
- package/dist/cli/render.d.ts +9 -3
- package/dist/cli/render.js +43 -2
- package/dist/config/claude-code.d.ts +2 -6
- package/dist/config/claude-code.js +6 -44
- package/dist/config/command.d.ts +10 -0
- package/dist/config/command.js +47 -0
- package/dist/config/env.d.ts +32 -19
- package/dist/config/env.js +95 -32
- package/dist/config/from-config.js +21 -3
- package/dist/config/resolve.d.ts +13 -1
- package/dist/config/resolve.js +27 -1
- package/dist/config/schema.d.ts +152 -5
- package/dist/config/schema.js +156 -12
- package/dist/core/clock.d.ts +17 -0
- package/dist/core/clock.js +14 -0
- package/dist/core/coverage.d.ts +74 -0
- package/dist/core/coverage.js +221 -0
- package/dist/core/diff.d.ts +101 -0
- package/dist/core/diff.js +178 -35
- package/dist/core/errors.d.ts +12 -3
- package/dist/core/errors.js +21 -2
- package/dist/core/estimate.d.ts +45 -0
- package/dist/core/estimate.js +188 -0
- package/dist/core/exclude.d.ts +42 -0
- package/dist/core/exclude.js +189 -0
- package/dist/core/flags.d.ts +14 -1
- package/dist/core/flags.js +56 -0
- package/dist/core/inputs.d.ts +38 -0
- package/dist/core/inputs.js +228 -0
- package/dist/core/limits.d.ts +83 -0
- package/dist/core/limits.js +169 -0
- package/dist/core/output.d.ts +3 -2
- package/dist/core/output.js +3 -2
- package/dist/core/parts.d.ts +146 -0
- package/dist/core/parts.js +388 -0
- package/dist/core/ports.d.ts +67 -2
- package/dist/core/ports.js +36 -1
- package/dist/core/prompts.d.ts +22 -3
- package/dist/core/prompts.js +43 -7
- package/dist/core/quorum.d.ts +26 -8
- package/dist/core/quorum.js +310 -73
- package/dist/core/retry.d.ts +36 -0
- package/dist/core/retry.js +46 -0
- package/dist/core/secrets.d.ts +98 -0
- package/dist/core/secrets.js +507 -0
- package/dist/core/stages/context.d.ts +131 -23
- package/dist/core/stages/context.js +331 -46
- package/dist/core/stages/panel.d.ts +13 -6
- package/dist/core/stages/panel.js +79 -19
- package/dist/core/stages/research.d.ts +110 -25
- package/dist/core/stages/research.js +338 -44
- package/dist/core/tokens.d.ts +31 -0
- package/dist/core/tokens.js +81 -0
- package/dist/core/types.d.ts +280 -25
- package/dist/core/verify.d.ts +6 -0
- package/dist/core/verify.js +6 -3
- package/dist/git/run.d.ts +223 -0
- package/dist/git/run.js +608 -0
- package/dist/git/tree.d.ts +43 -0
- package/dist/git/tree.js +171 -0
- package/dist/index.d.ts +26 -2
- package/dist/index.js +34 -2
- package/dist/providers/anthropic.d.ts +3 -1
- package/dist/providers/anthropic.js +10 -3
- package/dist/providers/claude-code.d.ts +56 -11
- package/dist/providers/claude-code.js +70 -25
- package/dist/providers/gemini.d.ts +5 -1
- package/dist/providers/gemini.js +65 -11
- package/dist/providers/http.d.ts +49 -4
- package/dist/providers/http.js +155 -14
- package/dist/providers/ollama.d.ts +4 -1
- package/dist/providers/ollama.js +13 -5
- package/dist/providers/openai-compatible.d.ts +4 -1
- package/dist/providers/openai-compatible.js +13 -5
- package/dist/providers/options.d.ts +12 -3
- package/dist/providers/options.js +30 -10
- package/dist/providers/size.d.ts +19 -15
- package/dist/providers/size.js +33 -73
- package/dist/sources/added.d.ts +49 -0
- package/dist/sources/added.js +134 -0
- package/dist/sources/diff.d.ts +13 -0
- package/dist/sources/diff.js +63 -0
- package/dist/sources/local.d.ts +41 -0
- package/dist/sources/local.js +251 -0
- package/dist/sources/paths.d.ts +45 -0
- package/dist/sources/paths.js +230 -0
- package/dist/sources/range.d.ts +21 -0
- package/dist/sources/range.js +75 -0
- package/dist/sources/read.d.ts +69 -0
- package/dist/sources/read.js +37 -0
- package/dist/sources/repository.d.ts +9 -0
- package/dist/sources/repository.js +28 -0
- package/dist/sources/skips.d.ts +49 -0
- package/dist/sources/skips.js +126 -0
- package/package.json +1 -1
- package/schema/quorum.schema.json +49 -1
package/README.md
CHANGED
|
@@ -43,6 +43,7 @@ export QUORUM_RESEARCH_MODEL=qwen3.6:27b # a model your local Ollama serves
|
|
|
43
43
|
export QUORUM_RESEARCH_CONTEXT_TOKENS=32768 # its context window, in tokens
|
|
44
44
|
quorum ./my.diff > result.json # review a diff file
|
|
45
45
|
git diff origin/main...HEAD | quorum > result.json # or what your branch changes
|
|
46
|
+
quorum --range origin/main...HEAD > result.json # what your branch changes, git run by quorum
|
|
46
47
|
```
|
|
47
48
|
|
|
48
49
|
`QUORUM_RESEARCH_CONTEXT_TOKENS` is the model's context window in tokens, which quorum checks every request against, and without it the run exits `2` with a message that names it. With no provider configured and no base URL, preset or protocol variable, quorum talks to the Ollama on this machine through its native API, `http://localhost:11434`, which Ollama listens on, on `127.0.0.1`, by default. That API takes `num_ctx`, `truncate: false` and `shift: false`, so quorum tells the server the context size and the server refuses an overflow, where Ollama's OpenAI-compatible `/v1` would cut the prompt in silence. Refusing needs Ollama 0.12.6 or later: an older server ignores `truncate` and `shift`. A base URL variable set on its own means an OpenAI-compatible server, as before. Keep the number at most what `ollama show <model>` reports for the model, since Ollama clamps a larger `num_ctx` to it, and read the context a running model loaded in the `CONTEXT` column of `ollama ps`.
|
|
@@ -58,7 +59,7 @@ quorum ./my.diff > result.json
|
|
|
58
59
|
|
|
59
60
|
Requires Claude Code 2.1.311 or newer. [`claudeCode`](./docs/20-using/30-providers.md#claudecode) has the command line, the isolation and the auth modes, and [Claude Code](./docs/20-using/20-configuration.md#claude-code) the setup, in CI too.
|
|
60
61
|
|
|
61
|
-
The JSON result goes to stdout, and the exit code says what to do with it: `0` for a complete review in which no gating profile kept a `high` finding, `1` when one did, complete or not, `2` for a usage or configuration error, `3` when
|
|
62
|
+
The JSON result goes to stdout, and the exit code says what to do with it: `0` for a complete review in which no gating profile kept a `high` finding, `1` when one did, complete or not, `2` for a usage or configuration error, `3` when a gating profile's research call failed, `4` when a gating profile's review is incomplete otherwise, and `130` when the run was interrupted. [Quick start](./docs/10-getting-started/20-quickstart.md) walks one review through, and [Start here](./docs/01-readme.md) lays out four tracks: review a diff, use it from code or review something else, know what it protects against, and measure configurations.
|
|
62
63
|
|
|
63
64
|
## Documentation
|
|
64
65
|
|
|
@@ -72,6 +73,7 @@ The JSON result goes to stdout, and the exit code says what to do with it: `0` f
|
|
|
72
73
|
|
|
73
74
|
*Using it*
|
|
74
75
|
- [The CLI](./docs/20-using/10-cli.md) *- The invocation, stdout and stderr, every exit code and its causes, and gating a CI job on them*
|
|
76
|
+
- [Sources](./docs/20-using/15-sources.md) *- A diff file or stdin, a range of commits, the local changes or the files at a commit, and how quorum runs git*
|
|
75
77
|
- [Configuration](./docs/20-using/20-configuration.md) *- The config file and its trust model in CI, the `QUORUM_*` variables and the voter's fallback, the context size each provider states, running Claude Code, the keys, `--print-config`, and the errors a bad setting causes*
|
|
76
78
|
- [Providers](./docs/20-using/30-providers.md) *- The provider port, the shipped adapters and their options, Claude Code run isolated as a provider, the presets, `ProviderError`, and writing a provider of your own*
|
|
77
79
|
- [Profiles](./docs/20-using/40-profiles.md) *- The security profile, writing your own, several in one review, gating and guidelines*
|
|
@@ -6,11 +6,14 @@
|
|
|
6
6
|
* The form holds every setting a review uses but the key: the profiles that run, in run order, each with
|
|
7
7
|
* its id, prompts, lenses, policy and gating, and its research provider and each lens's provider written
|
|
8
8
|
* out in place of their names; every field of a provider defined as data but its key, the values of
|
|
9
|
-
* `headers`, `query` and `extraBody` and the whole base URL included
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
9
|
+
* `headers`, `query` and `extraBody` and the whole base URL included, `timeoutMs` and `retries` among them;
|
|
10
|
+
* a provider object as its config key, its model and the `timeoutMs` and `retries` it states, so one that
|
|
11
|
+
* states none keeps the form it had, and never the `worstCase` or the `room` it states, which are code like its
|
|
12
|
+
* `complete`; `concurrency`, `maxFindings`, `partTokens`, `maxReviewTokens` and `reviewTimeoutMs` as the review applies
|
|
13
|
+
* them, defaults filled in, `exclude` when the config lists a pattern, and `send` when it lists a file. It
|
|
14
|
+
* holds no config-file path, no warning and no provider name, so the same settings under other names are one
|
|
15
|
+
* configuration, and a rotated key keeps a run. A `claude-code` provider's resolved command path is one of its
|
|
16
|
+
* settings, so it is in the form.
|
|
14
17
|
*
|
|
15
18
|
* The form can hold a secret (a credential kept in a header, a query parameter or the base path), so it
|
|
16
19
|
* never leaves this module: not written, not logged, not put in an error message. Only the HMAC does.
|
|
@@ -6,11 +6,14 @@
|
|
|
6
6
|
* The form holds every setting a review uses but the key: the profiles that run, in run order, each with
|
|
7
7
|
* its id, prompts, lenses, policy and gating, and its research provider and each lens's provider written
|
|
8
8
|
* out in place of their names; every field of a provider defined as data but its key, the values of
|
|
9
|
-
* `headers`, `query` and `extraBody` and the whole base URL included
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
9
|
+
* `headers`, `query` and `extraBody` and the whole base URL included, `timeoutMs` and `retries` among them;
|
|
10
|
+
* a provider object as its config key, its model and the `timeoutMs` and `retries` it states, so one that
|
|
11
|
+
* states none keeps the form it had, and never the `worstCase` or the `room` it states, which are code like its
|
|
12
|
+
* `complete`; `concurrency`, `maxFindings`, `partTokens`, `maxReviewTokens` and `reviewTimeoutMs` as the review applies
|
|
13
|
+
* them, defaults filled in, `exclude` when the config lists a pattern, and `send` when it lists a file. It
|
|
14
|
+
* holds no config-file path, no warning and no provider name, so the same settings under other names are one
|
|
15
|
+
* configuration, and a rotated key keeps a run. A `claude-code` provider's resolved command path is one of its
|
|
16
|
+
* settings, so it is in the form.
|
|
14
17
|
*
|
|
15
18
|
* The form can hold a secret (a credential kept in a header, a query parameter or the base path), so it
|
|
16
19
|
* never leaves this module: not written, not logged, not put in an error message. Only the HMAC does.
|
|
@@ -20,18 +23,25 @@
|
|
|
20
23
|
*/
|
|
21
24
|
import { createHmac, randomBytes } from 'node:crypto';
|
|
22
25
|
import { isPlainObject } from "../config/load.js";
|
|
23
|
-
import { isProvider } from "../config/schema.js";
|
|
26
|
+
import { isProvider, statedFacts } from "../config/schema.js";
|
|
24
27
|
import { DEFAULT_CONCURRENCY } from "../core/concurrency.js";
|
|
28
|
+
import { DEFAULT_MAX_REVIEW_TOKENS, DEFAULT_REVIEW_TIMEOUT_MS } from "../core/limits.js";
|
|
29
|
+
import { DEFAULT_PART_TOKENS } from "../core/parts.js";
|
|
25
30
|
import { DEFAULT_MAX_FINDINGS } from "../core/stages/research.js";
|
|
26
31
|
const SALT = /^[0-9a-f]{64}$/;
|
|
27
32
|
/** A fresh salt: 32 random bytes as 64 lowercase hex digits. */
|
|
28
33
|
export function newSalt() {
|
|
29
34
|
return randomBytes(32).toString('hex');
|
|
30
35
|
}
|
|
31
|
-
/** A provider as the form holds it: a provider object by its config key and
|
|
36
|
+
/** A provider as the form holds it: a provider object by its config key, its model and the facts it states, a spec by its fields less the key. */
|
|
32
37
|
function providerForm(provider) {
|
|
33
|
-
if (isProvider(provider))
|
|
34
|
-
return {
|
|
38
|
+
if (isProvider(provider)) {
|
|
39
|
+
return {
|
|
40
|
+
object: provider.id,
|
|
41
|
+
model: provider.model,
|
|
42
|
+
...statedFacts(provider).stated,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
35
45
|
return Object.fromEntries(Object.entries(provider).filter(([field]) => field !== 'apiKey'));
|
|
36
46
|
}
|
|
37
47
|
/** The form of the provider a profile binds by `name`; a name the config holds no provider under is a RangeError, never a silent gap. */
|
|
@@ -64,6 +74,15 @@ export function canonicalSettings(config) {
|
|
|
64
74
|
return JSON.stringify({
|
|
65
75
|
concurrency: config.concurrency ?? DEFAULT_CONCURRENCY,
|
|
66
76
|
maxFindings: config.maxFindings ?? DEFAULT_MAX_FINDINGS,
|
|
77
|
+
partTokens: config.partTokens ?? DEFAULT_PART_TOKENS,
|
|
78
|
+
maxReviewTokens: config.maxReviewTokens ?? DEFAULT_MAX_REVIEW_TOKENS,
|
|
79
|
+
reviewTimeoutMs: config.reviewTimeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS,
|
|
80
|
+
...config.exclude === undefined
|
|
81
|
+
? {}
|
|
82
|
+
: { exclude: config.exclude },
|
|
83
|
+
...config.send === undefined
|
|
84
|
+
? {}
|
|
85
|
+
: { send: config.send },
|
|
67
86
|
profiles,
|
|
68
87
|
}, sortKeys);
|
|
69
88
|
}
|
package/dist/bench/records.d.ts
CHANGED
|
@@ -106,6 +106,25 @@ export declare const resultSchema: z.ZodObject<{
|
|
|
106
106
|
reasoning: z.ZodString;
|
|
107
107
|
}, z.core.$strict>>;
|
|
108
108
|
error: z.ZodNullable<z.ZodString>;
|
|
109
|
+
failure: z.ZodNullable<z.ZodObject<{
|
|
110
|
+
kind: z.ZodEnum<{
|
|
111
|
+
http: "http";
|
|
112
|
+
network: "network";
|
|
113
|
+
process: "process";
|
|
114
|
+
timeout: "timeout";
|
|
115
|
+
aborted: "aborted";
|
|
116
|
+
refusal: "refusal";
|
|
117
|
+
truncated: "truncated";
|
|
118
|
+
"invalid-response": "invalid-response";
|
|
119
|
+
unsupported: "unsupported";
|
|
120
|
+
"too-large": "too-large";
|
|
121
|
+
"token-cap": "token-cap";
|
|
122
|
+
"review-deadline": "review-deadline";
|
|
123
|
+
"unusable-reply": "unusable-reply";
|
|
124
|
+
unknown: "unknown";
|
|
125
|
+
}>;
|
|
126
|
+
status: z.ZodOptional<z.ZodInt>;
|
|
127
|
+
}, z.core.$strict>>;
|
|
109
128
|
}, z.core.$strict>>;
|
|
110
129
|
}, z.core.$strict>>;
|
|
111
130
|
discarded: z.ZodArray<z.ZodObject<{
|
|
@@ -131,6 +150,25 @@ export declare const resultSchema: z.ZodObject<{
|
|
|
131
150
|
reasoning: z.ZodString;
|
|
132
151
|
}, z.core.$strict>>;
|
|
133
152
|
error: z.ZodNullable<z.ZodString>;
|
|
153
|
+
failure: z.ZodNullable<z.ZodObject<{
|
|
154
|
+
kind: z.ZodEnum<{
|
|
155
|
+
http: "http";
|
|
156
|
+
network: "network";
|
|
157
|
+
process: "process";
|
|
158
|
+
timeout: "timeout";
|
|
159
|
+
aborted: "aborted";
|
|
160
|
+
refusal: "refusal";
|
|
161
|
+
truncated: "truncated";
|
|
162
|
+
"invalid-response": "invalid-response";
|
|
163
|
+
unsupported: "unsupported";
|
|
164
|
+
"too-large": "too-large";
|
|
165
|
+
"token-cap": "token-cap";
|
|
166
|
+
"review-deadline": "review-deadline";
|
|
167
|
+
"unusable-reply": "unusable-reply";
|
|
168
|
+
unknown: "unknown";
|
|
169
|
+
}>;
|
|
170
|
+
status: z.ZodOptional<z.ZodInt>;
|
|
171
|
+
}, z.core.$strict>>;
|
|
134
172
|
}, z.core.$strict>>;
|
|
135
173
|
}, z.core.$strict>>;
|
|
136
174
|
unreviewed: z.ZodArray<z.ZodObject<{
|
|
@@ -156,12 +194,62 @@ export declare const resultSchema: z.ZodObject<{
|
|
|
156
194
|
reasoning: z.ZodString;
|
|
157
195
|
}, z.core.$strict>>;
|
|
158
196
|
error: z.ZodNullable<z.ZodString>;
|
|
197
|
+
failure: z.ZodNullable<z.ZodObject<{
|
|
198
|
+
kind: z.ZodEnum<{
|
|
199
|
+
http: "http";
|
|
200
|
+
network: "network";
|
|
201
|
+
process: "process";
|
|
202
|
+
timeout: "timeout";
|
|
203
|
+
aborted: "aborted";
|
|
204
|
+
refusal: "refusal";
|
|
205
|
+
truncated: "truncated";
|
|
206
|
+
"invalid-response": "invalid-response";
|
|
207
|
+
unsupported: "unsupported";
|
|
208
|
+
"too-large": "too-large";
|
|
209
|
+
"token-cap": "token-cap";
|
|
210
|
+
"review-deadline": "review-deadline";
|
|
211
|
+
"unusable-reply": "unusable-reply";
|
|
212
|
+
unknown: "unknown";
|
|
213
|
+
}>;
|
|
214
|
+
status: z.ZodOptional<z.ZodInt>;
|
|
215
|
+
}, z.core.$strict>>;
|
|
159
216
|
}, z.core.$strict>>;
|
|
160
217
|
}, z.core.$strict>>;
|
|
161
218
|
meta: z.ZodObject<{
|
|
162
219
|
version: z.ZodString;
|
|
163
220
|
startedAt: z.ZodString;
|
|
164
221
|
durationMs: z.ZodInt;
|
|
222
|
+
source: z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
223
|
+
kind: z.ZodLiteral<"diff">;
|
|
224
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
225
|
+
kind: z.ZodLiteral<"range">;
|
|
226
|
+
base: z.ZodString;
|
|
227
|
+
mergeBase: z.ZodString;
|
|
228
|
+
head: z.ZodString;
|
|
229
|
+
objectFormat: z.ZodEnum<{
|
|
230
|
+
sha1: "sha1";
|
|
231
|
+
sha256: "sha256";
|
|
232
|
+
}>;
|
|
233
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
234
|
+
kind: z.ZodEnum<{
|
|
235
|
+
local: "local";
|
|
236
|
+
staged: "staged";
|
|
237
|
+
}>;
|
|
238
|
+
head: z.ZodNullable<z.ZodString>;
|
|
239
|
+
objectFormat: z.ZodEnum<{
|
|
240
|
+
sha1: "sha1";
|
|
241
|
+
sha256: "sha256";
|
|
242
|
+
}>;
|
|
243
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
244
|
+
kind: z.ZodLiteral<"paths">;
|
|
245
|
+
commit: z.ZodString;
|
|
246
|
+
paths: z.ZodArray<z.ZodString>;
|
|
247
|
+
skips: z.ZodBoolean;
|
|
248
|
+
objectFormat: z.ZodEnum<{
|
|
249
|
+
sha1: "sha1";
|
|
250
|
+
sha256: "sha256";
|
|
251
|
+
}>;
|
|
252
|
+
}, z.core.$strict>], "kind">;
|
|
165
253
|
profiles: z.ZodRecord<z.ZodString, z.ZodObject<{
|
|
166
254
|
providers: z.ZodObject<{
|
|
167
255
|
research: z.ZodObject<{
|
|
@@ -180,6 +268,41 @@ export declare const resultSchema: z.ZodObject<{
|
|
|
180
268
|
}, z.core.$strict>;
|
|
181
269
|
gating: z.ZodBoolean;
|
|
182
270
|
invalidFindings: z.ZodInt;
|
|
271
|
+
coverage: z.ZodArray<z.ZodObject<{
|
|
272
|
+
file: z.ZodString;
|
|
273
|
+
status: z.ZodEnum<{
|
|
274
|
+
unreviewed: "unreviewed";
|
|
275
|
+
excluded: "excluded";
|
|
276
|
+
reviewed: "reviewed";
|
|
277
|
+
partial: "partial";
|
|
278
|
+
}>;
|
|
279
|
+
reason: z.ZodOptional<z.ZodEnum<{
|
|
280
|
+
"too-large": "too-large";
|
|
281
|
+
"token-cap": "token-cap";
|
|
282
|
+
"review-deadline": "review-deadline";
|
|
283
|
+
"research-failed": "research-failed";
|
|
284
|
+
"request-too-large": "request-too-large";
|
|
285
|
+
binary: "binary";
|
|
286
|
+
submodule: "submodule";
|
|
287
|
+
secret: "secret";
|
|
288
|
+
"deleted-secret": "deleted-secret";
|
|
289
|
+
}>>;
|
|
290
|
+
left: z.ZodOptional<z.ZodArray<z.ZodObject<{
|
|
291
|
+
start: z.ZodInt;
|
|
292
|
+
end: z.ZodInt;
|
|
293
|
+
reason: z.ZodEnum<{
|
|
294
|
+
"too-large": "too-large";
|
|
295
|
+
"token-cap": "token-cap";
|
|
296
|
+
"review-deadline": "review-deadline";
|
|
297
|
+
"research-failed": "research-failed";
|
|
298
|
+
"request-too-large": "request-too-large";
|
|
299
|
+
binary: "binary";
|
|
300
|
+
submodule: "submodule";
|
|
301
|
+
secret: "secret";
|
|
302
|
+
"deleted-secret": "deleted-secret";
|
|
303
|
+
}>;
|
|
304
|
+
}, z.core.$strict>>>;
|
|
305
|
+
}, z.core.$strict>>;
|
|
183
306
|
}, z.core.$strict>>;
|
|
184
307
|
usage: z.ZodOptional<z.ZodObject<{
|
|
185
308
|
inputTokens: z.ZodNumber;
|
|
@@ -189,19 +312,64 @@ export declare const resultSchema: z.ZodObject<{
|
|
|
189
312
|
incomplete: z.ZodArray<z.ZodObject<{
|
|
190
313
|
profile: z.ZodString;
|
|
191
314
|
reason: z.ZodEnum<{
|
|
315
|
+
"too-large": "too-large";
|
|
316
|
+
"token-cap": "token-cap";
|
|
317
|
+
"review-deadline": "review-deadline";
|
|
192
318
|
"research-unusable": "research-unusable";
|
|
319
|
+
"research-failed": "research-failed";
|
|
193
320
|
"findings-capped": "findings-capped";
|
|
194
321
|
"finding-unreadable": "finding-unreadable";
|
|
195
322
|
"vote-failed": "vote-failed";
|
|
196
323
|
"request-too-large": "request-too-large";
|
|
324
|
+
binary: "binary";
|
|
325
|
+
submodule: "submodule";
|
|
326
|
+
secret: "secret";
|
|
197
327
|
}>;
|
|
198
328
|
file: z.ZodOptional<z.ZodString>;
|
|
199
329
|
detail: z.ZodString;
|
|
200
330
|
}, z.core.$strict>>;
|
|
331
|
+
leftOut: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
332
|
+
file: z.ZodString;
|
|
333
|
+
reason: z.ZodEnum<{
|
|
334
|
+
binary: "binary";
|
|
335
|
+
submodule: "submodule";
|
|
336
|
+
excluded: "excluded";
|
|
337
|
+
}>;
|
|
338
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
339
|
+
file: z.ZodString;
|
|
340
|
+
reason: z.ZodLiteral<"skipped">;
|
|
341
|
+
rule: z.ZodEnum<{
|
|
342
|
+
binary: "binary";
|
|
343
|
+
submodule: "submodule";
|
|
344
|
+
secret: "secret";
|
|
345
|
+
vendored: "vendored";
|
|
346
|
+
lockfile: "lockfile";
|
|
347
|
+
minified: "minified";
|
|
348
|
+
lfs: "lfs";
|
|
349
|
+
}>;
|
|
350
|
+
}, z.core.$strict>, z.ZodObject<{
|
|
351
|
+
file: z.ZodString;
|
|
352
|
+
reason: z.ZodEnum<{
|
|
353
|
+
secret: "secret";
|
|
354
|
+
"deleted-secret": "deleted-secret";
|
|
355
|
+
}>;
|
|
356
|
+
rule: z.ZodEnum<{
|
|
357
|
+
"private-key": "private-key";
|
|
358
|
+
"aws-key": "aws-key";
|
|
359
|
+
"github-token": "github-token";
|
|
360
|
+
"slack-token": "slack-token";
|
|
361
|
+
"google-key": "google-key";
|
|
362
|
+
"stripe-key": "stripe-key";
|
|
363
|
+
"anthropic-key": "anthropic-key";
|
|
364
|
+
"openai-key": "openai-key";
|
|
365
|
+
name: "name";
|
|
366
|
+
}>;
|
|
367
|
+
}, z.core.$strict>], "reason">>;
|
|
201
368
|
flags: z.ZodArray<z.ZodObject<{
|
|
202
369
|
code: z.ZodEnum<{
|
|
203
370
|
"hidden-characters": "hidden-characters";
|
|
204
371
|
"ai-addressed-text": "ai-addressed-text";
|
|
372
|
+
"secret-file": "secret-file";
|
|
205
373
|
}>;
|
|
206
374
|
file: z.ZodOptional<z.ZodString>;
|
|
207
375
|
line: z.ZodOptional<z.ZodInt>;
|
package/dist/bench/records.js
CHANGED
|
@@ -14,7 +14,9 @@
|
|
|
14
14
|
* that a model's reply denied, which denial it was, never the reply.
|
|
15
15
|
*/
|
|
16
16
|
import { z } from 'zod';
|
|
17
|
+
import { SKIP_RULES, sourceIssues } from "../core/inputs.js";
|
|
17
18
|
import { inertJsonLine } from "../core/sanitize.js";
|
|
19
|
+
import { SECRET_RULES } from "../core/secrets.js";
|
|
18
20
|
import { MAX_TRIALS, SUITE_ID } from "./suite.js";
|
|
19
21
|
/** The `format` of the run file this quorum writes and reads. */
|
|
20
22
|
export const RUN_FORMAT = 2;
|
|
@@ -41,10 +43,30 @@ const vote = z.strictObject({
|
|
|
41
43
|
keep: z.boolean(),
|
|
42
44
|
reasoning: z.string(),
|
|
43
45
|
});
|
|
46
|
+
const voteFailure = z.strictObject({
|
|
47
|
+
kind: z.enum([
|
|
48
|
+
'http',
|
|
49
|
+
'network',
|
|
50
|
+
'process',
|
|
51
|
+
'timeout',
|
|
52
|
+
'aborted',
|
|
53
|
+
'refusal',
|
|
54
|
+
'truncated',
|
|
55
|
+
'invalid-response',
|
|
56
|
+
'unsupported',
|
|
57
|
+
'too-large',
|
|
58
|
+
'token-cap',
|
|
59
|
+
'review-deadline',
|
|
60
|
+
'unusable-reply',
|
|
61
|
+
'unknown',
|
|
62
|
+
]),
|
|
63
|
+
status: z.int().min(100).max(599).optional(),
|
|
64
|
+
});
|
|
44
65
|
const lensVote = z.strictObject({
|
|
45
66
|
angle: z.string(),
|
|
46
67
|
vote: vote.nullable(),
|
|
47
68
|
error: z.string().nullable(),
|
|
69
|
+
failure: voteFailure.nullable(),
|
|
48
70
|
});
|
|
49
71
|
const verdict = z.strictObject({
|
|
50
72
|
id: z.string(),
|
|
@@ -74,6 +96,33 @@ const usage = z.strictObject({
|
|
|
74
96
|
outputTokens: amount,
|
|
75
97
|
costUsd: amount.optional(),
|
|
76
98
|
});
|
|
99
|
+
const coverageReason = z.enum([
|
|
100
|
+
'binary',
|
|
101
|
+
'submodule',
|
|
102
|
+
'secret',
|
|
103
|
+
'deleted-secret',
|
|
104
|
+
'too-large',
|
|
105
|
+
'request-too-large',
|
|
106
|
+
'token-cap',
|
|
107
|
+
'review-deadline',
|
|
108
|
+
'research-failed',
|
|
109
|
+
]);
|
|
110
|
+
const leftRange = z.strictObject({
|
|
111
|
+
start: count,
|
|
112
|
+
end: count,
|
|
113
|
+
reason: coverageReason,
|
|
114
|
+
});
|
|
115
|
+
const coverageEntry = z.strictObject({
|
|
116
|
+
file: z.string(),
|
|
117
|
+
status: z.enum([
|
|
118
|
+
'reviewed',
|
|
119
|
+
'partial',
|
|
120
|
+
'unreviewed',
|
|
121
|
+
'excluded',
|
|
122
|
+
]),
|
|
123
|
+
reason: coverageReason.optional(),
|
|
124
|
+
left: z.array(leftRange).optional(),
|
|
125
|
+
});
|
|
77
126
|
const profileMeta = z.strictObject({
|
|
78
127
|
providers: z.strictObject({
|
|
79
128
|
research: providerRef,
|
|
@@ -82,21 +131,75 @@ const profileMeta = z.strictObject({
|
|
|
82
131
|
policy: thresholds,
|
|
83
132
|
gating: z.boolean(),
|
|
84
133
|
invalidFindings: count,
|
|
134
|
+
coverage: z.array(coverageEntry),
|
|
85
135
|
});
|
|
86
136
|
const incompleteEntry = z.strictObject({
|
|
87
137
|
profile: z.string(),
|
|
88
138
|
reason: z.enum([
|
|
89
139
|
'research-unusable',
|
|
140
|
+
'research-failed',
|
|
90
141
|
'findings-capped',
|
|
91
142
|
'finding-unreadable',
|
|
92
143
|
'vote-failed',
|
|
93
144
|
'request-too-large',
|
|
145
|
+
'too-large',
|
|
146
|
+
'token-cap',
|
|
147
|
+
'review-deadline',
|
|
148
|
+
'binary',
|
|
149
|
+
'submodule',
|
|
150
|
+
'secret',
|
|
94
151
|
]),
|
|
95
152
|
file: z.string().optional(),
|
|
96
153
|
detail: z.string(),
|
|
97
154
|
});
|
|
155
|
+
/** A file left out: a rule belongs to a skipped or a held file, and to none else. */
|
|
156
|
+
const leftOutFile = z.discriminatedUnion('reason', [
|
|
157
|
+
z.strictObject({
|
|
158
|
+
file: z.string(),
|
|
159
|
+
reason: z.enum(['excluded', 'binary', 'submodule']),
|
|
160
|
+
}),
|
|
161
|
+
z.strictObject({
|
|
162
|
+
file: z.string(),
|
|
163
|
+
reason: z.literal('skipped'),
|
|
164
|
+
rule: z.enum(SKIP_RULES),
|
|
165
|
+
}),
|
|
166
|
+
z.strictObject({
|
|
167
|
+
file: z.string(),
|
|
168
|
+
reason: z.enum(['secret', 'deleted-secret']),
|
|
169
|
+
rule: z.enum(SECRET_RULES),
|
|
170
|
+
}),
|
|
171
|
+
]);
|
|
172
|
+
const objectFormat = z.enum(['sha1', 'sha256']);
|
|
173
|
+
/**
|
|
174
|
+
* Where a review's diff came from. The shapes are strict, one object per kind; the ids are checked as a
|
|
175
|
+
* review checks its input's, by `sourceIssues`, so a record holds a full commit id of its object format.
|
|
176
|
+
*/
|
|
177
|
+
const reviewSource = z.discriminatedUnion('kind', [
|
|
178
|
+
z.strictObject({
|
|
179
|
+
kind: z.literal('diff'),
|
|
180
|
+
}),
|
|
181
|
+
z.strictObject({
|
|
182
|
+
kind: z.literal('range'),
|
|
183
|
+
base: z.string(),
|
|
184
|
+
mergeBase: z.string(),
|
|
185
|
+
head: z.string(),
|
|
186
|
+
objectFormat,
|
|
187
|
+
}),
|
|
188
|
+
z.strictObject({
|
|
189
|
+
kind: z.enum(['local', 'staged']),
|
|
190
|
+
head: z.string().nullable(),
|
|
191
|
+
objectFormat,
|
|
192
|
+
}),
|
|
193
|
+
z.strictObject({
|
|
194
|
+
kind: z.literal('paths'),
|
|
195
|
+
commit: z.string(),
|
|
196
|
+
paths: z.array(z.string()),
|
|
197
|
+
skips: z.boolean(),
|
|
198
|
+
objectFormat,
|
|
199
|
+
}),
|
|
200
|
+
]).refine(source => sourceIssues(source).length === 0);
|
|
98
201
|
const flag = z.strictObject({
|
|
99
|
-
code: z.enum(['hidden-characters', 'ai-addressed-text']),
|
|
202
|
+
code: z.enum(['hidden-characters', 'ai-addressed-text', 'secret-file']),
|
|
100
203
|
file: z.string().optional(),
|
|
101
204
|
line: count.optional(),
|
|
102
205
|
detail: z.string(),
|
|
@@ -114,9 +217,11 @@ export const resultSchema = z.strictObject({
|
|
|
114
217
|
version: z.string(),
|
|
115
218
|
startedAt: z.string(),
|
|
116
219
|
durationMs: count,
|
|
220
|
+
source: reviewSource,
|
|
117
221
|
profiles: z.record(z.string(), profileMeta),
|
|
118
222
|
usage: usage.optional(),
|
|
119
223
|
incomplete: z.array(incompleteEntry),
|
|
224
|
+
leftOut: z.array(leftOutFile),
|
|
120
225
|
flags: z.array(flag),
|
|
121
226
|
}),
|
|
122
227
|
});
|
package/dist/bench/runner.d.ts
CHANGED
|
@@ -44,12 +44,21 @@ export interface RunReviewsOptions {
|
|
|
44
44
|
readonly events: (event: RunnerEvent) => void;
|
|
45
45
|
/** The most a line may hold, its line break not counted, in bytes; `MAX_RECORD_BYTES` when left out. */
|
|
46
46
|
readonly maxRecordBytes?: number;
|
|
47
|
+
/**
|
|
48
|
+
* Once the run's reviews have spent this many tokens, as each review's cap counts them, no review starts.
|
|
49
|
+
* No cap when left out.
|
|
50
|
+
*/
|
|
51
|
+
readonly maxRunTokens?: number;
|
|
47
52
|
}
|
|
48
53
|
export interface RunReviewsOutcome {
|
|
49
54
|
/** The reviews written to the record. */
|
|
50
55
|
readonly recorded: number;
|
|
51
56
|
/** The reviews that failed, a rejection that is no denial, or whose record was too large to write. */
|
|
52
57
|
readonly failed: number;
|
|
58
|
+
/** The reviews that did not start because the run had spent `maxRunTokens`. */
|
|
59
|
+
readonly notRun: number;
|
|
60
|
+
/** The tokens the run's `call` events counted against their reviews' caps, as `maxRunTokens` reads them. */
|
|
61
|
+
readonly spent: number;
|
|
53
62
|
/** Whether the caller's signal was aborted by the time the run ended. */
|
|
54
63
|
readonly interrupted: boolean;
|
|
55
64
|
}
|
|
@@ -68,6 +77,8 @@ export declare function planReviews(options: {
|
|
|
68
77
|
* other rejection (an HTTP, network, timeout or process failure, a reply the server shaped wrong) says
|
|
69
78
|
* nothing of the model's answer, so the review fails and runs again. The classes are matched with
|
|
70
79
|
* `instanceof`: an error from a second copy of quorum is no denial, so it fails, which fails closed.
|
|
80
|
+
* Only a gating profile's failed research rejects a review: an advisory profile's, the call or its
|
|
81
|
+
* reply, resolves it with a `research-failed` gap, recorded with its result, neither failed nor denied.
|
|
71
82
|
*/
|
|
72
83
|
export declare function denialOf(error: unknown): Denial | undefined;
|
|
73
84
|
/**
|
|
@@ -87,6 +98,13 @@ export declare function denialOf(error: unknown): Denial | undefined;
|
|
|
87
98
|
* nothing for that review, cuts the others, and the run rejects with the error of the write once every
|
|
88
99
|
* review has settled. A callback of `events` that throws does the same.
|
|
89
100
|
*
|
|
101
|
+
* The run counts what each review's cap counts: the `spent` of every `call` event of every review, one that
|
|
102
|
+
* failed or was denied included. Before a review starts, and after the abort's check, the run checks that
|
|
103
|
+
* the total is below `maxRunTokens`: once it is not, the review does not start and counts as `notRun`, the
|
|
104
|
+
* reviews in flight finish and are recorded, and the outcome says what was spent. The check is made before
|
|
105
|
+
* each review starts, so the reviews in flight can pass the cap by up to `jobs` reviews' own caps. The calls
|
|
106
|
+
* a failing review's own abort cut send no event, so the count misses them.
|
|
107
|
+
*
|
|
90
108
|
* @throws the error of the first `append` (or `events`) that threw
|
|
91
109
|
*/
|
|
92
110
|
export declare function runReviews(options: RunReviewsOptions): Promise<RunReviewsOutcome>;
|
package/dist/bench/runner.js
CHANGED
|
@@ -66,6 +66,8 @@ function tallyCalls() {
|
|
|
66
66
|
* other rejection (an HTTP, network, timeout or process failure, a reply the server shaped wrong) says
|
|
67
67
|
* nothing of the model's answer, so the review fails and runs again. The classes are matched with
|
|
68
68
|
* `instanceof`: an error from a second copy of quorum is no denial, so it fails, which fails closed.
|
|
69
|
+
* Only a gating profile's failed research rejects a review: an advisory profile's, the call or its
|
|
70
|
+
* reply, resolves it with a `research-failed` gap, recorded with its result, neither failed nor denied.
|
|
69
71
|
*/
|
|
70
72
|
export function denialOf(error) {
|
|
71
73
|
if (error instanceof OutputError)
|
|
@@ -91,10 +93,17 @@ export function denialOf(error) {
|
|
|
91
93
|
* nothing for that review, cuts the others, and the run rejects with the error of the write once every
|
|
92
94
|
* review has settled. A callback of `events` that throws does the same.
|
|
93
95
|
*
|
|
96
|
+
* The run counts what each review's cap counts: the `spent` of every `call` event of every review, one that
|
|
97
|
+
* failed or was denied included. Before a review starts, and after the abort's check, the run checks that
|
|
98
|
+
* the total is below `maxRunTokens`: once it is not, the review does not start and counts as `notRun`, the
|
|
99
|
+
* reviews in flight finish and are recorded, and the outcome says what was spent. The check is made before
|
|
100
|
+
* each review starts, so the reviews in flight can pass the cap by up to `jobs` reviews' own caps. The calls
|
|
101
|
+
* a failing review's own abort cut send no event, so the count misses them.
|
|
102
|
+
*
|
|
94
103
|
* @throws the error of the first `append` (or `events`) that threw
|
|
95
104
|
*/
|
|
96
105
|
export async function runReviews(options) {
|
|
97
|
-
const { suite, configurations, reviews, jobs, env, append, events, maxRecordBytes = MAX_RECORD_BYTES } = options;
|
|
106
|
+
const { suite, configurations, reviews, jobs, env, append, events, maxRecordBytes = MAX_RECORD_BYTES, maxRunTokens = Infinity } = options;
|
|
98
107
|
const own = new AbortController();
|
|
99
108
|
const signal = options.signal === undefined
|
|
100
109
|
? own.signal
|
|
@@ -105,6 +114,8 @@ export async function runReviews(options) {
|
|
|
105
114
|
]));
|
|
106
115
|
let recorded = 0;
|
|
107
116
|
let failed = 0;
|
|
117
|
+
let notRun = 0;
|
|
118
|
+
let spent = 0;
|
|
108
119
|
let stopped;
|
|
109
120
|
const stop = (error) => {
|
|
110
121
|
stopped ??= { error };
|
|
@@ -128,7 +139,12 @@ export async function runReviews(options) {
|
|
|
128
139
|
if (configuration === undefined || diff === undefined)
|
|
129
140
|
throw new Error(UNKNOWN_REVIEW);
|
|
130
141
|
const tally = tallyCalls();
|
|
131
|
-
const
|
|
142
|
+
const sink = (event) => {
|
|
143
|
+
if (event.type === 'call')
|
|
144
|
+
spent += event.spent;
|
|
145
|
+
tally.sink(event);
|
|
146
|
+
};
|
|
147
|
+
const quorum = await fromConfig(configuration.config, { env, events: sink });
|
|
132
148
|
const started = performance.now();
|
|
133
149
|
const facts = () => ({
|
|
134
150
|
fingerprint: configuration.fingerprint,
|
|
@@ -139,7 +155,7 @@ export async function runReviews(options) {
|
|
|
139
155
|
});
|
|
140
156
|
let result;
|
|
141
157
|
try {
|
|
142
|
-
result = await quorum.review({ diff }, { signal });
|
|
158
|
+
result = await quorum.review({ diff, send: configuration.config.send }, { signal });
|
|
143
159
|
}
|
|
144
160
|
catch (error) {
|
|
145
161
|
const denied = signal.aborted
|
|
@@ -160,6 +176,10 @@ export async function runReviews(options) {
|
|
|
160
176
|
async function runOne(review) {
|
|
161
177
|
if (signal.aborted)
|
|
162
178
|
return;
|
|
179
|
+
if (spent >= maxRunTokens) {
|
|
180
|
+
notRun += 1;
|
|
181
|
+
return;
|
|
182
|
+
}
|
|
163
183
|
let record;
|
|
164
184
|
let line;
|
|
165
185
|
try {
|
|
@@ -191,5 +211,11 @@ export async function runReviews(options) {
|
|
|
191
211
|
await Promise.all(reviews.map(review => limit(() => runOne(review))));
|
|
192
212
|
if (stopped !== undefined)
|
|
193
213
|
throw stopped.error;
|
|
194
|
-
return {
|
|
214
|
+
return {
|
|
215
|
+
recorded,
|
|
216
|
+
failed,
|
|
217
|
+
notRun,
|
|
218
|
+
spent,
|
|
219
|
+
interrupted: options.signal?.aborted === true,
|
|
220
|
+
};
|
|
195
221
|
}
|
package/dist/cli/bench.d.ts
CHANGED
|
@@ -31,12 +31,19 @@ export declare const BENCH_USAGE: string;
|
|
|
31
31
|
* `results.jsonl` to append, so a cut last line stays, and warns of it. It refuses a run directory that
|
|
32
32
|
* holds no run, and `--retry-incomplete`, which runs reviews.
|
|
33
33
|
*
|
|
34
|
+
* `--max-run-tokens <n>` stops the run from starting reviews once its reviews have spent `n` tokens, counted
|
|
35
|
+
* from each call's `spent`, the tokens its review's own cap counted (`maxReviewTokens`): the reviews in flight
|
|
36
|
+
* finish and are recorded, the reports are written, one stderr line says how many reviews were not run, and
|
|
37
|
+
* the next run runs them. The check is made before each review starts, so the reviews in flight can pass
|
|
38
|
+
* the cap by up to `--jobs` reviews' own caps, and the calls a failing review's abort cut are not counted.
|
|
39
|
+
*
|
|
34
40
|
* Exit codes: 0 when every planned review is recorded, a denied one included, or `--report-only` wrote the
|
|
35
41
|
* reports, 2 for a usage, suite, configuration or run directory error, 3 when the run stops (a file cannot
|
|
36
|
-
* be written or read), 4 when a review failed (its research call failed, or its record
|
|
37
|
-
* write: the others are recorded, and the next run runs the failed ones)
|
|
38
|
-
* reports are written on 0, 4 and 130,
|
|
39
|
-
*
|
|
42
|
+
* be written or read), 4 when a review failed (its gating profile's research call failed, or its record
|
|
43
|
+
* was too large to write: the others are recorded, and the next run runs the failed ones) or
|
|
44
|
+
* `--max-run-tokens` left reviews unrun, 130 when interrupted. The reports are written on 0, 4 and 130,
|
|
45
|
+
* so a stopped run shows what it recorded; an interrupt during the checks and the plan leaves the run
|
|
46
|
+
* directory as it was, even one that does not exist yet.
|
|
40
47
|
*
|
|
41
48
|
* Every line it writes is inert (invariant I3).
|
|
42
49
|
*/
|