@tobi-or-not/quorum 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +3 -1
  2. package/dist/bench/fingerprint.d.ts +8 -5
  3. package/dist/bench/fingerprint.js +28 -9
  4. package/dist/bench/records.d.ts +168 -0
  5. package/dist/bench/records.js +106 -1
  6. package/dist/bench/runner.d.ts +18 -0
  7. package/dist/bench/runner.js +30 -4
  8. package/dist/cli/bench.d.ts +11 -4
  9. package/dist/cli/bench.js +34 -17
  10. package/dist/cli/bin.js +2 -2
  11. package/dist/cli/io.d.ts +5 -0
  12. package/dist/cli/main.d.ts +67 -24
  13. package/dist/cli/main.js +288 -59
  14. package/dist/cli/print-config.d.ts +8 -6
  15. package/dist/cli/print-config.js +26 -8
  16. package/dist/cli/render.d.ts +9 -3
  17. package/dist/cli/render.js +43 -2
  18. package/dist/config/claude-code.d.ts +2 -6
  19. package/dist/config/claude-code.js +6 -44
  20. package/dist/config/command.d.ts +10 -0
  21. package/dist/config/command.js +47 -0
  22. package/dist/config/env.d.ts +32 -19
  23. package/dist/config/env.js +95 -32
  24. package/dist/config/from-config.js +21 -3
  25. package/dist/config/resolve.d.ts +13 -1
  26. package/dist/config/resolve.js +27 -1
  27. package/dist/config/schema.d.ts +152 -5
  28. package/dist/config/schema.js +156 -12
  29. package/dist/core/clock.d.ts +17 -0
  30. package/dist/core/clock.js +14 -0
  31. package/dist/core/coverage.d.ts +74 -0
  32. package/dist/core/coverage.js +221 -0
  33. package/dist/core/diff.d.ts +101 -0
  34. package/dist/core/diff.js +178 -35
  35. package/dist/core/errors.d.ts +12 -3
  36. package/dist/core/errors.js +21 -2
  37. package/dist/core/estimate.d.ts +45 -0
  38. package/dist/core/estimate.js +188 -0
  39. package/dist/core/exclude.d.ts +42 -0
  40. package/dist/core/exclude.js +189 -0
  41. package/dist/core/flags.d.ts +14 -1
  42. package/dist/core/flags.js +56 -0
  43. package/dist/core/inputs.d.ts +38 -0
  44. package/dist/core/inputs.js +228 -0
  45. package/dist/core/limits.d.ts +83 -0
  46. package/dist/core/limits.js +169 -0
  47. package/dist/core/output.d.ts +3 -2
  48. package/dist/core/output.js +3 -2
  49. package/dist/core/parts.d.ts +146 -0
  50. package/dist/core/parts.js +388 -0
  51. package/dist/core/ports.d.ts +67 -2
  52. package/dist/core/ports.js +36 -1
  53. package/dist/core/prompts.d.ts +22 -3
  54. package/dist/core/prompts.js +43 -7
  55. package/dist/core/quorum.d.ts +26 -8
  56. package/dist/core/quorum.js +310 -73
  57. package/dist/core/retry.d.ts +36 -0
  58. package/dist/core/retry.js +46 -0
  59. package/dist/core/secrets.d.ts +98 -0
  60. package/dist/core/secrets.js +507 -0
  61. package/dist/core/stages/context.d.ts +131 -23
  62. package/dist/core/stages/context.js +331 -46
  63. package/dist/core/stages/panel.d.ts +13 -6
  64. package/dist/core/stages/panel.js +79 -19
  65. package/dist/core/stages/research.d.ts +110 -25
  66. package/dist/core/stages/research.js +338 -44
  67. package/dist/core/tokens.d.ts +31 -0
  68. package/dist/core/tokens.js +81 -0
  69. package/dist/core/types.d.ts +280 -25
  70. package/dist/core/verify.d.ts +6 -0
  71. package/dist/core/verify.js +6 -3
  72. package/dist/git/run.d.ts +223 -0
  73. package/dist/git/run.js +608 -0
  74. package/dist/git/tree.d.ts +43 -0
  75. package/dist/git/tree.js +171 -0
  76. package/dist/index.d.ts +26 -2
  77. package/dist/index.js +34 -2
  78. package/dist/providers/anthropic.d.ts +3 -1
  79. package/dist/providers/anthropic.js +10 -3
  80. package/dist/providers/claude-code.d.ts +56 -11
  81. package/dist/providers/claude-code.js +70 -25
  82. package/dist/providers/gemini.d.ts +5 -1
  83. package/dist/providers/gemini.js +65 -11
  84. package/dist/providers/http.d.ts +49 -4
  85. package/dist/providers/http.js +155 -14
  86. package/dist/providers/ollama.d.ts +4 -1
  87. package/dist/providers/ollama.js +13 -5
  88. package/dist/providers/openai-compatible.d.ts +4 -1
  89. package/dist/providers/openai-compatible.js +13 -5
  90. package/dist/providers/options.d.ts +12 -3
  91. package/dist/providers/options.js +30 -10
  92. package/dist/providers/size.d.ts +19 -15
  93. package/dist/providers/size.js +33 -73
  94. package/dist/sources/added.d.ts +49 -0
  95. package/dist/sources/added.js +134 -0
  96. package/dist/sources/diff.d.ts +13 -0
  97. package/dist/sources/diff.js +63 -0
  98. package/dist/sources/local.d.ts +41 -0
  99. package/dist/sources/local.js +251 -0
  100. package/dist/sources/paths.d.ts +45 -0
  101. package/dist/sources/paths.js +230 -0
  102. package/dist/sources/range.d.ts +21 -0
  103. package/dist/sources/range.js +75 -0
  104. package/dist/sources/read.d.ts +69 -0
  105. package/dist/sources/read.js +37 -0
  106. package/dist/sources/repository.d.ts +9 -0
  107. package/dist/sources/repository.js +28 -0
  108. package/dist/sources/skips.d.ts +49 -0
  109. package/dist/sources/skips.js +126 -0
  110. package/package.json +1 -1
  111. package/schema/quorum.schema.json +49 -1
package/README.md CHANGED
@@ -43,6 +43,7 @@ export QUORUM_RESEARCH_MODEL=qwen3.6:27b # a model your local Ollama serves
43
43
  export QUORUM_RESEARCH_CONTEXT_TOKENS=32768 # its context window, in tokens
44
44
  quorum ./my.diff > result.json # review a diff file
45
45
  git diff origin/main...HEAD | quorum > result.json # or what your branch changes
46
+ quorum --range origin/main...HEAD > result.json # what your branch changes, git run by quorum
46
47
  ```
47
48
 
48
49
  `QUORUM_RESEARCH_CONTEXT_TOKENS` is the model's context window in tokens, which quorum checks every request against, and without it the run exits `2` with a message that names it. With no provider configured and no base URL, preset or protocol variable, quorum talks to the Ollama on this machine through its native API, `http://localhost:11434`, which Ollama listens on, on `127.0.0.1`, by default. That API takes `num_ctx`, `truncate: false` and `shift: false`, so quorum tells the server the context size and the server refuses an overflow, where Ollama's OpenAI-compatible `/v1` would cut the prompt in silence. Refusing needs Ollama 0.12.6 or later: an older server ignores `truncate` and `shift`. A base URL variable set on its own means an OpenAI-compatible server, as before. Keep the number at most what `ollama show <model>` reports for the model, since Ollama clamps a larger `num_ctx` to it, and read the context a running model loaded in the `CONTEXT` column of `ollama ps`.
@@ -58,7 +59,7 @@ quorum ./my.diff > result.json
58
59
 
59
60
  Requires Claude Code 2.1.311 or newer. [`claudeCode`](./docs/20-using/30-providers.md#claudecode) has the command line, the isolation and the auth modes, and [Claude Code](./docs/20-using/20-configuration.md#claude-code) the setup, in CI too.
60
61
 
61
- The JSON result goes to stdout, and the exit code says what to do with it: `0` for a complete review in which no gating profile kept a `high` finding, `1` when one did, complete or not, `2` for a usage or configuration error, `3` when the research call failed, `4` when a gating profile's review is incomplete otherwise, and `130` when the run was interrupted. [Quick start](./docs/10-getting-started/20-quickstart.md) walks one review through, and [Start here](./docs/01-readme.md) lays out four tracks: review a diff, use it from code or review something else, know what it protects against, and measure configurations.
62
+ The JSON result goes to stdout, and the exit code says what to do with it: `0` for a complete review in which no gating profile kept a `high` finding, `1` when one did, complete or not, `2` for a usage or configuration error, `3` when a gating profile's research call failed, `4` when a gating profile's review is incomplete otherwise, and `130` when the run was interrupted. [Quick start](./docs/10-getting-started/20-quickstart.md) walks one review through, and [Start here](./docs/01-readme.md) lays out four tracks: review a diff, use it from code or review something else, know what it protects against, and measure configurations.
62
63
 
63
64
  ## Documentation
64
65
 
@@ -72,6 +73,7 @@ The JSON result goes to stdout, and the exit code says what to do with it: `0` f
72
73
 
73
74
  *Using it*
74
75
  - [The CLI](./docs/20-using/10-cli.md) *- The invocation, stdout and stderr, every exit code and its causes, and gating a CI job on them*
76
+ - [Sources](./docs/20-using/15-sources.md) *- A diff file or stdin, a range of commits, the local changes or the files at a commit, and how quorum runs git*
75
77
  - [Configuration](./docs/20-using/20-configuration.md) *- The config file and its trust model in CI, the `QUORUM_*` variables and the voter's fallback, the context size each provider states, running Claude Code, the keys, `--print-config`, and the errors a bad setting causes*
76
78
  - [Providers](./docs/20-using/30-providers.md) *- The provider port, the shipped adapters and their options, Claude Code run isolated as a provider, the presets, `ProviderError`, and writing a provider of your own*
77
79
  - [Profiles](./docs/20-using/40-profiles.md) *- The security profile, writing your own, several in one review, gating and guidelines*
@@ -6,11 +6,14 @@
6
6
  * The form holds every setting a review uses but the key: the profiles that run, in run order, each with
7
7
  * its id, prompts, lenses, policy and gating, and its research provider and each lens's provider written
8
8
  * out in place of their names; every field of a provider defined as data but its key, the values of
9
- * `headers`, `query` and `extraBody` and the whole base URL included; a provider object as its config key
10
- * and its model; `concurrency` and `maxFindings` as the review applies them, defaults filled in. It holds
11
- * no config-file path, no warning and no provider name, so the same settings under other names are one
12
- * configuration, and a rotated key keeps a run. A `claude-code` provider's resolved command path is one
13
- * of its settings, so it is in the form.
9
+ * `headers`, `query` and `extraBody` and the whole base URL included, `timeoutMs` and `retries` among them;
10
+ * a provider object as its config key, its model and the `timeoutMs` and `retries` it states, so one that
11
+ * states none keeps the form it had, and never the `worstCase` or the `room` it states, which are code like its
12
+ * `complete`; `concurrency`, `maxFindings`, `partTokens`, `maxReviewTokens` and `reviewTimeoutMs` as the review applies
13
+ * them, defaults filled in, `exclude` when the config lists a pattern, and `send` when it lists a file. It
14
+ * holds no config-file path, no warning and no provider name, so the same settings under other names are one
15
+ * configuration, and a rotated key keeps a run. A `claude-code` provider's resolved command path is one of its
16
+ * settings, so it is in the form.
14
17
  *
15
18
  * The form can hold a secret (a credential kept in a header, a query parameter or the base path), so it
16
19
  * never leaves this module: not written, not logged, not put in an error message. Only the HMAC does.
@@ -6,11 +6,14 @@
6
6
  * The form holds every setting a review uses but the key: the profiles that run, in run order, each with
7
7
  * its id, prompts, lenses, policy and gating, and its research provider and each lens's provider written
8
8
  * out in place of their names; every field of a provider defined as data but its key, the values of
9
- * `headers`, `query` and `extraBody` and the whole base URL included; a provider object as its config key
10
- * and its model; `concurrency` and `maxFindings` as the review applies them, defaults filled in. It holds
11
- * no config-file path, no warning and no provider name, so the same settings under other names are one
12
- * configuration, and a rotated key keeps a run. A `claude-code` provider's resolved command path is one
13
- * of its settings, so it is in the form.
9
+ * `headers`, `query` and `extraBody` and the whole base URL included, `timeoutMs` and `retries` among them;
10
+ * a provider object as its config key, its model and the `timeoutMs` and `retries` it states, so one that
11
+ * states none keeps the form it had, and never the `worstCase` or the `room` it states, which are code like its
12
+ * `complete`; `concurrency`, `maxFindings`, `partTokens`, `maxReviewTokens` and `reviewTimeoutMs` as the review applies
13
+ * them, defaults filled in, `exclude` when the config lists a pattern, and `send` when it lists a file. It
14
+ * holds no config-file path, no warning and no provider name, so the same settings under other names are one
15
+ * configuration, and a rotated key keeps a run. A `claude-code` provider's resolved command path is one of its
16
+ * settings, so it is in the form.
14
17
  *
15
18
  * The form can hold a secret (a credential kept in a header, a query parameter or the base path), so it
16
19
  * never leaves this module: not written, not logged, not put in an error message. Only the HMAC does.
@@ -20,18 +23,25 @@
20
23
  */
21
24
  import { createHmac, randomBytes } from 'node:crypto';
22
25
  import { isPlainObject } from "../config/load.js";
23
- import { isProvider } from "../config/schema.js";
26
+ import { isProvider, statedFacts } from "../config/schema.js";
24
27
  import { DEFAULT_CONCURRENCY } from "../core/concurrency.js";
28
+ import { DEFAULT_MAX_REVIEW_TOKENS, DEFAULT_REVIEW_TIMEOUT_MS } from "../core/limits.js";
29
+ import { DEFAULT_PART_TOKENS } from "../core/parts.js";
25
30
  import { DEFAULT_MAX_FINDINGS } from "../core/stages/research.js";
26
31
  const SALT = /^[0-9a-f]{64}$/;
27
32
  /** A fresh salt: 32 random bytes as 64 lowercase hex digits. */
28
33
  export function newSalt() {
29
34
  return randomBytes(32).toString('hex');
30
35
  }
31
- /** A provider as the form holds it: a provider object by its config key and model, a spec by its fields less the key. */
36
+ /** A provider as the form holds it: a provider object by its config key, its model and the facts it states, a spec by its fields less the key. */
32
37
  function providerForm(provider) {
33
- if (isProvider(provider))
34
- return { object: provider.id, model: provider.model };
38
+ if (isProvider(provider)) {
39
+ return {
40
+ object: provider.id,
41
+ model: provider.model,
42
+ ...statedFacts(provider).stated,
43
+ };
44
+ }
35
45
  return Object.fromEntries(Object.entries(provider).filter(([field]) => field !== 'apiKey'));
36
46
  }
37
47
  /** The form of the provider a profile binds by `name`; a name the config holds no provider under is a RangeError, never a silent gap. */
@@ -64,6 +74,15 @@ export function canonicalSettings(config) {
64
74
  return JSON.stringify({
65
75
  concurrency: config.concurrency ?? DEFAULT_CONCURRENCY,
66
76
  maxFindings: config.maxFindings ?? DEFAULT_MAX_FINDINGS,
77
+ partTokens: config.partTokens ?? DEFAULT_PART_TOKENS,
78
+ maxReviewTokens: config.maxReviewTokens ?? DEFAULT_MAX_REVIEW_TOKENS,
79
+ reviewTimeoutMs: config.reviewTimeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS,
80
+ ...config.exclude === undefined
81
+ ? {}
82
+ : { exclude: config.exclude },
83
+ ...config.send === undefined
84
+ ? {}
85
+ : { send: config.send },
67
86
  profiles,
68
87
  }, sortKeys);
69
88
  }
@@ -106,6 +106,25 @@ export declare const resultSchema: z.ZodObject<{
106
106
  reasoning: z.ZodString;
107
107
  }, z.core.$strict>>;
108
108
  error: z.ZodNullable<z.ZodString>;
109
+ failure: z.ZodNullable<z.ZodObject<{
110
+ kind: z.ZodEnum<{
111
+ http: "http";
112
+ network: "network";
113
+ process: "process";
114
+ timeout: "timeout";
115
+ aborted: "aborted";
116
+ refusal: "refusal";
117
+ truncated: "truncated";
118
+ "invalid-response": "invalid-response";
119
+ unsupported: "unsupported";
120
+ "too-large": "too-large";
121
+ "token-cap": "token-cap";
122
+ "review-deadline": "review-deadline";
123
+ "unusable-reply": "unusable-reply";
124
+ unknown: "unknown";
125
+ }>;
126
+ status: z.ZodOptional<z.ZodInt>;
127
+ }, z.core.$strict>>;
109
128
  }, z.core.$strict>>;
110
129
  }, z.core.$strict>>;
111
130
  discarded: z.ZodArray<z.ZodObject<{
@@ -131,6 +150,25 @@ export declare const resultSchema: z.ZodObject<{
131
150
  reasoning: z.ZodString;
132
151
  }, z.core.$strict>>;
133
152
  error: z.ZodNullable<z.ZodString>;
153
+ failure: z.ZodNullable<z.ZodObject<{
154
+ kind: z.ZodEnum<{
155
+ http: "http";
156
+ network: "network";
157
+ process: "process";
158
+ timeout: "timeout";
159
+ aborted: "aborted";
160
+ refusal: "refusal";
161
+ truncated: "truncated";
162
+ "invalid-response": "invalid-response";
163
+ unsupported: "unsupported";
164
+ "too-large": "too-large";
165
+ "token-cap": "token-cap";
166
+ "review-deadline": "review-deadline";
167
+ "unusable-reply": "unusable-reply";
168
+ unknown: "unknown";
169
+ }>;
170
+ status: z.ZodOptional<z.ZodInt>;
171
+ }, z.core.$strict>>;
134
172
  }, z.core.$strict>>;
135
173
  }, z.core.$strict>>;
136
174
  unreviewed: z.ZodArray<z.ZodObject<{
@@ -156,12 +194,62 @@ export declare const resultSchema: z.ZodObject<{
156
194
  reasoning: z.ZodString;
157
195
  }, z.core.$strict>>;
158
196
  error: z.ZodNullable<z.ZodString>;
197
+ failure: z.ZodNullable<z.ZodObject<{
198
+ kind: z.ZodEnum<{
199
+ http: "http";
200
+ network: "network";
201
+ process: "process";
202
+ timeout: "timeout";
203
+ aborted: "aborted";
204
+ refusal: "refusal";
205
+ truncated: "truncated";
206
+ "invalid-response": "invalid-response";
207
+ unsupported: "unsupported";
208
+ "too-large": "too-large";
209
+ "token-cap": "token-cap";
210
+ "review-deadline": "review-deadline";
211
+ "unusable-reply": "unusable-reply";
212
+ unknown: "unknown";
213
+ }>;
214
+ status: z.ZodOptional<z.ZodInt>;
215
+ }, z.core.$strict>>;
159
216
  }, z.core.$strict>>;
160
217
  }, z.core.$strict>>;
161
218
  meta: z.ZodObject<{
162
219
  version: z.ZodString;
163
220
  startedAt: z.ZodString;
164
221
  durationMs: z.ZodInt;
222
+ source: z.ZodDiscriminatedUnion<[z.ZodObject<{
223
+ kind: z.ZodLiteral<"diff">;
224
+ }, z.core.$strict>, z.ZodObject<{
225
+ kind: z.ZodLiteral<"range">;
226
+ base: z.ZodString;
227
+ mergeBase: z.ZodString;
228
+ head: z.ZodString;
229
+ objectFormat: z.ZodEnum<{
230
+ sha1: "sha1";
231
+ sha256: "sha256";
232
+ }>;
233
+ }, z.core.$strict>, z.ZodObject<{
234
+ kind: z.ZodEnum<{
235
+ local: "local";
236
+ staged: "staged";
237
+ }>;
238
+ head: z.ZodNullable<z.ZodString>;
239
+ objectFormat: z.ZodEnum<{
240
+ sha1: "sha1";
241
+ sha256: "sha256";
242
+ }>;
243
+ }, z.core.$strict>, z.ZodObject<{
244
+ kind: z.ZodLiteral<"paths">;
245
+ commit: z.ZodString;
246
+ paths: z.ZodArray<z.ZodString>;
247
+ skips: z.ZodBoolean;
248
+ objectFormat: z.ZodEnum<{
249
+ sha1: "sha1";
250
+ sha256: "sha256";
251
+ }>;
252
+ }, z.core.$strict>], "kind">;
165
253
  profiles: z.ZodRecord<z.ZodString, z.ZodObject<{
166
254
  providers: z.ZodObject<{
167
255
  research: z.ZodObject<{
@@ -180,6 +268,41 @@ export declare const resultSchema: z.ZodObject<{
180
268
  }, z.core.$strict>;
181
269
  gating: z.ZodBoolean;
182
270
  invalidFindings: z.ZodInt;
271
+ coverage: z.ZodArray<z.ZodObject<{
272
+ file: z.ZodString;
273
+ status: z.ZodEnum<{
274
+ unreviewed: "unreviewed";
275
+ excluded: "excluded";
276
+ reviewed: "reviewed";
277
+ partial: "partial";
278
+ }>;
279
+ reason: z.ZodOptional<z.ZodEnum<{
280
+ "too-large": "too-large";
281
+ "token-cap": "token-cap";
282
+ "review-deadline": "review-deadline";
283
+ "research-failed": "research-failed";
284
+ "request-too-large": "request-too-large";
285
+ binary: "binary";
286
+ submodule: "submodule";
287
+ secret: "secret";
288
+ "deleted-secret": "deleted-secret";
289
+ }>>;
290
+ left: z.ZodOptional<z.ZodArray<z.ZodObject<{
291
+ start: z.ZodInt;
292
+ end: z.ZodInt;
293
+ reason: z.ZodEnum<{
294
+ "too-large": "too-large";
295
+ "token-cap": "token-cap";
296
+ "review-deadline": "review-deadline";
297
+ "research-failed": "research-failed";
298
+ "request-too-large": "request-too-large";
299
+ binary: "binary";
300
+ submodule: "submodule";
301
+ secret: "secret";
302
+ "deleted-secret": "deleted-secret";
303
+ }>;
304
+ }, z.core.$strict>>>;
305
+ }, z.core.$strict>>;
183
306
  }, z.core.$strict>>;
184
307
  usage: z.ZodOptional<z.ZodObject<{
185
308
  inputTokens: z.ZodNumber;
@@ -189,19 +312,64 @@ export declare const resultSchema: z.ZodObject<{
189
312
  incomplete: z.ZodArray<z.ZodObject<{
190
313
  profile: z.ZodString;
191
314
  reason: z.ZodEnum<{
315
+ "too-large": "too-large";
316
+ "token-cap": "token-cap";
317
+ "review-deadline": "review-deadline";
192
318
  "research-unusable": "research-unusable";
319
+ "research-failed": "research-failed";
193
320
  "findings-capped": "findings-capped";
194
321
  "finding-unreadable": "finding-unreadable";
195
322
  "vote-failed": "vote-failed";
196
323
  "request-too-large": "request-too-large";
324
+ binary: "binary";
325
+ submodule: "submodule";
326
+ secret: "secret";
197
327
  }>;
198
328
  file: z.ZodOptional<z.ZodString>;
199
329
  detail: z.ZodString;
200
330
  }, z.core.$strict>>;
331
+ leftOut: z.ZodArray<z.ZodDiscriminatedUnion<[z.ZodObject<{
332
+ file: z.ZodString;
333
+ reason: z.ZodEnum<{
334
+ binary: "binary";
335
+ submodule: "submodule";
336
+ excluded: "excluded";
337
+ }>;
338
+ }, z.core.$strict>, z.ZodObject<{
339
+ file: z.ZodString;
340
+ reason: z.ZodLiteral<"skipped">;
341
+ rule: z.ZodEnum<{
342
+ binary: "binary";
343
+ submodule: "submodule";
344
+ secret: "secret";
345
+ vendored: "vendored";
346
+ lockfile: "lockfile";
347
+ minified: "minified";
348
+ lfs: "lfs";
349
+ }>;
350
+ }, z.core.$strict>, z.ZodObject<{
351
+ file: z.ZodString;
352
+ reason: z.ZodEnum<{
353
+ secret: "secret";
354
+ "deleted-secret": "deleted-secret";
355
+ }>;
356
+ rule: z.ZodEnum<{
357
+ "private-key": "private-key";
358
+ "aws-key": "aws-key";
359
+ "github-token": "github-token";
360
+ "slack-token": "slack-token";
361
+ "google-key": "google-key";
362
+ "stripe-key": "stripe-key";
363
+ "anthropic-key": "anthropic-key";
364
+ "openai-key": "openai-key";
365
+ name: "name";
366
+ }>;
367
+ }, z.core.$strict>], "reason">>;
201
368
  flags: z.ZodArray<z.ZodObject<{
202
369
  code: z.ZodEnum<{
203
370
  "hidden-characters": "hidden-characters";
204
371
  "ai-addressed-text": "ai-addressed-text";
372
+ "secret-file": "secret-file";
205
373
  }>;
206
374
  file: z.ZodOptional<z.ZodString>;
207
375
  line: z.ZodOptional<z.ZodInt>;
@@ -14,7 +14,9 @@
14
14
  * that a model's reply denied, which denial it was, never the reply.
15
15
  */
16
16
  import { z } from 'zod';
17
+ import { SKIP_RULES, sourceIssues } from "../core/inputs.js";
17
18
  import { inertJsonLine } from "../core/sanitize.js";
19
+ import { SECRET_RULES } from "../core/secrets.js";
18
20
  import { MAX_TRIALS, SUITE_ID } from "./suite.js";
19
21
  /** The `format` of the run file this quorum writes and reads. */
20
22
  export const RUN_FORMAT = 2;
@@ -41,10 +43,30 @@ const vote = z.strictObject({
41
43
  keep: z.boolean(),
42
44
  reasoning: z.string(),
43
45
  });
46
+ const voteFailure = z.strictObject({
47
+ kind: z.enum([
48
+ 'http',
49
+ 'network',
50
+ 'process',
51
+ 'timeout',
52
+ 'aborted',
53
+ 'refusal',
54
+ 'truncated',
55
+ 'invalid-response',
56
+ 'unsupported',
57
+ 'too-large',
58
+ 'token-cap',
59
+ 'review-deadline',
60
+ 'unusable-reply',
61
+ 'unknown',
62
+ ]),
63
+ status: z.int().min(100).max(599).optional(),
64
+ });
44
65
  const lensVote = z.strictObject({
45
66
  angle: z.string(),
46
67
  vote: vote.nullable(),
47
68
  error: z.string().nullable(),
69
+ failure: voteFailure.nullable(),
48
70
  });
49
71
  const verdict = z.strictObject({
50
72
  id: z.string(),
@@ -74,6 +96,33 @@ const usage = z.strictObject({
74
96
  outputTokens: amount,
75
97
  costUsd: amount.optional(),
76
98
  });
99
+ const coverageReason = z.enum([
100
+ 'binary',
101
+ 'submodule',
102
+ 'secret',
103
+ 'deleted-secret',
104
+ 'too-large',
105
+ 'request-too-large',
106
+ 'token-cap',
107
+ 'review-deadline',
108
+ 'research-failed',
109
+ ]);
110
+ const leftRange = z.strictObject({
111
+ start: count,
112
+ end: count,
113
+ reason: coverageReason,
114
+ });
115
+ const coverageEntry = z.strictObject({
116
+ file: z.string(),
117
+ status: z.enum([
118
+ 'reviewed',
119
+ 'partial',
120
+ 'unreviewed',
121
+ 'excluded',
122
+ ]),
123
+ reason: coverageReason.optional(),
124
+ left: z.array(leftRange).optional(),
125
+ });
77
126
  const profileMeta = z.strictObject({
78
127
  providers: z.strictObject({
79
128
  research: providerRef,
@@ -82,21 +131,75 @@ const profileMeta = z.strictObject({
82
131
  policy: thresholds,
83
132
  gating: z.boolean(),
84
133
  invalidFindings: count,
134
+ coverage: z.array(coverageEntry),
85
135
  });
86
136
  const incompleteEntry = z.strictObject({
87
137
  profile: z.string(),
88
138
  reason: z.enum([
89
139
  'research-unusable',
140
+ 'research-failed',
90
141
  'findings-capped',
91
142
  'finding-unreadable',
92
143
  'vote-failed',
93
144
  'request-too-large',
145
+ 'too-large',
146
+ 'token-cap',
147
+ 'review-deadline',
148
+ 'binary',
149
+ 'submodule',
150
+ 'secret',
94
151
  ]),
95
152
  file: z.string().optional(),
96
153
  detail: z.string(),
97
154
  });
155
+ /** A file left out: a rule belongs to a skipped or a held file, and to none else. */
156
+ const leftOutFile = z.discriminatedUnion('reason', [
157
+ z.strictObject({
158
+ file: z.string(),
159
+ reason: z.enum(['excluded', 'binary', 'submodule']),
160
+ }),
161
+ z.strictObject({
162
+ file: z.string(),
163
+ reason: z.literal('skipped'),
164
+ rule: z.enum(SKIP_RULES),
165
+ }),
166
+ z.strictObject({
167
+ file: z.string(),
168
+ reason: z.enum(['secret', 'deleted-secret']),
169
+ rule: z.enum(SECRET_RULES),
170
+ }),
171
+ ]);
172
+ const objectFormat = z.enum(['sha1', 'sha256']);
173
+ /**
174
+ * Where a review's diff came from. The shapes are strict, one object per kind; the ids are checked as a
175
+ * review checks its input's, by `sourceIssues`, so a record holds a full commit id of its object format.
176
+ */
177
+ const reviewSource = z.discriminatedUnion('kind', [
178
+ z.strictObject({
179
+ kind: z.literal('diff'),
180
+ }),
181
+ z.strictObject({
182
+ kind: z.literal('range'),
183
+ base: z.string(),
184
+ mergeBase: z.string(),
185
+ head: z.string(),
186
+ objectFormat,
187
+ }),
188
+ z.strictObject({
189
+ kind: z.enum(['local', 'staged']),
190
+ head: z.string().nullable(),
191
+ objectFormat,
192
+ }),
193
+ z.strictObject({
194
+ kind: z.literal('paths'),
195
+ commit: z.string(),
196
+ paths: z.array(z.string()),
197
+ skips: z.boolean(),
198
+ objectFormat,
199
+ }),
200
+ ]).refine(source => sourceIssues(source).length === 0);
98
201
  const flag = z.strictObject({
99
- code: z.enum(['hidden-characters', 'ai-addressed-text']),
202
+ code: z.enum(['hidden-characters', 'ai-addressed-text', 'secret-file']),
100
203
  file: z.string().optional(),
101
204
  line: count.optional(),
102
205
  detail: z.string(),
@@ -114,9 +217,11 @@ export const resultSchema = z.strictObject({
114
217
  version: z.string(),
115
218
  startedAt: z.string(),
116
219
  durationMs: count,
220
+ source: reviewSource,
117
221
  profiles: z.record(z.string(), profileMeta),
118
222
  usage: usage.optional(),
119
223
  incomplete: z.array(incompleteEntry),
224
+ leftOut: z.array(leftOutFile),
120
225
  flags: z.array(flag),
121
226
  }),
122
227
  });
@@ -44,12 +44,21 @@ export interface RunReviewsOptions {
44
44
  readonly events: (event: RunnerEvent) => void;
45
45
  /** The most a line may hold, its line break not counted, in bytes; `MAX_RECORD_BYTES` when left out. */
46
46
  readonly maxRecordBytes?: number;
47
+ /**
48
+ * Once the run's reviews have spent this many tokens, as each review's cap counts them, no review starts.
49
+ * No cap when left out.
50
+ */
51
+ readonly maxRunTokens?: number;
47
52
  }
48
53
  export interface RunReviewsOutcome {
49
54
  /** The reviews written to the record. */
50
55
  readonly recorded: number;
51
56
  /** The reviews that failed, a rejection that is no denial, or whose record was too large to write. */
52
57
  readonly failed: number;
58
+ /** The reviews that did not start because the run had spent `maxRunTokens`. */
59
+ readonly notRun: number;
60
+ /** The tokens the run's `call` events counted against their reviews' caps, as `maxRunTokens` reads them. */
61
+ readonly spent: number;
53
62
  /** Whether the caller's signal was aborted by the time the run ended. */
54
63
  readonly interrupted: boolean;
55
64
  }
@@ -68,6 +77,8 @@ export declare function planReviews(options: {
68
77
  * other rejection (an HTTP, network, timeout or process failure, a reply the server shaped wrong) says
69
78
  * nothing of the model's answer, so the review fails and runs again. The classes are matched with
70
79
  * `instanceof`: an error from a second copy of quorum is no denial, so it fails, which fails closed.
80
+ * Only a gating profile's failed research rejects a review: an advisory profile's, the call or its
81
+ * reply, resolves it with a `research-failed` gap, recorded with its result, neither failed nor denied.
71
82
  */
72
83
  export declare function denialOf(error: unknown): Denial | undefined;
73
84
  /**
@@ -87,6 +98,13 @@ export declare function denialOf(error: unknown): Denial | undefined;
87
98
  * nothing for that review, cuts the others, and the run rejects with the error of the write once every
88
99
  * review has settled. A callback of `events` that throws does the same.
89
100
  *
101
+ * The run counts what each review's cap counts: the `spent` of every `call` event of every review, one that
102
+ * failed or was denied included. Before a review starts, and after the abort's check, the run checks that
103
+ * the total is below `maxRunTokens`: once it is not, the review does not start and counts as `notRun`, the
104
+ * reviews in flight finish and are recorded, and the outcome says what was spent. The check is made before
105
+ * each review starts, so the reviews in flight can pass the cap by up to `jobs` reviews' own caps. The calls
106
+ * a failing review's own abort cut send no event, so the count misses them.
107
+ *
90
108
  * @throws the error of the first `append` (or `events`) that threw
91
109
  */
92
110
  export declare function runReviews(options: RunReviewsOptions): Promise<RunReviewsOutcome>;
@@ -66,6 +66,8 @@ function tallyCalls() {
66
66
  * other rejection (an HTTP, network, timeout or process failure, a reply the server shaped wrong) says
67
67
  * nothing of the model's answer, so the review fails and runs again. The classes are matched with
68
68
  * `instanceof`: an error from a second copy of quorum is no denial, so it fails, which fails closed.
69
+ * Only a gating profile's failed research rejects a review: an advisory profile's, the call or its
70
+ * reply, resolves it with a `research-failed` gap, recorded with its result, neither failed nor denied.
69
71
  */
70
72
  export function denialOf(error) {
71
73
  if (error instanceof OutputError)
@@ -91,10 +93,17 @@ export function denialOf(error) {
91
93
  * nothing for that review, cuts the others, and the run rejects with the error of the write once every
92
94
  * review has settled. A callback of `events` that throws does the same.
93
95
  *
96
+ * The run counts what each review's cap counts: the `spent` of every `call` event of every review, one that
97
+ * failed or was denied included. Before a review starts, and after the abort's check, the run checks that
98
+ * the total is below `maxRunTokens`: once it is not, the review does not start and counts as `notRun`, the
99
+ * reviews in flight finish and are recorded, and the outcome says what was spent. The check is made before
100
+ * each review starts, so the reviews in flight can pass the cap by up to `jobs` reviews' own caps. The calls
101
+ * a failing review's own abort cut send no event, so the count misses them.
102
+ *
94
103
  * @throws the error of the first `append` (or `events`) that threw
95
104
  */
96
105
  export async function runReviews(options) {
97
- const { suite, configurations, reviews, jobs, env, append, events, maxRecordBytes = MAX_RECORD_BYTES } = options;
106
+ const { suite, configurations, reviews, jobs, env, append, events, maxRecordBytes = MAX_RECORD_BYTES, maxRunTokens = Infinity } = options;
98
107
  const own = new AbortController();
99
108
  const signal = options.signal === undefined
100
109
  ? own.signal
@@ -105,6 +114,8 @@ export async function runReviews(options) {
105
114
  ]));
106
115
  let recorded = 0;
107
116
  let failed = 0;
117
+ let notRun = 0;
118
+ let spent = 0;
108
119
  let stopped;
109
120
  const stop = (error) => {
110
121
  stopped ??= { error };
@@ -128,7 +139,12 @@ export async function runReviews(options) {
128
139
  if (configuration === undefined || diff === undefined)
129
140
  throw new Error(UNKNOWN_REVIEW);
130
141
  const tally = tallyCalls();
131
- const quorum = await fromConfig(configuration.config, { env, events: tally.sink });
142
+ const sink = (event) => {
143
+ if (event.type === 'call')
144
+ spent += event.spent;
145
+ tally.sink(event);
146
+ };
147
+ const quorum = await fromConfig(configuration.config, { env, events: sink });
132
148
  const started = performance.now();
133
149
  const facts = () => ({
134
150
  fingerprint: configuration.fingerprint,
@@ -139,7 +155,7 @@ export async function runReviews(options) {
139
155
  });
140
156
  let result;
141
157
  try {
142
- result = await quorum.review({ diff }, { signal });
158
+ result = await quorum.review({ diff, send: configuration.config.send }, { signal });
143
159
  }
144
160
  catch (error) {
145
161
  const denied = signal.aborted
@@ -160,6 +176,10 @@ export async function runReviews(options) {
160
176
  async function runOne(review) {
161
177
  if (signal.aborted)
162
178
  return;
179
+ if (spent >= maxRunTokens) {
180
+ notRun += 1;
181
+ return;
182
+ }
163
183
  let record;
164
184
  let line;
165
185
  try {
@@ -191,5 +211,11 @@ export async function runReviews(options) {
191
211
  await Promise.all(reviews.map(review => limit(() => runOne(review))));
192
212
  if (stopped !== undefined)
193
213
  throw stopped.error;
194
- return { recorded, failed, interrupted: options.signal?.aborted === true };
214
+ return {
215
+ recorded,
216
+ failed,
217
+ notRun,
218
+ spent,
219
+ interrupted: options.signal?.aborted === true,
220
+ };
195
221
  }
@@ -31,12 +31,19 @@ export declare const BENCH_USAGE: string;
31
31
  * `results.jsonl` to append, so a cut last line stays, and warns of it. It refuses a run directory that
32
32
  * holds no run, and `--retry-incomplete`, which runs reviews.
33
33
  *
34
+ * `--max-run-tokens <n>` stops the run from starting reviews once its reviews have spent `n` tokens, counted
35
+ * from each call's `spent`, the tokens its review's own cap counted (`maxReviewTokens`): the reviews in flight
36
+ * finish and are recorded, the reports are written, one stderr line says how many reviews were not run, and
37
+ * the next run runs them. The check is made before each review starts, so the reviews in flight can pass
38
+ * the cap by up to `--jobs` reviews' own caps, and the calls a failing review's abort cut are not counted.
39
+ *
34
40
  * Exit codes: 0 when every planned review is recorded, a denied one included, or `--report-only` wrote the
35
41
  * reports, 2 for a usage, suite, configuration or run directory error, 3 when the run stops (a file cannot
36
- * be written or read), 4 when a review failed (its research call failed, or its record was too large to
37
- * write: the others are recorded, and the next run runs the failed ones), 130 when interrupted. The
38
- * reports are written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt during the
39
- * checks and the plan leaves the run directory as it was, even one that does not exist yet.
42
+ * be written or read), 4 when a review failed (its gating profile's research call failed, or its record
43
+ * was too large to write: the others are recorded, and the next run runs the failed ones) or
44
+ * `--max-run-tokens` left reviews unrun, 130 when interrupted. The reports are written on 0, 4 and 130,
45
+ * so a stopped run shows what it recorded; an interrupt during the checks and the plan leaves the run
46
+ * directory as it was, even one that does not exist yet.
40
47
  *
41
48
  * Every line it writes is inert (invariant I3).
42
49
  */