@tobi-or-not/quorum 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +3 -1
  2. package/dist/bench/fingerprint.d.ts +8 -5
  3. package/dist/bench/fingerprint.js +28 -9
  4. package/dist/bench/records.d.ts +168 -0
  5. package/dist/bench/records.js +106 -1
  6. package/dist/bench/runner.d.ts +18 -0
  7. package/dist/bench/runner.js +30 -4
  8. package/dist/cli/bench.d.ts +11 -4
  9. package/dist/cli/bench.js +34 -17
  10. package/dist/cli/bin.js +2 -2
  11. package/dist/cli/io.d.ts +5 -0
  12. package/dist/cli/main.d.ts +67 -24
  13. package/dist/cli/main.js +288 -59
  14. package/dist/cli/print-config.d.ts +8 -6
  15. package/dist/cli/print-config.js +26 -8
  16. package/dist/cli/render.d.ts +9 -3
  17. package/dist/cli/render.js +43 -2
  18. package/dist/config/claude-code.d.ts +2 -6
  19. package/dist/config/claude-code.js +6 -44
  20. package/dist/config/command.d.ts +10 -0
  21. package/dist/config/command.js +47 -0
  22. package/dist/config/env.d.ts +32 -19
  23. package/dist/config/env.js +95 -32
  24. package/dist/config/from-config.js +21 -3
  25. package/dist/config/resolve.d.ts +13 -1
  26. package/dist/config/resolve.js +27 -1
  27. package/dist/config/schema.d.ts +152 -5
  28. package/dist/config/schema.js +156 -12
  29. package/dist/core/clock.d.ts +17 -0
  30. package/dist/core/clock.js +14 -0
  31. package/dist/core/coverage.d.ts +74 -0
  32. package/dist/core/coverage.js +221 -0
  33. package/dist/core/diff.d.ts +101 -0
  34. package/dist/core/diff.js +178 -35
  35. package/dist/core/errors.d.ts +12 -3
  36. package/dist/core/errors.js +21 -2
  37. package/dist/core/estimate.d.ts +45 -0
  38. package/dist/core/estimate.js +188 -0
  39. package/dist/core/exclude.d.ts +42 -0
  40. package/dist/core/exclude.js +189 -0
  41. package/dist/core/flags.d.ts +14 -1
  42. package/dist/core/flags.js +56 -0
  43. package/dist/core/inputs.d.ts +38 -0
  44. package/dist/core/inputs.js +228 -0
  45. package/dist/core/limits.d.ts +83 -0
  46. package/dist/core/limits.js +169 -0
  47. package/dist/core/output.d.ts +3 -2
  48. package/dist/core/output.js +3 -2
  49. package/dist/core/parts.d.ts +146 -0
  50. package/dist/core/parts.js +388 -0
  51. package/dist/core/ports.d.ts +67 -2
  52. package/dist/core/ports.js +36 -1
  53. package/dist/core/prompts.d.ts +22 -3
  54. package/dist/core/prompts.js +43 -7
  55. package/dist/core/quorum.d.ts +26 -8
  56. package/dist/core/quorum.js +310 -73
  57. package/dist/core/retry.d.ts +36 -0
  58. package/dist/core/retry.js +46 -0
  59. package/dist/core/secrets.d.ts +98 -0
  60. package/dist/core/secrets.js +507 -0
  61. package/dist/core/stages/context.d.ts +131 -23
  62. package/dist/core/stages/context.js +331 -46
  63. package/dist/core/stages/panel.d.ts +13 -6
  64. package/dist/core/stages/panel.js +79 -19
  65. package/dist/core/stages/research.d.ts +110 -25
  66. package/dist/core/stages/research.js +338 -44
  67. package/dist/core/tokens.d.ts +31 -0
  68. package/dist/core/tokens.js +81 -0
  69. package/dist/core/types.d.ts +280 -25
  70. package/dist/core/verify.d.ts +6 -0
  71. package/dist/core/verify.js +6 -3
  72. package/dist/git/run.d.ts +223 -0
  73. package/dist/git/run.js +608 -0
  74. package/dist/git/tree.d.ts +43 -0
  75. package/dist/git/tree.js +171 -0
  76. package/dist/index.d.ts +26 -2
  77. package/dist/index.js +34 -2
  78. package/dist/providers/anthropic.d.ts +3 -1
  79. package/dist/providers/anthropic.js +10 -3
  80. package/dist/providers/claude-code.d.ts +56 -11
  81. package/dist/providers/claude-code.js +70 -25
  82. package/dist/providers/gemini.d.ts +5 -1
  83. package/dist/providers/gemini.js +65 -11
  84. package/dist/providers/http.d.ts +49 -4
  85. package/dist/providers/http.js +155 -14
  86. package/dist/providers/ollama.d.ts +4 -1
  87. package/dist/providers/ollama.js +13 -5
  88. package/dist/providers/openai-compatible.d.ts +4 -1
  89. package/dist/providers/openai-compatible.js +13 -5
  90. package/dist/providers/options.d.ts +12 -3
  91. package/dist/providers/options.js +30 -10
  92. package/dist/providers/size.d.ts +19 -15
  93. package/dist/providers/size.js +33 -73
  94. package/dist/sources/added.d.ts +49 -0
  95. package/dist/sources/added.js +134 -0
  96. package/dist/sources/diff.d.ts +13 -0
  97. package/dist/sources/diff.js +63 -0
  98. package/dist/sources/local.d.ts +41 -0
  99. package/dist/sources/local.js +251 -0
  100. package/dist/sources/paths.d.ts +45 -0
  101. package/dist/sources/paths.js +230 -0
  102. package/dist/sources/range.d.ts +21 -0
  103. package/dist/sources/range.js +75 -0
  104. package/dist/sources/read.d.ts +69 -0
  105. package/dist/sources/read.js +37 -0
  106. package/dist/sources/repository.d.ts +9 -0
  107. package/dist/sources/repository.js +28 -0
  108. package/dist/sources/skips.d.ts +49 -0
  109. package/dist/sources/skips.js +126 -0
  110. package/package.json +1 -1
  111. package/schema/quorum.schema.json +49 -1
package/dist/cli/bench.js CHANGED
@@ -26,22 +26,23 @@ export const BENCH_USAGE = [
26
26
  'Review every case of a suite with every config, --trials times each, and report what each caught.',
27
27
  '',
28
28
  'Options:',
29
- ' -c, --config <path> A config file to compare; repeat it, and the first is the reference',
30
- ' -p, --profile <id> Run only this profile; repeat it to run several',
31
- ' --trials <n> Reviews of each case per config: the suite\'s trials, else 1',
32
- ' --case <id> Run only this case; repeat it to run several',
33
- ' -j, --jobs <n> The reviews run at once (default 1)',
34
- ' -o, --out <dir> The run directory (default quorum-bench/<suite name>)',
35
- ' --retry-incomplete Run again the recorded reviews that came back incomplete',
36
- ' --report-only Write the reports of what is recorded, and run no review',
37
- ' --dry-run Check the suite and the configs, print the plan, and exit',
38
- ' -h, --help Print this help and exit',
29
+ ' -c, --config <path> A config file to compare; repeat it, and the first is the reference',
30
+ ' -p, --profile <id> Run only this profile; repeat it to run several',
31
+ ' --trials <n> Reviews of each case per config: the suite\'s trials, else 1',
32
+ ' --case <id> Run only this case; repeat it to run several',
33
+ ' -j, --jobs <n> The reviews run at once (default 1)',
34
+ ' --max-run-tokens <n> Start no review once the run has spent <n> tokens; the next run runs the rest',
35
+ ' -o, --out <dir> The run directory (default quorum-bench/<suite name>)',
36
+ ' --retry-incomplete Run again the recorded reviews that came back incomplete',
37
+ ' --report-only Write the reports of what is recorded, and run no review',
38
+ ' --dry-run Check the suite and the configs, print the plan, and exit',
39
+ ' -h, --help Print this help and exit',
39
40
  '',
40
41
  ].join('\n');
41
42
  /**
42
43
  * The bench's own exit codes (Decision 18), not the review's `EXIT`: `failed` means reviews failed and were
43
- * not recorded, not that a review left a gap or that a model's reply denied it, and `stopped` means the
44
- * run could not go on.
44
+ * not recorded, or that `--max-run-tokens` left reviews unrun, not that a review left a gap or that a
45
+ * model's reply denied it, and `stopped` means the run could not go on.
45
46
  */
46
47
  const BENCH_EXIT = {
47
48
  ok: 0,
@@ -61,6 +62,7 @@ const OPTIONS = {
61
62
  trials: { type: 'string' },
62
63
  case: { type: 'string', multiple: true },
63
64
  jobs: { type: 'string', short: 'j' },
65
+ 'max-run-tokens': { type: 'string' },
64
66
  out: { type: 'string', short: 'o' },
65
67
  'retry-incomplete': { type: 'boolean' },
66
68
  'report-only': { type: 'boolean' },
@@ -97,6 +99,7 @@ function checkOptions(values, positionals, cwd) {
97
99
  throw new Error(`unexpected arguments: ${extra.join(' ')}`);
98
100
  const trials = wholeNumber('--trials', values.trials, MAX_TRIALS);
99
101
  const jobs = wholeNumber('--jobs', values.jobs, MAX_JOBS);
102
+ const maxRunTokens = wholeNumber('--max-run-tokens', values['max-run-tokens'], Number.MAX_SAFE_INTEGER);
100
103
  const configs = values.config ?? [];
101
104
  if (configs.includes(''))
102
105
  throw usageIssue('--config', 'must name a file');
@@ -121,6 +124,7 @@ function checkOptions(values, positionals, cwd) {
121
124
  trials,
122
125
  cases,
123
126
  jobs: jobs ?? 1,
127
+ maxRunTokens,
124
128
  out: values.out,
125
129
  retryIncomplete: values['retry-incomplete'] === true,
126
130
  reportOnly: values['report-only'] === true,
@@ -355,7 +359,7 @@ async function writeRun({ out, state, runFile }) {
355
359
  * writer is closed whatever happens; when the run rejects, with the error of a failed write, that error
356
360
  * is the one thrown.
357
361
  */
358
- async function execute(io, { checked, scores, records, toRun, jobs }) {
362
+ async function execute(io, { checked, scores, records, toRun, jobs, maxRunTokens }) {
359
363
  const { suite, loaded, byId, out, runFile } = checked;
360
364
  await writeRun(checked);
361
365
  const writer = await openRecords(out, records);
@@ -367,6 +371,7 @@ async function execute(io, { checked, scores, records, toRun, jobs }) {
367
371
  configurations: loaded.map(({ config }, index) => ({ config, fingerprint: runFile.configurations[index].fingerprint })),
368
372
  reviews: toRun,
369
373
  jobs,
374
+ maxRunTokens,
370
375
  env: io.env,
371
376
  signal: io.signal,
372
377
  append: writer.append,
@@ -458,12 +463,17 @@ async function bench(io, options) {
458
463
  records,
459
464
  toRun,
460
465
  jobs: options.jobs,
466
+ maxRunTokens: options.maxRunTokens,
461
467
  });
462
468
  await writeReports(io, checked, trials, scores);
463
469
  if (outcome.interrupted) {
464
470
  io.stderr.write('interrupted\n');
465
471
  return BENCH_EXIT.interrupted;
466
472
  }
473
+ if (outcome.notRun > 0 && options.maxRunTokens !== undefined) {
474
+ writeLines(io.stderr, [`--max-run-tokens: the run reached its cap of ${options.maxRunTokens} tokens (it spent ${outcome.spent}), so ${outcome.notRun} review(s) were not run; the next run runs them`]);
475
+ return BENCH_EXIT.failed;
476
+ }
467
477
  return outcome.failed > 0
468
478
  ? BENCH_EXIT.failed
469
479
  : BENCH_EXIT.ok;
@@ -499,12 +509,19 @@ function usageError(io, error) {
499
509
  * `results.jsonl` to append, so a cut last line stays, and warns of it. It refuses a run directory that
500
510
  * holds no run, and `--retry-incomplete`, which runs reviews.
501
511
  *
512
+ * `--max-run-tokens <n>` stops the run from starting reviews once its reviews have spent `n` tokens, counted
513
+ * from each call's `spent`, the tokens its review's own cap counted (`maxReviewTokens`): the reviews in flight
514
+ * finish and are recorded, the reports are written, one stderr line says how many reviews were not run, and
515
+ * the next run runs them. The check is made before each review starts, so the reviews in flight can pass
516
+ * the cap by up to `--jobs` reviews' own caps, and the calls a failing review's abort cut are not counted.
517
+ *
502
518
  * Exit codes: 0 when every planned review is recorded, a denied one included, or `--report-only` wrote the
503
519
  * reports, 2 for a usage, suite, configuration or run directory error, 3 when the run stops (a file cannot
504
- * be written or read), 4 when a review failed (its research call failed, or its record was too large to
505
- * write: the others are recorded, and the next run runs the failed ones), 130 when interrupted. The
506
- * reports are written on 0, 4 and 130, so a stopped run shows what it recorded; an interrupt during the
507
- * checks and the plan leaves the run directory as it was, even one that does not exist yet.
520
+ * be written or read), 4 when a review failed (its gating profile's research call failed, or its record
521
+ * was too large to write: the others are recorded, and the next run runs the failed ones) or
522
+ * `--max-run-tokens` left reviews unrun, 130 when interrupted. The reports are written on 0, 4 and 130,
523
+ * so a stopped run shows what it recorded; an interrupt during the checks and the plan leaves the run
524
+ * directory as it was, even one that does not exist yet.
508
525
  *
509
526
  * Every line it writes is inert (invariant I3).
510
527
  */
package/dist/cli/bin.js CHANGED
@@ -13,8 +13,8 @@ function fail(error) {
13
13
  }
14
14
  process.on('uncaughtException', fail);
15
15
  process.on('unhandledRejection', fail);
16
- // Exit 3 until the run settles. A provider whose promise never settles, and holds no timer or socket,
17
- // empties the event loop: the runtime then exits with the code it holds, which must never be 0 with no result.
16
+ // Exit 3 until the run settles. A config module whose loading never finishes leaves nothing to wait
17
+ // for and empties the event loop: the runtime then exits with the code it holds, which must never be 0 with no result.
18
18
  process.exitCode = EXIT.runtime;
19
19
  let settled = false;
20
20
  process.once('beforeExit', () => {
package/dist/cli/io.d.ts CHANGED
@@ -5,6 +5,7 @@
5
5
  */
6
6
  import type { ParseArgsOptionsConfig } from 'node:util';
7
7
  import type { Env } from '../config/env.ts';
8
+ import type { GitSeam } from '../sources/read.ts';
8
9
  /** Exit codes (ADR-0008, amended by ADR-0012). */
9
10
  export declare const EXIT: {
10
11
  readonly ok: 0;
@@ -32,6 +33,10 @@ export interface CliIO {
32
33
  };
33
34
  /** The review's signal: once it is aborted, a run that ends without a result exits 130. */
34
35
  readonly signal?: AbortSignal;
36
+ /** The git a source runs, for the specs: its executable and its spawn. A run looks git up on `PATH` when unset. */
37
+ readonly git?: GitSeam;
38
+ /** The platform the git sources check, `process.platform` when unset: for the specs. */
39
+ readonly platform?: NodeJS.Platform;
35
40
  }
36
41
  /**
37
42
  * The `Error:` lines of a failure, each inert: a ConfigError's issues one per line, as its message
@@ -5,25 +5,49 @@ export { errorMessage, EXIT };
5
5
  export type { CliIO };
6
6
  /**
7
7
  * Run one review and return the process exit code. Never throws: usage and configuration problems
8
- * exit 2, runtime failures exit 3, an interrupt exits 130, and an incomplete review (a finding its
9
- * failed votes left undecided, or a research gap) of a gating profile exits 4 unless a `high` finding
10
- * was kept by a gating profile, so CI can tell a broken run and a partial review from a real finding
11
- * (exit 1) and from a clean one (exit 0). `exitCodeFor` maps the result; advisory flags never change it.
8
+ * exit 2, a gating profile's failed research and other runtime failures exit 3, an interrupt exits
9
+ * 130, and an incomplete review (a finding its failed votes left undecided, or a gap) of a gating
10
+ * profile exits 4 unless a `high` finding was kept by a gating profile, so CI can tell a broken run
11
+ * and a partial review from a real finding (exit 1) and from a clean one (exit 0). `--allow-incomplete`
12
+ * turns that exit 4 into 0 only when every failed vote behind it failed in passing (see `exitCodeFor`).
13
+ * `exitCodeFor` maps the result; advisory flags never change it.
12
14
  *
13
- * It reads at most 16 MiB of diff, from stdin or a file, counted in bytes: a larger diff is a usage error,
14
- * and reading stops at the chunk that crosses the cap.
15
+ * It reads at most 16 MiB of diff, from stdin, a file, a range of commits, the local changes or the files at
16
+ * a commit, counted in bytes: a larger diff is a usage error. A diff file or stdin stops at the chunk that
17
+ * crosses the cap, and the child of a git source is stopped at its own cap.
15
18
  *
16
- * Its flags are `--config`, `--profile`, `--allow-incomplete`, `--print-config`, `--help` and `--version`;
17
- * a usage error prints the usage after its message. Its settings are the config file `--config` or
18
- * `QUORUM_CONFIG` names (`--config` resolved against `io.cwd`, `QUORUM_CONFIG` an absolute path), and the
19
- * `QUORUM_*` variables of `io.env` with their deprecated aliases. Each warning is printed once, before the
20
- * banner, and when the configuration is refused the environment's warnings come before the `Error:` lines,
21
- * since a deprecated or mistyped variable is often what caused the error.
19
+ * The diff comes from one source: a diff file, stdin, `--range <base>...<head>`, `--local` (what `git add
20
+ * -A` would commit), `--staged` (what the next commit holds) or `--path <path>` (the files under it at the
21
+ * commit `--at` names, HEAD by default, skipping by default what `--no-default-skips` reviews), the last four
22
+ * running git itself (see `rangeSource`, `localSource` and `pathsSource`). Every usage rule of the source is
23
+ * checked first, before any git and any config, the form of a range included (`parseRange`), and a diff file
24
+ * or stdin is read then. A git source is read once the config is: it needs the config's `exclude` and `send`,
25
+ * and a source git refuses, or a repository whose own config names a filter without `--trust-repo-config`, exits 2
26
+ * before the banner, a git command that fails exits 3.
27
+ *
28
+ * A path review is a codebase and not a change, so its size is the tree's. After the banner and before any
29
+ * call it prints its estimate on stderr (see `Quorum.estimate`), and exits 2 when its research's expected
30
+ * spend and what the calls in flight reserve pass `maxReviewTokens`, unless `--large` is given: the review then
31
+ * reviews what the cap allows and ends incomplete, exit 4, where the cap stops it. A diff, a range and the
32
+ * local changes print no estimate and are never gated, since their size is the change's.
33
+ *
34
+ * Its flags are `--config`, `--profile`, `--range`, `--local`, `--staged`, `--trust-repo-config`, `--path`,
35
+ * `--at`, `--no-default-skips`, `--large`, `--send`, `--allow-incomplete`, `--print-config`, `--help` and
36
+ * `--version`; a usage error prints the usage after its message. Its settings are the config file `--config` or `QUORUM_CONFIG`
37
+ * names (`--config` resolved against `io.cwd`, `QUORUM_CONFIG` an absolute path), and the `QUORUM_*`
38
+ * variables of `io.env` with their deprecated aliases. Each warning is printed once, before the banner, and
39
+ * when the configuration is refused the environment's warnings come before the `Error:` lines, since a
40
+ * deprecated or mistyped variable is often what caused the error.
22
41
  *
23
42
  * `--print-config` reads no diff and calls no model: it writes the resolved config as YAML on stdout
24
- * (see `printConfig`), warnings included, and returns 0; a diff argument with it is a usage error, and
43
+ * (see `printConfig`), warnings included, and returns 0; a diff argument or a git source with it is a usage error, and
25
44
  * a configuration the run refuses, one of its providers that cannot be built included, exits 2 as in a run.
26
45
  *
46
+ * A file held back as a secret is sent all the same when `--send <path>` names it, repeatable, or the config's
47
+ * `send` lists it: the path is the one the diff writes and the `left out:` line prints, matched exactly. The
48
+ * CLI merges both lists, the config's first, and gives them to the source, the estimate and the review, so
49
+ * the operator alone, never the change, lifts the hold. An empty `--send` is a usage error, before any config.
50
+ *
27
51
  * The run is interrupted when `io.signal` aborts: it then writes `interrupted` and returns 130, whatever
28
52
  * error ended the review, and writes no result. A review that has already resolved is complete.
29
53
  *
@@ -40,22 +64,41 @@ export interface SignalSource {
40
64
  }
41
65
  /**
42
66
  * The signal of one review, aborted by the first SIGINT or SIGTERM. The next one calls `exit(130)`: the
43
- * way out of a provider that ignores its signal, since no timer waits for it.
67
+ * way out when what a provider still runs, after the review settled its call, keeps the process alive.
44
68
  */
45
69
  export declare function interruptSignal(source: SignalSource, exit: (code: number) => void): AbortSignal;
46
70
  /**
47
71
  * The exit code of a finished review: 1 when a kept finding is `high` and its profile gates, 4 when
48
- * an unreviewed finding or a gap belongs to a gating profile (unless `allowIncomplete` turns it into
49
- * 0, which it does for every gap but a `request-too-large` one), 0 otherwise. A profile gates
50
- * unless `meta.profiles` holds it with `gating: false`, so a profile missing from the result counts
51
- * as gating and a gap never hides.
52
- *
53
- * A `request-too-large` gap of a gating profile exits 4 whatever `allowIncomplete` says: a pull
54
- * request can cause that gap by padding the diff until a request is refused, so no flag lifts it
55
- * (ADR-0012). The flag lifts every other gap, including gaps a pull request can cause: decoy findings
56
- * that push a real one past `maxFindings`, or text that trips a hosted voter's content filter. On a
57
- * gating profile, do not use it for a pull request you do not trust.
72
+ * an unreviewed finding or a gap belongs to a gating profile, 0 otherwise. `allowIncomplete` turns
73
+ * that 4 into 0 only when every gap of a gating profile is a `vote-failed` one (an unreviewed finding
74
+ * is no gap) and, for each gating profile that has a gap, every failed vote of its own kept and
75
+ * unreviewed findings failed in passing, as `failedInPassing` says: a dropped connection, a timeout,
76
+ * or an HTTP 408, 409, 429 or 5xx. A gating profile with no gap blocks no other profile's lift. A gap
77
+ * with no failed vote behind it lifts nothing: an unreviewed finding whose votes hold none, or a
78
+ * `vote-failed` gap beside kept findings that hold none, which no review writes.
79
+ * A profile gates unless `meta.profiles` holds it with `gating: false`, so a profile missing from the
80
+ * result counts as gating and a gap never hides.
81
+ *
82
+ * Why so narrow: a pull request causes the other gaps directly (ADR-0012). It pads a hunk until no
83
+ * research request holds it (`too-large`), floods the research with decoy findings past `maxFindings`, or
84
+ * plants text that a voter refuses or that cuts its reply off or leaves it unusable, one
85
+ * `.gitattributes` line, one NUL byte or a submodule's bump leaves a file's change out of the diff, and
86
+ * a file named as a secret, or one that holds a credential, is held back from every request (`secret`).
87
+ * The flag lifts only what the code names, so a reason added later is out of its reach by default,
88
+ * and so is what a result built by hand or by an older quorum leaves unsaid: a failed vote with no
89
+ * `failure`, a verdict whose `votes` is no array, a reason no quorum writes. A gating profile's
90
+ * `research-failed` gap, which no review records, is out too: a review that read nothing is never
91
+ * turned into exit 0. Planted text can still draw out a reply that ends in a lifted failure, a
92
+ * timeout of its own, so on a gating profile do not use the flag for a pull request you do not trust.
58
93
  */
59
94
  export declare function exitCodeFor(result: QuorumResult, options?: {
60
95
  readonly allowIncomplete?: boolean;
61
96
  }): number;
97
+ /**
98
+ * True for a vote's failure that is infrastructure's: its own data `kind` is `network` or `timeout`, or
99
+ * `http` with an own data `status` of 408, 409, 429 or an integer from 500 to 599. Anything else is false:
100
+ * a refusal, a reply cut off or unusable, a request too large, any other status, a status that is no
101
+ * number, a kind no provider has, `null` and a value that is no object. It reads own data only, never
102
+ * runs a getter and never throws.
103
+ */
104
+ export declare function failedInPassing(failure: unknown): boolean;