@tobi-or-not/quorum 0.4.0 → 0.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +3 -1
  2. package/dist/bench/fingerprint.d.ts +8 -5
  3. package/dist/bench/fingerprint.js +28 -9
  4. package/dist/bench/records.d.ts +168 -0
  5. package/dist/bench/records.js +106 -1
  6. package/dist/bench/runner.d.ts +18 -0
  7. package/dist/bench/runner.js +30 -4
  8. package/dist/cli/bench.d.ts +11 -4
  9. package/dist/cli/bench.js +34 -17
  10. package/dist/cli/bin.js +2 -2
  11. package/dist/cli/io.d.ts +5 -0
  12. package/dist/cli/main.d.ts +67 -24
  13. package/dist/cli/main.js +288 -59
  14. package/dist/cli/print-config.d.ts +8 -6
  15. package/dist/cli/print-config.js +26 -8
  16. package/dist/cli/render.d.ts +9 -3
  17. package/dist/cli/render.js +43 -2
  18. package/dist/config/claude-code.d.ts +2 -6
  19. package/dist/config/claude-code.js +6 -44
  20. package/dist/config/command.d.ts +10 -0
  21. package/dist/config/command.js +47 -0
  22. package/dist/config/env.d.ts +32 -19
  23. package/dist/config/env.js +95 -32
  24. package/dist/config/from-config.js +21 -3
  25. package/dist/config/resolve.d.ts +13 -1
  26. package/dist/config/resolve.js +27 -1
  27. package/dist/config/schema.d.ts +152 -5
  28. package/dist/config/schema.js +156 -12
  29. package/dist/core/clock.d.ts +17 -0
  30. package/dist/core/clock.js +14 -0
  31. package/dist/core/coverage.d.ts +74 -0
  32. package/dist/core/coverage.js +221 -0
  33. package/dist/core/diff.d.ts +101 -0
  34. package/dist/core/diff.js +178 -35
  35. package/dist/core/errors.d.ts +12 -3
  36. package/dist/core/errors.js +21 -2
  37. package/dist/core/estimate.d.ts +45 -0
  38. package/dist/core/estimate.js +188 -0
  39. package/dist/core/exclude.d.ts +42 -0
  40. package/dist/core/exclude.js +189 -0
  41. package/dist/core/flags.d.ts +14 -1
  42. package/dist/core/flags.js +56 -0
  43. package/dist/core/inputs.d.ts +38 -0
  44. package/dist/core/inputs.js +228 -0
  45. package/dist/core/limits.d.ts +83 -0
  46. package/dist/core/limits.js +169 -0
  47. package/dist/core/output.d.ts +3 -2
  48. package/dist/core/output.js +3 -2
  49. package/dist/core/parts.d.ts +146 -0
  50. package/dist/core/parts.js +388 -0
  51. package/dist/core/ports.d.ts +67 -2
  52. package/dist/core/ports.js +36 -1
  53. package/dist/core/prompts.d.ts +22 -3
  54. package/dist/core/prompts.js +43 -7
  55. package/dist/core/quorum.d.ts +26 -8
  56. package/dist/core/quorum.js +310 -73
  57. package/dist/core/retry.d.ts +36 -0
  58. package/dist/core/retry.js +46 -0
  59. package/dist/core/secrets.d.ts +98 -0
  60. package/dist/core/secrets.js +507 -0
  61. package/dist/core/stages/context.d.ts +131 -23
  62. package/dist/core/stages/context.js +331 -46
  63. package/dist/core/stages/panel.d.ts +13 -6
  64. package/dist/core/stages/panel.js +79 -19
  65. package/dist/core/stages/research.d.ts +110 -25
  66. package/dist/core/stages/research.js +338 -44
  67. package/dist/core/tokens.d.ts +31 -0
  68. package/dist/core/tokens.js +81 -0
  69. package/dist/core/types.d.ts +280 -25
  70. package/dist/core/verify.d.ts +6 -0
  71. package/dist/core/verify.js +6 -3
  72. package/dist/git/run.d.ts +223 -0
  73. package/dist/git/run.js +608 -0
  74. package/dist/git/tree.d.ts +43 -0
  75. package/dist/git/tree.js +171 -0
  76. package/dist/index.d.ts +26 -2
  77. package/dist/index.js +34 -2
  78. package/dist/providers/anthropic.d.ts +3 -1
  79. package/dist/providers/anthropic.js +10 -3
  80. package/dist/providers/claude-code.d.ts +56 -11
  81. package/dist/providers/claude-code.js +70 -25
  82. package/dist/providers/gemini.d.ts +5 -1
  83. package/dist/providers/gemini.js +65 -11
  84. package/dist/providers/http.d.ts +49 -4
  85. package/dist/providers/http.js +155 -14
  86. package/dist/providers/ollama.d.ts +4 -1
  87. package/dist/providers/ollama.js +13 -5
  88. package/dist/providers/openai-compatible.d.ts +4 -1
  89. package/dist/providers/openai-compatible.js +13 -5
  90. package/dist/providers/options.d.ts +12 -3
  91. package/dist/providers/options.js +30 -10
  92. package/dist/providers/size.d.ts +19 -15
  93. package/dist/providers/size.js +33 -73
  94. package/dist/sources/added.d.ts +49 -0
  95. package/dist/sources/added.js +134 -0
  96. package/dist/sources/diff.d.ts +13 -0
  97. package/dist/sources/diff.js +63 -0
  98. package/dist/sources/local.d.ts +41 -0
  99. package/dist/sources/local.js +251 -0
  100. package/dist/sources/paths.d.ts +45 -0
  101. package/dist/sources/paths.js +230 -0
  102. package/dist/sources/range.d.ts +21 -0
  103. package/dist/sources/range.js +75 -0
  104. package/dist/sources/read.d.ts +69 -0
  105. package/dist/sources/read.js +37 -0
  106. package/dist/sources/repository.d.ts +9 -0
  107. package/dist/sources/repository.js +28 -0
  108. package/dist/sources/skips.d.ts +49 -0
  109. package/dist/sources/skips.js +126 -0
  110. package/package.json +1 -1
  111. package/schema/quorum.schema.json +49 -1
@@ -1,15 +1,23 @@
1
1
  import packageJson from '../../package.json' with { type: 'json' };
2
+ import { systemClock } from "./clock.js";
2
3
  import { createLimiter, DEFAULT_CONCURRENCY } from "./concurrency.js";
3
- import { splitDiff } from "./diff.js";
4
+ import { leftOutGaps, reviewScope } from "./coverage.js";
5
+ import { diffSections, splitDiff } from "./diff.js";
4
6
  import { ConfigError } from "./errors.js";
5
- import { diffFlags } from "./flags.js";
7
+ import { estimateReview } from "./estimate.js";
8
+ import { excludeIssues, excludeMatcher } from "./exclude.js";
9
+ import { diffFlags, secretFlags } from "./flags.js";
10
+ import { leftOutIssues, mergeLeftOut, sendIssues, sourceIssues, sourceOf } from "./inputs.js";
11
+ import { createLedger, DEFAULT_MAX_REVIEW_TOKENS, DEFAULT_REVIEW_TIMEOUT_MS, LimitError, limitGaps, maxReviewTokensProblem } from "./limits.js";
6
12
  import { isRecord } from "./output.js";
13
+ import { coverageOf, DEFAULT_PART_TOKENS, partTokensProblem, tooLargeGap } from "./parts.js";
14
+ import { MAX_STATED_TIMEOUT_MS, retriesProblem, timeoutProblem } from "./ports.js";
7
15
  import { profileIssues, profileWarnings, thresholdsFor } from "./profile.js";
8
16
  import { renderTemplate } from "./prompts.js";
9
- import { elapsed, stage } from "./stages/context.js";
17
+ import { addUsage, elapsed, ownData, stage, targetOf } from "./stages/context.js";
10
18
  import { dedup } from "./stages/dedup.js";
11
19
  import { panel } from "./stages/panel.js";
12
- import { DEFAULT_MAX_FINDINGS, research } from "./stages/research.js";
20
+ import { DEFAULT_MAX_FINDINGS, planResearch, research } from "./stages/research.js";
13
21
  const PROVIDER_SHAPE = 'must be a provider: an object with a string id, a string model and a complete function';
14
22
  /** The options that must be positive integers when they are set. */
15
23
  const LIMITS = ['concurrency', 'maxFindings'];
@@ -20,16 +28,25 @@ const LIMITS = ['concurrency', 'maxFindings'];
20
28
  * Every problem is reported at once, in one ConfigError, never as a TypeError, so an untyped caller
21
29
  * gets the checks a config file gets: options and ports that are not objects, no profile, each
22
30
  * profile's own issues, a repeated profile id, a `concurrency` or `maxFindings` that is not a
23
- * positive integer, a missing or malformed research provider or voter, ports for a profile or a
24
- * lens the run does not have (a likely typo), and an event sink that is not a function. Ports are
25
- * checked only for the profiles that have no issue of their own, and not at all when `profiles` is
26
- * not an array. Every record keyed by a caller's id is read as an own property, since `constructor`
27
- * is a valid id.
31
+ * positive integer, a `maxReviewTokens` that is not a whole number of tokens from 1 to
32
+ * 9007199254740991, a `reviewTimeoutMs` that is not a whole number of milliseconds from 1000 to
33
+ * 86400000, a `partTokens` that is not a whole number of tokens from 1024 to 9007199254740991, an
34
+ * `exclude` that is not a list of at most 100 patterns that can be taken, a missing or malformed
35
+ * research provider or voter, a provider's stated `timeoutMs` that is not a
36
+ * whole number of milliseconds from 1000 to 518400000, `retries` that is not a whole number from
37
+ * 0 to 5 or `worstCase` or `room` that is not a function, ports for a profile or a lens the run does not have
38
+ * (a likely typo), and an event sink that is not a function. Ports are checked only for the profiles
39
+ * that have no issue of their own, and not at all when `profiles` is not an array. Every record keyed
40
+ * by a caller's id is read as an own property, since `constructor` is a valid id. `maxReviewTokens`,
41
+ * the most tokens one review spends, input and output, is 2000000 when unset, `reviewTimeoutMs`,
42
+ * how long one review runs, is 1800000 (30 minutes) when unset, and `partTokens`, the most tokens of
43
+ * diff text one research part holds, is 32768 when unset: see `review`.
28
44
  *
29
45
  * Each profile is compiled into a frozen `ProfileRun`, its instructions rendered, its thresholds
30
- * resolved and its providers captured, so later changes to the caller's profiles or ports are not
31
- * seen. `warnings` lists what the profiles set that cannot take full effect. Each review can be
32
- * stopped through its own signal; a throwing event sink or a failing profile stops it too.
46
+ * resolved and its providers captured, and `exclude` is compiled into a matcher, so later changes to
47
+ * the caller's profiles, ports or `exclude` are not seen. `warnings` lists what the profiles set
48
+ * that cannot take full effect. Each review can be stopped through its own signal; a throwing event
49
+ * sink or a failing profile stops it too.
33
50
  *
34
51
  * @throws ConfigError listing every issue of the options and the ports
35
52
  *
@@ -39,6 +56,16 @@ const LIMITS = ['concurrency', 'maxFindings'];
39
56
  * const { kept, meta } = await quorum.review({ diff: diffText }, { signal: controller.signal })
40
57
  */
41
58
  export function createQuorum(options, ports) {
59
+ return createQuorumWith(options, ports, systemClock);
60
+ }
61
+ /**
62
+ * `createQuorum` on a clock of the caller's: the time, the randomness and the one timer the call path and the
63
+ * review's deadline read. It is the specs' seam, so a review's deadline and a call's waits run on a fake clock
64
+ * and cost no real time; the package does not export it, and `createQuorum` passes the system's clock.
65
+ *
66
+ * @throws ConfigError listing every issue of the options and the ports, as `createQuorum` does
67
+ */
68
+ export function createQuorumWith(options, ports, clock) {
42
69
  const issues = inputIssues(options, ports);
43
70
  if (issues.length > 0)
44
71
  throw new ConfigError(issues);
@@ -47,43 +74,151 @@ export function createQuorum(options, ports) {
47
74
  runs: Object.freeze(options.profiles.map(profile => compile(profile, ports.profiles[profile.id]))),
48
75
  concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,
49
76
  maxFindings: options.maxFindings ?? DEFAULT_MAX_FINDINGS,
77
+ maxReviewTokens: options.maxReviewTokens ?? DEFAULT_MAX_REVIEW_TOKENS,
78
+ reviewTimeoutMs: options.reviewTimeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS,
79
+ partTokens: options.partTokens ?? DEFAULT_PART_TOKENS,
80
+ clock,
81
+ excluded: excludeMatcher(options.exclude ?? []),
50
82
  sink: ports.events,
51
83
  };
52
84
  const warnings = Object.freeze(options.profiles.flatMap(profile => profileWarnings(profile)));
53
85
  return Object.freeze({
54
86
  review: (input, options) => review(compiled, input, options),
87
+ estimate: (input) => estimate(compiled, input),
55
88
  warnings,
56
89
  });
57
90
  }
91
+ /**
92
+ * What one review of `input.diff` may cost, before any call: no provider is called and nothing is read or
93
+ * written. Per profile, it states what the review's ledger reserves for each research part, in order, exactly
94
+ * as `review` plans and builds them (the diff less the sections `exclude` leaves out, parted on each research
95
+ * provider's stated room), and a bound for the votes: `maxFindings` votes of each lens, each at the request for
96
+ * a finding at its longest citing the largest section of the filtered diff. `expected` counts the research
97
+ * parts alone, each at a third of its reservation, rounded up (`BOUND_OVER_CODE`), and `worst` the parts in full
98
+ * and every vote at its bound, so the votes are in the worst case only. `inFlight` is what the calls that run
99
+ * at once reserve, the `concurrency` largest bounds among all the profiles' calls, and `fits` is whether
100
+ * `expected` and `inFlight` are within `maxReviewTokens`. A diff with nothing left to read, blank or whose every
101
+ * file is excluded or held back as a secret, makes no call and costs nothing; a file `input.send` names is not
102
+ * held back, so it is counted, as `review` sends it. It is an estimate, not a limit: a review reserves each call
103
+ * against the cap, which still holds whatever `fits` says.
104
+ *
105
+ * @throws ConfigError listing every issue of the input that `review` refuses: an input that is not an object, a `diff` that is not a string, a `source`, a `leftOut` or a `send` it cannot take
106
+ */
107
+ function estimate(compiled, input) {
108
+ const issues = reviewIssues(input, {});
109
+ if (issues.length > 0)
110
+ throw new ConfigError(issues);
111
+ const sent = new Set(input.send);
112
+ return estimateReview(compiled.runs, compiled, input.diff, path => sent.has(path));
113
+ }
58
114
  /**
59
115
  * One review of a diff by every profile of the quorum, concurrently, each through research, dedup
60
- * and panel under the review's one concurrency limit; a blank diff runs no stage. The diff is split
61
- * once, for every profile and for the advisory flags, which are read from it before any stage runs.
62
- * Each is sent as a `flag` event right after `run:start` and kept in `meta.flags`, and none of them
63
- * changes a verdict. Each event is handed to the sink as a copy, so a sink that changes one changes
64
- * nothing else. The review reports itself through events, from `run:start` to `run:end`, and its
65
- * result merges the profiles' verdicts and gaps in profile order.
66
- * `meta.profiles[<id>].invalidFindings` and `meta.usage` are counted from the events before the
67
- * sink receives them.
116
+ * and panel under the review's one concurrency limit; a blank diff runs no stage. The diff is
117
+ * filtered once: the sections the quorum's `exclude` patterns match, and the files held back as secrets
118
+ * (below), leave every request, so research, the check of each finding and the votes read the diff less
119
+ * those sections, split into one section per file once for every profile, and a finding that cites an
120
+ * excluded or a held file is dropped as outside the diff. A filtered diff that is blank, with every file
121
+ * excluded or held and nothing before the first header, runs no stage, as a blank diff does, and still
122
+ * records the gaps its held files make. The advisory flags are read before any stage runs, from the
123
+ * whole diff, the excluded and the held files included. The review lists the files it leaves out in
124
+ * `meta.leftOut`: the excluded ones, which are no gap, the files whose change the diff does not carry,
125
+ * binary files and submodules' commits, and the files held back, for each reason of which it records a
126
+ * gap for every profile; the sections of a file whose change the diff does not carry still reach the
127
+ * stages. The input's own `leftOut`, the files its
128
+ * source left out before it made the diff, comes first in the list, in the `left-out` event and in the gaps'
129
+ * counts, each file, reason and rule once, and a skipped one is no gap; the input's `source` is checked,
130
+ * copied and recorded in `meta.source`, `{ kind: 'diff' }` when it names none. Its `skipped` files of the
131
+ * rule `secret`, and the `secret` and `deleted-secret` files `meta.leftOut` lists, each raise one `secret-file`
132
+ * flag, as many as `meta.leftOut` lists, ahead of the diff's flags and whatever the diff is.
133
+ * Everything it knows before a call is sent in this order: `run:start`, then a `left-out` event naming those files when it leaves one
134
+ * out, then the gaps' `incomplete` events (each profile's `binary`, `submodule` and `secret` gaps, then its
135
+ * `too-large` gap), then the `flag` events of the advisory flags, the `secret-file` flags first,
136
+ * kept in `meta.flags`, none of which changes a verdict, then the stages. Each event is handed to the sink
137
+ * as a copy, so a sink that changes one changes nothing else. The review reports itself through
138
+ * events, from `run:start` to `run:end`, and its result merges the profiles' verdicts and gaps in
139
+ * profile order. `meta.profiles[<id>].invalidFindings` and `meta.usage` are counted from the
140
+ * events before the sink receives them.
141
+ *
142
+ * No model is sent a line of a file held back as a secret. A file is held when its name is a secret's
143
+ * (`.env`, `id_ed25519`, a `.pem` file) or a line of it, removed and context lines included, holds a credential
144
+ * quorum recognizes (a private key's armor and body, the keys of AWS, GitHub, Slack, Google, Stripe, Anthropic
145
+ * and OpenAI), by any path its section names; every section of it leaves the diff, and a credential in one
146
+ * holds all. It is listed in `meta.leftOut` as `secret` with its `rule` (`name`, or the kind of credential),
147
+ * raises a `secret-file` flag, and is a `secret` gap of every profile that no flag lifts; a change that deletes
148
+ * a file named as a secret whole lists it as `deleted-secret` with its flag and no gap, while a deletion of a
149
+ * file held for a credential in it, whose removed code would go unread, is a `secret` gap. The text no header names is one file
150
+ * of the empty name, held as any other. `input.send` names the files to send though held, each the path exactly
151
+ * as `meta.leftOut` lists it, never the empty name: such a file is sent, neither listed nor flagged, and one
152
+ * `exclude` matches is excluded first. `send` is copied before the review's first await.
68
153
  *
69
154
  * Abort never fails open. The review's signal joins the caller's `signal` to an internal one; every
70
- * stage and every call checks it, every request carries it, and it is checked once more after the
71
- * profiles have run, a blank diff included, so an aborted review rejects with the signal's reason,
72
- * whatever error a call it cut raised, and never resolves. No event is delivered once the review is
73
- * aborted. A sink that throws aborts the review with its error, and a research failure (the call or
155
+ * stage and every call checks it, every attempt carries a signal of its own, which the review's
156
+ * abort and the attempt's deadline abort, and it is checked once more after the profiles have run,
157
+ * a blank diff included, so an aborted review rejects with the signal's reason, whatever error a
158
+ * call it cut raised, and never resolves. No event is delivered once the review is aborted. A sink
159
+ * that throws aborts the review with its error, and a gating profile's research failure (the call or
74
160
  * its reply) aborts it with that failure where it fails, so no other profile starts a call after
75
161
  * it; any other failure aborts the review when the profiles' run rejects. Either way the review
76
- * rejects with the first reason. An abort after `run:end` has no effect: the review is over.
162
+ * rejects with the first reason. An advisory profile's research failure is its `research-failed`
163
+ * gap and aborts nothing. An abort after `run:end` has no effect: the review is over.
164
+ *
165
+ * A review ends at its deadline, `reviewTimeoutMs` after it starts, with an answer: at it every call in
166
+ * flight and every wait is stopped at once, no call still queued starts, and what was not done is a
167
+ * `review-deadline` gap of its profile, its research request (the parts of it not yet answered) or its
168
+ * votes, each a failed vote of the deadline. The review then resolves with its findings so far, exiting
169
+ * `4` for a gating profile whatever the flags; the deadline is no abort, so an interrupt still rejects
170
+ * with its reason, before the deadline or after it, and a review that reaches its deadline is never a
171
+ * failed run. The deadline's timer is armed once the input checks pass and cleared however the review
172
+ * ends: it resolves, it rejects, its sink throws or it is aborted.
173
+ *
174
+ * A change larger than one research request is researched in parts, not refused. Each profile's research
175
+ * is planned once the scope is known, on its own research provider's stated room: a provider that
176
+ * states none, or a diff within `partTokens` and within the room, is sent the whole diff in one request,
177
+ * as it always was; else the filtered diff's sections, in order, are packed into parts of at most
178
+ * `partTokens` whose request fits the room, a section too large for a part split between its hunks and
179
+ * a hunk never cut. Each part is one research call, in order, that names its position and the change's
180
+ * files; the replies are read in order, up to `maxFindings` findings in all, each checked against the
181
+ * whole diff. A hunk, or a section with no hunk, that fits in no research part of the window is no part's:
182
+ * the profile records one `too-large` gap for them before any call is sent, which no flag lifts for a
183
+ * gating profile. `meta.profiles.<id>.coverage` lists every file of the input diff once, in its order,
184
+ * with how much of it the profile's research reviewed: reviewed, partly (with the ranges left), not
185
+ * (with the reason), or excluded by `exclude`; each profile has its own, since each has its own
186
+ * research model.
77
187
  *
78
- * @throws ConfigError listing every issue of the input and the options: an input that is not an object, a `diff` that is not a string, options that are not an object, a `signal` that is not an AbortSignal
188
+ * A review spends at most `maxReviewTokens` tokens. Each review has its own ledger of that cap: every
189
+ * attempt of every call reserves its worst case in it before it is sent and settles at what it spent,
190
+ * so concurrent calls cannot overshoot it. A call the cap stops is no failed review: its profile's
191
+ * research or vote is a `token-cap` gap, which exits 4 for a gating profile, and the other profiles
192
+ * and votes run as far as the cap lets them. A research in parts spends up to the cap, each part reserved
193
+ * before it is sent: the parts the cap cannot fund are not sent, and one gap counts them.
194
+ *
195
+ * @throws ConfigError listing every issue of the input and the options: an input that is not an object, a `diff` that is not a string, a `source`, a `leftOut` or a `send` it cannot take, options that are not an object, a `signal` that is not an AbortSignal
79
196
  * @throws the signal's reason once the review is aborted, before it starts or while it runs
80
- * @throws the error of a sink that throws, or of a profile that fails: a ProviderError from its research call, an OutputError for its reply
197
+ * @throws the error of a sink that throws, or of a gating profile that fails: a ProviderError from its research call, an OutputError for its reply
81
198
  */
82
199
  async function review(compiled, input, options = {}) {
83
200
  const issues = reviewIssues(input, options);
84
201
  if (issues.length > 0)
85
202
  throw new ConfigError(issues);
86
203
  options.signal?.throwIfAborted();
204
+ const { clock, reviewTimeoutMs } = compiled;
205
+ const deadline = { at: clock.now() + reviewTimeoutMs, ms: reviewTimeoutMs };
206
+ const expired = new AbortController();
207
+ const cancel = clock.timer(reviewTimeoutMs, () => {
208
+ expired.abort(new LimitError('review-deadline', `the review reached its deadline of ${reviewTimeoutMs} ms`));
209
+ });
210
+ try {
211
+ return await reviewUntil(compiled, input, options, deadline, expired.signal);
212
+ }
213
+ finally {
214
+ cancel();
215
+ }
216
+ }
217
+ /**
218
+ * What `review` does once its deadline is armed: everything from its signals to its result, with `expired`, the
219
+ * deadline's own signal, joined to the review's into the one `stop` the call path reads.
220
+ */
221
+ async function reviewUntil(compiled, input, options, deadline, expired) {
87
222
  const internal = new AbortController();
88
223
  const signal = options.signal === undefined
89
224
  ? internal.signal
@@ -95,12 +230,48 @@ async function review(compiled, input, options = {}) {
95
230
  const startedAt = new Date().toISOString();
96
231
  const started = performance.now();
97
232
  const { runs, sink } = compiled;
98
- const sections = input.diff.trim() === ''
233
+ const whole = input.diff.trim() === ''
99
234
  ? undefined
100
235
  : splitDiff(input.diff);
101
- const flags = sections === undefined
236
+ // Copied before the first await, so the caller's later changes to its input reach neither the result nor the events.
237
+ const sent = new Set(input.send);
238
+ const source = sourceOf(input.source);
239
+ const scope = whole === undefined
240
+ ? { diff: input.diff, leftOut: [], files: [] }
241
+ : reviewScope(input.diff, compiled.excluded, path => sent.has(path));
242
+ const leftOut = mergeLeftOut(input.leftOut ?? [], scope.leftOut);
243
+ // The source's secrets are read from the merged list, so a file it lists twice is flagged once, as `meta.leftOut` lists it once.
244
+ const flags = [
245
+ ...secretFlags(leftOut),
246
+ ...whole === undefined
247
+ ? []
248
+ : diffFlags(whole),
249
+ ];
250
+ // What the stages read: none when the filtered diff is blank, else the sections the flags read when nothing was left out.
251
+ const sections = scope.diff.trim() === ''
252
+ ? undefined
253
+ : scope.diff === input.diff
254
+ ? whole
255
+ : splitDiff(scope.diff);
256
+ // The same sections as the research packs them, and how each profile's research goes: none when no stage runs.
257
+ const filtered = sections === undefined
258
+ ? []
259
+ : diffSections(scope.diff);
260
+ const plans = sections === undefined
102
261
  ? []
103
- : diffFlags(sections);
262
+ : runs.map(run => planResearch(run, scope.diff, filtered, compiled.partTokens));
263
+ const gaps = new Map(runs.map((run, index) => {
264
+ const tooLarge = tooLargeGap(run.id, filtered, plans[index]?.packing, scope.leftOut);
265
+ return [
266
+ run.id,
267
+ [
268
+ ...leftOutGaps(run.id, leftOut),
269
+ ...tooLarge === undefined
270
+ ? []
271
+ : [tooLarge],
272
+ ],
273
+ ];
274
+ }));
104
275
  const tally = {
105
276
  invalidFindings: new Map(),
106
277
  };
@@ -122,10 +293,15 @@ async function review(compiled, input, options = {}) {
122
293
  };
123
294
  const ctx = {
124
295
  limit: createLimiter(compiled.concurrency),
296
+ ledger: createLedger(compiled.maxReviewTokens),
125
297
  emit,
126
298
  signal,
299
+ stop: AbortSignal.any([signal, expired]),
300
+ deadline,
127
301
  fail,
128
302
  maxFindings: compiled.maxFindings,
303
+ concurrency: compiled.concurrency,
304
+ clock: compiled.clock,
129
305
  };
130
306
  emit({
131
307
  type: 'run:start',
@@ -135,10 +311,16 @@ async function review(compiled, input, options = {}) {
135
311
  providersOf(run),
136
312
  ])),
137
313
  });
314
+ if (leftOut.length > 0)
315
+ emit({ type: 'left-out', files: leftOut });
316
+ for (const run of runs) {
317
+ for (const entry of gaps.get(run.id) ?? [])
318
+ emit({ type: 'incomplete', entry });
319
+ }
138
320
  for (const flag of flags)
139
321
  emit({ type: 'flag', flag });
140
- const outcomes = await reviewAll(ctx, runs, input.diff, sections).catch((error) => {
141
- // A research failure aborted the review where it failed; any other failure aborts it here. The first reason stays the reason.
322
+ const outcomes = await reviewAll(ctx, runs, plans, scope.diff, sections).catch((error) => {
323
+ // A gating profile's research failure aborted the review where it failed; any other failure aborts it here. The first reason stays the reason.
142
324
  const reason = signal.aborted ? signal.reason : error;
143
325
  internal.abort(reason);
144
326
  throw reason;
@@ -147,20 +329,25 @@ async function review(compiled, input, options = {}) {
147
329
  const kept = outcomes.flatMap(outcome => outcome.kept);
148
330
  const discarded = outcomes.flatMap(outcome => outcome.discarded);
149
331
  const unreviewed = outcomes.flatMap(outcome => outcome.unreviewed);
150
- const incomplete = outcomes.flatMap(outcome => outcome.incomplete);
332
+ const incomplete = runs.flatMap((run, index) => [
333
+ ...gaps.get(run.id) ?? [],
334
+ ...outcomes[index]?.incomplete ?? [],
335
+ ]);
151
336
  const durationMs = elapsed(started);
152
337
  const meta = {
153
338
  version: packageJson.version,
154
339
  startedAt,
155
340
  durationMs,
156
- profiles: Object.fromEntries(runs.map(run => [
341
+ source,
342
+ profiles: Object.fromEntries(runs.map((run, index) => [
157
343
  run.id,
158
- profileMeta(run, tally.invalidFindings.get(run.id) ?? 0),
344
+ profileMeta(run, tally.invalidFindings.get(run.id) ?? 0, coverageOf(scope, filtered, plans[index]?.packing, outcomes[index]?.parts ?? [])),
159
345
  ])),
160
346
  ...(tally.usage === undefined
161
347
  ? {}
162
348
  : { usage: tally.usage }),
163
349
  incomplete,
350
+ leftOut,
164
351
  flags,
165
352
  };
166
353
  emit({
@@ -173,53 +360,60 @@ async function review(compiled, input, options = {}) {
173
360
  });
174
361
  return { kept, discarded, unreviewed, meta };
175
362
  }
176
- /** Run every profile through its stages at once on the diff's sections; a blank diff, which has none, runs no stage. */
177
- async function reviewAll(ctx, runs, diff, sections) {
363
+ /** Run every profile through its stages at once on the diff's sections, each on its own plan; a blank diff, which has none, runs no stage. */
364
+ async function reviewAll(ctx, runs, plans, diff, sections) {
178
365
  if (sections === undefined)
179
366
  return [];
180
- return Promise.all(runs.map(run => reviewProfile(ctx, run, diff, sections)));
367
+ return Promise.all(runs.map((run, index) => reviewProfile(ctx, run, plans[index], diff, sections)));
181
368
  }
182
- /** One profile's review: its panel's outcome, with the research gaps before the panel's own. */
183
- async function reviewProfile(ctx, run, diff, sections) {
184
- const researched = await stage(ctx, run.id, 'research', () => research(ctx, run, diff, sections), outcome => outcome.findings.length);
369
+ /**
370
+ * One profile's review: its panel's outcome, with the research gaps before the panel's own, and last the
371
+ * gap of each limit that stopped its work, reported by an `incomplete` event once the panel's stage has ended.
372
+ */
373
+ async function reviewProfile(ctx, run, plan, diff, sections) {
374
+ const researched = await stage(ctx, run.id, 'research', () => research(ctx, run, plan, diff, sections), outcome => outcome.findings.length);
185
375
  const candidates = await stage(ctx, run.id, 'dedup', () => dedup(researched.findings), found => found.length);
186
376
  const judged = await stage(ctx, run.id, 'panel', () => panel(ctx, run, candidates, sections, diff), outcome => outcome.kept.length + outcome.discarded.length + outcome.unreviewed.length);
377
+ const limits = limitGaps(run.id, researched.parts, [
378
+ ...judged.kept,
379
+ ...judged.discarded,
380
+ ...judged.unreviewed,
381
+ ], { cap: ctx.ledger.cap, deadlineMs: ctx.deadline.ms });
382
+ for (const entry of limits)
383
+ ctx.emit({ type: 'incomplete', entry });
187
384
  return {
188
385
  ...judged,
189
386
  incomplete: [
190
387
  ...researched.incomplete,
191
388
  ...judged.incomplete,
389
+ ...limits,
192
390
  ],
391
+ parts: researched.parts,
193
392
  };
194
393
  }
195
394
  /**
196
395
  * Every problem with one review's input and options, so an untyped caller gets issues, never a
197
396
  * TypeError. An input that is not an object is named as the input, not as its diff: the likely
198
- * mistake is the diff text passed on its own, as `runQuorum` takes it. The signal must be a real
199
- * AbortSignal: `AbortSignal.any` accepts nothing else.
397
+ * mistake is the diff text passed on its own, as `runQuorum` takes it. The `source`, the `leftOut` and the
398
+ * `send`, when set, are checked after the diff, so a caller sees every issue of the input at once. The signal
399
+ * must be a real AbortSignal: `AbortSignal.any` accepts nothing else.
200
400
  */
201
401
  function reviewIssues(input, options) {
202
402
  const issues = [];
203
- if (!isRecord(input))
403
+ if (!isRecord(input)) {
204
404
  issues.push({ path: 'input', message: 'must be an object holding the diff string' });
205
- else if (typeof input.diff !== 'string')
206
- issues.push({ path: 'diff', message: 'must be a string' });
405
+ }
406
+ else {
407
+ if (typeof input.diff !== 'string')
408
+ issues.push({ path: 'diff', message: 'must be a string' });
409
+ issues.push(...sourceIssues(input.source), ...leftOutIssues(input.leftOut), ...sendIssues(input.send));
410
+ }
207
411
  if (!isRecord(options))
208
412
  issues.push({ path: 'options', message: 'must be an object' });
209
413
  else if (options.signal !== undefined && !(options.signal instanceof AbortSignal))
210
414
  issues.push({ path: 'signal', message: 'must be an AbortSignal' });
211
415
  return issues;
212
416
  }
213
- /** The usage of one more call added to a running total; `costUsd` is summed over the calls that reported one. */
214
- function addUsage(total, call) {
215
- const sum = { inputTokens: (total?.inputTokens ?? 0) + call.inputTokens, outputTokens: (total?.outputTokens ?? 0) + call.outputTokens };
216
- const costUsd = call.costUsd === undefined
217
- ? total?.costUsd
218
- : (total?.costUsd ?? 0) + call.costUsd;
219
- if (costUsd !== undefined)
220
- sum.costUsd = costUsd;
221
- return sum;
222
- }
223
417
  /** Who reviews for a profile, as new objects the caller may keep or change: never the frozen refs of the run. */
224
418
  function providersOf(run) {
225
419
  return {
@@ -230,31 +424,33 @@ function providersOf(run) {
230
424
  ])),
231
425
  };
232
426
  }
233
- function profileMeta(run, invalidFindings) {
427
+ function profileMeta(run, invalidFindings, coverage) {
234
428
  return {
235
429
  providers: providersOf(run),
236
430
  policy: { ...run.thresholds },
237
431
  gating: run.gating,
238
432
  invalidFindings,
433
+ coverage,
239
434
  };
240
435
  }
241
436
  /**
242
437
  * Compile a checked profile and its checked ports into the frozen run every review reads: the vote
243
438
  * template rendered once per lens, the thresholds resolved, `gating` true unless the profile says
244
- * otherwise, and each provider captured beside the `{ id, model }` ref events and meta report.
439
+ * otherwise, and each provider captured beside the `{ id, model }` ref events and meta report and
440
+ * the deadline of its attempts.
245
441
  */
246
442
  function compile(profile, ports) {
247
443
  const panelSize = String(profile.lenses.length);
248
444
  const lenses = profile.lenses.map(lens => Object.freeze({
249
445
  id: lens.id,
250
446
  instructions: renderTemplate(profile.prompts.vote, { 'lens.id': lens.id, 'lens.question': lens.question, 'panel.size': panelSize }),
251
- ...target(ports.voters[lens.id]),
447
+ ...targetOf(ports.voters[lens.id]),
252
448
  }));
253
449
  return Object.freeze({
254
450
  id: profile.id,
255
451
  research: Object.freeze({
256
452
  instructions: profile.prompts.research,
257
- ...target(ports.research),
453
+ ...targetOf(ports.research),
258
454
  }),
259
455
  lenses: Object.freeze(lenses),
260
456
  thresholds: Object.freeze(thresholdsFor(profile)),
@@ -262,15 +458,10 @@ function compile(profile, ports) {
262
458
  guidelines: profile.guidelines,
263
459
  });
264
460
  }
265
- function target(provider) {
266
- return {
267
- provider,
268
- ref: Object.freeze({ id: provider.id, model: provider.model }),
269
- };
270
- }
271
461
  /**
272
462
  * Every problem with the options and the ports, in this order: options and ports that are not
273
- * objects, `profiles`, each profile's own issues and repeated ids, the limits, then the ports.
463
+ * objects, `profiles`, each profile's own issues and repeated ids, the limits, `exclude`, then the
464
+ * ports.
274
465
  */
275
466
  function inputIssues(options, ports) {
276
467
  const issues = [];
@@ -285,11 +476,12 @@ function inputIssues(options, ports) {
285
476
  ...issues,
286
477
  { path: 'profiles', message: 'must be an array of profiles' },
287
478
  ...limitIssues(options),
479
+ ...excludeIssues(options.exclude),
288
480
  ];
289
481
  }
290
482
  // `Array.from` reads a hole as `undefined`, so it is reported like any other profile that is not an object.
291
483
  const profiles = profileListIssues(Array.from(options.profiles));
292
- issues.push(...profiles.issues, ...limitIssues(options));
484
+ issues.push(...profiles.issues, ...limitIssues(options), ...excludeIssues(options.exclude));
293
485
  if (isRecord(ports))
294
486
  issues.push(...portIssues(ports, profiles.checked, profiles.named));
295
487
  return issues;
@@ -323,9 +515,14 @@ function profileListIssues(profiles) {
323
515
  });
324
516
  return { issues, checked, named };
325
517
  }
326
- /** `concurrency` and `maxFindings` are positive integers when set. A number is quoted; anything else is described by the rule alone. */
518
+ /**
519
+ * `concurrency` and `maxFindings` are positive integers when set, `maxReviewTokens` a whole number of
520
+ * tokens from 1 to 9007199254740991, `reviewTimeoutMs` a whole number of milliseconds from 1000 to
521
+ * 86400000, and `partTokens` a whole number of tokens from 1024 to 9007199254740991. A number is
522
+ * quoted in the first two; anything else is described by the rule alone.
523
+ */
327
524
  function limitIssues(options) {
328
- return LIMITS.flatMap((path) => {
525
+ const issues = LIMITS.flatMap((path) => {
329
526
  const value = options[path];
330
527
  if (value === undefined || (typeof value === 'number' && Number.isInteger(value) && value >= 1))
331
528
  return [];
@@ -336,6 +533,22 @@ function limitIssues(options) {
336
533
  { path, message: `must be a positive integer${got}` },
337
534
  ];
338
535
  });
536
+ const problem = options.maxReviewTokens === undefined
537
+ ? undefined
538
+ : maxReviewTokensProblem(options.maxReviewTokens);
539
+ if (problem !== undefined)
540
+ issues.push({ path: 'maxReviewTokens', message: problem });
541
+ const timeout = options.reviewTimeoutMs === undefined
542
+ ? undefined
543
+ : timeoutProblem(options.reviewTimeoutMs);
544
+ if (timeout !== undefined)
545
+ issues.push({ path: 'reviewTimeoutMs', message: timeout });
546
+ const partTokens = options.partTokens === undefined
547
+ ? undefined
548
+ : partTokensProblem(options.partTokens);
549
+ if (partTokens !== undefined)
550
+ issues.push({ path: 'partTokens', message: partTokens });
551
+ return issues;
339
552
  }
340
553
  /** The ports of each checked profile in order, then the ports no profile of the run names, then the sink. */
341
554
  function portIssues(ports, profiles, named) {
@@ -393,17 +606,41 @@ function profilePortIssues(entries, profile) {
393
606
  }
394
607
  return issues;
395
608
  }
609
+ /**
610
+ * The issues of one provider: it is missing, it is not one, the `timeoutMs` or the `retries` it
611
+ * states, as own data, is out of range, or the `worstCase` or the `room` it states, as own data, is not a function.
612
+ */
396
613
  function providerIssues(path, value) {
397
614
  if (value === undefined) {
398
615
  return [
399
616
  { path, message: 'is missing' },
400
617
  ];
401
618
  }
402
- return isProvider(value)
403
- ? []
404
- : [
619
+ if (!isProvider(value)) {
620
+ return [
405
621
  { path, message: PROVIDER_SHAPE },
406
622
  ];
623
+ }
624
+ const issues = [];
625
+ const timeoutMs = ownData(value, 'timeoutMs');
626
+ const timeout = timeoutMs === undefined
627
+ ? undefined
628
+ : timeoutProblem(timeoutMs, MAX_STATED_TIMEOUT_MS);
629
+ if (timeout !== undefined)
630
+ issues.push({ path: `${path}.timeoutMs`, message: timeout });
631
+ const retries = ownData(value, 'retries');
632
+ const retry = retries === undefined
633
+ ? undefined
634
+ : retriesProblem(retries);
635
+ if (retry !== undefined)
636
+ issues.push({ path: `${path}.retries`, message: retry });
637
+ const worstCase = ownData(value, 'worstCase');
638
+ if (worstCase !== undefined && typeof worstCase !== 'function')
639
+ issues.push({ path: `${path}.worstCase`, message: 'must be a function' });
640
+ const room = ownData(value, 'room');
641
+ if (room !== undefined && typeof room !== 'function')
642
+ issues.push({ path: `${path}.room`, message: 'must be a function' });
643
+ return issues;
407
644
  }
408
645
  /** A provider is a non-null object with a string id, a string model and a complete function. */
409
646
  function isProvider(value) {
@@ -0,0 +1,36 @@
1
+ /**
2
+ * The retry policy: which failures a call retries and how long it waits before the next attempt.
3
+ * It is plain data in and a number out, so it holds no clock and no thrown value: the call path
4
+ * reads a failure's facts and the call's state, and gives it the randomness.
5
+ */
6
+ /** The longest wait a server may ask for, in milliseconds: a longer hint ends the call. */
7
+ export declare const MAX_HINT_MS = 60000;
8
+ /** What the policy reads of a failure: each as the call path found it, so any of them may be anything. */
9
+ export interface FailureFacts {
10
+ readonly kind: unknown;
11
+ readonly status: unknown;
12
+ readonly retryable: unknown;
13
+ readonly retryAfterMs: unknown;
14
+ }
15
+ /** Where a call stands: the retries it has made, the retries its provider states, whether it timed out already, and the time its review has left. */
16
+ export interface CallState {
17
+ /** Retries made so far: 0 once the first attempt has failed. */
18
+ readonly retried: number;
19
+ readonly retries: number;
20
+ readonly timedOut: boolean;
21
+ /** The milliseconds until the review's deadline: a server's hint longer than this ends the call, since the review would be over before the wait. */
22
+ readonly timeLeftMs: number;
23
+ }
24
+ /** True for the deadline of the core's own, or a gateway's 504 or 524: a slow request is likely slow again, and may be billed each time. */
25
+ export declare function isTimeout(failure: FailureFacts): boolean;
26
+ /**
27
+ * How long to wait before the next attempt of a call, in milliseconds, or undefined when the call
28
+ * ends with this failure. It ends, in this order, when the retries are spent, the failure is not a
29
+ * `network`, `timeout` or `http` one, it does not say `retryable` is `true`, it is a second
30
+ * timeout of the call, or the server asks for more than `MAX_HINT_MS` or for more than the time the
31
+ * review has left. Otherwise the wait is the server's hint, when it is a finite number above 0, plus
32
+ * up to a second; else a random time from 0 to the lesser of 8 s and 0.5 s times 2 to the number of
33
+ * retries made. The call path asks this first, with infinite time left once the review's stop is aborted, so a
34
+ * retry that only the stop prevents is the stop's doing, and it checks the stop after.
35
+ */
36
+ export declare function retryWait(failure: FailureFacts, call: CallState, random: () => number): number | undefined;