@tobi-or-not/quorum 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bench/fingerprint.d.ts +7 -5
- package/dist/bench/fingerprint.js +22 -9
- package/dist/bench/records.d.ts +70 -0
- package/dist/bench/records.js +30 -0
- package/dist/bench/runner.d.ts +18 -0
- package/dist/bench/runner.js +29 -3
- package/dist/cli/bench.d.ts +11 -4
- package/dist/cli/bench.js +34 -17
- package/dist/cli/bin.js +2 -2
- package/dist/cli/main.d.ts +34 -14
- package/dist/cli/main.js +99 -20
- package/dist/cli/print-config.d.ts +8 -6
- package/dist/cli/print-config.js +21 -8
- package/dist/cli/render.d.ts +2 -2
- package/dist/cli/render.js +21 -2
- package/dist/config/env.d.ts +32 -19
- package/dist/config/env.js +95 -32
- package/dist/config/from-config.js +15 -3
- package/dist/config/resolve.d.ts +9 -1
- package/dist/config/resolve.js +17 -1
- package/dist/config/schema.d.ts +141 -5
- package/dist/config/schema.js +125 -12
- package/dist/core/clock.d.ts +17 -0
- package/dist/core/clock.js +14 -0
- package/dist/core/coverage.d.ts +49 -0
- package/dist/core/coverage.js +144 -0
- package/dist/core/diff.d.ts +24 -0
- package/dist/core/diff.js +63 -18
- package/dist/core/errors.d.ts +12 -3
- package/dist/core/errors.js +21 -2
- package/dist/core/exclude.d.ts +42 -0
- package/dist/core/exclude.js +189 -0
- package/dist/core/limits.d.ts +82 -0
- package/dist/core/limits.js +163 -0
- package/dist/core/output.d.ts +3 -2
- package/dist/core/output.js +3 -2
- package/dist/core/ports.d.ts +57 -2
- package/dist/core/ports.js +36 -1
- package/dist/core/quorum.d.ts +24 -8
- package/dist/core/quorum.js +187 -56
- package/dist/core/retry.d.ts +36 -0
- package/dist/core/retry.js +46 -0
- package/dist/core/stages/context.d.ts +118 -23
- package/dist/core/stages/context.js +304 -46
- package/dist/core/stages/panel.d.ts +5 -4
- package/dist/core/stages/panel.js +30 -11
- package/dist/core/stages/research.d.ts +15 -6
- package/dist/core/stages/research.js +75 -18
- package/dist/core/tokens.d.ts +31 -0
- package/dist/core/tokens.js +81 -0
- package/dist/core/types.d.ts +77 -10
- package/dist/index.d.ts +1 -1
- package/dist/providers/anthropic.d.ts +2 -1
- package/dist/providers/anthropic.js +8 -3
- package/dist/providers/claude-code.d.ts +51 -11
- package/dist/providers/claude-code.js +60 -25
- package/dist/providers/gemini.d.ts +4 -1
- package/dist/providers/gemini.js +63 -11
- package/dist/providers/http.d.ts +49 -4
- package/dist/providers/http.js +155 -14
- package/dist/providers/ollama.d.ts +3 -1
- package/dist/providers/ollama.js +11 -5
- package/dist/providers/openai-compatible.d.ts +3 -1
- package/dist/providers/openai-compatible.js +11 -5
- package/dist/providers/options.d.ts +12 -3
- package/dist/providers/options.js +30 -10
- package/dist/providers/size.d.ts +10 -14
- package/dist/providers/size.js +15 -69
- package/package.json +1 -1
- package/schema/quorum.schema.json +34 -1
package/dist/core/quorum.js
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
import packageJson from '../../package.json' with { type: 'json' };
|
|
2
|
+
import { systemClock } from "./clock.js";
|
|
2
3
|
import { createLimiter, DEFAULT_CONCURRENCY } from "./concurrency.js";
|
|
4
|
+
import { leftOutGaps, reviewScope } from "./coverage.js";
|
|
3
5
|
import { splitDiff } from "./diff.js";
|
|
4
6
|
import { ConfigError } from "./errors.js";
|
|
7
|
+
import { excludeIssues, excludeMatcher } from "./exclude.js";
|
|
5
8
|
import { diffFlags } from "./flags.js";
|
|
9
|
+
import { createLedger, DEFAULT_MAX_REVIEW_TOKENS, DEFAULT_REVIEW_TIMEOUT_MS, LimitError, limitGaps, maxReviewTokensProblem } from "./limits.js";
|
|
6
10
|
import { isRecord } from "./output.js";
|
|
11
|
+
import { MAX_STATED_TIMEOUT_MS, retriesProblem, timeoutProblem } from "./ports.js";
|
|
7
12
|
import { profileIssues, profileWarnings, thresholdsFor } from "./profile.js";
|
|
8
13
|
import { renderTemplate } from "./prompts.js";
|
|
9
|
-
import { elapsed, stage } from "./stages/context.js";
|
|
14
|
+
import { addUsage, elapsed, ownData, stage, targetOf } from "./stages/context.js";
|
|
10
15
|
import { dedup } from "./stages/dedup.js";
|
|
11
16
|
import { panel } from "./stages/panel.js";
|
|
12
17
|
import { DEFAULT_MAX_FINDINGS, research } from "./stages/research.js";
|
|
@@ -20,16 +25,23 @@ const LIMITS = ['concurrency', 'maxFindings'];
|
|
|
20
25
|
* Every problem is reported at once, in one ConfigError, never as a TypeError, so an untyped caller
|
|
21
26
|
* gets the checks a config file gets: options and ports that are not objects, no profile, each
|
|
22
27
|
* profile's own issues, a repeated profile id, a `concurrency` or `maxFindings` that is not a
|
|
23
|
-
* positive integer, a
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
* is a
|
|
28
|
+
* positive integer, a `maxReviewTokens` that is not a whole number of tokens from 1 to
|
|
29
|
+
* 9007199254740991, a `reviewTimeoutMs` that is not a whole number of milliseconds from 1000 to
|
|
30
|
+
* 86400000, an `exclude` that is not a list of at most 100 patterns that can be taken, a
|
|
31
|
+
* missing or malformed research provider or voter, a provider's stated `timeoutMs` that is not a
|
|
32
|
+
* whole number of milliseconds from 1000 to 518400000, `retries` that is not a whole number from
|
|
33
|
+
* 0 to 5 or `worstCase` that is not a function, ports for a profile or a lens the run does not have
|
|
34
|
+
* (a likely typo), and an event sink that is not a function. Ports are checked only for the profiles
|
|
35
|
+
* that have no issue of their own, and not at all when `profiles` is not an array. Every record keyed
|
|
36
|
+
* by a caller's id is read as an own property, since `constructor` is a valid id. `maxReviewTokens`,
|
|
37
|
+
* the most tokens one review spends, input and output, is 2000000 when unset, and `reviewTimeoutMs`,
|
|
38
|
+
* how long one review runs, is 1800000 (30 minutes) when unset: see `review`.
|
|
28
39
|
*
|
|
29
40
|
* Each profile is compiled into a frozen `ProfileRun`, its instructions rendered, its thresholds
|
|
30
|
-
* resolved and its providers captured,
|
|
31
|
-
* seen. `warnings` lists what the profiles set
|
|
32
|
-
* stopped through its own signal; a throwing event
|
|
41
|
+
* resolved and its providers captured, and `exclude` is compiled into a matcher, so later changes to
|
|
42
|
+
* the caller's profiles, ports or `exclude` are not seen. `warnings` lists what the profiles set
|
|
43
|
+
* that cannot take full effect. Each review can be stopped through its own signal; a throwing event
|
|
44
|
+
* sink or a failing profile stops it too.
|
|
33
45
|
*
|
|
34
46
|
* @throws ConfigError listing every issue of the options and the ports
|
|
35
47
|
*
|
|
@@ -39,6 +51,16 @@ const LIMITS = ['concurrency', 'maxFindings'];
|
|
|
39
51
|
* const { kept, meta } = await quorum.review({ diff: diffText }, { signal: controller.signal })
|
|
40
52
|
*/
|
|
41
53
|
export function createQuorum(options, ports) {
|
|
54
|
+
return createQuorumWith(options, ports, systemClock);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* `createQuorum` on a clock of the caller's: the time, the randomness and the one timer the call path and the
|
|
58
|
+
* review's deadline read. It is the specs' seam, so a review's deadline and a call's waits run on a fake clock
|
|
59
|
+
* and cost no real time; the package does not export it, and `createQuorum` passes the system's clock.
|
|
60
|
+
*
|
|
61
|
+
* @throws ConfigError listing every issue of the options and the ports, as `createQuorum` does
|
|
62
|
+
*/
|
|
63
|
+
export function createQuorumWith(options, ports, clock) {
|
|
42
64
|
const issues = inputIssues(options, ports);
|
|
43
65
|
if (issues.length > 0)
|
|
44
66
|
throw new ConfigError(issues);
|
|
@@ -47,6 +69,10 @@ export function createQuorum(options, ports) {
|
|
|
47
69
|
runs: Object.freeze(options.profiles.map(profile => compile(profile, ports.profiles[profile.id]))),
|
|
48
70
|
concurrency: options.concurrency ?? DEFAULT_CONCURRENCY,
|
|
49
71
|
maxFindings: options.maxFindings ?? DEFAULT_MAX_FINDINGS,
|
|
72
|
+
maxReviewTokens: options.maxReviewTokens ?? DEFAULT_MAX_REVIEW_TOKENS,
|
|
73
|
+
reviewTimeoutMs: options.reviewTimeoutMs ?? DEFAULT_REVIEW_TIMEOUT_MS,
|
|
74
|
+
clock,
|
|
75
|
+
excluded: excludeMatcher(options.exclude ?? []),
|
|
50
76
|
sink: ports.events,
|
|
51
77
|
};
|
|
52
78
|
const warnings = Object.freeze(options.profiles.flatMap(profile => profileWarnings(profile)));
|
|
@@ -57,33 +83,77 @@ export function createQuorum(options, ports) {
|
|
|
57
83
|
}
|
|
58
84
|
/**
|
|
59
85
|
* One review of a diff by every profile of the quorum, concurrently, each through research, dedup
|
|
60
|
-
* and panel under the review's one concurrency limit; a blank diff runs no stage. The diff is
|
|
61
|
-
* once
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
86
|
+
* and panel under the review's one concurrency limit; a blank diff runs no stage. The diff is
|
|
87
|
+
* filtered once: the sections the quorum's `exclude` patterns match leave every request, so
|
|
88
|
+
* research, the check of each finding and the votes read the diff less those sections, split into
|
|
89
|
+
* one section per file once for every profile, and a finding that cites an excluded file is dropped
|
|
90
|
+
* as outside the diff. A filtered diff that is blank, with every file excluded and nothing before
|
|
91
|
+
* the first header, runs no stage, as a blank diff does. The advisory flags are read before any
|
|
92
|
+
* stage runs, from the whole diff, the excluded files included. The review lists the files it
|
|
93
|
+
* leaves out in `meta.leftOut`: the excluded ones, which are no gap, and the files whose change the
|
|
94
|
+
* diff does not carry, binary files and submodules' commits, for each reason of which it records a
|
|
95
|
+
* gap for every profile; their sections still reach the stages. Everything it knows before a call
|
|
96
|
+
* is sent in this order: `run:start`, then a `left-out` event naming those files when it leaves one
|
|
97
|
+
* out, then the gaps' `incomplete` events, then the `flag` events of the advisory flags, kept in
|
|
98
|
+
* `meta.flags`, none of which changes a verdict, then the stages. Each event is handed to the sink
|
|
99
|
+
* as a copy, so a sink that changes one changes nothing else. The review reports itself through
|
|
100
|
+
* events, from `run:start` to `run:end`, and its result merges the profiles' verdicts and gaps in
|
|
101
|
+
* profile order. `meta.profiles[<id>].invalidFindings` and `meta.usage` are counted from the
|
|
102
|
+
* events before the sink receives them.
|
|
68
103
|
*
|
|
69
104
|
* Abort never fails open. The review's signal joins the caller's `signal` to an internal one; every
|
|
70
|
-
* stage and every call checks it, every
|
|
71
|
-
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
105
|
+
* stage and every call checks it, every attempt carries a signal of its own, which the review's
|
|
106
|
+
* abort and the attempt's deadline abort, and it is checked once more after the profiles have run,
|
|
107
|
+
* a blank diff included, so an aborted review rejects with the signal's reason, whatever error a
|
|
108
|
+
* call it cut raised, and never resolves. No event is delivered once the review is aborted. A sink
|
|
109
|
+
* that throws aborts the review with its error, and a gating profile's research failure (the call or
|
|
74
110
|
* its reply) aborts it with that failure where it fails, so no other profile starts a call after
|
|
75
111
|
* it; any other failure aborts the review when the profiles' run rejects. Either way the review
|
|
76
|
-
* rejects with the first reason. An
|
|
112
|
+
* rejects with the first reason. An advisory profile's research failure is its `research-failed`
|
|
113
|
+
* gap and aborts nothing. An abort after `run:end` has no effect: the review is over.
|
|
114
|
+
*
|
|
115
|
+
* A review ends at its deadline, `reviewTimeoutMs` after it starts, with an answer: at it every call in
|
|
116
|
+
* flight and every wait is stopped at once, no call still queued starts, and what was not done is a
|
|
117
|
+
* `review-deadline` gap of its profile, its research request or its votes, each a failed vote of the
|
|
118
|
+
* deadline. The review then resolves with its findings so far, exiting `4` for a gating profile whatever
|
|
119
|
+
* the flags; the deadline is no abort, so an interrupt still rejects with its reason, before the deadline
|
|
120
|
+
* or after it, and a review that reaches its deadline is never a failed run. The deadline's timer is armed
|
|
121
|
+
* once the input checks pass and cleared however the review ends: it resolves, it rejects, its sink throws
|
|
122
|
+
* or it is aborted.
|
|
123
|
+
*
|
|
124
|
+
* A review spends at most `maxReviewTokens` tokens. Each review has its own ledger of that cap: every
|
|
125
|
+
* attempt of every call reserves its worst case in it before it is sent and settles at what it spent,
|
|
126
|
+
* so concurrent calls cannot overshoot it. A call the cap stops is no failed review: its profile's
|
|
127
|
+
* research or vote is a `token-cap` gap, which exits 4 for a gating profile, and the other profiles
|
|
128
|
+
* and votes run as far as the cap lets them.
|
|
77
129
|
*
|
|
78
130
|
* @throws ConfigError listing every issue of the input and the options: an input that is not an object, a `diff` that is not a string, options that are not an object, a `signal` that is not an AbortSignal
|
|
79
131
|
* @throws the signal's reason once the review is aborted, before it starts or while it runs
|
|
80
|
-
* @throws the error of a sink that throws, or of a profile that fails: a ProviderError from its research call, an OutputError for its reply
|
|
132
|
+
* @throws the error of a sink that throws, or of a gating profile that fails: a ProviderError from its research call, an OutputError for its reply
|
|
81
133
|
*/
|
|
82
134
|
async function review(compiled, input, options = {}) {
|
|
83
135
|
const issues = reviewIssues(input, options);
|
|
84
136
|
if (issues.length > 0)
|
|
85
137
|
throw new ConfigError(issues);
|
|
86
138
|
options.signal?.throwIfAborted();
|
|
139
|
+
const { clock, reviewTimeoutMs } = compiled;
|
|
140
|
+
const deadline = { at: clock.now() + reviewTimeoutMs, ms: reviewTimeoutMs };
|
|
141
|
+
const expired = new AbortController();
|
|
142
|
+
const cancel = clock.timer(reviewTimeoutMs, () => {
|
|
143
|
+
expired.abort(new LimitError('review-deadline', `the review reached its deadline of ${reviewTimeoutMs} ms`));
|
|
144
|
+
});
|
|
145
|
+
try {
|
|
146
|
+
return await reviewUntil(compiled, input, options, deadline, expired.signal);
|
|
147
|
+
}
|
|
148
|
+
finally {
|
|
149
|
+
cancel();
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
/**
|
|
153
|
+
* What `review` does once its deadline is armed: everything from its signals to its result, with `expired`, the
|
|
154
|
+
* deadline's own signal, joined to the review's into the one `stop` the call path reads.
|
|
155
|
+
*/
|
|
156
|
+
async function reviewUntil(compiled, input, options, deadline, expired) {
|
|
87
157
|
const internal = new AbortController();
|
|
88
158
|
const signal = options.signal === undefined
|
|
89
159
|
? internal.signal
|
|
@@ -95,12 +165,25 @@ async function review(compiled, input, options = {}) {
|
|
|
95
165
|
const startedAt = new Date().toISOString();
|
|
96
166
|
const started = performance.now();
|
|
97
167
|
const { runs, sink } = compiled;
|
|
98
|
-
const
|
|
168
|
+
const whole = input.diff.trim() === ''
|
|
99
169
|
? undefined
|
|
100
170
|
: splitDiff(input.diff);
|
|
101
|
-
const flags =
|
|
171
|
+
const flags = whole === undefined
|
|
102
172
|
? []
|
|
103
|
-
: diffFlags(
|
|
173
|
+
: diffFlags(whole);
|
|
174
|
+
const scope = whole === undefined
|
|
175
|
+
? { diff: input.diff, leftOut: [] }
|
|
176
|
+
: reviewScope(input.diff, compiled.excluded);
|
|
177
|
+
// What the stages read: none when the filtered diff is blank, else the sections the flags read when nothing was left out.
|
|
178
|
+
const sections = scope.diff.trim() === ''
|
|
179
|
+
? undefined
|
|
180
|
+
: scope.diff === input.diff
|
|
181
|
+
? whole
|
|
182
|
+
: splitDiff(scope.diff);
|
|
183
|
+
const gaps = new Map(runs.map(run => [
|
|
184
|
+
run.id,
|
|
185
|
+
leftOutGaps(run.id, scope.leftOut),
|
|
186
|
+
]));
|
|
104
187
|
const tally = {
|
|
105
188
|
invalidFindings: new Map(),
|
|
106
189
|
};
|
|
@@ -122,10 +205,14 @@ async function review(compiled, input, options = {}) {
|
|
|
122
205
|
};
|
|
123
206
|
const ctx = {
|
|
124
207
|
limit: createLimiter(compiled.concurrency),
|
|
208
|
+
ledger: createLedger(compiled.maxReviewTokens),
|
|
125
209
|
emit,
|
|
126
210
|
signal,
|
|
211
|
+
stop: AbortSignal.any([signal, expired]),
|
|
212
|
+
deadline,
|
|
127
213
|
fail,
|
|
128
214
|
maxFindings: compiled.maxFindings,
|
|
215
|
+
clock: compiled.clock,
|
|
129
216
|
};
|
|
130
217
|
emit({
|
|
131
218
|
type: 'run:start',
|
|
@@ -135,10 +222,16 @@ async function review(compiled, input, options = {}) {
|
|
|
135
222
|
providersOf(run),
|
|
136
223
|
])),
|
|
137
224
|
});
|
|
225
|
+
if (scope.leftOut.length > 0)
|
|
226
|
+
emit({ type: 'left-out', files: scope.leftOut });
|
|
227
|
+
for (const run of runs) {
|
|
228
|
+
for (const entry of gaps.get(run.id) ?? [])
|
|
229
|
+
emit({ type: 'incomplete', entry });
|
|
230
|
+
}
|
|
138
231
|
for (const flag of flags)
|
|
139
232
|
emit({ type: 'flag', flag });
|
|
140
|
-
const outcomes = await reviewAll(ctx, runs,
|
|
141
|
-
// A research failure aborted the review where it failed; any other failure aborts it here. The first reason stays the reason.
|
|
233
|
+
const outcomes = await reviewAll(ctx, runs, scope.diff, sections).catch((error) => {
|
|
234
|
+
// A gating profile's research failure aborted the review where it failed; any other failure aborts it here. The first reason stays the reason.
|
|
142
235
|
const reason = signal.aborted ? signal.reason : error;
|
|
143
236
|
internal.abort(reason);
|
|
144
237
|
throw reason;
|
|
@@ -147,7 +240,10 @@ async function review(compiled, input, options = {}) {
|
|
|
147
240
|
const kept = outcomes.flatMap(outcome => outcome.kept);
|
|
148
241
|
const discarded = outcomes.flatMap(outcome => outcome.discarded);
|
|
149
242
|
const unreviewed = outcomes.flatMap(outcome => outcome.unreviewed);
|
|
150
|
-
const incomplete =
|
|
243
|
+
const incomplete = runs.flatMap((run, index) => [
|
|
244
|
+
...gaps.get(run.id) ?? [],
|
|
245
|
+
...outcomes[index]?.incomplete ?? [],
|
|
246
|
+
]);
|
|
151
247
|
const durationMs = elapsed(started);
|
|
152
248
|
const meta = {
|
|
153
249
|
version: packageJson.version,
|
|
@@ -161,6 +257,7 @@ async function review(compiled, input, options = {}) {
|
|
|
161
257
|
? {}
|
|
162
258
|
: { usage: tally.usage }),
|
|
163
259
|
incomplete,
|
|
260
|
+
leftOut: scope.leftOut,
|
|
164
261
|
flags,
|
|
165
262
|
};
|
|
166
263
|
emit({
|
|
@@ -179,16 +276,27 @@ async function reviewAll(ctx, runs, diff, sections) {
|
|
|
179
276
|
return [];
|
|
180
277
|
return Promise.all(runs.map(run => reviewProfile(ctx, run, diff, sections)));
|
|
181
278
|
}
|
|
182
|
-
/**
|
|
279
|
+
/**
|
|
280
|
+
* One profile's review: its panel's outcome, with the research gaps before the panel's own, and last the
|
|
281
|
+
* gap of each limit that stopped its work, reported by an `incomplete` event once the panel's stage has ended.
|
|
282
|
+
*/
|
|
183
283
|
async function reviewProfile(ctx, run, diff, sections) {
|
|
184
284
|
const researched = await stage(ctx, run.id, 'research', () => research(ctx, run, diff, sections), outcome => outcome.findings.length);
|
|
185
285
|
const candidates = await stage(ctx, run.id, 'dedup', () => dedup(researched.findings), found => found.length);
|
|
186
286
|
const judged = await stage(ctx, run.id, 'panel', () => panel(ctx, run, candidates, sections, diff), outcome => outcome.kept.length + outcome.discarded.length + outcome.unreviewed.length);
|
|
287
|
+
const limits = limitGaps(run.id, researched.stopped, [
|
|
288
|
+
...judged.kept,
|
|
289
|
+
...judged.discarded,
|
|
290
|
+
...judged.unreviewed,
|
|
291
|
+
], { cap: ctx.ledger.cap, deadlineMs: ctx.deadline.ms });
|
|
292
|
+
for (const entry of limits)
|
|
293
|
+
ctx.emit({ type: 'incomplete', entry });
|
|
187
294
|
return {
|
|
188
295
|
...judged,
|
|
189
296
|
incomplete: [
|
|
190
297
|
...researched.incomplete,
|
|
191
298
|
...judged.incomplete,
|
|
299
|
+
...limits,
|
|
192
300
|
],
|
|
193
301
|
};
|
|
194
302
|
}
|
|
@@ -210,16 +318,6 @@ function reviewIssues(input, options) {
|
|
|
210
318
|
issues.push({ path: 'signal', message: 'must be an AbortSignal' });
|
|
211
319
|
return issues;
|
|
212
320
|
}
|
|
213
|
-
/** The usage of one more call added to a running total; `costUsd` is summed over the calls that reported one. */
|
|
214
|
-
function addUsage(total, call) {
|
|
215
|
-
const sum = { inputTokens: (total?.inputTokens ?? 0) + call.inputTokens, outputTokens: (total?.outputTokens ?? 0) + call.outputTokens };
|
|
216
|
-
const costUsd = call.costUsd === undefined
|
|
217
|
-
? total?.costUsd
|
|
218
|
-
: (total?.costUsd ?? 0) + call.costUsd;
|
|
219
|
-
if (costUsd !== undefined)
|
|
220
|
-
sum.costUsd = costUsd;
|
|
221
|
-
return sum;
|
|
222
|
-
}
|
|
223
321
|
/** Who reviews for a profile, as new objects the caller may keep or change: never the frozen refs of the run. */
|
|
224
322
|
function providersOf(run) {
|
|
225
323
|
return {
|
|
@@ -241,20 +339,21 @@ function profileMeta(run, invalidFindings) {
|
|
|
241
339
|
/**
|
|
242
340
|
* Compile a checked profile and its checked ports into the frozen run every review reads: the vote
|
|
243
341
|
* template rendered once per lens, the thresholds resolved, `gating` true unless the profile says
|
|
244
|
-
* otherwise, and each provider captured beside the `{ id, model }` ref events and meta report
|
|
342
|
+
* otherwise, and each provider captured beside the `{ id, model }` ref events and meta report and
|
|
343
|
+
* the deadline of its attempts.
|
|
245
344
|
*/
|
|
246
345
|
function compile(profile, ports) {
|
|
247
346
|
const panelSize = String(profile.lenses.length);
|
|
248
347
|
const lenses = profile.lenses.map(lens => Object.freeze({
|
|
249
348
|
id: lens.id,
|
|
250
349
|
instructions: renderTemplate(profile.prompts.vote, { 'lens.id': lens.id, 'lens.question': lens.question, 'panel.size': panelSize }),
|
|
251
|
-
...
|
|
350
|
+
...targetOf(ports.voters[lens.id]),
|
|
252
351
|
}));
|
|
253
352
|
return Object.freeze({
|
|
254
353
|
id: profile.id,
|
|
255
354
|
research: Object.freeze({
|
|
256
355
|
instructions: profile.prompts.research,
|
|
257
|
-
...
|
|
356
|
+
...targetOf(ports.research),
|
|
258
357
|
}),
|
|
259
358
|
lenses: Object.freeze(lenses),
|
|
260
359
|
thresholds: Object.freeze(thresholdsFor(profile)),
|
|
@@ -262,15 +361,10 @@ function compile(profile, ports) {
|
|
|
262
361
|
guidelines: profile.guidelines,
|
|
263
362
|
});
|
|
264
363
|
}
|
|
265
|
-
function target(provider) {
|
|
266
|
-
return {
|
|
267
|
-
provider,
|
|
268
|
-
ref: Object.freeze({ id: provider.id, model: provider.model }),
|
|
269
|
-
};
|
|
270
|
-
}
|
|
271
364
|
/**
|
|
272
365
|
* Every problem with the options and the ports, in this order: options and ports that are not
|
|
273
|
-
* objects, `profiles`, each profile's own issues and repeated ids, the limits, then the
|
|
366
|
+
* objects, `profiles`, each profile's own issues and repeated ids, the limits, `exclude`, then the
|
|
367
|
+
* ports.
|
|
274
368
|
*/
|
|
275
369
|
function inputIssues(options, ports) {
|
|
276
370
|
const issues = [];
|
|
@@ -285,11 +379,12 @@ function inputIssues(options, ports) {
|
|
|
285
379
|
...issues,
|
|
286
380
|
{ path: 'profiles', message: 'must be an array of profiles' },
|
|
287
381
|
...limitIssues(options),
|
|
382
|
+
...excludeIssues(options.exclude),
|
|
288
383
|
];
|
|
289
384
|
}
|
|
290
385
|
// `Array.from` reads a hole as `undefined`, so it is reported like any other profile that is not an object.
|
|
291
386
|
const profiles = profileListIssues(Array.from(options.profiles));
|
|
292
|
-
issues.push(...profiles.issues, ...limitIssues(options));
|
|
387
|
+
issues.push(...profiles.issues, ...limitIssues(options), ...excludeIssues(options.exclude));
|
|
293
388
|
if (isRecord(ports))
|
|
294
389
|
issues.push(...portIssues(ports, profiles.checked, profiles.named));
|
|
295
390
|
return issues;
|
|
@@ -323,9 +418,13 @@ function profileListIssues(profiles) {
|
|
|
323
418
|
});
|
|
324
419
|
return { issues, checked, named };
|
|
325
420
|
}
|
|
326
|
-
/**
|
|
421
|
+
/**
|
|
422
|
+
* `concurrency` and `maxFindings` are positive integers when set, `maxReviewTokens` a whole number of
|
|
423
|
+
* tokens from 1 to 9007199254740991, and `reviewTimeoutMs` a whole number of milliseconds from 1000 to
|
|
424
|
+
* 86400000. A number is quoted in the first two; anything else is described by the rule alone.
|
|
425
|
+
*/
|
|
327
426
|
function limitIssues(options) {
|
|
328
|
-
|
|
427
|
+
const issues = LIMITS.flatMap((path) => {
|
|
329
428
|
const value = options[path];
|
|
330
429
|
if (value === undefined || (typeof value === 'number' && Number.isInteger(value) && value >= 1))
|
|
331
430
|
return [];
|
|
@@ -336,6 +435,17 @@ function limitIssues(options) {
|
|
|
336
435
|
{ path, message: `must be a positive integer${got}` },
|
|
337
436
|
];
|
|
338
437
|
});
|
|
438
|
+
const problem = options.maxReviewTokens === undefined
|
|
439
|
+
? undefined
|
|
440
|
+
: maxReviewTokensProblem(options.maxReviewTokens);
|
|
441
|
+
if (problem !== undefined)
|
|
442
|
+
issues.push({ path: 'maxReviewTokens', message: problem });
|
|
443
|
+
const timeout = options.reviewTimeoutMs === undefined
|
|
444
|
+
? undefined
|
|
445
|
+
: timeoutProblem(options.reviewTimeoutMs);
|
|
446
|
+
if (timeout !== undefined)
|
|
447
|
+
issues.push({ path: 'reviewTimeoutMs', message: timeout });
|
|
448
|
+
return issues;
|
|
339
449
|
}
|
|
340
450
|
/** The ports of each checked profile in order, then the ports no profile of the run names, then the sink. */
|
|
341
451
|
function portIssues(ports, profiles, named) {
|
|
@@ -393,17 +503,38 @@ function profilePortIssues(entries, profile) {
|
|
|
393
503
|
}
|
|
394
504
|
return issues;
|
|
395
505
|
}
|
|
506
|
+
/**
|
|
507
|
+
* The issues of one provider: it is missing, it is not one, the `timeoutMs` or the `retries` it
|
|
508
|
+
* states, as own data, is out of range, or the `worstCase` it states, as own data, is not a function.
|
|
509
|
+
*/
|
|
396
510
|
function providerIssues(path, value) {
|
|
397
511
|
if (value === undefined) {
|
|
398
512
|
return [
|
|
399
513
|
{ path, message: 'is missing' },
|
|
400
514
|
];
|
|
401
515
|
}
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
: [
|
|
516
|
+
if (!isProvider(value)) {
|
|
517
|
+
return [
|
|
405
518
|
{ path, message: PROVIDER_SHAPE },
|
|
406
519
|
];
|
|
520
|
+
}
|
|
521
|
+
const issues = [];
|
|
522
|
+
const timeoutMs = ownData(value, 'timeoutMs');
|
|
523
|
+
const timeout = timeoutMs === undefined
|
|
524
|
+
? undefined
|
|
525
|
+
: timeoutProblem(timeoutMs, MAX_STATED_TIMEOUT_MS);
|
|
526
|
+
if (timeout !== undefined)
|
|
527
|
+
issues.push({ path: `${path}.timeoutMs`, message: timeout });
|
|
528
|
+
const retries = ownData(value, 'retries');
|
|
529
|
+
const retry = retries === undefined
|
|
530
|
+
? undefined
|
|
531
|
+
: retriesProblem(retries);
|
|
532
|
+
if (retry !== undefined)
|
|
533
|
+
issues.push({ path: `${path}.retries`, message: retry });
|
|
534
|
+
const worstCase = ownData(value, 'worstCase');
|
|
535
|
+
if (worstCase !== undefined && typeof worstCase !== 'function')
|
|
536
|
+
issues.push({ path: `${path}.worstCase`, message: 'must be a function' });
|
|
537
|
+
return issues;
|
|
407
538
|
}
|
|
408
539
|
/** A provider is a non-null object with a string id, a string model and a complete function. */
|
|
409
540
|
function isProvider(value) {
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The retry policy: which failures a call retries and how long it waits before the next attempt.
|
|
3
|
+
* It is plain data in and a number out, so it holds no clock and no thrown value: the call path
|
|
4
|
+
* reads a failure's facts and the call's state, and gives it the randomness.
|
|
5
|
+
*/
|
|
6
|
+
/** The longest wait a server may ask for, in milliseconds: a longer hint ends the call. */
|
|
7
|
+
export declare const MAX_HINT_MS = 60000;
|
|
8
|
+
/** What the policy reads of a failure: each as the call path found it, so any of them may be anything. */
|
|
9
|
+
export interface FailureFacts {
|
|
10
|
+
readonly kind: unknown;
|
|
11
|
+
readonly status: unknown;
|
|
12
|
+
readonly retryable: unknown;
|
|
13
|
+
readonly retryAfterMs: unknown;
|
|
14
|
+
}
|
|
15
|
+
/** Where a call stands: the retries it has made, the retries its provider states, whether it timed out already, and the time its review has left. */
|
|
16
|
+
export interface CallState {
|
|
17
|
+
/** Retries made so far: 0 once the first attempt has failed. */
|
|
18
|
+
readonly retried: number;
|
|
19
|
+
readonly retries: number;
|
|
20
|
+
readonly timedOut: boolean;
|
|
21
|
+
/** The milliseconds until the review's deadline: a server's hint longer than this ends the call, since the review would be over before the wait. */
|
|
22
|
+
readonly timeLeftMs: number;
|
|
23
|
+
}
|
|
24
|
+
/** True for the deadline of the core's own, or a gateway's 504 or 524: a slow request is likely slow again, and may be billed each time. */
|
|
25
|
+
export declare function isTimeout(failure: FailureFacts): boolean;
|
|
26
|
+
/**
|
|
27
|
+
* How long to wait before the next attempt of a call, in milliseconds, or undefined when the call
|
|
28
|
+
* ends with this failure. It ends, in this order, when the retries are spent, the failure is not a
|
|
29
|
+
* `network`, `timeout` or `http` one, it does not say `retryable` is `true`, it is a second
|
|
30
|
+
* timeout of the call, or the server asks for more than `MAX_HINT_MS` or for more than the time the
|
|
31
|
+
* review has left. Otherwise the wait is the server's hint, when it is a finite number above 0, plus
|
|
32
|
+
* up to a second; else a random time from 0 to the lesser of 8 s and 0.5 s times 2 to the number of
|
|
33
|
+
* retries made. The call path asks this first, with infinite time left once the review's stop is aborted, so a
|
|
34
|
+
* retry that only the stop prevents is the stop's doing, and it checks the stop after.
|
|
35
|
+
*/
|
|
36
|
+
export declare function retryWait(failure: FailureFacts, call: CallState, random: () => number): number | undefined;
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The retry policy: which failures a call retries and how long it waits before the next attempt.
|
|
3
|
+
* It is plain data in and a number out, so it holds no clock and no thrown value: the call path
|
|
4
|
+
* reads a failure's facts and the call's state, and gives it the randomness.
|
|
5
|
+
*/
|
|
6
|
+
/** The longest wait a server may ask for, in milliseconds: a longer hint ends the call. */
|
|
7
|
+
export const MAX_HINT_MS = 60_000;
|
|
8
|
+
/** The backoff before retry n is random from 0 to the lesser of `BACKOFF_CAP_MS` and `BACKOFF_BASE_MS` times 2 to the n. */
|
|
9
|
+
const BACKOFF_BASE_MS = 500;
|
|
10
|
+
const BACKOFF_CAP_MS = 8000;
|
|
11
|
+
/** What a server's hint is stretched by, at most, so calls that were told the same wait do not retry together. */
|
|
12
|
+
const HINT_JITTER_MS = 1000;
|
|
13
|
+
/** The only failures a call retries, whatever a failure says of itself. */
|
|
14
|
+
const RETRIED_KINDS = new Set(['network', 'timeout', 'http']);
|
|
15
|
+
/** The statuses of a gateway that gave up waiting for the server, which count as a timeout. */
|
|
16
|
+
const GATEWAY_TIMEOUTS = new Set([504, 524]);
|
|
17
|
+
/** True for the deadline of the core's own, or a gateway's 504 or 524: a slow request is likely slow again, and may be billed each time. */
|
|
18
|
+
export function isTimeout(failure) {
|
|
19
|
+
return failure.kind === 'timeout' || (failure.kind === 'http' && GATEWAY_TIMEOUTS.has(failure.status));
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* How long to wait before the next attempt of a call, in milliseconds, or undefined when the call
|
|
23
|
+
* ends with this failure. It ends, in this order, when the retries are spent, the failure is not a
|
|
24
|
+
* `network`, `timeout` or `http` one, it does not say `retryable` is `true`, it is a second
|
|
25
|
+
* timeout of the call, or the server asks for more than `MAX_HINT_MS` or for more than the time the
|
|
26
|
+
* review has left. Otherwise the wait is the server's hint, when it is a finite number above 0, plus
|
|
27
|
+
* up to a second; else a random time from 0 to the lesser of 8 s and 0.5 s times 2 to the number of
|
|
28
|
+
* retries made. The call path asks this first, with infinite time left once the review's stop is aborted, so a
|
|
29
|
+
* retry that only the stop prevents is the stop's doing, and it checks the stop after.
|
|
30
|
+
*/
|
|
31
|
+
export function retryWait(failure, call, random) {
|
|
32
|
+
// Not `>=`: a retry count that is not a number compares false both ways, and must end the call.
|
|
33
|
+
if (!(call.retried < call.retries))
|
|
34
|
+
return undefined;
|
|
35
|
+
if (!RETRIED_KINDS.has(failure.kind) || failure.retryable !== true)
|
|
36
|
+
return undefined;
|
|
37
|
+
if (call.timedOut && isTimeout(failure))
|
|
38
|
+
return undefined;
|
|
39
|
+
const hint = failure.retryAfterMs;
|
|
40
|
+
if (typeof hint === 'number' && Number.isFinite(hint) && hint > 0) {
|
|
41
|
+
return hint > MAX_HINT_MS || hint > call.timeLeftMs
|
|
42
|
+
? undefined
|
|
43
|
+
: hint + random() * HINT_JITTER_MS;
|
|
44
|
+
}
|
|
45
|
+
return random() * Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * 2 ** call.retried);
|
|
46
|
+
}
|