@tobi-or-not/quorum 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/bench/fingerprint.d.ts +7 -5
- package/dist/bench/fingerprint.js +22 -9
- package/dist/bench/records.d.ts +70 -0
- package/dist/bench/records.js +30 -0
- package/dist/bench/runner.d.ts +18 -0
- package/dist/bench/runner.js +29 -3
- package/dist/cli/bench.d.ts +11 -4
- package/dist/cli/bench.js +34 -17
- package/dist/cli/bin.js +2 -2
- package/dist/cli/main.d.ts +34 -14
- package/dist/cli/main.js +99 -20
- package/dist/cli/print-config.d.ts +8 -6
- package/dist/cli/print-config.js +21 -8
- package/dist/cli/render.d.ts +2 -2
- package/dist/cli/render.js +21 -2
- package/dist/config/env.d.ts +32 -19
- package/dist/config/env.js +95 -32
- package/dist/config/from-config.js +15 -3
- package/dist/config/resolve.d.ts +9 -1
- package/dist/config/resolve.js +17 -1
- package/dist/config/schema.d.ts +141 -5
- package/dist/config/schema.js +125 -12
- package/dist/core/clock.d.ts +17 -0
- package/dist/core/clock.js +14 -0
- package/dist/core/coverage.d.ts +49 -0
- package/dist/core/coverage.js +144 -0
- package/dist/core/diff.d.ts +24 -0
- package/dist/core/diff.js +63 -18
- package/dist/core/errors.d.ts +12 -3
- package/dist/core/errors.js +21 -2
- package/dist/core/exclude.d.ts +42 -0
- package/dist/core/exclude.js +189 -0
- package/dist/core/limits.d.ts +82 -0
- package/dist/core/limits.js +163 -0
- package/dist/core/output.d.ts +3 -2
- package/dist/core/output.js +3 -2
- package/dist/core/ports.d.ts +57 -2
- package/dist/core/ports.js +36 -1
- package/dist/core/quorum.d.ts +24 -8
- package/dist/core/quorum.js +187 -56
- package/dist/core/retry.d.ts +36 -0
- package/dist/core/retry.js +46 -0
- package/dist/core/stages/context.d.ts +118 -23
- package/dist/core/stages/context.js +304 -46
- package/dist/core/stages/panel.d.ts +5 -4
- package/dist/core/stages/panel.js +30 -11
- package/dist/core/stages/research.d.ts +15 -6
- package/dist/core/stages/research.js +75 -18
- package/dist/core/tokens.d.ts +31 -0
- package/dist/core/tokens.js +81 -0
- package/dist/core/types.d.ts +77 -10
- package/dist/index.d.ts +1 -1
- package/dist/providers/anthropic.d.ts +2 -1
- package/dist/providers/anthropic.js +8 -3
- package/dist/providers/claude-code.d.ts +51 -11
- package/dist/providers/claude-code.js +60 -25
- package/dist/providers/gemini.d.ts +4 -1
- package/dist/providers/gemini.js +63 -11
- package/dist/providers/http.d.ts +49 -4
- package/dist/providers/http.js +155 -14
- package/dist/providers/ollama.d.ts +3 -1
- package/dist/providers/ollama.js +11 -5
- package/dist/providers/openai-compatible.d.ts +3 -1
- package/dist/providers/openai-compatible.js +11 -5
- package/dist/providers/options.d.ts +12 -3
- package/dist/providers/options.js +30 -10
- package/dist/providers/size.d.ts +10 -14
- package/dist/providers/size.js +15 -69
- package/package.json +1 -1
- package/schema/quorum.schema.json +34 -1
|
@@ -1,10 +1,23 @@
|
|
|
1
|
+
import type { Clock } from '../clock.ts';
|
|
1
2
|
import type { Limiter } from '../concurrency.ts';
|
|
3
|
+
import type { ProviderErrorKind } from '../errors.ts';
|
|
4
|
+
import type { Ledger } from '../limits.ts';
|
|
2
5
|
import type { CompletionRequest, EventSink, Provider } from '../ports.ts';
|
|
3
6
|
import type { ProviderRef, StageName, Thresholds, Usage } from '../types.ts';
|
|
4
|
-
/**
|
|
7
|
+
/**
|
|
8
|
+
* A provider as a compiled profile holds it: the object it calls, the ref captured at creation that
|
|
9
|
+
* events and meta report, the deadline of each of its attempts, the retries of a call and the worst case
|
|
10
|
+
* it states.
|
|
11
|
+
*/
|
|
5
12
|
export interface Target {
|
|
6
13
|
readonly provider: Provider;
|
|
7
14
|
readonly ref: Readonly<ProviderRef>;
|
|
15
|
+
/** The deadline of one attempt, in milliseconds: what the provider stated when the quorum was created, else the default. */
|
|
16
|
+
readonly timeoutMs: number;
|
|
17
|
+
/** The retries of a call after its first attempt: what the provider stated when the quorum was created, else none. */
|
|
18
|
+
readonly retries: number;
|
|
19
|
+
/** The worst case of one attempt the provider stated when the quorum was created, when it is a function; `reservationOf` reads its result. */
|
|
20
|
+
readonly worstCase: ((request: CompletionRequest) => unknown) | undefined;
|
|
8
21
|
}
|
|
9
22
|
/** One lens of a compiled profile: its vote instructions, rendered once, and its own provider. */
|
|
10
23
|
export interface LensRun extends Target {
|
|
@@ -33,24 +46,47 @@ export interface ProfileRun {
|
|
|
33
46
|
export interface RunContext {
|
|
34
47
|
/** The review's one concurrency limit, shared by every profile. */
|
|
35
48
|
readonly limit: Limiter;
|
|
49
|
+
/**
|
|
50
|
+
* The review's one token ledger, shared by every profile: each attempt of a call reserves its worst case in
|
|
51
|
+
* it before the attempt is sent and settles after, inside the call's slot, so the attempts in flight can
|
|
52
|
+
* never pass the cap between them.
|
|
53
|
+
*/
|
|
54
|
+
readonly ledger: Ledger;
|
|
36
55
|
/** Delivers one event of the review to the sink, as a copy; it delivers nothing once the review is aborted. */
|
|
37
56
|
readonly emit: EventSink;
|
|
38
57
|
/**
|
|
39
|
-
* Aborted when the caller's signal is, when the event sink throws, when a profile's research fails
|
|
40
|
-
* or when the profiles' run rejects: every stage and every call checks it, and every
|
|
41
|
-
* carries
|
|
58
|
+
* Aborted when the caller's signal is, when the event sink throws, when a gating profile's research fails
|
|
59
|
+
* or when the profiles' run rejects: every stage and every call checks it, and every attempt
|
|
60
|
+
* carries a signal of its own, which `stop` and the attempt's own deadline abort.
|
|
42
61
|
*/
|
|
43
62
|
readonly signal: AbortSignal;
|
|
63
|
+
/**
|
|
64
|
+
* The review's signal joined to its deadline's: aborted by whichever comes first, with that one's reason, which
|
|
65
|
+
* is the deadline's `LimitError` when it is the deadline. The slot's entry, the ledger's wait, every attempt and
|
|
66
|
+
* every backoff read it, so the deadline ends them all at once and no queued call starts. It is no abort of the
|
|
67
|
+
* review: `emit`, `stage` and the stages' catches keep reading `signal`, so an interrupt still rejects with its
|
|
68
|
+
* reason and the events still flow after the deadline, and what the deadline stopped is a gap.
|
|
69
|
+
*/
|
|
70
|
+
readonly stop: AbortSignal;
|
|
71
|
+
/** When the review's deadline falls, `at` on the review's clock, and its length `ms`: the retry policy reads the time left from it. */
|
|
72
|
+
readonly deadline: {
|
|
73
|
+
readonly at: number;
|
|
74
|
+
readonly ms: number;
|
|
75
|
+
};
|
|
44
76
|
/**
|
|
45
77
|
* Aborts the review with `error`, unless it is already aborted, so a caller's abort reason or a
|
|
46
|
-
* sink's error stays the reason. A research failure, the call or its reply, calls
|
|
47
|
-
* fails, so no other profile starts a call after it;
|
|
48
|
-
* a gap,
|
|
78
|
+
* sink's error stays the reason. A gating profile's research failure, the call or its reply, calls
|
|
79
|
+
* it where it fails, so no other profile starts a call after it; an advisory profile's research
|
|
80
|
+
* failure is a gap, like a research request refused as too large, and does not.
|
|
49
81
|
*/
|
|
50
82
|
readonly fail: (error: unknown) => void;
|
|
51
83
|
/** How many findings of a research reply each profile sends to the panel. */
|
|
52
84
|
readonly maxFindings: number;
|
|
85
|
+
/** The time and the randomness the call path reads: the system's, unless a spec injects its own. */
|
|
86
|
+
readonly clock: Clock;
|
|
53
87
|
}
|
|
88
|
+
/** The value of an own data property of an object, else undefined: a getter is never run, an inherited property or a throwing proxy never matches. */
|
|
89
|
+
export declare function ownData(value: unknown, key: string): unknown;
|
|
54
90
|
/**
|
|
55
91
|
* True for a refusal as too large: an object whose own data property `kind` is `'too-large'`, which is what a
|
|
56
92
|
* provider raises when a request may not fit its model's context, before sending it. It is matched by that
|
|
@@ -59,8 +95,24 @@ export interface RunContext {
|
|
|
59
95
|
* never runs a getter and never throws.
|
|
60
96
|
*/
|
|
61
97
|
export declare function isTooLarge(error: unknown): boolean;
|
|
98
|
+
/**
|
|
99
|
+
* What a failed call says of itself, as facts: the own data `kind` when it is one of `PROVIDER_ERROR_KINDS`,
|
|
100
|
+
* and for an `http` failure its own data `status` when that is an integer from 100 to 599. A message, which can
|
|
101
|
+
* quote a server, is never read. It is matched by those properties, not by class, as `isTooLarge` is, and it
|
|
102
|
+
* never runs a getter and never throws: any other value, a kind of no provider's included, is undefined.
|
|
103
|
+
*/
|
|
104
|
+
export declare function providerFailure(error: unknown): {
|
|
105
|
+
readonly kind: ProviderErrorKind;
|
|
106
|
+
readonly status?: number;
|
|
107
|
+
} | undefined;
|
|
62
108
|
/** What a refusal as too large says: its own data string `message`, else a fixed text, never anything else from the object. */
|
|
63
109
|
export declare function refusalMessage(refusal: unknown): string;
|
|
110
|
+
/**
|
|
111
|
+
* The usage of one more attempt added to a running total: both sums are of the attempts that
|
|
112
|
+
* reported valid counts, and `costUsd` is summed over those that reported one. With no more usage
|
|
113
|
+
* the total stays as it is.
|
|
114
|
+
*/
|
|
115
|
+
export declare function addUsage(total: Usage | undefined, more: Usage | undefined): Usage | undefined;
|
|
64
116
|
/**
|
|
65
117
|
* The usage a failed call's thrown value carries, when its counts can be trusted: its own data property
|
|
66
118
|
* `usage`, and that object's own data `inputTokens`, `outputTokens` and `costUsd`, checked as a reply's
|
|
@@ -75,28 +127,71 @@ export interface Requester {
|
|
|
75
127
|
readonly role: 'research' | 'vote';
|
|
76
128
|
readonly lens?: string;
|
|
77
129
|
}
|
|
130
|
+
/**
|
|
131
|
+
* A provider as a run holds it: the ref captured now, which events and meta report, and its
|
|
132
|
+
* deadline and retries, each read as own data and taken only when it passes its check, else the
|
|
133
|
+
* default: 600000 ms and no retry. The quorum's checks ran first, so a plain object's values are the
|
|
134
|
+
* ones they passed; a proxy that answers the check and this read differently still gets an in-range
|
|
135
|
+
* deadline and no retry, never an unbounded one. The worst case it states is its own data value when
|
|
136
|
+
* that is a function, else none: a getter is never run, and an inherited method states nothing.
|
|
137
|
+
*/
|
|
138
|
+
export declare function targetOf(provider: Provider): Target;
|
|
139
|
+
/**
|
|
140
|
+
* The tokens one attempt of `request` is reserved: the worst case the provider states, called once with the
|
|
141
|
+
* request and the provider as `this`, when it returns a safe integer of at least 1. A provider that states
|
|
142
|
+
* none, a result that is anything else (a promise included, whose rejection is marked handled) and a throw
|
|
143
|
+
* all reserve the core's own count of the request plus
|
|
144
|
+
* `TEMPLATE_TOKENS` for what a server adds and `DEFAULT_REPLY_TOKENS` for the reply, so a worst case that
|
|
145
|
+
* misbehaves never reserves nothing. It never throws.
|
|
146
|
+
*/
|
|
147
|
+
export declare function reservationOf(target: Target, request: CompletionRequest): number;
|
|
78
148
|
/** Whole milliseconds since `started`, a `performance.now()` reading. */
|
|
79
149
|
export declare function elapsed(started: number): number;
|
|
80
150
|
/**
|
|
81
151
|
* Call a provider inside the review's concurrency limit and return its reply text. The review's
|
|
82
|
-
*
|
|
83
|
-
*
|
|
84
|
-
* request holds the diff or a section of it, and building it only there keeps the requests a
|
|
152
|
+
* stop is checked inside the slot, before the request is built, so a call still queued when the review
|
|
153
|
+
* is aborted or reaches its deadline never starts. `buildRequest` runs inside the slot once that check
|
|
154
|
+
* passes: a request holds the diff or a section of it, and building it only there keeps the requests a
|
|
85
155
|
* review holds at once to its concurrency, whatever the number of findings and lenses (ADR-0009).
|
|
86
|
-
* The
|
|
87
|
-
*
|
|
88
|
-
*
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
156
|
+
* The call runs its attempts inside its slot, retried as `retryWait` decides, the waits between
|
|
157
|
+
* them holding the slot: the review's abort ends a wait at once, and no failure is retried once the
|
|
158
|
+
* review is aborted. Each attempt has the provider's deadline, and the provider is handed the
|
|
159
|
+
* attempt's own signal, so it can stop a call in flight: the review's abort settles the attempt at
|
|
160
|
+
* once, without waiting for the provider.
|
|
161
|
+
*
|
|
162
|
+
* The review's deadline ends the call's attempt and its waits for room and between attempts at once, as an
|
|
163
|
+
* abort does, but with its own failure, a `LimitError` that is no attempt's `timeout`; a call still queued for
|
|
164
|
+
* its slot ends there when its turn comes, before it builds its request; and no failure is retried once the
|
|
165
|
+
* deadline has passed. A failure the retry policy would have retried, which the stop alone prevents, ends the
|
|
166
|
+
* call with the stop's reason, the deadline's `LimitError` or the interrupt's, never with the failure, so a
|
|
167
|
+
* deadline is not recorded as a failure it cut short; a failure the policy would not have retried keeps its own
|
|
168
|
+
* error. The retry policy is told the time the review has left, so a server's hint longer than it ends the
|
|
169
|
+
* call with its last failure.
|
|
170
|
+
*
|
|
171
|
+
* Each attempt, the first and every retry, reserves the call's worst case (`reservationOf`, taken once
|
|
172
|
+
* the request is built) in the review's ledger before it is sent, and settles it once it ends, inside the
|
|
173
|
+
* slot: at the usage its reply or its failure reports, at nothing when the server refused it before any
|
|
174
|
+
* work, else at the whole reservation. An attempt that does not fit beside what was settled is never
|
|
175
|
+
* sent, and the call fails with the cap's `LimitError`; one that fits but not beside the attempts in
|
|
176
|
+
* flight waits for them, holding its slot, and the review's abort or deadline ends that wait too. A call a limit
|
|
177
|
+
* stops before its first attempt sends no `call` event, since no provider was called; one it stops
|
|
178
|
+
* after an attempt sends its failed event first, as any failed call does.
|
|
179
|
+
*
|
|
180
|
+
* A call that fails once its request is built, for any reason, ends through one place: it sends its
|
|
181
|
+
* failed `call` event when an attempt was made, then hands that failure and the attempts made, 0 when
|
|
182
|
+
* none, to `onFailure`, once, inside its slot, and rethrows. The research stage aborts the review there
|
|
183
|
+
* for a gating profile, so the queued call the slot goes to next never starts; a failed vote has no
|
|
184
|
+
* `onFailure`: the panel records it. The duration is measured inside the slot, from the first attempt,
|
|
185
|
+
* once the request is built, to the call's end, the waits included: the provider's latency, not the time
|
|
186
|
+
* the call waited for a slot or for room, or took to build its request. A reply whose text is not a string
|
|
187
|
+
* is a `ProviderError`. Each call emits one `call` event, which names the provider by the ref captured at
|
|
188
|
+
* creation, never by the live object, and counts its attempts when it made more than one: failed, before
|
|
189
|
+
* the error is rethrown, with the usage summed over its attempts when their counts are valid, or ok, with
|
|
190
|
+
* that usage; both carry `spent`, the sum of what its attempts settled at. The ok event is emitted outside
|
|
191
|
+
* the `try` that catches the provider's failure, so a sink that throws on it is never reported as a failed
|
|
192
|
+
* call.
|
|
98
193
|
*/
|
|
99
|
-
export declare function callProvider(ctx: RunContext, requester: Requester, target: Target, buildRequest: () => CompletionRequest): Promise<string>;
|
|
194
|
+
export declare function callProvider(ctx: RunContext, requester: Requester, target: Target, buildRequest: () => CompletionRequest, onFailure?: (error: unknown, attempts: number) => void): Promise<string>;
|
|
100
195
|
/**
|
|
101
196
|
* Run one stage of a profile, sync or async, between its `stage:start` and `stage:end` events. The
|
|
102
197
|
* review's signal is checked before the stage starts, not after its step: an abort during the step
|
|
@@ -1,9 +1,13 @@
|
|
|
1
|
-
import { ProviderError } from "../errors.js";
|
|
1
|
+
import { PROVIDER_ERROR_KINDS, ProviderError } from "../errors.js";
|
|
2
|
+
import { settlementOf } from "../limits.js";
|
|
2
3
|
import { isRecord } from "../output.js";
|
|
4
|
+
import { DEFAULT_TIMEOUT_MS, MAX_STATED_TIMEOUT_MS, retriesProblem, timeoutProblem } from "../ports.js";
|
|
5
|
+
import { isTimeout, retryWait } from "../retry.js";
|
|
6
|
+
import { DEFAULT_REPLY_TOKENS, requestTokens, TEMPLATE_TOKENS } from "../tokens.js";
|
|
3
7
|
/** What a refusal as too large says when it holds no message of its own. */
|
|
4
8
|
const REFUSAL_FALLBACK = 'refused as too large';
|
|
5
9
|
/** The value of an own data property of an object, else undefined: a getter is never run, an inherited property or a throwing proxy never matches. */
|
|
6
|
-
function ownData(value, key) {
|
|
10
|
+
export function ownData(value, key) {
|
|
7
11
|
if (typeof value !== 'object' || value === null)
|
|
8
12
|
return undefined;
|
|
9
13
|
try {
|
|
@@ -26,6 +30,21 @@ function ownData(value, key) {
|
|
|
26
30
|
export function isTooLarge(error) {
|
|
27
31
|
return ownData(error, 'kind') === 'too-large';
|
|
28
32
|
}
|
|
33
|
+
/**
|
|
34
|
+
* What a failed call says of itself, as facts: the own data `kind` when it is one of `PROVIDER_ERROR_KINDS`,
|
|
35
|
+
* and for an `http` failure its own data `status` when that is an integer from 100 to 599. A message, which can
|
|
36
|
+
* quote a server, is never read. It is matched by those properties, not by class, as `isTooLarge` is, and it
|
|
37
|
+
* never runs a getter and never throws: any other value, a kind of no provider's included, is undefined.
|
|
38
|
+
*/
|
|
39
|
+
export function providerFailure(error) {
|
|
40
|
+
const kind = PROVIDER_ERROR_KINDS.find(known => known === ownData(error, 'kind'));
|
|
41
|
+
if (kind === undefined)
|
|
42
|
+
return undefined;
|
|
43
|
+
const status = ownData(error, 'status');
|
|
44
|
+
return kind === 'http' && typeof status === 'number' && Number.isInteger(status) && status >= 100 && status <= 599
|
|
45
|
+
? { kind, status }
|
|
46
|
+
: { kind };
|
|
47
|
+
}
|
|
29
48
|
/** What a refusal as too large says: its own data string `message`, else a fixed text, never anything else from the object. */
|
|
30
49
|
export function refusalMessage(refusal) {
|
|
31
50
|
const message = ownData(refusal, 'message');
|
|
@@ -33,6 +52,22 @@ export function refusalMessage(refusal) {
|
|
|
33
52
|
? message
|
|
34
53
|
: REFUSAL_FALLBACK;
|
|
35
54
|
}
|
|
55
|
+
/**
|
|
56
|
+
* The usage of one more attempt added to a running total: both sums are of the attempts that
|
|
57
|
+
* reported valid counts, and `costUsd` is summed over those that reported one. With no more usage
|
|
58
|
+
* the total stays as it is.
|
|
59
|
+
*/
|
|
60
|
+
export function addUsage(total, more) {
|
|
61
|
+
if (more === undefined)
|
|
62
|
+
return total;
|
|
63
|
+
const sum = { inputTokens: (total?.inputTokens ?? 0) + more.inputTokens, outputTokens: (total?.outputTokens ?? 0) + more.outputTokens };
|
|
64
|
+
const costUsd = more.costUsd === undefined
|
|
65
|
+
? total?.costUsd
|
|
66
|
+
: (total?.costUsd ?? 0) + more.costUsd;
|
|
67
|
+
if (costUsd !== undefined)
|
|
68
|
+
sum.costUsd = costUsd;
|
|
69
|
+
return sum;
|
|
70
|
+
}
|
|
36
71
|
/**
|
|
37
72
|
* The usage a failed call's thrown value carries, when its counts can be trusted: its own data property
|
|
38
73
|
* `usage`, and that object's own data `inputTokens`, `outputTokens` and `costUsd`, checked as a reply's
|
|
@@ -48,71 +83,294 @@ export function failureUsage(error) {
|
|
|
48
83
|
costUsd: ownData(usage, 'costUsd'),
|
|
49
84
|
});
|
|
50
85
|
}
|
|
86
|
+
/**
|
|
87
|
+
* A provider as a run holds it: the ref captured now, which events and meta report, and its
|
|
88
|
+
* deadline and retries, each read as own data and taken only when it passes its check, else the
|
|
89
|
+
* default: 600000 ms and no retry. The quorum's checks ran first, so a plain object's values are the
|
|
90
|
+
* ones they passed; a proxy that answers the check and this read differently still gets an in-range
|
|
91
|
+
* deadline and no retry, never an unbounded one. The worst case it states is its own data value when
|
|
92
|
+
* that is a function, else none: a getter is never run, and an inherited method states nothing.
|
|
93
|
+
*/
|
|
94
|
+
export function targetOf(provider) {
|
|
95
|
+
const timeoutMs = ownData(provider, 'timeoutMs');
|
|
96
|
+
const retries = ownData(provider, 'retries');
|
|
97
|
+
const worstCase = ownData(provider, 'worstCase');
|
|
98
|
+
return {
|
|
99
|
+
provider,
|
|
100
|
+
ref: Object.freeze({ id: provider.id, model: provider.model }),
|
|
101
|
+
timeoutMs: timeoutProblem(timeoutMs, MAX_STATED_TIMEOUT_MS) === undefined
|
|
102
|
+
? timeoutMs
|
|
103
|
+
: DEFAULT_TIMEOUT_MS,
|
|
104
|
+
retries: retriesProblem(retries) === undefined
|
|
105
|
+
? retries
|
|
106
|
+
: 0,
|
|
107
|
+
worstCase: typeof worstCase === 'function'
|
|
108
|
+
? worstCase
|
|
109
|
+
: undefined,
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* The tokens one attempt of `request` is reserved: the worst case the provider states, called once with the
|
|
114
|
+
* request and the provider as `this`, when it returns a safe integer of at least 1. A provider that states
|
|
115
|
+
* none, a result that is anything else (a promise included, whose rejection is marked handled) and a throw
|
|
116
|
+
* all reserve the core's own count of the request plus
|
|
117
|
+
* `TEMPLATE_TOKENS` for what a server adds and `DEFAULT_REPLY_TOKENS` for the reply, so a worst case that
|
|
118
|
+
* misbehaves never reserves nothing. It never throws.
|
|
119
|
+
*/
|
|
120
|
+
export function reservationOf(target, request) {
|
|
121
|
+
const { worstCase } = target;
|
|
122
|
+
if (worstCase !== undefined) {
|
|
123
|
+
try {
|
|
124
|
+
const stated = Reflect.apply(worstCase, target.provider, [request]);
|
|
125
|
+
if (typeof stated === 'number' && Number.isSafeInteger(stated) && stated >= 1)
|
|
126
|
+
return stated;
|
|
127
|
+
// It is called synchronously: a promise states nothing, and a rejected one must not crash the process.
|
|
128
|
+
if ((typeof stated === 'object' && stated !== null) || typeof stated === 'function') {
|
|
129
|
+
Promise.resolve(stated).then(undefined, () => { });
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
catch {
|
|
133
|
+
// A worst case that throws states nothing.
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
return requestTokens(request) + TEMPLATE_TOKENS + DEFAULT_REPLY_TOKENS;
|
|
137
|
+
}
|
|
51
138
|
/** Whole milliseconds since `started`, a `performance.now()` reading. */
|
|
52
139
|
export function elapsed(started) {
|
|
53
140
|
return Math.round(performance.now() - started);
|
|
54
141
|
}
|
|
55
142
|
/**
|
|
56
143
|
* Call a provider inside the review's concurrency limit and return its reply text. The review's
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
* request holds the diff or a section of it, and building it only there keeps the requests a
|
|
144
|
+
* stop is checked inside the slot, before the request is built, so a call still queued when the review
|
|
145
|
+
* is aborted or reaches its deadline never starts. `buildRequest` runs inside the slot once that check
|
|
146
|
+
* passes: a request holds the diff or a section of it, and building it only there keeps the requests a
|
|
60
147
|
* review holds at once to its concurrency, whatever the number of findings and lenses (ADR-0009).
|
|
61
|
-
* The
|
|
62
|
-
*
|
|
63
|
-
*
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
*
|
|
69
|
-
*
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
*
|
|
148
|
+
* The call runs its attempts inside its slot, retried as `retryWait` decides, the waits between
|
|
149
|
+
* them holding the slot: the review's abort ends a wait at once, and no failure is retried once the
|
|
150
|
+
* review is aborted. Each attempt has the provider's deadline, and the provider is handed the
|
|
151
|
+
* attempt's own signal, so it can stop a call in flight: the review's abort settles the attempt at
|
|
152
|
+
* once, without waiting for the provider.
|
|
153
|
+
*
|
|
154
|
+
* The review's deadline ends the call's attempt and its waits for room and between attempts at once, as an
|
|
155
|
+
* abort does, but with its own failure, a `LimitError` that is no attempt's `timeout`; a call still queued for
|
|
156
|
+
* its slot ends there when its turn comes, before it builds its request; and no failure is retried once the
|
|
157
|
+
* deadline has passed. A failure the retry policy would have retried, which the stop alone prevents, ends the
|
|
158
|
+
* call with the stop's reason, the deadline's `LimitError` or the interrupt's, never with the failure, so a
|
|
159
|
+
* deadline is not recorded as a failure it cut short; a failure the policy would not have retried keeps its own
|
|
160
|
+
* error. The retry policy is told the time the review has left, so a server's hint longer than it ends the
|
|
161
|
+
* call with its last failure.
|
|
162
|
+
*
|
|
163
|
+
* Each attempt, the first and every retry, reserves the call's worst case (`reservationOf`, taken once
|
|
164
|
+
* the request is built) in the review's ledger before it is sent, and settles it once it ends, inside the
|
|
165
|
+
* slot: at the usage its reply or its failure reports, at nothing when the server refused it before any
|
|
166
|
+
* work, else at the whole reservation. An attempt that does not fit beside what was settled is never
|
|
167
|
+
* sent, and the call fails with the cap's `LimitError`; one that fits but not beside the attempts in
|
|
168
|
+
* flight waits for them, holding its slot, and the review's abort or deadline ends that wait too. A call a limit
|
|
169
|
+
* stops before its first attempt sends no `call` event, since no provider was called; one it stops
|
|
170
|
+
* after an attempt sends its failed event first, as any failed call does.
|
|
171
|
+
*
|
|
172
|
+
* A call that fails once its request is built, for any reason, ends through one place: it sends its
|
|
173
|
+
* failed `call` event when an attempt was made, then hands that failure and the attempts made, 0 when
|
|
174
|
+
* none, to `onFailure`, once, inside its slot, and rethrows. The research stage aborts the review there
|
|
175
|
+
* for a gating profile, so the queued call the slot goes to next never starts; a failed vote has no
|
|
176
|
+
* `onFailure`: the panel records it. The duration is measured inside the slot, from the first attempt,
|
|
177
|
+
* once the request is built, to the call's end, the waits included: the provider's latency, not the time
|
|
178
|
+
* the call waited for a slot or for room, or took to build its request. A reply whose text is not a string
|
|
179
|
+
* is a `ProviderError`. Each call emits one `call` event, which names the provider by the ref captured at
|
|
180
|
+
* creation, never by the live object, and counts its attempts when it made more than one: failed, before
|
|
181
|
+
* the error is rethrown, with the usage summed over its attempts when their counts are valid, or ok, with
|
|
182
|
+
* that usage; both carry `spent`, the sum of what its attempts settled at. The ok event is emitted outside
|
|
183
|
+
* the `try` that catches the provider's failure, so a sink that throws on it is never reported as a failed
|
|
184
|
+
* call.
|
|
73
185
|
*/
|
|
74
|
-
export async function callProvider(ctx, requester, target, buildRequest) {
|
|
186
|
+
export async function callProvider(ctx, requester, target, buildRequest, onFailure) {
|
|
75
187
|
return ctx.limit(async () => {
|
|
76
|
-
ctx.
|
|
188
|
+
ctx.stop.throwIfAborted();
|
|
77
189
|
const request = buildRequest();
|
|
78
|
-
const
|
|
190
|
+
const reservation = reservationOf(target, request);
|
|
191
|
+
const started = ctx.clock.now();
|
|
192
|
+
const took = () => Math.round(ctx.clock.now() - started);
|
|
193
|
+
let attempts = 0;
|
|
194
|
+
let usage;
|
|
195
|
+
let spent = 0;
|
|
196
|
+
let timedOut = false;
|
|
79
197
|
let response;
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
198
|
+
let replied;
|
|
199
|
+
/** Settles the attempt's reservation at `tokens`, which the call's `spent` adds. */
|
|
200
|
+
const settle = (tokens) => {
|
|
201
|
+
ctx.ledger.settle(reservation, tokens);
|
|
202
|
+
spent += tokens;
|
|
203
|
+
};
|
|
204
|
+
/** Ends the call with `error`: its failed event when an attempt was made, then `onFailure`; the error to throw. */
|
|
205
|
+
const end = (error) => {
|
|
206
|
+
if (attempts > 0) {
|
|
207
|
+
ctx.emit({
|
|
208
|
+
type: 'call',
|
|
209
|
+
...requester,
|
|
210
|
+
provider: target.ref,
|
|
211
|
+
durationMs: took(),
|
|
212
|
+
ok: false,
|
|
213
|
+
...counted(attempts, usage, spent),
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
// Still inside the slot: a review aborted here never starts the queued call the slot goes to next.
|
|
217
|
+
onFailure?.(error, attempts);
|
|
218
|
+
return error;
|
|
219
|
+
};
|
|
220
|
+
while (response === undefined) {
|
|
221
|
+
try {
|
|
222
|
+
await ctx.ledger.reserve(reservation, ctx.stop);
|
|
223
|
+
}
|
|
224
|
+
catch (error) {
|
|
225
|
+
throw end(error);
|
|
226
|
+
}
|
|
227
|
+
attempts += 1;
|
|
228
|
+
try {
|
|
229
|
+
const reply = await attempt(ctx, target, request);
|
|
230
|
+
if (typeof reply?.text !== 'string')
|
|
231
|
+
throw new ProviderError({ providerId: target.ref.id, kind: 'invalid-response', message: 'returned no reply text' });
|
|
232
|
+
// The reply's usage is read once, so its settlement and its event cannot disagree.
|
|
233
|
+
replied = checkedUsage(reply.usage);
|
|
234
|
+
settle(settlementOf(reservation, replied));
|
|
235
|
+
response = reply;
|
|
236
|
+
}
|
|
237
|
+
catch (error) {
|
|
238
|
+
const reported = failureUsage(error);
|
|
239
|
+
usage = addUsage(usage, reported);
|
|
240
|
+
const facts = failureFacts(error);
|
|
241
|
+
settle(settlementOf(reservation, reported, facts));
|
|
242
|
+
// A stopped review is asked as if it had time to spare, so a retry only the stop prevents is the stop's doing.
|
|
243
|
+
const wait = retryWait(facts, {
|
|
244
|
+
retried: attempts - 1,
|
|
245
|
+
retries: target.retries,
|
|
246
|
+
timedOut,
|
|
247
|
+
timeLeftMs: ctx.stop.aborted
|
|
248
|
+
? Number.POSITIVE_INFINITY
|
|
249
|
+
: ctx.deadline.at - ctx.clock.now(),
|
|
250
|
+
}, ctx.clock.random);
|
|
251
|
+
if (wait === undefined)
|
|
252
|
+
throw end(error);
|
|
253
|
+
if (ctx.stop.aborted)
|
|
254
|
+
throw end(ctx.stop.reason);
|
|
255
|
+
timedOut ||= isTimeout(facts);
|
|
256
|
+
try {
|
|
257
|
+
await pause(ctx, wait);
|
|
258
|
+
}
|
|
259
|
+
catch (stopped) {
|
|
260
|
+
throw end(stopped);
|
|
261
|
+
}
|
|
262
|
+
}
|
|
101
263
|
}
|
|
102
|
-
const usage = checkedUsage(response.usage);
|
|
103
264
|
ctx.emit({
|
|
104
265
|
type: 'call',
|
|
105
266
|
...requester,
|
|
106
267
|
provider: target.ref,
|
|
107
|
-
durationMs:
|
|
268
|
+
durationMs: took(),
|
|
108
269
|
ok: true,
|
|
109
|
-
...(usage
|
|
110
|
-
? {}
|
|
111
|
-
: { usage }),
|
|
270
|
+
...counted(attempts, addUsage(usage, replied), spent),
|
|
112
271
|
});
|
|
113
272
|
return response.text;
|
|
114
273
|
});
|
|
115
274
|
}
|
|
275
|
+
/** The fields of a `call` event that hold what the call's attempts add up to: their number when above one, their summed usage when valid, and the tokens the cap counted. */
|
|
276
|
+
function counted(attempts, usage, spent) {
|
|
277
|
+
return {
|
|
278
|
+
...(attempts > 1
|
|
279
|
+
? { attempts }
|
|
280
|
+
: {}),
|
|
281
|
+
...(usage === undefined
|
|
282
|
+
? {}
|
|
283
|
+
: { usage }),
|
|
284
|
+
spent,
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
/** What the retry policy reads of a thrown value: its `kind`, `status`, `retryable` and `retryAfterMs` as own data, so a getter decides nothing and never runs. */
|
|
288
|
+
function failureFacts(error) {
|
|
289
|
+
return {
|
|
290
|
+
kind: ownData(error, 'kind'),
|
|
291
|
+
status: ownData(error, 'status'),
|
|
292
|
+
retryable: ownData(error, 'retryable'),
|
|
293
|
+
retryAfterMs: ownData(error, 'retryAfterMs'),
|
|
294
|
+
};
|
|
295
|
+
}
|
|
296
|
+
/**
|
|
297
|
+
* Wait `ms` milliseconds between two attempts of a call, which holds its slot meanwhile. The review's
|
|
298
|
+
* abort or deadline ends the wait at once: it rejects with the stop's reason, and the timer is cancelled.
|
|
299
|
+
* Each of the timer and the stop listener removes the other when it goes first.
|
|
300
|
+
*/
|
|
301
|
+
function pause(ctx, ms) {
|
|
302
|
+
if (ctx.stop.aborted)
|
|
303
|
+
return Promise.reject(ctx.stop.reason);
|
|
304
|
+
return new Promise((resolve, reject) => {
|
|
305
|
+
let cancel = () => { };
|
|
306
|
+
function onAbort() {
|
|
307
|
+
cancel();
|
|
308
|
+
reject(ctx.stop.reason);
|
|
309
|
+
}
|
|
310
|
+
cancel = ctx.clock.timer(ms, () => {
|
|
311
|
+
ctx.stop.removeEventListener('abort', onAbort);
|
|
312
|
+
resolve();
|
|
313
|
+
});
|
|
314
|
+
ctx.stop.addEventListener('abort', onAbort, { once: true });
|
|
315
|
+
});
|
|
316
|
+
}
|
|
317
|
+
/**
|
|
318
|
+
* One attempt of a call, settled once, by the first of: the provider's reply or failure; its own
|
|
319
|
+
* deadline, as a `timeout` error of the core's own with the attempt's signal aborted with it as the
|
|
320
|
+
* reason; the review's stop, its abort or its deadline, with the stop's reason. The attempt's timer
|
|
321
|
+
* and the join to the review's stop go when it settles, on every path. A provider that throws before
|
|
322
|
+
* it returns a promise, or returns a plain object, is handled; whatever it does once the attempt
|
|
323
|
+
* settled is caught and ignored, and it is never an unhandled rejection. The join is one listener,
|
|
324
|
+
* not `AbortSignal.any`, so nothing builds up on a signal that outlives thousands of attempts.
|
|
325
|
+
*/
|
|
326
|
+
function attempt(ctx, target, request) {
|
|
327
|
+
if (ctx.stop.aborted)
|
|
328
|
+
return Promise.reject(ctx.stop.reason);
|
|
329
|
+
const controller = new AbortController();
|
|
330
|
+
return new Promise((resolve, reject) => {
|
|
331
|
+
let settled = false;
|
|
332
|
+
let cancel = () => { };
|
|
333
|
+
/** True the first time, when it clears the timer and the listener; false once the attempt has settled. */
|
|
334
|
+
function done() {
|
|
335
|
+
if (settled)
|
|
336
|
+
return false;
|
|
337
|
+
settled = true;
|
|
338
|
+
cancel();
|
|
339
|
+
ctx.stop.removeEventListener('abort', onAbort);
|
|
340
|
+
return true;
|
|
341
|
+
}
|
|
342
|
+
function onAbort() {
|
|
343
|
+
if (!done())
|
|
344
|
+
return;
|
|
345
|
+
controller.abort(ctx.stop.reason);
|
|
346
|
+
reject(ctx.stop.reason);
|
|
347
|
+
}
|
|
348
|
+
cancel = ctx.clock.timer(target.timeoutMs, () => {
|
|
349
|
+
if (!done())
|
|
350
|
+
return;
|
|
351
|
+
const error = new ProviderError({ providerId: target.ref.id, kind: 'timeout', message: `no reply within ${target.timeoutMs} ms` });
|
|
352
|
+
controller.abort(error);
|
|
353
|
+
reject(error);
|
|
354
|
+
});
|
|
355
|
+
ctx.stop.addEventListener('abort', onAbort, { once: true });
|
|
356
|
+
let call;
|
|
357
|
+
try {
|
|
358
|
+
call = Promise.resolve(target.provider.complete({ ...request, signal: controller.signal }));
|
|
359
|
+
}
|
|
360
|
+
catch (error) {
|
|
361
|
+
if (done())
|
|
362
|
+
reject(error);
|
|
363
|
+
return;
|
|
364
|
+
}
|
|
365
|
+
call.then((response) => {
|
|
366
|
+
if (done())
|
|
367
|
+
resolve(response);
|
|
368
|
+
}, (error) => {
|
|
369
|
+
if (done())
|
|
370
|
+
reject(error);
|
|
371
|
+
});
|
|
372
|
+
});
|
|
373
|
+
}
|
|
116
374
|
/**
|
|
117
375
|
* Run one stage of a profile, sync or async, between its `stage:start` and `stage:end` events. The
|
|
118
376
|
* review's signal is checked before the stage starts, not after its step: an abort during the step
|
|
@@ -21,10 +21,11 @@ export interface PanelOutcome {
|
|
|
21
21
|
* self-reported confidence gets to override: with the security profile, unanimous 3/3 is a ceiling
|
|
22
22
|
* of 'high', a 2/3 split is capped at 'medium' no matter how certain any one voter sounded.
|
|
23
23
|
*
|
|
24
|
-
* A failed vote counts as a reject and is recorded with its error, never as a crash,
|
|
25
|
-
* by the review's abort is not a failed vote: it rejects the panel with the abort's
|
|
26
|
-
* without waiting for the votes still in flight. A vote request its provider refused
|
|
27
|
-
* is a failed vote that also records a `request-too-large` gap. Each vote emits a `vote`
|
|
24
|
+
* A failed vote counts as a reject and is recorded with its error and its failure, never as a crash,
|
|
25
|
+
* but a vote cut by the review's abort is not a failed vote: it rejects the panel with the abort's
|
|
26
|
+
* reason at once, without waiting for the votes still in flight. A vote request its provider refused
|
|
27
|
+
* as too large is a failed vote that also records a `request-too-large` gap. Each vote emits a `vote`
|
|
28
|
+
* event.
|
|
28
29
|
* Once a finding's votes are in, its verdict emits a `verdict` event, then its gaps are reported by
|
|
29
30
|
* `incomplete` events: one `request-too-large` gap when any vote request was refused, and a
|
|
30
31
|
* `vote-failed` gap when a kept verdict's failed votes leave a higher level undecided (ADR-0012).
|