@tobi-or-not/quorum 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +2 -1
  2. package/dist/bench/attacks.d.ts +103 -0
  3. package/dist/bench/attacks.js +272 -0
  4. package/dist/bench/fingerprint.d.ts +7 -4
  5. package/dist/bench/fingerprint.js +22 -8
  6. package/dist/bench/independence.d.ts +28 -0
  7. package/dist/bench/independence.js +67 -0
  8. package/dist/bench/records.d.ts +241 -13
  9. package/dist/bench/records.js +64 -11
  10. package/dist/bench/report.d.ts +44 -11
  11. package/dist/bench/report.js +238 -32
  12. package/dist/bench/run-dir.d.ts +52 -26
  13. package/dist/bench/run-dir.js +124 -45
  14. package/dist/bench/runner.d.ts +47 -17
  15. package/dist/bench/runner.js +82 -21
  16. package/dist/bench/score.d.ts +38 -7
  17. package/dist/bench/score.js +64 -17
  18. package/dist/bench/stats.d.ts +61 -7
  19. package/dist/bench/stats.js +82 -7
  20. package/dist/bench/suite.d.ts +54 -9
  21. package/dist/bench/suite.js +264 -96
  22. package/dist/cli/bench.d.ts +27 -9
  23. package/dist/cli/bench.js +174 -83
  24. package/dist/cli/bin.js +2 -2
  25. package/dist/cli/io.d.ts +9 -2
  26. package/dist/cli/io.js +20 -2
  27. package/dist/cli/main.d.ts +34 -14
  28. package/dist/cli/main.js +100 -38
  29. package/dist/cli/print-config.d.ts +8 -6
  30. package/dist/cli/print-config.js +21 -8
  31. package/dist/cli/render.d.ts +2 -2
  32. package/dist/cli/render.js +21 -2
  33. package/dist/config/env.d.ts +32 -19
  34. package/dist/config/env.js +95 -32
  35. package/dist/config/from-config.js +15 -3
  36. package/dist/config/load.d.ts +3 -0
  37. package/dist/config/load.js +17 -8
  38. package/dist/config/resolve.d.ts +9 -1
  39. package/dist/config/resolve.js +17 -1
  40. package/dist/config/schema.d.ts +141 -5
  41. package/dist/config/schema.js +125 -12
  42. package/dist/core/clock.d.ts +17 -0
  43. package/dist/core/clock.js +14 -0
  44. package/dist/core/coverage.d.ts +49 -0
  45. package/dist/core/coverage.js +144 -0
  46. package/dist/core/diff.d.ts +34 -0
  47. package/dist/core/diff.js +140 -18
  48. package/dist/core/errors.d.ts +18 -3
  49. package/dist/core/errors.js +26 -2
  50. package/dist/core/exclude.d.ts +42 -0
  51. package/dist/core/exclude.js +189 -0
  52. package/dist/core/limits.d.ts +82 -0
  53. package/dist/core/limits.js +163 -0
  54. package/dist/core/output.d.ts +3 -2
  55. package/dist/core/output.js +3 -2
  56. package/dist/core/ports.d.ts +57 -2
  57. package/dist/core/ports.js +36 -1
  58. package/dist/core/quorum.d.ts +24 -8
  59. package/dist/core/quorum.js +187 -56
  60. package/dist/core/retry.d.ts +36 -0
  61. package/dist/core/retry.js +46 -0
  62. package/dist/core/stages/context.d.ts +126 -22
  63. package/dist/core/stages/context.js +319 -41
  64. package/dist/core/stages/panel.d.ts +5 -4
  65. package/dist/core/stages/panel.js +30 -11
  66. package/dist/core/stages/research.d.ts +15 -6
  67. package/dist/core/stages/research.js +75 -18
  68. package/dist/core/tokens.d.ts +31 -0
  69. package/dist/core/tokens.js +81 -0
  70. package/dist/core/types.d.ts +77 -10
  71. package/dist/index.d.ts +1 -1
  72. package/dist/providers/anthropic.d.ts +2 -1
  73. package/dist/providers/anthropic.js +14 -6
  74. package/dist/providers/claude-code.d.ts +51 -11
  75. package/dist/providers/claude-code.js +69 -33
  76. package/dist/providers/gemini.d.ts +4 -1
  77. package/dist/providers/gemini.js +72 -15
  78. package/dist/providers/http.d.ts +49 -4
  79. package/dist/providers/http.js +155 -14
  80. package/dist/providers/ollama.d.ts +3 -1
  81. package/dist/providers/ollama.js +17 -9
  82. package/dist/providers/openai-compatible.d.ts +3 -1
  83. package/dist/providers/openai-compatible.js +17 -9
  84. package/dist/providers/options.d.ts +12 -3
  85. package/dist/providers/options.js +30 -10
  86. package/dist/providers/reply.d.ts +6 -2
  87. package/dist/providers/reply.js +13 -3
  88. package/dist/providers/size.d.ts +10 -14
  89. package/dist/providers/size.js +15 -69
  90. package/package.json +1 -1
  91. package/schema/quorum.schema.json +34 -1
@@ -1,10 +1,23 @@
1
+ import type { Clock } from '../clock.ts';
1
2
  import type { Limiter } from '../concurrency.ts';
3
+ import type { ProviderErrorKind } from '../errors.ts';
4
+ import type { Ledger } from '../limits.ts';
2
5
  import type { CompletionRequest, EventSink, Provider } from '../ports.ts';
3
6
  import type { ProviderRef, StageName, Thresholds, Usage } from '../types.ts';
4
- /** A provider as a compiled profile holds it: the object it calls, and the ref captured at creation that events and meta report. */
7
+ /**
8
+ * A provider as a compiled profile holds it: the object it calls, the ref captured at creation that
9
+ * events and meta report, the deadline of each of its attempts, the retries of a call and the worst case
10
+ * it states.
11
+ */
5
12
  export interface Target {
6
13
  readonly provider: Provider;
7
14
  readonly ref: Readonly<ProviderRef>;
15
+ /** The deadline of one attempt, in milliseconds: what the provider stated when the quorum was created, else the default. */
16
+ readonly timeoutMs: number;
17
+ /** The retries of a call after its first attempt: what the provider stated when the quorum was created, else none. */
18
+ readonly retries: number;
19
+ /** The worst case of one attempt the provider stated when the quorum was created, when it is a function; `reservationOf` reads its result. */
20
+ readonly worstCase: ((request: CompletionRequest) => unknown) | undefined;
8
21
  }
9
22
  /** One lens of a compiled profile: its vote instructions, rendered once, and its own provider. */
10
23
  export interface LensRun extends Target {
@@ -33,24 +46,47 @@ export interface ProfileRun {
33
46
  export interface RunContext {
34
47
  /** The review's one concurrency limit, shared by every profile. */
35
48
  readonly limit: Limiter;
49
+ /**
50
+ * The review's one token ledger, shared by every profile: each attempt of a call reserves its worst case in
51
+ * it before the attempt is sent and settles after, inside the call's slot, so the attempts in flight can
52
+ * never pass the cap between them.
53
+ */
54
+ readonly ledger: Ledger;
36
55
  /** Delivers one event of the review to the sink, as a copy; it delivers nothing once the review is aborted. */
37
56
  readonly emit: EventSink;
38
57
  /**
39
- * Aborted when the caller's signal is, when the event sink throws, when a profile's research fails
40
- * or when the profiles' run rejects: every stage and every call checks it, and every request
41
- * carries it.
58
+ * Aborted when the caller's signal is, when the event sink throws, when a gating profile's research fails
59
+ * or when the profiles' run rejects: every stage and every call checks it, and every attempt
60
+ * carries a signal of its own, which `stop` and the attempt's own deadline abort.
42
61
  */
43
62
  readonly signal: AbortSignal;
63
+ /**
64
+ * The review's signal joined to its deadline's: aborted by whichever comes first, with that one's reason, which
65
+ * is the deadline's `LimitError` when it is the deadline. The slot's entry, the ledger's wait, every attempt and
66
+ * every backoff read it, so the deadline ends them all at once and no queued call starts. It is no abort of the
67
+ * review: `emit`, `stage` and the stages' catches keep reading `signal`, so an interrupt still rejects with its
68
+ * reason and the events still flow after the deadline, and what the deadline stopped is a gap.
69
+ */
70
+ readonly stop: AbortSignal;
71
+ /** When the review's deadline falls, `at` on the review's clock, and its length `ms`: the retry policy reads the time left from it. */
72
+ readonly deadline: {
73
+ readonly at: number;
74
+ readonly ms: number;
75
+ };
44
76
  /**
45
77
  * Aborts the review with `error`, unless it is already aborted, so a caller's abort reason or a
46
- * sink's error stays the reason. A research failure, the call or its reply, calls it where it
47
- * fails, so no other profile starts a call after it; a research request refused as too large is
48
- * a gap, not a failure, and does not.
78
+ * sink's error stays the reason. A gating profile's research failure, the call or its reply, calls
79
+ * it where it fails, so no other profile starts a call after it; an advisory profile's research
80
+ * failure is a gap, like a research request refused as too large, and does not.
49
81
  */
50
82
  readonly fail: (error: unknown) => void;
51
83
  /** How many findings of a research reply each profile sends to the panel. */
52
84
  readonly maxFindings: number;
85
+ /** The time and the randomness the call path reads: the system's, unless a spec injects its own. */
86
+ readonly clock: Clock;
53
87
  }
88
+ /** The value of an own data property of an object, else undefined: a getter is never run, an inherited property or a throwing proxy never matches. */
89
+ export declare function ownData(value: unknown, key: string): unknown;
54
90
  /**
55
91
  * True for a refusal as too large: an object whose own data property `kind` is `'too-large'`, which is what a
56
92
  * provider raises when a request may not fit its model's context, before sending it. It is matched by that
@@ -59,35 +95,103 @@ export interface RunContext {
59
95
  * never runs a getter and never throws.
60
96
  */
61
97
  export declare function isTooLarge(error: unknown): boolean;
98
+ /**
99
+ * What a failed call says of itself, as facts: the own data `kind` when it is one of `PROVIDER_ERROR_KINDS`,
100
+ * and for an `http` failure its own data `status` when that is an integer from 100 to 599. A message, which can
101
+ * quote a server, is never read. It is matched by those properties, not by class, as `isTooLarge` is, and it
102
+ * never runs a getter and never throws: any other value, a kind of no provider's included, is undefined.
103
+ */
104
+ export declare function providerFailure(error: unknown): {
105
+ readonly kind: ProviderErrorKind;
106
+ readonly status?: number;
107
+ } | undefined;
62
108
  /** What a refusal as too large says: its own data string `message`, else a fixed text, never anything else from the object. */
63
109
  export declare function refusalMessage(refusal: unknown): string;
110
+ /**
111
+ * The usage of one more attempt added to a running total: both sums are of the attempts that
112
+ * reported valid counts, and `costUsd` is summed over those that reported one. With no more usage
113
+ * the total stays as it is.
114
+ */
115
+ export declare function addUsage(total: Usage | undefined, more: Usage | undefined): Usage | undefined;
116
+ /**
117
+ * The usage a failed call's thrown value carries, when its counts can be trusted: its own data property
118
+ * `usage`, and that object's own data `inputTokens`, `outputTokens` and `costUsd`, checked as a reply's
119
+ * usage is. It is matched by those properties, not by class, as `isTooLarge` is, so a `ProviderError`
120
+ * from a second copy of quorum or a library provider's own error object counts too. It never runs a
121
+ * getter and never throws: an inherited property, a getter or a throwing proxy reads as no usage.
122
+ */
123
+ export declare function failureUsage(error: unknown): Usage | undefined;
64
124
  /** Who requests a provider call, as its `call` event names it: `lens` only for a vote. */
65
125
  export interface Requester {
66
126
  readonly profile: string;
67
127
  readonly role: 'research' | 'vote';
68
128
  readonly lens?: string;
69
129
  }
130
+ /**
131
+ * A provider as a run holds it: the ref captured now, which events and meta report, and its
132
+ * deadline and retries, each read as own data and taken only when it passes its check, else the
133
+ * default: 600000 ms and no retry. The quorum's checks ran first, so a plain object's values are the
134
+ * ones they passed; a proxy that answers the check and this read differently still gets an in-range
135
+ * deadline and no retry, never an unbounded one. The worst case it states is its own data value when
136
+ * that is a function, else none: a getter is never run, and an inherited method states nothing.
137
+ */
138
+ export declare function targetOf(provider: Provider): Target;
139
+ /**
140
+ * The tokens one attempt of `request` is reserved: the worst case the provider states, called once with the
141
+ * request and the provider as `this`, when it returns a safe integer of at least 1. A provider that states
142
+ * none, a result that is anything else (a promise included, whose rejection is marked handled) and a throw
143
+ * all reserve the core's own count of the request plus
144
+ * `TEMPLATE_TOKENS` for what a server adds and `DEFAULT_REPLY_TOKENS` for the reply, so a worst case that
145
+ * misbehaves never reserves nothing. It never throws.
146
+ */
147
+ export declare function reservationOf(target: Target, request: CompletionRequest): number;
70
148
  /** Whole milliseconds since `started`, a `performance.now()` reading. */
71
149
  export declare function elapsed(started: number): number;
72
150
  /**
73
151
  * Call a provider inside the review's concurrency limit and return its reply text. The review's
74
- * signal is checked inside the slot, before the request is built, so a call still queued when the
75
- * review is aborted never starts. `buildRequest` runs inside the slot once that check passes: a
76
- * request holds the diff or a section of it, and building it only there keeps the requests a
152
+ * stop is checked inside the slot, before the request is built, so a call still queued when the review
153
+ * is aborted or reaches its deadline never starts. `buildRequest` runs inside the slot once that check
154
+ * passes: a request holds the diff or a section of it, and building it only there keeps the requests a
77
155
  * review holds at once to its concurrency, whatever the number of findings and lenses (ADR-0009).
78
- * The request carries the signal, so the provider can stop a call in flight. A failed research
79
- * call aborts the review through `ctx.fail` before it leaves its slot, so the queued call the slot
80
- * goes to next never starts, unless the provider refused the request as too large: that refusal is
81
- * a gap, never a reason to stop the other profiles, so the research stage records it. A failed vote
82
- * does not abort the review, since the panel records it. The duration is measured inside the slot,
83
- * once the request is built: the provider's latency, not the time the call waited for a slot or
84
- * took to build its request. A reply whose text is not a string is a `ProviderError`. Each call
85
- * emits one `call` event, which names the provider by the ref captured at creation, never by the
86
- * live object: failed, before the error is rethrown, or ok, with the usage when its counts are
87
- * valid. The ok event is emitted outside the `try` that catches the provider's failure, so a sink
88
- * that throws on it is never reported as a failed call.
156
+ * The call runs its attempts inside its slot, retried as `retryWait` decides, the waits between
157
+ * them holding the slot: the review's abort ends a wait at once, and no failure is retried once the
158
+ * review is aborted. Each attempt has the provider's deadline, and the provider is handed the
159
+ * attempt's own signal, so it can stop a call in flight: the review's abort settles the attempt at
160
+ * once, without waiting for the provider.
161
+ *
162
+ * The review's deadline ends the call's attempt and its waits for room and between attempts at once, as an
163
+ * abort does, but with its own failure, a `LimitError` that is no attempt's `timeout`; a call still queued for
164
+ * its slot ends there when its turn comes, before it builds its request; and no failure is retried once the
165
+ * deadline has passed. A failure the retry policy would have retried, which the stop alone prevents, ends the
166
+ * call with the stop's reason, the deadline's `LimitError` or the interrupt's, never with the failure, so a
167
+ * deadline is not recorded as a failure it cut short; a failure the policy would not have retried keeps its own
168
+ * error. The retry policy is told the time the review has left, so a server's hint longer than it ends the
169
+ * call with its last failure.
170
+ *
171
+ * Each attempt, the first and every retry, reserves the call's worst case (`reservationOf`, taken once
172
+ * the request is built) in the review's ledger before it is sent, and settles it once it ends, inside the
173
+ * slot: at the usage its reply or its failure reports, at nothing when the server refused it before any
174
+ * work, else at the whole reservation. An attempt that does not fit beside what was settled is never
175
+ * sent, and the call fails with the cap's `LimitError`; one that fits but not beside the attempts in
176
+ * flight waits for them, holding its slot, and the review's abort or deadline ends that wait too. A call a limit
177
+ * stops before its first attempt sends no `call` event, since no provider was called; one it stops
178
+ * after an attempt sends its failed event first, as any failed call does.
179
+ *
180
+ * A call that fails once its request is built, for any reason, ends through one place: it sends its
181
+ * failed `call` event when an attempt was made, then hands that failure and the attempts made, 0 when
182
+ * none, to `onFailure`, once, inside its slot, and rethrows. The research stage aborts the review there
183
+ * for a gating profile, so the queued call the slot goes to next never starts; a failed vote has no
184
+ * `onFailure`: the panel records it. The duration is measured inside the slot, from the first attempt,
185
+ * once the request is built, to the call's end, the waits included: the provider's latency, not the time
186
+ * the call waited for a slot or for room, or took to build its request. A reply whose text is not a string
187
+ * is a `ProviderError`. Each call emits one `call` event, which names the provider by the ref captured at
188
+ * creation, never by the live object, and counts its attempts when it made more than one: failed, before
189
+ * the error is rethrown, with the usage summed over its attempts when their counts are valid, or ok, with
190
+ * that usage; both carry `spent`, the sum of what its attempts settled at. The ok event is emitted outside
191
+ * the `try` that catches the provider's failure, so a sink that throws on it is never reported as a failed
192
+ * call.
89
193
  */
90
- export declare function callProvider(ctx: RunContext, requester: Requester, target: Target, buildRequest: () => CompletionRequest): Promise<string>;
194
+ export declare function callProvider(ctx: RunContext, requester: Requester, target: Target, buildRequest: () => CompletionRequest, onFailure?: (error: unknown, attempts: number) => void): Promise<string>;
91
195
  /**
92
196
  * Run one stage of a profile, sync or async, between its `stage:start` and `stage:end` events. The
93
197
  * review's signal is checked before the stage starts, not after its step: an abort during the step
@@ -1,9 +1,13 @@
1
- import { ProviderError } from "../errors.js";
1
+ import { PROVIDER_ERROR_KINDS, ProviderError } from "../errors.js";
2
+ import { settlementOf } from "../limits.js";
2
3
  import { isRecord } from "../output.js";
4
+ import { DEFAULT_TIMEOUT_MS, MAX_STATED_TIMEOUT_MS, retriesProblem, timeoutProblem } from "../ports.js";
5
+ import { isTimeout, retryWait } from "../retry.js";
6
+ import { DEFAULT_REPLY_TOKENS, requestTokens, TEMPLATE_TOKENS } from "../tokens.js";
3
7
  /** What a refusal as too large says when it holds no message of its own. */
4
8
  const REFUSAL_FALLBACK = 'refused as too large';
5
9
  /** The value of an own data property of an object, else undefined: a getter is never run, an inherited property or a throwing proxy never matches. */
6
- function ownData(value, key) {
10
+ export function ownData(value, key) {
7
11
  if (typeof value !== 'object' || value === null)
8
12
  return undefined;
9
13
  try {
@@ -26,6 +30,21 @@ function ownData(value, key) {
26
30
  export function isTooLarge(error) {
27
31
  return ownData(error, 'kind') === 'too-large';
28
32
  }
33
+ /**
34
+ * What a failed call says of itself, as facts: the own data `kind` when it is one of `PROVIDER_ERROR_KINDS`,
35
+ * and for an `http` failure its own data `status` when that is an integer from 100 to 599. A message, which can
36
+ * quote a server, is never read. It is matched by those properties, not by class, as `isTooLarge` is, and it
37
+ * never runs a getter and never throws: any other value, a kind of no provider's included, is undefined.
38
+ */
39
+ export function providerFailure(error) {
40
+ const kind = PROVIDER_ERROR_KINDS.find(known => known === ownData(error, 'kind'));
41
+ if (kind === undefined)
42
+ return undefined;
43
+ const status = ownData(error, 'status');
44
+ return kind === 'http' && typeof status === 'number' && Number.isInteger(status) && status >= 100 && status <= 599
45
+ ? { kind, status }
46
+ : { kind };
47
+ }
29
48
  /** What a refusal as too large says: its own data string `message`, else a fixed text, never anything else from the object. */
30
49
  export function refusalMessage(refusal) {
31
50
  const message = ownData(refusal, 'message');
@@ -33,66 +52,325 @@ export function refusalMessage(refusal) {
33
52
  ? message
34
53
  : REFUSAL_FALLBACK;
35
54
  }
55
+ /**
56
+ * The usage of one more attempt added to a running total: both sums are of the attempts that
57
+ * reported valid counts, and `costUsd` is summed over those that reported one. With no more usage
58
+ * the total stays as it is.
59
+ */
60
+ export function addUsage(total, more) {
61
+ if (more === undefined)
62
+ return total;
63
+ const sum = { inputTokens: (total?.inputTokens ?? 0) + more.inputTokens, outputTokens: (total?.outputTokens ?? 0) + more.outputTokens };
64
+ const costUsd = more.costUsd === undefined
65
+ ? total?.costUsd
66
+ : (total?.costUsd ?? 0) + more.costUsd;
67
+ if (costUsd !== undefined)
68
+ sum.costUsd = costUsd;
69
+ return sum;
70
+ }
71
+ /**
72
+ * The usage a failed call's thrown value carries, when its counts can be trusted: its own data property
73
+ * `usage`, and that object's own data `inputTokens`, `outputTokens` and `costUsd`, checked as a reply's
74
+ * usage is. It is matched by those properties, not by class, as `isTooLarge` is, so a `ProviderError`
75
+ * from a second copy of quorum or a library provider's own error object counts too. It never runs a
76
+ * getter and never throws: an inherited property, a getter or a throwing proxy reads as no usage.
77
+ */
78
+ export function failureUsage(error) {
79
+ const usage = ownData(error, 'usage');
80
+ return checkedUsage({
81
+ inputTokens: ownData(usage, 'inputTokens'),
82
+ outputTokens: ownData(usage, 'outputTokens'),
83
+ costUsd: ownData(usage, 'costUsd'),
84
+ });
85
+ }
86
+ /**
87
+ * A provider as a run holds it: the ref captured now, which events and meta report, and its
88
+ * deadline and retries, each read as own data and taken only when it passes its check, else the
89
+ * default: 600000 ms and no retry. The quorum's checks ran first, so a plain object's values are the
90
+ * ones they passed; a proxy that answers the check and this read differently still gets an in-range
91
+ * deadline and no retry, never an unbounded one. The worst case it states is its own data value when
92
+ * that is a function, else none: a getter is never run, and an inherited method states nothing.
93
+ */
94
+ export function targetOf(provider) {
95
+ const timeoutMs = ownData(provider, 'timeoutMs');
96
+ const retries = ownData(provider, 'retries');
97
+ const worstCase = ownData(provider, 'worstCase');
98
+ return {
99
+ provider,
100
+ ref: Object.freeze({ id: provider.id, model: provider.model }),
101
+ timeoutMs: timeoutProblem(timeoutMs, MAX_STATED_TIMEOUT_MS) === undefined
102
+ ? timeoutMs
103
+ : DEFAULT_TIMEOUT_MS,
104
+ retries: retriesProblem(retries) === undefined
105
+ ? retries
106
+ : 0,
107
+ worstCase: typeof worstCase === 'function'
108
+ ? worstCase
109
+ : undefined,
110
+ };
111
+ }
112
+ /**
113
+ * The tokens one attempt of `request` is reserved: the worst case the provider states, called once with the
114
+ * request and the provider as `this`, when it returns a safe integer of at least 1. A provider that states
115
+ * none, a result that is anything else (a promise included, whose rejection is marked handled) and a throw
116
+ * all reserve the core's own count of the request plus
117
+ * `TEMPLATE_TOKENS` for what a server adds and `DEFAULT_REPLY_TOKENS` for the reply, so a worst case that
118
+ * misbehaves never reserves nothing. It never throws.
119
+ */
120
+ export function reservationOf(target, request) {
121
+ const { worstCase } = target;
122
+ if (worstCase !== undefined) {
123
+ try {
124
+ const stated = Reflect.apply(worstCase, target.provider, [request]);
125
+ if (typeof stated === 'number' && Number.isSafeInteger(stated) && stated >= 1)
126
+ return stated;
127
+ // It is called synchronously: a promise states nothing, and a rejected one must not crash the process.
128
+ if ((typeof stated === 'object' && stated !== null) || typeof stated === 'function') {
129
+ Promise.resolve(stated).then(undefined, () => { });
130
+ }
131
+ }
132
+ catch {
133
+ // A worst case that throws states nothing.
134
+ }
135
+ }
136
+ return requestTokens(request) + TEMPLATE_TOKENS + DEFAULT_REPLY_TOKENS;
137
+ }
36
138
  /** Whole milliseconds since `started`, a `performance.now()` reading. */
37
139
  export function elapsed(started) {
38
140
  return Math.round(performance.now() - started);
39
141
  }
40
142
  /**
41
143
  * Call a provider inside the review's concurrency limit and return its reply text. The review's
42
- * signal is checked inside the slot, before the request is built, so a call still queued when the
43
- * review is aborted never starts. `buildRequest` runs inside the slot once that check passes: a
44
- * request holds the diff or a section of it, and building it only there keeps the requests a
144
+ * stop is checked inside the slot, before the request is built, so a call still queued when the review
145
+ * is aborted or reaches its deadline never starts. `buildRequest` runs inside the slot once that check
146
+ * passes: a request holds the diff or a section of it, and building it only there keeps the requests a
45
147
  * review holds at once to its concurrency, whatever the number of findings and lenses (ADR-0009).
46
- * The request carries the signal, so the provider can stop a call in flight. A failed research
47
- * call aborts the review through `ctx.fail` before it leaves its slot, so the queued call the slot
48
- * goes to next never starts, unless the provider refused the request as too large: that refusal is
49
- * a gap, never a reason to stop the other profiles, so the research stage records it. A failed vote
50
- * does not abort the review, since the panel records it. The duration is measured inside the slot,
51
- * once the request is built: the provider's latency, not the time the call waited for a slot or
52
- * took to build its request. A reply whose text is not a string is a `ProviderError`. Each call
53
- * emits one `call` event, which names the provider by the ref captured at creation, never by the
54
- * live object: failed, before the error is rethrown, or ok, with the usage when its counts are
55
- * valid. The ok event is emitted outside the `try` that catches the provider's failure, so a sink
56
- * that throws on it is never reported as a failed call.
148
+ * The call runs its attempts inside its slot, retried as `retryWait` decides, the waits between
149
+ * them holding the slot: the review's abort ends a wait at once, and no failure is retried once the
150
+ * review is aborted. Each attempt has the provider's deadline, and the provider is handed the
151
+ * attempt's own signal, so it can stop a call in flight: the review's abort settles the attempt at
152
+ * once, without waiting for the provider.
153
+ *
154
+ * The review's deadline ends the call's attempt and its waits for room and between attempts at once, as an
155
+ * abort does, but with its own failure, a `LimitError` that is no attempt's `timeout`; a call still queued for
156
+ * its slot ends there when its turn comes, before it builds its request; and no failure is retried once the
157
+ * deadline has passed. A failure the retry policy would have retried, which the stop alone prevents, ends the
158
+ * call with the stop's reason, the deadline's `LimitError` or the interrupt's, never with the failure, so a
159
+ * deadline is not recorded as a failure it cut short; a failure the policy would not have retried keeps its own
160
+ * error. The retry policy is told the time the review has left, so a server's hint longer than it ends the
161
+ * call with its last failure.
162
+ *
163
+ * Each attempt, the first and every retry, reserves the call's worst case (`reservationOf`, taken once
164
+ * the request is built) in the review's ledger before it is sent, and settles it once it ends, inside the
165
+ * slot: at the usage its reply or its failure reports, at nothing when the server refused it before any
166
+ * work, else at the whole reservation. An attempt that does not fit beside what was settled is never
167
+ * sent, and the call fails with the cap's `LimitError`; one that fits but not beside the attempts in
168
+ * flight waits for them, holding its slot, and the review's abort or deadline ends that wait too. A call a limit
169
+ * stops before its first attempt sends no `call` event, since no provider was called; one it stops
170
+ * after an attempt sends its failed event first, as any failed call does.
171
+ *
172
+ * A call that fails once its request is built, for any reason, ends through one place: it sends its
173
+ * failed `call` event when an attempt was made, then hands that failure and the attempts made, 0 when
174
+ * none, to `onFailure`, once, inside its slot, and rethrows. The research stage aborts the review there
175
+ * for a gating profile, so the queued call the slot goes to next never starts; a failed vote has no
176
+ * `onFailure`: the panel records it. The duration is measured inside the slot, from the first attempt,
177
+ * once the request is built, to the call's end, the waits included: the provider's latency, not the time
178
+ * the call waited for a slot or for room, or took to build its request. A reply whose text is not a string
179
+ * is a `ProviderError`. Each call emits one `call` event, which names the provider by the ref captured at
180
+ * creation, never by the live object, and counts its attempts when it made more than one: failed, before
181
+ * the error is rethrown, with the usage summed over its attempts when their counts are valid, or ok, with
182
+ * that usage; both carry `spent`, the sum of what its attempts settled at. The ok event is emitted outside
183
+ * the `try` that catches the provider's failure, so a sink that throws on it is never reported as a failed
184
+ * call.
57
185
  */
58
- export async function callProvider(ctx, requester, target, buildRequest) {
186
+ export async function callProvider(ctx, requester, target, buildRequest, onFailure) {
59
187
  return ctx.limit(async () => {
60
- ctx.signal.throwIfAborted();
188
+ ctx.stop.throwIfAborted();
61
189
  const request = buildRequest();
62
- const started = performance.now();
190
+ const reservation = reservationOf(target, request);
191
+ const started = ctx.clock.now();
192
+ const took = () => Math.round(ctx.clock.now() - started);
193
+ let attempts = 0;
194
+ let usage;
195
+ let spent = 0;
196
+ let timedOut = false;
63
197
  let response;
64
- try {
65
- response = await target.provider.complete({ ...request, signal: ctx.signal });
66
- if (typeof response?.text !== 'string')
67
- throw new ProviderError({ providerId: target.ref.id, kind: 'invalid-response', message: 'returned no reply text' });
198
+ let replied;
199
+ /** Settles the attempt's reservation at `tokens`, which the call's `spent` adds. */
200
+ const settle = (tokens) => {
201
+ ctx.ledger.settle(reservation, tokens);
202
+ spent += tokens;
203
+ };
204
+ /** Ends the call with `error`: its failed event when an attempt was made, then `onFailure`; the error to throw. */
205
+ const end = (error) => {
206
+ if (attempts > 0) {
207
+ ctx.emit({
208
+ type: 'call',
209
+ ...requester,
210
+ provider: target.ref,
211
+ durationMs: took(),
212
+ ok: false,
213
+ ...counted(attempts, usage, spent),
214
+ });
215
+ }
216
+ // Still inside the slot: a review aborted here never starts the queued call the slot goes to next.
217
+ onFailure?.(error, attempts);
218
+ return error;
219
+ };
220
+ while (response === undefined) {
221
+ try {
222
+ await ctx.ledger.reserve(reservation, ctx.stop);
223
+ }
224
+ catch (error) {
225
+ throw end(error);
226
+ }
227
+ attempts += 1;
228
+ try {
229
+ const reply = await attempt(ctx, target, request);
230
+ if (typeof reply?.text !== 'string')
231
+ throw new ProviderError({ providerId: target.ref.id, kind: 'invalid-response', message: 'returned no reply text' });
232
+ // The reply's usage is read once, so its settlement and its event cannot disagree.
233
+ replied = checkedUsage(reply.usage);
234
+ settle(settlementOf(reservation, replied));
235
+ response = reply;
236
+ }
237
+ catch (error) {
238
+ const reported = failureUsage(error);
239
+ usage = addUsage(usage, reported);
240
+ const facts = failureFacts(error);
241
+ settle(settlementOf(reservation, reported, facts));
242
+ // A stopped review is asked as if it had time to spare, so a retry only the stop prevents is the stop's doing.
243
+ const wait = retryWait(facts, {
244
+ retried: attempts - 1,
245
+ retries: target.retries,
246
+ timedOut,
247
+ timeLeftMs: ctx.stop.aborted
248
+ ? Number.POSITIVE_INFINITY
249
+ : ctx.deadline.at - ctx.clock.now(),
250
+ }, ctx.clock.random);
251
+ if (wait === undefined)
252
+ throw end(error);
253
+ if (ctx.stop.aborted)
254
+ throw end(ctx.stop.reason);
255
+ timedOut ||= isTimeout(facts);
256
+ try {
257
+ await pause(ctx, wait);
258
+ }
259
+ catch (stopped) {
260
+ throw end(stopped);
261
+ }
262
+ }
68
263
  }
69
- catch (error) {
70
- ctx.emit({
71
- type: 'call',
72
- ...requester,
73
- provider: target.ref,
74
- durationMs: elapsed(started),
75
- ok: false,
76
- });
77
- // Still inside the slot: the queued call it is handed to next sees the abort and never starts.
78
- if (requester.role === 'research' && !isTooLarge(error))
79
- ctx.fail(error);
80
- throw error;
81
- }
82
- const usage = checkedUsage(response.usage);
83
264
  ctx.emit({
84
265
  type: 'call',
85
266
  ...requester,
86
267
  provider: target.ref,
87
- durationMs: elapsed(started),
268
+ durationMs: took(),
88
269
  ok: true,
89
- ...(usage === undefined
90
- ? {}
91
- : { usage }),
270
+ ...counted(attempts, addUsage(usage, replied), spent),
92
271
  });
93
272
  return response.text;
94
273
  });
95
274
  }
275
+ /** The fields of a `call` event that hold what the call's attempts add up to: their number when above one, their summed usage when valid, and the tokens the cap counted. */
276
+ function counted(attempts, usage, spent) {
277
+ return {
278
+ ...(attempts > 1
279
+ ? { attempts }
280
+ : {}),
281
+ ...(usage === undefined
282
+ ? {}
283
+ : { usage }),
284
+ spent,
285
+ };
286
+ }
287
+ /** What the retry policy reads of a thrown value: its `kind`, `status`, `retryable` and `retryAfterMs` as own data, so a getter decides nothing and never runs. */
288
+ function failureFacts(error) {
289
+ return {
290
+ kind: ownData(error, 'kind'),
291
+ status: ownData(error, 'status'),
292
+ retryable: ownData(error, 'retryable'),
293
+ retryAfterMs: ownData(error, 'retryAfterMs'),
294
+ };
295
+ }
296
+ /**
297
+ * Wait `ms` milliseconds between two attempts of a call, which holds its slot meanwhile. The review's
298
+ * abort or deadline ends the wait at once: it rejects with the stop's reason, and the timer is cancelled.
299
+ * Each of the timer and the stop listener removes the other when it goes first.
300
+ */
301
+ function pause(ctx, ms) {
302
+ if (ctx.stop.aborted)
303
+ return Promise.reject(ctx.stop.reason);
304
+ return new Promise((resolve, reject) => {
305
+ let cancel = () => { };
306
+ function onAbort() {
307
+ cancel();
308
+ reject(ctx.stop.reason);
309
+ }
310
+ cancel = ctx.clock.timer(ms, () => {
311
+ ctx.stop.removeEventListener('abort', onAbort);
312
+ resolve();
313
+ });
314
+ ctx.stop.addEventListener('abort', onAbort, { once: true });
315
+ });
316
+ }
317
+ /**
318
+ * One attempt of a call, settled once, by the first of: the provider's reply or failure; its own
319
+ * deadline, as a `timeout` error of the core's own with the attempt's signal aborted with it as the
320
+ * reason; the review's stop, its abort or its deadline, with the stop's reason. The attempt's timer
321
+ * and the join to the review's stop go when it settles, on every path. A provider that throws before
322
+ * it returns a promise, or returns a plain object, is handled; whatever it does once the attempt
323
+ * settled is caught and ignored, and it is never an unhandled rejection. The join is one listener,
324
+ * not `AbortSignal.any`, so nothing builds up on a signal that outlives thousands of attempts.
325
+ */
326
+ function attempt(ctx, target, request) {
327
+ if (ctx.stop.aborted)
328
+ return Promise.reject(ctx.stop.reason);
329
+ const controller = new AbortController();
330
+ return new Promise((resolve, reject) => {
331
+ let settled = false;
332
+ let cancel = () => { };
333
+ /** True the first time, when it clears the timer and the listener; false once the attempt has settled. */
334
+ function done() {
335
+ if (settled)
336
+ return false;
337
+ settled = true;
338
+ cancel();
339
+ ctx.stop.removeEventListener('abort', onAbort);
340
+ return true;
341
+ }
342
+ function onAbort() {
343
+ if (!done())
344
+ return;
345
+ controller.abort(ctx.stop.reason);
346
+ reject(ctx.stop.reason);
347
+ }
348
+ cancel = ctx.clock.timer(target.timeoutMs, () => {
349
+ if (!done())
350
+ return;
351
+ const error = new ProviderError({ providerId: target.ref.id, kind: 'timeout', message: `no reply within ${target.timeoutMs} ms` });
352
+ controller.abort(error);
353
+ reject(error);
354
+ });
355
+ ctx.stop.addEventListener('abort', onAbort, { once: true });
356
+ let call;
357
+ try {
358
+ call = Promise.resolve(target.provider.complete({ ...request, signal: controller.signal }));
359
+ }
360
+ catch (error) {
361
+ if (done())
362
+ reject(error);
363
+ return;
364
+ }
365
+ call.then((response) => {
366
+ if (done())
367
+ resolve(response);
368
+ }, (error) => {
369
+ if (done())
370
+ reject(error);
371
+ });
372
+ });
373
+ }
96
374
  /**
97
375
  * Run one stage of a profile, sync or async, between its `stage:start` and `stage:end` events. The
98
376
  * review's signal is checked before the stage starts, not after its step: an abort during the step
@@ -21,10 +21,11 @@ export interface PanelOutcome {
21
21
  * self-reported confidence gets to override: with the security profile, unanimous 3/3 is a ceiling
22
22
  * of 'high', a 2/3 split is capped at 'medium' no matter how certain any one voter sounded.
23
23
  *
24
- * A failed vote counts as a reject and is recorded with its error, never as a crash, but a vote cut
25
- * by the review's abort is not a failed vote: it rejects the panel with the abort's reason at once,
26
- * without waiting for the votes still in flight. A vote request its provider refused as too large
27
- * is a failed vote that also records a `request-too-large` gap. Each vote emits a `vote` event.
24
+ * A failed vote counts as a reject and is recorded with its error and its failure, never as a crash,
25
+ * but a vote cut by the review's abort is not a failed vote: it rejects the panel with the abort's
26
+ * reason at once, without waiting for the votes still in flight. A vote request its provider refused
27
+ * as too large is a failed vote that also records a `request-too-large` gap. Each vote emits a `vote`
28
+ * event.
28
29
  * Once a finding's votes are in, its verdict emits a `verdict` event, then its gaps are reported by
29
30
  * `incomplete` events: one `request-too-large` gap when any vote request was refused, and a
30
31
  * `vote-failed` gap when a kept verdict's failed votes leave a higher level undecided (ADR-0012).