@aragon-agent/core 0.2.8 → 0.2.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/dist/engine/agent.d.ts +17 -0
- package/dist/engine/agent.js +30 -0
- package/dist/index.d.ts +4 -1
- package/dist/index.js +11 -0
- package/dist/llm/provider.d.ts +25 -2
- package/dist/llm/provider.js +40 -3
- package/dist/llm/providers/anthropic.js +42 -8
- package/dist/llm/providers/google.js +29 -1
- package/dist/llm/providers/index.d.ts +61 -2
- package/dist/llm/providers/index.js +70 -9
- package/dist/llm/providers/openai.js +40 -2
- package/dist/llm/retry.d.ts +211 -0
- package/dist/llm/retry.js +458 -0
- package/dist/llm/types.d.ts +40 -1
- package/package.json +1 -1
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Retry & exponential backoff — THE authority for "a provider call failed, now
|
|
3
|
+
* what" (llm-api-retry-backoff §4).
|
|
4
|
+
*
|
|
5
|
+
* One module owns the policy type, the classification predicate, the backoff
|
|
6
|
+
* arithmetic and a single generator `withRetry()` that wraps ANY
|
|
7
|
+
* `AsyncIterableIterator<StreamEvent>`. `ProviderRegistry.stream()` applies it,
|
|
8
|
+
* so all three adapters, the lead agent and every subagent inherit retry with
|
|
9
|
+
* zero call-site changes.
|
|
10
|
+
*
|
|
11
|
+
* THE WRAPPER IS THE ONLY LAYER THAT CAN DO THIS JOB, and that is why it lives
|
|
12
|
+
* here rather than in the adapters or in `runAgentLoop` (§3.1 / §3.2):
|
|
13
|
+
*
|
|
14
|
+
* - it knows whether any CONTENT has already been forwarded downstream (the
|
|
15
|
+
* commit point), which decides whether a replay would duplicate text on the
|
|
16
|
+
* user's screen;
|
|
17
|
+
* - it holds the `AbortSignal`, which is what keeps a user pressing Esc from
|
|
18
|
+
* being mistaken for a retryable timeout;
|
|
19
|
+
* - it can emit its own events, which is what turns a silent 40-second stall
|
|
20
|
+
* into a countdown a user can read.
|
|
21
|
+
*
|
|
22
|
+
* NO `node:*` IMPORTS. `no-host-coupling.test.ts` rule 2 bans every one of them
|
|
23
|
+
* under core `src/` with a single grandfathered exception, so the abortable
|
|
24
|
+
* `sleep` below uses the GLOBAL `setTimeout` plus one `abort` listener rather
|
|
25
|
+
* than `node:timers/promises` (whose `setTimeout` takes `{ signal }` natively
|
|
26
|
+
* and would be the idiomatic choice anywhere else).
|
|
27
|
+
*/
|
|
28
|
+
import { LLMError } from './provider.js';
|
|
29
|
+
// ---------------------------------------------------------------------------
|
|
30
|
+
// Structural bounds
|
|
31
|
+
// ---------------------------------------------------------------------------
|
|
32
|
+
/**
|
|
33
|
+
* Structural bounds. NOT user policy: these describe what the mechanism can
|
|
34
|
+
* physically do, and a hand-edited config must not be able to exceed them.
|
|
35
|
+
* (The same split `TEAM_LIMITS` states for team mode, one package over.)
|
|
36
|
+
*
|
|
37
|
+
* ENFORCED AT THIS BOUNDARY by `normalizePolicy`, not only by the CLI's
|
|
38
|
+
* `clampRetryConfig` (§4.5 G4). Hardening one path and leaving the other open
|
|
39
|
+
* would let an embedder passing `{ maxRetries: 5000 }` get 5000.
|
|
40
|
+
*/
|
|
41
|
+
export const RETRY_LIMITS = {
|
|
42
|
+
/** Ceiling on `maxRetries` from any layer. */
|
|
43
|
+
hardMaxRetries: 20,
|
|
44
|
+
/** Ceiling on one wait, whatever the config or a `Retry-After` header says. */
|
|
45
|
+
absoluteMaxDelayMs: 120_000,
|
|
46
|
+
/**
|
|
47
|
+
* A `Retry-After` LARGER than this is not honoured and not retried — it is
|
|
48
|
+
* surfaced immediately with the provider's own number in the message. A
|
|
49
|
+
* provider asking us to come back in an hour is telling the user something,
|
|
50
|
+
* and silently wedging the terminal for an hour is not relaying it.
|
|
51
|
+
*/
|
|
52
|
+
retryAfterCeilingMs: 60_000,
|
|
53
|
+
/** Floor under any computed wait, so jitter can never produce a 0 ms "retry". */
|
|
54
|
+
minDelayMs: 100,
|
|
55
|
+
};
|
|
56
|
+
export const DEFAULT_RETRY_POLICY = {
|
|
57
|
+
maxRetries: 10,
|
|
58
|
+
initialDelayMs: 1_000,
|
|
59
|
+
/**
|
|
60
|
+
* Core's default `idleTimeout` is 60 000 ms. Even with the watchdog pause of
|
|
61
|
+
* §4.6 in place, a single wait longer than half the idle window is a design
|
|
62
|
+
* smell: it means the mechanism depends on the pause being correct rather
|
|
63
|
+
* than merely benefiting from it. Two independent guards, and the numeric one
|
|
64
|
+
* costs nothing.
|
|
65
|
+
*/
|
|
66
|
+
maxDelayMs: 30_000,
|
|
67
|
+
multiplier: 2,
|
|
68
|
+
jitter: true,
|
|
69
|
+
respectRetryAfter: true,
|
|
70
|
+
/**
|
|
71
|
+
* 240 s, NOT 300 s. The CLI's `DEFAULT_TEAM_CONFIG.subagentTimeoutMs` is
|
|
72
|
+
* 300 000, and a retry budget EQUAL to the per-child timeout means the child
|
|
73
|
+
* is killed at the same instant its ladder ends — the user is then told
|
|
74
|
+
* "subagent timed out" for what was a provider outage. 240 s fits the full
|
|
75
|
+
* ten-step ladder (181 s worst case) with 59 s of headroom and still leaves
|
|
76
|
+
* the child 60 s of its own window to actually produce an answer.
|
|
77
|
+
*
|
|
78
|
+
* THE INVARIANT IS `maxElapsedMs + one model round-trip <
|
|
79
|
+
* team.subagentTimeoutMs` (§5.6). It is invisible at both edit sites, so the
|
|
80
|
+
* CLI's `retry-config.test.ts` pins the inequality statically.
|
|
81
|
+
*/
|
|
82
|
+
maxElapsedMs: 240_000,
|
|
83
|
+
onPartialStream: true,
|
|
84
|
+
};
|
|
85
|
+
/**
|
|
86
|
+
* The events that mean "the consumer has already rendered answer content", i.e.
|
|
87
|
+
* the commit point (§4.5a).
|
|
88
|
+
*
|
|
89
|
+
* `thinking_start` is in the set even though it carries no text: it flips
|
|
90
|
+
* `thinkingOpen` in the CLI reducer, which is visible state a restart must
|
|
91
|
+
* rewind. The tool events are in the set because `tool_call_start` appends a
|
|
92
|
+
* transcript card — that is why `stream_restart` carries `discardedToolCallIds`
|
|
93
|
+
* at all. `done` and `error` are terminal, not content; the three `retry_*`
|
|
94
|
+
* variants are ours and never round-trip through `makeStream()`.
|
|
95
|
+
*/
|
|
96
|
+
export const CONTENT_EVENTS = new Set([
|
|
97
|
+
'text_delta',
|
|
98
|
+
'thinking_start',
|
|
99
|
+
'thinking_delta',
|
|
100
|
+
'tool_call_start',
|
|
101
|
+
'tool_call_delta',
|
|
102
|
+
'tool_call_end',
|
|
103
|
+
]);
|
|
104
|
+
// ---------------------------------------------------------------------------
|
|
105
|
+
// normalizePolicy
|
|
106
|
+
// ---------------------------------------------------------------------------
|
|
107
|
+
function finiteOr(value, fallback) {
|
|
108
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : fallback;
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* Clamp a policy to `RETRY_LIMITS`. Pure, total, and it NEVER THROWS — a bad
|
|
112
|
+
* policy is corrected rather than rejected, because the alternative is a
|
|
113
|
+
* library that crashes on a config typo.
|
|
114
|
+
*
|
|
115
|
+
* Module-internal by intent: exported so `llm-retry.test.ts` may import it
|
|
116
|
+
* directly, but deliberately NOT re-exported from the barrel. The public
|
|
117
|
+
* assertion is made THROUGH `withRetry`, which is also the only way to catch
|
|
118
|
+
* this being present but unwired.
|
|
119
|
+
*/
|
|
120
|
+
export function normalizePolicy(policy) {
|
|
121
|
+
const maxRetries = Math.min(RETRY_LIMITS.hardMaxRetries, Math.max(0, Math.floor(finiteOr(policy.maxRetries, DEFAULT_RETRY_POLICY.maxRetries))));
|
|
122
|
+
const maxDelayMs = Math.min(RETRY_LIMITS.absoluteMaxDelayMs, Math.max(RETRY_LIMITS.minDelayMs, Math.floor(finiteOr(policy.maxDelayMs, DEFAULT_RETRY_POLICY.maxDelayMs))));
|
|
123
|
+
const initialDelayMs = Math.min(maxDelayMs, Math.max(RETRY_LIMITS.minDelayMs, Math.floor(finiteOr(policy.initialDelayMs, DEFAULT_RETRY_POLICY.initialDelayMs))));
|
|
124
|
+
const multiplier = Math.max(1, finiteOr(policy.multiplier, DEFAULT_RETRY_POLICY.multiplier));
|
|
125
|
+
/**
|
|
126
|
+
* FLOORED AT `minDelayMs`, NOT AT `initialDelayMs`.
|
|
127
|
+
*
|
|
128
|
+
* `>= initialDelayMs` reads like the tighter bound and is the wrong one: it
|
|
129
|
+
* would make "give up after 500 ms in total" unexpressible by silently raising
|
|
130
|
+
* the budget to the first delay, at which point the first retry always fires —
|
|
131
|
+
* so a budget SMALLER than one wait would stop being a budget at all, silently.
|
|
132
|
+
* The CLI's own range for this key already starts at 10 000, so nothing a user
|
|
133
|
+
* can type reaches the difference; what does reach it is a unit test asserting
|
|
134
|
+
* that the budget is evaluated on the FIRST failure, which is the one property
|
|
135
|
+
* a mis-ordered implementation gets wrong. See the spec's implementation-findings
|
|
136
|
+
* section (IF-2).
|
|
137
|
+
*/
|
|
138
|
+
const maxElapsedMs = Math.max(RETRY_LIMITS.minDelayMs, Math.floor(finiteOr(policy.maxElapsedMs, DEFAULT_RETRY_POLICY.maxElapsedMs)));
|
|
139
|
+
return {
|
|
140
|
+
maxRetries,
|
|
141
|
+
initialDelayMs,
|
|
142
|
+
maxDelayMs,
|
|
143
|
+
multiplier,
|
|
144
|
+
jitter: policy.jitter !== false,
|
|
145
|
+
respectRetryAfter: policy.respectRetryAfter !== false,
|
|
146
|
+
maxElapsedMs,
|
|
147
|
+
onPartialStream: policy.onPartialStream !== false,
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Whether this failure may be retried at all, ignoring budgets and counters.
|
|
152
|
+
*
|
|
153
|
+
* THE ORDER OF THE TWO CHECKS IS LOAD-BEARING.
|
|
154
|
+
*
|
|
155
|
+
* 1. `signal?.aborted === true` COMES FIRST. `wrapFetchError` classifies an
|
|
156
|
+
* `AbortError` as `errorType: 'timeout', retryable: true`, so every user Esc,
|
|
157
|
+
* every `AgentController.abort()`, every dispatch timeout and every
|
|
158
|
+
* tool-timeout cancellation arrives here wearing a retryable badge. The CLI's
|
|
159
|
+
* `team/retry.ts` records this trap in thirty lines of comment because it
|
|
160
|
+
* cost someone an afternoon; reading the signal BEFORE `err.retryable` is
|
|
161
|
+
* what makes Esc mean Esc.
|
|
162
|
+
* 2. `retryable` is read STRUCTURALLY, never with `instanceof LLMError`. The
|
|
163
|
+
* error crosses a package boundary, and a duplicated core instance (a linked
|
|
164
|
+
* checkout, a hoisting accident) would make `instanceof` false while the
|
|
165
|
+
* field is plainly there — and the failure mode would be "retries silently
|
|
166
|
+
* stopped happening". This is the rule `isRetryableStreamError` and
|
|
167
|
+
* `formatStreamError` already follow.
|
|
168
|
+
*/
|
|
169
|
+
export function isRetryableError(err, signal) {
|
|
170
|
+
if (signal?.aborted === true)
|
|
171
|
+
return { retry: false, reason: 'aborted' };
|
|
172
|
+
if (err?.retryable !== true) {
|
|
173
|
+
return { retry: false, reason: 'not_retryable' };
|
|
174
|
+
}
|
|
175
|
+
return { retry: true, reason: 'retryable' };
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* The retry decision, taking every input EXPLICITLY — no closure reads — and
|
|
179
|
+
* returning the first matching reason (§4.5).
|
|
180
|
+
*
|
|
181
|
+
* Module-internal by intent, like `normalizePolicy` above.
|
|
182
|
+
*/
|
|
183
|
+
export function decide(input) {
|
|
184
|
+
const base = isRetryableError(input.error, input.signal);
|
|
185
|
+
if (!base.retry)
|
|
186
|
+
return base;
|
|
187
|
+
if (input.committed && !input.policy.onPartialStream) {
|
|
188
|
+
return { retry: false, reason: 'committed' };
|
|
189
|
+
}
|
|
190
|
+
// PRE-INCREMENT, so `maxRetries: 10` permits retries at `retryIndex` 0...9 and
|
|
191
|
+
// stops the 11th failure.
|
|
192
|
+
if (input.retryIndex >= input.policy.maxRetries) {
|
|
193
|
+
return { retry: false, reason: 'attempts_exhausted' };
|
|
194
|
+
}
|
|
195
|
+
// `firstFailureAt` is a number by construction — the caller anchors it BEFORE
|
|
196
|
+
// deriving `candidateDelay` and before calling here. An ordering that read it
|
|
197
|
+
// first would compute `NaN`, and `NaN > maxElapsedMs` is `false`: a budget
|
|
198
|
+
// that never triggers, silently.
|
|
199
|
+
if (input.now() - input.firstFailureAt + input.candidateDelay > input.policy.maxElapsedMs) {
|
|
200
|
+
return { retry: false, reason: 'budget_exhausted' };
|
|
201
|
+
}
|
|
202
|
+
if (input.retryAfterMs !== undefined &&
|
|
203
|
+
input.retryAfterMs > RETRY_LIMITS.retryAfterCeilingMs) {
|
|
204
|
+
return { retry: false, reason: 'retry_after_too_long' };
|
|
205
|
+
}
|
|
206
|
+
return { retry: true, reason: 'retryable' };
|
|
207
|
+
}
|
|
208
|
+
// ---------------------------------------------------------------------------
|
|
209
|
+
// Backoff arithmetic
|
|
210
|
+
// ---------------------------------------------------------------------------
|
|
211
|
+
/**
|
|
212
|
+
* The wait before retry number `retryIndex` (1-BASED: the first retry is 1).
|
|
213
|
+
*
|
|
214
|
+
* EQUAL JITTER, NOT FULL JITTER. Full jitter (`rand() * base`) is the textbook
|
|
215
|
+
* contention minimiser, but it can return 40 ms on the sixth retry, and a UI
|
|
216
|
+
* that just announced "retrying in 30s" and then fires in 40 ms reads as a bug.
|
|
217
|
+
* Equal jitter keeps the announced number within a factor of two of the truth
|
|
218
|
+
* while still de-correlating N concurrent subagents hitting the same 429. The
|
|
219
|
+
* user-visible countdown is computed from the ACTUAL chosen delay, never from
|
|
220
|
+
* `base`, so what the card says is always what happens.
|
|
221
|
+
*/
|
|
222
|
+
export function computeBackoffDelay(retryIndex, policy, opts) {
|
|
223
|
+
const rand = opts?.random ?? Math.random;
|
|
224
|
+
const raw = policy.initialDelayMs * Math.pow(policy.multiplier, Math.max(0, retryIndex - 1));
|
|
225
|
+
const base = Math.min(raw, policy.maxDelayMs, RETRY_LIMITS.absoluteMaxDelayMs);
|
|
226
|
+
const jittered = policy.jitter ? base / 2 + rand() * (base / 2) : base;
|
|
227
|
+
// MAX, not replace. A `Retry-After: 1` arriving on the ninth retry must not
|
|
228
|
+
// undo eight retries' worth of backoff — the server is stating a FLOOR ("not
|
|
229
|
+
// before this"), not a ceiling.
|
|
230
|
+
const withHeader = policy.respectRetryAfter && opts?.retryAfterMs !== undefined
|
|
231
|
+
? Math.max(jittered, opts.retryAfterMs)
|
|
232
|
+
: jittered;
|
|
233
|
+
return Math.round(Math.min(Math.max(withHeader, RETRY_LIMITS.minDelayMs), RETRY_LIMITS.absoluteMaxDelayMs));
|
|
234
|
+
}
|
|
235
|
+
// ---------------------------------------------------------------------------
|
|
236
|
+
// Retry-After parsing
|
|
237
|
+
// ---------------------------------------------------------------------------
|
|
238
|
+
/** Seconds-or-HTTP-date -> ms, or `undefined` when unusable. Never negative. */
|
|
239
|
+
function parseDelaySeconds(raw, now) {
|
|
240
|
+
if (raw === null)
|
|
241
|
+
return undefined;
|
|
242
|
+
const trimmed = raw.trim();
|
|
243
|
+
if (trimmed.length === 0)
|
|
244
|
+
return undefined;
|
|
245
|
+
if (/^\d+(?:\.\d+)?$/.test(trimmed)) {
|
|
246
|
+
const seconds = Number.parseFloat(trimmed);
|
|
247
|
+
if (!Number.isFinite(seconds))
|
|
248
|
+
return undefined;
|
|
249
|
+
return Math.max(0, Math.round(seconds * 1000));
|
|
250
|
+
}
|
|
251
|
+
return parseInstant(trimmed, now);
|
|
252
|
+
}
|
|
253
|
+
/** An RFC-3339 / HTTP-date instant -> ms from now, or `undefined`. Never negative. */
|
|
254
|
+
function parseInstant(raw, now) {
|
|
255
|
+
const at = Date.parse(raw);
|
|
256
|
+
if (!Number.isFinite(at))
|
|
257
|
+
return undefined;
|
|
258
|
+
return Math.max(0, Math.round(at - now()));
|
|
259
|
+
}
|
|
260
|
+
/**
|
|
261
|
+
* The server-stated wait, in ms, or `undefined` when absent or unparseable.
|
|
262
|
+
*
|
|
263
|
+
* Read order: `retry-after` (integer seconds, or an HTTP-date), then
|
|
264
|
+
* `x-ratelimit-reset-after` (seconds, OpenAI-flavoured), then
|
|
265
|
+
* `anthropic-ratelimit-requests-reset`, then
|
|
266
|
+
* `anthropic-ratelimit-input-tokens-reset` (both RFC-3339 instants).
|
|
267
|
+
*
|
|
268
|
+
* THE TOKEN HEADER IS READ, not skipped: on a long context it is the limit that
|
|
269
|
+
* actually bites, and a request throttled on tokens that waits out only the
|
|
270
|
+
* REQUEST window comes back to the same 429.
|
|
271
|
+
*
|
|
272
|
+
* A value above `RETRY_LIMITS.retryAfterCeilingMs` is returned AS-IS — the
|
|
273
|
+
* CALLER (`withRetry`) decides to stop, so the honest number reaches the user's
|
|
274
|
+
* error message rather than being silently rounded down to a wait we would then
|
|
275
|
+
* actually perform.
|
|
276
|
+
*/
|
|
277
|
+
export function parseRetryAfterMs(headers, now = Date.now) {
|
|
278
|
+
if (!headers || typeof headers.get !== 'function')
|
|
279
|
+
return undefined;
|
|
280
|
+
const direct = parseDelaySeconds(headers.get('retry-after'), now);
|
|
281
|
+
if (direct !== undefined)
|
|
282
|
+
return direct;
|
|
283
|
+
const openai = parseDelaySeconds(headers.get('x-ratelimit-reset-after'), now);
|
|
284
|
+
if (openai !== undefined)
|
|
285
|
+
return openai;
|
|
286
|
+
const requests = headers.get('anthropic-ratelimit-requests-reset');
|
|
287
|
+
if (requests !== null && requests.trim().length > 0) {
|
|
288
|
+
const parsed = parseInstant(requests.trim(), now);
|
|
289
|
+
if (parsed !== undefined)
|
|
290
|
+
return parsed;
|
|
291
|
+
}
|
|
292
|
+
const inputTokens = headers.get('anthropic-ratelimit-input-tokens-reset');
|
|
293
|
+
if (inputTokens !== null && inputTokens.trim().length > 0) {
|
|
294
|
+
const parsed = parseInstant(inputTokens.trim(), now);
|
|
295
|
+
if (parsed !== undefined)
|
|
296
|
+
return parsed;
|
|
297
|
+
}
|
|
298
|
+
return undefined;
|
|
299
|
+
}
|
|
300
|
+
/**
|
|
301
|
+
* Sleep, resolving on EITHER the timer or the signal's `abort` event.
|
|
302
|
+
*
|
|
303
|
+
* A bare `setTimeout` would make Esc take up to 30 s to be felt, which is the
|
|
304
|
+
* single most visible way this feature can be made to feel cheap. The listener
|
|
305
|
+
* is removed on both paths so a long-lived signal does not accumulate them.
|
|
306
|
+
*/
|
|
307
|
+
function defaultSleep(ms, signal) {
|
|
308
|
+
if (signal?.aborted === true)
|
|
309
|
+
return Promise.resolve();
|
|
310
|
+
return new Promise((resolve) => {
|
|
311
|
+
const done = () => {
|
|
312
|
+
clearTimeout(timer);
|
|
313
|
+
signal?.removeEventListener('abort', done);
|
|
314
|
+
resolve();
|
|
315
|
+
};
|
|
316
|
+
const timer = setTimeout(done, ms);
|
|
317
|
+
signal?.addEventListener('abort', done, { once: true });
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
function errorTypeOf(err) {
|
|
321
|
+
const t = err?.errorType;
|
|
322
|
+
return typeof t === 'string' ? t : 'unknown';
|
|
323
|
+
}
|
|
324
|
+
function messageOf(err) {
|
|
325
|
+
const m = err?.message;
|
|
326
|
+
return typeof m === 'string' && m.length > 0 ? m : 'Unknown provider error';
|
|
327
|
+
}
|
|
328
|
+
function retryAfterOf(err) {
|
|
329
|
+
const v = err?.retryAfterMs;
|
|
330
|
+
return typeof v === 'number' && Number.isFinite(v) && v >= 0 ? v : undefined;
|
|
331
|
+
}
|
|
332
|
+
/**
|
|
333
|
+
* Wrap a stream factory with retry.
|
|
334
|
+
*
|
|
335
|
+
* Six properties are load-bearing and easy to break:
|
|
336
|
+
*
|
|
337
|
+
* - G1 — THE ABORT GUARD IS CHECKED ON EVERY ITERATION, matching
|
|
338
|
+
* `agent-loop.ts`'s own guard. Combined with G3 below, an aborted request
|
|
339
|
+
* never forwards a trailing error to a subscriber.
|
|
340
|
+
* - G2 — `sleep` MUST OBSERVE `signal`. See `defaultSleep`.
|
|
341
|
+
* - G3 — `aborted` RETURNS SILENTLY. It is the one non-retry reason that must
|
|
342
|
+
* not `yield` an `error`. `consumeStream` (the body of `complete()`) does not
|
|
343
|
+
* consult the signal, so a yielded error would become a thrown, FABRICATED
|
|
344
|
+
* `network_error` on user Esc — blaming the network for something the user
|
|
345
|
+
* just did is worse than saying nothing. Three silent return sites: the
|
|
346
|
+
* in-loop guard, the `decide` short-circuit, and the post-sleep check.
|
|
347
|
+
* - G4 — `normalizePolicy` RUNS ONCE, at generator entry, so `RETRY_LIMITS`
|
|
348
|
+
* bounds an embedder's policy and not merely a CLI-resolved one.
|
|
349
|
+
* - THE SUCCESS PATH IS UNTOUCHED. A request that works yields exactly the
|
|
350
|
+
* events the adapter produced, in the same order, with no extra allocation
|
|
351
|
+
* beyond one boolean and one empty array. The wrapper is not installed at all
|
|
352
|
+
* when `maxRetries <= 0`, so an opted-out embedder is byte-identical to a
|
|
353
|
+
* pre-feature build.
|
|
354
|
+
* - THE TERMINAL ERROR IS FORWARDED, NOT REPLACED. After exhaustion the
|
|
355
|
+
* consumer sees the PROVIDER's own `LLMError`, so `formatStreamError`, the
|
|
356
|
+
* `retryable` field and any downstream `instanceof` continue to behave. Only
|
|
357
|
+
* the retry COUNT is new information, and it travels on the events.
|
|
358
|
+
*/
|
|
359
|
+
export async function* withRetry(makeStream, opts) {
|
|
360
|
+
const policy = normalizePolicy(opts.policy);
|
|
361
|
+
const now = opts.now ?? Date.now;
|
|
362
|
+
const sleep = opts.sleep ?? defaultSleep;
|
|
363
|
+
const random = opts.random ?? Math.random;
|
|
364
|
+
const signal = opts.signal;
|
|
365
|
+
let firstFailureAt;
|
|
366
|
+
/** Retries COMPLETED so far. */
|
|
367
|
+
let retryIndex = 0;
|
|
368
|
+
for (;;) {
|
|
369
|
+
let committed = false;
|
|
370
|
+
const toolCallIds = [];
|
|
371
|
+
let sawDone = false;
|
|
372
|
+
let terminalError;
|
|
373
|
+
for await (const event of makeStream()) {
|
|
374
|
+
if (signal?.aborted === true)
|
|
375
|
+
return; // G1
|
|
376
|
+
if (CONTENT_EVENTS.has(event.type))
|
|
377
|
+
committed = true;
|
|
378
|
+
if (event.type === 'tool_call_start')
|
|
379
|
+
toolCallIds.push(event.toolCallId);
|
|
380
|
+
if (event.type === 'error') {
|
|
381
|
+
// NOT FORWARDED YET: whether the consumer ever sees it depends on the
|
|
382
|
+
// decision below. Breaking here also calls the source iterator's
|
|
383
|
+
// `return()`, which runs its `finally` and releases the SSE reader.
|
|
384
|
+
terminalError = event.error;
|
|
385
|
+
break;
|
|
386
|
+
}
|
|
387
|
+
if (event.type === 'done')
|
|
388
|
+
sawDone = true;
|
|
389
|
+
yield event;
|
|
390
|
+
}
|
|
391
|
+
if (sawDone)
|
|
392
|
+
return; // success; the hot path ends here
|
|
393
|
+
if (terminalError === undefined) {
|
|
394
|
+
// Defensive only: with the adapters' terminal-sentinel contract in place
|
|
395
|
+
// (§4.5a) every adapter terminates in `done` or `error`. Reaching here
|
|
396
|
+
// means an adapter broke its own contract, and a silent success would be
|
|
397
|
+
// the worst possible reading of that.
|
|
398
|
+
// NAMES THE PROVIDER *AND* THE MODEL. `agent-loop.ts` throws an almost
|
|
399
|
+
// identical sentence one layer up ("LLM stream ended without producing a
|
|
400
|
+
// done event"), and both can reach a user; without the ids the two are
|
|
401
|
+
// indistinguishable in a bug report, and the id is the first thing worth
|
|
402
|
+
// knowing when one adapter of three is misbehaving. This is also the only
|
|
403
|
+
// read of `opts.modelId` — a required option nothing consumes is one the
|
|
404
|
+
// next caller will start passing wrong.
|
|
405
|
+
terminalError = new LLMError(`${opts.providerId}/${opts.modelId} stream ended without producing a done or error event`, opts.providerId, 'network_error', true);
|
|
406
|
+
}
|
|
407
|
+
// ---- ORDER IS NORMATIVE FROM HERE -----------------------------------
|
|
408
|
+
// 1. Anchor the budget clock on the FIRST failure, before anything reads it.
|
|
409
|
+
firstFailureAt ??= now();
|
|
410
|
+
// 2. Derive the candidate wait for the NEXT attempt. `retryIndex + 1`
|
|
411
|
+
// because `computeBackoffDelay` is 1-based and `retryIndex` counts
|
|
412
|
+
// COMPLETED retries.
|
|
413
|
+
const retryAfterMs = policy.respectRetryAfter ? retryAfterOf(terminalError) : undefined;
|
|
414
|
+
const candidateDelay = computeBackoffDelay(retryIndex + 1, policy, {
|
|
415
|
+
...(retryAfterMs !== undefined ? { retryAfterMs } : {}),
|
|
416
|
+
random,
|
|
417
|
+
});
|
|
418
|
+
// 3. Only now can `decide` answer the budget question.
|
|
419
|
+
const decision = decide({
|
|
420
|
+
error: terminalError,
|
|
421
|
+
...(signal ? { signal } : {}),
|
|
422
|
+
committed,
|
|
423
|
+
retryIndex,
|
|
424
|
+
firstFailureAt,
|
|
425
|
+
candidateDelay,
|
|
426
|
+
...(retryAfterMs !== undefined ? { retryAfterMs } : {}),
|
|
427
|
+
policy,
|
|
428
|
+
now,
|
|
429
|
+
});
|
|
430
|
+
if (decision.reason === 'aborted')
|
|
431
|
+
return; // G3 — no trailing error event
|
|
432
|
+
if (!decision.retry) {
|
|
433
|
+
yield { type: 'error', error: terminalError };
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
retryIndex += 1;
|
|
437
|
+
// What we ANNOUNCE is what we WAIT. Recomputing here would let the card and
|
|
438
|
+
// the timer disagree by a jitter draw.
|
|
439
|
+
const delay = candidateDelay;
|
|
440
|
+
if (committed) {
|
|
441
|
+
yield { type: 'stream_restart', attempt: retryIndex, discardedToolCallIds: toolCallIds };
|
|
442
|
+
}
|
|
443
|
+
yield {
|
|
444
|
+
type: 'retry_scheduled',
|
|
445
|
+
attempt: retryIndex,
|
|
446
|
+
maxRetries: policy.maxRetries,
|
|
447
|
+
delayMs: delay,
|
|
448
|
+
resumeAt: now() + delay,
|
|
449
|
+
errorType: errorTypeOf(terminalError),
|
|
450
|
+
message: messageOf(terminalError),
|
|
451
|
+
...(retryAfterMs !== undefined ? { retryAfterMs } : {}),
|
|
452
|
+
};
|
|
453
|
+
await sleep(delay, signal); // G2
|
|
454
|
+
if (signal?.aborted === true)
|
|
455
|
+
return; // G3 again: silent
|
|
456
|
+
yield { type: 'retry_attempt', attempt: retryIndex, maxRetries: policy.maxRetries };
|
|
457
|
+
}
|
|
458
|
+
}
|
package/dist/llm/types.d.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
* into these types so the rest of the system never touches provider-specific
|
|
7
7
|
* structures.
|
|
8
8
|
*/
|
|
9
|
+
import type { LLMErrorType } from './provider.js';
|
|
9
10
|
export interface TextContentPart {
|
|
10
11
|
type: 'text';
|
|
11
12
|
text: string;
|
|
@@ -98,7 +99,45 @@ export interface ErrorEvent {
|
|
|
98
99
|
type: 'error';
|
|
99
100
|
error: Error;
|
|
100
101
|
}
|
|
101
|
-
|
|
102
|
+
/** A retryable failure occurred; the next attempt fires at `resumeAt`. */
|
|
103
|
+
export interface RetryScheduledEvent {
|
|
104
|
+
type: 'retry_scheduled';
|
|
105
|
+
/** 1-based retry index. */
|
|
106
|
+
attempt: number;
|
|
107
|
+
maxRetries: number;
|
|
108
|
+
delayMs: number;
|
|
109
|
+
/**
|
|
110
|
+
* Epoch ms the next attempt fires. THE UI TICKS AGAINST THIS, not `delayMs`.
|
|
111
|
+
*
|
|
112
|
+
* An absolute instant rather than a stream of countdown ticks is a deliberate
|
|
113
|
+
* boundary: a 1 Hz tick from core would be 300 events per exhausted retry
|
|
114
|
+
* sequence, would set core's render cadence for every embedder, and would
|
|
115
|
+
* make the idle watchdog un-fireable during a backoff for reasons unrelated to
|
|
116
|
+
* the explicit pause in `Agent`. The consumer owns its own clock.
|
|
117
|
+
*/
|
|
118
|
+
resumeAt: number;
|
|
119
|
+
errorType: LLMErrorType;
|
|
120
|
+
/** Already truncated by `classifyHttpError`. */
|
|
121
|
+
message: string;
|
|
122
|
+
/** Present only when the provider stated one. */
|
|
123
|
+
retryAfterMs?: number;
|
|
124
|
+
}
|
|
125
|
+
/** The wait is over; attempt `attempt` is going out now. */
|
|
126
|
+
export interface RetryAttemptEvent {
|
|
127
|
+
type: 'retry_attempt';
|
|
128
|
+
attempt: number;
|
|
129
|
+
maxRetries: number;
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Content already forwarded this turn is being discarded and replayed.
|
|
133
|
+
* Emitted ONLY when the failed attempt had reached its commit point.
|
|
134
|
+
*/
|
|
135
|
+
export interface StreamRestartEvent {
|
|
136
|
+
type: 'stream_restart';
|
|
137
|
+
attempt: number;
|
|
138
|
+
discardedToolCallIds: string[];
|
|
139
|
+
}
|
|
140
|
+
export type StreamEvent = TextDeltaEvent | ThinkingStartEvent | ThinkingDeltaEvent | ToolCallStartEvent | ToolCallDeltaEvent | ToolCallEndEvent | DoneEvent | ErrorEvent | RetryScheduledEvent | RetryAttemptEvent | StreamRestartEvent;
|
|
102
141
|
/** JSON Schema object — kept deliberately loose to avoid coupling to a schema lib. */
|
|
103
142
|
export type JSONSchema = Record<string, unknown>;
|
|
104
143
|
export interface ToolDefinition {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@aragon-agent/core",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.9",
|
|
4
4
|
"description": "AragonAgent core engine — LLM provider adapters, streaming, tool registry/execution, steering, idle watchdog, and an optional isolated-vm sandbox.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "AragonAgent contributors",
|