@aragon-agent/core 0.2.8 → 0.2.9

package/dist/llm/providers/index.js

@@ -3,18 +3,36 @@
  * convenience methods for streaming and completion.
  */
 import { LLMError } from '../provider.js';
+import { DEFAULT_RETRY_POLICY, withRetry } from '../retry.js';
+import { consumeStream } from '../stream-utils.js';
 import { AnthropicProvider } from './anthropic.js';
 import { OpenAIProvider } from './openai.js';
 import { GoogleProvider } from './google.js';
-// ---------------------------------------------------------------------------
-// ProviderRegistry
-// ---------------------------------------------------------------------------
 export class ProviderRegistry {
     providers = new Map();
+    retryPolicy;
+    constructor(options = {}) {
+        this.retryPolicy =
+            options.retryPolicy === undefined ? DEFAULT_RETRY_POLICY : options.retryPolicy;
+    }
     /** Register a provider. Overwrites any existing registration with the same id. */
     register(provider) {
         this.providers.set(provider.id, provider);
     }
+    /**
+     * Install a retry policy live, or `null` to opt out entirely.
+     *
+     * A METHOD RATHER THAN A CONSTRUCTOR-ONLY CONCERN because the CLI's `/retry
+     * off` has to take effect in the running session: persisting the setting and
+     * changing nothing until the next launch is the failure users report as "the
+     * setting does nothing".
+     */
+    setRetryPolicy(policy) {
+        this.retryPolicy = policy;
+    }
+    getRetryPolicy() {
+        return this.retryPolicy;
+    }
     /** Retrieve a provider by id. */
     get(id) {
         return this.providers.get(id);
@@ -28,22 +46,52 @@ export class ProviderRegistry {
         return this.providers.has(id);
     }
     /**
-     * Convenience: stream a completion through the named provider.
+     * Convenience: stream a completion through the named provider, WITH RETRY.
+     *
+     * THIS IS THE SEAM THE WHOLE RETRY FEATURE HANGS ON (llm-api-retry-backoff
+     * §5.2). Wrapping here rather than inside each adapter is what lets the retry
+     * see the commit point, the abort signal and the whole event stream at once,
+     * and it is why the lead agent, every subagent and `completeLLM` all inherit
+     * retry with zero call-site changes.
+     *
+     * The thunk captures `request` WHOLE, so every attempt sends the same body —
+     * including any `max_tokens` that `sendWithOutputLimitRecovery` repaired on the
+     * way through, because that repair lives inside `provider.stream()` and re-runs
+     * per attempt.
+     *
+     * `maxRetries <= 0` returns the adapter's iterator DIRECTLY rather than a
+     * wrapper that happens to never retry: an opted-out caller is then byte- and
+     * allocation-identical to a pre-feature build.
      *
      * @throws {LLMError} if the provider is not registered.
      */
     stream(providerId, request) {
         const provider = this.resolveProvider(providerId);
-        return provider.stream(request);
+        const policy = this.retryPolicy;
+        if (!policy || policy.maxRetries <= 0)
+            return provider.stream(request);
+        return withRetry(() => provider.stream(request), {
+            policy,
+            providerId,
+            modelId: request.model,
+            ...(request.signal ? { signal: request.signal } : {}),
+        });
     }
     /**
      * Convenience: non-streaming completion through the named provider.
      *
+     * ROUTED THROUGH `this.stream()`, NOT `provider.complete()`, and that is not
+     * optional (§5.4). All three adapters implement `complete` as exactly
+     * `consumeStream(this.stream(request))`, so this is behaviour-preserving — the
+     * only difference is that the registry's version is now the RETRYING one.
+     * Calling `provider.complete()` here would leave `completeLLM` as the one
+     * public entry point with no retry at all, which is exactly the kind of hole
+     * that gets discovered in production.
+     *
      * @throws {LLMError} if the provider is not registered.
      */
     async complete(providerId, request) {
-        const provider = this.resolveProvider(providerId);
-        return provider.complete(request);
+        return consumeStream(this.stream(providerId, request));
     }
     /**
      * Convenience: list models from the named provider.
@@ -75,9 +123,12 @@ export class ProviderRegistry {
 /**
  * Create and populate a {@link ProviderRegistry} with all built-in providers.
  * Call this once at server startup.
+ *
+ * `options` is additive and source-compatible: every existing `initProviders()`
+ * call site compiles unchanged and gets `DEFAULT_RETRY_POLICY`.
  */
-export function initProviders() {
-    const registry = new ProviderRegistry();
+export function initProviders(options) {
+    const registry = new ProviderRegistry(options);
     registry.register(new AnthropicProvider());
     registry.register(new OpenAIProvider());
     registry.register(new GoogleProvider());
@@ -93,6 +144,16 @@ let _singleton;
  * Prefer calling `initProviders()` explicitly at startup and passing the
  * registry around.  This singleton exists as a convenience for modules that
  * cannot easily receive the registry through dependency injection.
+ *
+ * IT IS A SEPARATE REGISTRY AND IT KEEPS THE PRODUCT DEFAULT RETRY POLICY
+ * (llm-api-retry-backoff §5.3). It is built here with `initProviders()` and no
+ * arguments, and a host that constructs its own registry (the CLI does) never
+ * installs it — so `streamLLM` / `completeLLM` retry with
+ * `DEFAULT_RETRY_POLICY` and do NOT observe that host's `/retry off` equivalent.
+ * Harmless while nothing calls those two helpers, and deliberately not "fixed"
+ * by having a UI mutate process-wide state. An embedder that wants one policy
+ * everywhere calls `setProviderRegistry(initProviders({ retryPolicy }))` once at
+ * startup.
  */
 export function getProviderRegistry() {
     if (!_singleton) {

package/dist/llm/providers/openai.js

@@ -98,7 +98,12 @@ export class OpenAIProvider {
         }
         if (!response.ok) {
             // `errorBody` is the ALREADY-CONSUMED body; re-reading it would throw.
-            yield { type: 'error', error: classifyHttpError(response.status, errorBody, this.id) };
+            // `response.headers` carries `Retry-After` / `x-ratelimit-reset-after` out
+            // on `LLMError.retryAfterMs` so a 429 waits as long as the server asked.
+            yield {
+                type: 'error',
+                error: classifyHttpError(response.status, errorBody, this.id, response.headers),
+            };
             return;
         }
         if (!response.body) {
@@ -112,12 +117,30 @@ export class OpenAIProvider {
         let stopReason;
         // Accumulate tool calls by index
         const toolAccumulators = new Map();
+        /**
+         * Whether this stream reached a terminal frame (§4.5a). Unlike Anthropic's
+         * `message_stop`, neither of OpenAI's terminal signals `return`s, so the
+         * absence of a return cannot stand in for the flag.
+         *
+         * TWO SIGNALS, NOT ONE, and the second is not optional. The `[DONE]`
+         * sentinel is the documented terminator, but it is a convention of OpenAI's
+         * own server rather than a property of the wire format: several
+         * OpenAI-compatible endpoints (the ones `--base-url` exists for) close the
+         * connection after the last chunk without sending it. Keying truncation on
+         * `[DONE]` alone would make every request against such a server a retryable
+         * error, i.e. ten retries and then a failure, for a stream that completed
+         * perfectly. A `finish_reason` on any choice is the model's own statement
+         * that it stopped, and a stream cut mid-answer carries neither.
+         */
+        let sawTerminal = false;
         try {
             for await (const sse of parseSSEStream(response.body, request.signal)) {
                 const raw = sse.data.trim();
                 // [DONE] sentinel
-                if (raw === '[DONE]')
+                if (raw === '[DONE]') {
+                    sawTerminal = true;
                     break;
+                }
                 let data;
                 try {
                     data = JSON.parse(raw);
@@ -183,9 +206,24 @@ export class OpenAIProvider {
                 }
                 // Finish reason handling
                 if (finishReason) {
+                    sawTerminal = true;
                     stopReason = mapFinishReason(finishReason);
                 }
             }
+            // ---- The terminal-sentinel contract (§4.5a) -------------------------
+            //
+            // An abort is NOT a truncation: `parseSSEStream` `break`s on
+            // `signal.aborted`, which lands on exactly this path, and turning that into
+            // a retryable `network_error` would re-create the trap `isRetryableError`
+            // check 1 exists to close (constraint 1 / R-17). The signal is therefore
+            // tested FIRST and that path keeps yielding `done`.
+            if (!sawTerminal && !request.signal?.aborted) {
+                yield {
+                    type: 'error',
+                    error: new LLMError(`${this.id} stream truncated before a terminal chunk`, this.id, 'network_error', true),
+                };
+                return;
+            }
             // Flush text block
             if (textAccumulator) {
                 contentBlocks.push({ type: 'text', text: textAccumulator });

package/dist/llm/retry.d.ts

@@ -0,0 +1,211 @@
+/**
+ * Retry & exponential backoff — THE authority for "a provider call failed, now
+ * what" (llm-api-retry-backoff §4).
+ *
+ * One module owns the policy type, the classification predicate, the backoff
+ * arithmetic and a single generator `withRetry()` that wraps ANY
+ * `AsyncIterableIterator<StreamEvent>`. `ProviderRegistry.stream()` applies it,
+ * so all three adapters, the lead agent and every subagent inherit retry with
+ * zero call-site changes.
+ *
+ * THE WRAPPER IS THE ONLY LAYER THAT CAN DO THIS JOB, and that is why it lives
+ * here rather than in the adapters or in `runAgentLoop` (§3.1 / §3.2):
+ *
+ *   - it knows whether any CONTENT has already been forwarded downstream (the
+ *     commit point), which decides whether a replay would duplicate text on the
+ *     user's screen;
+ *   - it holds the `AbortSignal`, which is what keeps a user pressing Esc from
+ *     being mistaken for a retryable timeout;
+ *   - it can emit its own events, which is what turns a silent 40-second stall
+ *     into a countdown a user can read.
+ *
+ * NO `node:*` IMPORTS. `no-host-coupling.test.ts` rule 2 bans every one of them
+ * under core `src/` with a single grandfathered exception, so the abortable
+ * `sleep` below uses the GLOBAL `setTimeout` plus one `abort` listener rather
+ * than `node:timers/promises` (whose `setTimeout` takes `{ signal }` natively
+ * and would be the idiomatic choice anywhere else).
+ */
+import type { StreamEvent } from './types.js';
+/**
+ * Structural bounds. NOT user policy: these describe what the mechanism can
+ * physically do, and a hand-edited config must not be able to exceed them.
+ * (The same split `TEAM_LIMITS` states for team mode, one package over.)
+ *
+ * ENFORCED AT THIS BOUNDARY by `normalizePolicy`, not only by the CLI's
+ * `clampRetryConfig` (§4.5 G4). Hardening one path and leaving the other open
+ * would let an embedder passing `{ maxRetries: 5000 }` get 5000.
+ */
+export declare const RETRY_LIMITS: {
+    /** Ceiling on `maxRetries` from any layer. */
+    readonly hardMaxRetries: 20;
+    /** Ceiling on one wait, whatever the config or a `Retry-After` header says. */
+    readonly absoluteMaxDelayMs: 120000;
+    /**
+     * A `Retry-After` LARGER than this is not honoured and not retried — it is
+     * surfaced immediately with the provider's own number in the message. A
+     * provider asking us to come back in an hour is telling the user something,
+     * and silently wedging the terminal for an hour is not relaying it.
+     */
+    readonly retryAfterCeilingMs: 60000;
+    /** Floor under any computed wait, so jitter can never produce a 0 ms "retry". */
+    readonly minDelayMs: 100;
+};
+export interface RetryPolicy {
+    /** Retries AFTER the initial attempt. `10` => up to 11 total requests. */
+    maxRetries: number;
+    initialDelayMs: number;
+    maxDelayMs: number;
+    /** Growth factor per retry. `2` doubles. */
+    multiplier: number;
+    /** Equal jitter (`d/2 + rand*d/2`). See `computeBackoffDelay` for why not full. */
+    jitter: boolean;
+    /** Honour `Retry-After` / `anthropic-ratelimit-*-reset` when present. */
+    respectRetryAfter: boolean;
+    /** Wall-clock budget from the FIRST failure of this request. */
+    maxElapsedMs: number;
+    /** Allow a restart after content has already been forwarded (§3.5). */
+    onPartialStream: boolean;
+}
+export declare const DEFAULT_RETRY_POLICY: RetryPolicy;
+/**
+ * The events that mean "the consumer has already rendered answer content", i.e.
+ * the commit point (§4.5a).
+ *
+ * `thinking_start` is in the set even though it carries no text: it flips
+ * `thinkingOpen` in the CLI reducer, which is visible state a restart must
+ * rewind. The tool events are in the set because `tool_call_start` appends a
+ * transcript card — that is why `stream_restart` carries `discardedToolCallIds`
+ * at all. `done` and `error` are terminal, not content; the three `retry_*`
+ * variants are ours and never round-trip through `makeStream()`.
+ */
+export declare const CONTENT_EVENTS: ReadonlySet<StreamEvent['type']>;
+/**
+ * Clamp a policy to `RETRY_LIMITS`. Pure, total, and it NEVER THROWS — a bad
+ * policy is corrected rather than rejected, because the alternative is a
+ * library that crashes on a config typo.
+ *
+ * Module-internal by intent: exported so `llm-retry.test.ts` may import it
+ * directly, but deliberately NOT re-exported from the barrel. The public
+ * assertion is made THROUGH `withRetry`, which is also the only way to catch
+ * this being present but unwired.
+ */
+export declare function normalizePolicy(policy: RetryPolicy): RetryPolicy;
+export interface RetryDecision {
+    retry: boolean;
+    /** Machine reason, for logs and for the acceptance tests. */
+    reason: 'retryable' | 'aborted' | 'not_retryable' | 'budget_exhausted' | 'attempts_exhausted' | 'committed' | 'retry_after_too_long';
+}
+/**
+ * Whether this failure may be retried at all, ignoring budgets and counters.
+ *
+ * THE ORDER OF THE TWO CHECKS IS LOAD-BEARING.
+ *
+ * 1. `signal?.aborted === true` COMES FIRST. `wrapFetchError` classifies an
+ *    `AbortError` as `errorType: 'timeout', retryable: true`, so every user Esc,
+ *    every `AgentController.abort()`, every dispatch timeout and every
+ *    tool-timeout cancellation arrives here wearing a retryable badge. The CLI's
+ *    `team/retry.ts` records this trap in thirty lines of comment because it
+ *    cost someone an afternoon; reading the signal BEFORE `err.retryable` is
+ *    what makes Esc mean Esc.
+ * 2. `retryable` is read STRUCTURALLY, never with `instanceof LLMError`. The
+ *    error crosses a package boundary, and a duplicated core instance (a linked
+ *    checkout, a hoisting accident) would make `instanceof` false while the
+ *    field is plainly there — and the failure mode would be "retries silently
+ *    stopped happening". This is the rule `isRetryableStreamError` and
+ *    `formatStreamError` already follow.
+ */
+export declare function isRetryableError(err: unknown, signal?: AbortSignal): RetryDecision;
+interface DecideInput {
+    error: unknown;
+    signal?: AbortSignal;
+    /** Any content event was forwarded during the failed attempt. */
+    committed: boolean;
+    /** Retries COMPLETED so far (pre-increment). */
+    retryIndex: number;
+    /** Anchored on the FIRST failure of this request; a number by construction. */
+    firstFailureAt: number;
+    /** The wait that WOULD be used for the next attempt. */
+    candidateDelay: number;
+    retryAfterMs?: number;
+    policy: RetryPolicy;
+    now: () => number;
+}
+/**
+ * The retry decision, taking every input EXPLICITLY — no closure reads — and
+ * returning the first matching reason (§4.5).
+ *
+ * Module-internal by intent, like `normalizePolicy` above.
+ */
+export declare function decide(input: DecideInput): RetryDecision;
+/**
+ * The wait before retry number `retryIndex` (1-BASED: the first retry is 1).
+ *
+ * EQUAL JITTER, NOT FULL JITTER. Full jitter (`rand() * base`) is the textbook
+ * contention minimiser, but it can return 40 ms on the sixth retry, and a UI
+ * that just announced "retrying in 30s" and then fires in 40 ms reads as a bug.
+ * Equal jitter keeps the announced number within a factor of two of the truth
+ * while still de-correlating N concurrent subagents hitting the same 429. The
+ * user-visible countdown is computed from the ACTUAL chosen delay, never from
+ * `base`, so what the card says is always what happens.
+ */
+export declare function computeBackoffDelay(retryIndex: number, policy: RetryPolicy, opts?: {
+    retryAfterMs?: number;
+    random?: () => number;
+}): number;
+/**
+ * The server-stated wait, in ms, or `undefined` when absent or unparseable.
+ *
+ * Read order: `retry-after` (integer seconds, or an HTTP-date), then
+ * `x-ratelimit-reset-after` (seconds, OpenAI-flavoured), then
+ * `anthropic-ratelimit-requests-reset`, then
+ * `anthropic-ratelimit-input-tokens-reset` (both RFC-3339 instants).
+ *
+ * THE TOKEN HEADER IS READ, not skipped: on a long context it is the limit that
+ * actually bites, and a request throttled on tokens that waits out only the
+ * REQUEST window comes back to the same 429.
+ *
+ * A value above `RETRY_LIMITS.retryAfterCeilingMs` is returned AS-IS — the
+ * CALLER (`withRetry`) decides to stop, so the honest number reaches the user's
+ * error message rather than being silently rounded down to a wait we would then
+ * actually perform.
+ */
+export declare function parseRetryAfterMs(headers: Headers | undefined, now?: () => number): number | undefined;
+export interface WithRetryOptions {
+    policy: RetryPolicy;
+    providerId: string;
+    modelId: string;
+    signal?: AbortSignal;
+    /** All three injected so tests are deterministic and never sleep. */
+    now?: () => number;
+    sleep?: (ms: number, signal?: AbortSignal) => Promise<void>;
+    random?: () => number;
+}
+/**
+ * Wrap a stream factory with retry.
+ *
+ * Six properties are load-bearing and easy to break:
+ *
+ *  - G1 — THE ABORT GUARD IS CHECKED ON EVERY ITERATION, matching
+ *    `agent-loop.ts`'s own guard. Combined with G3 below, an aborted request
+ *    never forwards a trailing error to a subscriber.
+ *  - G2 — `sleep` MUST OBSERVE `signal`. See `defaultSleep`.
+ *  - G3 — `aborted` RETURNS SILENTLY. It is the one non-retry reason that must
+ *    not `yield` an `error`. `consumeStream` (the body of `complete()`) does not
+ *    consult the signal, so a yielded error would become a thrown, FABRICATED
+ *    `network_error` on user Esc — blaming the network for something the user
+ *    just did is worse than saying nothing. Three silent return sites: the
+ *    in-loop guard, the `decide` short-circuit, and the post-sleep check.
+ *  - G4 — `normalizePolicy` RUNS ONCE, at generator entry, so `RETRY_LIMITS`
+ *    bounds an embedder's policy and not merely a CLI-resolved one.
+ *  - THE SUCCESS PATH IS UNTOUCHED. A request that works yields exactly the
+ *    events the adapter produced, in the same order, with no extra allocation
+ *    beyond one boolean and one empty array. The wrapper is not installed at all
+ *    when `maxRetries <= 0`, so an opted-out embedder is byte-identical to a
+ *    pre-feature build.
+ *  - THE TERMINAL ERROR IS FORWARDED, NOT REPLACED. After exhaustion the
+ *    consumer sees the PROVIDER's own `LLMError`, so `formatStreamError`, the
+ *    `retryable` field and any downstream `instanceof` continue to behave. Only
+ *    the retry COUNT is new information, and it travels on the events.
+ */
+export declare function withRetry(makeStream: () => AsyncIterableIterator<StreamEvent>, opts: WithRetryOptions): AsyncIterableIterator<StreamEvent>;
+export {};