@hicaru/pi-rlm 0.3.9 → 0.3.13

package/README.md

@@ -156,8 +156,7 @@ on `poolside/laguna-xs-2.1:free` (a free ~32B model):
 | Coding (retry fix) | orchestrator | **correct** (file edited) |
 | Live smoke needle | classic RLM | **hit** (~5k tokens) |
 
-> On a *free* model. Frontier models do even better. See `rlm_test/RESULTS_AGENT.md`
-> and `rlm_test/RESULTS.md` for full methodology.
+> On a *free* model. Frontier models do even better.
 
 ## Security
 

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@hicaru/pi-rlm",
-	"version": "0.3.9",
+	"version": "0.3.13",
 	"author": "hicaru",
 	"repository": {
 		"type": "git",

package/src/bridge/handlers/completion.ts

@@ -12,6 +12,7 @@ import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
 import { type ChatMsg, modelComplete } from "../model.ts";
 import { checkResourceLimits } from "../../core/resource-limits.ts";
 import { errorMessage, formatError } from "../../util/errors.ts";
+import { retryPolicy } from "../../util/retry.ts";
 import type { Semaphore } from "../../util/concurrency.ts";
 import type { Invocation, SubcallConfig } from "./types.ts";
 
@@ -33,6 +34,7 @@ export async function complete1(
   prompt: string,
   track: (usage: Usage) => void,
   deps: Complete1Deps,
+  hooks?: { readonly onThrottlePark?: (ms: number) => void; readonly onThrottleRelease?: () => void },
 ): Promise<string> {
   const config = deps.getConfig();
   const limitError = checkResourceLimits({
@@ -55,6 +57,9 @@ export async function complete1(
         maxTokens: config.subSampling?.maxTokens,
         temperature: config.subSampling?.temperature,
         reasoning: config.subSampling?.reasoning,
+        retry: retryPolicy(config),
+        onThrottlePark: hooks?.onThrottlePark,
+        onThrottleRelease: hooks?.onThrottleRelease,
         signal: deps.signal,
       }),
     );

package/src/bridge/handlers/emitting.ts

@@ -8,6 +8,7 @@
 import type { Usage } from "@earendil-works/pi-ai";
 import { isErrorText } from "../../util/errors.ts";
 import { previewText } from "../../text/preview.ts";
+import type { SubcallPhase } from "../../tool/rlm-details.ts";
 import type { Invocation } from "./types.ts";
 
 export interface EmitOpts {
@@ -24,14 +25,19 @@ export interface EmitSummary {
   readonly total?: number;
 }
 
+/** Targeted update for THIS node — the 2nd arg every `fn` handed to emitting() receives. */
+export type EmitNote = (u: { readonly phase?: SubcallPhase; readonly detail?: string }) => void;
+
 /**
  * Create a subcall node, run `fn`, then update the node with status/cost/preview.
  * `fn` should not throw for soft failures (prefer Error: strings). Hard throws mark error.
+ * The `note` arg lets `fn` surface live per-node state — e.g. parking on the rate-limit
+ * cooldown (throttleHooks below).
  */
 export async function emitting<T>(
   inv: Invocation,
   opts: EmitOpts,
-  fn: (track: (usage: Usage) => void) => Promise<T>,
+  fn: (track: (usage: Usage) => void, note: EmitNote) => Promise<T>,
   summarize: (out: T) => EmitSummary,
 ): Promise<T> {
   const id = inv.emitter.emitSubcallCreated({
@@ -44,6 +50,9 @@ export async function emitting<T>(
   });
   // Leaf nodes spend their whole lifetime waiting on the model — say so from birth.
   inv.emitter.emitSubcallUpdated({ id, phase: "waiting" });
+  const note: EmitNote = (u): void => {
+    inv.emitter.emitSubcallUpdated({ id, ...u });
+  };
 
   let costUsd = 0;
   let tokens = 0;
@@ -53,7 +62,7 @@ export async function emitting<T>(
   };
 
   try {
-    const out = await fn(track);
+    const out = await fn(track, note);
     const summary = summarize(out);
     inv.emitter.emitSubcallUpdated({
       id,
@@ -84,3 +93,22 @@ export async function emitting<T>(
 export function summarizeLeaf(out: string): EmitSummary {
   return { preview: previewText(out), error: isErrorText(out) ? out : undefined };
 }
+
+/**
+ * Wire a leaf's `note` into modelComplete's throttle callbacks: parked → "queued" with the
+ * pending seconds in detail; released → back to plain "waiting". One helper so every leaf
+ * call site reports the queue state identically.
+ */
+export function throttleHooks(note: EmitNote): {
+  readonly onThrottlePark: (ms: number) => void;
+  readonly onThrottleRelease: () => void;
+} {
+  return {
+    onThrottlePark: (ms): void => {
+      note({ phase: "queued", detail: `rate limit — waiting ${Math.max(1, Math.round(ms / 1000))}s` });
+    },
+    onThrottleRelease: (): void => {
+      note({ phase: "waiting" });
+    },
+  };
+}

package/src/bridge/handlers/llm-query.ts

@@ -5,7 +5,7 @@
 import type { Usage } from "@earendil-works/pi-ai";
 import { modelRef } from "../../config/settings.ts";
 import { complete1, type Complete1Deps } from "./completion.ts";
-import { emitting, summarizeLeaf } from "./emitting.ts";
+import { emitting, summarizeLeaf, throttleHooks } from "./emitting.ts";
 import { formatError, errorMessage } from "../../util/errors.ts";
 import { previewText } from "../../text/preview.ts";
 import type { SpawnResult, SubcallHandlerDeps } from "./types.ts";
@@ -106,7 +106,7 @@ export function createLlmQueryHandler(
           args: `prompt: ${previewText(prompt)}`,
           model: displayModel(deps),
         },
-        (track: (u: Usage) => void) => complete1(inv, prompt, track, cdeps),
+        (track: (u: Usage) => void, note) => complete1(inv, prompt, track, cdeps, throttleHooks(note)),
         summarizeLeaf,
       );
     // v5 TaskLedger for leaves: identical prompts coalesce onto one completion (key has no
@@ -166,13 +166,13 @@ export function createLlmBatchHandler(
               // NO outer gate — complete1 takes the single leaf slot per prompt.
               // v5 (audit H3): every item routes through the ledger — duplicate prompts inside
               // one batch (or twins of other in-flight leaves) coalesce instead of paying N times.
-              (track: (u: Usage) => void) =>
+              (track: (u: Usage) => void, note) =>
                 runClaimedLeaf(
                   ledger,
                   ledger === undefined ? undefined : leafClaimKey(deps, p),
                   p,
                   inv.depth,
-                  () => complete1(inv, p, track, cdeps),
+                  () => complete1(inv, p, track, cdeps, throttleHooks(note)),
                 ),
               summarizeLeaf,
             ),

package/src/bridge/handlers/rlm-query.ts

@@ -15,7 +15,7 @@ import type { Invocation, SpawnResult, SubcallHandlerDeps } from "./types.ts";
 import type { SubcallOpts } from "../../sandbox/interrupts.ts";
 import { SPAWN_HINT, spawnAndRun, type SpawnDeps } from "./task-registry.ts";
 import { complete1, type Complete1Deps } from "./completion.ts";
-import { emitting, summarizeLeaf } from "./emitting.ts";
+import { emitting, summarizeLeaf, throttleHooks } from "./emitting.ts";
 import { leafClaimKey, runClaimedLeaf } from "./llm-query.ts";
 
 const UNWIRED = formatError("RLM bridge not wired for this invocation");
@@ -265,7 +265,7 @@ export function createRlmQueryHandler(deps: SubcallHandlerDeps, sd: SpawnDeps) {
                   label: "rlm_query→llm (demoted)",
                   args: previewText(task),
                 },
-                (track) => complete1(inv, task, track, completeDeps(deps)),
+                (track, note) => complete1(inv, task, track, completeDeps(deps), throttleHooks(note)),
                 summarizeLeaf,
               ),
           ),

package/src/bridge/handlers/types.ts

@@ -3,7 +3,7 @@
  *
  * Every subcall (llm_query, llm_batch, rlm_query, rlm_batch) returns a SpawnResult
  * immediately with a task_id. The model must call await(task_id) to collect the
- * real answer. This contract is proven in rlm_test (api_v5 + batch, scores 0.89–1.0).
+ * real answer. This contract is proven in bake-off runs (api_v5 + batch, scores 0.89–1.0).
  */
 
 import type { Api, Model, Usage } from "@earendil-works/pi-ai";
@@ -93,6 +93,13 @@ export interface SubcallConfig {
   readonly rlmBudget?: number;
   /** v5 durable memory gates (optional; omitted → memory off). */
   readonly enableMemory?: boolean;
+  /** v5.1 retry knobs — structural slice of RlmConfig so retryPolicy() can read them. */
+  readonly retryMaxAttempts?: number;
+  readonly rateLimitMaxAttempts?: number;
+  readonly retryBaseDelayMs?: number;
+  readonly retryMaxDelayMs?: number;
+  readonly throttleBaseMs?: number;
+  readonly throttleMaxMs?: number;
 }
 
 export interface SubcallHandlerDeps {

package/src/bridge/model.ts

@@ -2,12 +2,16 @@
  * modelComplete — a single, serverless, in-process LLM completion.
  *
  * This is the one place that talks to a provider. It resolves the API key from pi's
- * ModelRegistry (keys live here, never in the sandbox) and calls pi-ai's `completeSimple`.
- * Used both for `llm_query` (one user prompt) and for the headless RLM root (full history).
+ * ModelRegistry (keys live here, never in the sandbox) and calls pi-ai's `completeSimple`,
+ * wrapped in completeWithRetry: transient 429/5xx get exponential backoff (honoring
+ * `retry-after` via the onResponse hook) and rate limits additionally cool the shared
+ * per-provider throttle. Used both for `llm_query` (one user prompt) and for the headless
+ * RLM root (full history).
  */
 
 import { type Api, completeSimple, type Message, type Model, type ThinkingLevel, type Usage } from "@earendil-works/pi-ai/compat";
 import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
+import { completeWithRetry, DEFAULT_RETRY_POLICY, type RetryPolicy } from "../util/retry.ts";
 
 export type Role = "system" | "user" | "assistant";
 export interface ChatMsg {
@@ -23,6 +27,11 @@ export interface CompleteOptions {
   readonly temperature?: number;
   readonly reasoning?: ThinkingLevel;
   readonly signal?: AbortSignal;
+  /** Retry + adaptive throttle for transient 429/5xx; defaults apply when omitted. */
+  readonly retry?: RetryPolicy;
+  /** v5.1 UX: fired while parked on the rate-limit cooldown ("queued") / when released. */
+  readonly onThrottlePark?: (ms: number) => void;
+  readonly onThrottleRelease?: () => void;
 }
 
 export interface CompleteResult {
@@ -78,20 +87,29 @@ export async function modelComplete(messages: readonly ChatMsg[], opts: Complete
       : opts.system
     : built.systemPrompt;
 
-  const msg = await completeSimple(
-    opts.model,
-    { systemPrompt, messages: built.messages },
-    {
-      apiKey: auth.apiKey,
-      headers: auth.headers,
-      maxTokens: opts.maxTokens,
-      temperature: opts.temperature,
-      reasoning: opts.reasoning,
-      signal: opts.signal,
+  const msg = await completeWithRetry(
+    async (note) => {
+      const response = await completeSimple(
+        opts.model,
+        { systemPrompt, messages: built.messages },
+        {
+          apiKey: auth.apiKey,
+          headers: auth.headers,
+          maxTokens: opts.maxTokens,
+          temperature: opts.temperature,
+          reasoning: opts.reasoning,
+          signal: opts.signal,
+          onResponse: (res) => { note(res.status, res.headers); },
+        },
+      );
+      // pi-ai folds provider failures into the message: "error"/"aborted" + errorMessage.
+      // Throwing here puts every completion failure onto ONE path — the retry classifier.
+      if (response.stopReason === "error" || response.stopReason === "aborted") {
+        throw new Error(response.errorMessage ?? response.stopReason);
+      }
+      return response;
     },
+    { policy: opts.retry ?? DEFAULT_RETRY_POLICY, provider: opts.model.provider, signal: opts.signal, onPark: opts.onThrottlePark, onRelease: opts.onThrottleRelease },
   );
-  if (msg.stopReason === "error" || msg.stopReason === "aborted") {
-    throw new Error(msg.errorMessage ?? msg.stopReason);
-  }
   return { text: extractText(msg.content), usage: msg.usage };
 }

package/src/config/defaults.ts

@@ -13,12 +13,22 @@ export const DEFAULT_CONFIG: Readonly<RlmConfig> = Object.freeze({
   execTimeoutS: 120,
   requestTimeoutMs: 15 * 60_000,
   // Session-wide, not per-batch: spawn() puts many requests on the wire at once, so this is
-  // the only thing bounding leaf fan-out.
-  maxConcurrentSubcalls: 16,
+  // the only thing bounding leaf fan-out. Default 8 — a sane rate for per-account limits
+  // (raise to 16/32 via /rlm-config when the provider allows).
+  maxConcurrentSubcalls: 8,
   // Children are bounded separately and lower: each is a Python subprocess holding its own copy
   // of the context it inherited, where a leaf is one HTTP request. Worst case is
-  // (maxDepth - 1) × this many concurrent child engines.
-  maxConcurrentChildren: 6,
+  // (maxDepth - 1) × this many concurrent child engines. Default 4.
+  maxConcurrentChildren: 4,
+  // v5.1 rate-limit resilience (util/retry.ts): 3 total attempts, 500ms→15s backoff,
+  // 2s→60s adaptive per-provider cooldown. All overridable in rlm.json.
+  retryMaxAttempts: 3,
+  // 429s park on the cooldown instead of dying — up to 8 windows (2s→4s→…≤60s ≈ 4 min).
+  rateLimitMaxAttempts: 8,
+  retryBaseDelayMs: 500,
+  retryMaxDelayMs: 15_000,
+  throttleBaseMs: 2_000,
+  throttleMaxMs: 60_000,
   maxPromptChars: 400_000,
   maxErrors: 5,
   orchestrator: true,

package/src/config/settings.ts

@@ -68,6 +68,19 @@ export function validateConfig(raw: unknown): Partial<RlmConfig> {
   if (maxConcurrentSubcalls !== undefined) out.maxConcurrentSubcalls = maxConcurrentSubcalls;
   const maxConcurrentChildren = validateNumber(r.maxConcurrentChildren, 1);
   if (maxConcurrentChildren !== undefined) out.maxConcurrentChildren = maxConcurrentChildren;
+  // v5.1 rate-limit resilience knobs
+  const retryMaxAttempts = validateNumber(r.retryMaxAttempts, 1);
+  if (retryMaxAttempts !== undefined) out.retryMaxAttempts = retryMaxAttempts;
+  const rateLimitMaxAttempts = validateNumber(r.rateLimitMaxAttempts, 1);
+  if (rateLimitMaxAttempts !== undefined) out.rateLimitMaxAttempts = rateLimitMaxAttempts;
+  const retryBaseDelayMs = validateNumber(r.retryBaseDelayMs, 0);
+  if (retryBaseDelayMs !== undefined) out.retryBaseDelayMs = retryBaseDelayMs;
+  const retryMaxDelayMs = validateNumber(r.retryMaxDelayMs, 100);
+  if (retryMaxDelayMs !== undefined) out.retryMaxDelayMs = retryMaxDelayMs;
+  const throttleBaseMs = validateNumber(r.throttleBaseMs, 0);
+  if (throttleBaseMs !== undefined) out.throttleBaseMs = throttleBaseMs;
+  const throttleMaxMs = validateNumber(r.throttleMaxMs, 100);
+  if (throttleMaxMs !== undefined) out.throttleMaxMs = throttleMaxMs;
   const maxPromptChars = validateNumber(r.maxPromptChars, 1000);
   if (maxPromptChars !== undefined) out.maxPromptChars = maxPromptChars;
   const maxTimeoutMs = validateNumber(r.maxTimeoutMs, 1000);

package/src/core/budget.ts

@@ -1,5 +1,5 @@
 /**
- * Token budget cascade (port of rlm_test v4/v5 `budget.py`).
+ * Token budget cascade (port of the v4/v5 `budget.py` engine).
  *
  * The budget is the PRIMARY run-length control: cap = budgetShare × model context window,
  * one soft wrap-up turn at `softFrac` of the cap, and at the hard cap a deterministic

package/src/core/compaction.ts

@@ -9,6 +9,7 @@
 import type { Api, Model, Usage } from "@earendil-works/pi-ai";
 import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
 import { type ChatMsg, modelComplete } from "../bridge/model.ts";
+import type { RetryPolicy } from "../util/retry.ts";
 import { estimateMessageTokens } from "../text/tokens.ts";
 
 const DEFAULT_CONTEXT_WINDOW = 128_000;
@@ -24,6 +25,8 @@ export interface CompactionDeps {
   readonly contextWindow?: number;
   readonly thresholdPct?: number;
   readonly signal?: AbortSignal;
+  /** v5.1 retry policy for modelComplete; defaults apply when omitted. */
+  readonly retry?: RetryPolicy;
 }
 
 /** True if the history is at/over the compaction threshold. */
@@ -93,6 +96,7 @@ export async function compactHistory(
     model: deps.model,
     registry: deps.registry,
     signal: deps.signal,
+    retry: deps.retry,
   });
   onUsage?.(usage);
   const system = history.find((m) => m.role === "system");

package/src/core/engine.ts

@@ -29,6 +29,7 @@ import { previewStdout, previewText } from "../text/preview.ts";
 import { contextLength, contextSizeStats, contextTypeLabel } from "../text/tokens.ts";
 import { finalAnswerOf, formatReplOutputs, latestAnswerContentOf, turnHadError } from "./answer.ts";
 import { compactHistory, elideOldToolPayloads, shouldCompact } from "./compaction.ts";
+import { retryPolicy } from "../util/retry.ts";
 import { appendUserMessage } from "./history.ts";
 import { runTurn } from "./iteration.ts";
 import { type Limits, LimitError, LimitGuard } from "./limits.ts";
@@ -138,7 +139,7 @@ export function createEngine(deps: EngineDeps): RunRlm {
     if (deps.config.enableLedger) runLedger.beginRun(input.rootPrompt);
 
     // v5 durable memory: read-only root replay — an identical prompt over an identical
-    // context answers for zero API calls (measured 10,051 → 0 tok in rlm_test).
+    // context answers for zero API calls (measured 10,051 → 0 tok in bake-off runs).
     const rootMemory =
       deps.memory !== undefined && deps.config.enableMemory ? deps.memory : undefined;
     const modelRefStr = `${model.provider}/${model.id}`;
@@ -329,6 +330,7 @@ export function createEngine(deps: EngineDeps): RunRlm {
             registry: deps.registry,
             contextWindow: model.contextWindow,
             thresholdPct: deps.config.compactionThresholdPct,
+            retry: retryPolicy(deps.config),
             signal: deps.signal,
           };
           if (shouldCompact(history, compactionDeps)) {
@@ -341,7 +343,7 @@ export function createEngine(deps: EngineDeps): RunRlm {
           pendingReplOutputs = undefined;
         }
 
-        // Soft runtime nudge (rlm_test parity): remind the model to await pending host tasks.
+        // Soft runtime nudge (engine parity): remind the model to await pending host tasks.
         const pendingIds = taskRegistry.awaitDeps.unawaitedIds();
         if (pendingIds.length > 0) {
           appendUserMessage(
@@ -372,6 +374,7 @@ export function createEngine(deps: EngineDeps): RunRlm {
           model: model,
           registry: deps.registry,
           sampling: rootSampling,
+          retry: deps.complete === undefined ? retryPolicy(deps.config) : undefined,
           signal: deps.signal,
           complete: deps.complete,
           onPhase: reportPhase,

package/src/core/iteration.ts

@@ -7,6 +7,7 @@
 import type { Api, Model, Usage } from "@earendil-works/pi-ai";
 import type { ModelRegistry } from "@earendil-works/pi-coding-agent";
 import { type ChatMsg, type CompleteOptions, type CompleteResult, modelComplete } from "../bridge/model.ts";
+import type { RetryPolicy } from "../util/retry.ts";
 import type { ReplResult } from "../sandbox/protocol.ts";
 import type { PythonSandbox } from "../sandbox/sandbox.ts";
 import { findReplBlocks } from "../text/parsing.ts";
@@ -31,6 +32,8 @@ export interface TurnDeps {
   readonly signal?: AbortSignal;
   /** Test-only override for model completion (scripted responses). */
   readonly complete?: CompleteFn;
+  /** v5.1 retry policy for modelComplete (rate-limit resilience); defaults apply when omitted. */
+  readonly retry?: RetryPolicy;
   /** Live activity reporting for the tree UI (thinking → repl/texting per turn). */
   readonly onPhase?: (phase: SubcallPhase) => void;
 }
@@ -44,6 +47,10 @@ export async function runTurn(history: readonly ChatMsg[], sandbox: PythonSandbo
     maxTokens: deps.sampling?.maxTokens,
     temperature: deps.sampling?.temperature,
     reasoning: deps.sampling?.reasoning,
+    retry: deps.retry,
+    // Rate-limit parking is a visible phase too — the root/child spinner says "queued".
+    onThrottlePark: deps.onPhase ? () => deps.onPhase?.("queued") : undefined,
+    onThrottleRelease: deps.onPhase ? () => deps.onPhase?.("thinking") : undefined,
     signal: deps.signal,
   });
 

package/src/core/ledger.ts

@@ -1,5 +1,5 @@
 /**
- * TaskLedger — the session blackboard (port of rlm_test v5 `ledger.py`).
+ * TaskLedger — the session blackboard (port of the v5 `ledger.py` engine).
  *
  * One instance per root run (engine) or per session (native repl tool), threaded down to every
  * child through `SubcallHandlerDeps.ledger` / `RlmInput.ledger` — the same seam as

package/src/core/memory.ts

@@ -1,5 +1,5 @@
 /**
- * Durable memory (port of rlm_test v5 `memory/store.py`).
+ * Durable memory (port of the v5 `memory/store.py` engine).
  *
  * L1 episodes: content-addressed replay — a recorded child/root answer replays for ZERO
  * API calls while every file it touched still hashes to the recorded sha256.

package/src/core/model-registry.ts

@@ -1,5 +1,5 @@
 /**
- * Model context-window registry (port of rlm_test v4/v5 `models.py`).
+ * Model context-window registry (port of the v4/v5 `models.py` engine).
  *
  * The plugin already knows context windows from model metadata (`Model.contextWindow`); this
  * registry is the offline fallback for models whose metadata carries none: a conservative

package/src/core/types.ts

@@ -25,6 +25,20 @@ export interface RlmConfig {
   /** Concurrent recursive child engines admitted per depth. Lower than maxConcurrentSubcalls:
    *  each child is a Python subprocess holding its own copy of the inherited context. */
   readonly maxConcurrentChildren: number;
+  /** v5.1 rate-limit resilience (see util/retry.ts): transient 429/5xx are retried with
+   *  backoff, and rate limits additionally cool a shared per-provider throttle.
+   *  retryMaxAttempts counts TOTAL attempts per call (1 = never retry).
+   *  rateLimitMaxAttempts is the SEPARATE budget a 429 may burn while parking on the
+   *  cooldown — generous, because "come back later" is a queue, not a failure. */
+  readonly retryMaxAttempts?: number;
+  readonly rateLimitMaxAttempts?: number;
+  readonly retryBaseDelayMs?: number;
+  /** Cap for any single retry delay, including a parsed `retry-after`. */
+  readonly retryMaxDelayMs?: number;
+  /** First cooldown when a provider 429s without timing; doubles per consecutive strike. */
+  readonly throttleBaseMs?: number;
+  /** Ceiling for the adaptive per-provider cooldown. */
+  readonly throttleMaxMs?: number;
   /** Reject sub-LLM prompts larger than this many chars. */
   readonly maxPromptChars: number;
   /** Max wall-clock ms across the whole tree before the engine stops (undefined = no cap). */

package/src/index.ts

@@ -23,6 +23,7 @@ import { buildSessionGates, type SubcallGates } from "./util/concurrency.ts";
 import { BackgroundTasks } from "./tool/background-tasks.ts";
 import { MemoryStore } from "./core/memory.ts";
 import { modelComplete } from "./bridge/model.ts";
+import { retryPolicy } from "./util/retry.ts";
 import { resolve } from "node:path";
 import { resolveSource } from "./context/resolve.ts";
 import { formatContextListing } from "./context/listing.ts";
@@ -256,7 +257,11 @@ export default function rlmExtension(pi: ExtensionAPI): void {
       // the workspace root is only known once the session starts.
       const consolidateModel = llmModel;
       memory.setLlm((prompt) =>
-        modelComplete([{ role: "user", content: prompt }], { model: consolidateModel, registry: ctx.modelRegistry })
+        modelComplete([{ role: "user", content: prompt }], {
+          model: consolidateModel,
+          registry: ctx.modelRegistry,
+          retry: retryPolicy(controller.config),
+        })
           .then((r) => r.text));
       memory.setRoot(ctx.cwd ?? process.cwd());
       // v5 provider caps (audit C1/C6): ONE resolver shared by both composition roots — the

package/src/prompts/native.ts

@@ -1,6 +1,6 @@
 /** Native-mode prompts — the main Pi agent drives the sandbox through the `repl` tool.
  *
- * Structure mirrors rlm_test api_v5_anthropic (best bake-off arm): role → contract → routing →
+ * Structure mirrors api_v5_anthropic (best bake-off arm): role → contract → routing →
  * few-shots → anti-patterns → REPL surface. Goal: multi-area work fires rlm_batch / rlm_query
  * as Task (BG), not serial repl+native read.
  */

package/src/tool/rlm-details.ts

@@ -9,8 +9,9 @@
 export type SubcallKind = "root" | "rlm" | "llm" | "batch" | "tool";
 export type SubcallStatus = "running" | "done" | "error";
 
-/** Live activity of a node while status is "running" — powers the tree/modal UI. */
-export type SubcallPhase = "thinking" | "texting" | "repl" | "waiting" | "spawning";
+/** Live activity of a node while status is "running" — powers the tree/modal UI.
+ *  "queued" = parked on the rate-limit cooldown (util/throttle.ts), not in flight. */
+export type SubcallPhase = "thinking" | "texting" | "repl" | "waiting" | "spawning" | "queued";
 export type RlmRunStatus = "running" | "done" | "error" | "aborted";
 
 export interface RlmSubcall {

package/src/ui/tree/tree-model.ts

@@ -40,7 +40,8 @@ export interface NodeRow {
   readonly prefix: string;
   readonly expandable: boolean;
   readonly expanded: boolean;
-  readonly icon: SubcallStatus;
+  /** SubcallStatus, or "queued" while the node parks on the rate-limit cooldown. */
+  readonly icon: SubcallStatus | "queued";
   readonly phase?: SubcallPhase;
   readonly label: string;
   /** The row's OWN token spend for its OWN model — never a subtree sum. */
@@ -63,7 +64,8 @@ export interface GroupRow {
   readonly model?: string;
   /** Sum over members — one model only (the group key pins it), so never a blend. */
   readonly tokens: number;
-  readonly icon: SubcallStatus;
+  /** SubcallStatus, or "queued" while any member parks on the rate-limit cooldown. */
+  readonly icon: SubcallStatus | "queued";
   readonly expandable: boolean;
   readonly expanded: boolean;
 }
@@ -98,9 +100,12 @@ function partition(children: readonly RlmSubcall[], byParent: ReadonlyMap<string
   return out;
 }
 
-/** RlmRunStatus has "aborted"; the row icon set does not — aborted renders as error. */
-function iconOf(status: SubcallStatus | RlmRunStatus): SubcallStatus {
-  return status === "aborted" ? "error" : status;
+/** RlmRunStatus has "aborted"; the row icon set does not — aborted renders as error.
+ *  A RUNNING node parked on the rate-limit cooldown renders as "queued" (◷). */
+function iconOf(status: SubcallStatus | RlmRunStatus, phase: SubcallPhase | undefined): SubcallStatus | "queued" {
+  if (status === "aborted") return "error";
+  if (status === "running" && phase === "queued") return "queued";
+  return status;
 }
 
 /**
@@ -138,7 +143,7 @@ export function buildRows(
       prefix,
       expandable: children.length > 0,
       expanded,
-      icon: iconOf(sc.status),
+      icon: iconOf(sc.status, sc.phase),
       phase: sc.phase,
       label: sc.label,
       tokens: sc.tokens,
@@ -187,7 +192,7 @@ export function buildRows(
       label: entry.label,
       model: entry.model,
       tokens,
-      icon: iconOf(entry.status),
+      icon: iconOf(entry.status, entry.members.some((m) => m.phase === "queued") ? "queued" : undefined),
       expandable: true,
       expanded,
     });
@@ -209,7 +214,7 @@ export function buildRows(
     prefix: "",
     expandable: roots.length > 0,
     expanded: !collapsed.has(run.runId),
-    icon: iconOf(run.status),
+    icon: iconOf(run.status, run.rootPhase),
     phase: run.rootPhase,
     label: run.rootLabel,
     tokens: run.rootTokens,

package/src/ui/tree/tree-rows.ts

@@ -11,7 +11,7 @@ import { truncateToWidth, visibleWidth } from "@earendil-works/pi-tui";
 import { formatTokens, spinnerFrame } from "../theme.ts";
 import type { GroupRow, NodeRow, TreeRow } from "./tree-model.ts";
 
-const GLYPHS = Object.freeze({ done: "✓", error: "✗", expanded: "▾", collapsed: "▸", leaf: " " } as const);
+const GLYPHS = Object.freeze({ done: "✓", error: "✗", queued: "◷", expanded: "▾", collapsed: "▸", leaf: " " } as const);
 const MODEL_MAX = 14;
 
 /** "openai/gpt-5-mini" → "gpt-5-mini", hard-capped so rows stay on one line. */
@@ -24,6 +24,7 @@ function iconGlyph(row: NodeRow, theme: Theme): string {
   switch (row.icon) {
     case "done": return theme.fg("success", GLYPHS.done);
     case "error": return theme.fg("error", GLYPHS.error);
+    case "queued": return theme.fg("warning", GLYPHS.queued);
     default: return theme.fg("warning", spinnerFrame());
   }
 }

package/src/util/retry.ts

@@ -0,0 +1,180 @@
+/**
+ * Retry + rate-limit classification for LLM completions — the retry half of resilience.
+ *
+ * pi-ai surfaces provider failures as `stopReason:"error"` plus an errorMessage STRING
+ * (the HTTP status baked into the text, e.g. `429: {"code":"1302",...}`), but its
+ * `onResponse` hook still hands us the raw `{status, headers}` of every HTTP response.
+ * Classification therefore runs on both: captured status first, text patterns as the
+ * fallback — the same shape pi-ai's own codex provider uses internally. Timing, when the
+ * provider sends it, comes from `retry-after` / `retry-after-ms` headers (numeric seconds,
+ * milliseconds, or an HTTP-date); otherwise exponential backoff with jitter, capped.
+ *
+ * Rate-limit errors additionally penalize the per-provider cooldown (util/throttle.ts):
+ * the retry sleeps, and every OTHER request queued behind the same provider waits too.
+ */
+
+import { ProviderCooldown, sleepMs, sharedCooldown } from "./throttle.ts";
+
+// Auth/quota failures must not be retried — they burn attempts and never recover.
+const NON_RETRYABLE_TEXT =
+  /api[ -]?key|unauthorized|forbidden|permission denied|billing|insufficient|balance|quota exceeded|not.?found|context length|too large|invalid request|malformed/i;
+// Transport/server transients — worth another attempt.
+const RETRYABLE_TEXT =
+  /\b429\b|rate.?limit|overloaded|service.?unavailable|upstream|timeout|timed.?out|temporarily|try.?again|econnreset|econnrefused|etimedout|socket hang up|network|1302|速率|频率/i;
+const RATE_LIMIT_TEXT = /\b429\b|rate.?limit|1302|速率|频率/i;
+
+const NON_RETRYABLE_STATUS = new Set([400, 401, 402, 403, 404, 413, 422]);
+const RETRYABLE_STATUS = new Set([408, 429, 500, 502, 503, 504]);
+
+/** Did this failure mean "too many requests"? Drives the cooldown penalty. */
+export function isRateLimited(status: number | undefined, text: string): boolean {
+  return status === 429 || (status === undefined && RATE_LIMIT_TEXT.test(text));
+}
+
+/** Should this failure get another attempt? Explicit non-retryables win over patterns. */
+export function retryableError(status: number | undefined, text: string): boolean {
+  if (status !== undefined) {
+    if (NON_RETRYABLE_STATUS.has(status)) return false;
+    if (RETRYABLE_STATUS.has(status)) return true;
+  }
+  if (NON_RETRYABLE_TEXT.test(text)) return false;
+  return RETRYABLE_TEXT.test(text);
+}
+
+/** Header lookup that tolerates any key casing the provider layer kept. */
+function header(headers: Record<string, string>, name: string): string | undefined {
+  const lower = name.toLowerCase();
+  for (const k of Object.keys(headers)) {
+    if (k.toLowerCase() === lower) return headers[k];
+  }
+  return undefined;
+}
+
+/** Parse `retry-after` / `retry-after-ms` into ms; undefined when absent or garbage. */
+export function retryAfterMs(headers: Record<string, string> | undefined): number | undefined {
+  if (headers === undefined) return undefined;
+  const ms = header(headers, "retry-after-ms");
+  if (ms !== undefined) {
+    const n = Number(ms);
+    if (Number.isFinite(n) && n >= 0) return n;
+  }
+  const ra = header(headers, "retry-after");
+  if (ra === undefined) return undefined;
+  const seconds = Number(ra);
+  if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000);
+  const date = Date.parse(ra);
+  return Number.isNaN(date) ? undefined : Math.max(0, date - Date.now());
+}
+
+/** Exponential backoff with ±30% jitter, hard-capped at `maxMs`. */
+export function backoffMs(attempt: number, baseMs: number, maxMs: number): number {
+  const raw = baseMs * 2 ** attempt * (0.7 + Math.random() * 0.6);
+  return Math.min(Math.round(raw), maxMs);
+}
+
+/** Numbers knobs — mirrors the optional RlmConfig fields (core/types.ts). */
+export interface RetryPolicy {
+  /** TOTAL attempts per call, including the first. 1 = never retry. */
+  readonly maxAttempts: number;
+  /** Separate, GENEROUS budget for rate limits only: a 429 means "come back later",
+   *  not "fail" — the call keeps parking on the cooldown window instead of dying.
+   *  Burned only by rate-limited failures, never by 5xx/timeouts. */
+  readonly rateLimitMaxAttempts: number;
+  readonly baseDelayMs: number;
+  /** Cap for any single retry delay, including a parsed `retry-after`. */
+  readonly maxDelayMs: number;
+  /** First cooldown when a provider 429s without timing; doubles per consecutive strike. */
+  readonly throttleBaseMs: number;
+  /** Ceiling for the adaptive per-provider cooldown. */
+  readonly throttleMaxMs: number;
+  /** Isolation for tests; defaults to the process-wide shared cooldown. */
+  readonly cooldown?: ProviderCooldown;
+}
+
+export const DEFAULT_RETRY_POLICY: Readonly<RetryPolicy> = Object.freeze({
+  maxAttempts: 3,
+  rateLimitMaxAttempts: 8,
+  baseDelayMs: 500,
+  maxDelayMs: 15_000,
+  throttleBaseMs: 2_000,
+  throttleMaxMs: 60_000,
+});
+
+/** Shape of the optional retry knobs on RlmConfig — kept structural to avoid a cycle. */
+export interface RetryConfigNumbers {
+  readonly retryMaxAttempts?: number;
+  readonly rateLimitMaxAttempts?: number;
+  readonly retryBaseDelayMs?: number;
+  readonly retryMaxDelayMs?: number;
+  readonly throttleBaseMs?: number;
+  readonly throttleMaxMs?: number;
+}
+
+/** Derive a policy from persisted config knobs, falling back to the defaults. */
+export function retryPolicy(from: RetryConfigNumbers = {}): RetryPolicy {
+  return {
+    maxAttempts: from.retryMaxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
+    rateLimitMaxAttempts: from.rateLimitMaxAttempts ?? DEFAULT_RETRY_POLICY.rateLimitMaxAttempts,
+    baseDelayMs: from.retryBaseDelayMs ?? DEFAULT_RETRY_POLICY.baseDelayMs,
+    maxDelayMs: from.retryMaxDelayMs ?? DEFAULT_RETRY_POLICY.maxDelayMs,
+    throttleBaseMs: from.throttleBaseMs ?? DEFAULT_RETRY_POLICY.throttleBaseMs,
+    throttleMaxMs: from.throttleMaxMs ?? DEFAULT_RETRY_POLICY.throttleMaxMs,
+  };
+}
+
+/**
+ * Run `attempt` under the policy. `note` is how the caller feeds captured HTTP
+ * `{status, headers}` back (pi-ai's onResponse hook) — cleared before every attempt so a
+ * stale capture never misclassifies a fresh failure. Rate-limit failures park on the
+ * cooldown — their own, generous budget — so sibling requests slow down with us;
+ * `onPark`/`onRelease` surface the parking to the UI as a "queued" phase.
+ */
+export async function completeWithRetry<T>(
+  attempt: (note: (status: number, headers: Record<string, string>) => void) => Promise<T>,
+  opts: {
+    readonly policy: RetryPolicy;
+    readonly provider: string;
+    readonly signal?: AbortSignal;
+    readonly onPark?: (ms: number) => void;
+    readonly onRelease?: () => void;
+  },
+): Promise<T> {
+  const { policy, provider, signal, onPark, onRelease } = opts;
+  const cooldown = policy.cooldown ?? sharedCooldown;
+  let status: number | undefined;
+  let headers: Record<string, string> | undefined;
+  const note = (s: number, h: Record<string, string>): void => {
+    status = s;
+    headers = h;
+  };
+  let rlTries = 0; // rate-limit failures burn their OWN budget, never maxAttempts
+  for (let tries = 0; ; tries++) {
+    await cooldown.wait(provider, signal, onPark, onRelease);
+    status = undefined;
+    headers = undefined;
+    try {
+      const out = await attempt(note);
+      cooldown.success(provider);
+      return out;
+    } catch (err: unknown) {
+      const msg = err instanceof Error ? err.message : String(err);
+      if (signal?.aborted) throw err;
+      if (isRateLimited(status, msg)) {
+        // "Come back later" — park on the shared cooldown instead of dying. The strike
+        // heuristic escalates the window; the wait itself happens at the loop top, so
+        // sibling requests behind the same provider slow down together.
+        if (rlTries + 1 >= policy.rateLimitMaxAttempts) throw err;
+        rlTries++;
+        const hint = retryAfterMs(headers);
+        cooldown.penalize(provider, hint !== undefined ? Math.min(hint, policy.throttleMaxMs) : undefined);
+        continue;
+      }
+      if (tries + 1 >= policy.maxAttempts) throw err;
+      if (!retryableError(status, msg)) throw err;
+      await sleepMs(
+        Math.min(retryAfterMs(headers) ?? backoffMs(tries, policy.baseDelayMs, policy.maxDelayMs), policy.maxDelayMs),
+        signal,
+      );
+    }
+  }
+}

package/src/util/throttle.ts

@@ -0,0 +1,90 @@
+/**
+ * Per-provider adaptive cooldown — the throttle half of rate-limit resilience.
+ *
+ * A 429 means the provider wants FEWER requests for a while, not just this one retried.
+ * `ProviderCooldown` holds each provider's admission time: `wait()` parks new requests
+ * until the window opens; `penalize()` extends it, escalating on consecutive strikes
+ * (base × 2^strikes, capped) so a persistent limit backs the whole fan-out off
+ * exponentially. Any success clears the strike counter — providers rarely announce
+ * recovery, so we probe again at full concurrency rather than assume the worst.
+ *
+ * `sharedCooldown` is process-wide on purpose: the provider's limit is process-wide.
+ * Tests inject a fresh instance via `RetryPolicy.cooldown` for isolation.
+ */
+
+/** Abort-aware sleep. Rejects with "aborted" the moment `signal` fires. */
+export function sleepMs(ms: number, signal?: AbortSignal): Promise<void> {
+  if (ms <= 0) return Promise.resolve();
+  return new Promise<void>((resolve, reject) => {
+    const timer = setTimeout((): void => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = (): void => {
+      clearTimeout(timer);
+      reject(new Error("aborted"));
+    };
+    if (signal?.aborted) {
+      clearTimeout(timer);
+      reject(new Error("aborted"));
+      return;
+    }
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+export class ProviderCooldown {
+  private readonly blockedUntil = new Map<string, number>();
+  private readonly strikes = new Map<string, number>();
+
+  constructor(private readonly baseMs: number, private readonly maxMs: number) {}
+
+  /** ms until `provider` may be admitted again; 0 = free. */
+  waitMs(provider: string): number {
+    return Math.max(0, (this.blockedUntil.get(provider) ?? 0) - Date.now());
+  }
+
+  /** Park until the window opens. Re-checks after every wake — penalize() may extend it.
+   *  `onPark` fires with the pending ms before each sleep (UI: "queued on rate limit"),
+   *  `onRelease` once right after the wait ends — but only if we actually parked. */
+  async wait(provider: string, signal?: AbortSignal, onPark?: (ms: number) => void, onRelease?: () => void): Promise<void> {
+    let parked = false;
+    for (;;) {
+      const ms = this.waitMs(provider);
+      if (ms <= 0) break;
+      parked = true;
+      onPark?.(ms);
+      await sleepMs(ms, signal);
+    }
+    if (parked) onRelease?.();
+  }
+
+  /**
+   * Extend the window. `ms` (e.g. a parsed `retry-after`) floors the penalty; the strike
+   * heuristic (base × 2^strikes, capped) always applies on top so repeated 429s escalate
+   * even when the provider sends no timing at all — zai's `{"code":"1302",...}` body, for
+   * one, carries none.
+   */
+  penalize(provider: string, ms?: number): void {
+    const strikes = (this.strikes.get(provider) ?? 0) + 1;
+    this.strikes.set(provider, strikes);
+    const heuristic = Math.min(this.baseMs * 2 ** (strikes - 1), this.maxMs);
+    const until = Date.now() + Math.max(ms ?? 0, heuristic);
+    const prev = this.blockedUntil.get(provider) ?? 0;
+    if (until > prev) this.blockedUntil.set(provider, until);
+  }
+
+  /** A success means the window opened — reset escalation for the next burst. */
+  success(provider: string): void {
+    this.strikes.delete(provider);
+  }
+}
+
+/** Mirrored into DEFAULT_RETRY_POLICY (util/retry.ts imports these — keep one-way). */
+export const THROTTLE_DEFAULTS = Object.freeze({ baseMs: 2_000, maxMs: 60_000 } as const);
+
+/** Process-wide instance: every completion in this pi session shares it. */
+export const sharedCooldown: ProviderCooldown = new ProviderCooldown(
+  THROTTLE_DEFAULTS.baseMs,
+  THROTTLE_DEFAULTS.maxMs,
+);