@possumtech/rummy 2.0.0 → 2.1.0

package/src/llm/LlmProvider.js

@@ -1,23 +1,33 @@
+import config from "../agent/config.js";
 import msg from "../agent/messages.js";
 import {
 	ContextExceededError,
+	classifyTransient,
 	isContextExceededMessage,
-	isTransientMessage,
 } from "./errors.js";
+import { retryClassified } from "./retry.js";
 
-const MAX_TRANSIENT_RETRIES = 3;
+const { LLM_DEADLINE, LLM_MAX_BACKOFF } = config;
 
-/**
- * Thin dispatcher over the LLM provider registry (`hooks.llm.providers`).
- * Resolves the model alias via the DB, finds the highest-priority provider
- * whose `matches()` returns true, and delegates. Wraps the call with
- * transient-error retry and surfaces context-exceeded as a typed
- * ContextExceededError.
- *
- * Vendor-specific HTTP is owned by per-vendor plugins under
- * `src/plugins/{openai,ollama,xai,openrouter,...}/`. Adding a new vendor
- * is a matter of adding a plugin — no changes here.
- */
+// Per-category retry policies. Gateway/server are bounded short because
+// upstream-down won't recover by waiting; warmup/rate_limit get the full
+// LLM deadline because they're recoverable wait states with knowable bounds.
+const POLICIES = Object.freeze({
+	gateway: { deadlineMs: 30_000, baseDelayMs: 500, maxDelayMs: 5_000 },
+	warmup: {
+		deadlineMs: LLM_DEADLINE,
+		baseDelayMs: 2000,
+		maxDelayMs: LLM_MAX_BACKOFF,
+	},
+	rate_limit: {
+		deadlineMs: LLM_DEADLINE,
+		baseDelayMs: 1000,
+		maxDelayMs: LLM_MAX_BACKOFF,
+	},
+	server: { deadlineMs: 60_000, baseDelayMs: 1000, maxDelayMs: 10_000 },
+});
+
+// Dispatches to hooks.llm.providers; per-category transient retry; ContextExceededError surface.
 export default class LlmProvider {
 	#db;
 	#hooks;
@@ -55,27 +65,25 @@ export default class LlmProvider {
 			);
 		}
 
-		for (let attempt = 0; ; attempt++) {
-			try {
-				return await provider.completion(
-					messages,
-					resolvedModel,
-					resolvedOptions,
-				);
-			} catch (err) {
-				if (isContextExceededMessage(err.message)) {
-					throw new ContextExceededError(err.message, { cause: err });
-				}
-				if (
-					isTransientMessage(err.message) &&
-					attempt < MAX_TRANSIENT_RETRIES
-				) {
-					const delay = 1000 * 2 ** attempt;
-					await new Promise((r) => setTimeout(r, delay));
-					continue;
-				}
-				throw err;
+		try {
+			return await retryClassified(
+				() => provider.completion(messages, resolvedModel, resolvedOptions),
+				{
+					signal: options.signal,
+					classify: classifyTransient,
+					policies: POLICIES,
+					onRetry: (err, category, attempt, delayMs, remainingMs) => {
+						console.error(
+							`[LLM] ${category} on ${provider.name} attempt ${attempt}: ${err.message}; retrying in ${delayMs}ms (${Math.round(remainingMs / 1000)}s ${category} budget remaining)`,
+						);
+					},
+				},
+			);
+		} catch (err) {
+			if (isContextExceededMessage(err.message)) {
+				throw new ContextExceededError(err.message, { cause: err });
 			}
+			throw err;
 		}
 	}
 

package/src/llm/errors.js

@@ -13,9 +13,46 @@ export function isContextExceededMessage(message) {
 	return CONTEXT_EXCEEDED_PATTERN.test(String(message));
 }
 
-const TRANSIENT_PATTERN =
-	/\b(503|429|timeout|ECONNREFUSED|ECONNRESET|unavailable)\b/i;
+const ABORT_PATTERN = /\b(aborted|AbortError|TimeoutError)\b/;
+// `terminated` is undici's err.message when the underlying socket closes
+// mid-fetch (TLSSocket.onHttpSocketClose → Fetch.onAborted) — same lane
+// as ECONNRESET, just surfaced through a streaming-fetch path.
+const GATEWAY_PATTERN =
+	/\b(502|504|ECONNREFUSED|ECONNRESET|ENOTFOUND|EHOSTUNREACH|ETIMEDOUT|EPIPE|ECONNABORTED|fetch failed|terminated)\b/i;
+const RATE_LIMIT_PATTERN = /\b429\b/;
+const STATUS_503_PATTERN = /\b503\b/;
+const STATUS_500_PATTERN = /\b500\b/;
+// llamacpp / OpenAI-compatible servers signal model-warmup with this body.
+const MODEL_WARMUP_PATTERN = /\bLoading model\b/i;
 
-export function isTransientMessage(message) {
-	return TRANSIENT_PATTERN.test(String(message));
+// Returns "gateway" | "warmup" | "rate_limit" | "server" | null.
+// null = do not retry, propagate immediately. Operator/internal aborts,
+// auth failures, malformed-request errors, unknown shapes all fall here.
+export function classifyTransient(err) {
+	if (!err || typeof err.message !== "string") return null;
+	const { message } = err;
+
+	if (ABORT_PATTERN.test(message)) return null;
+	if (GATEWAY_PATTERN.test(message)) return "gateway";
+	if (RATE_LIMIT_PATTERN.test(message)) return "rate_limit";
+	if (STATUS_503_PATTERN.test(message)) {
+		// 503 + explicit warmup signal → wait it out.
+		if (MODEL_WARMUP_PATTERN.test(message)) return "warmup";
+		if (typeof err.body === "string" && MODEL_WARMUP_PATTERN.test(err.body)) {
+			return "warmup";
+		}
+		return "server";
+	}
+	if (STATUS_500_PATTERN.test(message)) return "server";
+	return null;
+}
+
+// HTTP Retry-After: integer seconds (most common form). Returns
+// undefined for missing, malformed, or HTTP-date forms — callers
+// fall through to backoff in those cases.
+export function parseRetryAfter(value) {
+	if (!value) return undefined;
+	const seconds = Number(value);
+	if (Number.isFinite(seconds) && seconds >= 0) return seconds;
+	return undefined;
 }

package/src/llm/openaiStream.js

@@ -0,0 +1,125 @@
+import { parseRetryAfter } from "./errors.js";
+
+/**
+ * Shared streaming client for OpenAI-compatible /chat/completions endpoints.
+ *
+ * Provider plugins (openai, openrouter, ollama) construct the request body
+ * and headers; this module handles the SSE parsing, accumulates deltas into
+ * a non-streaming-shape response, and surfaces errors with the same ergonomics
+ * as the previous fetch-then-json pattern.
+ *
+ * Streaming is preferred over non-streaming for two reasons:
+ *
+ *   1. Long-running completions through CDN proxies (e.g. Cloudflare's 100s
+ *      origin-timeout) can't survive a non-streaming hold; streaming keeps
+ *      the connection alive byte-by-byte.
+ *   2. Future UI surfaces ("thinking" displays) want the deltas live; a
+ *      streaming-first plugin layer gives them a hook.
+ *
+ * The xAI Responses API (`/v1/responses`) uses a different streaming format
+ * and is out of scope for this client.
+ */
+
+/**
+ * @param {Object} args
+ * @param {string} args.url            Full POST URL (e.g. `${baseUrl}/v1/chat/completions`).
+ * @param {Object} args.headers        Plugin-specific headers (Authorization, etc.).
+ * @param {Object} args.body           Request body (without `stream` — added here).
+ * @param {AbortSignal} [args.signal]  Cancellation signal.
+ * @returns {Promise<Object>}          Non-streaming-shape response: `{ choices, usage, model }`.
+ *                                     Throws on non-2xx with `err.status` and `err.body` populated.
+ */
+export async function chatCompletionStream({ url, headers, body, signal }) {
+	const requestBody = {
+		...body,
+		stream: true,
+		// Tells OpenAI / OpenAI-compatible servers to emit a final usage chunk.
+		stream_options: { include_usage: true },
+	};
+
+	const response = await fetch(url, {
+		method: "POST",
+		headers: { "Content-Type": "application/json", ...headers },
+		body: JSON.stringify(requestBody),
+		signal,
+	});
+
+	if (!response.ok) {
+		const errorBody = await response.text();
+		const err = new Error(`${response.status} - ${errorBody}`);
+		err.status = response.status;
+		err.body = errorBody;
+		err.retryAfter = parseRetryAfter(response.headers.get("retry-after"));
+		throw err;
+	}
+
+	const reader = response.body.getReader();
+	const decoder = new TextDecoder();
+
+	let buffer = "";
+	let content = "";
+	let reasoningContent = "";
+	let usage = null;
+	let model = null;
+	let finishReason = null;
+
+	while (true) {
+		const { done, value } = await reader.read();
+		if (done) break;
+		buffer += decoder.decode(value, { stream: true });
+
+		// SSE frames are separated by blank lines; within a frame, a `data:`
+		// line carries the JSON payload. Process complete lines and keep any
+		// trailing partial-line in the buffer for the next read.
+		const lines = buffer.split("\n");
+		buffer = lines.pop();
+
+		for (const rawLine of lines) {
+			const line = rawLine.trim();
+			if (!line.startsWith("data:")) continue;
+			const payload = line.slice(5).trimStart();
+			if (payload === "[DONE]" || payload === "") continue;
+
+			let chunk;
+			try {
+				chunk = JSON.parse(payload);
+			} catch {
+				continue;
+			}
+
+			if (chunk.model) model = chunk.model;
+			if (chunk.usage) usage = chunk.usage;
+
+			const choice = chunk.choices?.[0];
+			if (!choice) continue;
+			if (choice.finish_reason) finishReason = choice.finish_reason;
+
+			const delta = choice.delta;
+			if (!delta) continue;
+			if (typeof delta.content === "string") content += delta.content;
+			// Different providers surface reasoning under different field names.
+			// Concatenate any that show up.
+			if (typeof delta.reasoning_content === "string")
+				reasoningContent += delta.reasoning_content;
+			if (typeof delta.reasoning === "string")
+				reasoningContent += delta.reasoning;
+			if (typeof delta.thinking === "string")
+				reasoningContent += delta.thinking;
+		}
+	}
+
+	return {
+		model,
+		choices: [
+			{
+				message: {
+					role: "assistant",
+					content,
+					reasoning_content: reasoningContent,
+				},
+				finish_reason: finishReason,
+			},
+		],
+		usage,
+	};
+}

package/src/llm/retry.js

@@ -0,0 +1,109 @@
+// Time-bounded exponential backoff with full jitter; mid-sleep AbortSignal-aware.
+export async function retryWithBackoff(
+	fn,
+	{
+		signal,
+		deadlineMs,
+		baseDelayMs = 1000,
+		maxDelayMs = 30_000,
+		isRetryable,
+		onRetry,
+	} = {},
+) {
+	const startTime = Date.now();
+	let attempt = 0;
+	while (true) {
+		signal?.throwIfAborted();
+		try {
+			return await fn();
+		} catch (err) {
+			if (!isRetryable(err)) throw err;
+			const elapsedMs = Date.now() - startTime;
+			const remainingMs = deadlineMs - elapsedMs;
+			if (remainingMs <= 0) {
+				throw new Error(
+					`transient failures persisted ${Math.round(elapsedMs / 1000)}s past deadline; last error: ${err.message}`,
+					{ cause: err },
+				);
+			}
+			const expCap = Math.min(maxDelayMs, baseDelayMs * 2 ** attempt);
+			const jittered = Math.floor(Math.random() * expCap);
+			const delayMs = Math.min(remainingMs, jittered);
+			onRetry?.(err, attempt + 1, delayMs, remainingMs);
+			await sleep(delayMs, signal);
+			attempt++;
+		}
+	}
+}
+
+// Per-category retry. Each category gets its own deadline budget; a
+// category transition resets prior category state — the rationale being
+// that seeing a different category proves upstream is alive in some way,
+// so prior gateway/server storms aren't relevant to the new attempt.
+// Honors err.retryAfter (seconds) as a delay floor for rate-limit hints.
+export async function retryClassified(
+	fn,
+	{ signal, classify, policies, onRetry } = {},
+) {
+	const state = new Map(); // category → { start: ms, attempts: number }
+	let lastCategory = null;
+
+	while (true) {
+		signal?.throwIfAborted();
+		try {
+			return await fn();
+		} catch (err) {
+			const category = classify(err);
+			if (!category) throw err;
+			const policy = policies[category];
+			if (!policy) {
+				throw new Error(
+					`retryClassified: no policy for category "${category}"`,
+					{ cause: err },
+				);
+			}
+
+			if (lastCategory !== category) state.clear();
+			if (!state.has(category)) {
+				state.set(category, { start: Date.now(), attempts: 0 });
+			}
+			lastCategory = category;
+
+			const s = state.get(category);
+			const elapsedMs = Date.now() - s.start;
+			const remainingMs = policy.deadlineMs - elapsedMs;
+			if (remainingMs <= 0) {
+				throw new Error(
+					`${category} retry exhausted after ${Math.round(elapsedMs / 1000)}s; last error: ${err.message}`,
+					{ cause: err },
+				);
+			}
+
+			const expCap = Math.min(
+				policy.maxDelayMs,
+				policy.baseDelayMs * 2 ** s.attempts,
+			);
+			const jittered = Math.floor(Math.random() * expCap);
+			const delayMs =
+				err.retryAfter !== undefined
+					? Math.min(remainingMs, Math.max(err.retryAfter * 1000, jittered))
+					: Math.min(remainingMs, jittered);
+
+			onRetry?.(err, category, s.attempts + 1, delayMs, remainingMs);
+			await sleep(delayMs, signal);
+			s.attempts++;
+		}
+	}
+}
+
+function sleep(ms, signal) {
+	return new Promise((resolve, reject) => {
+		const t = setTimeout(resolve, ms);
+		if (!signal) return;
+		const onAbort = () => {
+			clearTimeout(t);
+			reject(signal.reason || new Error("aborted"));
+		};
+		signal.addEventListener("abort", onAbort, { once: true });
+	});
+}

package/src/plugins/budget/budget.js

@@ -2,15 +2,7 @@ import { ceiling, computeBudget, measureMessages } from "../../agent/budget.js";
 import materializeContext from "../../agent/materializeContext.js";
 import { countTokens } from "../../agent/tokens.js";
 
-/**
- * Delta-from-actual baseline. The pre-call <prompt tokenUsage> reports
- * the prior turn's actual API prompt_tokens; post-dispatch predicts
- * next turn's packet = this turn's actual tokens + tokens of new rows
- * written this turn. Keeps the 413 body on the same scale as the
- * model's <prompt> arithmetic — a 60% divergence between pre-call
- * (actual) and post-check (conservative estimator) makes the model
- * dismiss the system as janky and stop following rules.
- */
+// Delta-from-actual; same scale as <prompt tokenUsage>. SPEC #budget_enforcement.
 function predictNextPacket(rows, currentTurn, baseline) {
 	let delta = 0;
 	for (const r of rows) {
@@ -19,13 +11,7 @@ function predictNextPacket(rows, currentTurn, baseline) {
 	return baseline + delta;
 }
 
-/**
- * Format the 413 error body. Names each demoted path with its turn
- * and token count so the model can avoid re-promoting them next turn.
- * Exported (not private) so unit tests can assert the exact wire
- * format — the model reads this string, so its shape is part of the
- * contract.
- */
+// 413 error body; wire format is part of the model contract.
 export function overflowBody(overflow, contextSize, demoted) {
 	const cap = ceiling(contextSize);
 	const size = cap + overflow;
@@ -50,72 +36,95 @@ export default class Budget {
 			enforce: this.enforce.bind(this),
 			postDispatch: this.postDispatch.bind(this),
 		};
-		core.filter("assembly.user", this.assembleBudget.bind(this), 275);
+		core.filter("assembly.user", this.assembleBudget.bind(this), 175);
 	}
 
-	/**
-	 * Render the <budget> table between <instructions> and <prompt>.
-	 * See SPEC @token_accounting for the contract: per-row tokens are
-	 * aTokens (the promotion premium = vTokens − sTokens), summarized
-	 * entries collapse into a single aggregate line, system overhead
-	 * (system prompt + tool defs) gets its own line.
-	 */
+	// Renders <budget> at priority 275; see SPEC #token_accounting.
 	assembleBudget(content, ctx) {
 		const { rows, contextSize, systemPrompt } = ctx;
 		if (!contextSize) return content;
 
 		const cap = ceiling(contextSize);
 
-		const visibleByScheme = new Map();
+		const byScheme = new Map();
 		let visibleCount = 0;
 		let premiumTokens = 0;
 		let summarizedCount = 0;
-		let summarizedTokens = 0;
+		let _summarizedTokens = 0;
 		let floorTokens = 0;
 
+		const schemeEntry = (s) => {
+			let e = byScheme.get(s);
+			if (!e) {
+				e = {
+					vis: 0,
+					sum: 0,
+					visTokens: 0, // current cost of visible entries
+					visIfSumTokens: 0, // sTokens of visible (what they'd cost demoted)
+					sumTokens: 0, // current cost of summarized entries
+					premium: 0, // savings from demoting visible → summarized
+				};
+				byScheme.set(s, e);
+			}
+			return e;
+		};
+
 		for (const r of rows) {
 			if (r.aTokens == null) continue;
 			const s = r.scheme || "file";
+			const entry = schemeEntry(s);
 			if (r.visibility === "visible") {
-				const entry = visibleByScheme.get(s) ?? { count: 0, tokens: 0 };
-				entry.count += 1;
-				entry.tokens += r.aTokens;
-				visibleByScheme.set(s, entry);
+				entry.vis += 1;
+				entry.visTokens += r.vTokens;
+				entry.visIfSumTokens += r.sTokens;
+				entry.premium += r.aTokens;
 				visibleCount += 1;
 				premiumTokens += r.aTokens;
 				floorTokens += r.sTokens;
 			} else if (r.visibility === "summarized") {
+				entry.sum += 1;
+				entry.sumTokens += r.sTokens;
 				summarizedCount += 1;
-				summarizedTokens += r.sTokens;
+				_summarizedTokens += r.sTokens;
 				floorTokens += r.sTokens;
 			}
 		}
 
-		const systemTokens = countTokens(systemPrompt || "");
+		const systemTokens = countTokens(systemPrompt);
 		const tokenUsage = floorTokens + premiumTokens + systemTokens;
 		const tokensFree = Math.max(0, cap - tokenUsage);
 
-		const schemeRows = [...visibleByScheme.entries()]
-			.toSorted((a, b) => b[1].tokens - a[1].tokens)
-			.map(([scheme, v]) => {
-				const pct = Math.round((v.tokens / cap) * 100);
-				return `| ${scheme} | ${v.count} | ${v.tokens} | ${pct}% |`;
+		// Sort by current cost desc so biggest-impact rows are top.
+		const schemeRows = [...byScheme.entries()]
+			.toSorted(
+				([, a], [, b]) =>
+					b.visTokens + b.sumTokens - (a.visTokens + a.sumTokens),
+			)
+			.map(([scheme, e]) => {
+				const cost = e.visTokens + e.sumTokens;
+				const ifAllSum = e.visIfSumTokens + e.sumTokens;
+				return `| ${scheme} | ${e.vis} | ${e.sum} | ${cost} | ${ifAllSum} | ${e.premium} |`;
 			});
 
-		const summarizedPct = Math.round((summarizedTokens / cap) * 100);
-		const systemPct = Math.round((systemTokens / cap) * 100);
+		const systemPct =
+			tokenUsage > 0 ? Math.round((systemTokens / tokenUsage) * 100) : 0;
 
 		const table = [
-			"| scheme | visible | tokens | % |",
-			"|---|---|---|---|",
+			"| scheme | vis | sum | cost | if-all-sum | premium |",
+			"|---|---|---|---|---|---|",
 			...schemeRows,
 		].join("\n");
 
-		const summarizedLine = `Summarized: ${summarizedCount} entries, ${summarizedTokens} tokens (${summarizedPct}% of budget).`;
 		const systemLine = `System: ${systemTokens} tokens (${systemPct}% of budget).`;
 		const totalLine = `Total: ${visibleCount} visible + ${summarizedCount} summarized entries; tokenUsage ${tokenUsage} / ceiling ${cap}. ${tokensFree} tokens free.`;
+		const legend = [
+			"Columns:",
+			"- cost: current cost of this scheme (vTokens for visible + sTokens for summarized)",
+			"- if-all-sum: cost if every entry of this scheme were demoted to summarized",
+			"- premium: savings from demoting visible → summarized (cost − if-all-sum)",
+		].join("\n");
 
-		return `${content}<budget tokenUsage="${tokenUsage}" tokensFree="${tokensFree}">\n${table}\n\n${summarizedLine}\n${systemLine}\n${totalLine}\n</budget>\n`;
+		return `${content}<budget tokenUsage="${tokenUsage}" tokensFree="${tokensFree}">\n${table}\n\n${legend}\n${systemLine}\n${totalLine}\n</budget>\n`;
 	}
 
 	#check({ contextSize, messages, rows, lastPromptTokens = 0 }) {
@@ -151,16 +160,7 @@ export default class Budget {
 		});
 	}
 
-	/**
-	 * Pre-LLM budget enforcement. On first-turn overflow, demotes the
-	 * incoming prompt and re-materializes; re-checks and returns the
-	 * post-demotion result. If overflow persists after demotion (or on
-	 * later iterations), emits a 413 error (strike) and returns !ok so
-	 * TurnExecutor can skip the LLM call this turn.
-	 *
-	 * ctx = { runId, loopId, turn, systemPrompt, mode, toolSet, demoted,
-	 *         loopIteration }
-	 */
+	// Pre-LLM enforce: SPEC #budget_enforcement.
 	async enforce({
 		contextSize,
 		messages,
@@ -213,7 +213,6 @@ export default class Budget {
 			mode: ctx.mode,
 			toolSet: ctx.toolSet,
 			contextSize,
-			demoted: ctx.demoted,
 		});
 		const rechecked = this.#check({
 			contextSize,
@@ -234,14 +233,7 @@ export default class Budget {
 		return rechecked;
 	}
 
-	/**
-	 * Post-dispatch Turn Demotion. Re-materializes end-of-turn context and
-	 * checks against the ceiling. On overflow, demotes this turn's promoted
-	 * entries and emits a 413 error (strike) with the descriptive body so
-	 * the model sees it next turn via the unified error channel.
-	 *
-	 * ctx = { runId, loopId, turn, systemPrompt, mode, toolSet, demoted }
-	 */
+	// Post-dispatch Turn Demotion: SPEC #budget_enforcement.
 	async postDispatch({ contextSize, ctx, rummy }) {
 		if (!contextSize) return { failed: false };
 		const postMat = await materializeContext({
@@ -254,13 +246,7 @@ export default class Budget {
 			mode: ctx.mode,
 			toolSet: ctx.toolSet,
 			contextSize,
-			demoted: ctx.demoted,
 		});
-		// Baseline from this turn's actual API tokens (telemetry wrote it
-		// before post-dispatch runs). Delta from rows added this turn.
-		// Predicted next-turn packet stays on the tokenUsage scale the
-		// model can verify against its own arithmetic. materializeContext
-		// guarantees a number (0 when no prior API call exists).
 		const baseline = postMat.lastContextTokens;
 		const predicted = predictNextPacket(postMat.rows, ctx.turn, baseline);
 		const cap = ceiling(contextSize);
@@ -269,14 +255,7 @@ export default class Budget {
 
 		const store = rummy.entries;
 		let demotedEntries = await store.demoteTurnEntries(ctx.runId, ctx.turn);
-		// Fallback: if this turn had nothing to demote but the packet still
-		// overflows, the pressure is coming from prior-turn promotions the
-		// model never demoted itself. Widen to all currently-visible
-		// entries in the run. Without this fallback, overflow-with-nothing
-		// strikes out runs where the base context has drifted over ceiling
-		// through no fault of the current turn (observed: runs where 3
-		// stale promotions from turns 12–14 saturate every subsequent
-		// turn's budget).
+		// Prior-turn-pressure fallback; SPEC #budget_enforcement.
 		if (demotedEntries.length === 0) {
 			demotedEntries = await store.demoteRunVisibleEntries(ctx.runId);
 		}