@iowarp/clio-coder 0.4.9 → 0.5.0

package/src/engine/acp/tool-mediator.ts

@@ -92,6 +92,19 @@ const SINGLE_MUTATION_PATH_KEYS = [
 	"uri",
 ] as const;
 
+/**
+ * A read or a search also names the directory it reaches under these keys,
+ * and that directory is judged like any other path target.
+ */
+const READ_PATH_KEYS = [
+	...SINGLE_MUTATION_PATH_KEYS,
+	"directory",
+	"dir",
+	"dirPath",
+	"dir_path",
+	"folder",
+	"root",
+] as const;
 const MULTI_MUTATION_PATH_KEYS = ["paths", "files", "filePaths", "file_paths", "targets", "targetPaths"] as const;
 const MOVE_SOURCE_PATH_KEYS = [
 	"source",
@@ -415,7 +428,7 @@ function mapPathBearingToolCall(
 	targetRequired: boolean,
 ): MappedToolCall {
 	const rawInput = isRecord(toolCall?.rawInput) ? toolCall.rawInput : {};
-	const raw = extractPathFields(rawInput, SINGLE_MUTATION_PATH_KEYS, MULTI_MUTATION_PATH_KEYS, cwd, "rawInput");
+	const raw = extractPathFields(rawInput, READ_PATH_KEYS, MULTI_MUTATION_PATH_KEYS, cwd, "rawInput");
 	const locations = extractLocationPaths(toolCall?.locations, cwd);
 	const errors = [...raw.errors, ...locations.errors];
 	if (toolCall?.rawInput !== undefined && !isRecord(toolCall.rawInput)) errors.push("rawInput must be an object");
@@ -512,10 +525,10 @@ function mapToolCall(toolCall: AcpToolCallUpdate | undefined, cwd: string): Mapp
 		const args = commandArgs(rawInput, cwd);
 		return { tool: ToolNames.Bash, args, evaluations: [{ tool: ToolNames.Bash, args }], known: true, displayTool };
 	}
-	const inferredPattern = stringField(rawInput, "pattern");
-	if (inferredPattern) {
-		const args = { pattern: inferredPattern };
-		return { tool: ToolNames.Grep, args, evaluations: [{ tool: ToolNames.Grep, args }], known: true, displayTool };
+	// A search names where it looks as well as what it looks for, so the
+	// inferred grep keeps its path targets for the policy to judge.
+	if (stringField(rawInput, "pattern")) {
+		return mapPathBearingToolCall(toolCall, ToolNames.Grep, displayTool, cwd, false);
 	}
 	const inferredPath = stringField(rawInput, "file_path", "filePath", "path", "notebook_path");
 	if (inferredPath) {
@@ -621,6 +634,7 @@ export class AcpToolMediator {
 					candidate,
 					disposition: mapAutonomy(level, candidate.classification.actionClass, {
 						executeRecognized: candidate.policy?.execRecognition !== "unrecognized",
+						...(candidate.policy?.readScope === "outside-workspace" ? { readOutsideWorkspace: true } : {}),
 					}),
 				}));
 				const deniedDisposition = dispositions.find((candidate) => candidate.disposition === "deny");
@@ -628,11 +642,20 @@ export class AcpToolMediator {
 				if (deniedDisposition !== undefined) {
 					safetyDecision = deniedDisposition.candidate;
 					decision = "denied";
-					reason = autonomyDenyRejection(level, mapped.tool, deniedDisposition.candidate.classification.actionClass).short;
+					reason = autonomyDenyRejection(
+						level,
+						mapped.tool,
+						deniedDisposition.candidate.classification.actionClass,
+						deniedDisposition.candidate.policy?.readScope === "outside-workspace",
+					).short;
 				} else if (askDisposition !== undefined) {
 					safetyDecision = askDisposition.candidate;
 					decision = "denied";
-					reason = `permission_required: autonomy ${level} requires approval for ${askDisposition.candidate.classification.actionClass}; denied by non-stall policy (no interactive operator in delegation context)`;
+					const asked =
+						askDisposition.candidate.policy?.readScope === "outside-workspace"
+							? "a path outside the workspace"
+							: askDisposition.candidate.classification.actionClass;
+					reason = `permission_required: autonomy ${level} requires approval for ${asked}; denied by non-stall policy (no interactive operator in delegation context)`;
 				} else {
 					decision = "approved";
 					reason = safetyDecision?.policy?.reasonCode ?? "allowed";

package/src/engine/acp/types.ts

@@ -15,6 +15,30 @@ export const ACP_SESSION_META_KEY = "clio-coder/session";
  * client that does not read it sees exactly the frames it saw before.
  */
 export const ACP_AGENT_META_KEY = "clio-coder/agent";
+/**
+ * Opt-in key for non-terminal `tool_call_update` frames carrying a running
+ * tool's cumulative output. It is an opt-in and not a default because a client
+ * that renders every frame it receives without collapsing repeated updates for
+ * one call would redraw the whole tool segment several times a second.
+ */
+export const ACP_TOOL_PROGRESS_META_KEY = "clio-coder/toolProgress";
+/**
+ * Decision facts attached to `session/request_permission`. The server already
+ * classifies the ask to pick the option labels; without this the tier, the
+ * consequence, and the reversibility it computed are discarded and a client is
+ * left re-deriving them from a tool name.
+ */
+export const ACP_DECISION_META_KEY = "clio-coder/decision";
+/**
+ * Operator-command catalog and invocation. The names live here, in the leaf
+ * that holds every other wire constant, rather than beside the projection in
+ * `./commands.js`: that module value-imports the whole slash registry, and the
+ * ACP server may not drag the registry's closure into the Stage 0 chunk budget
+ * just to know what a method is called (rule6, tests/boundaries).
+ */
+export const ACP_COMMANDS_META_KEY = "clio-coder/commands";
+export const ACP_COMMANDS_LIST_METHOD = "clio-coder/commands/list";
+export const ACP_COMMANDS_INVOKE_METHOD = "clio-coder/commands/invoke";
 
 /**
  * Wire bounds for one prompt turn (CONTRACT C001 §3). A client renders every
@@ -31,6 +55,40 @@ export const ACP_MAX_STRING_BYTES = 4096;
 export const ACP_MAX_RAW_RECORD_BYTES = 32768;
 export const ACP_MAX_CHUNK_BYTES = 16384;
 
+/**
+ * The one raw-record string the generic 4 KiB cap actively harmed. `edit` and
+ * `write` put a rendered unified diff on `details.diff`, already capped by the
+ * engine's own `MAX_DIFF_BYTES` (32 KiB, `src/tools/edit-diff.ts`). At four
+ * context lines and roughly 60 bytes a line, 4 KiB is about 65 diff lines, so a
+ * multi-hunk refactor arrived cut mid-hunk and a client could not tell a
+ * truncated hunk from a hunk that ended there.
+ *
+ * It sits below {@link ACP_MAX_RAW_RECORD_BYTES} rather than at it because the
+ * record cap elides the WHOLE record when it trips: a diff sized at exactly the
+ * record bound would serialize past it once JSON escaping and the rest of the
+ * result are counted, and the client would have received `{truncated:true}`
+ * instead of the 4 KiB it used to get. The reserve is one
+ * {@link ACP_MAX_STRING_BYTES} budget for everything else in the record.
+ */
+export const ACP_MAX_RAW_DIFF_BYTES = ACP_MAX_RAW_RECORD_BYTES - ACP_MAX_STRING_BYTES;
+
+/**
+ * Per-call ceiling on non-terminal tool-progress frames. The tool's own
+ * throttle bounds the rate, not the total: a build that prints for ten minutes
+ * is a bounded stream of unbounded length, and a client holding per-call state
+ * has to be told where the stream stops.
+ */
+export const ACP_MAX_TOOL_PROGRESS_FRAMES_PER_CALL = 64;
+
+/**
+ * Floor between two progress frames for one call, on top of the 100 ms throttle
+ * `src/core/bash-exec.ts` already applies upstream. The upstream throttle is a
+ * rendering budget for a terminal that redraws in place; a JSON-RPC peer pays
+ * serialization and a full segment re-render per frame, so the wire gets its
+ * own, slower floor.
+ */
+export const ACP_MIN_TOOL_PROGRESS_INTERVAL_MS = 250;
+
 /**
  * Upper bound on a `toolCallId` this server puts on the wire, in UTF-8 bytes
  * (CONTRACT C001 §3). Identity is one call per id in both directions: an engine

package/src/engine/agent.ts

@@ -1,3 +1,5 @@
+import { createInitialSystemMessage, toToolDeclaration } from "@earendil-works/pi-ai";
+import { resolvedRequestContext } from "./context.js";
 /**
  * Thin wrapper over Clio's engine Agent class.
  *
@@ -6,16 +8,22 @@
  * from options.initialState on instantiation.
  *
  * pi-agent-core 0.84 requires an explicit stream function on every agent.
- * Clio's default is the compat `streamSimple` dispatcher (the same function the
- * pre-0.81 engine used implicitly); callers may override it for tests.
+ * Clio's default dispatcher preserves Pi's transcript. Production adapters can
+ * wrap that path with transcriptStreamFn; legacy custom/test delegates receive
+ * Pi's resolved prompt/tool request view through streamFn.
  */
 
 import { Agent, type AgentOptions, type StreamFn } from "@earendil-works/pi-agent-core";
 import { isDispositionedToolResultError } from "../tools/result-disposition.js";
 import { engineStreamSimple } from "./api-registry.js";
 
+export type EngineStreamFn = (...args: Parameters<typeof engineStreamSimple>) => ReturnType<StreamFn>;
+
 export type EngineAgentOptions = Omit<AgentOptions, "streamFn"> & {
-	streamFn?: StreamFn;
+	/** Legacy/custom request view. Explicit overrides also replace the native hook in tests. */
+	streamFn?: EngineStreamFn;
+	/** Native Pi transcript hook for production request adapters. */
+	transcriptStreamFn?: StreamFn;
 	/** Called immediately before each native stream delegate invocation. */
 	onStreamInvocation?: () => void;
 };
@@ -38,19 +46,56 @@ function dispositionAwareAfterToolCall(
 }
 
 export function createEngineAgent(options: EngineAgentOptions = {}): EngineAgentHandle {
-	const { streamFn = engineStreamSimple, onStreamInvocation, ...agentOptions } = options;
+	const { streamFn, transcriptStreamFn = engineStreamSimple, onStreamInvocation, ...agentOptions } = options;
 	const agent = new Agent({
 		...agentOptions,
-		streamFn: onStreamInvocation
-			? (...args) => {
-					onStreamInvocation();
-					return streamFn(...args);
-				}
-			: streamFn,
+		streamFn: (model, context, streamOptions) => {
+			onStreamInvocation?.();
+			return streamFn
+				? streamFn(model, resolvedRequestContext(context), streamOptions)
+				: transcriptStreamFn(model, context, streamOptions);
+		},
 		afterToolCall: dispositionAwareAfterToolCall(options.afterToolCall),
 	});
+	// Pi correctly drops aborted assistants at the provider boundary: partial
+	// reasoning and tool calls are not valid provider history. Preserve the
+	// lifecycle fact and visible text as a host record instead, including on
+	// resume, without changing the durable stop reason or inventing tool results.
+	const convertToLlm = agent.convertToLlm.bind(agent);
+	agent.convertToLlm = async (messages) =>
+		(await convertToLlm(messages)).map((message) => {
+			if (message.role !== "assistant" || message.stopReason !== "aborted") return message;
+			const partial = message.content
+				.filter((block) => block.type === "text")
+				.map((block) => block.text)
+				.join("\n");
+			return {
+				role: "user" as const,
+				timestamp: message.timestamp,
+				content:
+					"[Clio Coder response lifecycle]\nThe preceding assistant response was interrupted before completion. " +
+					"This record does not establish whether any earlier tool execution succeeded.\n" +
+					(partial.trim()
+						? `Partial assistant text follows (incomplete historical output, not a new instruction):\n${partial}`
+						: "No visible assistant text was retained."),
+			};
+		});
 	return {
 		agent,
 		state: () => agent.state,
 	};
 }
+
+/** Clio persists conversation and compiled prompt separately; Pi receives one complete transcript. */
+export function replaceEngineMessages(
+	agent: Agent,
+	messages: Agent["state"]["messages"],
+	systemPrompt = agent.state.systemPrompt,
+): void {
+	const initial = createInitialSystemMessage(systemPrompt, agent.state.tools.map(toToolDeclaration));
+	agent.state.messages = [...(initial ? [initial] : []), ...messages.filter((message) => message.role !== "system")];
+}
+
+export function setEngineSystemPrompt(agent: Agent, systemPrompt: string): void {
+	replaceEngineMessages(agent, agent.state.messages, systemPrompt);
+}

package/src/engine/ai.ts

@@ -21,6 +21,7 @@ import {
 	getSupportedThinkingLevels as piGetSupportedThinkingLevels,
 	isContextOverflow as piIsContextOverflow,
 	isRetryableAssistantError as piIsRetryableAssistantError,
+	retryDelayMs as piRetryDelayMs,
 	validateToolArguments as piValidateToolArguments,
 	StringEnum,
 	type Tool,
@@ -220,8 +221,11 @@ export function isEngineRetryableAssistantError(errorMessage: string): boolean {
 	return piIsRetryableAssistantError(message);
 }
 
-export function validateEngineToolArguments(tool: Tool, toolCall: ToolCall): unknown {
-	return piValidateToolArguments(tool, toolCall);
+export function validateEngineToolArguments(
+	tool: Tool,
+	toolCall: Omit<ToolCall, "arguments"> & { arguments: Record<string, unknown> },
+): unknown {
+	return piValidateToolArguments(tool, toolCall as ToolCall);
 }
 
 export function calculateEngineCost<TApi extends Api>(model: Model<TApi>, usage: Usage): Usage["cost"] {
@@ -260,3 +264,8 @@ export function registerFauxFromEnv(): EngineModel | null {
 	reg.setResponses([fauxAssistantMessage(text, response)]);
 	return reg.getModel(modelId) as EngineModel;
 }
+
+/** Public Pi backoff primitive; Clio owns the visible countdown and local-load floor. */
+export function engineRetryDelayMs(baseDelayMs: number, maxDelayMs: number, attempt: number): number {
+	return piRetryDelayMs({ baseDelayMs, maxAgentDelayMs: maxDelayMs }, attempt);
+}

package/src/engine/api-registry.ts

@@ -25,6 +25,7 @@ import {
 } from "@earendil-works/pi-ai/providers/faux";
 import "@earendil-works/pi-ai/providers/images/register-builtins";
 
+import { normalizeContext } from "./context.js";
 import { getEngineEnvApiKey } from "./env-api-keys.js";
 import { engineModels } from "./models.js";
 import { instrumentProviderCall } from "./provider-diagnostics.js";
@@ -147,9 +148,9 @@ function dispatchEngineStream(
 		if (model.provider.startsWith("cloudflare-") && !hasCloudflareAuth(options)) {
 			return engineModels.stream(model, context, options);
 		}
-		return provider.stream(model, context, withEnvApiKey(model, options));
+		return provider.stream(model, normalizeContext(context), withEnvApiKey(model, options));
 	}
-	return resolved(model.api).stream(model, context, withEnvApiKey(model, options));
+	return resolved(model.api).stream(model, normalizeContext(context), withEnvApiKey(model, options));
 }
 
 function dispatchEngineStreamSimple(
@@ -164,9 +165,9 @@ function dispatchEngineStreamSimple(
 		if (model.provider.startsWith("cloudflare-") && !hasCloudflareAuth(options)) {
 			return engineModels.streamSimple(model, context, options);
 		}
-		return provider.streamSimple(model, context, withEnvApiKey(model, options));
+		return provider.streamSimple(model, normalizeContext(context), withEnvApiKey(model, options));
 	}
-	return resolved(model.api).streamSimple(model, context, withEnvApiKey(model, options));
+	return resolved(model.api).streamSimple(model, normalizeContext(context), withEnvApiKey(model, options));
 }
 
 export function engineStream(

package/src/engine/apis/degraded-inference.ts

@@ -0,0 +1,243 @@
+/**
+ * Degraded-inference watchdog for local runtimes.
+ *
+ * A local server whose weights or KV cache spilled to CPU does not fail: it
+ * answers, at roughly a hundredth of the speed, and the operator sees only a
+ * spinner. The observed case was 2m18s of streaming for 25 tokens with no
+ * indication that anything was wrong. This watchdog turns that silence into a
+ * notice: once a prediction has run past a grace period, a sustained token rate
+ * below the floor is reported once per turn, along with what is resident on the
+ * target so the cause is visible in the same line.
+ *
+ * The watchdog observes; it never cancels a turn. A slow model is still a
+ * working model, and the operator decides whether to abort.
+ */
+
+import {
+	type AssistantMessage,
+	type AssistantMessageEvent,
+	type AssistantMessageEventStream,
+	createAssistantMessageEventStream,
+} from "@earendil-works/pi-ai";
+import { rawDurationMs } from "../../core/timers.js";
+import { ceilChars } from "../../domains/session/context-accounting.js";
+import { declareRuntimeNoticeProducer } from "./residency.js";
+import type { ResidentModelInfo } from "./resident-models.js";
+
+/** Time a prediction gets before its rate is judged. */
+export const DEGRADED_GRACE_MS = 30_000;
+/** Sustained output tokens per second below which inference is reported as degraded. */
+export const DEGRADED_FLOOR_TOKENS_PER_SECOND = 2;
+
+const emitDegradedNotice = declareRuntimeNoticeProducer("degraded-inference", ["degraded"]);
+
+export interface DegradedInferenceReport {
+	elapsedMs: number;
+	tokens: number;
+	tokensPerSecond: number;
+}
+
+export interface DegradedInferenceTiming {
+	graceMs?: number;
+	floorTokensPerSecond?: number;
+	/** Sampling interval; the watchdog checks the rate this often. */
+	pollMs?: number;
+	/**
+	 * Monotonic clock for the turn's span, defaulting to `performance.now`. A
+	 * stepped wall clock would fake a collapse on a forward step and hide one on
+	 * a backward step.
+	 */
+	monotonicNow?: () => number;
+	setTimer?: (fn: () => void, ms: number) => { cancel: () => void };
+}
+
+export interface DegradedInferenceOptions extends DegradedInferenceTiming {
+	/** Called at most once, when the observed rate stays under the floor past the grace period. */
+	onDegraded: (report: DegradedInferenceReport) => void;
+}
+
+export interface DegradedInferenceWatchdog {
+	/** Record generated tokens (output plus reasoning; both are work the server did). */
+	addTokens: (count: number) => void;
+	stop: () => void;
+}
+
+function defaultTimer(fn: () => void, ms: number): { cancel: () => void } {
+	const handle = setInterval(fn, ms);
+	handle.unref?.();
+	return { cancel: () => clearInterval(handle) };
+}
+
+/**
+ * Start watching a prediction's token rate. Call {@link
+ * DegradedInferenceWatchdog.addTokens} as fragments arrive and `stop()` when the
+ * turn ends, in a `finally`, so a failed turn never leaves a timer behind.
+ */
+function startDegradedInferenceWatchdog(options: DegradedInferenceOptions): DegradedInferenceWatchdog {
+	const monotonicNow = options.monotonicNow ?? (() => performance.now());
+	const graceMs = options.graceMs ?? DEGRADED_GRACE_MS;
+	const floor = options.floorTokensPerSecond ?? DEGRADED_FLOOR_TOKENS_PER_SECOND;
+	const pollMs = options.pollMs ?? 5_000;
+	const startedAt = monotonicNow();
+	let tokens = 0;
+	let reported = false;
+	let stopped = false;
+
+	const check = (): void => {
+		if (reported || stopped) return;
+		const elapsedMs = rawDurationMs(startedAt, monotonicNow());
+		if (elapsedMs < graceMs) return;
+		const tokensPerSecond = tokens / (elapsedMs / 1000);
+		if (tokensPerSecond >= floor) return;
+		reported = true;
+		timer.cancel();
+		try {
+			options.onDegraded({ elapsedMs, tokens, tokensPerSecond });
+		} catch {
+			// A notice sink failure must never escape into the turn it describes.
+		}
+	};
+
+	const timer = (options.setTimer ?? defaultTimer)(check, pollMs);
+
+	return {
+		addTokens: (count) => {
+			if (Number.isFinite(count) && count > 0) tokens += count;
+		},
+		stop: () => {
+			if (stopped) return;
+			stopped = true;
+			timer.cancel();
+		},
+	};
+}
+
+export interface WatchDegradedInferenceOptions {
+	targetId: string;
+	runtimeId: string;
+	model: string;
+	signal?: AbortSignal;
+	/** Lists what is resident on the target when the notice fires. */
+	listResident?: () => Promise<ResidentModelInfo[]>;
+	timing?: DegradedInferenceTiming;
+}
+
+function describeResident(entry: ResidentModelInfo): string {
+	const { sizeBytes, sizeVramBytes } = entry;
+	if (sizeBytes === undefined || sizeVramBytes === undefined || sizeBytes <= 0 || sizeVramBytes >= sizeBytes) {
+		return entry.modelId;
+	}
+	return `${entry.modelId} (${Math.round((sizeVramBytes / sizeBytes) * 100)}% on GPU)`;
+}
+
+// A server crawling on CPU can answer its listing slowly too; the notice must
+// not wait on it indefinitely.
+const RESIDENT_LISTING_TIMEOUT_MS = 3_000;
+
+async function residentSummary(options: WatchDegradedInferenceOptions): Promise<string | undefined> {
+	const listResident = options.listResident;
+	if (!listResident) return undefined;
+	let timeout: ReturnType<typeof setTimeout> | undefined;
+	try {
+		const resident = await Promise.race([
+			listResident(),
+			new Promise<never>((_, reject) => {
+				timeout = setTimeout(() => reject(new Error("resident listing timed out")), RESIDENT_LISTING_TIMEOUT_MS);
+				timeout.unref?.();
+			}),
+		]);
+		return resident.length > 0 ? resident.map(describeResident).join(", ") : "none reported";
+	} catch {
+		return undefined;
+	} finally {
+		if (timeout !== undefined) clearTimeout(timeout);
+	}
+}
+
+function generatedChars(event: AssistantMessageEvent): number {
+	if (event.type === "text_delta" || event.type === "thinking_delta" || event.type === "toolcall_delta") {
+		return event.delta.length;
+	}
+	return 0;
+}
+
+const runningWatchdogs = new Set<DegradedInferenceWatchdog>();
+
+/** How many watchdogs currently hold a poll timer. Every finished turn returns this to zero. */
+export function runningDegradedInferenceWatchdogs(): number {
+	return runningWatchdogs.size;
+}
+
+/**
+ * The event stream a local runtime pushes into, watched for its token rate as
+ * events go through. The producer keeps its own loop and its own `try/catch`,
+ * so a failure mid-stream reaches the consumer exactly as it did unwatched;
+ * there is no second loop between them that could swallow a throw.
+ *
+ * The clock starts at the `start` event, which both local paths push once the
+ * server has answered the request, so a model load is never judged as slow
+ * generation. Text, reasoning, and tool-call fragments all count. The watch
+ * ends at `done`, `error`, `end()`, or abort, and an abort also suppresses a
+ * notice still waiting on its resident listing.
+ */
+export function createDegradedInferenceStream(options: WatchDegradedInferenceOptions): AssistantMessageEventStream {
+	const stream = createAssistantMessageEventStream();
+	const push = stream.push.bind(stream);
+	const end = stream.end.bind(stream);
+	let watchdog: DegradedInferenceWatchdog | null = null;
+	let stopped = false;
+	let generated = 0;
+	const aborted = (): boolean => options.signal?.aborted === true;
+	const stop = (): void => {
+		stopped = true;
+		options.signal?.removeEventListener("abort", stop);
+		if (watchdog === null) return;
+		watchdog.stop();
+		runningWatchdogs.delete(watchdog);
+	};
+	options.signal?.addEventListener("abort", stop, { once: true });
+	const onDegraded = (report: DegradedInferenceReport): void => {
+		void residentSummary(options).then((residents) => {
+			if (stopped || aborted()) return;
+			const seconds = Math.round(report.elapsedMs / 1000);
+			const rate = report.tokensPerSecond.toFixed(2);
+			const residentLine =
+				residents === undefined
+					? "The resident models on the target could not be listed."
+					: `Resident there: ${residents}.`;
+			emitDegradedNotice({
+				kind: "degraded",
+				level: "warning",
+				targetId: options.targetId,
+				runtimeId: options.runtimeId,
+				model: options.model,
+				message: `'${options.model}' on '${options.targetId}' has generated ${report.tokens} tokens in ${seconds}s (${rate} tok/s); inference is running far below GPU speed, which is what a spill to CPU looks like. ${residentLine} Unload a co-resident model or lower the context window.`,
+				detail: {
+					tokens: report.tokens,
+					elapsedMs: report.elapsedMs,
+					tokensPerSecond: Number(rate),
+					...(residents !== undefined ? { residents } : {}),
+				},
+			});
+		});
+	};
+	stream.push = (event: AssistantMessageEvent): void => {
+		if (event.type === "start" && watchdog === null && !stopped && !aborted()) {
+			watchdog = startDegradedInferenceWatchdog({ ...options.timing, onDegraded });
+			runningWatchdogs.add(watchdog);
+		}
+		const chars = generatedChars(event);
+		if (chars > 0) {
+			const before = ceilChars(generated);
+			generated += chars;
+			watchdog?.addTokens(ceilChars(generated) - before);
+		}
+		if (event.type === "done" || event.type === "error") stop();
+		push(event);
+	};
+	stream.end = (result?: AssistantMessage): void => {
+		stop();
+		end(result);
+	};
+	return stream;
+}

package/src/engine/apis/llamacpp-residency.ts

@@ -1,12 +1,13 @@
 import { performance } from "node:perf_hooks";
+import { setTimeout as sleep } from "node:timers/promises";
 import { residencyTargetKey } from "../../core/residency-target-key.js";
-import { sleep } from "../../core/timers.js";
 import {
 	type ResidencyAdapter,
 	ResidencyPreconditionError,
 	reconcileResidency,
 	residentTagProtected,
 } from "./residency.js";
+import type { ResidentModelInfo } from "./resident-models.js";
 
 /**
  * llama.cpp router adapter for the shared residency reconciler
@@ -30,6 +31,8 @@ export interface LlamaCppResidencyInput {
 	 * to managed for direct adapter callers.
 	 */
 	managed?: boolean;
+	signal?: AbortSignal;
+	headers?: Record<string, string>;
 	fetchImpl?: typeof fetch;
 	now?: () => number;
 	ttlMs?: number;
@@ -42,6 +45,17 @@ export interface LlamaCppResidencyInput {
 const LOAD_TIMEOUT_MS = 120_000;
 const POLL_INTERVAL_MS = 500;
 
+function routerFetch(input: Pick<LlamaCppResidencyInput, "fetchImpl" | "headers" | "signal">): typeof fetch {
+	return (url, init) => {
+		const headers = new Headers(input.headers);
+		new Headers(init?.headers).forEach((value, name) => {
+			headers.set(name, value);
+		});
+		const signal = input.signal ? AbortSignal.any([input.signal, ...(init?.signal ? [init.signal] : [])]) : init?.signal;
+		return (input.fetchImpl ?? fetch)(url, { ...init, headers, ...(signal ? { signal } : {}) });
+	};
+}
+
 /**
  * `sleeping` is a resident state, not an unloaded one. A router started with
  * `--sleep-idle-seconds` parks an idle model's compute while it keeps the slot
@@ -136,12 +150,13 @@ async function fetchRouterProps(input: LlamaCppResidencyInput, fetchImpl: typeof
 		if (!response.ok) return {};
 		return parseLlamaCppRouterProps(await response.json());
 	} catch {
+		input.signal?.throwIfAborted();
 		return {};
 	}
 }
 
 async function fetchRouterModels(
-	input: LlamaCppResidencyInput,
+	input: Pick<LlamaCppResidencyInput, "baseUrl" | "timeoutMs">,
 	fetchImpl: typeof fetch,
 ): Promise<LlamaCppRouterModel[]> {
 	const response = await fetchImpl(modelsUrl(input.baseUrl), {
@@ -223,7 +238,7 @@ async function waitForLoaded(input: LlamaCppResidencyInput, fetchImpl: typeof fe
 		// the whole load timeout on an idle router.
 		if (model?.state === "loaded" || model?.state === "sleeping") return;
 		if (model?.state === "failed") throw new Error(`llama.cpp router reports '${modelId}' failed to load`);
-		await sleep(POLL_INTERVAL_MS);
+		await sleep(POLL_INTERVAL_MS, undefined, input.signal ? { signal: input.signal } : {});
 	}
 	throw new Error(`timed out waiting for '${modelId}' to load`);
 }
@@ -275,8 +290,28 @@ async function restoreDisplacedPinned(
 }
 
 /** Ensure the selected llama.cpp router model is resident before inference. */
+/**
+ * Models resident on a llama.cpp server now. A router reports per-model load
+ * states and only the resident ones are returned; a single-model server
+ * reports no states, so every model it lists is the one it serves. The
+ * degraded-inference watchdog calls this when a turn slows down.
+ */
+export async function listLlamaCppResidentModels(
+	baseUrl: string,
+	fetchImpl: typeof fetch = fetch,
+	options: Pick<LlamaCppResidencyInput, "headers" | "signal"> = {},
+): Promise<ResidentModelInfo[]> {
+	const models = await fetchRouterModels({ baseUrl }, routerFetch({ ...options, fetchImpl }));
+	const router = models.some((entry) => entry.state !== "unknown");
+	return models
+		.filter((entry) => !router || residentModel(entry))
+		.map((entry) => ({ modelId: entry.id, tags: entry.tags }));
+}
+
 export async function ensureLlamaCppResidency(input: LlamaCppResidencyInput): Promise<void> {
-	const fetchImpl = input.fetchImpl ?? fetch;
+	input.signal?.throwIfAborted();
+	if (input.managed === false) return;
+	const fetchImpl = routerFetch(input);
 	let snapshot: LlamaCppRouterModel[] = [];
 	const adapter: ResidencyAdapter = {
 		targetKey: residencyTargetKey("llamacpp", input.baseUrl),
@@ -285,6 +320,7 @@ export async function ensureLlamaCppResidency(input: LlamaCppResidencyInput): Pr
 		keepModelId: input.keepModelId,
 		managed: input.managed ?? true,
 		strategy: "router",
+		...(input.signal ? { signal: input.signal } : {}),
 		listResident: async () => {
 			const models = await fetchRouterModels(input, fetchImpl);
 			if (!models.some((entry) => entry.state !== "unknown")) {
@@ -315,11 +351,19 @@ export async function ensureLlamaCppResidency(input: LlamaCppResidencyInput): Pr
 		// `snapshot` predates the eviction and still calls this model resident, so
 		// the state is deliberately not read here: the reload has to be forced.
 		reloadEvicted: async (modelId) => {
-			await ensureModelLoaded(input, fetchImpl, modelId, undefined);
+			// Recovery must survive request cancellation, but has its own total deadline.
+			const recovery = { ...input, signal: AbortSignal.timeout(LOAD_TIMEOUT_MS) };
+			await ensureModelLoaded(recovery, routerFetch(recovery), modelId, undefined);
 		},
 	};
 	if (input.withLock) adapter.withLock = input.withLock;
 	if (input.now) adapter.now = input.now;
 	if (input.ttlMs !== undefined) adapter.ttlMs = input.ttlMs;
-	await reconcileResidency(adapter);
+	const plan = await reconcileResidency(adapter);
+	if (plan.decision === "decline") {
+		throw new ResidencyPreconditionError(
+			plan.notices.find((notice) => notice.level === "error")?.message ??
+				`llama.cpp residency declined loading '${input.keepModelId}' on '${input.targetId}'`,
+		);
+	}
 }