@iowarp/clio-coder 0.4.7 → 0.4.8

package/src/interactive/turn-prewarm.ts

@@ -10,10 +10,16 @@
  * nobody watching.
  */
 
+import { createHash } from "node:crypto";
 import { BusChannels } from "../core/bus-events.js";
 import type { ClioSettings } from "../core/config.js";
 import type { SafeEventBus } from "../core/event-bus.js";
-import type { ProvidersContract } from "../domains/providers/index.js";
+import { endpointCapacityUsage } from "../domains/dispatch/index.js";
+import {
+	type CacheDeploymentObservation,
+	observeCacheDeployment,
+	type ProvidersContract,
+} from "../domains/providers/index.js";
 import type { SessionContract } from "../domains/session/contract.js";
 import type { Usage } from "../engine/types.js";
 import { type PrewarmTrigger, prewarmPromptTokens, runPrewarmRound } from "./prewarm.js";
@@ -28,9 +34,14 @@ export type PrewarmSkipReason =
 	| "dispatch-active"
 	| "tier"
 	| "unresolved"
+	| "deployment"
+	| "cooldown"
+	| "expired"
 	| "superseded";
 
-export type PrewarmOutcome = { ran: true; trigger: PrewarmTrigger } | { ran: false; reason: PrewarmSkipReason };
+export type PrewarmOutcome =
+	| { ran: true; trigger: PrewarmTrigger }
+	| { ran: false; reason: PrewarmSkipReason; detail?: string };
 
 export interface TurnPrewarmDeps {
 	state: ChatTurnState;
@@ -78,6 +89,7 @@ export interface TurnPrewarmDeps {
 	registerEndpointSlot?: (runtime: AgentRuntime) => (() => void) | null;
 	/** Test seam. Production uses the real provider round. */
 	runPrewarm?: typeof runPrewarmRound;
+	observeDeployment?: typeof observeCacheDeployment;
 	/**
 	 * Whether a submit aborts the round's request. Defaults to the measured
 	 * backend behavior; see {@link ABORT_ROUND_ON_SUBMIT}.
@@ -127,6 +139,13 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 	let pendingTrigger: PrewarmTrigger | null = null;
 	let timer: ReturnType<typeof setTimeout> | null = null;
 	let active: AbortController | null = null;
+	// Ownership outlives detachment: one request and at most one newer trigger.
+	let running: AbortController | null = null;
+	let disposed = false;
+	let pendingSince = 0;
+	let lastFinished = 0;
+	let lastPrefix: string | null = null;
+	let lastSkip: string | null = null;
 	// Rounds a submit or a session switch let go of. Their result is still
 	// recorded, but it no longer describes the prefix the next turn will send, so
 	// it never reaches the `/context` line.
@@ -167,13 +186,33 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 		// notice from either.
 		const targetId = deps.getSettings().chat?.target?.trim();
 		const runtimeId = targetId ? deps.providers.getTarget(targetId)?.runtime : undefined;
-		if (runtimeId === undefined || deps.providers.getRuntime(runtimeId)?.tier !== "local-native") return "tier";
+		if (
+			runtimeId === undefined ||
+			(deps.providers.getRuntime(runtimeId)?.tier !== "local-native" &&
+				!(runtimeId === "litellm" && targetId && deps.providers.getTarget(targetId)?.cache?.deployment))
+		)
+			return "tier";
 		return null;
 	};
 
 	const runOnce = async (trigger: PrewarmTrigger, controller: AbortController): Promise<PrewarmOutcome> => {
 		const refusal = admissionRefusal();
 		if (refusal) return { ran: false, reason: refusal };
+		const targetId = deps.getSettings().chat.target;
+		const target = targetId ? deps.providers.getTarget(targetId) : null;
+		if (!target) return { ran: false, reason: "unresolved" };
+		const targetSettings = JSON.stringify(target);
+		const started = Date.now();
+		if (target.cache?.retention === "none") return { ran: false, reason: "disabled" };
+		if (target.pricing && Object.values(target.pricing).some((rate) => rate > 0))
+			return { ran: false, reason: "deployment", detail: "paid-warming-disabled" };
+		const cooldown = Math.max(1000, target.cache?.warm?.cooldownMs ?? 60000);
+		if (Date.now() - lastFinished < cooldown) return { ran: false, reason: "cooldown" };
+		const observe = deps.observeDeployment ?? observeCacheDeployment;
+		let deployment: CacheDeploymentObservation = await observe(target, deps.getSettings().chat.model ?? "", {
+			signal: controller.signal,
+		});
+		if (deployment.warm !== "bounded") return { ran: false, reason: "deployment", detail: deployment.reason };
 
 		const prepared = await deps.prepareRuntime(controller.signal);
 		if (!prepared.ok) return { ran: false, reason: "unresolved" };
@@ -194,15 +233,48 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 		const lateRefusal = admissionRefusal();
 		if (lateRefusal) return { ran: false, reason: lateRefusal };
 		if (detached.has(controller) || controller.signal.aborted) return { ran: false, reason: "superseded" };
+		// Binding/residency can change during preparation. Discovery itself never
+		// loads models and missing evidence leaves ordinary inference untouched.
+		deployment = await observe(target, runtime.wireModelId, { signal: controller.signal });
+		if (deployment.warm !== "bounded") return { ran: false, reason: "deployment", detail: deployment.reason };
+		const finalRefusal = admissionRefusal();
+		if (finalRefusal) return { ran: false, reason: finalRefusal };
+		if (
+			detached.has(controller) ||
+			controller.signal.aborted ||
+			JSON.stringify(deps.providers.getTarget(target.id)) !== targetSettings
+		)
+			return { ran: false, reason: "superseded" };
+		if (Date.now() - started > 30000) return { ran: false, reason: "expired" };
+		// This includes shared dispatch leases/reservations and this process's
+		// foreground streams. It is a fresh check, not an atomic global warm lease;
+		// server slot observations above also include unrelated active traffic.
+		if (deployment.endpoint && (endpointCapacityUsage()[deployment.endpoint] ?? 0) > 0)
+			return { ran: false, reason: "dispatch-active" };
+		const prefix = createHash("sha256")
+			.update(
+				JSON.stringify({
+					model: runtime.agent.state.model,
+					system: runtime.agent.state.systemPrompt,
+					messages: runtime.agent.state.messages,
+					tools: runtime.agent.state.tools,
+					thinking: runtime.agent.state.thinkingLevel,
+				}),
+			)
+			.digest("hex");
+		if (prefix === lastPrefix && Date.now() - lastFinished < Math.max(cooldown, 60000))
+			return { ran: false, reason: "cooldown" };
 
 		// The claim is taken before the request goes out and released in the
 		// finally, including on the detach path, because the request the server is
 		// still finishing is the one occupying the slot.
 		const releaseEndpointSlot = deps.registerEndpointSlot?.(runtime) ?? null;
 		let result: Awaited<ReturnType<typeof round>>;
+		const deadline = setTimeout(() => controller.abort(), Math.min(target.cache?.warm?.maxDurationMs ?? 30000, 120000));
 		try {
 			result = await round({
 				model: runtime.agent.state.model,
+				agent: runtime.agent,
 				state: {
 					systemPrompt: runtime.agent.state.systemPrompt,
 					messages: runtime.agent.state.messages,
@@ -211,10 +283,14 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 				},
 				...(prepared.apiKey !== undefined ? { apiKey: prepared.apiKey } : {}),
 				signal: controller.signal,
+				maxInputTokens: target.cache?.warm?.maxInputTokens ?? 8192,
 			});
 		} finally {
+			clearTimeout(deadline);
+			lastFinished = Date.now();
 			releaseEndpointSlot?.();
 		}
+		lastPrefix = result.aborted || result.errorMessage ? null : prefix;
 
 		// The prefill the server did is spend whatever the operator did next, so a
 		// let-go round is recorded exactly like a completed one. Only the
@@ -223,7 +299,7 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 		// prefix.
 		const wasDetached = detached.has(controller) || controller.signal.aborted;
 		const promptTokens = prewarmPromptTokens(result);
-		if (!wasDetached) {
+		if (!wasDetached && !result.errorMessage) {
 			deps.context.notePrewarm({
 				tokens: promptTokens,
 				ms: result.timing.apiMs,
@@ -242,6 +318,7 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 				parentTurnId: deps.state.lastTurnId,
 				data: {
 					trigger,
+					deployment,
 					target: runtime.targetId,
 					model: runtime.wireModelId,
 					promptTokens,
@@ -249,9 +326,10 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 					detached: wasDetached,
 					timing: result.timing,
 					promptCache: {
-						input: result.usage?.input ?? 0,
-						cacheRead: result.usage?.cacheRead ?? 0,
-						cacheWrite: result.usage?.cacheWrite ?? 0,
+						input: result.usage?.input ?? null,
+						cacheRead: result.usage?.cacheRead ?? null,
+						cacheWrite: result.usage?.cacheWrite ?? null,
+						...(result.usage?.cacheWrite1h === undefined ? {} : { cacheWrite1h: result.usage.cacheWrite1h }),
 						...(result.backend ? { backend: { ...result.backend } } : {}),
 					},
 					...(result.errorMessage ? { error: result.errorMessage } : {}),
@@ -279,28 +357,55 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 
 	const fire = (): void => {
 		timer = null;
+		if (disposed || running !== null) return;
 		const trigger = pendingTrigger;
 		pendingTrigger = null;
 		if (trigger === null) return;
 		const controller = new AbortController();
 		active = controller;
-		const previous = inFlight;
-		inFlight = previous
-			.catch(() => null)
-			.then(() => runOnce(trigger, controller))
+		running = controller;
+		inFlight = (
+			Date.now() - pendingSince > 30000
+				? Promise.resolve<PrewarmOutcome>({ ran: false, reason: "expired" })
+				: runOnce(trigger, controller)
+		)
+			.then((outcome) => {
+				if (!outcome.ran && deps.getSettings().chat.prewarm && outcome.reason !== "surface") {
+					const key = `${outcome.reason}:${outcome.detail ?? ""}`;
+					if (lastSkip !== key) {
+						lastSkip = key;
+						try {
+							deps.session?.appendEntry({
+								kind: "custom",
+								customType: "prewarmSkipped",
+								display: false,
+								parentTurnId: deps.state.lastTurnId,
+								data: { trigger, ...outcome },
+							});
+						} catch {
+							/* Optional diagnostics. */
+						}
+					}
+				} else if (outcome.ran) lastSkip = null;
+				return outcome;
+			})
 			.catch(() => null)
 			.finally(() => {
 				if (active === controller) active = null;
+				running = null;
+				if (!disposed && pendingTrigger !== null && timer === null) timer = setTimeout(fire, 0);
 			});
 	};
 
 	return {
 		schedule(trigger: PrewarmTrigger): void {
+			if (disposed) return;
 			// A newer trigger describes a newer prefix; the older round is no longer
 			// warming history this session has.
 			releaseActiveRound();
 			pendingTrigger = trigger;
-			if (timer !== null) return;
+			pendingSince = Date.now();
+			if (timer !== null || running !== null) return;
 			// Ref'd on purpose, same reasoning as `settled()` below: the timer is due
 			// in 0 ms, so it holds the event loop for one tick at most, and a Node 22
 			// loop that drains past a due unref'd timer would otherwise never start
@@ -317,7 +422,7 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 			releaseActiveRound();
 		},
 
-		settled(): Promise<PrewarmOutcome | null> {
+		async settled(): Promise<PrewarmOutcome | null> {
 			// One hop past the scheduling tick so a caller that just scheduled a
 			// round waits for that round rather than for the previous one. The timer
 			// is deliberately ref'd, unlike the scheduling timer above: the caller is
@@ -327,14 +432,16 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 			// pending forever and cancelled every later test in the lane; Node 24
 			// happens to fire the due timer first, which is why the hang never
 			// reproduced on a 24.x development machine.
-			return new Promise((resolve) => {
-				setTimeout(() => {
-					resolve(inFlight.catch(() => null));
-				}, 0);
-			});
+			let outcome: PrewarmOutcome | null;
+			do {
+				await new Promise<void>((resolve) => setTimeout(resolve, 0));
+				outcome = await inFlight;
+			} while (timer !== null || running !== null || pendingTrigger !== null);
+			return outcome;
 		},
 
 		dispose(): void {
+			disposed = true;
 			for (const unsubscribe of unsubscribeDispatch) unsubscribe?.();
 			pendingTrigger = null;
 			if (timer !== null) {
@@ -343,7 +450,7 @@ export function createTurnPrewarm(deps: TurnPrewarmDeps): TurnPrewarm {
 			}
 			// Shutdown always aborts: the process is going away, and a socket held
 			// open for a round nobody will read is not a latency question.
-			active?.abort();
+			running?.abort();
 			active = null;
 		},
 	};

package/src/interactive/turn-runtime.ts

@@ -25,6 +25,7 @@ import {
 import type { RetrySettings } from "../domains/session/retry.js";
 import type { createEngineAgent } from "../engine/agent.js";
 import { cleanupEngineSessionResources } from "../engine/ai.js";
+import { setGlobalDefaultMaxOutputTokens } from "../engine/apis/index.js";
 import { sanitizeLockedSynthesisMessage } from "../engine/loop-guard.js";
 import {
 	patchProviderThinkingPayload,
@@ -380,9 +381,17 @@ export function createTurnRuntime(deps: TurnRuntimeDeps): TurnRuntime {
 		);
 	};
 
+	// A target id/model names the selection, not its current URL, auth, pricing
+	// or cache policy. Keep the constructed descriptor identity across calls.
+	let runtimeTargetSettings: string | null = null;
 	const ensureRuntime = (): AgentRuntime | null => {
 		const target = readTarget();
 		if (!target) return null;
+		const targetSettings = JSON.stringify(target.target);
+		const sameTargetSettings = runtimeTargetSettings === targetSettings;
+		// Preflight and the wire budget share the session's effective settings,
+		// including overrides that never reach the saved providers snapshot.
+		setGlobalDefaultMaxOutputTokens(deps.getSettings().chat.maxOutputTokens);
 		context.emitContextWindowWarningTransition(target.runtimeResolution?.contextWindowDetails?.warning ?? null);
 		if (!deps.knownTargets().has(target.target.id)) {
 			throw new TurnAdmissionError(
@@ -394,6 +403,7 @@ export function createTurnRuntime(deps: TurnRuntimeDeps): TurnRuntime {
 			state.runtime &&
 			state.runtime.targetId === target.target.id &&
 			state.runtime.runtimeId === target.runtime.id &&
+			sameTargetSettings &&
 			state.runtime.wireModelId === target.wireModelId
 		) {
 			// Same target+runtime+model. Settings may still have moved
@@ -427,6 +437,7 @@ export function createTurnRuntime(deps: TurnRuntimeDeps): TurnRuntime {
 			state.runtime &&
 			state.runtime.targetId === target.target.id &&
 			state.runtime.runtimeId === target.runtime.id &&
+			sameTargetSettings &&
 			state.runtime.wireModelId !== target.wireModelId
 		) {
 			const nextModel = synthesizeModel(target);
@@ -472,6 +483,7 @@ export function createTurnRuntime(deps: TurnRuntimeDeps): TurnRuntime {
 			maxRetryDelayMs: deps.retrySettings().maxDelayMs,
 			...(gatewaySessionId ? { sessionId: gatewaySessionId } : {}),
 			onStreamInvocation: () => {
+				setGlobalDefaultMaxOutputTokens(deps.getSettings().chat.maxOutputTokens);
 				apiCallStartedAt = performance.now();
 				apiCallFirstDeltaAt = null;
 			},
@@ -777,6 +789,7 @@ export function createTurnRuntime(deps: TurnRuntimeDeps): TurnRuntime {
 							output: summary.output,
 							cacheRead: summary.cacheRead,
 							cacheWrite: summary.cacheWrite,
+							...(summary.cacheWrite1h === undefined ? {} : { cacheWrite1h: summary.cacheWrite1h }),
 							reasoningTokens: summary.reasoning,
 							totalTokens: summary.tokens,
 							apiCalls: summary.apiCalls,
@@ -911,6 +924,7 @@ export function createTurnRuntime(deps: TurnRuntimeDeps): TurnRuntime {
 		});
 
 		state.runtime = localRuntime;
+		runtimeTargetSettings = targetSettings;
 		// Append a modelChange marker only when this rebuild replaces a prior
 		// runtime, which is the cross-target swap case (mid-session change of
 		// target or runtime id). On the initial build, the session header's

package/src/tools/builtin-tool-catalog.ts

@@ -346,12 +346,16 @@ function withBuiltinMetadata<T extends ToolSpec>(spec: T): T {
 	if (!metadata) return spec;
 	// The read cap is a guardrail (`safety.limits.readBytesPerCall`) installed
 	// from settings after this module loads, so the table's import-time value
-	// is the default, not the operator's. Recompute the policy cap when the tool
-	// is registered; the drift check in policy.ts compares against the live cap
-	// and a settings file raising the read cap used to fail boot.
+	// is the default, not the operator's. Resolve the policy cap when consumed
+	// so registration checks and result shaping follow later settings changes.
 	const resultSizePolicy =
 		spec.name === ToolNames.Read
-			? observePolicy(readMaxBytes(), metadata.resultSizePolicy?.followUpHint ?? "")
+			? {
+					...metadata.resultSizePolicy,
+					get maxBytes() {
+						return policyCap(readMaxBytes());
+					},
+				}
 			: metadata.resultSizePolicy;
 	return withMetadata(spec, {
 		...metadata,

package/src/tools/registry.ts

@@ -486,7 +486,12 @@ export function createRegistry(deps: RegistryDeps): ToolRegistry {
 				});
 				const digest = toolResultDigestFor(spec, result, resultDisposition, options);
 				const afterEffects = runToolHook("after_tool", spec, call, decision, options, result, digest);
-				const finalResult = shapeToolResult(spec, applyToolResultEffects(result, afterEffects), options, resultDisposition);
+				const finalResult = shapeToolResult(
+					spec,
+					applyToolResultEffects(result, [...beforeEffects, ...afterEffects]),
+					options,
+					resultDisposition,
+				);
 				return { kind: "ok", result: finalResult, decision };
 			} catch (err) {
 				const message = err instanceof Error ? err.message : String(err);
@@ -495,7 +500,12 @@ export function createRegistry(deps: RegistryDeps): ToolRegistry {
 				const afterEffects = runToolHook("after_tool", spec, call, decision, options, result, digest);
 				return {
 					kind: "ok",
-					result: shapeToolResult(spec, applyToolResultEffects(result, afterEffects), options, resultDisposition),
+					result: shapeToolResult(
+						spec,
+						applyToolResultEffects(result, [...beforeEffects, ...afterEffects]),
+						options,
+						resultDisposition,
+					),
 					decision,
 				};
 			}

package/src/worker/spec-contract.ts

@@ -381,6 +381,33 @@ function validateTarget(value: unknown, runtimeId: string, runtimeAliases: Reado
 	readOptionalEnum(target, "lifecycle", "WorkerSpec.target", TARGET_LIFECYCLES);
 	if (target.auth !== undefined) validateTargetAuth(target.auth);
 	if (target.pricing !== undefined) validateTargetPricing(target.pricing);
+	if (target.cache !== undefined) {
+		const cache = readRecord(target.cache, "WorkerSpec.target.cache");
+		readOptionalEnum(cache, "retention", "WorkerSpec.target.cache", ["none", "short", "long"] as const);
+		if (cache.deployment !== undefined) {
+			const binding = readRecord(cache.deployment, "WorkerSpec.target.cache.deployment");
+			readString(binding.backend, "WorkerSpec.target.cache.deployment.backend");
+			readOptionalEnum(binding, "backend", "WorkerSpec.target.cache.deployment", ["llamacpp", "vllm"] as const);
+			for (const key of ["controlUrl", "model", "build"])
+				readString(binding[key], `WorkerSpec.target.cache.deployment.${key}`);
+			readOptionalString(binding, "gatewayDeploymentId", "WorkerSpec.target.cache.deployment");
+			const url = new URL(readString(binding.controlUrl, "WorkerSpec.target.cache.deployment.controlUrl"));
+			if (!["http:", "https:"].includes(url.protocol) || url.username || url.password || url.search || url.hash)
+				throw new Error(
+					"WorkerSpec.target.cache.deployment.controlUrl must be HTTP(S) without credentials, query, or fragment",
+				);
+		}
+		if (cache.warm !== undefined) {
+			const warm = readRecord(cache.warm, "WorkerSpec.target.cache.warm");
+			for (const key of ["maxInputTokens", "maxDurationMs", "cooldownMs"]) {
+				if (
+					warm[key] !== undefined &&
+					(typeof warm[key] !== "number" || !Number.isSafeInteger(warm[key]) || (warm[key] as number) < 1)
+				)
+					throw new Error(`WorkerSpec.target.cache.warm.${key} must be a positive safe integer`);
+			}
+		}
+	}
 	if (target.capabilities !== undefined) validateCapabilityPatch(target.capabilities, "WorkerSpec.target.capabilities");
 }