@iowarp/clio-coder 0.3.1 → 0.3.3

package/src/engine/apis/residency.ts

@@ -456,8 +456,8 @@ export function decideResidency(facts: ResidencyFacts): ResidencyPlan {
 
 /**
  * Per-runtime hooks the reconciler drives. Each manageable runtime builds an
- * adapter that closes over its own client (the LM Studio SDK socket, the
- * Ollama HTTP client, the llama.cpp router's HTTP surface) so the reconciler
+ * adapter that closes over its own transport (LM Studio REST, the Ollama HTTP
+ * client, the llama.cpp router's HTTP surface) so the reconciler
  * itself stays runtime-agnostic.
  */
 export interface ResidencyAdapter {
@@ -479,6 +479,16 @@ export interface ResidencyAdapter {
 	 */
 	keepModelTags?(): Promise<ReadonlyArray<string> | undefined>;
 	unload(modelId: string): Promise<void>;
+	/**
+	 * Reject with a {@link ResidencyPreconditionError} when the server cannot
+	 * make the keep model resident at all, most often because the model is not
+	 * in its catalog. Consulted only when the plan would evict, and only from
+	 * facts the adapter already holds, so it costs no extra round trip.
+	 *
+	 * Adapters that cannot know this ahead of the load omit the hook, which
+	 * keeps the previous behaviour of letting the load itself report the failure.
+	 */
+	assertLoadable?(): Promise<void>;
 	/** Router-style servers: read the max co-resident instance count. */
 	capacity?(): Promise<number | undefined>;
 	/**
@@ -488,6 +498,19 @@ export interface ResidencyAdapter {
 	 * its message.
 	 */
 	load?(modelId: string): Promise<void>;
+	/**
+	 * Make a model the reconciler just evicted resident again, after the load
+	 * that eviction made room for was rejected. Distinct from
+	 * {@link ResidencyAdapter.load}, which serves the keep model and may consult
+	 * a listing snapshot taken before the eviction: that snapshot still shows
+	 * the evicted model as resident, so reusing it here would restore nothing.
+	 * Implementations must therefore force the load rather than short-circuit on
+	 * cached state.
+	 *
+	 * Adapters that omit the hook keep the previous behaviour, where a failed
+	 * load leaves the evicted model unloaded.
+	 */
+	reloadEvicted?(modelId: string): Promise<void>;
 	/** Cross-process mutation serializer; defaults to the state-dir lock file. */
 	withLock?<T>(targetKey: string, fn: () => Promise<T>): Promise<T>;
 	now?: () => number;
@@ -496,6 +519,76 @@ export interface ResidencyAdapter {
 
 export type ReconcileResult = ResidencyPlan;
 
+/**
+ * Put back the models an eviction removed once the load it made room for has
+ * failed. Best-effort by construction: the load failure is the error the caller
+ * must see, so a restore that also fails is reported as a notice and never
+ * replaces it. Restores run in eviction order, and the freed slots are known to
+ * be available because the keep model did not take them.
+ */
+async function restoreEvictedModels(
+	adapter: ResidencyAdapter,
+	evicted: ReadonlyArray<ResidentModelInfo>,
+	loadError: unknown,
+): Promise<void> {
+	if (evicted.length === 0) return;
+	const reason = loadError instanceof Error ? loadError.message : String(loadError);
+	const restored: string[] = [];
+	const failed: string[] = [];
+	for (const entry of evicted) {
+		if (!adapter.reloadEvicted) {
+			failed.push(entry.modelId);
+			continue;
+		}
+		try {
+			await adapter.reloadEvicted(entry.modelId);
+			restored.push(entry.modelId);
+		} catch {
+			failed.push(entry.modelId);
+		}
+	}
+	if (restored.length > 0) {
+		emitResidencyNotice({
+			kind: "swap",
+			level: "warning",
+			targetId: adapter.targetId,
+			runtimeId: adapter.runtimeId,
+			model: adapter.keepModelId,
+			message: `loading '${adapter.keepModelId}' on '${adapter.targetId}' failed (${reason}); restored ${restored.map((id) => `'${id}'`).join(", ")} so the target keeps serving.`,
+			detail: { restored: restored.join(", ") },
+		});
+	}
+	if (failed.length > 0) {
+		emitResidencyNotice({
+			kind: "will-not-fit",
+			level: "error",
+			targetId: adapter.targetId,
+			runtimeId: adapter.runtimeId,
+			model: adapter.keepModelId,
+			message: `loading '${adapter.keepModelId}' on '${adapter.targetId}' failed (${reason}) and ${failed.map((id) => `'${id}'`).join(", ")} could not be restored; the target may now serve nothing. Reload it on the server.`,
+			detail: { unrestored: failed.join(", ") },
+		});
+	}
+}
+
+/**
+ * Thrown by an adapter's {@link ResidencyAdapter.assertLoadable} when the
+ * server cannot make the keep model resident at all, most often because the
+ * model is not in the server's catalog. It is a precondition failure, not a
+ * transport failure: the reconciler must not evict anything for a load that is
+ * already known to be impossible, because the eviction would succeed, the load
+ * would fail, and the target would be left holding nothing (#127).
+ *
+ * The reconciler answers it by declining the reconcile and emitting a
+ * `will-not-fit` notice, so the turn continues to the provider call and fails
+ * with the server's own error for the unknown model instead of with a silent
+ * outage on the node. It is the fast path in front of the restore that
+ * {@link reconcileResidency} performs when a load fails for any other reason.
+ */
+export class ResidencyPreconditionError extends Error {
+	override readonly name = "ResidencyPreconditionError";
+}
+
 /**
  * Gather the resident set and capacity facts, decide, emit notices, and
  * perform the mutations under the cross-process lock. List and unload probes
@@ -575,19 +668,62 @@ export async function reconcileResidency(adapter: ResidencyAdapter): Promise<Rec
 	};
 
 	const plan = decideResidency(facts);
+
+	// An eviction is only justified if the load it makes room for can succeed.
+	// Unloading first and discovering afterwards that the keep model does not
+	// exist leaves the target serving nothing, which took a shared node out of
+	// service on the live fleet (#127). Nothing has mutated yet at this point,
+	// so refusing here is a clean no-op: the turn continues to the provider call
+	// and fails with the server's own error for the unknown model.
+	if (plan.decision === "reconcile" && plan.evict.length > 0 && adapter.assertLoadable) {
+		try {
+			await adapter.assertLoadable();
+		} catch (error) {
+			if (!(error instanceof ResidencyPreconditionError)) throw error;
+			const notice: ResidencyNotice = {
+				kind: "will-not-fit",
+				level: "error",
+				targetId: adapter.targetId,
+				runtimeId: adapter.runtimeId,
+				model: adapter.keepModelId,
+				message: error.message,
+			};
+			emitResidencyNotice(notice);
+			return { decision: "decline", evict: [], fallbackEvict: [], keepResident: false, notices: [notice] };
+		}
+	}
+
+	// Emitted after the loadability gate so the operator never reads an eviction
+	// notice for a swap that the gate then refuses.
 	for (const notice of plan.notices) emitResidencyNotice(notice);
 
 	if (plan.decision === "reconcile") {
 		const mutate = async (): Promise<void> => {
+			const evicted: ResidentModelInfo[] = [];
 			for (const entry of plan.evict) {
 				try {
 					await adapter.unload(entry.modelId);
 					forgetClioLoaded(adapter.targetKey, entry.modelId);
+					evicted.push(entry);
 				} catch {
 					// Best-effort: a failed unload self-heals on the next reconcile.
 				}
 			}
-			if (adapter.load) await adapter.load(adapter.keepModelId);
+			if (!adapter.load) return;
+			try {
+				await adapter.load(adapter.keepModelId);
+			} catch (error) {
+				// The eviction bought nothing: the slot it freed is still empty and
+				// the target is now serving less than before the reconcile. Put back
+				// what was taken before surfacing the load's own failure, so a
+				// rejected load costs the node a request rather than its resident
+				// model (#127). The gate above catches the knowable case ahead of any
+				// mutation; this covers the rest, including an out-of-memory load, a
+				// 5xx, a load wait that times out, and a preset removed between the
+				// listing and the load.
+				await restoreEvictedModels(adapter, evicted, error);
+				throw error;
+			}
 		};
 		// Serialize actual mutations across processes; a keep model that is
 		// already fully resident needs no lock (the load hook is a no-op) and

package/src/engine/apis/types.ts

@@ -0,0 +1,16 @@
+import type {
+	Api,
+	AssistantMessageEventStream,
+	Context,
+	Model,
+	SimpleStreamOptions,
+	StreamFunction,
+	StreamOptions,
+} from "@earendil-works/pi-ai";
+
+/** Engine-owned API implementation shape; avoids the deprecated compat registry type. */
+export interface EngineApiProvider<TApi extends Api = Api, TOptions extends StreamOptions = StreamOptions> {
+	api: TApi;
+	stream: StreamFunction<TApi, TOptions>;
+	streamSimple(model: Model<TApi>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream;
+}

package/src/engine/env-api-keys.ts

@@ -0,0 +1,98 @@
+import { existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+import type { ProviderEnv } from "@earendil-works/pi-ai";
+
+export const ANTHROPIC_AUTH_TOKEN_ENV = "ANTHROPIC_AUTH_TOKEN";
+export const ANTHROPIC_OAUTH_TOKEN_ENV = "ANTHROPIC_OAUTH_TOKEN";
+export const ANTHROPIC_API_KEY_ENV = "ANTHROPIC_API_KEY";
+
+const ENV_KEYS: Readonly<Record<string, string>> = {
+	"ant-ling": "ANT_LING_API_KEY",
+	"qwen-token-plan": "QWEN_TOKEN_PLAN_API_KEY",
+	"qwen-token-plan-cn": "QWEN_TOKEN_PLAN_CN_API_KEY",
+	openai: "OPENAI_API_KEY",
+	"azure-openai-responses": "AZURE_OPENAI_API_KEY",
+	nvidia: "NVIDIA_API_KEY",
+	deepseek: "DEEPSEEK_API_KEY",
+	google: "GEMINI_API_KEY",
+	"google-vertex": "GOOGLE_CLOUD_API_KEY",
+	groq: "GROQ_API_KEY",
+	cerebras: "CEREBRAS_API_KEY",
+	xai: "XAI_API_KEY",
+	radius: "RADIUS_API_KEY",
+	openrouter: "OPENROUTER_API_KEY",
+	"vercel-ai-gateway": "AI_GATEWAY_API_KEY",
+	zai: "ZAI_API_KEY",
+	"zai-coding-cn": "ZAI_CODING_CN_API_KEY",
+	mistral: "MISTRAL_API_KEY",
+	minimax: "MINIMAX_API_KEY",
+	"minimax-cn": "MINIMAX_CN_API_KEY",
+	moonshotai: "MOONSHOT_API_KEY",
+	"moonshotai-cn": "MOONSHOT_API_KEY",
+	huggingface: "HF_TOKEN",
+	fireworks: "FIREWORKS_API_KEY",
+	together: "TOGETHER_API_KEY",
+	baseten: "BASETEN_API_KEY",
+	opencode: "OPENCODE_API_KEY",
+	"opencode-go": "OPENCODE_API_KEY",
+	"kimi-coding": "KIMI_API_KEY",
+	"cloudflare-workers-ai": "CLOUDFLARE_API_KEY",
+	"cloudflare-ai-gateway": "CLOUDFLARE_API_KEY",
+	xiaomi: "XIAOMI_API_KEY",
+	"xiaomi-token-plan-cn": "XIAOMI_TOKEN_PLAN_CN_API_KEY",
+	"xiaomi-token-plan-ams": "XIAOMI_TOKEN_PLAN_AMS_API_KEY",
+	"xiaomi-token-plan-sgp": "XIAOMI_TOKEN_PLAN_SGP_API_KEY",
+};
+
+function envValue(name: string, env?: ProviderEnv): string | undefined {
+	return env?.[name] || process.env[name] || undefined;
+}
+
+function apiKeyEnvVars(provider: string): readonly string[] | undefined {
+	if (provider === "github-copilot") return ["COPILOT_GITHUB_TOKEN"];
+	if (provider === "anthropic") {
+		return [ANTHROPIC_AUTH_TOKEN_ENV, ANTHROPIC_OAUTH_TOKEN_ENV, ANTHROPIC_API_KEY_ENV];
+	}
+	const name = ENV_KEYS[provider];
+	return name ? [name] : undefined;
+}
+
+/** Synchronous provider-key discovery pinned to pi-ai 0.84's public behavior. */
+export function findEngineEnvKeys(provider: string, env?: ProviderEnv): string[] | undefined {
+	const names = apiKeyEnvVars(provider);
+	if (!names) return undefined;
+	const found = names.filter((name) => !!envValue(name, env));
+	return found.length > 0 ? found : undefined;
+}
+
+/** Resolve API-key and ambient-auth markers without importing pi-ai/compat. */
+export function getEngineEnvApiKey(provider: string, env?: ProviderEnv): string | undefined {
+	const keys = findEngineEnvKeys(provider, env);
+	if (keys?.[0]) {
+		const name = provider === "anthropic" ? keys.find((key) => key !== ANTHROPIC_AUTH_TOKEN_ENV) : keys[0];
+		if (name) return envValue(name, env);
+	}
+	if (provider === "google-vertex") {
+		const credentialsPath = envValue("GOOGLE_APPLICATION_CREDENTIALS", env);
+		const hasCredentials = credentialsPath
+			? existsSync(credentialsPath)
+			: existsSync(join(homedir(), ".config", "gcloud", "application_default_credentials.json"));
+		const hasProject = !!(envValue("GOOGLE_CLOUD_PROJECT", env) || envValue("GCLOUD_PROJECT", env));
+		const hasLocation = !!envValue("GOOGLE_CLOUD_LOCATION", env);
+		if (hasCredentials && hasProject && hasLocation) return "<authenticated>";
+	}
+	if (
+		provider === "amazon-bedrock" &&
+		(envValue("AWS_PROFILE", env) ||
+			(envValue("AWS_ACCESS_KEY_ID", env) && envValue("AWS_SECRET_ACCESS_KEY", env)) ||
+			envValue("AWS_BEARER_TOKEN_BEDROCK", env) ||
+			envValue("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI", env) ||
+			envValue("AWS_CONTAINER_CREDENTIALS_FULL_URI", env) ||
+			envValue("AWS_WEB_IDENTITY_TOKEN_FILE", env))
+	) {
+		return "<authenticated>";
+	}
+	return undefined;
+}

package/src/engine/gemma-channel-filter.ts

@@ -0,0 +1,223 @@
+import type {
+	AssistantMessage,
+	AssistantMessageEvent,
+	AssistantMessageEventStream,
+	TextContent,
+	ThinkingContent,
+} from "@earendil-works/pi-ai";
+import { createAssistantMessageEventStream } from "@earendil-works/pi-ai";
+
+const THOUGHT_START = /<\|channel>[^\n]*\n/;
+const BARE_THOUGHT_START = /^\s*(?:thought|own[- ]?(?:thought|think))\s*\n/i;
+const BARE_THOUGHT_ONLY = /^\s*(?:thought|own[- ]?(?:thought|think))\s*$/i;
+const THOUGHT_END = "<channel|>";
+const TOOL_CALL_START = "<tool_call|>";
+const TOOL_CALL_END = "<|tool_call|>";
+const LOOKAHEAD_BYTES = 64;
+
+export interface GemmaChannelSegment {
+	kind: "text" | "thinking";
+	content: string;
+}
+
+export interface GemmaChannelFilter {
+	push(chunk: string): GemmaChannelSegment[];
+	flush(): GemmaChannelSegment[];
+}
+
+function boundedPrefix(value: string): { prefix: string; tail: string } {
+	let tailStart = value.length;
+	while (tailStart > 0) {
+		const candidate = tailStart - 1;
+		if (Buffer.byteLength(value.slice(candidate), "utf8") > LOOKAHEAD_BYTES) break;
+		tailStart = candidate;
+	}
+	return { prefix: value.slice(0, tailStart), tail: value.slice(tailStart) };
+}
+
+export function createGemmaChannelFilter(): GemmaChannelFilter {
+	let pending = "";
+	let state: "idle" | "thought" | "toolcall" = "idle";
+	const route = (chunk: string): GemmaChannelSegment[] => {
+		const output: GemmaChannelSegment[] = [];
+		const emit = (kind: GemmaChannelSegment["kind"], content: string) => {
+			if (!content) return;
+			const previous = output.at(-1);
+			if (previous?.kind === kind) previous.content += content;
+			else output.push({ kind, content });
+		};
+		pending += chunk;
+		while (true) {
+			if (state === "thought") {
+				const end = pending.indexOf(THOUGHT_END);
+				const toolCall = pending.indexOf(TOOL_CALL_START);
+				if (toolCall >= 0 && (end < 0 || toolCall < end)) {
+					emit("thinking", pending.slice(0, toolCall));
+					pending = pending.slice(toolCall + TOOL_CALL_START.length);
+					state = "toolcall";
+					continue;
+				}
+				if (end >= 0) {
+					emit("thinking", pending.slice(0, end));
+					pending = pending.slice(end + THOUGHT_END.length);
+					state = "idle";
+					continue;
+				}
+				const bounded = boundedPrefix(pending);
+				emit("thinking", bounded.prefix);
+				pending = bounded.tail;
+				return output;
+			}
+			if (state === "toolcall") {
+				const end = pending.indexOf(TOOL_CALL_END);
+				if (end >= 0) {
+					pending = pending.slice(end + TOOL_CALL_END.length);
+					state = "idle";
+					continue;
+				}
+				pending = boundedPrefix(pending).tail;
+				return output;
+			}
+			const thought = THOUGHT_START.exec(pending);
+			const bare = BARE_THOUGHT_START.exec(pending);
+			const candidates = [
+				{ index: thought?.index ?? -1, kind: "thought" as const, length: thought?.[0].length ?? 0 },
+				{ index: bare ? 0 : -1, kind: "thought" as const, length: bare?.[0].length ?? 0 },
+				{ index: pending.indexOf(TOOL_CALL_START), kind: "toolcall" as const, length: TOOL_CALL_START.length },
+				{ index: pending.indexOf(THOUGHT_END), kind: "orphan" as const, length: THOUGHT_END.length },
+			]
+				.filter((candidate) => candidate.index >= 0)
+				.sort((left, right) => left.index - right.index);
+			const next = candidates[0];
+			if (!next) {
+				const bounded = boundedPrefix(pending);
+				emit("text", bounded.prefix);
+				pending = bounded.tail;
+				return output;
+			}
+			emit("text", pending.slice(0, next.index));
+			pending = pending.slice(next.index + next.length);
+			if (next.kind === "thought") state = "thought";
+			else if (next.kind === "toolcall") state = "toolcall";
+		}
+	};
+	return {
+		push: route,
+		flush() {
+			const output: GemmaChannelSegment[] = [];
+			if (state === "thought" && pending) output.push({ kind: "thinking", content: pending });
+			else if (state === "idle" && pending && !BARE_THOUGHT_ONLY.test(pending)) {
+				output.push({ kind: "text", content: pending });
+			}
+			pending = "";
+			state = "idle";
+			return output;
+		},
+	};
+}
+
+type ReplacementBlock = TextContent | ThinkingContent;
+
+export function filterGemmaChannelStream(
+	source: AssistantMessageEventStream,
+	enabled: boolean,
+): AssistantMessageEventStream {
+	if (!enabled) return source;
+	const filtered = createAssistantMessageEventStream();
+	const replacements = new Map<number, ReplacementBlock[]>();
+	const filters = new Map<number, GemmaChannelFilter>();
+	const activeKinds = new Map<number, GemmaChannelSegment["kind"]>();
+
+	const projectContent = (content: AssistantMessage["content"]): AssistantMessage["content"] =>
+		content.flatMap((block, index) => replacements.get(index) ?? [block]);
+	const projectMessage = (message: AssistantMessage): AssistantMessage => ({
+		...message,
+		content: projectContent(message.content),
+	});
+	const projectedIndex = (originalIndex: number, localIndex = 0): number => {
+		let offset = 0;
+		for (let index = 0; index < originalIndex; index += 1) offset += (replacements.get(index)?.length ?? 1) - 1;
+		return originalIndex + offset + localIndex;
+	};
+	const projectedPartial = (event: { partial: AssistantMessage }): AssistantMessage => projectMessage(event.partial);
+
+	(async () => {
+		try {
+			const closeActive = (originalIndex: number, partial: AssistantMessage) => {
+				const kind = activeKinds.get(originalIndex);
+				const blocks = replacements.get(originalIndex) ?? [];
+				const localIndex = blocks.length - 1;
+				const block = blocks[localIndex];
+				if (!kind || !block) return;
+				const contentIndex = projectedIndex(originalIndex, localIndex);
+				if (kind === "text" && block.type === "text") {
+					filtered.push({ type: "text_end", contentIndex, content: block.text, partial: projectMessage(partial) });
+				} else if (kind === "thinking" && block.type === "thinking") {
+					filtered.push({ type: "thinking_end", contentIndex, content: block.thinking, partial: projectMessage(partial) });
+				}
+				activeKinds.delete(originalIndex);
+			};
+			const emitSegments = (originalIndex: number, partial: AssistantMessage, segments: GemmaChannelSegment[]) => {
+				for (const segment of segments) {
+					if (activeKinds.get(originalIndex) !== segment.kind) {
+						closeActive(originalIndex, partial);
+						const blocks = replacements.get(originalIndex) ?? [];
+						blocks.push(segment.kind === "text" ? { type: "text", text: "" } : { type: "thinking", thinking: "" });
+						replacements.set(originalIndex, blocks);
+						activeKinds.set(originalIndex, segment.kind);
+						const contentIndex = projectedIndex(originalIndex, blocks.length - 1);
+						filtered.push({
+							type: segment.kind === "text" ? "text_start" : "thinking_start",
+							contentIndex,
+							partial: projectMessage(partial),
+						});
+					}
+					const blocks = replacements.get(originalIndex) ?? [];
+					const localIndex = blocks.length - 1;
+					const block = blocks[localIndex];
+					if (!block) continue;
+					if (segment.kind === "text" && block.type === "text") block.text += segment.content;
+					if (segment.kind === "thinking" && block.type === "thinking") block.thinking += segment.content;
+					filtered.push({
+						type: segment.kind === "text" ? "text_delta" : "thinking_delta",
+						contentIndex: projectedIndex(originalIndex, localIndex),
+						delta: segment.content,
+						partial: projectMessage(partial),
+					});
+				}
+			};
+
+			for await (const event of source) {
+				if (event.type === "text_start") {
+					replacements.set(event.contentIndex, []);
+					filters.set(event.contentIndex, createGemmaChannelFilter());
+					continue;
+				}
+				if (event.type === "text_delta") {
+					emitSegments(event.contentIndex, event.partial, filters.get(event.contentIndex)?.push(event.delta) ?? []);
+					continue;
+				}
+				if (event.type === "text_end") {
+					emitSegments(event.contentIndex, event.partial, filters.get(event.contentIndex)?.flush() ?? []);
+					closeActive(event.contentIndex, event.partial);
+					filters.delete(event.contentIndex);
+					continue;
+				}
+				if (event.type === "done") filtered.push({ ...event, message: projectMessage(event.message) });
+				else if (event.type === "error") filtered.push({ ...event, error: projectMessage(event.error) });
+				else if ("partial" in event) {
+					const contentIndex = "contentIndex" in event ? projectedIndex(event.contentIndex as number) : undefined;
+					filtered.push({
+						...event,
+						...(contentIndex === undefined ? {} : { contentIndex }),
+						partial: projectedPartial(event),
+					} as AssistantMessageEvent);
+				} else filtered.push(event as AssistantMessageEvent);
+			}
+			filtered.end();
+		} catch {
+			filtered.end();
+		}
+	})();
+	return filtered;
+}