@iowarp/clio-coder 0.4.9 → 0.5.0

package/src/engine/apis/lmstudio-residency.ts

@@ -2,11 +2,9 @@
  * LM Studio-specific residency arithmetic, kept pure so the fit and duplicate
  * rules are testable without a server connection.
  *
- * LM Studio's `/api/v1/models` response exposes no total or free GPU memory,
- * and a GPU offload cap does not prove that a load fits
- * rather than failing an oversized load, so a request that does not fit is
- * served from CPU at a fraction of the speed instead of erroring. Fit therefore
- * cannot be computed from memory arithmetic; it is bounded by evidence.
+ * LM Studio's `/api/v1/models` response exposes no total or free GPU memory.
+ * A successful load can still offload work to CPU, so fit cannot be inferred
+ * from load success alone; the context ceiling is an empirical safeguard.
  *
  * The evidence rule: while another model is resident on the same server, an
  * explicit load is capped at {@link CO_RESIDENT_CONTEXT_CEILING} tokens.

package/src/engine/apis/lmstudio.ts

@@ -2,18 +2,24 @@ import type { Api, Model } from "@earendil-works/pi-ai";
 
 import { residencyTargetKey } from "../../core/residency-target-key.js";
 import {
-	type LmStudioCatalog,
-	type LmStudioLoadedInstance,
-	type LmStudioModelInfo,
+	invalidateLmStudioCatalog,
 	listLmStudioModels,
 	lmStudioRootUrl,
+	loadedContextLength,
 	requestLmStudioJson,
 	resolveLmStudioInstance,
 } from "../../domains/providers/runtimes/common/lmstudio-http.js";
 import type { TargetDescriptor } from "../../domains/providers/types/target-descriptor.js";
 import { coResidentContextCeiling, fitLoadContextLength } from "./lmstudio-residency.js";
-import { emitResidencyMutation, emitResidencyNotice, reconcileResidency, residencyManagedFor } from "./residency.js";
+import {
+	declareRuntimeNoticeProducer,
+	emitResidencyMutation,
+	markClioLoaded,
+	reconcileResidency,
+	residencyManagedFor,
+} from "./residency.js";
 import { withResidencyLock } from "./residency-lock.js";
+import type { ResidentModelInfo } from "./resident-models.js";
 
 interface LmStudioModelMetadata {
 	clioCoder?: {
@@ -60,18 +66,6 @@ function targetForModel(model: Model<"openai-completions">): TargetDescriptor |
 	};
 }
 
-function resolveModel(
-	catalog: LmStudioCatalog,
-	id: string,
-): { model: LmStudioModelInfo; instance?: LmStudioLoadedInstance } | null {
-	for (const model of catalog.models) {
-		if (model.key === id) return { model, ...(model.loadedInstances[0] ? { instance: model.loadedInstances[0] } : {}) };
-		const instance = model.loadedInstances.find((entry) => entry.id === id);
-		if (instance) return { model, instance };
-	}
-	return null;
-}
-
 function lmStudioLoadBody(
 	modelKey: string,
 	settings: NonNullable<NonNullable<TargetDescriptor["lmstudio"]>["load"]>,
@@ -92,15 +86,18 @@ async function post(
 	apiKey: string | undefined,
 	signal: AbortSignal | undefined,
 ): Promise<unknown> {
-	const headers: Record<string, string> = { "content-type": "application/json", ...(target.auth?.headers ?? {}) };
-	if (apiKey?.trim()) headers.authorization = `Bearer ${apiKey.trim()}`;
+	const headers = new Headers(target.auth?.headers);
+	headers.set("content-type", "application/json");
+	if (apiKey?.trim() && !headers.has("authorization")) headers.set("authorization", `Bearer ${apiKey.trim()}`);
+	invalidateLmStudioCatalog(target);
 	const response = await requestLmStudioJson(
 		`${lmStudioRootUrl(target.url ?? "")}${path}`,
 		{ method: "POST", headers, body: JSON.stringify(body) },
-		5_000,
+		path === "/api/v1/models/load" ? 120_000 : 5_000,
 		signal,
 	);
 	if (!response.ok) throw new Error(response.error ?? `LM Studio ${path} returned HTTP ${response.status}`);
+	invalidateLmStudioCatalog(target);
 	return response.data;
 }
 
@@ -124,13 +121,25 @@ async function loadOwnedInstance(
 	body: Record<string, unknown>,
 	apiKey: string | undefined,
 	signal: AbortSignal | undefined,
+	requestModel?: Model<"openai-completions">,
 ): Promise<string | undefined> {
 	const data = await post(target, "/api/v1/models/load", body, apiKey, signal);
+	const loadedWindow = (data as { load_config?: { context_length?: unknown } } | null)?.load_config?.context_length;
+	if (requestModel) capToLoadedContext(requestModel, loadedWindow);
 	const instanceId = responseInstanceId(data);
-	if (instanceId) ownedInstances(targetKey).add(instanceId);
+	if (instanceId) {
+		ownedInstances(targetKey).add(instanceId);
+		if (typeof body.model === "string") markClioLoaded(targetKey, body.model);
+	}
 	return instanceId;
 }
 
+function capToLoadedContext(model: Model<"openai-completions">, window: unknown): void {
+	if (typeof window === "number" && Number.isFinite(window) && window > 0) {
+		model.contextWindow = Math.min(model.contextWindow > 0 ? model.contextWindow : window, window);
+	}
+}
+
 /**
  * Residency runs before every request, and the same co-residency facts held
  * every turn, so the peer warning printed once per turn (issue #185). Each
@@ -139,6 +148,8 @@ async function loadOwnedInstance(
  */
 const announcedResidencyFacts = new Set<string>();
 
+const emitResidencyNotice = declareRuntimeNoticeProducer("lmstudio-residency", ["co-resident", "stress"]);
+
 function emitResidencyNoticeOnce(key: string, notice: Parameters<typeof emitResidencyNotice>[0]): void {
 	if (announcedResidencyFacts.has(key)) return;
 	announcedResidencyFacts.add(key);
@@ -148,22 +159,52 @@ function emitResidencyNoticeOnce(key: string, notice: Parameters<typeof emitResi
 export async function ensureLmStudioResidency(
 	model: Model<"openai-completions">,
 	options: { apiKey?: string; signal?: AbortSignal } = {},
+): Promise<string> {
+	const info = metadata(model);
+	const load = info?.lmstudio?.load;
+	if (
+		info?.runtimeId === "lmstudio" &&
+		model.baseUrl &&
+		residencyManagedFor(info.lifecycle) &&
+		load &&
+		Object.keys(load).length > 0
+	) {
+		// Read the catalog after taking the lock so concurrent loads reuse the new instance.
+		return withResidencyLock(
+			residencyTargetKey("lmstudio", lmStudioRootUrl(model.baseUrl)),
+			() => ensureLmStudioResidencyUnlocked(model, options),
+			options.signal,
+		);
+	}
+	return ensureLmStudioResidencyUnlocked(model, options);
+}
+
+async function ensureLmStudioResidencyUnlocked(
+	model: Model<"openai-completions">,
+	options: { apiKey?: string; signal?: AbortSignal } = {},
 ): Promise<string> {
 	const target = targetForModel(model);
 	if (!target) return model.id;
 	const info = metadata(model);
 	if (!info) return model.id;
+	options.signal?.throwIfAborted();
+	const managed = residencyManagedFor(info.lifecycle);
 	const load = target.lmstudio?.load;
+	const explicitLoad = managed && load !== undefined && Object.keys(load).length > 0;
 	const ctx = {
 		credentialsPresent: new Set<string>(),
 		httpTimeoutMs: 5_000,
 		...(options.apiKey ? { authToken: options.apiKey } : {}),
 		...(options.signal ? { signal: options.signal } : {}),
 	};
-	const catalog = await listLmStudioModels(target, ctx);
-	if (!catalog.ok) throw new Error(catalog.error ?? "LM Studio model listing failed");
+	const catalog = await listLmStudioModels(target, ctx, explicitLoad ? 0 : 3000);
+	if (!catalog.ok) {
+		if (!explicitLoad) return model.id;
+		throw new Error(catalog.error ?? "LM Studio model listing failed");
+	}
 	const resolution = resolveLmStudioInstance(target, catalog.models, model.id, info.lmstudioDefaultModel);
 	if (resolution.state === "unknown") {
+		if (!explicitLoad) return model.id;
 		const resident = catalog.models.flatMap((entry) => entry.loadedInstances.map((instance) => instance.id));
 		throw new Error(
 			`LM Studio target '${info.targetId}' does not advertise model '${model.id}'. Resident instances: ${resident.length > 0 ? resident.join(", ") : "none"}. Configure an explicit LM Studio load before requesting an unlisted model.`,
@@ -193,14 +234,13 @@ export async function ensureLmStudioResidency(
 			detail: { requestedModel: model.id, wireModel: resolution.wireModelId, peerTargets: peers },
 		});
 	}
-	if (!load || Object.keys(load).length === 0 || resolution.instance) return resolution.wireModelId;
+	capToLoadedContext(model, loadedContextLength(resolution.instance));
+	if (!explicitLoad || !load || resolution.instance) return resolution.wireModelId;
 	if (catalog.tier !== "0.4+") return resolution.wireModelId;
-	const selected = resolveModel(catalog, model.id);
-	const modelKey = selected?.model.key ?? model.id;
+	const modelKey = resolution.model?.key ?? model.id;
 	let instances = catalog.models.flatMap((entry) =>
 		entry.loadedInstances.map((instance) => ({ modelKey: entry.key, identifier: instance.id, instance })),
 	);
-	const managed = residencyManagedFor(info.lifecycle);
 	const targetKey = residencyTargetKey("lmstudio", target.url ?? model.baseUrl);
 	const contextLength = target.lmstudio?.load?.contextLength;
 	const plan = await reconcileResidency({
@@ -210,6 +250,8 @@ export async function ensureLmStudioResidency(
 		keepModelId: modelKey,
 		managed,
 		strategy: "jit",
+		ttlMs: 0, // The fresh catalog, not a previous load attempt, determines residency.
+		...(options.signal ? { signal: options.signal } : {}),
 		...(contextLength !== undefined ? { contextLength } : {}),
 		...(model.contextWindow > 0 ? { modelMaxContext: model.contextWindow } : {}),
 		listResident: async () => [...new Set(instances.map((entry) => entry.modelKey))].map((modelId) => ({ modelId })),
@@ -245,7 +287,7 @@ export async function ensureLmStudioResidency(
 		}
 	}
 	const loadAndReport = async (): Promise<string> => {
-		const instanceId = await loadOwnedInstance(target, targetKey, body, options.apiKey, options.signal);
+		const instanceId = await loadOwnedInstance(target, targetKey, body, options.apiKey, options.signal, model);
 		emitResidencyMutation({
 			targetKey,
 			targetId: info.targetId,
@@ -258,11 +300,19 @@ export async function ensureLmStudioResidency(
 	try {
 		return await loadAndReport();
 	} catch (error) {
-		if (plan.decision === "observe" || plan.fallbackEvict.length === 0) throw error;
-		return withResidencyLock(targetKey, async () => {
+		options.signal?.throwIfAborted();
+		const message = error instanceof Error ? error.message : String(error);
+		const capacityError =
+			/insufficient[_ ](?:system[_ ])?(?:resources|memory)|out of (?:device |gpu |system )?memory|not enough (?:free )?(?:vram|memory)/i.test(
+				message,
+			);
+		if (!capacityError || plan.decision !== "reconcile" || plan.fallbackEvict.length === 0) throw error;
+		const evicted: typeof instances = [];
+		try {
 			for (const candidate of plan.fallbackEvict) {
 				for (const entry of instances.filter((resident) => resident.modelKey === candidate.modelId)) {
 					if (await unloadOwnedInstance(target, targetKey, entry.identifier, options.apiKey, options.signal)) {
+						evicted.push(entry);
 						emitResidencyMutation({
 							targetKey,
 							targetId: info.targetId,
@@ -273,7 +323,63 @@ export async function ensureLmStudioResidency(
 					}
 				}
 			}
-			return loadAndReport();
-		});
+			if (evicted.length === 0) throw error;
+			return await loadAndReport();
+		} catch (retryError) {
+			// A rejected replacement must not leave an otherwise working server empty.
+			for (const entry of evicted) {
+				const restore: Record<string, unknown> = { model: entry.modelKey };
+				for (const key of [
+					"context_length",
+					"flash_attention",
+					"eval_batch_size",
+					"num_experts",
+					"offload_kv_cache_to_gpu",
+				]) {
+					if (entry.instance.config[key] !== undefined) restore[key] = entry.instance.config[key];
+				}
+				try {
+					await loadOwnedInstance(target, targetKey, restore, options.apiKey, undefined);
+					emitResidencyMutation({
+						targetKey,
+						targetId: info.targetId,
+						runtimeId: "lmstudio",
+						model: entry.modelKey,
+						operation: "load",
+					});
+				} catch {
+					emitResidencyNotice({
+						kind: "stress",
+						level: "error",
+						targetId: info.targetId,
+						runtimeId: "lmstudio",
+						model: entry.modelKey,
+						message: `LM Studio could not restore '${entry.modelKey}' after the replacement load failed. Reload it on the server.`,
+					});
+				}
+			}
+			throw retryError;
+		}
 	}
 }
+
+/**
+ * Model keys with at least one loaded instance on the model's LM Studio target,
+ * as the REST listing reports them now. The degraded-inference watchdog calls
+ * this when a turn slows down, so the notice names what shares the server.
+ */
+export async function listLmStudioResidentModels(
+	model: Model<"openai-completions">,
+	options: { apiKey?: string; signal?: AbortSignal } = {},
+): Promise<ResidentModelInfo[]> {
+	const target = targetForModel(model);
+	if (!target) throw new Error("no LM Studio target for model");
+	const catalog = await listLmStudioModels(target, {
+		credentialsPresent: new Set<string>(),
+		httpTimeoutMs: 1_500,
+		...(options.apiKey ? { authToken: options.apiKey } : {}),
+		...(options.signal ? { signal: options.signal } : {}),
+	});
+	if (!catalog.ok) throw new Error(catalog.error ?? "LM Studio model listing failed");
+	return catalog.models.filter((entry) => entry.loadedInstances.length > 0).map((entry) => ({ modelId: entry.key }));
+}

package/src/engine/apis/ollama-http.ts

@@ -0,0 +1,118 @@
+import type { ChatRequest, ChatResponse } from "ollama";
+
+/** Preserve structured Ollama errors, including context overflows inside a 200 stream. */
+function errorText(detail: unknown): string {
+	if (typeof detail === "string") return detail;
+	if (detail && typeof detail === "object") {
+		const { message, type } = detail as { message?: unknown; type?: unknown };
+		const text = typeof message === "string" ? message : JSON.stringify(detail);
+		return typeof type === "string" ? `${type}: ${text}` : text;
+	}
+	return String(detail);
+}
+
+interface RequestOptions {
+	headers?: Record<string, string> | undefined;
+	signal?: AbortSignal | undefined;
+	fetch?: typeof fetch | undefined;
+}
+
+async function request(baseUrl: string, path: string, body: unknown, options: RequestOptions): Promise<Response> {
+	options.signal?.throwIfAborted();
+	const headers = new Headers(options.headers);
+	headers.set("content-type", "application/json");
+	const url = `${baseUrl.replace(/\/+$/, "").replace(/\/v1$/, "")}/api/${path}`;
+	if (new URL(url).origin === "https://ollama.com" && !headers.has("authorization") && process.env.OLLAMA_API_KEY) {
+		headers.set("authorization", `Bearer ${process.env.OLLAMA_API_KEY}`);
+	}
+	const response = await (options.fetch ?? fetch)(url, {
+		method: body === undefined ? "GET" : "POST",
+		headers,
+		...(body === undefined ? {} : { body: JSON.stringify(body) }),
+		...(options.signal ? { signal: options.signal } : {}),
+	});
+	if (!response.ok) {
+		const text = await response.text();
+		let detail = text;
+		try {
+			const parsed = JSON.parse(text) as { error?: unknown };
+			if (parsed?.error !== undefined) detail = errorText(parsed.error);
+		} catch {
+			// Proxies also return plain text or HTML. Keep that response out of stdout.
+		}
+		throw new Error(detail.slice(0, 4000) || `HTTP ${response.status}: ${response.statusText}`);
+	}
+	return response;
+}
+
+/** Small metadata and unload requests have a deadline as well as caller cancellation. */
+export async function ollamaJson<T>(
+	baseUrl: string,
+	path: string,
+	body: unknown,
+	options: RequestOptions = {},
+): Promise<T> {
+	const timeout = AbortSignal.timeout(3000);
+	const signal = options.signal ? AbortSignal.any([options.signal, timeout]) : timeout;
+	const response = await request(baseUrl, path, body, { ...options, signal });
+	const result = (await response.json()) as T & { error?: unknown };
+	if (result?.error !== undefined) throw new Error(errorText(result.error));
+	return result;
+}
+
+/** Own the response until done, cancellation, or failure; never print from the transport. */
+export async function* streamOllamaChat(
+	baseUrl: string,
+	body: ChatRequest & { stream: true },
+	options: RequestOptions,
+): AsyncGenerator<ChatResponse> {
+	const response = await request(baseUrl, "chat", body, options);
+	if (!response.body) throw new Error("Ollama returned an empty response body");
+	const reader = response.body.getReader();
+	const decoder = new TextDecoder();
+	let pending = "";
+	try {
+		while (true) {
+			const { done, value } = await reader.read();
+			pending += done ? decoder.decode() : decoder.decode(value, { stream: true });
+			while (pending.length > 0) {
+				const newline = pending.indexOf("\n");
+				if (newline < 0 && !done) break;
+				const line = newline < 0 ? pending : pending.slice(0, newline);
+				pending = newline < 0 ? "" : pending.slice(newline + 1);
+				if (line.length > 16 * 1024 * 1024) throw new Error("Ollama response frame exceeds 16 MiB");
+				if (!line.trim()) continue;
+				let chunk: ChatResponse & { error?: unknown };
+				try {
+					chunk = JSON.parse(line);
+				} catch {
+					throw new Error("Ollama returned invalid JSON in its response stream");
+				}
+				if (!chunk || typeof chunk !== "object" || Array.isArray(chunk)) {
+					throw new Error("Ollama returned an invalid response frame");
+				}
+				if (chunk.error !== undefined) throw new Error(errorText(chunk.error));
+				if (
+					typeof chunk.done !== "boolean" ||
+					(chunk.model !== undefined && typeof chunk.model !== "string") ||
+					(!chunk.done && !chunk.message) ||
+					(chunk.message !== undefined &&
+						(!chunk.message || typeof chunk.message !== "object" || Array.isArray(chunk.message))) ||
+					(chunk.message?.content !== undefined && typeof chunk.message.content !== "string") ||
+					(chunk.message?.thinking !== undefined && typeof chunk.message.thinking !== "string") ||
+					(chunk.message?.tool_calls !== undefined && !Array.isArray(chunk.message.tool_calls))
+				) {
+					throw new Error("Ollama returned an invalid response frame");
+				}
+				options.signal?.throwIfAborted();
+				yield chunk;
+				if (chunk.done) return;
+			}
+			if (done) throw new Error("Ollama response stream ended before its completion frame");
+			if (pending.length > 16 * 1024 * 1024) throw new Error("Ollama response frame exceeds 16 MiB");
+		}
+	} finally {
+		await reader.cancel().catch(() => {});
+		reader.releaseLock();
+	}
+}