@iowarp/clio-coder 0.4.2 → 0.4.3

package/src/engine/apis/openai-completions.ts

@@ -17,6 +17,11 @@ import {
 } from "@earendil-works/pi-ai";
 import { openAICompletionsApi } from "@earendil-works/pi-ai/api/openai-completions.lazy";
 import type { BackendCompletionTimings, BackendTimingsSource } from "../../core/cache-telemetry.js";
+import {
+	type GatewayRoutingObservation,
+	liteLLMGatewayRoutingFromHeaders,
+	liteLLMRouteFailureMessage,
+} from "../../core/gateway-routing.js";
 import type { ResponseModelIdObservation } from "../../core/response-model-id.js";
 import {
 	type AppliedThinking,
@@ -27,7 +32,7 @@ import {
 import { lmStudioReasoningEffort } from "../../domains/providers/runtimes/common/lmstudio-http.js";
 import type { ThinkingLevel } from "../../domains/providers/types/capability-flags.js";
 import type { LocalModelQuirks, SamplingProfile } from "../../domains/providers/types/local-model-quirks.js";
-import type { LmStudioTargetSettings } from "../../domains/providers/types/target-descriptor.js";
+import type { LiteLLMTargetSettings, LmStudioTargetSettings } from "../../domains/providers/types/target-descriptor.js";
 import { filterGemmaChannelStream, usesGemmaChannelMarkers } from "../gemma-channel-filter.js";
 import { HarmonyResponseParser } from "../harmony-response.js";
 import { createSentinelStripper, stripTokenizerSentinels } from "../strip-tokenizer-sentinels.js";
@@ -42,6 +47,8 @@ declare module "@earendil-works/pi-ai" {
 	interface AssistantMessage {
 		/** Direct observation of model-id presence in an OpenAI-compatible response. */
 		responseModelIdObservation?: ResponseModelIdObservation;
+		/** Route, fallback, retry, and proxy timing facts reported by LiteLLM. */
+		gatewayRouting?: GatewayRoutingObservation;
 		/** Backend-reported prefill and prediction timings when available. */
 		backendTimings?: BackendCompletionTimings;
 	}
@@ -69,6 +76,7 @@ interface ClioRuntimeMetadata {
 		quirks?: LocalModelQuirks;
 		chatTemplateKwargsUnsupported?: boolean;
 		lmstudio?: LmStudioTargetSettings;
+		litellm?: LiteLLMTargetSettings;
 		lmstudioReasoningOptions?: ReadonlyArray<string>;
 		lmstudioDefaultModel?: string;
 	};
@@ -101,6 +109,7 @@ interface ResponseModelIdCapture {
 	modelIdDone: boolean;
 	backendTimings: BackendCompletionTimings | null;
 	backendTimingsSource: BackendTimingsSource | null;
+	gatewayRouting: GatewayRoutingObservation | null;
 	buffer: string;
 	decoder: TextDecoder | null;
 }
@@ -199,7 +208,10 @@ function observeResponseModelIdBytes(
 	if (captureCanStopEarly(capture) || flush) capture.decoder = null;
 }
 
-function captureResponseModelId(response: Response, capture: ResponseModelIdCapture): Response {
+function captureResponseModelId(response: Response, capture: ResponseModelIdCapture, model: Model<Api>): Response {
+	if (runtimeMetadata(model)?.runtimeId === "litellm") {
+		capture.gatewayRouting = liteLLMGatewayRoutingFromHeaders(response.headers);
+	}
 	if (!response.body || !response.headers.get("content-type")?.toLowerCase().includes("text/event-stream")) {
 		return response;
 	}
@@ -234,6 +246,7 @@ function withResponseModelIdCapture<TOptions extends StreamOptions>(
 		modelIdDone: false,
 		backendTimings: null,
 		backendTimingsSource: backendTimingsSourceForModel(model),
+		gatewayRouting: null,
 		buffer: "",
 		decoder: new TextDecoder(),
 	};
@@ -241,7 +254,7 @@ function withResponseModelIdCapture<TOptions extends StreamOptions>(
 		options.fetch ?? ((input, init) => globalThis.fetch(input, init));
 	const capturedOptions = {
 		...options,
-		fetch: async (input, init) => captureResponseModelId(await fetchImpl(input, init), capture),
+		fetch: async (input, init) => captureResponseModelId(await fetchImpl(input, init), capture, model),
 	} as TOptions;
 	const source = sourceFactory(capturedOptions);
 	const annotated = createAssistantMessageEventStream();
@@ -255,9 +268,11 @@ function withResponseModelIdCapture<TOptions extends StreamOptions>(
 					: { state: "not-observed" };
 				if (event.type === "done") {
 					event.message.responseModelIdObservation = observation;
+					if (capture.gatewayRouting !== null) event.message.gatewayRouting = capture.gatewayRouting;
 					if (capture.backendTimings !== null) event.message.backendTimings = capture.backendTimings;
 				} else if (event.type === "error") {
 					event.error.responseModelIdObservation = observation;
+					if (capture.gatewayRouting !== null) event.error.gatewayRouting = capture.gatewayRouting;
 					if (capture.backendTimings !== null) event.error.backendTimings = capture.backendTimings;
 				}
 				annotated.push(event as AssistantMessageEvent);
@@ -271,6 +286,31 @@ function withResponseModelIdCapture<TOptions extends StreamOptions>(
 	return annotated;
 }
 
+/** Make a failed physical gateway route final and useful to an operator. */
+function withLiteLLMRouteFailureAdvice(
+	model: Model<"openai-completions">,
+	source: AssistantMessageEventStream,
+): AssistantMessageEventStream {
+	const metadata = runtimeMetadata(model);
+	if (metadata?.runtimeId !== "litellm") return source;
+	const advised = createAssistantMessageEventStream();
+	(async () => {
+		try {
+			for await (const event of source) {
+				if (event.type === "error") {
+					event.error.errorMessage = liteLLMRouteFailureMessage(event.error.errorMessage, metadata.targetId, model.id);
+				}
+				advised.push(event as AssistantMessageEvent);
+			}
+			advised.end();
+		} catch (err) {
+			const message = err instanceof Error ? err.message : String(err);
+			throw new Error(liteLLMRouteFailureMessage(message, metadata.targetId, model.id));
+		}
+	})();
+	return advised;
+}
+
 function samplingParamsFromProfile(profile: SamplingProfile): Record<string, unknown> {
 	return {
 		...(profile.topP !== undefined ? { top_p: profile.topP } : {}),
@@ -395,6 +435,17 @@ function applyThinkingPayload(
 	) {
 		next.reasoning_effort = resolved.request.reasoningEffort;
 	}
+	// LiteLLM generic openai/<local model> routes otherwise drop this standard
+	// parameter. Allow only the effort this model/runtime actually resolved;
+	// unknown off controls and unrelated caller parameters gain no allowance.
+	if (
+		resolved.runtimeId === "litellm" &&
+		resolved.request.reasoningEffort &&
+		next.reasoning_effort === resolved.request.reasoningEffort
+	) {
+		const allowed = Array.isArray(next.allowed_openai_params) ? next.allowed_openai_params : [];
+		next.allowed_openai_params = [...new Set([...allowed, "reasoning_effort"])];
+	}
 	if (resolved.request.chatTemplateKwargs && !chatTemplateKwargsUnsupported(model)) {
 		const existing = isPlainRecord(next.chat_template_kwargs) ? next.chat_template_kwargs : {};
 		next.chat_template_kwargs = { ...existing, ...resolved.request.chatTemplateKwargs };
@@ -433,15 +484,15 @@ function applyLmStudioPayload(
 	if (request?.ttlSeconds !== undefined) next.ttl = request.ttlSeconds;
 	if (request?.draftModel !== undefined) next.draft_model = request.draftModel;
 	if (resolved.thinking.mechanism === "none" || resolved.thinking.mechanism === "always-on") return next;
-	const resolvedEffort =
-		resolved.request.reasoningEffort ??
-		lmStudioReasoningEffort(resolved.thinking.effectiveLevel, runtimeMetadata(model)?.lmstudioReasoningOptions);
+	const resolvedEffort = runtimeMetadata(model)?.lmstudioReasoningOptions
+		? lmStudioReasoningEffort(resolved.thinking.effectiveLevel, runtimeMetadata(model)?.lmstudioReasoningOptions)
+		: (resolved.request.reasoningEffort ?? lmStudioReasoningEffort(resolved.thinking.effectiveLevel));
 	switch (request?.reasoning) {
 		case "off":
-			next.reasoning_effort = "none";
+			next.reasoning_effort = lmStudioReasoningEffort("off", runtimeMetadata(model)?.lmstudioReasoningOptions);
 			break;
 		case "on":
-			next.reasoning_effort = "low";
+			next.reasoning_effort = lmStudioReasoningEffort("low", runtimeMetadata(model)?.lmstudioReasoningOptions);
 			break;
 		case "low":
 		case "medium":
@@ -584,6 +635,50 @@ function withRemainingContextBudget<TOptions extends StreamOptions>(
 	} as TOptions;
 }
 
+function hasHeader(headers: Readonly<Record<string, unknown>> | undefined, name: string): boolean {
+	if (!headers) return false;
+	const wanted = name.toLowerCase();
+	return Object.keys(headers).some((key) => key.toLowerCase() === wanted);
+}
+
+/**
+ * Apply LiteLLM's request-control headers at the final transport boundary.
+ *
+ * The OpenAI SDK otherwise retries twice below Clio's visible recovery loop and
+ * below LiteLLM's own router, multiplying a single failure into several hidden
+ * attempts. Gateway requests use zero client retries by default; optional
+ * `numRetries` controls the proxy router itself, where any configured attempts
+ * remain observable. A physical-routing gateway should leave it at zero.
+ */
+function withLiteLLMRequestOptions<TOptions extends StreamOptions>(
+	model: Model<"openai-completions">,
+	options: TOptions,
+): TOptions {
+	const metadata = runtimeMetadata(model);
+	if (metadata?.runtimeId !== "litellm") return options;
+	const request = metadata.litellm?.request;
+	const modelHeaders = model.headers as Readonly<Record<string, unknown>> | undefined;
+	const optionHeaders = options.headers as Readonly<Record<string, unknown>> | undefined;
+	const headers: Record<string, string | null> = { ...(options.headers ?? {}) };
+	const setDefault = (name: string, value: string | undefined): void => {
+		if (value === undefined || hasHeader(modelHeaders, name) || hasHeader(optionHeaders, name)) return;
+		headers[name] = value;
+	};
+	setDefault("x-litellm-tags", ["clio-coder", ...(request?.tags ?? [])].join(","));
+	if (request?.sendSessionId !== false) setDefault("x-litellm-session-id", options.sessionId);
+	setDefault("x-litellm-timeout", request?.timeoutSeconds?.toString());
+	setDefault("x-litellm-stream-timeout", request?.streamTimeoutSeconds?.toString());
+	setDefault("x-litellm-num-retries", request?.numRetries?.toString());
+	return {
+		...options,
+		headers,
+		// Never hide duplicate attempts inside the OpenAI SDK beneath Clio's
+		// operator-visible failure. The gateway may still be configured to retry,
+		// but physical-routing deployments should keep that policy at zero too.
+		maxRetries: options.maxRetries ?? 0,
+	} as TOptions;
+}
+
 function requiredToolArguments(tool: Tool): ReadonlyArray<string> {
 	const schema = tool.parameters as unknown;
 	if (schema === null || typeof schema !== "object" || Array.isArray(schema)) return [];
@@ -921,15 +1016,18 @@ export const openAICompletionsApiProvider: EngineApiProvider<"openai-completions
 		const resolved = resolvedCapabilitiesForModel(model, defaultLevel);
 		const effectiveContext = stripsThinking(resolved) ? stripThinkingFromContext(context) : context;
 		const withSamplers = withSamplingOverrides(model, options, resolved);
+		const withGatewayOptions = withLiteLLMRequestOptions(
+			model,
+			withRemainingContextBudget(model, effectiveContext, withSamplers),
+		);
 		return guardMalformedToolCalls(
 			withReasoningTokenEstimate(
 				stripSentinelsFromStream(
 					stripNeverReasoningFromStream(
 						filterGemmaChannelStream(
-							withResponseModelIdCapture(
+							withLiteLLMRouteFailureAdvice(
 								model,
-								withRemainingContextBudget(model, effectiveContext, withSamplers),
-								(capturedOptions) =>
+								withResponseModelIdCapture(model, withGatewayOptions, (capturedOptions) =>
 									withLocalResidency(
 										model,
 										{
@@ -938,6 +1036,7 @@ export const openAICompletionsApiProvider: EngineApiProvider<"openai-completions
 										},
 										(requestModel) => piOpenAICompletions.stream(requestModel, effectiveContext, capturedOptions),
 									),
+								),
 							),
 							usesGemmaChannelMarkers(resolved.modelId),
 						),
@@ -954,15 +1053,18 @@ export const openAICompletionsApiProvider: EngineApiProvider<"openai-completions
 		const resolved = resolvedCapabilitiesForModel(model, thinkingLevelFromSimple(options));
 		const effectiveContext = stripsThinking(resolved) ? stripThinkingFromContext(context) : context;
 		const withSamplers = withSamplingOverrides(model, options, resolved);
+		const withGatewayOptions = withLiteLLMRequestOptions(
+			model,
+			withRemainingContextBudget(model, effectiveContext, withSamplers),
+		);
 		return guardMalformedToolCalls(
 			withReasoningTokenEstimate(
 				stripSentinelsFromStream(
 					stripNeverReasoningFromStream(
 						filterGemmaChannelStream(
-							withResponseModelIdCapture(
+							withLiteLLMRouteFailureAdvice(
 								model,
-								withRemainingContextBudget(model, effectiveContext, withSamplers),
-								(capturedOptions) =>
+								withResponseModelIdCapture(model, withGatewayOptions, (capturedOptions) =>
 									withLocalResidency(
 										model,
 										{
@@ -971,6 +1073,7 @@ export const openAICompletionsApiProvider: EngineApiProvider<"openai-completions
 										},
 										(requestModel) => piOpenAICompletions.streamSimple(requestModel, effectiveContext, capturedOptions),
 									),
+								),
 							),
 							usesGemmaChannelMarkers(resolved.modelId),
 						),

package/src/engine/external-subprocess.ts

@@ -1,15 +1,117 @@
 import type { ChildProcessByStdio } from "node:child_process";
 import type { Readable } from "node:stream";
 
+import { boundedExternalDiagnostic } from "../core/external-diagnostic.js";
+
 export type SubprocessWithStdio = ChildProcessByStdio<null, Readable, Readable>;
 
+export type BoundedLine = { kind: "line"; line: string } | { kind: "oversized" };
+
+/**
+ * Split a byte stream without ever retaining more than one bounded line.
+ * Node has already allocated each incoming chunk, but an unterminated provider
+ * line cannot make this decoder accumulate without limit.
+ */
+export async function* readBoundedLines(
+	readable: Readable,
+	options: { maxLineBytes: number; maxTotalBytes: number },
+): AsyncGenerator<BoundedLine> {
+	let segments: Buffer[] = [];
+	let lineBytes = 0;
+	let totalBytes = 0;
+	let discarding = false;
+	const finishLine = (): BoundedLine | null => {
+		if (discarding) {
+			segments = [];
+			lineBytes = 0;
+			discarding = false;
+			return { kind: "oversized" };
+		}
+		if (lineBytes === 0) return { kind: "line", line: "" };
+		const joined = Buffer.concat(segments, lineBytes);
+		segments = [];
+		lineBytes = 0;
+		const end = joined.at(-1) === 0x0d ? joined.length - 1 : joined.length;
+		return { kind: "line", line: joined.subarray(0, end).toString("utf8") };
+	};
+
+	for await (const raw of readable) {
+		const chunk = Buffer.isBuffer(raw) ? raw : Buffer.from(String(raw));
+		totalBytes += chunk.byteLength;
+		if (totalBytes > options.maxTotalBytes) throw new Error("external stdout exceeded the cumulative output limit");
+		let offset = 0;
+		while (offset < chunk.length) {
+			const newline = chunk.indexOf(0x0a, offset);
+			const end = newline === -1 ? chunk.length : newline;
+			const piece = chunk.subarray(offset, end);
+			if (!discarding && piece.length > 0) {
+				if (lineBytes + piece.length > options.maxLineBytes) {
+					segments = [];
+					lineBytes = 0;
+					discarding = true;
+				} else {
+					segments.push(piece);
+					lineBytes += piece.length;
+				}
+			}
+			if (newline === -1) break;
+			const line = finishLine();
+			if (line) yield line;
+			offset = newline + 1;
+		}
+	}
+	if (discarding || lineBytes > 0) {
+		const line = finishLine();
+		if (line) yield line;
+	}
+}
+
 export async function readStderr(child: { stderr: Readable }): Promise<string> {
-	let stderr = "";
+	let tail = Buffer.alloc(0);
 	for await (const chunk of child.stderr) {
-		stderr += String(chunk);
-		if (stderr.length > 8192) stderr = stderr.slice(-8192);
+		const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk));
+		tail = Buffer.concat([tail, bytes]);
+		if (tail.byteLength > 8192) tail = tail.subarray(tail.byteLength - 8192);
 	}
-	return stderr;
+	return boundedExternalDiagnostic(tail.toString("utf8"));
+}
+
+export interface ProcessTreeTerminator {
+	terminate(): void;
+	cleanup(): void;
+}
+
+/** POSIX process-group termination with a direct-child fallback; Windows is direct-child only. */
+export function createProcessTreeTerminator(
+	child: { pid?: number | undefined; exitCode: number | null; kill(signal?: NodeJS.Signals): boolean },
+	graceMs = 1500,
+): ProcessTreeTerminator {
+	let sent = false;
+	let timer: ReturnType<typeof setTimeout> | null = null;
+	const signal = (name: NodeJS.Signals): void => {
+		if (child.exitCode !== null) return;
+		if (child.pid && process.platform !== "win32") {
+			try {
+				process.kill(-child.pid, name);
+				return;
+			} catch {
+				// The child may not have established its group yet.
+			}
+		}
+		child.kill(name);
+	};
+	return {
+		terminate() {
+			if (sent) return;
+			sent = true;
+			signal("SIGTERM");
+			timer = setTimeout(() => signal("SIGKILL"), graceMs);
+		},
+		cleanup() {
+			if (timer) clearTimeout(timer);
+			timer = null;
+		},
+	};
 }
 
 export function waitForClose(child: {
@@ -17,7 +119,13 @@ export function waitForClose(child: {
 	once(event: "close", listener: (code: number | null) => void): unknown;
 }): Promise<number> {
 	return new Promise((resolve) => {
-		child.once("error", () => resolve(1));
-		child.once("close", (code: number | null) => resolve(code ?? 1));
+		let settled = false;
+		const finish = (code: number): void => {
+			if (settled) return;
+			settled = true;
+			resolve(code);
+		};
+		child.once("error", () => finish(1));
+		child.once("close", (code: number | null) => finish(code ?? 1));
 	});
 }

package/src/entry/background-model-metadata.ts

@@ -0,0 +1,18 @@
+import type { ProvidersContract } from "../domains/providers/contract.js";
+
+/** Cold gateway roles need their own declared model controls before synthesis.
+ * Read metadata only; an unknown/mixed successful probe remains authoritative.
+ * Memory supplies its existing deadline/generation signal, including auth.
+ */
+export async function prepareBackgroundModelMetadata(
+	providers: ProvidersContract,
+	targetId: string,
+	signal?: AbortSignal,
+): Promise<void> {
+	signal?.throwIfAborted();
+	const status = providers.list().find((entry) => entry.target.id === targetId);
+	if (status?.target.runtime === "litellm" && status.health.lastCheckAt === null) {
+		await providers.probeTarget(targetId, { reasoning: false, ...(signal ? { signal } : {}) });
+	}
+	signal?.throwIfAborted();
+}

package/src/entry/compaction-prompt.ts

@@ -0,0 +1,57 @@
+import { constants } from "node:fs";
+import { open } from "node:fs/promises";
+import { resolve } from "node:path";
+
+const MAX_PROMPT_BYTES = 64 * 1024;
+
+/** Absolute paths remain supported; relative paths are based on the current session workspace. */
+export async function readCompactionSystemPrompt(
+	path: string | null | undefined,
+	workspace: string,
+): Promise<string | undefined> {
+	if (path === undefined || path === null) return undefined;
+	// Quote the configured path, not file contents. Escape invisible directional
+	// formatting too, and bound diagnostics for an invalid oversized filename.
+	const quotedPath = JSON.stringify(path).replace(
+		/[\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/gu,
+		(character) => `\\u{${character.codePointAt(0)?.toString(16)}}`,
+	);
+	const displayPath = quotedPath.length <= 1024 ? quotedPath : `${quotedPath.slice(0, 1024)}...`;
+	const failure = (reason: string) =>
+		new Error(
+			`context.compaction.systemPrompt ${displayPath} ${reason}; set a readable, nonempty UTF-8 file of at most ${MAX_PROMPT_BYTES} bytes (relative to the session workspace), or unset the override`,
+		);
+	if (!path.trim()) throw failure("path is empty");
+	let file: Awaited<ReturnType<typeof open>>;
+	try {
+		// Nonblocking open prevents a named pipe from hanging compaction before stat.
+		file = await open(resolve(workspace, path), constants.O_RDONLY | constants.O_NONBLOCK);
+	} catch {
+		throw failure("cannot be opened");
+	}
+	try {
+		if (!(await file.stat()).isFile()) throw failure("must refer to a regular file");
+		// Read only limit + 1, even if the file grows after stat. No unbounded readFile.
+		const buffer = Buffer.alloc(MAX_PROMPT_BYTES + 1);
+		let length = 0;
+		while (length < buffer.length) {
+			const { bytesRead } = await file.read(buffer, length, buffer.length - length, null);
+			if (bytesRead === 0) break;
+			length += bytesRead;
+		}
+		if (length > MAX_PROMPT_BYTES) throw failure("file is oversized");
+		let text: string;
+		try {
+			text = new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, length));
+		} catch {
+			throw failure("file is not valid UTF-8");
+		}
+		if (!text.trim()) throw failure("file is empty");
+		return text;
+	} catch (error) {
+		if (error instanceof Error && error.message.startsWith("context.compaction.systemPrompt ")) throw error;
+		throw failure("file cannot be read");
+	} finally {
+		await file.close();
+	}
+}