@iowarp/clio-coder 0.3.1 → 0.3.2

package/src/engine/apis/ollama-native.ts

@@ -15,7 +15,6 @@ import type {
 	Usage,
 } from "@earendil-works/pi-ai";
 import { createAssistantMessageEventStream } from "@earendil-works/pi-ai";
-import type { ApiProvider } from "@earendil-works/pi-ai/compat";
 import {
 	type ChatRequest,
 	type ChatResponse,
@@ -32,11 +31,13 @@ import {
 } from "../../domains/providers/index.js";
 import type { LocalModelQuirks, SamplingProfile } from "../../domains/providers/types/local-model-quirks.js";
 import { calculateEngineCost } from "../ai.js";
+import { createGemmaChannelFilter } from "../gemma-channel-filter.js";
 import { createSentinelStripper } from "../strip-tokenizer-sentinels.js";
 import { remainingContextMaxTokens } from "./output-budget.js";
 import { type ResidencyAdapter, reconcileResidency, residencyManagedFor } from "./residency.js";
 import type { ResidentModelInfo } from "./resident-models.js";
 import { mergeSamplingOverride } from "./sampling-overrides.js";
+import type { EngineApiProvider } from "./types.js";
 
 const REASONING_CHARS_PER_TOKEN = 4;
 
@@ -381,6 +382,8 @@ function runStream(
 			let hadToolCall = false;
 			let doneReason: string | undefined;
 			const sentinelStripper = createSentinelStripper();
+			const resolved = resolveModelRuntimeCapabilitiesForModel(model, thinkingLevel);
+			const channelFilter = resolved.family === "gemma-4" ? createGemmaChannelFilter() : null;
 			const closeActiveThinking = () => {
 				if (!activeThinking) return;
 				stream.push({
@@ -446,6 +449,22 @@ function runStream(
 					partial: output,
 				});
 			};
+			const routeText = (content: string) => {
+				if (!channelFilter) {
+					emitText(content);
+					return;
+				}
+				for (const segment of channelFilter.push(content)) {
+					if (segment.kind === "thinking") emitThinking(segment.content);
+					else emitText(segment.content);
+				}
+			};
+			const flushChannel = () => {
+				for (const segment of channelFilter?.flush() ?? []) {
+					if (segment.kind === "thinking") emitThinking(segment.content);
+					else emitText(segment.content);
+				}
+			};
 			for await (const chunk of iterator) {
 				const response = chunk as ChatResponse;
 				const msg = response.message;
@@ -453,15 +472,17 @@ function runStream(
 					emitThinking(msg.thinking);
 				}
 				if (msg?.content && msg.content.length > 0) {
-					emitText(msg.content);
+					routeText(msg.content);
 				}
 				if (msg?.tool_calls && msg.tool_calls.length > 0) {
+					flushChannel();
 					closeActiveText();
 					closeActiveThinking();
 					for (const raw of msg.tool_calls) emitToolCall(raw, output, stream);
 					hadToolCall = true;
 				}
 				if (response.done) {
+					flushChannel();
 					closeActiveText();
 					closeActiveThinking();
 					output.usage.input = response.prompt_eval_count ?? 0;
@@ -499,7 +520,7 @@ function stripReasoning(options: SimpleStreamOptions | undefined): StreamOptions
 	return rest;
 }
 
-export const ollamaNativeApiProvider: ApiProvider<"ollama-native"> = {
+export const ollamaNativeApiProvider: EngineApiProvider<"ollama-native"> = {
 	api: "ollama-native",
 	stream: (model, context, options) => runStream(model, context, options, fallbackThinkingLevel(model)),
 	streamSimple: (model, context, options?: SimpleStreamOptions) =>

package/src/engine/apis/openai-completions.ts

@@ -2,31 +2,40 @@ import {
 	type Api,
 	type AssistantMessage,
 	type AssistantMessageEvent,
+	type AssistantMessageEventStream,
 	type Context,
 	createAssistantMessageEventStream,
 	type Model,
 	type OpenAICompletionsOptions,
 	type SimpleStreamOptions,
 	type StreamOptions,
+	type ThinkingBudgets,
 	type ThinkingContent,
 	type Tool,
 	type Usage,
 } from "@earendil-works/pi-ai";
-import { type ApiProvider, streamOpenAICompletions, streamSimpleOpenAICompletions } from "@earendil-works/pi-ai/compat";
+import { openAICompletionsApi } from "@earendil-works/pi-ai/api/openai-completions.lazy";
 import {
 	type AppliedThinking,
 	type ResolvedModelRuntimeCapabilities,
 	reasoningClassForMechanism,
 	resolveModelRuntimeCapabilitiesForModel,
 } from "../../domains/providers/model-runtime-capabilities.js";
+import { lmStudioReasoningEffort } from "../../domains/providers/runtimes/common/lmstudio-http.js";
 import type { ThinkingLevel } from "../../domains/providers/types/capability-flags.js";
 import type { LocalModelQuirks, SamplingProfile } from "../../domains/providers/types/local-model-quirks.js";
+import type { LmStudioTargetSettings } from "../../domains/providers/types/target-descriptor.js";
+import { filterGemmaChannelStream } from "../gemma-channel-filter.js";
 import { HarmonyResponseParser } from "../harmony-response.js";
 import { createSentinelStripper, stripTokenizerSentinels } from "../strip-tokenizer-sentinels.js";
 import { ensureLlamaCppResidency } from "./llamacpp-residency.js";
+import { ensureLmStudioResidency } from "./lmstudio.js";
 import { LOCAL_TOOL_TURN_MAX_OUTPUT_TOKENS, remainingContextMaxTokens } from "./output-budget.js";
 import { residencyManagedFor } from "./residency.js";
 import { mergeSamplingOverride } from "./sampling-overrides.js";
+import type { EngineApiProvider } from "./types.js";
+
+const piOpenAICompletions = openAICompletionsApi();
 
 /**
  * Average characters-per-token for the English/code reasoning streams pi-ai
@@ -47,6 +56,9 @@ interface ClioRuntimeMetadata {
 		gateway?: boolean;
 		quirks?: LocalModelQuirks;
 		chatTemplateKwargsUnsupported?: boolean;
+		lmstudio?: LmStudioTargetSettings;
+		lmstudioReasoningOptions?: ReadonlyArray<string>;
+		lmstudioDefaultModel?: string;
 	};
 }
 
@@ -71,33 +83,30 @@ function isPlainRecord(value: unknown): value is Record<string, unknown> {
 	return value !== null && typeof value === "object" && !Array.isArray(value);
 }
 
-/**
- * Install catalog sampling quirks on the OpenAI-compat request body. Standard
- * OpenAI fields (`temperature`, `top_p`, `presence_penalty`,
- * `frequency_penalty`) and LM Studio's extra fields (`top_k`, `min_p`,
- * `repeat_penalty`) are added only when the request body does not already
- * carry them. This keeps explicit StreamOptions and any user-supplied
- * `onPayload` overrides authoritative.
- */
-function applyOpenAISamplingProfile(
-	payload: Record<string, unknown>,
-	profile: SamplingProfile,
-): Record<string, unknown> {
-	const next: Record<string, unknown> = { ...payload };
-	if (profile.temperature !== undefined && next.temperature === undefined) next.temperature = profile.temperature;
-	if (profile.topP !== undefined && next.top_p === undefined) next.top_p = profile.topP;
-	if (profile.topK !== undefined && next.top_k === undefined) next.top_k = profile.topK;
-	if (profile.minP !== undefined && next.min_p === undefined) next.min_p = profile.minP;
-	if (profile.repeatPenalty !== undefined && next.repeat_penalty === undefined)
-		next.repeat_penalty = profile.repeatPenalty;
-	if (profile.presencePenalty !== undefined && next.presence_penalty === undefined)
-		next.presence_penalty = profile.presencePenalty;
-	if (profile.frequencyPenalty !== undefined && next.frequency_penalty === undefined)
-		next.frequency_penalty = profile.frequencyPenalty;
-	return next;
+function samplingParamsFromProfile(profile: SamplingProfile): Record<string, unknown> {
+	return {
+		...(profile.topP !== undefined ? { top_p: profile.topP } : {}),
+		...(profile.topK !== undefined ? { top_k: profile.topK } : {}),
+		...(profile.minP !== undefined ? { min_p: profile.minP } : {}),
+		...(profile.repeatPenalty !== undefined ? { repeat_penalty: profile.repeatPenalty } : {}),
+		...(profile.presencePenalty !== undefined ? { presence_penalty: profile.presencePenalty } : {}),
+		...(profile.frequencyPenalty !== undefined ? { frequency_penalty: profile.frequencyPenalty } : {}),
+	};
+}
+
+function piThinkingBudgets(quirks: LocalModelQuirks | undefined): ThinkingBudgets | undefined {
+	const budgets = quirks?.thinking?.budgetByLevel;
+	if (!budgets) return undefined;
+	return {
+		...(budgets.minimal !== undefined ? { minimal: budgets.minimal } : {}),
+		...(budgets.low !== undefined ? { low: budgets.low } : {}),
+		...(budgets.medium !== undefined ? { medium: budgets.medium } : {}),
+		...(budgets.high !== undefined ? { high: budgets.high } : {}),
+	};
 }
 
 type AnyOnPayload = (payload: unknown, model: Model<Api>) => unknown | undefined | Promise<unknown | undefined>;
+type StreamOptionsWithThinkingBudgets = StreamOptions & { thinkingBudgets?: ThinkingBudgets };
 
 const THINKING_CHAT_TEMPLATE_KWARGS = new Set([
 	"enable_thinking",
@@ -156,7 +165,9 @@ function withStrippedPartial<TEvent extends AssistantMessageEvent>(event: TEvent
  * Apply thinking-mechanism payload mutations to an openai-compat request body
  * after the catalog sampler is in place. Each mechanism owns the wire fields
  * it touches:
- *   - `effort-levels` writes `reasoning_effort` when the family resolved one.
+ *   - `effort-levels` writes `reasoning_effort` when the family resolved one;
+ *     off also carries `chat_template_kwargs.enable_thinking=false` for strict
+ *     templates whose effort vocabulary has no off value.
  *   - `budget-tokens` writes a vendor-specific budget object when the family
  *     declares a `thinkingFormat` of `anthropic-extended`; otherwise the
  *     budget remains informational and surfaces through the prompt only.
@@ -189,7 +200,8 @@ function applyThinkingPayload(
 		applied.mechanism === "budget-tokens" &&
 		resolved.request.budgetTokens !== undefined &&
 		next.thinking === undefined &&
-		resolved.request.budgetEnforcement === "enforced"
+		resolved.request.budgetEnforcement === "enforced" &&
+		resolved.runtimeId !== "vllm"
 	) {
 		// Only vendors whose openai-compat surface advertises a structured
 		// thinking budget (e.g. anthropic-extended on routed providers) get
@@ -201,6 +213,44 @@ function applyThinkingPayload(
 	return next;
 }
 
+function isLmStudioModel(model: Model<Api>): boolean {
+	const metadata = runtimeMetadata(model);
+	return model.provider === "lmstudio" && metadata?.runtimeId === "lmstudio";
+}
+
+function applyLmStudioPayload(
+	payload: Record<string, unknown>,
+	model: Model<Api>,
+	resolved: ResolvedModelRuntimeCapabilities,
+): Record<string, unknown> {
+	if (!isLmStudioModel(model)) return payload;
+	const request = runtimeMetadata(model)?.lmstudio?.request;
+	const next: Record<string, unknown> = { ...payload };
+	delete next.chat_template_kwargs;
+	if (request?.ttlSeconds !== undefined) next.ttl = request.ttlSeconds;
+	if (request?.draftModel !== undefined) next.draft_model = request.draftModel;
+	if (resolved.thinking.mechanism === "none" || resolved.thinking.mechanism === "always-on") return next;
+	const resolvedEffort =
+		resolved.request.reasoningEffort ??
+		lmStudioReasoningEffort(resolved.thinking.effectiveLevel, runtimeMetadata(model)?.lmstudioReasoningOptions);
+	switch (request?.reasoning) {
+		case "off":
+			next.reasoning_effort = "none";
+			break;
+		case "on":
+			next.reasoning_effort = "low";
+			break;
+		case "low":
+		case "medium":
+		case "high":
+			next.reasoning_effort = lmStudioReasoningEffort(request.reasoning, runtimeMetadata(model)?.lmstudioReasoningOptions);
+			break;
+		default:
+			next.reasoning_effort = resolvedEffort;
+	}
+	return next;
+}
+
 function applyLlamaCppPromptCachePayload(payload: Record<string, unknown>, model: Model<Api>): Record<string, unknown> {
 	const metadata = runtimeMetadata(model);
 	if (model.provider !== "llamacpp" || metadata?.runtimeId !== "llamacpp") return payload;
@@ -213,35 +263,7 @@ function shouldApplyLlamaCppPromptCache(model: Model<"openai-completions">): boo
 	return model.provider === "llamacpp" && metadata?.runtimeId === "llamacpp";
 }
 
-/**
- * Compose an `onPayload` hook over any caller-supplied one. Catalog overrides
- * apply first so the caller's hook sees the mutated body and can override or
- * inspect it. The result is returned only when the body actually changed
- * (pi-ai treats `undefined` as "use the original").
- */
-function composeSamplingOnPayload(
-	profile: SamplingProfile,
-	resolved: ResolvedModelRuntimeCapabilities | undefined,
-	base: AnyOnPayload | undefined,
-): AnyOnPayload {
-	return async (payload, model) => {
-		if (!isPlainRecord(payload)) {
-			return base ? await base(payload, model) : undefined;
-		}
-		let next = applyLlamaCppPromptCachePayload(applyOpenAISamplingProfile(payload, profile), model);
-		if (resolved) next = applyThinkingPayload(next, resolved.thinking, resolved, model);
-		if (base) {
-			const fromBase = await base(next, model);
-			if (fromBase !== undefined) return fromBase;
-		}
-		return next;
-	};
-}
-
-/**
- * Variant of `composeSamplingOnPayload` for cases where there is no catalog
- * sampler but we still need to inject thinking-mechanism fields.
- */
+/** Compose Clio-only payload deltas over the caller hook after pi applies sampling. */
 function composeThinkingOnPayload(
 	resolved: ResolvedModelRuntimeCapabilities,
 	base: AnyOnPayload | undefined,
@@ -250,11 +272,10 @@ function composeThinkingOnPayload(
 		if (!isPlainRecord(payload)) {
 			return base ? await base(payload, model) : undefined;
 		}
-		const next = applyThinkingPayload(
-			applyLlamaCppPromptCachePayload(payload, model),
-			resolved.thinking,
-			resolved,
+		const next = applyLmStudioPayload(
+			applyThinkingPayload(applyLlamaCppPromptCachePayload(payload, model), resolved.thinking, resolved, model),
 			model,
+			resolved,
 		);
 		if (base) {
 			const fromBase = await base(next, model);
@@ -270,11 +291,16 @@ function withSamplingOverrides<TOptions extends StreamOptions>(
 	resolved: ResolvedModelRuntimeCapabilities,
 ): TOptions | undefined {
 	const applied = resolved.thinking;
-	const profile = pickSamplingProfile(clioQuirks(model), applied.thinkingActive);
+	const quirks = clioQuirks(model);
+	const profile = pickSamplingProfile(quirks, applied.thinkingActive);
 	const promptCache = shouldApplyLlamaCppPromptCache(model);
+	const lmstudio = isLmStudioModel(model);
+	const vllmThinkingBudgets = resolved.runtimeId === "vllm" ? piThinkingBudgets(quirks) : undefined;
 	if (
 		!promptCache &&
+		!lmstudio &&
 		!profile &&
+		!vllmThinkingBudgets &&
 		applied.mechanism !== "effort-levels" &&
 		applied.mechanism !== "budget-tokens" &&
 		applied.mechanism !== "on-off" &&
@@ -285,17 +311,34 @@ function withSamplingOverrides<TOptions extends StreamOptions>(
 	const merged: Record<string, unknown> = { ...(options ?? {}) };
 	if (profile?.temperature !== undefined && merged.temperature === undefined) merged.temperature = profile.temperature;
 	if (profile) {
-		merged.onPayload = composeSamplingOnPayload(profile, resolved, options?.onPayload);
-	} else {
+		merged.samplingParams = {
+			...samplingParamsFromProfile(profile),
+			...options?.samplingParams,
+		};
+	}
+	if (vllmThinkingBudgets) {
+		merged.thinkingBudgets = {
+			...vllmThinkingBudgets,
+			...(options as StreamOptionsWithThinkingBudgets | undefined)?.thinkingBudgets,
+		};
+	}
+	if (
+		promptCache ||
+		lmstudio ||
+		applied.mechanism === "effort-levels" ||
+		applied.mechanism === "budget-tokens" ||
+		applied.mechanism === "on-off" ||
+		applied.mechanism === "none"
+	) {
 		merged.onPayload = composeThinkingOnPayload(resolved, options?.onPayload);
 	}
 	return merged as TOptions;
 }
 
 function stripNeverReasoningFromStream(
-	source: ReturnType<typeof streamOpenAICompletions>,
+	source: AssistantMessageEventStream,
 	resolved: ResolvedModelRuntimeCapabilities,
-): ReturnType<typeof streamOpenAICompletions> {
+): AssistantMessageEventStream {
 	if (!stripsThinking(resolved)) return source;
 	const stripped = createAssistantMessageEventStream();
 	(async () => {
@@ -399,7 +442,7 @@ function malformedToolArgsMessage(
 	const workaround =
 		model.provider === "llamacpp" || runtime === "llamacpp"
 			? "For llama.cpp, verify --jinja, the model chat template, reasoning flags, and tool parser support for this model."
-			: "For LM Studio, use the verified openai-compat gateway fallback when native SDK tool extraction is unreliable.";
+			: "For LM Studio, verify the model's tool-use capability and active chat template in LM Studio.";
 	return `OpenAI-compatible runtime returned empty tool-call arguments for target '${target}' model '${model.id}' tool '${toolName}'.${required} ${workaround}`;
 }
 
@@ -450,15 +493,18 @@ function hasReportedReasoningUsage(usage: Usage): boolean {
  */
 export function applyOpenAICompatReasoningEstimate(message: AssistantMessage): void {
 	if (hasReportedReasoningUsage(message.usage)) return;
-	const reasoningTokens = estimateReasoningTokens(message.content);
+	const estimated = estimateReasoningTokens(message.content);
+	const reportedOutput = message.usage.output;
+	const reasoningTokens =
+		typeof reportedOutput === "number" && Number.isFinite(reportedOutput)
+			? Math.min(estimated, Math.max(0, reportedOutput))
+			: estimated;
 	if (reasoningTokens > 0) {
 		(message.usage as UsageWithReasoningAliases).reasoningTokens = reasoningTokens;
 	}
 }
 
-function withReasoningTokenEstimate(
-	source: ReturnType<typeof streamOpenAICompletions>,
-): ReturnType<typeof streamOpenAICompletions> {
+function withReasoningTokenEstimate(source: AssistantMessageEventStream): AssistantMessageEventStream {
 	const annotated = createAssistantMessageEventStream();
 	(async () => {
 		try {
@@ -488,9 +534,9 @@ function withReasoningTokenEstimate(
  * tool-call events pass through unchanged.
  */
 function stripSentinelsFromStream(
-	source: ReturnType<typeof streamOpenAICompletions>,
+	source: AssistantMessageEventStream,
 	resolved: ResolvedModelRuntimeCapabilities,
-): ReturnType<typeof streamOpenAICompletions> {
+): AssistantMessageEventStream {
 	const sanitized = createAssistantMessageEventStream();
 	(async () => {
 		try {
@@ -576,10 +622,10 @@ function stripSentinelsFromStream(
 }
 
 function guardMalformedToolCalls(
-	source: ReturnType<typeof streamOpenAICompletions>,
+	source: AssistantMessageEventStream,
 	model: Model<"openai-completions">,
 	context: Context,
-): ReturnType<typeof streamOpenAICompletions> {
+): AssistantMessageEventStream {
 	const requiredByTool = new Map<string, ReadonlyArray<string>>();
 	for (const tool of context.tools ?? []) {
 		const required = requiredToolArguments(tool);
@@ -638,16 +684,29 @@ async function ensureResidencyForModel(model: Model<"openai-completions">): Prom
 	});
 }
 
-function withLlamaCppResidency(
+async function ensureLocalResidency(
+	model: Model<"openai-completions">,
+	options: { apiKey?: string; signal?: AbortSignal },
+): Promise<Model<"openai-completions">> {
+	if (isLmStudioModel(model)) {
+		const wireModelId = await ensureLmStudioResidency(model, options);
+		return wireModelId === model.id ? model : { ...model, id: wireModelId };
+	}
+	await ensureResidencyForModel(model);
+	return model;
+}
+
+function withLocalResidency(
 	model: Model<"openai-completions">,
-	sourceFactory: () => ReturnType<typeof streamOpenAICompletions>,
-): ReturnType<typeof streamOpenAICompletions> {
-	if (!isManagedLlamaCppModel(model)) return sourceFactory();
+	options: { apiKey?: string; signal?: AbortSignal },
+	sourceFactory: (requestModel: Model<"openai-completions">) => AssistantMessageEventStream,
+): AssistantMessageEventStream {
+	if (!isManagedLlamaCppModel(model) && !isLmStudioModel(model)) return sourceFactory(model);
 	const stream = createAssistantMessageEventStream();
 	(async () => {
 		try {
-			await ensureResidencyForModel(model);
-			for await (const event of sourceFactory()) {
+			const requestModel = await ensureLocalResidency(model, options);
+			for await (const event of sourceFactory(requestModel)) {
 				stream.push(event as AssistantMessageEvent);
 			}
 			stream.end();
@@ -661,24 +720,35 @@ function withLlamaCppResidency(
 	return stream;
 }
 
-export const openAICompletionsApiProvider: ApiProvider<"openai-completions", OpenAICompletionsOptions> = {
+export const openAICompletionsApiProvider: EngineApiProvider<"openai-completions", OpenAICompletionsOptions> = {
 	api: "openai-completions",
 	stream: (model, context, options) => {
 		// Bare `stream` callers don't communicate thinking state; fall back to
-		// the model's reasoning capability so the catalog still applies.
-		const resolved = resolvedCapabilitiesForModel(model, model.reasoning === true ? "medium" : "off");
+		// the model's reasoning capability so the catalog still applies. LM Studio
+		// is the exception because its HTTP server defaults to thinking off.
+		const defaultLevel = isLmStudioModel(model) ? "off" : model.reasoning === true ? "medium" : "off";
+		const resolved = resolvedCapabilitiesForModel(model, defaultLevel);
 		const effectiveContext = stripsThinking(resolved) ? stripThinkingFromContext(context) : context;
 		const withSamplers = withSamplingOverrides(model, options, resolved);
 		return guardMalformedToolCalls(
 			withReasoningTokenEstimate(
 				stripSentinelsFromStream(
 					stripNeverReasoningFromStream(
-						withLlamaCppResidency(model, () =>
-							streamOpenAICompletions(
+						filterGemmaChannelStream(
+							withLocalResidency(
 								model,
-								effectiveContext,
-								withRemainingContextBudget(model, effectiveContext, withSamplers),
+								{
+									...(options?.apiKey !== undefined ? { apiKey: options.apiKey } : {}),
+									...(options?.signal !== undefined ? { signal: options.signal } : {}),
+								},
+								(requestModel) =>
+									piOpenAICompletions.stream(
+										requestModel,
+										effectiveContext,
+										withRemainingContextBudget(model, effectiveContext, withSamplers),
+									),
 							),
+							resolved.family === "gemma-4",
 						),
 						resolved,
 					),
@@ -697,12 +767,21 @@ export const openAICompletionsApiProvider: ApiProvider<"openai-completions", Ope
 			withReasoningTokenEstimate(
 				stripSentinelsFromStream(
 					stripNeverReasoningFromStream(
-						withLlamaCppResidency(model, () =>
-							streamSimpleOpenAICompletions(
+						filterGemmaChannelStream(
+							withLocalResidency(
 								model,
-								effectiveContext,
-								withRemainingContextBudget(model, effectiveContext, withSamplers),
+								{
+									...(options?.apiKey !== undefined ? { apiKey: options.apiKey } : {}),
+									...(options?.signal !== undefined ? { signal: options.signal } : {}),
+								},
+								(requestModel) =>
+									piOpenAICompletions.streamSimple(
+										requestModel,
+										effectiveContext,
+										withRemainingContextBudget(model, effectiveContext, withSamplers),
+									),
 							),
+							resolved.family === "gemma-4",
 						),
 						resolved,
 					),