@oh-my-pi/pi-ai 14.7.1 → 14.7.3

package/CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## [Unreleased]
 
+## [14.7.2] - 2026-05-06
+
+### Fixed
+
+- Fixed VLLM model discovery to use `max_model_len` as the context window when the endpoint reports it.
+- Fixed custom Ollama Cloud/local-proxy model aliases (for example `deepseek-v4-pro:cloud`) to inherit bundled cache-pricing metadata when the upstream model is known ([#937](https://github.com/can1357/oh-my-pi/issues/937)).
+- Fixed local Ollama model discovery to apply `/api/show` thinking and vision capabilities in addition to native context windows ([#928](https://github.com/can1357/oh-my-pi/issues/928)).
+
 ## [14.7.0] - 2026-05-04
 ### Breaking Changes
 

package/package.json

@@ -1,7 +1,7 @@
 {
 	"type": "module",
 	"name": "@oh-my-pi/pi-ai",
-	"version": "14.7.1",
+	"version": "14.7.3",
 	"description": "Unified LLM API with automatic model discovery and provider configuration",
 	"homepage": "https://github.com/can1357/oh-my-pi",
 	"author": "Can Boluk",
@@ -46,8 +46,8 @@
 		"@aws-sdk/credential-provider-node": "^3.972.39",
 		"@bufbuild/protobuf": "^2.12.0",
 		"@google/genai": "^1.52.0",
-		"@oh-my-pi/pi-natives": "14.7.1",
-		"@oh-my-pi/pi-utils": "14.7.1",
+		"@oh-my-pi/pi-natives": "14.7.3",
+		"@oh-my-pi/pi-utils": "14.7.3",
 		"@sinclair/typebox": "^0.34.49",
 		"@smithy/node-http-handler": "^4.6.1",
 		"ajv": "^8.20.0",

package/src/provider-models/openai-compat.ts

@@ -1,6 +1,7 @@
 import type { ModelManagerOptions } from "../model-manager";
+import { Effort } from "../model-thinking";
 import { getBundledModels } from "../models";
-import type { Api, Model } from "../types";
+import type { Api, Model, ThinkingConfig } from "../types";
 import { isAnthropicOAuthToken, isRecord, toBoolean, toNumber, toPositiveNumber } from "../utils";
 import {
 	fetchOpenAICompatibleModels,
@@ -192,7 +193,7 @@ function toOllamaNativeBaseUrl(baseUrl: string): string {
 
 async function fetchOllamaNativeModels(
 	baseUrl: string,
-	resolveLimits: (modelId: string) => Promise<OllamaModelLimits>,
+	resolveMetadata: (modelId: string) => Promise<OllamaResolvedMetadata>,
 ): Promise<Model<"openai-responses">[] | null> {
 	const nativeBaseUrl = toOllamaNativeBaseUrl(baseUrl);
 	let response: Response;
@@ -213,18 +214,19 @@ async function fetchOllamaNativeModels(
 		entries.map(async (entry): Promise<Model<"openai-responses"> | null> => {
 			const id = entry.model ?? entry.name;
 			if (!id) return null;
-			const { contextWindow, maxTokens } = await resolveLimits(id);
+			const metadata = await resolveMetadata(id);
 			return {
 				id,
 				name: entry.name ?? id,
 				api: "openai-responses",
 				provider: "ollama",
 				baseUrl,
-				reasoning: false,
-				input: ["text"],
+				reasoning: metadata.reasoning ?? false,
+				thinking: metadata.thinking,
+				input: metadata.input ?? ["text"],
 				cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
-				contextWindow,
-				maxTokens,
+				contextWindow: metadata.contextWindow,
+				maxTokens: metadata.maxTokens,
 			};
 		}),
 	);
@@ -241,18 +243,65 @@ const OLLAMA_FALLBACK_CONTEXT_WINDOW = 128_000;
 /** Cap max output tokens at a value that matches OMP's other openai-responses defaults. */
 const OLLAMA_DEFAULT_MAX_TOKENS = 8192;
 
-interface OllamaModelLimits {
+interface OllamaResolvedMetadata {
 	contextWindow: number;
 	maxTokens: number;
+	capabilities?: string[];
+	reasoning?: boolean;
+	thinking?: ThinkingConfig;
+	input?: ("text" | "image")[];
+}
+
+interface OllamaShowMetadata {
+	contextWindow?: number;
+	maxTokens?: number;
+	capabilities?: string[];
+	reasoning?: boolean;
+	thinking?: ThinkingConfig;
+	input?: ("text" | "image")[];
+}
+
+function getOllamaContextWindow(modelInfo: Record<string, unknown> | undefined): number | undefined {
+	if (!modelInfo) {
+		return undefined;
+	}
+	for (const [key, value] of Object.entries(modelInfo)) {
+		if (typeof value !== "number" || value <= 0) {
+			continue;
+		}
+		if (key.endsWith(".context_length") || key.endsWith(".num_ctx") || key.endsWith(".context_window")) {
+			return value;
+		}
+	}
+}
+
+function getOllamaCapabilities(value: unknown): string[] | undefined {
+	if (!Array.isArray(value)) {
+		return undefined;
+	}
+	return value.filter((item): item is string => typeof item === "string");
+}
+
+function getOllamaThinkingConfig(capabilities: string[] | undefined): ThinkingConfig | undefined {
+	if (!capabilities?.includes("thinking")) {
+		return undefined;
+	}
+	return {
+		mode: "effort",
+		minLevel: Effort.Minimal,
+		maxLevel: Effort.High,
+	};
 }
 
 /**
- * Query Ollama's `/api/show` endpoint for a single model and pull its native
- * context length out of `model_info.<arch>.context_length`. Returns the
- * discovered limits, or `undefined` when the endpoint or field is
- * unavailable so callers can layer their own fallback.
+ * Query Ollama's `/api/show` endpoint for a single model and pull native
+ * context and capability metadata from the response. Returns `undefined` when
+ * the endpoint is unavailable so callers can layer their own fallback.
  */
-async function fetchOllamaShowLimits(nativeBaseUrl: string, modelId: string): Promise<OllamaModelLimits | undefined> {
+async function fetchOllamaShowMetadata(
+	nativeBaseUrl: string,
+	modelId: string,
+): Promise<OllamaShowMetadata | undefined> {
 	try {
 		const response = await fetch(`${nativeBaseUrl}/api/show`, {
 			method: "POST",
@@ -262,13 +311,21 @@ async function fetchOllamaShowLimits(nativeBaseUrl: string, modelId: string): Pr
 		if (!response.ok) {
 			return undefined;
 		}
-		const payload = (await response.json()) as { model_info?: Record<string, unknown> };
-		const info = payload.model_info ?? {};
-		for (const [key, value] of Object.entries(info)) {
-			if (key.endsWith(".context_length") && typeof value === "number" && value > 0) {
-				return { contextWindow: value, maxTokens: OLLAMA_DEFAULT_MAX_TOKENS };
-			}
-		}
+		const payload = (await response.json()) as { capabilities?: unknown; model_info?: Record<string, unknown> };
+		const capabilities = getOllamaCapabilities(payload.capabilities);
+		const contextWindow = getOllamaContextWindow(payload.model_info);
+		return {
+			contextWindow,
+			maxTokens: contextWindow ? OLLAMA_DEFAULT_MAX_TOKENS : undefined,
+			capabilities,
+			reasoning: capabilities ? capabilities.includes("thinking") : undefined,
+			thinking: getOllamaThinkingConfig(capabilities),
+			input: capabilities
+				? capabilities.includes("vision")
+					? (["text", "image"] as Array<"text" | "image">)
+					: (["text"] as Array<"text">)
+				: undefined,
+		};
 	} catch {
 		// fall through; caller decides on the fallback
 	}
@@ -276,23 +333,27 @@ async function fetchOllamaShowLimits(nativeBaseUrl: string, modelId: string): Pr
 }
 
 /**
- * Build a resolver that fetches `/api/show` limits per model id and caches the
- * result in-memory for the lifetime of the manager. Successful lookups are
+ * Build a resolver that fetches `/api/show` metadata per model id and caches
+ * the result in-memory for the lifetime of the manager. Successful lookups are
  * cached so repeated `fetchDynamicModels` calls do not refetch; failed
  * lookups stay uncached so a later refresh can recover.
  */
-function createOllamaLimitsResolver(nativeBaseUrl: string): (modelId: string) => Promise<OllamaModelLimits> {
-	const cache = new Map<string, Promise<OllamaModelLimits>>();
+function createOllamaMetadataResolver(nativeBaseUrl: string): (modelId: string) => Promise<OllamaResolvedMetadata> {
+	const cache = new Map<string, Promise<OllamaResolvedMetadata>>();
 	return modelId => {
 		const cached = cache.get(modelId);
 		if (cached) return cached;
 		const pending = (async () => {
-			const limits = await fetchOllamaShowLimits(nativeBaseUrl, modelId);
-			if (!limits) {
+			const metadata = await fetchOllamaShowMetadata(nativeBaseUrl, modelId);
+			if (!metadata) {
 				cache.delete(modelId);
 				return { contextWindow: OLLAMA_FALLBACK_CONTEXT_WINDOW, maxTokens: OLLAMA_DEFAULT_MAX_TOKENS };
 			}
-			return limits;
+			return {
+				...metadata,
+				contextWindow: metadata.contextWindow ?? OLLAMA_FALLBACK_CONTEXT_WINDOW,
+				maxTokens: metadata.maxTokens ?? OLLAMA_DEFAULT_MAX_TOKENS,
+			};
 		})();
 		cache.set(modelId, pending);
 		void pending.catch(() => cache.delete(modelId));
@@ -702,7 +763,7 @@ export function ollamaModelManagerOptions(config?: OllamaModelManagerConfig): Mo
 	const baseUrl = normalizeOllamaBaseUrl(config?.baseUrl);
 	const nativeBaseUrl = toOllamaNativeBaseUrl(baseUrl);
 	const references = createBundledReferenceMap<"openai-responses">("ollama" as Parameters<typeof getBundledModels>[0]);
-	const resolveLimits = createOllamaLimitsResolver(nativeBaseUrl);
+	const resolveMetadata = createOllamaMetadataResolver(nativeBaseUrl);
 	return {
 		providerId: "ollama",
 		fetchDynamicModels: async () => {
@@ -727,13 +788,20 @@ export function ollamaModelManagerOptions(config?: OllamaModelManagerConfig): Mo
 			if (openAiCompatible && openAiCompatible.length > 0) {
 				await Promise.all(
 					openAiCompatible.map(async model => {
-						const limits = await resolveLimits(model.id);
-						model.contextWindow = limits.contextWindow;
+						const metadata = await resolveMetadata(model.id);
+						model.contextWindow = metadata.contextWindow;
+						if (metadata.reasoning !== undefined) {
+							model.reasoning = metadata.reasoning;
+							model.thinking = metadata.thinking;
+						}
+						if (metadata.input) {
+							model.input = metadata.input;
+						}
 					}),
 				);
 				return openAiCompatible;
 			}
-			const nativeFallback = await fetchOllamaNativeModels(baseUrl, resolveLimits);
+			const nativeFallback = await fetchOllamaNativeModels(baseUrl, resolveMetadata);
 			if (nativeFallback && nativeFallback.length > 0) {
 				return nativeFallback;
 			}
@@ -1407,8 +1475,11 @@ export function vllmModelManagerOptions(config?: VllmModelManagerConfig): ModelM
 				baseUrl,
 				apiKey,
 				mapModel: (entry, defaults) => {
-					const reference = references.get(defaults.id);
-					return mapWithBundledReference(entry, defaults, reference);
+					const model = mapWithBundledReference(entry, defaults, references.get(defaults.id));
+					return {
+						...model,
+						contextWindow: toPositiveNumber(entry.max_model_len, model.contextWindow),
+					};
 				},
 			}),
 	};

package/src/providers/anthropic.ts

@@ -8,7 +8,7 @@ import type {
 	MessageParam,
 	RawMessageStreamEvent,
 } from "@anthropic-ai/sdk/resources/messages";
-import { $env, abortableSleep, isEnoent } from "@oh-my-pi/pi-utils";
+import { $env, abortableSleep, isEnoent, readSseEvents } from "@oh-my-pi/pi-utils";
 import { hasOpus47ApiRestrictions, mapEffortToAnthropicAdaptiveEffort } from "../model-thinking";
 import { calculateCost } from "../models";
 import { getEnvApiKey, OUTPUT_FALLBACK_BUFFER } from "../stream";
@@ -658,18 +658,6 @@ function mergeHeaders(...headerSources: (Record<string, string> | undefined)[]):
 // We surface the resulting provider error ourselves, so keep the SDK quiet.
 const ANTHROPIC_SDK_LOG_LEVEL = "off" as const;
 
-interface ServerSentEvent {
-	event: string | null;
-	data: string;
-	raw: string[];
-}
-
-interface SseDecoderState {
-	event: string | null;
-	data: string[];
-	raw: string[];
-}
-
 const ANTHROPIC_MESSAGE_EVENTS: ReadonlySet<string> = new Set([
 	"message_start",
 	"message_delta",
@@ -679,136 +667,6 @@ const ANTHROPIC_MESSAGE_EVENTS: ReadonlySet<string> = new Set([
 	"content_block_stop",
 ]);
 
-function flushSseEvent(state: SseDecoderState): ServerSentEvent | null {
-	if (!state.event && state.data.length === 0) {
-		return null;
-	}
-
-	const event: ServerSentEvent = {
-		event: state.event,
-		data: state.data.join("\n"),
-		raw: [...state.raw],
-	};
-	state.event = null;
-	state.data = [];
-	state.raw = [];
-	return event;
-}
-
-function decodeSseLine(line: string, state: SseDecoderState): ServerSentEvent | null {
-	if (line === "") {
-		return flushSseEvent(state);
-	}
-
-	state.raw.push(line);
-	if (line.startsWith(":")) {
-		return null;
-	}
-
-	const delimiterIndex = line.indexOf(":");
-	const fieldName = delimiterIndex === -1 ? line : line.slice(0, delimiterIndex);
-	let value = delimiterIndex === -1 ? "" : line.slice(delimiterIndex + 1);
-	if (value.startsWith(" ")) {
-		value = value.slice(1);
-	}
-
-	if (fieldName === "event") {
-		state.event = value;
-	} else if (fieldName === "data") {
-		state.data.push(value);
-	}
-
-	return null;
-}
-
-function nextLineBreakIndex(text: string): number {
-	const carriageReturnIndex = text.indexOf("\r");
-	const newlineIndex = text.indexOf("\n");
-	if (carriageReturnIndex === -1) {
-		return newlineIndex;
-	}
-	if (newlineIndex === -1) {
-		return carriageReturnIndex;
-	}
-	return Math.min(carriageReturnIndex, newlineIndex);
-}
-
-function consumeLine(text: string): { line: string; rest: string } | null {
-	const lineBreakIndex = nextLineBreakIndex(text);
-	if (lineBreakIndex === -1) {
-		return null;
-	}
-
-	let nextIndex = lineBreakIndex + 1;
-	if (text[lineBreakIndex] === "\r" && text[nextIndex] === "\n") {
-		nextIndex += 1;
-	}
-
-	return {
-		line: text.slice(0, lineBreakIndex),
-		rest: text.slice(nextIndex),
-	};
-}
-
-async function* iterateSseMessages(
-	body: ReadableStream<Uint8Array>,
-	signal?: AbortSignal,
-): AsyncGenerator<ServerSentEvent> {
-	const reader = body.getReader();
-	const decoder = new TextDecoder();
-	const state: SseDecoderState = { event: null, data: [], raw: [] };
-	let buffer = "";
-
-	try {
-		while (true) {
-			if (signal?.aborted) {
-				throw new Error("Request was aborted");
-			}
-
-			const { value, done } = await reader.read();
-			if (done) {
-				break;
-			}
-
-			buffer += decoder.decode(value, { stream: true });
-			let consumed = consumeLine(buffer);
-			while (consumed) {
-				buffer = consumed.rest;
-				const event = decodeSseLine(consumed.line, state);
-				if (event) {
-					yield event;
-				}
-				consumed = consumeLine(buffer);
-			}
-		}
-
-		buffer += decoder.decode();
-		let consumed = consumeLine(buffer);
-		while (consumed) {
-			buffer = consumed.rest;
-			const event = decodeSseLine(consumed.line, state);
-			if (event) {
-				yield event;
-			}
-			consumed = consumeLine(buffer);
-		}
-
-		if (buffer.length > 0) {
-			const event = decodeSseLine(buffer, state);
-			if (event) {
-				yield event;
-			}
-		}
-
-		const trailingEvent = flushSseEvent(state);
-		if (trailingEvent) {
-			yield trailingEvent;
-		}
-	} finally {
-		reader.releaseLock();
-	}
-}
-
 async function* iterateAnthropicEvents(
 	response: Response,
 	signal?: AbortSignal,
@@ -820,7 +678,7 @@ async function* iterateAnthropicEvents(
 	let sawMessageStart = false;
 	let sawMessageEnd = false;
 
-	for await (const sse of iterateSseMessages(response.body, signal)) {
+	for await (const sse of readSseEvents(response.body, signal)) {
 		if (sse.event === "error") {
 			throw new Error(sse.data);
 		}

package/src/utils/event-stream.ts

@@ -105,23 +105,7 @@ export class EventStream<T, R = T> implements AsyncIterable<T> {
 	}
 }
 
-// Delta events that can be batched for throttling
-type DeltaEvent =
-	| { type: "text_delta"; contentIndex: number; delta: string; partial: AssistantMessage }
-	| { type: "thinking_delta"; contentIndex: number; delta: string; partial: AssistantMessage }
-	| { type: "toolcall_delta"; contentIndex: number; delta: string; partial: AssistantMessage };
-
-function isDeltaEvent(event: AssistantMessageEvent): event is DeltaEvent {
-	return event.type === "text_delta" || event.type === "thinking_delta" || event.type === "toolcall_delta";
-}
-
 export class AssistantMessageEventStream extends EventStream<AssistantMessageEvent, AssistantMessage> {
-	// Throttling state
-	#deltaBuffer: DeltaEvent[] = [];
-	#flushTimer?: NodeJS.Timeout;
-	#lastFlushTime = 0;
-	readonly #throttleMs = 50; // 20 updates/sec
-
 	constructor() {
 		super(
 			event => event.type === "done" || event.type === "error",
@@ -139,103 +123,20 @@ export class AssistantMessageEventStream extends EventStream<AssistantMessageEve
 	override push(event: AssistantMessageEvent): void {
 		if (this.done) return;
 
-		// Check for completion first
+		// Completion resolves the final result and still emits the terminal event.
 		if (this.isComplete(event)) {
-			this.#flushDeltas(); // Flush any pending deltas before completing
 			this.done = true;
 			this.resolveFinalResult(this.extractResult(event));
 		}
 
-		// Delta events get batched and throttled
-		if (isDeltaEvent(event)) {
-			this.#deltaBuffer.push(event);
-			this.#scheduleFlush();
-			return;
-		}
-
-		// Non-delta events flush pending deltas immediately, then emit
-		this.#flushDeltas();
 		this.deliver(event);
 	}
 
 	override end(result?: AssistantMessage): void {
-		this.#flushDeltas();
 		this.done = true;
 		if (result !== undefined) {
 			this.resolveFinalResult(result);
 		}
 		this.endWaiting();
 	}
-
-	override fail(err: unknown): void {
-		if (this.#flushTimer) {
-			clearTimeout(this.#flushTimer);
-			this.#flushTimer = undefined;
-		}
-		this.#deltaBuffer = [];
-		super.fail(err);
-	}
-
-	#scheduleFlush(): void {
-		if (this.#flushTimer) return; // Already scheduled
-
-		const now = Bun.nanoseconds();
-		const timeSinceLastFlush = (now - this.#lastFlushTime) / 1e6;
-
-		if (timeSinceLastFlush >= this.#throttleMs) {
-			// Flush immediately if throttle window has passed
-			this.#flushDeltas();
-		} else {
-			// Schedule flush for when throttle window expires
-			const delay = this.#throttleMs - timeSinceLastFlush;
-			this.#flushTimer = setTimeout(() => {
-				this.#flushTimer = undefined;
-				this.#flushDeltas();
-			}, delay);
-		}
-	}
-
-	#flushDeltas(): void {
-		if (this.#flushTimer) {
-			clearTimeout(this.#flushTimer);
-			this.#flushTimer = undefined;
-		}
-
-		if (this.#deltaBuffer.length === 0) return;
-
-		// Merge consecutive deltas for the same content block and type
-		const merged = this.#mergeDeltas(this.#deltaBuffer);
-		this.#deltaBuffer = [];
-		this.#lastFlushTime = Bun.nanoseconds();
-
-		for (const event of merged) {
-			this.deliver(event);
-		}
-	}
-
-	#mergeDeltas(deltas: DeltaEvent[]): AssistantMessageEvent[] {
-		if (deltas.length === 0) return [];
-		if (deltas.length === 1) return [deltas[0]];
-
-		const result: AssistantMessageEvent[] = [];
-		let current = deltas[0];
-
-		for (let i = 1; i < deltas.length; i++) {
-			const next = deltas[i];
-			// Can merge if same type, same content index
-			if (next.type === current.type && next.contentIndex === current.contentIndex) {
-				current = {
-					...current,
-					delta: current.delta + next.delta,
-					partial: next.partial, // Use latest partial
-				} as DeltaEvent;
-			} else {
-				result.push(current);
-				current = next;
-			}
-		}
-		result.push(current);
-
-		return result;
-	}
 }