@clinebot/llms 0.0.0

package/src/providers/handlers/openai-responses.ts

@@ -0,0 +1,598 @@
+/**
+ * OpenAI Responses API Handler
+ *
+ * Handler for OpenAI's Responses API format, which is used by newer models
+ * that require native tool calling (e.g., GPT-5, o3, codex).
+ *
+ * The Responses API has a different structure than Chat Completions:
+ * - Uses `instructions` instead of system messages
+ * - Uses `input` instead of messages array
+ * - Has different streaming event types (response.*, not choices.delta)
+ * - Supports reasoning with encrypted content
+ */
+
+import OpenAI from "openai";
+import type {
+	ApiStream,
+	HandlerModelInfo,
+	ModelInfo,
+	ProviderConfig,
+} from "../types";
+import type { Message, ToolDefinition } from "../types/messages";
+import { retryStream } from "../utils/retry";
+import { getMissingApiKeyError, resolveApiKeyForProvider } from "./auth";
+import { BaseHandler } from "./base";
+
+const DEFAULT_REASONING_EFFORT = "medium" as const;
+
+function normalizeStrictToolSchema(
+	schema: unknown,
+	options?: { stripFormat?: boolean },
+): unknown {
+	if (!schema || typeof schema !== "object" || Array.isArray(schema)) {
+		return schema;
+	}
+
+	const normalized = { ...(schema as Record<string, unknown>) };
+	if (options?.stripFormat && "format" in normalized) {
+		delete normalized.format;
+	}
+	const type = normalized.type;
+
+	if (type === "object") {
+		if (!Object.hasOwn(normalized, "additionalProperties")) {
+			normalized.additionalProperties = false;
+		}
+		const properties = normalized.properties;
+		if (
+			properties &&
+			typeof properties === "object" &&
+			!Array.isArray(properties)
+		) {
+			const nextProperties: Record<string, unknown> = {};
+			for (const [key, value] of Object.entries(
+				properties as Record<string, unknown>,
+			)) {
+				nextProperties[key] = normalizeStrictToolSchema(value, options);
+			}
+			normalized.properties = nextProperties;
+		}
+	}
+
+	if (Array.isArray(normalized.anyOf)) {
+		normalized.anyOf = normalized.anyOf.map((item) =>
+			normalizeStrictToolSchema(item, options),
+		);
+	}
+	if (Array.isArray(normalized.oneOf)) {
+		normalized.oneOf = normalized.oneOf.map((item) =>
+			normalizeStrictToolSchema(item, options),
+		);
+	}
+	if (Array.isArray(normalized.allOf)) {
+		normalized.allOf = normalized.allOf.map((item) =>
+			normalizeStrictToolSchema(item, options),
+		);
+	}
+	if (normalized.not && typeof normalized.not === "object") {
+		normalized.not = normalizeStrictToolSchema(normalized.not, options);
+	}
+	if (normalized.items) {
+		if (Array.isArray(normalized.items)) {
+			normalized.items = normalized.items.map((item) =>
+				normalizeStrictToolSchema(item, options),
+			);
+		} else {
+			normalized.items = normalizeStrictToolSchema(normalized.items, options);
+		}
+	}
+
+	return normalized;
+}
+
+/**
+ * Convert tool definitions to Responses API format
+ */
+function convertToolsToResponsesFormat(
+	tools?: ToolDefinition[],
+	options?: { stripFormat?: boolean },
+) {
+	if (!tools?.length) return undefined;
+
+	return tools.map((tool) => ({
+		type: "function" as const,
+		name: tool.name,
+		description: tool.description,
+		parameters: normalizeStrictToolSchema(tool.inputSchema, options),
+		strict: true, // Responses API defaults to strict mode
+	}));
+}
+
+/**
+ * Convert messages to Responses API input format
+ */
+function convertToResponsesInput(messages: Message[]) {
+	// Responses API uses a flat input array with specific item types
+	const input: Array<{
+		type: "message";
+		role: "user" | "assistant";
+		content:
+			| string
+			| Array<{ type: "input_text" | "output_text"; text: string }>;
+	}> = [];
+
+	for (const msg of messages) {
+		if (msg.role === "user" || msg.role === "assistant") {
+			// Handle content blocks
+			const contentBlocks = Array.isArray(msg.content)
+				? msg.content
+				: [{ type: "text" as const, text: msg.content }];
+
+			const textContent = contentBlocks
+				.filter(
+					(block): block is { type: "text"; text: string } =>
+						block.type === "text",
+				)
+				.map((block) => block.text)
+				.join("\n");
+
+			if (textContent) {
+				input.push({
+					type: "message",
+					role: msg.role,
+					content:
+						msg.role === "user"
+							? [{ type: "input_text", text: textContent }]
+							: [{ type: "output_text", text: textContent }],
+				});
+			}
+		}
+	}
+
+	return input;
+}
+
+/**
+ * Handler for OpenAI Responses API
+ *
+ * Uses ProviderConfig fields:
+ * - baseUrl: Base URL for the API
+ * - modelId: Model ID
+ * - knownModels: Known models with their info
+ * - headers: Custom headers
+ */
+export class OpenAIResponsesHandler extends BaseHandler {
+	protected client: OpenAI | undefined;
+
+	/**
+	 * Ensure the OpenAI client is initialized
+	 */
+	protected ensureClient(): OpenAI {
+		if (!this.client) {
+			const baseURL = this.config.baseUrl;
+
+			if (!baseURL) {
+				throw new Error("Base URL is required. Set baseUrl in config.");
+			}
+			const apiKey = resolveApiKeyForProvider(
+				this.config.providerId,
+				this.config.apiKey,
+			);
+			if (!apiKey) {
+				throw new Error(getMissingApiKeyError(this.config.providerId));
+			}
+			const requestHeaders = this.getRequestHeaders();
+			const hasAuthorizationHeader = Object.keys(requestHeaders).some(
+				(key) => key.toLowerCase() === "authorization",
+			);
+
+			this.client = new OpenAI({
+				apiKey,
+				baseURL,
+				defaultHeaders: hasAuthorizationHeader
+					? requestHeaders
+					: { ...requestHeaders, Authorization: `Bearer ${apiKey}` },
+			});
+		}
+		return this.client;
+	}
+
+	/**
+	 * Get model info, falling back to provider defaults
+	 */
+	getModel(): HandlerModelInfo {
+		const modelId = this.config.modelId;
+		if (!modelId) {
+			throw new Error("Model ID is required. Set modelId in config.");
+		}
+
+		const modelInfo =
+			this.config.modelInfo ??
+			this.config.knownModels?.[modelId] ??
+			this.getDefaultModelInfo();
+
+		return { id: modelId, info: { ...modelInfo, id: modelId } };
+	}
+
+	protected getDefaultModelInfo(): ModelInfo {
+		// Responses API models don't support prompt caching
+		const capabilities = [].filter((c) => c !== "prompt-cache");
+		return {
+			id: this.config.modelId,
+			capabilities,
+		};
+	}
+
+	getMessages(
+		_systemPrompt: string,
+		messages: Message[],
+	): ReturnType<typeof convertToResponsesInput> {
+		return convertToResponsesInput(messages);
+	}
+
+	/**
+	 * Create a streaming message using the Responses API
+	 */
+	async *createMessage(
+		systemPrompt: string,
+		messages: Message[],
+		tools?: ToolDefinition[],
+	): ApiStream {
+		yield* retryStream(() =>
+			this.createMessageInternal(systemPrompt, messages, tools),
+		);
+	}
+
+	private async *createMessageInternal(
+		systemPrompt: string,
+		messages: Message[],
+		tools?: ToolDefinition[],
+	): ApiStream {
+		const client = this.ensureClient();
+		const { id: modelId, info: modelInfo } = this.getModel();
+		const abortSignal = this.getAbortSignal();
+		const fallbackResponseId = this.createResponseId();
+		let resolvedResponseId: string | undefined;
+
+		// Convert messages to Responses API input format
+		const input = this.getMessages(systemPrompt, messages);
+
+		// Convert tools to Responses API format
+		const responseTools = convertToolsToResponsesFormat(tools, {
+			stripFormat: this.config.providerId === "openai-codex",
+		});
+
+		// Responses API requires tools for native tool calling
+		if (!responseTools?.length) {
+			throw new Error(
+				"OpenAI Responses API requires tools to be provided. Enable native tool calling in settings.",
+			);
+		}
+
+		// Build reasoning config
+		const supportsReasoning =
+			modelInfo.capabilities?.includes("reasoning") ?? false;
+		const effectiveReasoningEffort =
+			this.config.reasoningEffort ??
+			(this.config.thinking ? DEFAULT_REASONING_EFFORT : undefined);
+		const reasoningConfig =
+			supportsReasoning && effectiveReasoningEffort
+				? {
+						effort: effectiveReasoningEffort,
+						summary: "auto" as const,
+					}
+				: undefined;
+		const requestHeaders = this.getRequestHeaders();
+		const hasAuthorizationHeader = Object.keys(requestHeaders).some(
+			(key) => key.toLowerCase() === "authorization",
+		);
+		const apiKey = resolveApiKeyForProvider(
+			this.config.providerId,
+			this.config.apiKey,
+		);
+		if (!hasAuthorizationHeader && apiKey) {
+			requestHeaders.Authorization = `Bearer ${apiKey}`;
+		}
+		if (
+			this.config.providerId === "openai-codex" &&
+			typeof this.config.accountId === "string" &&
+			this.config.accountId.trim().length > 0
+		) {
+			const accountId = this.config.accountId.trim();
+			// ChatGPT Codex endpoints may require an explicit account identifier.
+			requestHeaders["chatgpt-account-id"] = accountId;
+			requestHeaders["openai-account-id"] = accountId;
+		}
+
+		// Create the response using Responses API
+		let stream: AsyncIterable<any>;
+		try {
+			stream = await (client as any).responses.create(
+				{
+					model: modelId,
+					instructions: systemPrompt,
+					input,
+					// ChatGPT account Codex rejects requests unless explicit non-storage is set.
+					store: this.config.providerId === "openai-codex" ? false : undefined,
+					stream: true,
+					tools: responseTools,
+					reasoning: reasoningConfig,
+				},
+				{ signal: abortSignal, headers: requestHeaders },
+			);
+		} catch (error) {
+			if (this.config.providerId === "openai-codex") {
+				const rawError = error as
+					| (Error & {
+							status?: number;
+							message?: string;
+							error?: { message?: string; detail?: string };
+							response?: { status?: number };
+					  })
+					| undefined;
+				const status =
+					rawError?.status ??
+					rawError?.response?.status ??
+					(typeof rawError?.message === "string" &&
+					rawError.message.includes("400")
+						? 400
+						: undefined);
+				if (status === 400) {
+					const detail =
+						rawError?.error?.detail ??
+						rawError?.error?.message ??
+						(typeof rawError?.message === "string" ? rawError.message : "");
+					throw new Error(
+						`OpenAI Codex request was rejected (HTTP 400). ${detail ? `Detail: ${detail}` : "Re-run 'clite auth openai-codex', verify model access, and ensure accountId is present in provider settings."}`,
+						{ cause: error },
+					);
+				}
+			}
+			throw error;
+		}
+
+		// Process the response stream
+		for await (const chunk of stream) {
+			const apiResponseId = this.getApiResponseId(chunk);
+			if (apiResponseId) {
+				resolvedResponseId = apiResponseId;
+			}
+
+			yield* this.processResponseChunk(
+				chunk,
+				modelInfo,
+				resolvedResponseId ?? fallbackResponseId,
+			);
+		}
+	}
+
+	/**
+	 * Process a single chunk from the Responses API stream
+	 */
+	protected *processResponseChunk(
+		chunk: any,
+		_modelInfo: ModelInfo,
+		responseId: string,
+	): Generator<import("../types").ApiStreamChunk> {
+		// Handle different event types from Responses API
+		switch (chunk.type) {
+			case "response.output_item.added": {
+				const item = chunk.item;
+				if (item.type === "function_call" && item.id) {
+					yield {
+						type: "tool_calls",
+						id: item.id || responseId,
+						tool_call: {
+							call_id: item.call_id,
+							function: {
+								id: item.id,
+								name: item.name,
+								arguments: item.arguments,
+							},
+						},
+					};
+				}
+				if (item.type === "reasoning" && item.encrypted_content && item.id) {
+					yield {
+						type: "reasoning",
+						id: item.id || responseId,
+						reasoning: "",
+						redacted_data: item.encrypted_content,
+					};
+				}
+				break;
+			}
+
+			case "response.output_item.done": {
+				const item = chunk.item;
+				if (item.type === "function_call") {
+					yield {
+						type: "tool_calls",
+						id: item.id || responseId,
+						tool_call: {
+							call_id: item.call_id,
+							function: {
+								id: item.id,
+								name: item.name,
+								arguments: item.arguments,
+							},
+						},
+					};
+				}
+				if (item.type === "reasoning") {
+					yield {
+						type: "reasoning",
+						id: item.id || responseId,
+						details: item.summary,
+						reasoning: "",
+					};
+				}
+				break;
+			}
+
+			case "response.reasoning_summary_part.added":
+				yield {
+					type: "reasoning",
+					id: chunk.item_id || responseId,
+					reasoning: chunk.part?.text || "",
+				};
+				break;
+
+			case "response.reasoning_summary_text.delta":
+				yield {
+					type: "reasoning",
+					id: chunk.item_id || responseId,
+					reasoning: chunk.delta || "",
+				};
+				break;
+
+			case "response.reasoning_summary_part.done":
+				yield {
+					type: "reasoning",
+					id: chunk.item_id || responseId,
+					details: chunk.part,
+					reasoning: "",
+				};
+				break;
+
+			case "response.output_text.delta":
+				if (chunk.delta) {
+					yield {
+						id: chunk.item_id || responseId,
+						type: "text",
+						text: chunk.delta,
+					};
+				}
+				break;
+
+			case "response.reasoning_text.delta":
+				if (chunk.delta) {
+					yield {
+						id: chunk.item_id || responseId,
+						type: "reasoning",
+						reasoning: chunk.delta,
+					};
+				}
+				break;
+
+			case "response.function_call_arguments.delta":
+				yield {
+					type: "tool_calls",
+					id: chunk.item_id || responseId,
+					tool_call: {
+						function: {
+							id: chunk.item_id,
+							name: chunk.item_id,
+							arguments: chunk.delta,
+						},
+					},
+				};
+				break;
+
+			case "response.function_call_arguments.done":
+				if (chunk.item_id && chunk.name && chunk.arguments) {
+					yield {
+						type: "tool_calls",
+						id: chunk.item_id || responseId,
+						tool_call: {
+							function: {
+								id: chunk.item_id,
+								name: chunk.name,
+								arguments: chunk.arguments,
+							},
+						},
+					};
+				}
+				break;
+
+			case "response.incomplete": {
+				const incompleteReason = chunk.response?.incomplete_details?.reason;
+				yield {
+					type: "done",
+					success: false,
+					incompleteReason,
+					id: chunk.response?.id || responseId,
+				};
+				break;
+			}
+
+			case "response.failed": {
+				const error = chunk.response?.error;
+				yield {
+					type: "done",
+					success: false,
+					error: error?.message || "Unknown error",
+					id: chunk.response?.id || responseId,
+				};
+				break;
+			}
+
+			case "response.completed": {
+				if (chunk.response?.usage) {
+					const usage = chunk.response.usage;
+					const inputTokens = usage.input_tokens || 0;
+					const outputTokens = usage.output_tokens || 0;
+					const cacheReadTokens =
+						usage.output_tokens_details?.reasoning_tokens || 0;
+					const cacheWriteTokens =
+						usage.input_tokens_details?.cached_tokens || 0;
+
+					const totalCost = this.calculateCost(
+						inputTokens,
+						outputTokens,
+						cacheReadTokens,
+					);
+					const nonCachedInputTokens = Math.max(
+						0,
+						inputTokens - cacheReadTokens - cacheWriteTokens,
+					);
+
+					yield {
+						type: "usage",
+						inputTokens: nonCachedInputTokens,
+						outputTokens,
+						cacheWriteTokens,
+						cacheReadTokens,
+						totalCost,
+						id: chunk.response.id || responseId,
+					};
+				}
+
+				// Yield done chunk to indicate streaming completed successfully
+				yield {
+					type: "done",
+					success: true,
+					id: chunk.response?.id || responseId,
+				};
+				break;
+			}
+		}
+	}
+
+	private getApiResponseId(chunk: any): string | undefined {
+		if (
+			typeof chunk?.response?.id === "string" &&
+			chunk.response.id.length > 0
+		) {
+			return chunk.response.id;
+		}
+
+		if (
+			typeof chunk?.response_id === "string" &&
+			chunk.response_id.length > 0
+		) {
+			return chunk.response_id;
+		}
+
+		return undefined;
+	}
+}
+
+/**
+ * Create an OpenAI Responses API handler
+ */
+export function createOpenAIResponsesHandler(
+	config: ProviderConfig,
+): OpenAIResponsesHandler {
+	return new OpenAIResponsesHandler(config);
+}

package/src/providers/handlers/providers.test.ts

@@ -0,0 +1,120 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import type { ModelInfo, ProviderConfig } from "../types/index";
+import {
+	clearLiveModelsCatalogCache,
+	clearPrivateModelsCatalogCache,
+	OPENAI_COMPATIBLE_PROVIDERS,
+	resolveProviderConfig,
+} from "./providers";
+
+function createLiteLlmConfig(
+	overrides?: Partial<ProviderConfig>,
+): ProviderConfig {
+	return {
+		providerId: "litellm",
+		modelId: "gpt-4o",
+		apiKey: "test-key",
+		baseUrl: "http://localhost:4000",
+		...overrides,
+	};
+}
+
+describe("resolveProviderConfig", () => {
+	const originalFetch = globalThis.fetch;
+
+	beforeEach(() => {
+		clearLiveModelsCatalogCache();
+		clearPrivateModelsCatalogCache();
+	});
+
+	afterEach(() => {
+		globalThis.fetch = originalFetch;
+		vi.restoreAllMocks();
+	});
+
+	it("loads auth-gated private models and gives user knownModels highest priority", async () => {
+		const fetchMock = vi.fn(async () => ({
+			ok: true,
+			json: async () => ({
+				data: [
+					{
+						model_name: "gpt-4o",
+						litellm_params: { model: "gpt-4o" },
+						model_info: {
+							max_tokens: 32000,
+							max_input_tokens: 128000,
+						},
+					},
+				],
+			}),
+		}));
+		globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+		const userModel: ModelInfo = {
+			id: "gpt-4o",
+			name: "User Override",
+			contextWindow: 999_999,
+			maxTokens: 9_999,
+			capabilities: ["streaming", "tools"],
+		};
+
+		const resolved = await resolveProviderConfig(
+			"litellm",
+			{ loadPrivateOnAuth: true },
+			createLiteLlmConfig({
+				knownModels: {
+					"gpt-4o": userModel,
+				},
+			}),
+		);
+
+		expect(fetchMock).toHaveBeenCalledTimes(1);
+		expect(resolved?.knownModels?.["gpt-4o"]).toEqual(userModel);
+	});
+
+	it("caches auth-gated private model responses by provider+baseUrl+token", async () => {
+		const fetchMock = vi.fn(async () => ({
+			ok: true,
+			json: async () => ({
+				data: [
+					{
+						model_name: "proxy-model",
+						litellm_params: { model: "proxy-model" },
+						model_info: {},
+					},
+				],
+			}),
+		}));
+		globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+		const config = createLiteLlmConfig();
+		await resolveProviderConfig("litellm", { loadPrivateOnAuth: true }, config);
+		await resolveProviderConfig("litellm", { loadPrivateOnAuth: true }, config);
+
+		expect(fetchMock).toHaveBeenCalledTimes(1);
+	});
+
+	it("does not load auth-gated private models when disabled", async () => {
+		const fetchMock = vi.fn();
+		globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+		await resolveProviderConfig(
+			"litellm",
+			{ loadPrivateOnAuth: false },
+			createLiteLlmConfig(),
+		);
+
+		expect(fetchMock).not.toHaveBeenCalled();
+	});
+
+	it("includes lmstudio and zai as OpenAI-compatible providers", () => {
+		expect(OPENAI_COMPATIBLE_PROVIDERS).toHaveProperty("lmstudio");
+		expect(OPENAI_COMPATIBLE_PROVIDERS).toHaveProperty("zai");
+		expect(OPENAI_COMPATIBLE_PROVIDERS.lmstudio?.baseUrl).toBe(
+			"http://localhost:1234/v1",
+		);
+		expect(OPENAI_COMPATIBLE_PROVIDERS.zai?.baseUrl).toBe(
+			"https://api.z.ai/api/paas/v4",
+		);
+	});
+});