@clinebot/llms 0.0.6 → 0.0.10

package/src/providers/handlers/gemini-base.test.ts

@@ -218,4 +218,44 @@ describe("GeminiHandler", () => {
 		expect(secondId).toBeTruthy();
 		expect(firstId).not.toBe(secondId);
 	});
+
+	it("defaults maxOutputTokens to 8192 for gemini-3-flash when no model or config limit is provided", async () => {
+		generateContentStreamSpy.mockResolvedValue(createAsyncIterable([]));
+
+		const handler = new GeminiHandler({
+			providerId: "gemini",
+			modelId: "gemini-3-flash",
+			apiKey: "test-key",
+		});
+
+		await collectChunks(
+			handler.createMessage("System", [{ role: "user", content: "go" }]),
+		);
+
+		expect(generateContentStreamSpy).toHaveBeenCalledTimes(1);
+		const request = generateContentStreamSpy.mock.calls[0]?.[0] as {
+			config?: { maxOutputTokens?: number };
+		};
+		expect(request.config?.maxOutputTokens).toBe(8192);
+	});
+
+	it("defaults maxOutputTokens to 128000 for non gemini-3-flash models when no model or config limit is provided", async () => {
+		generateContentStreamSpy.mockResolvedValue(createAsyncIterable([]));
+
+		const handler = new GeminiHandler({
+			providerId: "gemini",
+			modelId: "gemini-2.5-flash",
+			apiKey: "test-key",
+		});
+
+		await collectChunks(
+			handler.createMessage("System", [{ role: "user", content: "go" }]),
+		);
+
+		expect(generateContentStreamSpy).toHaveBeenCalledTimes(1);
+		const request = generateContentStreamSpy.mock.calls[0]?.[0] as {
+			config?: { maxOutputTokens?: number };
+		};
+		expect(request.config?.maxOutputTokens).toBe(128000);
+	});
 });

package/src/providers/handlers/gemini-base.ts

@@ -27,6 +27,16 @@ import { RetriableError, retryStream } from "../utils/retry";
 import { BaseHandler } from "./base";
 
 const DEFAULT_THINKING_BUDGET_TOKENS = 1024;
+const DEFAULT_MAX_OUTPUT_TOKENS = 128_000;
+const GEMINI_3_FLASH_MAX_OUTPUT_TOKENS = 8192;
+
+function isGemini3FlashModel(modelId: string): boolean {
+	const normalized = modelId.toLowerCase();
+	return (
+		normalized.includes("gemini-3-flash") ||
+		normalized.includes("gemini-3.0-flash")
+	);
+}
 
 /**
  * Handler for Google's Gemini API
@@ -131,6 +141,11 @@ export class GeminiHandler extends BaseHandler {
 		}
 
 		// Build request config with abort signal
+		const fallbackMaxOutputTokens = isGemini3FlashModel(modelId)
+			? GEMINI_3_FLASH_MAX_OUTPUT_TOKENS
+			: DEFAULT_MAX_OUTPUT_TOKENS;
+		const maxOutputTokens =
+			info.maxTokens ?? this.config.maxOutputTokens ?? fallbackMaxOutputTokens;
 		const requestConfig: GenerateContentConfig = {
 			httpOptions: this.config.baseUrl
 				? { baseUrl: this.config.baseUrl, headers: this.getRequestHeaders() }
@@ -138,7 +153,7 @@ export class GeminiHandler extends BaseHandler {
 			abortSignal,
 			systemInstruction: systemPrompt,
 			temperature: info.temperature ?? 1,
-			maxOutputTokens: info.maxTokens ?? this.config.maxOutputTokens,
+			maxOutputTokens,
 		};
 
 		// Add thinking config only when explicitly requested and supported.

package/src/providers/handlers/openai-base.ts

@@ -22,6 +22,7 @@ import type {
 	ModelInfo,
 	ProviderConfig,
 } from "../types";
+import { hasModelCapability } from "../types";
 import type { Message, ToolDefinition } from "../types/messages";
 import { retryStream } from "../utils/retry";
 import { ToolCallProcessor } from "../utils/tool-processor";
@@ -106,9 +107,26 @@ export class OpenAIBaseHandler extends BaseHandler {
 		systemPrompt: string,
 		messages: Message[],
 	): OpenAI.Chat.ChatCompletionMessageParam[] {
+		const model = this.getModel();
+		const supportsPromptCache =
+			hasModelCapability(model.info, "prompt-cache") ||
+			this.config.capabilities?.includes("prompt-cache") === true;
+		const systemMessage = supportsPromptCache
+			? ({
+					role: "system",
+					content: [
+						{
+							type: "text",
+							text: systemPrompt,
+							cache_control: { type: "ephemeral" },
+						},
+					],
+				} as unknown as OpenAI.Chat.ChatCompletionMessageParam)
+			: { role: "system" as const, content: systemPrompt };
+
 		return [
-			{ role: "system", content: systemPrompt },
-			...convertToOpenAIMessages(messages),
+			systemMessage,
+			...convertToOpenAIMessages(messages, supportsPromptCache),
 		];
 	}
 
@@ -171,7 +189,11 @@ export class OpenAIBaseHandler extends BaseHandler {
 			this.config.reasoningEffort ??
 			(this.config.thinking ? DEFAULT_REASONING_EFFORT : undefined);
 		if (supportsReasoningEffort && effectiveReasoningEffort) {
-			(requestOptions as any).reasoning_effort = effectiveReasoningEffort;
+			(
+				requestOptions as OpenAI.ChatCompletionCreateParamsStreaming & {
+					reasoning_effort?: string;
+				}
+			).reasoning_effort = effectiveReasoningEffort;
 		}
 
 		const requestHeaders = this.getRequestHeaders();
@@ -191,16 +213,25 @@ export class OpenAIBaseHandler extends BaseHandler {
 			headers: requestHeaders,
 		});
 		const toolCallProcessor = new ToolCallProcessor();
+		let finishReason: string | null = null;
 
 		for await (const chunk of stream) {
+			const choice = chunk.choices?.[0];
+			if (choice?.finish_reason) {
+				finishReason = choice.finish_reason;
+			}
 			yield* this.withResponseIdForAll(
 				this.processChunk(chunk, toolCallProcessor, modelInfo, responseId),
 				responseId,
 			);
 		}
 
-		// Yield done chunk to indicate streaming completed successfully
-		yield { type: "done", success: true, id: responseId };
+		yield {
+			type: "done",
+			success: true,
+			id: responseId,
+			incompleteReason: finishReason === "length" ? "max_tokens" : undefined,
+		};
 	}
 
 	/**
@@ -213,9 +244,11 @@ export class OpenAIBaseHandler extends BaseHandler {
 		_modelInfo: ModelInfo,
 		responseId: string,
 	): Generator<import("../types").ApiStreamChunk> {
-		const delta = chunk.choices?.[0]?.delta && {
-			...chunk.choices[0].delta,
-			reasoning_content: (chunk.choices[0].delta as any).reasoning_content,
+		const rawDelta = chunk.choices?.[0]?.delta;
+		const delta = rawDelta && {
+			...rawDelta,
+			reasoning_content: (rawDelta as { reasoning_content?: string })
+				.reasoning_content,
 		};
 
 		// Handle text content
@@ -227,7 +260,7 @@ export class OpenAIBaseHandler extends BaseHandler {
 		if (delta?.reasoning_content) {
 			yield {
 				type: "reasoning",
-				reasoning: (delta as any).reasoning_content,
+				reasoning: delta.reasoning_content,
 				id: responseId,
 			};
 		}
@@ -248,10 +281,21 @@ export class OpenAIBaseHandler extends BaseHandler {
 		if (chunk.usage) {
 			const inputTokens = chunk.usage.prompt_tokens ?? 0;
 			const outputTokens = chunk.usage.completion_tokens ?? 0;
+			const usageWithCache = chunk.usage as typeof chunk.usage & {
+				prompt_tokens_details?: {
+					cached_tokens?: number;
+					cache_write_tokens?: number;
+				};
+				prompt_cache_miss_tokens?: number;
+				cache_creation_input_tokens?: number;
+				cache_read_input_tokens?: number;
+			};
 			const cacheReadTokens =
-				(chunk.usage as any).prompt_tokens_details?.cached_tokens ?? 0;
+				usageWithCache.prompt_tokens_details?.cached_tokens ?? 0;
 			const cacheWriteTokens =
-				(chunk.usage as any).prompt_cache_miss_tokens ?? 0;
+				usageWithCache.prompt_tokens_details?.cache_write_tokens ??
+				usageWithCache.prompt_cache_miss_tokens ??
+				0;
 
 			yield {
 				type: "usage",

package/src/providers/handlers/vertex.ts

@@ -241,7 +241,7 @@ export class VertexHandler extends BaseHandler {
 				promptCacheOn,
 			}),
 			tools: toAiSdkTools(tools),
-			maxTokens: model.info.maxTokens ?? this.config.maxOutputTokens ?? 8192,
+			maxTokens: model.info.maxTokens ?? this.config.maxOutputTokens ?? 128_000,
 			temperature: reasoningOn ? undefined : 0,
 			providerOptions:
 				Object.keys(providerOptions).length > 0 ? providerOptions : undefined,

package/src/providers/transform/format-conversion.test.ts

@@ -216,6 +216,32 @@ describe("format conversion", () => {
 		expect(openai[1].tool_calls[0].extra_content).toBeUndefined();
 	});
 
+	it("applies OpenAI cache markers only to the final user message", () => {
+		const messages: Message[] = [
+			{ role: "user", content: "first prompt" },
+			{ role: "assistant", content: "intermediate response" },
+			{ role: "user", content: "second prompt" },
+		];
+
+		const openai = convertToOpenAIMessages(messages, true) as any[];
+		expect(openai[0]).toMatchObject({ role: "user", content: "first prompt" });
+		expect(openai[2].role).toBe("user");
+		expect(openai[2].content).toMatchObject([
+			{
+				type: "text",
+				text: "second prompt",
+				cache_control: { type: "ephemeral" },
+			},
+		]);
+
+		const cacheMarkerCount = openai
+			.flatMap((message) =>
+				Array.isArray(message.content) ? message.content : [],
+			)
+			.filter((part) => part?.cache_control?.type === "ephemeral").length;
+		expect(cacheMarkerCount).toBe(1);
+	});
+
 	it("normalizes array-shaped tool_use input for openai replay", () => {
 		const messages: Message[] = [
 			{ role: "user", content: "run these" },

package/src/providers/transform/openai-format.ts

@@ -26,23 +26,49 @@ type OpenAIContentPart = OpenAI.Chat.ChatCompletionContentPart;
 /**
  * Convert messages to OpenAI format
  */
-export function convertToOpenAIMessages(messages: Message[]): OpenAIMessage[] {
-	return messages.flatMap(convertMessage);
+export function convertToOpenAIMessages(
+	messages: Message[],
+	enableCaching = false,
+): OpenAIMessage[] {
+	const lastUserIndex = enableCaching
+		? messages.map((m) => m.role).lastIndexOf("user")
+		: -1;
+	return messages.flatMap((message, index) =>
+		convertMessage(message, enableCaching && index === lastUserIndex),
+	);
 }
 
-function convertMessage(message: Message): OpenAIMessage[] {
+function convertMessage(
+	message: Message,
+	addCacheControl: boolean,
+): OpenAIMessage[] {
 	const { role, content } = message;
 
 	// Simple string content
 	if (typeof content === "string") {
-		return [{ role, content } as OpenAIMessage];
+		if (role !== "user" || !addCacheControl) {
+			return [{ role, content } as OpenAIMessage];
+		}
+
+		return [
+			{
+				role,
+				content: [
+					{
+						type: "text",
+						text: content,
+						cache_control: { type: "ephemeral" },
+					},
+				],
+			} as unknown as OpenAIMessage,
+		];
 	}
 
 	// Array content - need to process blocks
 	if (role === "assistant") {
 		return [convertAssistantMessage(content)];
 	} else {
-		return convertUserMessage(content);
+		return convertUserMessage(content, addCacheControl);
 	}
 }
 
@@ -85,7 +111,10 @@ function convertAssistantMessage(content: ContentBlock[]): OpenAIMessage {
 	return message;
 }
 
-function convertUserMessage(content: ContentBlock[]): OpenAIMessage[] {
+function convertUserMessage(
+	content: ContentBlock[],
+	addCacheControl: boolean,
+): OpenAIMessage[] {
 	const messages: OpenAIMessage[] = [];
 
 	// Convert all tool results to separate tool messages
@@ -137,10 +166,24 @@ function convertUserMessage(content: ContentBlock[]): OpenAIMessage[] {
 		return messages;
 	}
 
+	if (addCacheControl) {
+		for (let i = parts.length - 1; i >= 0; i--) {
+			if (parts[i].type === "text") {
+				parts[i] = {
+					...(parts[i] as OpenAI.Chat.ChatCompletionContentPartText),
+					cache_control: { type: "ephemeral" },
+				} as unknown as OpenAIContentPart;
+				break;
+			}
+		}
+	}
+
 	messages.push({
 		role: "user",
 		content:
-			parts.length === 1 && parts[0].type === "text" ? parts[0].text : parts,
+			parts.length === 1 && parts[0].type === "text" && !addCacheControl
+				? parts[0].text
+				: (parts as unknown as OpenAI.Chat.ChatCompletionUserMessageParam["content"]),
 	});
 
 	return messages;

package/src/providers/types/config.ts

@@ -244,6 +244,11 @@ export interface ProviderOptions {
 	modelCatalog?: ModelCatalogConfig;
 }
 
+/**
+ * Provider-specific options that don't fit other categories
+ */
+import type { BasicLogger } from "@clinebot/shared";
+
 /**
  * Runtime model catalog refresh options
  */
@@ -299,6 +304,9 @@ export interface ProviderConfig
 	/** AbortSignal for cancelling requests */
 	abortSignal?: AbortSignal;
 
+	/** Optional runtime logger for provider-level diagnostics */
+	logger?: BasicLogger;
+
 	/** Codex CLI-specific options */
 	codex?: CodexConfig;
 

package/src/providers/types/messages.ts

@@ -121,6 +121,8 @@ export interface Message {
 export interface MessageWithMetadata extends Message {
 	/** Unique message ID */
 	id?: string;
+	/** Additional message metadata for storage/history consumers */
+	metadata?: Record<string, unknown>;
 	/** Provider ID used to generate this message */
 	providerId?: string;
 	/** Model ID used to generate this message */

package/src/providers/utils/tool-processor.test.ts

@@ -33,6 +33,66 @@ describe("ToolCallProcessor", () => {
 		expect(second[0].tool_call.function.arguments).toBe(' -la"]}');
 	});
 
+	it("normalizes cumulative argument snapshots into deltas", () => {
+		const processor = new ToolCallProcessor();
+
+		const first = processor.processToolCallDeltas(
+			[
+				{
+					index: 0,
+					id: "call_1",
+					function: { name: "editor", arguments: '{"command":"create"' },
+				},
+			],
+			"resp_1",
+		);
+
+		const second = processor.processToolCallDeltas(
+			[
+				{
+					index: 0,
+					function: {
+						arguments: '{"command":"create","path":"/tmp/file.txt"}',
+					},
+				},
+			],
+			"resp_1",
+		);
+
+		expect(first).toHaveLength(1);
+		expect(second).toHaveLength(1);
+		expect(first[0].tool_call.function.arguments).toBe('{"command":"create"');
+		expect(second[0].tool_call.function.arguments).toBe(
+			',"path":"/tmp/file.txt"}',
+		);
+	});
+
+	it("serializes object-shaped arguments instead of concatenating [object Object]", () => {
+		const processor = new ToolCallProcessor();
+
+		const result = processor.processToolCallDeltas(
+			[
+				{
+					index: 0,
+					id: "call_1",
+					function: {
+						name: "editor",
+						arguments: {
+							command: "create",
+							path: "/tmp/file.txt",
+						},
+					},
+				},
+			],
+			"resp_1",
+		);
+
+		expect(result).toHaveLength(1);
+		expect(result[0].tool_call.function.arguments).toBe(
+			'{"command":"create","path":"/tmp/file.txt"}',
+		);
+	});
+
 	it("preserves tool call id/name for interleaved parallel deltas", () => {
 		const processor = new ToolCallProcessor();
 

package/src/providers/utils/tool-processor.ts

@@ -18,7 +18,7 @@ interface ToolCallDelta {
 	id?: string;
 	function?: {
 		name?: string;
-		arguments?: string;
+		arguments?: unknown;
 	};
 }
 
@@ -61,7 +61,11 @@ export class ToolCallProcessor {
 			if (fn?.name) {
 				toolCall.name = fn.name;
 			}
-			const deltaArguments = fn?.arguments ?? "";
+			const rawArguments = fn?.arguments;
+			const deltaArguments = this.normalizeArgumentsDelta(
+				toolCall.arguments,
+				rawArguments,
+			);
 			if (deltaArguments) {
 				toolCall.arguments += deltaArguments;
 			}
@@ -108,4 +112,35 @@ export class ToolCallProcessor {
 	reset(): void {
 		this.toolCalls.clear();
 	}
+
+	private normalizeArgumentsDelta(
+		accumulatedArguments: string,
+		rawArguments: unknown,
+	): string {
+		if (rawArguments == null) {
+			return "";
+		}
+
+		const nextArguments =
+			typeof rawArguments === "string"
+				? rawArguments
+				: JSON.stringify(rawArguments);
+
+		if (!nextArguments) {
+			return "";
+		}
+
+		// Some OpenAI-compatible providers emit cumulative argument snapshots
+		// instead of true deltas. Convert those snapshots back into a suffix so
+		// downstream accumulation only happens once.
+		if (
+			accumulatedArguments &&
+			nextArguments.length >= accumulatedArguments.length &&
+			nextArguments.startsWith(accumulatedArguments)
+		) {
+			return nextArguments.slice(accumulatedArguments.length);
+		}
+
+		return nextArguments;
+	}
 }