@clinebot/llms 0.0.7 → 0.0.11

package/src/providers/handlers/r1-base.ts

@@ -255,19 +255,18 @@ export class R1BaseHandler extends BaseHandler {
 		const cacheReadTokens = r1Usage.prompt_cache_hit_tokens ?? 0;
 		const cacheWriteTokens = r1Usage.prompt_cache_miss_tokens ?? 0;
 
-		// Calculate non-cached input tokens (will always be 0 for DeepSeek since input = read + write)
-		const nonCachedInputTokens = Math.max(
-			0,
-			inputTokens - cacheReadTokens - cacheWriteTokens,
-		);
-
 		yield {
 			type: "usage",
-			inputTokens: nonCachedInputTokens,
+			inputTokens,
 			outputTokens,
 			cacheReadTokens,
 			cacheWriteTokens,
-			totalCost: this.calculateCost(inputTokens, outputTokens, cacheReadTokens),
+			totalCost: this.calculateCost(
+				inputTokens,
+				outputTokens,
+				cacheReadTokens,
+				cacheWriteTokens,
+			),
 			id: responseId,
 		};
 	}

package/src/providers/handlers/vertex.ts

@@ -189,7 +189,7 @@ export class VertexHandler extends BaseHandler {
 		if (!isClaudeModel(model.id)) {
 			return this.ensureGeminiHandler().getMessages(systemPrompt, messages);
 		}
-		const supportsPromptCache = hasModelCapability(model.info, "prompt-cache");
+		const supportsPromptCache = this.supportsPromptCache(model.info);
 		return convertToAnthropicMessages(messages, supportsPromptCache);
 	}
 
@@ -226,7 +226,7 @@ export class VertexHandler extends BaseHandler {
 		const budgetTokens = this.config.thinkingBudgetTokens ?? 0;
 		const reasoningOn =
 			hasModelCapability(model.info, "reasoning") && budgetTokens > 0;
-		const promptCacheOn = hasModelCapability(model.info, "prompt-cache");
+		const promptCacheOn = this.supportsPromptCache(model.info);
 
 		const providerOptions: Record<string, unknown> = {};
 		if (reasoningOn) {
@@ -241,7 +241,7 @@ export class VertexHandler extends BaseHandler {
 				promptCacheOn,
 			}),
 			tools: toAiSdkTools(tools),
-			maxTokens: model.info.maxTokens ?? this.config.maxOutputTokens ?? 8192,
+			maxTokens: model.info.maxTokens ?? this.config.maxOutputTokens ?? 128_000,
 			temperature: reasoningOn ? undefined : 0,
 			providerOptions:
 				Object.keys(providerOptions).length > 0 ? providerOptions : undefined,
@@ -251,8 +251,18 @@ export class VertexHandler extends BaseHandler {
 		yield* emitAiSdkStream(stream, {
 			responseId,
 			errorMessage: "Vertex Anthropic stream failed",
-			calculateCost: (inputTokens, outputTokens, cacheReadTokens) =>
-				this.calculateCost(inputTokens, outputTokens, cacheReadTokens),
+			calculateCost: (
+				inputTokens,
+				outputTokens,
+				cacheReadTokens,
+				cacheWriteTokens,
+			) =>
+				this.calculateCost(
+					inputTokens,
+					outputTokens,
+					cacheReadTokens,
+					cacheWriteTokens,
+				),
 			reasoningTypes: ["reasoning-delta"],
 			enableToolCalls: true,
 			toolCallArgsOrder: ["input", "args"],

package/src/providers/transform/anthropic-format.ts

@@ -32,12 +32,24 @@ export function convertToAnthropicMessages(
 	messages: Message[],
 	enableCaching = false,
 ): AnthropicMessage[] {
+	const userMessageIndices = messages.reduce<number[]>(
+		(indices, message, index) => {
+			if (message.role === "user") {
+				indices.push(index);
+			}
+			return indices;
+		},
+		[],
+	);
+	const cacheableMessageIndices = enableCaching
+		? new Set(userMessageIndices.slice(-2))
+		: new Set<number>();
 	const result: AnthropicMessage[] = [];
 
-	for (const message of messages) {
+	for (const [index, message] of messages.entries()) {
 		const converted = convertMessage(
 			message,
-			enableCaching && messages.indexOf(message) === messages.length - 1,
+			cacheableMessageIndices.has(index),
 		);
 		if (converted) {
 			result.push(converted);

package/src/providers/transform/format-conversion.test.ts

@@ -216,6 +216,32 @@ describe("format conversion", () => {
 		expect(openai[1].tool_calls[0].extra_content).toBeUndefined();
 	});
 
+	it("applies OpenAI cache markers only to the final user message", () => {
+		const messages: Message[] = [
+			{ role: "user", content: "first prompt" },
+			{ role: "assistant", content: "intermediate response" },
+			{ role: "user", content: "second prompt" },
+		];
+
+		const openai = convertToOpenAIMessages(messages, true) as any[];
+		expect(openai[0]).toMatchObject({ role: "user", content: "first prompt" });
+		expect(openai[2].role).toBe("user");
+		expect(openai[2].content).toMatchObject([
+			{
+				type: "text",
+				text: "second prompt",
+				cache_control: { type: "ephemeral" },
+			},
+		]);
+
+		const cacheMarkerCount = openai
+			.flatMap((message) =>
+				Array.isArray(message.content) ? message.content : [],
+			)
+			.filter((part) => part?.cache_control?.type === "ephemeral").length;
+		expect(cacheMarkerCount).toBe(1);
+	});
+
 	it("normalizes array-shaped tool_use input for openai replay", () => {
 		const messages: Message[] = [
 			{ role: "user", content: "run these" },
@@ -259,10 +285,33 @@ describe("format conversion", () => {
 		];
 
 		const anthropic = convertToAnthropicMessages(messages, true) as any[];
+		expect(anthropic[0].content[0].cache_control).toEqual({
+			type: "ephemeral",
+		});
 		expect(anthropic[1].content[0].type).toBe("thinking");
 		expect(anthropic[1].content[0].signature).toBe("anthropic-sig");
 	});
 
+	it("applies anthropic cache markers to the last two user messages", () => {
+		const messages: Message[] = [
+			{ role: "user", content: "first prompt" },
+			{ role: "assistant", content: "intermediate response" },
+			{ role: "user", content: "second prompt" },
+			{ role: "assistant", content: "another response" },
+			{ role: "user", content: "third prompt" },
+		];
+
+		const anthropic = convertToAnthropicMessages(messages, true) as any[];
+
+		expect(anthropic[0].content[0].cache_control).toBeUndefined();
+		expect(anthropic[2].content[0].cache_control).toEqual({
+			type: "ephemeral",
+		});
+		expect(anthropic[4].content[0].cache_control).toEqual({
+			type: "ephemeral",
+		});
+	});
+
 	it("normalizes array-shaped tool_use input for anthropic replay", () => {
 		const messages: Message[] = [
 			{ role: "user", content: "run these" },

package/src/providers/transform/openai-format.ts

@@ -26,23 +26,49 @@ type OpenAIContentPart = OpenAI.Chat.ChatCompletionContentPart;
 /**
  * Convert messages to OpenAI format
  */
-export function convertToOpenAIMessages(messages: Message[]): OpenAIMessage[] {
-	return messages.flatMap(convertMessage);
+export function convertToOpenAIMessages(
+	messages: Message[],
+	enableCaching = false,
+): OpenAIMessage[] {
+	const lastUserIndex = enableCaching
+		? messages.map((m) => m.role).lastIndexOf("user")
+		: -1;
+	return messages.flatMap((message, index) =>
+		convertMessage(message, enableCaching && index === lastUserIndex),
+	);
 }
 
-function convertMessage(message: Message): OpenAIMessage[] {
+function convertMessage(
+	message: Message,
+	addCacheControl: boolean,
+): OpenAIMessage[] {
 	const { role, content } = message;
 
 	// Simple string content
 	if (typeof content === "string") {
-		return [{ role, content } as OpenAIMessage];
+		if (role !== "user" || !addCacheControl) {
+			return [{ role, content } as OpenAIMessage];
+		}
+
+		return [
+			{
+				role,
+				content: [
+					{
+						type: "text",
+						text: content,
+						cache_control: { type: "ephemeral" },
+					},
+				],
+			} as unknown as OpenAIMessage,
+		];
 	}
 
 	// Array content - need to process blocks
 	if (role === "assistant") {
 		return [convertAssistantMessage(content)];
 	} else {
-		return convertUserMessage(content);
+		return convertUserMessage(content, addCacheControl);
 	}
 }
 
@@ -85,7 +111,10 @@ function convertAssistantMessage(content: ContentBlock[]): OpenAIMessage {
 	return message;
 }
 
-function convertUserMessage(content: ContentBlock[]): OpenAIMessage[] {
+function convertUserMessage(
+	content: ContentBlock[],
+	addCacheControl: boolean,
+): OpenAIMessage[] {
 	const messages: OpenAIMessage[] = [];
 
 	// Convert all tool results to separate tool messages
@@ -137,10 +166,24 @@ function convertUserMessage(content: ContentBlock[]): OpenAIMessage[] {
 		return messages;
 	}
 
+	if (addCacheControl) {
+		for (let i = parts.length - 1; i >= 0; i--) {
+			if (parts[i].type === "text") {
+				parts[i] = {
+					...(parts[i] as OpenAI.Chat.ChatCompletionContentPartText),
+					cache_control: { type: "ephemeral" },
+				} as unknown as OpenAIContentPart;
+				break;
+			}
+		}
+	}
+
 	messages.push({
 		role: "user",
 		content:
-			parts.length === 1 && parts[0].type === "text" ? parts[0].text : parts,
+			parts.length === 1 && parts[0].type === "text" && !addCacheControl
+				? parts[0].text
+				: (parts as unknown as OpenAI.Chat.ChatCompletionUserMessageParam["content"]),
 	});
 
 	return messages;

package/src/providers/types/config.ts

@@ -244,6 +244,11 @@ export interface ProviderOptions {
 	modelCatalog?: ModelCatalogConfig;
 }
 
+/**
+ * Provider-specific options that don't fit other categories
+ */
+import type { BasicLogger } from "@clinebot/shared";
+
 /**
  * Runtime model catalog refresh options
  */
@@ -299,6 +304,9 @@ export interface ProviderConfig
 	/** AbortSignal for cancelling requests */
 	abortSignal?: AbortSignal;
 
+	/** Optional runtime logger for provider-level diagnostics */
+	logger?: BasicLogger;
+
 	/** Codex CLI-specific options */
 	codex?: CodexConfig;
 

package/src/providers/types/stream.ts

@@ -55,7 +55,7 @@ export interface ApiStreamReasoningChunk {
  */
 export interface ApiStreamUsageChunk {
 	type: "usage";
-	/** Number of input tokens (excluding cached) */
+	/** Total number of input tokens reported by the provider */
 	inputTokens: number;
 	/** Number of output tokens */
 	outputTokens: number;