@clinebot/llms 0.0.7 → 0.0.11

package/dist/providers/handlers/ai-sdk-community.d.ts

@@ -28,7 +28,7 @@ type AiSdkUsageMetrics = {
 export type EmitAiSdkStreamOptions = {
     responseId: string;
     errorMessage: string;
-    calculateCost: (inputTokens: number, outputTokens: number, cacheReadTokens: number) => number | undefined;
+    calculateCost: (inputTokens: number, outputTokens: number, cacheReadTokens: number, cacheWriteTokens?: number) => number | undefined;
     reasoningTypes?: string[];
     enableToolCalls?: boolean;
     toolCallArgsOrder?: Array<"args" | "input">;

package/dist/providers/handlers/base.d.ts

@@ -3,7 +3,7 @@
  *
  * Abstract base class that provides common functionality for all handlers.
  */
-import type { ApiHandler, ApiStream, ApiStreamUsageChunk, HandlerModelInfo, ProviderConfig } from "../types";
+import type { ApiHandler, ApiStream, ApiStreamUsageChunk, HandlerModelInfo, ModelInfo, ProviderConfig } from "../types";
 import type { Message, ToolDefinition } from "../types/messages";
 import type { ApiStreamChunk } from "../types/stream";
 export declare const DEFAULT_REQUEST_HEADERS: Record<string, string>;
@@ -13,42 +13,18 @@ export declare const DEFAULT_REQUEST_HEADERS: Record<string, string>;
 export declare abstract class BaseHandler implements ApiHandler {
     protected config: ProviderConfig;
     protected abortController: AbortController | undefined;
+    private abortSignalSequence;
     constructor(config: ProviderConfig);
-    /**
-     * Convert Cline messages to provider-specific format
-     * Must be implemented by subclasses
-     */
     abstract getMessages(systemPrompt: string, messages: Message[]): unknown;
-    /**
-     * Create a streaming message completion
-     * Must be implemented by subclasses
-     */
     abstract createMessage(systemPrompt: string, messages: Message[], tools?: ToolDefinition[]): ApiStream;
-    /**
-     * Get the current model configuration
-     * Can be overridden by subclasses for provider-specific logic
-     */
     getModel(): HandlerModelInfo;
-    /**
-     * Get usage information (optional)
-     * Override in subclasses that support this
-     */
     getApiStreamUsage(): Promise<ApiStreamUsageChunk | undefined>;
-    /**
-     * Get the abort signal for the current request
-     * Creates a new AbortController if one doesn't exist or was already aborted
-     * Combines with config.abortSignal if provided
-     */
     protected getAbortSignal(): AbortSignal;
-    /**
-     * Abort the current request
-     */
     abort(): void;
     setAbortSignal(signal: AbortSignal | undefined): void;
-    /**
-     * Helper to calculate cost from usage
-     */
-    protected calculateCost(inputTokens: number, outputTokens: number, cacheReadTokens?: number): number | undefined;
+    private logAbort;
+    protected supportsPromptCache(modelInfo?: ModelInfo): boolean;
+    protected calculateCost(inputTokens: number, outputTokens: number, cacheReadTokens?: number, cacheWriteTokens?: number): number | undefined;
     protected createResponseId(): string;
     protected withResponseId<T extends ApiStreamChunk>(chunk: T, responseId: string): T;
     protected withResponseIdForAll(chunks: Iterable<ApiStreamChunk>, responseId: string): Generator<ApiStreamChunk>;

package/dist/providers/transform/openai-format.d.ts

@@ -9,7 +9,7 @@ type OpenAIMessage = OpenAI.Chat.ChatCompletionMessageParam;
 /**
  * Convert messages to OpenAI format
  */
-export declare function convertToOpenAIMessages(messages: Message[]): OpenAIMessage[];
+export declare function convertToOpenAIMessages(messages: Message[], enableCaching?: boolean): OpenAIMessage[];
 /**
  * Convert tool definitions to OpenAI format
  */

package/dist/providers/types/config.d.ts

@@ -187,6 +187,10 @@ export interface ProviderOptions {
     /** Runtime model catalog refresh configuration */
     modelCatalog?: ModelCatalogConfig;
 }
+/**
+ * Provider-specific options that don't fit other categories
+ */
+import type { BasicLogger } from "@clinebot/shared";
 /**
  * Runtime model catalog refresh options
  */
@@ -219,6 +223,8 @@ export interface ProviderConfig extends AuthConfig, EndpointConfig, ModelConfig,
     onRetryAttempt?: (attempt: number, maxRetries: number, delay: number, error: unknown) => void;
     /** AbortSignal for cancelling requests */
     abortSignal?: AbortSignal;
+    /** Optional runtime logger for provider-level diagnostics */
+    logger?: BasicLogger;
     /** Codex CLI-specific options */
     codex?: CodexConfig;
     /** Claude Code-specific options */

package/dist/providers/types/stream.d.ts

@@ -47,7 +47,7 @@ export interface ApiStreamReasoningChunk {
  */
 export interface ApiStreamUsageChunk {
     type: "usage";
-    /** Number of input tokens (excluding cached) */
+    /** Total number of input tokens reported by the provider */
     inputTokens: number;
     /** Number of output tokens */
     outputTokens: number;

package/package.json

@@ -1,10 +1,11 @@
 {
 	"name": "@clinebot/llms",
-	"version": "0.0.7",
+	"version": "0.0.11",
 	"description": "Config-driven SDK for selecting, extending, and instantiating LLM providers and models",
 	"main": "./dist/index.js",
 	"module": "./dist/index.js",
 	"dependencies": {
+		"@clinebot/shared": "0.0.11",
 		"@ai-sdk/amazon-bedrock": "^4.0.67",
 		"@ai-sdk/google-vertex": "^4.0.74",
 		"@ai-sdk/mistral": "^3.0.24",

package/src/index.browser.ts

@@ -1,6 +1,6 @@
 export { defineLlmsConfig, loadLlmsConfigFromFile } from "./config-browser";
-export * as models from "./models/index";
-export * as providers from "./providers/public.browser";
+export * as LlmsModels from "./models/index";
+export * as LlmsProviders from "./providers/public.browser";
 export type {
 	CustomProviderConfig,
 	LlmsConfig,

package/src/index.ts

@@ -1,6 +1,6 @@
 export { defineLlmsConfig } from "./config";
-export * as models from "./models/index";
-export * as providers from "./providers/public";
+export * as LlmsModels from "./models/index";
+export * as LlmsProviders from "./providers/public";
 export { createLlmsSdk } from "./sdk";
 export type {
 	CustomProviderConfig,

package/src/models/providers/vercel-ai-gateway.ts

@@ -14,7 +14,7 @@ export const VERCEL_AI_GATEWAY_PROVIDER: ModelCollection = {
 		name: "Vercel AI Gateway",
 		description: "Vercel's AI gateway service",
 		protocol: "openai-chat",
-		baseUrl: "https://ai-gateway.vercel.app/v1",
+		baseUrl: "https://ai-gateway.vercel.sh/v1",
 		defaultModelId: Object.keys(VERCEL_AI_GATEWAY_MODELS)[0],
 		capabilities: ["reasoning"],
 		env: ["AI_GATEWAY_API_KEY"],

package/src/providers/handlers/ai-sdk-community.ts

@@ -41,6 +41,7 @@ export type EmitAiSdkStreamOptions = {
 		inputTokens: number,
 		outputTokens: number,
 		cacheReadTokens: number,
+		cacheWriteTokens?: number,
 	) => number | undefined;
 	reasoningTypes?: string[];
 	enableToolCalls?: boolean;
@@ -168,10 +169,7 @@ export async function* emitAiSdkStream(
 
 				yield {
 					type: "usage",
-					inputTokens: Math.max(
-						0,
-						usageMetrics.inputTokens - usageMetrics.cacheReadTokens,
-					),
+					inputTokens: usageMetrics.inputTokens,
 					outputTokens: usageMetrics.outputTokens,
 					thoughtsTokenCount: usageMetrics.thoughtsTokenCount,
 					cacheReadTokens: usageMetrics.cacheReadTokens,
@@ -180,6 +178,7 @@ export async function* emitAiSdkStream(
 						usageMetrics.inputTokens,
 						usageMetrics.outputTokens,
 						usageMetrics.cacheReadTokens,
+						usageMetrics.cacheWriteTokens,
 					),
 					id: responseId,
 				};
@@ -205,10 +204,7 @@ export async function* emitAiSdkStream(
 		const usageMetrics = resolveUsageMetrics(usage);
 		yield {
 			type: "usage",
-			inputTokens: Math.max(
-				0,
-				usageMetrics.inputTokens - usageMetrics.cacheReadTokens,
-			),
+			inputTokens: usageMetrics.inputTokens,
 			outputTokens: usageMetrics.outputTokens,
 			thoughtsTokenCount: usageMetrics.thoughtsTokenCount,
 			cacheReadTokens: usageMetrics.cacheReadTokens,
@@ -217,6 +213,7 @@ export async function* emitAiSdkStream(
 				usageMetrics.inputTokens,
 				usageMetrics.outputTokens,
 				usageMetrics.cacheReadTokens,
+				usageMetrics.cacheWriteTokens,
 			),
 			id: responseId,
 		};

package/src/providers/handlers/ai-sdk-provider-base.ts

@@ -185,8 +185,18 @@ export abstract class AiSdkProviderHandler extends BaseHandler {
 		yield* emitAiSdkStream(stream, {
 			responseId,
 			errorMessage: this.getStreamErrorMessage(),
-			calculateCost: (inputTokens, outputTokens, cacheReadTokens) =>
-				this.calculateCost(inputTokens, outputTokens, cacheReadTokens),
+			calculateCost: (
+				inputTokens,
+				outputTokens,
+				cacheReadTokens,
+				cacheWriteTokens,
+			) =>
+				this.calculateCost(
+					inputTokens,
+					outputTokens,
+					cacheReadTokens,
+					cacheWriteTokens,
+				),
 			...this.getEmitStreamOptions(),
 		});
 	}

package/src/providers/handlers/anthropic-base.test.ts

@@ -0,0 +1,30 @@
+import { describe, expect, it } from "vitest";
+import { AnthropicHandler } from "./anthropic-base";
+
+describe("AnthropicHandler prompt cache detection", () => {
+	it("enables prompt caching when model pricing includes cache pricing", () => {
+		const handler = new AnthropicHandler({
+			providerId: "anthropic",
+			modelId: "claude-sonnet-4-6",
+			apiKey: "test-key",
+			modelInfo: {
+				id: "claude-sonnet-4-6",
+				pricing: {
+					input: 3,
+					output: 15,
+					cacheRead: 0.3,
+					cacheWrite: 3.75,
+				},
+			},
+		});
+
+		const messages = handler.getMessages("system", [
+			{ role: "user", content: "Tell me about this repo" },
+		]);
+		const userTextBlock = messages[0]?.content?.[0] as
+			| { cache_control?: { type: string } }
+			| undefined;
+
+		expect(userTextBlock?.cache_control).toEqual({ type: "ephemeral" });
+	});
+});

package/src/providers/handlers/anthropic-base.ts

@@ -17,7 +17,6 @@ import {
 import {
 	type ApiStream,
 	type HandlerModelInfo,
-	hasModelCapability,
 	type ProviderConfig,
 	supportsModelThinking,
 } from "../types";
@@ -76,10 +75,7 @@ export class AnthropicHandler extends BaseHandler {
 		_systemPrompt: string,
 		messages: Message[],
 	): Anthropic.MessageParam[] {
-		const supportsPromptCache = hasModelCapability(
-			this.getModel().info,
-			"prompt-cache",
-		);
+		const supportsPromptCache = this.supportsPromptCache(this.getModel().info);
 		return convertToAnthropicMessages(
 			messages,
 			supportsPromptCache,
@@ -113,7 +109,7 @@ export class AnthropicHandler extends BaseHandler {
 		const budgetTokens =
 			thinkingSupported && requestedBudget > 0 ? requestedBudget : 0;
 		const nativeToolsOn = tools && tools.length > 0;
-		const supportsPromptCache = hasModelCapability(model.info, "prompt-cache");
+		const supportsPromptCache = this.supportsPromptCache(model.info);
 		const reasoningOn = thinkingSupported && budgetTokens > 0;
 		const debugThinking = isThinkingDebugEnabled();
 		const debugChunkCounts: Record<string, number> = {};
@@ -139,29 +135,34 @@ export class AnthropicHandler extends BaseHandler {
 		const requestOptions = { signal: abortSignal };
 
 		// Create the request
+		// Use top-level automatic caching so the entire prefix (system +
+		// messages) is cached and the breakpoint advances each turn.
+		const createParams: Record<string, unknown> &
+			Anthropic.MessageCreateParamsStreaming = {
+			model: model.id,
+			thinking: reasoningOn
+				? { type: "enabled", budget_tokens: budgetTokens }
+				: undefined,
+			max_tokens:
+				model.info.maxTokens ?? this.config.maxOutputTokens ?? 128_000,
+			temperature: reasoningOn ? undefined : 0,
+			system: [
+				supportsPromptCache
+					? {
+							text: systemPrompt,
+							type: "text",
+							cache_control: { type: "ephemeral" },
+						}
+					: { text: systemPrompt, type: "text" },
+			],
+			messages: anthropicMessages as Anthropic.MessageParam[],
+			stream: true,
+			tools: anthropicTools,
+			tool_choice: nativeToolsOn && !reasoningOn ? { type: "auto" } : undefined,
+		};
+
 		const stream = await client.messages.create(
-			{
-				model: model.id,
-				thinking: reasoningOn
-					? { type: "enabled", budget_tokens: budgetTokens }
-					: undefined,
-				max_tokens: model.info.maxTokens ?? this.config.maxOutputTokens ?? 8192,
-				temperature: reasoningOn ? undefined : 0,
-				system: supportsPromptCache
-					? [
-							{
-								text: systemPrompt,
-								type: "text",
-								cache_control: { type: "ephemeral" },
-							},
-						]
-					: [{ text: systemPrompt, type: "text" }],
-				messages: anthropicMessages as Anthropic.MessageParam[],
-				stream: true,
-				tools: anthropicTools,
-				tool_choice:
-					nativeToolsOn && !reasoningOn ? { type: "auto" } : undefined,
-			},
+			createParams as Anthropic.MessageCreateParamsStreaming,
 			requestOptions,
 		);
 
@@ -173,6 +174,7 @@ export class AnthropicHandler extends BaseHandler {
 			cacheReadTokens: 0,
 			cacheWriteTokens: 0,
 		};
+		let stopReason: string | null = null;
 
 		for await (const chunk of stream) {
 			if (debugThinking) {
@@ -185,6 +187,11 @@ export class AnthropicHandler extends BaseHandler {
 					countChunk(`content_block_delta:${chunk.delta?.type ?? "unknown"}`);
 				}
 			}
+			if (chunk.type === "message_delta") {
+				stopReason =
+					(chunk as { delta?: { stop_reason?: string } }).delta?.stop_reason ??
+					stopReason;
+			}
 			yield* this.withResponseIdForAll(
 				this.processChunk(chunk, currentToolCall, usageSnapshot, responseId),
 				responseId,
@@ -199,8 +206,12 @@ export class AnthropicHandler extends BaseHandler {
 			console.error(`[thinking-debug][anthropic][stream] ${summary}`);
 		}
 
-		// Yield done chunk to indicate streaming completed successfully
-		yield { type: "done", success: true, id: responseId };
+		yield {
+			type: "done",
+			success: true,
+			id: responseId,
+			incompleteReason: stopReason === "max_tokens" ? "max_tokens" : undefined,
+		};
 	}
 
 	protected *processChunk(
@@ -233,6 +244,7 @@ export class AnthropicHandler extends BaseHandler {
 						usageSnapshot.inputTokens,
 						usageSnapshot.outputTokens,
 						usageSnapshot.cacheReadTokens,
+						usageSnapshot.cacheWriteTokens,
 					),
 					id: responseId,
 				};
@@ -252,6 +264,7 @@ export class AnthropicHandler extends BaseHandler {
 						usageSnapshot.inputTokens,
 						usageSnapshot.outputTokens,
 						usageSnapshot.cacheReadTokens,
+						usageSnapshot.cacheWriteTokens,
 					),
 					id: responseId,
 				};

package/src/providers/handlers/base.test.ts

@@ -1,4 +1,4 @@
-import { describe, expect, it } from "vitest";
+import { describe, expect, it, vi } from "vitest";
 import type { ApiStream, ProviderConfig } from "../types/index";
 import { BaseHandler } from "./base";
 
@@ -15,8 +15,18 @@ class TestHandler extends BaseHandler {
 		inputTokens: number,
 		outputTokens: number,
 		cacheReadTokens = 0,
+		cacheWriteTokens = 0,
 	): number | undefined {
-		return this.calculateCost(inputTokens, outputTokens, cacheReadTokens);
+		return this.calculateCost(
+			inputTokens,
+			outputTokens,
+			cacheReadTokens,
+			cacheWriteTokens,
+		);
+	}
+
+	public exposeAbortSignal(): AbortSignal {
+		return this.getAbortSignal();
 	}
 }
 
@@ -41,6 +51,61 @@ describe("BaseHandler.calculateCost", () => {
 
 		const cost = handler.computeCost(1_000_000, 1_000_000, 100_000);
 
-		expect(cost).toBeCloseTo(17.73, 6);
+		expect(cost).toBeCloseTo(18.03, 6);
+	});
+});
+
+describe("BaseHandler abort signal wiring", () => {
+	it("does not let a stale request signal abort a newer request", () => {
+		const logger = {
+			debug: vi.fn(),
+			warn: vi.fn(),
+		};
+		const request1 = new AbortController();
+		const handler = new TestHandler({
+			providerId: "openrouter",
+			modelId: "mock-model",
+			apiKey: "test-key",
+			baseUrl: "https://example.com/v1",
+			abortSignal: request1.signal,
+			logger,
+		});
+
+		const signal1 = handler.exposeAbortSignal();
+		expect(signal1.aborted).toBe(false);
+
+		const request2 = new AbortController();
+		handler.setAbortSignal(request2.signal);
+		const signal2 = handler.exposeAbortSignal();
+		expect(signal2).not.toBe(signal1);
+		expect(signal2.aborted).toBe(false);
+
+		request1.abort(new Error("stale timeout"));
+
+		expect(signal1.aborted).toBe(true);
+		expect(signal2.aborted).toBe(false);
+		expect(logger.warn).toHaveBeenCalledWith(
+			"Provider request abort signal fired",
+			expect.objectContaining({
+				reason: expect.objectContaining({ message: "stale timeout" }),
+			}),
+		);
+	});
+
+	it("creates a fresh controller for each request", () => {
+		const handler = new TestHandler({
+			providerId: "openrouter",
+			modelId: "mock-model",
+			apiKey: "test-key",
+			baseUrl: "https://example.com/v1",
+			abortSignal: new AbortController().signal,
+		});
+
+		const signal1 = handler.exposeAbortSignal();
+		const signal2 = handler.exposeAbortSignal();
+
+		expect(signal2).not.toBe(signal1);
+		expect(signal1.aborted).toBe(false);
+		expect(signal2.aborted).toBe(false);
 	});
 });