@clinebot/llms 0.0.0

package/src/providers/handlers/anthropic-base.ts

@@ -0,0 +1,372 @@
+/**
+ * Anthropic Base Handler
+ *
+ * Handler for Anthropic's API using the official SDK.
+ * Supports prompt caching, extended thinking, and native tool calling.
+ */
+
+import { Anthropic } from "@anthropic-ai/sdk";
+import type {
+	Tool as AnthropicTool,
+	RawMessageStreamEvent,
+} from "@anthropic-ai/sdk/resources";
+import {
+	convertToAnthropicMessages,
+	convertToolsToAnthropic,
+} from "../transform/anthropic-format";
+import {
+	type ApiStream,
+	type HandlerModelInfo,
+	hasModelCapability,
+	type ProviderConfig,
+	supportsModelThinking,
+} from "../types";
+import type { Message, ToolDefinition } from "../types/messages";
+import { retryStream } from "../utils/retry";
+import { getMissingApiKeyError, resolveApiKeyForProvider } from "./auth";
+import { BaseHandler } from "./base";
+
+const DEFAULT_THINKING_BUDGET_TOKENS = 1024;
+const THINKING_DEBUG_ENV = "CLINE_DEBUG_THINKING";
+
+function isThinkingDebugEnabled(): boolean {
+	const raw = process.env[THINKING_DEBUG_ENV];
+	if (!raw) {
+		return false;
+	}
+	const normalized = raw.trim().toLowerCase();
+	return normalized === "1" || normalized === "true" || normalized === "yes";
+}
+
+/**
+ * Handler for Anthropic's API
+ */
+export class AnthropicHandler extends BaseHandler {
+	private client: Anthropic | undefined;
+
+	private ensureClient(): Anthropic {
+		if (!this.client) {
+			const apiKey = resolveApiKeyForProvider(
+				this.config.providerId,
+				this.config.apiKey,
+			);
+			if (!apiKey) {
+				throw new Error(getMissingApiKeyError(this.config.providerId));
+			}
+
+			this.client = new Anthropic({
+				apiKey,
+				baseURL: this.config.baseUrl || undefined,
+				defaultHeaders: this.getRequestHeaders(),
+			});
+		}
+		return this.client;
+	}
+
+	getModel(): HandlerModelInfo {
+		const modelId = this.config.modelId;
+		const knownModels = this.config.knownModels ?? {};
+		const fallbackModel = knownModels[modelId] ?? {};
+		const modelInfo = this.config.modelInfo ?? fallbackModel;
+
+		return { id: modelId, info: { ...modelInfo, id: modelId } };
+	}
+
+	getMessages(
+		_systemPrompt: string,
+		messages: Message[],
+	): Anthropic.MessageParam[] {
+		const supportsPromptCache = hasModelCapability(
+			this.getModel().info,
+			"prompt-cache",
+		);
+		return convertToAnthropicMessages(
+			messages,
+			supportsPromptCache,
+		) as Anthropic.MessageParam[];
+	}
+
+	async *createMessage(
+		systemPrompt: string,
+		messages: Message[],
+		tools?: ToolDefinition[],
+	): ApiStream {
+		yield* retryStream(() =>
+			this.createMessageInternal(systemPrompt, messages, tools),
+		);
+	}
+
+	private async *createMessageInternal(
+		systemPrompt: string,
+		messages: Message[],
+		tools?: ToolDefinition[],
+	): ApiStream {
+		const client = this.ensureClient();
+		const model = this.getModel();
+		const abortSignal = this.getAbortSignal();
+		const responseId = this.createResponseId();
+
+		const thinkingSupported = supportsModelThinking(model.info);
+		const requestedBudget =
+			this.config.thinkingBudgetTokens ??
+			(this.config.thinking ? DEFAULT_THINKING_BUDGET_TOKENS : 0);
+		const budgetTokens =
+			thinkingSupported && requestedBudget > 0 ? requestedBudget : 0;
+		const nativeToolsOn = tools && tools.length > 0;
+		const supportsPromptCache = hasModelCapability(model.info, "prompt-cache");
+		const reasoningOn = thinkingSupported && budgetTokens > 0;
+		const debugThinking = isThinkingDebugEnabled();
+		const debugChunkCounts: Record<string, number> = {};
+		const countChunk = (type: string): void => {
+			debugChunkCounts[type] = (debugChunkCounts[type] ?? 0) + 1;
+		};
+
+		if (debugThinking) {
+			console.error(
+				`[thinking-debug][anthropic][request] model=${model.id} thinkingFlag=${this.config.thinking === true} supportsModelThinking=${thinkingSupported} requestedBudget=${requestedBudget} effectiveBudget=${budgetTokens} reasoningOn=${reasoningOn} promptCache=${supportsPromptCache}`,
+			);
+		}
+
+		// Convert messages
+		const anthropicMessages = this.getMessages(systemPrompt, messages);
+
+		// Convert tools
+		const anthropicTools: AnthropicTool[] | undefined = nativeToolsOn
+			? convertToolsToAnthropic(tools)
+			: undefined;
+
+		// Request options with abort signal
+		const requestOptions = { signal: abortSignal };
+
+		// Create the request
+		const stream = await client.messages.create(
+			{
+				model: model.id,
+				thinking: reasoningOn
+					? { type: "enabled", budget_tokens: budgetTokens }
+					: undefined,
+				max_tokens: model.info.maxTokens || 8192,
+				temperature: reasoningOn ? undefined : 0,
+				system: supportsPromptCache
+					? [
+							{
+								text: systemPrompt,
+								type: "text",
+								cache_control: { type: "ephemeral" },
+							},
+						]
+					: [{ text: systemPrompt, type: "text" }],
+				messages: anthropicMessages as Anthropic.MessageParam[],
+				stream: true,
+				tools: anthropicTools,
+				tool_choice:
+					nativeToolsOn && !reasoningOn ? { type: "auto" } : undefined,
+			},
+			requestOptions,
+		);
+
+		// Track tool call state
+		const currentToolCall = { id: "", name: "", arguments: "" };
+		const usageSnapshot = {
+			inputTokens: 0,
+			outputTokens: 0,
+			cacheReadTokens: 0,
+			cacheWriteTokens: 0,
+		};
+
+		for await (const chunk of stream) {
+			if (debugThinking) {
+				countChunk(`event:${chunk.type}`);
+				if (chunk.type === "content_block_start") {
+					countChunk(
+						`content_block_start:${chunk.content_block?.type ?? "unknown"}`,
+					);
+				} else if (chunk.type === "content_block_delta") {
+					countChunk(`content_block_delta:${chunk.delta?.type ?? "unknown"}`);
+				}
+			}
+			yield* this.withResponseIdForAll(
+				this.processChunk(chunk, currentToolCall, usageSnapshot, responseId),
+				responseId,
+			);
+		}
+
+		if (debugThinking) {
+			const summary = Object.entries(debugChunkCounts)
+				.map(([key, count]) => `${key}=${count}`)
+				.sort()
+				.join(" ");
+			console.error(`[thinking-debug][anthropic][stream] ${summary}`);
+		}
+
+		// Yield done chunk to indicate streaming completed successfully
+		yield { type: "done", success: true, id: responseId };
+	}
+
+	private *processChunk(
+		chunk: RawMessageStreamEvent,
+		currentToolCall: { id: string; name: string; arguments: string },
+		usageSnapshot: {
+			inputTokens: number;
+			outputTokens: number;
+			cacheReadTokens: number;
+			cacheWriteTokens: number;
+		},
+		responseId: string,
+	): Generator<import("../types").ApiStreamChunk> {
+		switch (chunk.type) {
+			case "message_start": {
+				const usage = chunk.message.usage;
+				usageSnapshot.inputTokens = usage.input_tokens || 0;
+				usageSnapshot.outputTokens = usage.output_tokens || 0;
+				usageSnapshot.cacheWriteTokens =
+					(usage as any).cache_creation_input_tokens || 0;
+				usageSnapshot.cacheReadTokens =
+					(usage as any).cache_read_input_tokens || 0;
+				yield {
+					type: "usage",
+					inputTokens: usageSnapshot.inputTokens,
+					outputTokens: usageSnapshot.outputTokens,
+					cacheWriteTokens: usageSnapshot.cacheWriteTokens,
+					cacheReadTokens: usageSnapshot.cacheReadTokens,
+					totalCost: this.calculateCost(
+						usageSnapshot.inputTokens,
+						usageSnapshot.outputTokens,
+						usageSnapshot.cacheReadTokens,
+					),
+					id: responseId,
+				};
+				break;
+			}
+
+			case "message_delta": {
+				usageSnapshot.outputTokens =
+					chunk.usage.output_tokens || usageSnapshot.outputTokens;
+				yield {
+					type: "usage",
+					inputTokens: usageSnapshot.inputTokens,
+					outputTokens: usageSnapshot.outputTokens,
+					cacheWriteTokens: usageSnapshot.cacheWriteTokens,
+					cacheReadTokens: usageSnapshot.cacheReadTokens,
+					totalCost: this.calculateCost(
+						usageSnapshot.inputTokens,
+						usageSnapshot.outputTokens,
+						usageSnapshot.cacheReadTokens,
+					),
+					id: responseId,
+				};
+				break;
+			}
+
+			case "content_block_start": {
+				const block = chunk.content_block;
+				switch (block.type) {
+					case "thinking":
+						yield {
+							type: "reasoning",
+							reasoning:
+								typeof (block as { thinking?: unknown }).thinking === "string"
+									? ((block as { thinking: string }).thinking ?? "")
+									: "",
+							signature:
+								typeof (block as { signature?: unknown }).signature === "string"
+									? ((block as { signature: string }).signature ?? undefined)
+									: undefined,
+							id: responseId,
+						};
+						break;
+					case "redacted_thinking":
+						yield {
+							type: "reasoning",
+							reasoning: "",
+							redacted_data:
+								typeof (block as { data?: unknown }).data === "string"
+									? ((block as { data: string }).data ?? undefined)
+									: undefined,
+							id: responseId,
+						};
+						break;
+					case "text":
+						yield { type: "text", text: "", id: responseId };
+						break;
+					case "tool_use":
+						currentToolCall.id = block.id;
+						currentToolCall.name = block.name;
+						currentToolCall.arguments = "";
+						break;
+				}
+				break;
+			}
+
+			case "content_block_delta": {
+				const delta = chunk.delta;
+				switch (delta.type) {
+					case "thinking_delta":
+						yield {
+							type: "reasoning",
+							reasoning: delta.thinking,
+							id: responseId,
+						};
+						break;
+					case "signature_delta":
+						yield {
+							type: "reasoning",
+							reasoning: "",
+							signature:
+								typeof (delta as { signature?: unknown }).signature === "string"
+									? ((delta as { signature: string }).signature ?? undefined)
+									: undefined,
+							id: responseId,
+						};
+						break;
+					case "text_delta":
+						yield { type: "text", text: delta.text, id: responseId };
+						break;
+					case "input_json_delta":
+						currentToolCall.arguments += delta.partial_json;
+						break;
+				}
+				break;
+			}
+
+			case "content_block_stop": {
+				// If we have a tool call, yield it
+				if (currentToolCall.id) {
+					let parsedArgs: Record<string, unknown>;
+					try {
+						parsedArgs = JSON.parse(currentToolCall.arguments || "{}");
+					} catch {
+						parsedArgs = {};
+					}
+
+					yield {
+						type: "tool_calls",
+						id: responseId,
+						tool_call: {
+							call_id: currentToolCall.id,
+							function: {
+								name: currentToolCall.name,
+								arguments: parsedArgs,
+							},
+						},
+					};
+
+					// Reset tool call state
+					currentToolCall.id = "";
+					currentToolCall.name = "";
+					currentToolCall.arguments = "";
+				}
+				break;
+			}
+		}
+	}
+}
+
+/**
+ * Create an Anthropic handler
+ */
+export function createAnthropicHandler(
+	config: ProviderConfig,
+): AnthropicHandler {
+	return new AnthropicHandler(config);
+}

package/src/providers/handlers/asksage.test.ts

@@ -0,0 +1,103 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { createHandler } from "../index";
+import type { ApiStreamChunk } from "../types";
+import { AskSageHandler } from "./asksage";
+
+vi.mock("./auth", async () => {
+	const actual = await vi.importActual("./auth");
+	return {
+		...(actual as object),
+		resolveApiKeyForProvider: (_providerId: string, explicitApiKey?: string) =>
+			explicitApiKey?.trim() || undefined,
+	};
+});
+
+describe("AskSageHandler", () => {
+	beforeEach(() => {
+		vi.restoreAllMocks();
+	});
+
+	it("formats request payload and emits text/usage/done chunks", async () => {
+		const fetchMock = vi.fn(async () => ({
+			ok: true,
+			json: async () => ({
+				message: "final answer",
+				tool_responses: [{ name: "search", ok: true }],
+				usage: {
+					model_tokens: {
+						prompt_tokens: 123,
+						completion_tokens: 45,
+						total_tokens: 168,
+					},
+					asksage_tokens: 17.5,
+				},
+			}),
+		}));
+		globalThis.fetch = fetchMock as unknown as typeof fetch;
+
+		const handler = new AskSageHandler({
+			providerId: "asksage",
+			modelId: "gpt-4o",
+			apiKey: "ask-key",
+		});
+
+		const chunks: ApiStreamChunk[] = [];
+		for await (const chunk of handler.createMessage("system prompt", [
+			{ role: "user", content: [{ type: "text", text: "hello" }] },
+			{ role: "assistant", content: "hi there" },
+		])) {
+			chunks.push(chunk);
+		}
+
+		expect(fetchMock).toHaveBeenCalledTimes(1);
+		const [url, init] = fetchMock.mock.calls[0] as unknown as [
+			string,
+			RequestInit & { body?: string },
+		];
+		expect(url).toBe("https://api.asksage.ai/server/query");
+		expect(init.method).toBe("POST");
+		expect(init.headers).toMatchObject({
+			"Content-Type": "application/json",
+			"x-access-tokens": "ask-key",
+		});
+		expect(JSON.parse(init.body ?? "{}")).toEqual({
+			system_prompt: "system prompt",
+			message: [
+				{ user: "me", message: "hello" },
+				{ user: "gpt", message: "hi there" },
+			],
+			model: "gpt-4o",
+			dataset: "none",
+			usage: true,
+		});
+
+		expect(chunks.map((chunk) => chunk.type)).toEqual([
+			"text",
+			"text",
+			"usage",
+			"done",
+		]);
+	});
+
+	it("is used by createHandler for built-in asksage provider id", () => {
+		const handler = createHandler({
+			providerId: "asksage",
+			modelId: "gpt-4o",
+			apiKey: "ask-key",
+		});
+		expect(handler).toBeInstanceOf(AskSageHandler);
+	});
+
+	it("throws when API key is missing", async () => {
+		const handler = new AskSageHandler({
+			providerId: "asksage",
+			modelId: "gpt-4o",
+		});
+
+		await expect(async () => {
+			for await (const _chunk of handler.createMessage("system", [])) {
+				// noop
+			}
+		}).rejects.toThrow("AskSage API key is required");
+	});
+});

package/src/providers/handlers/asksage.ts

@@ -0,0 +1,138 @@
+import type { ApiStream, HandlerModelInfo, ProviderConfig } from "../types";
+import type { ContentBlock, Message } from "../types/messages";
+import { resolveApiKeyForProvider } from "./auth";
+import { FetchBaseHandler } from "./fetch-base";
+
+export const DEFAULT_ASKSAGE_BASE_URL = "https://api.asksage.ai/server";
+const DEFAULT_ASKSAGE_MODEL_ID = "gpt-4o";
+
+type AskSageRequest = {
+	system_prompt: string;
+	message: Array<{
+		user: "gpt" | "me";
+		message: string;
+	}>;
+	model: string;
+	dataset: "none";
+	usage: boolean;
+};
+
+type AskSageUsage = {
+	model_tokens: {
+		completion_tokens: number;
+		prompt_tokens: number;
+		total_tokens: number;
+	};
+	asksage_tokens: number;
+};
+
+type AskSageResponse = {
+	message?: string;
+	usage?: AskSageUsage | null;
+	tool_responses?: unknown[];
+};
+
+export class AskSageHandler extends FetchBaseHandler {
+	protected getDefaultBaseUrl(): string {
+		return DEFAULT_ASKSAGE_BASE_URL;
+	}
+
+	getModel(): HandlerModelInfo {
+		const modelId = this.config.modelId?.trim() || DEFAULT_ASKSAGE_MODEL_ID;
+		const modelInfo = this.config.modelInfo ??
+			this.config.knownModels?.[modelId] ?? {
+				id: modelId,
+				capabilities: ["tools"],
+			};
+		return { id: modelId, info: { ...modelInfo, id: modelId } };
+	}
+
+	protected getJsonHeaders(
+		extra?: Record<string, string>,
+	): Record<string, string> {
+		const apiKey = resolveApiKeyForProvider(
+			this.config.providerId,
+			this.config.apiKey,
+		);
+		if (!apiKey) {
+			throw new Error("AskSage API key is required");
+		}
+		return super.getJsonHeaders({
+			"x-access-tokens": apiKey,
+			...(extra ?? {}),
+		});
+	}
+
+	protected async *createMessageWithFetch(
+		systemPrompt: string,
+		messages: Message[],
+	): ApiStream {
+		const responseId = this.createResponseId();
+		const { id: modelId } = this.getModel();
+
+		const payload: AskSageRequest = {
+			system_prompt: systemPrompt,
+			message: messages.map((message) => ({
+				user: message.role === "assistant" ? "gpt" : "me",
+				message: this.serializeMessageContent(message.content),
+			})),
+			model: modelId,
+			dataset: "none",
+			usage: true,
+		};
+
+		let result: AskSageResponse;
+		try {
+			result = await this.fetchJson<AskSageResponse>("/query", {
+				method: "POST",
+				body: payload,
+			});
+		} catch (error) {
+			const details = error instanceof Error ? error.message : String(error);
+			throw new Error(`AskSage request failed: ${details}`);
+		}
+
+		for (const toolResponse of result.tool_responses ?? []) {
+			yield {
+				type: "text",
+				text: `[Tool Response: ${JSON.stringify(toolResponse)}]\n`,
+				id: responseId,
+			};
+		}
+
+		const text = result.message?.trim();
+		if (!text) {
+			throw new Error("AskSage request failed: no content in response");
+		}
+
+		yield { type: "text", text, id: responseId };
+
+		if (result.usage) {
+			yield {
+				type: "usage",
+				inputTokens: result.usage.model_tokens.prompt_tokens,
+				outputTokens: result.usage.model_tokens.completion_tokens,
+				cacheReadTokens: 0,
+				cacheWriteTokens: 0,
+				totalCost: result.usage.asksage_tokens,
+				id: responseId,
+			};
+		}
+
+		yield { type: "done", success: true, id: responseId };
+	}
+
+	private serializeMessageContent(content: string | ContentBlock[]): string {
+		if (typeof content === "string") {
+			return content;
+		}
+		return content
+			.map((block) => ("text" in block ? block.text : ""))
+			.join("")
+			.trim();
+	}
+}
+
+export function createAskSageHandler(config: ProviderConfig): AskSageHandler {
+	return new AskSageHandler(config);
+}

package/src/providers/handlers/auth.test.ts

@@ -0,0 +1,19 @@
+import { describe, expect, it } from "vitest";
+import { resolveApiKeyForProvider } from "./auth";
+
+describe("resolveApiKeyForProvider", () => {
+	it("returns noop for lmstudio when no key is provided", () => {
+		const apiKey = resolveApiKeyForProvider("lmstudio", undefined, {});
+		expect(apiKey).toBe("noop");
+	});
+
+	it("prefers explicit api keys over provider defaults", () => {
+		const apiKey = resolveApiKeyForProvider("lmstudio", "real-key", {});
+		expect(apiKey).toBe("real-key");
+	});
+
+	it("does not apply lmstudio fallback to zai", () => {
+		const apiKey = resolveApiKeyForProvider("zai", undefined, {});
+		expect(apiKey).toBeUndefined();
+	});
+});