@clinebot/llms 0.0.0

package/src/providers/handlers/auth.ts

@@ -0,0 +1,121 @@
+import * as modelProviderExports from "../../models/providers/index";
+import type { ModelCollection } from "../../models/schemas/index";
+import { BUILT_IN_PROVIDER, normalizeProviderId } from "../types/provider-ids";
+
+const DEFAULT_FALLBACK_PROVIDER_IDS = [
+	BUILT_IN_PROVIDER.CLINE,
+	BUILT_IN_PROVIDER.ANTHROPIC,
+	BUILT_IN_PROVIDER.OPENAI_NATIVE,
+	BUILT_IN_PROVIDER.GEMINI,
+	BUILT_IN_PROVIDER.OPENROUTER,
+] as const;
+
+function isModelCollection(value: unknown): value is ModelCollection {
+	if (!value || typeof value !== "object") {
+		return false;
+	}
+
+	const maybeCollection = value as Partial<ModelCollection>;
+	return (
+		typeof maybeCollection.provider === "object" &&
+		typeof maybeCollection.models === "object"
+	);
+}
+
+function dedupe(values: readonly string[]): string[] {
+	return [...new Set(values)];
+}
+
+function buildProviderEnvKeys(): Record<string, readonly string[]> {
+	const envKeysByProvider: Record<string, readonly string[]> = {};
+
+	for (const value of Object.values(modelProviderExports)) {
+		if (!isModelCollection(value)) {
+			continue;
+		}
+
+		const providerId = value.provider.id;
+		envKeysByProvider[providerId] = dedupe(value.provider.env ?? []);
+	}
+
+	return envKeysByProvider;
+}
+
+const ENV_KEYS_BY_PROVIDER = buildProviderEnvKeys();
+const DEFAULT_FALLBACK_ENV_KEYS = dedupe(
+	DEFAULT_FALLBACK_PROVIDER_IDS.flatMap(
+		(providerId) => ENV_KEYS_BY_PROVIDER[providerId] ?? [],
+	),
+);
+
+function readTrimmed(
+	env: Record<string, string | undefined>,
+	key: string,
+): string | undefined {
+	const value = env[key];
+	if (typeof value !== "string") {
+		return undefined;
+	}
+	const trimmed = value.trim();
+	return trimmed.length > 0 ? trimmed : undefined;
+}
+
+function resolveFromKeys(
+	keys: readonly string[],
+	env: Record<string, string | undefined>,
+): string | undefined {
+	for (const key of keys) {
+		const value = readTrimmed(env, key);
+		if (value) {
+			return value;
+		}
+	}
+	return undefined;
+}
+
+export { normalizeProviderId };
+
+export function getProviderEnvKeys(providerId: string): readonly string[] {
+	return ENV_KEYS_BY_PROVIDER[normalizeProviderId(providerId)] ?? [];
+}
+
+export function resolveApiKeyForProvider(
+	providerId: string,
+	explicitApiKey: string | undefined,
+	env: Record<string, string | undefined> = process.env,
+): string | undefined {
+	const normalizedProviderId = normalizeProviderId(providerId);
+	const explicit = explicitApiKey?.trim();
+	if (explicit) {
+		return explicit;
+	}
+
+	const providerKey = resolveFromKeys(
+		getProviderEnvKeys(normalizedProviderId),
+		env,
+	);
+	if (providerKey) {
+		return providerKey;
+	}
+
+	// LM Studio local runtime typically does not require auth.
+	if (normalizedProviderId === BUILT_IN_PROVIDER.LMSTUDIO) {
+		return "noop";
+	}
+
+	return resolveFromKeys(DEFAULT_FALLBACK_ENV_KEYS, env);
+}
+
+export function getMissingApiKeyError(providerId: string): string {
+	const expectedKeys = [
+		...new Set([
+			...getProviderEnvKeys(providerId),
+			...DEFAULT_FALLBACK_ENV_KEYS,
+		]),
+	];
+	const keysMessage =
+		expectedKeys.length > 0
+			? expectedKeys.join(", ")
+			: "provider-specific API key env var";
+	return `Missing API key for provider "${normalizeProviderId(providerId)}". Set apiKey explicitly or one of: ${keysMessage}.`;
+}

package/src/providers/handlers/base.test.ts

@@ -0,0 +1,46 @@
+import { describe, expect, it } from "vitest";
+import type { ApiStream, ProviderConfig } from "../types/index";
+import { BaseHandler } from "./base";
+
+class TestHandler extends BaseHandler {
+	getMessages(): unknown {
+		return [];
+	}
+
+	createMessage(): ApiStream {
+		throw new Error("not implemented");
+	}
+
+	public computeCost(
+		inputTokens: number,
+		outputTokens: number,
+		cacheReadTokens = 0,
+	): number | undefined {
+		return this.calculateCost(inputTokens, outputTokens, cacheReadTokens);
+	}
+}
+
+describe("BaseHandler.calculateCost", () => {
+	it("uses known model pricing when modelInfo is not provided", () => {
+		const config: ProviderConfig = {
+			providerId: "anthropic",
+			modelId: "claude-sonnet-test",
+			apiKey: "test-key",
+			knownModels: {
+				"claude-sonnet-test": {
+					id: "claude-sonnet-test",
+					pricing: {
+						input: 3,
+						output: 15,
+						cacheRead: 0.3,
+					},
+				},
+			},
+		};
+		const handler = new TestHandler(config);
+
+		const cost = handler.computeCost(1_000_000, 1_000_000, 100_000);
+
+		expect(cost).toBeCloseTo(17.73, 6);
+	});
+});

package/src/providers/handlers/base.ts

@@ -0,0 +1,160 @@
+/**
+ * Base Handler
+ *
+ * Abstract base class that provides common functionality for all handlers.
+ */
+
+import { nanoid } from "nanoid";
+import type {
+	ApiHandler,
+	ApiStream,
+	ApiStreamUsageChunk,
+	HandlerModelInfo,
+	ProviderConfig,
+} from "../types";
+import type { Message, ToolDefinition } from "../types/messages";
+import type { ApiStreamChunk } from "../types/stream";
+
+export const DEFAULT_REQUEST_HEADERS: Record<string, string> = {
+	"HTTP-Referer": "https://cline.bot",
+	"X-Title": "Cline",
+	"X-IS-MULTIROOT": "false",
+	"X-CLIENT-TYPE": "cline-sdk",
+};
+
+/**
+ * Base handler class with common functionality
+ */
+export abstract class BaseHandler implements ApiHandler {
+	protected config: ProviderConfig;
+	protected abortController: AbortController | undefined;
+
+	constructor(config: ProviderConfig) {
+		this.config = config;
+	}
+
+	/**
+	 * Convert Cline messages to provider-specific format
+	 * Must be implemented by subclasses
+	 */
+	abstract getMessages(systemPrompt: string, messages: Message[]): unknown;
+
+	/**
+	 * Create a streaming message completion
+	 * Must be implemented by subclasses
+	 */
+	abstract createMessage(
+		systemPrompt: string,
+		messages: Message[],
+		tools?: ToolDefinition[],
+	): ApiStream;
+
+	/**
+	 * Get the current model configuration
+	 * Can be overridden by subclasses for provider-specific logic
+	 */
+	getModel(): HandlerModelInfo {
+		const modelId = this.config.modelId;
+		return {
+			id: modelId,
+			info: { ...(this.config.modelInfo ?? {}), id: modelId },
+		};
+	}
+
+	/**
+	 * Get usage information (optional)
+	 * Override in subclasses that support this
+	 */
+	async getApiStreamUsage(): Promise<ApiStreamUsageChunk | undefined> {
+		return undefined;
+	}
+
+	/**
+	 * Get the abort signal for the current request
+	 * Creates a new AbortController if one doesn't exist or was already aborted
+	 * Combines with config.abortSignal if provided
+	 */
+	protected getAbortSignal(): AbortSignal {
+		// Create a new controller if needed
+		if (!this.abortController || this.abortController.signal.aborted) {
+			this.abortController = new AbortController();
+		}
+
+		// If a signal was provided in config, chain it
+		if (this.config.abortSignal) {
+			const configSignal = this.config.abortSignal;
+			if (configSignal.aborted) {
+				this.abortController.abort(configSignal.reason);
+			} else {
+				configSignal.addEventListener("abort", () => {
+					this.abortController?.abort(configSignal.reason);
+				});
+			}
+		}
+
+		return this.abortController.signal;
+	}
+
+	/**
+	 * Abort the current request
+	 */
+	abort(): void {
+		this.abortController?.abort();
+	}
+
+	/**
+	 * Helper to calculate cost from usage
+	 */
+	protected calculateCost(
+		inputTokens: number,
+		outputTokens: number,
+		cacheReadTokens = 0,
+	): number | undefined {
+		const modelPricingSource =
+			this.config.modelInfo ??
+			(this.config.modelId
+				? this.config.knownModels?.[this.config.modelId]
+				: undefined);
+		const pricing = modelPricingSource?.pricing;
+		if (!pricing?.input || !pricing?.output) {
+			return undefined;
+		}
+
+		const uncachedInputTokens = inputTokens - cacheReadTokens;
+		const inputCost = (uncachedInputTokens / 1_000_000) * pricing.input;
+		const outputCost = (outputTokens / 1_000_000) * pricing.output;
+		const cacheReadCost =
+			cacheReadTokens > 0
+				? (cacheReadTokens / 1_000_000) * (pricing.cacheRead ?? 0)
+				: 0;
+
+		return inputCost + outputCost + cacheReadCost;
+	}
+
+	protected createResponseId(): string {
+		return nanoid();
+	}
+
+	protected withResponseId<T extends ApiStreamChunk>(
+		chunk: T,
+		responseId: string,
+	): T {
+		return { ...chunk, id: responseId };
+	}
+
+	protected *withResponseIdForAll(
+		chunks: Iterable<ApiStreamChunk>,
+		responseId: string,
+	): Generator<ApiStreamChunk> {
+		for (const chunk of chunks) {
+			yield this.withResponseId(chunk, responseId);
+		}
+	}
+
+	protected getRequestHeaders(): Record<string, string> {
+		return {
+			...DEFAULT_REQUEST_HEADERS,
+			...(this.config.headers ?? {}),
+		};
+	}
+}

package/src/providers/handlers/bedrock-base.ts

@@ -0,0 +1,390 @@
+import { convertToolsToAnthropic } from "../transform/anthropic-format";
+import {
+	type ApiStream,
+	type HandlerModelInfo,
+	type ProviderConfig,
+	supportsModelThinking,
+} from "../types";
+import type { Message, ToolDefinition } from "../types/messages";
+import { retryStream } from "../utils/retry";
+import { BaseHandler } from "./base";
+import { createBedrockClient } from "./bedrock-client";
+
+const CLAUDE_SONNET_1M_SUFFIX = ":1m";
+
+type AiModule = {
+	streamText: (input: Record<string, unknown>) => {
+		fullStream?: AsyncIterable<{ type?: string; [key: string]: unknown }>;
+		textStream?: AsyncIterable<string>;
+		usage?: Promise<{
+			inputTokens?: number;
+			outputTokens?: number;
+			reasoningTokens?: number;
+			cachedInputTokens?: number;
+			[key: string]: unknown;
+		}>;
+	};
+};
+
+let cachedAiModule: AiModule | undefined;
+const DEFAULT_THINKING_BUDGET_TOKENS = 1024;
+const DEFAULT_REASONING_EFFORT = "medium" as const;
+
+async function loadAiModule(): Promise<AiModule> {
+	if (cachedAiModule) {
+		return cachedAiModule;
+	}
+
+	const moduleName = "ai";
+	cachedAiModule = (await import(moduleName)) as AiModule;
+	return cachedAiModule;
+}
+
+type ModelMessagePart = Record<string, unknown>;
+type ModelMessage = {
+	role: "system" | "user" | "assistant" | "tool";
+	content: string | ModelMessagePart[];
+};
+
+/**
+ * Handler for AWS Bedrock using AI SDK's Amazon Bedrock provider.
+ *
+ * This handler is async-lazy loaded via createHandlerAsync.
+ */
+export class BedrockHandler extends BaseHandler {
+	private clientFactory: ((modelId: string) => unknown) | undefined;
+
+	private async ensureClientFactory(): Promise<(modelId: string) => unknown> {
+		if (!this.clientFactory) {
+			this.clientFactory = await createBedrockClient(
+				this.config,
+				this.getRequestHeaders(),
+			);
+		}
+		return this.clientFactory;
+	}
+
+	getModel(): HandlerModelInfo {
+		const modelId = this.config.modelId;
+		if (!modelId) {
+			throw new Error("Model ID is required. Set modelId in config.");
+		}
+
+		const modelInfo =
+			this.config.modelInfo ?? this.config.knownModels?.[modelId] ?? {};
+		return { id: modelId, info: { ...modelInfo, id: modelId } };
+	}
+
+	getMessages(systemPrompt: string, messages: Message[]): ModelMessage[] {
+		return toModelMessages(systemPrompt, messages);
+	}
+
+	async *createMessage(
+		systemPrompt: string,
+		messages: Message[],
+		tools?: ToolDefinition[],
+	): ApiStream {
+		yield* retryStream(
+			() => this.createMessageInternal(systemPrompt, messages, tools),
+			{ maxRetries: 4 },
+		);
+	}
+
+	private async *createMessageInternal(
+		systemPrompt: string,
+		messages: Message[],
+		tools?: ToolDefinition[],
+	): ApiStream {
+		const ai = await loadAiModule();
+		const factory = await this.ensureClientFactory();
+		const responseId = this.createResponseId();
+		const abortSignal = this.getAbortSignal();
+		const model = this.getModel();
+
+		let modelId = model.id;
+		const providerOptions: Record<string, unknown> = {};
+		const bedrockOptions: Record<string, unknown> = {};
+
+		if (modelId.endsWith(CLAUDE_SONNET_1M_SUFFIX)) {
+			modelId = modelId.slice(0, -CLAUDE_SONNET_1M_SUFFIX.length);
+			bedrockOptions.anthropicBeta = ["context-1m-2025-08-07"];
+		}
+
+		const thinkingSupported = supportsModelThinking(model.info);
+		const budgetTokens =
+			this.config.thinkingBudgetTokens ??
+			(this.config.thinking ? DEFAULT_THINKING_BUDGET_TOKENS : 0);
+		let reasoningEnabled = false;
+		if (
+			thinkingSupported &&
+			budgetTokens > 0 &&
+			modelId.includes("anthropic")
+		) {
+			bedrockOptions.reasoningConfig = { type: "enabled", budgetTokens };
+			reasoningEnabled = true;
+		} else if (thinkingSupported && modelId.includes("amazon.nova")) {
+			const reasoningEffort =
+				this.config.reasoningEffort ??
+				(this.config.thinking ? DEFAULT_REASONING_EFFORT : undefined);
+			if (reasoningEffort) {
+				bedrockOptions.reasoningConfig = {
+					type: "enabled",
+					maxReasoningEffort: reasoningEffort,
+				};
+				reasoningEnabled = true;
+			}
+		}
+
+		if (Object.keys(bedrockOptions).length > 0) {
+			providerOptions.bedrock = bedrockOptions;
+		}
+
+		const stream = ai.streamText({
+			model: factory(modelId),
+			messages: this.getMessages(systemPrompt, messages),
+			tools: toAiSdkTools(tools),
+			maxTokens: model.info.maxTokens ?? undefined,
+			temperature: reasoningEnabled ? undefined : (model.info.temperature ?? 0),
+			providerOptions:
+				Object.keys(providerOptions).length > 0 ? providerOptions : undefined,
+			abortSignal,
+		});
+
+		let usageEmitted = false;
+
+		if (stream.fullStream) {
+			for await (const part of stream.fullStream) {
+				const partType = part.type;
+
+				if (partType === "text-delta") {
+					const text =
+						(part.textDelta as string | undefined) ??
+						(part.delta as string | undefined);
+					if (text) {
+						yield { type: "text", text, id: responseId };
+					}
+					continue;
+				}
+
+				if (partType === "reasoning-delta" || partType === "reasoning") {
+					const reasoning =
+						(part.textDelta as string | undefined) ??
+						(part.reasoning as string | undefined);
+					if (reasoning) {
+						yield { type: "reasoning", reasoning, id: responseId };
+					}
+					continue;
+				}
+
+				if (partType === "tool-call") {
+					const toolCallId =
+						(part.toolCallId as string | undefined) ??
+						(part.id as string | undefined);
+					const toolName =
+						(part.toolName as string | undefined) ??
+						(part.name as string | undefined);
+					const args = (part.args as Record<string, unknown> | undefined) ?? {};
+
+					yield {
+						type: "tool_calls",
+						id: responseId,
+						tool_call: {
+							call_id: toolCallId,
+							function: {
+								name: toolName,
+								arguments: args,
+							},
+						},
+					};
+					continue;
+				}
+
+				if (partType === "error") {
+					const message =
+						(part.error as Error | undefined)?.message ??
+						"Bedrock stream failed";
+					throw new Error(message);
+				}
+
+				if (partType === "finish") {
+					const usage =
+						(part.usage as Record<string, unknown> | undefined) ?? {};
+					const inputTokens = numberOrZero(usage.inputTokens);
+					const outputTokens = numberOrZero(usage.outputTokens);
+					const thoughtsTokenCount = numberOrZero(usage.reasoningTokens);
+					const cacheReadTokens = numberOrZero(usage.cachedInputTokens);
+
+					yield {
+						type: "usage",
+						inputTokens: inputTokens - cacheReadTokens,
+						outputTokens,
+						thoughtsTokenCount,
+						cacheReadTokens,
+						totalCost: this.calculateCost(
+							inputTokens,
+							outputTokens,
+							cacheReadTokens,
+						),
+						id: responseId,
+					};
+					usageEmitted = true;
+				}
+			}
+		} else if (stream.textStream) {
+			for await (const text of stream.textStream) {
+				yield { type: "text", text, id: responseId };
+			}
+		}
+
+		if (!usageEmitted && stream.usage) {
+			const usage = await stream.usage;
+			const inputTokens = numberOrZero(usage.inputTokens);
+			const outputTokens = numberOrZero(usage.outputTokens);
+			const thoughtsTokenCount = numberOrZero(usage.reasoningTokens);
+			const cacheReadTokens = numberOrZero(usage.cachedInputTokens);
+
+			yield {
+				type: "usage",
+				inputTokens: inputTokens - cacheReadTokens,
+				outputTokens,
+				thoughtsTokenCount,
+				cacheReadTokens,
+				totalCost: this.calculateCost(
+					inputTokens,
+					outputTokens,
+					cacheReadTokens,
+				),
+				id: responseId,
+			};
+		}
+
+		yield { type: "done", success: true, id: responseId };
+	}
+}
+
+export function createBedrockHandler(config: ProviderConfig): BedrockHandler {
+	return new BedrockHandler(config);
+}
+
+function numberOrZero(value: unknown): number {
+	return typeof value === "number" && Number.isFinite(value) ? value : 0;
+}
+
+function toAiSdkTools(
+	tools: ToolDefinition[] | undefined,
+): Record<string, unknown> | undefined {
+	if (!tools || tools.length === 0) {
+		return undefined;
+	}
+
+	// We keep the same schema shape used by Anthropic conversion.
+	const anthropicTools = convertToolsToAnthropic(tools);
+	return Object.fromEntries(
+		anthropicTools.map((tool) => [
+			tool.name,
+			{
+				description: tool.description,
+				inputSchema: tool.input_schema,
+			},
+		]),
+	);
+}
+
+function toModelMessages(
+	systemPrompt: string,
+	messages: Message[],
+): ModelMessage[] {
+	const result: ModelMessage[] = [{ role: "system", content: systemPrompt }];
+	const toolNamesById = new Map<string, string>();
+
+	for (const message of messages) {
+		if (typeof message.content === "string") {
+			result.push({ role: message.role, content: message.content });
+			continue;
+		}
+
+		if (message.role === "assistant") {
+			const parts: ModelMessagePart[] = [];
+			for (const block of message.content) {
+				if (block.type === "text") {
+					parts.push({ type: "text", text: block.text });
+					continue;
+				}
+
+				if (block.type === "tool_use") {
+					toolNamesById.set(block.id, block.name);
+					parts.push({
+						type: "tool-call",
+						toolCallId: block.id,
+						toolName: block.name,
+						args: block.input,
+					});
+				}
+			}
+
+			if (parts.length > 0) {
+				result.push({ role: "assistant", content: parts });
+			}
+			continue;
+		}
+
+		// User message (can include text/image/tool_result blocks)
+		const userParts: ModelMessagePart[] = [];
+
+		for (const block of message.content) {
+			if (block.type === "text") {
+				userParts.push({ type: "text", text: block.text });
+				continue;
+			}
+
+			if (block.type === "image") {
+				userParts.push({
+					type: "image",
+					image: Buffer.from(block.data, "base64"),
+					mediaType: block.mediaType,
+				});
+				continue;
+			}
+
+			if (block.type === "tool_result") {
+				if (userParts.length > 0) {
+					result.push({
+						role: "user",
+						content: userParts.splice(0, userParts.length),
+					});
+				}
+
+				result.push({
+					role: "tool",
+					content: [
+						{
+							type: "tool-result",
+							toolCallId: block.tool_use_id,
+							toolName: toolNamesById.get(block.tool_use_id) ?? "tool",
+							output: serializeToolResult(block.content),
+							isError: block.is_error ?? false,
+						},
+					],
+				});
+			}
+		}
+
+		if (userParts.length > 0) {
+			result.push({ role: "user", content: userParts });
+		}
+	}
+
+	return result;
+}
+
+function serializeToolResult(content: Message["content"] | string): string {
+	if (typeof content === "string") {
+		return content;
+	}
+
+	try {
+		return JSON.stringify(content);
+	} catch {
+		return String(content);
+	}
+}