@roj-ai/sdk 0.0.2

package/src/core/llm/provider-integration.test.ts

@@ -0,0 +1,408 @@
+/**
+ * Integration tests for LLM providers against real APIs.
+ *
+ * Uses snapshot-caching fetch: first run hits the real API and saves the
+ * request+response; subsequent runs with the same request use the cache.
+ *
+ * Env vars: ANTHROPIC_API_KEY, OPENROUTER_API_KEY
+ * When snapshots exist, tests run without API keys (using cached responses).
+ * API keys are only needed to record new snapshots.
+ */
+
+import { describe, expect, test } from 'bun:test'
+import { existsSync } from 'node:fs'
+import { join } from 'node:path'
+import z4 from 'zod/v4'
+import type { LLMMessage } from '~/core/agents/state.js'
+import type { ImageProcessor } from '~/core/image/types.js'
+import type { ToolDefinition } from '~/core/tools/definition.js'
+import { ToolCallId } from '~/core/tools/schema.js'
+import { AnthropicProvider } from './anthropic.js'
+import { OpenRouterProvider } from './openrouter.js'
+import type { InferenceRequest, InferenceResponse, LLMProvider } from './provider.js'
+import { ModelId } from './schema.js'
+import { createSnapshotFetch } from './snapshot-fetch.js'
+
+// ============================================================================
+// Setup
+// ============================================================================
+
+const SNAPSHOTS_DIR = join(import.meta.dir, '__snapshots__')
+
+const anthropicApiKey = process.env.ANTHROPIC_API_KEY ?? 'snapshot-only'
+const openRouterApiKey = process.env.OPENROUTER_API_KEY ?? 'snapshot-only'
+
+const noopImageProcessor: ImageProcessor = {
+	resolveContent: async (content) => content,
+}
+
+// 1x1 red pixel PNG
+const TINY_PNG = 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg=='
+
+const weatherTool: ToolDefinition<{ city: string }> = {
+	name: 'get_weather',
+	description: 'Get current weather for a city',
+	input: z4.object({ city: z4.string() }),
+	execute: async () => ({ ok: true, value: 'not implemented' }),
+}
+
+const searchTool: ToolDefinition<{ query: string }> = {
+	name: 'search',
+	description: 'Search the web',
+	input: z4.object({ query: z4.string() }),
+	execute: async () => ({ ok: true, value: 'not implemented' }),
+}
+
+// ============================================================================
+// Provider factories
+// ============================================================================
+
+interface ProviderSetup {
+	name: string
+	model: ModelId
+	hasApiKey: boolean
+	snapshotPrefix: string
+	create: (testName: string) => LLMProvider
+}
+
+const providers: ProviderSetup[] = [
+	{
+		name: 'Anthropic',
+		model: ModelId('claude-haiku-4-5-20251001'),
+		hasApiKey: !!process.env.ANTHROPIC_API_KEY,
+		snapshotPrefix: 'anthropic',
+		create: (testName) =>
+			new AnthropicProvider({
+				apiKey: anthropicApiKey!,
+				imageProcessor: noopImageProcessor,
+				defaultModel: 'claude-haiku-4-5-20251001',
+				fetch: createSnapshotFetch(SNAPSHOTS_DIR, `anthropic-${testName}`),
+			}),
+	},
+	{
+		name: 'OpenRouter',
+		model: ModelId('anthropic/claude-haiku-4.5'),
+		hasApiKey: !!process.env.OPENROUTER_API_KEY,
+		snapshotPrefix: 'openrouter',
+		create: (testName) =>
+			new OpenRouterProvider({
+				apiKey: openRouterApiKey!,
+				imageProcessor: noopImageProcessor,
+				defaultModel: 'anthropic/claude-haiku-4.5',
+				fetch: createSnapshotFetch(SNAPSHOTS_DIR, `openrouter-${testName}`),
+			}),
+	},
+]
+
+// ============================================================================
+// Test scenarios
+// ============================================================================
+
+interface Scenario {
+	name: string
+	request: Omit<InferenceRequest, 'model'>
+	assert: (response: InferenceResponse) => void
+}
+
+const scenarios: Scenario[] = [
+	// --- Basic messages ---
+
+	{
+		name: 'simple-user-message',
+		request: {
+			systemPrompt: 'You are a helpful assistant. Reply with exactly one word.',
+			messages: [{ role: 'user', content: 'What color is the sky? Reply with one word.' }],
+		},
+		assert: (r) => {
+			expect(r.content).toBeTruthy()
+			expect(r.finishReason).toBe('stop')
+			expect(r.metrics.promptTokens).toBeGreaterThan(0)
+			expect(r.metrics.completionTokens).toBeGreaterThan(0)
+			expect(r.providerRequestId).toBeTruthy()
+		},
+	},
+
+	{
+		name: 'system-message-in-conversation',
+		request: {
+			systemPrompt: 'You are a helpful assistant.',
+			messages: [
+				{ role: 'user', content: 'Hi' },
+				{ role: 'assistant', content: 'Hello!' },
+				{ role: 'system', content: 'The user prefers Czech language.' },
+				{ role: 'user', content: 'How are you?' },
+			],
+		},
+		assert: (r) => {
+			expect(r.content).toBeTruthy()
+			expect(r.finishReason).toBe('stop')
+		},
+	},
+
+	// --- Tool calls ---
+
+	{
+		name: 'tool-call',
+		request: {
+			systemPrompt: 'You have access to tools. Use the get_weather tool to answer weather questions.',
+			messages: [{ role: 'user', content: 'What is the weather in Prague?' }],
+			tools: [weatherTool],
+		},
+		assert: (r) => {
+			expect(r.finishReason).toBe('tool_calls')
+			expect(r.toolCalls.length).toBeGreaterThan(0)
+			expect(r.toolCalls[0].name).toBe('get_weather')
+			expect(r.toolCalls[0].input).toHaveProperty('city')
+			expect(r.toolCalls[0].id).toBeTruthy()
+		},
+	},
+
+	{
+		name: 'tool-roundtrip',
+		request: {
+			systemPrompt: 'You have access to tools. Use them as needed. Be brief.',
+			messages: [
+				{ role: 'user', content: 'What is the weather in Prague?' },
+				{
+					role: 'assistant',
+					content: '',
+					toolCalls: [{ id: ToolCallId('call_001'), name: 'get_weather', input: { city: 'Prague' } }],
+				},
+				{
+					role: 'tool',
+					toolCallId: ToolCallId('call_001'),
+					content: 'Sunny, 22°C',
+				},
+			],
+			tools: [weatherTool],
+		},
+		assert: (r) => {
+			expect(r.content).toBeTruthy()
+			expect(r.finishReason).toBe('stop')
+		},
+	},
+
+	{
+		name: 'tool-error-result',
+		request: {
+			systemPrompt: 'You have access to tools. If a tool fails, explain the error to the user briefly.',
+			messages: [
+				{ role: 'user', content: 'What is the weather in Prague?' },
+				{
+					role: 'assistant',
+					content: '',
+					toolCalls: [{ id: ToolCallId('call_001'), name: 'get_weather', input: { city: 'Prague' } }],
+				},
+				{
+					role: 'tool',
+					toolCallId: ToolCallId('call_001'),
+					content: 'Service unavailable: weather API is down',
+					isError: true,
+				},
+			],
+			tools: [weatherTool],
+		},
+		assert: (r) => {
+			expect(r.content).toBeTruthy()
+			expect(r.finishReason).toBe('stop')
+		},
+	},
+
+	{
+		name: 'assistant-text-with-tool-calls',
+		request: {
+			systemPrompt: 'You have access to tools. When asked about weather, first say you will check, then use the tool.',
+			messages: [
+				{ role: 'user', content: 'What is the weather in Prague and London?' },
+				{
+					role: 'assistant',
+					content: 'Let me check the weather for both cities.',
+					toolCalls: [
+						{ id: ToolCallId('call_001'), name: 'get_weather', input: { city: 'Prague' } },
+						{ id: ToolCallId('call_002'), name: 'get_weather', input: { city: 'London' } },
+					],
+				},
+				{
+					role: 'tool',
+					toolCallId: ToolCallId('call_001'),
+					content: 'Sunny, 22°C',
+				},
+				{
+					role: 'tool',
+					toolCallId: ToolCallId('call_002'),
+					content: 'Rainy, 14°C',
+				},
+			],
+			tools: [weatherTool],
+		},
+		assert: (r) => {
+			expect(r.content).toBeTruthy()
+			expect(r.finishReason).toBe('stop')
+		},
+	},
+
+	{
+		name: 'multiple-tool-calls-requested',
+		request: {
+			systemPrompt: 'You have access to tools. Use ALL relevant tools in a SINGLE response. Always call both tools when asked about weather and search.',
+			messages: [{ role: 'user', content: 'Search for "Prague weather forecast" and also get the current weather in Prague using the weather tool.' }],
+			tools: [weatherTool, searchTool],
+		},
+		assert: (r) => {
+			expect(r.finishReason).toBe('tool_calls')
+			expect(r.toolCalls.length).toBeGreaterThanOrEqual(2)
+			const toolNames = r.toolCalls.map((tc) => tc.name)
+			expect(toolNames).toContain('get_weather')
+			expect(toolNames).toContain('search')
+		},
+	},
+
+	// --- Images ---
+
+	{
+		name: 'image-data-url',
+		request: {
+			systemPrompt: 'Describe images briefly. One sentence.',
+			messages: [{
+				role: 'user',
+				content: [
+					{ type: 'text', text: 'What do you see in this image?' },
+					{ type: 'image_url', imageUrl: { url: `data:image/png;base64,${TINY_PNG}` } },
+				],
+			}],
+		},
+		assert: (r) => {
+			expect(r.content).toBeTruthy()
+			expect(r.finishReason).toBe('stop')
+		},
+	},
+
+	{
+		name: 'image-http-url',
+		request: {
+			systemPrompt: 'Describe images briefly. One sentence.',
+			messages: [{
+				role: 'user',
+				content: [
+					{ type: 'text', text: 'What do you see in this image?' },
+					{ type: 'image_url', imageUrl: { url: 'https://www.google.com/images/branding/googlelogo/2x/googlelogo_color_272x92dp.png' } },
+				],
+			}],
+		},
+		assert: (r) => {
+			expect(r.content).toBeTruthy()
+			expect(r.finishReason).toBe('stop')
+		},
+	},
+
+	{
+		name: 'multiple-images',
+		request: {
+			systemPrompt: 'Describe what you see. Be brief.',
+			messages: [{
+				role: 'user',
+				content: [
+					{ type: 'text', text: 'How many images do you see? What colors?' },
+					{ type: 'image_url', imageUrl: { url: `data:image/png;base64,${TINY_PNG}` } },
+					{ type: 'image_url', imageUrl: { url: `data:image/png;base64,${TINY_PNG}` } },
+				],
+			}],
+		},
+		assert: (r) => {
+			expect(r.content).toBeTruthy()
+			expect(r.finishReason).toBe('stop')
+		},
+	},
+
+	{
+		name: 'tool-result-with-image',
+		request: {
+			systemPrompt: 'You have access to tools. Describe tool results briefly.',
+			messages: [
+				{ role: 'user', content: 'Take a screenshot' },
+				{
+					role: 'assistant',
+					content: '',
+					toolCalls: [{ id: ToolCallId('call_001'), name: 'search', input: { query: 'screenshot' } }],
+				},
+				{
+					role: 'tool',
+					toolCallId: ToolCallId('call_001'),
+					content: [
+						{ type: 'text', text: 'Here is the screenshot:' },
+						{ type: 'image_url', imageUrl: { url: `data:image/png;base64,${TINY_PNG}` } },
+					],
+				},
+			],
+			tools: [searchTool],
+		},
+		assert: (r) => {
+			expect(r.content).toBeTruthy()
+			expect(r.finishReason).toBe('stop')
+		},
+	},
+
+	// --- Edge cases / message merging ---
+
+	{
+		name: 'consecutive-tool-results',
+		request: {
+			systemPrompt: 'You have access to tools. Summarize all results briefly.',
+			messages: [
+				{ role: 'user', content: 'Get weather in Prague and London.' },
+				{
+					role: 'assistant',
+					content: '',
+					toolCalls: [
+						{ id: ToolCallId('call_001'), name: 'get_weather', input: { city: 'Prague' } },
+						{ id: ToolCallId('call_002'), name: 'get_weather', input: { city: 'London' } },
+					],
+				},
+				{
+					role: 'tool',
+					toolCallId: ToolCallId('call_001'),
+					content: 'Sunny, 22°C',
+				},
+				{
+					role: 'tool',
+					toolCallId: ToolCallId('call_002'),
+					content: 'Rainy, 14°C',
+				},
+			],
+			tools: [weatherTool],
+		},
+		assert: (r) => {
+			expect(r.content).toBeTruthy()
+			expect(r.finishReason).toBe('stop')
+		},
+	},
+]
+
+// ============================================================================
+// Run all scenarios against all providers
+// ============================================================================
+
+for (const provider of providers) {
+	describe(`${provider.name} integration`, () => {
+		for (const scenario of scenarios) {
+			const snapshotPath = join(SNAPSHOTS_DIR, `${provider.snapshotPrefix}-${scenario.name}.json`)
+			const hasSnapshot = existsSync(snapshotPath)
+			const shouldSkip = !provider.hasApiKey && !hasSnapshot
+
+			test.skipIf(shouldSkip)(scenario.name, async () => {
+				const llm = provider.create(scenario.name)
+
+				const result = await llm.inference({
+					model: provider.model,
+					...scenario.request,
+				})
+
+				if (!result.ok) {
+					throw new Error(`LLM inference failed: ${JSON.stringify(result.error)}`)
+				}
+
+				scenario.assert(result.value)
+			})
+		}
+	})
+}

package/src/core/llm/provider.ts

@@ -0,0 +1,238 @@
+import type { AssistantLLMMessage, LLMMessage, SystemLLMMessage, ToolCall, ToolLLMMessage, UserLLMMessage } from '~/core/agents/state.js'
+import type { FileStore } from '~/core/file-store/types.js'
+import type { ToolResultContent } from '~/core/llm/llm-log-types.js'
+import type { ToolDefinition } from '~/core/tools/definition.js'
+import type { ToolCallId } from '~/core/tools/schema.js'
+import type { Result } from '~/lib/utils/result.js'
+import { ModelId } from './schema'
+
+// Re-export LLMMessage types from agents/state for backwards compatibility
+export type { AssistantLLMMessage, LLMMessage, SystemLLMMessage, ToolCall, ToolLLMMessage, UserLLMMessage }
+
+// ============================================================================
+// Request types
+// ============================================================================
+
+/**
+ * Raw tool spec without Zod dependency (for buildHttpRequest / curl export).
+ */
+export interface RawToolSpec {
+	name: string
+	description: string
+	parameters: Record<string, unknown>
+}
+
+/**
+ * OpenRouter-specific request options.
+ * Set via `withOpenRouter()` middleware.
+ *
+ * @see https://openrouter.ai/docs/api-reference/overview
+ */
+export interface OpenRouterRequestOptions {
+	/** Provider routing preferences */
+	providers?: {
+		/** Preferred provider ordering */
+		order?: string[]
+		/** Only allow these providers */
+		allow?: string[]
+		/** Exclude these providers */
+		deny?: string[]
+		/** Allowed quantizations (e.g. "bf16", "int8") */
+		quantizations?: string[]
+	}
+	/** Routing strategy */
+	route?: 'fallback'
+	/** Model transforms */
+	transforms?: string[]
+	/** OpenRouter saved preset ID */
+	preset?: string
+}
+
+/**
+ * Anthropic-specific request options.
+ * Set via `withAnthropic()` middleware.
+ */
+export interface AnthropicRequestOptions {
+	/** Extended thinking token budget. Min 1024. Overrides server-level default. */
+	thinkingBudget?: number
+}
+
+/**
+ * Request pro LLM inference
+ */
+export interface InferenceRequest {
+	model: ModelId
+	systemPrompt: string
+	messages: LLMMessage[]
+	tools?: ToolDefinition<any>[]
+	maxTokens?: number
+	temperature?: number
+	/** Stop sequences - LLM will stop generating when any of these are encountered */
+	stopSequences?: string[]
+	/** OpenRouter-specific options, set by middleware */
+	openrouter?: OpenRouterRequestOptions
+	/** Anthropic-specific options, set by middleware */
+	anthropic?: AnthropicRequestOptions
+}
+
+/**
+ * Raw inference request — uses plain JSON schemas instead of ToolDefinition.
+ * Used by buildHttpRequest (and curl export from stored log entries).
+ */
+export interface RawInferenceRequest {
+	model: ModelId
+	systemPrompt: string
+	messages: LLMMessage[]
+	tools?: RawToolSpec[]
+	maxTokens?: number
+	temperature?: number
+	stopSequences?: string[]
+	/** OpenRouter-specific options, set by middleware */
+	openrouter?: OpenRouterRequestOptions
+	/** Anthropic-specific options, set by middleware */
+	anthropic?: AnthropicRequestOptions
+}
+
+/**
+ * Context for inference (for logging and call tracking).
+ */
+export interface InferenceContext {
+	sessionId: string
+	agentId: string
+	/** Callback invoked when an LLM call entry is created, providing the call ID */
+	onLLMCallCreated?: (callId: string) => void
+	/** Signal to abort the inference request */
+	signal?: AbortSignal
+	/** FileStore for resolving file:// URLs in message content */
+	fileStore: FileStore
+	/** Named provider instances, available for middleware routing via useProvider() */
+	providers?: ReadonlyMap<string, LLMProvider>
+}
+
+// ============================================================================
+// HTTP request types (for curl export)
+// ============================================================================
+
+/**
+ * Raw HTTP request as it would be sent to the provider API (without auth).
+ */
+export interface ProviderHttpRequest {
+	url: string
+	method: 'POST'
+	headers: Record<string, string>
+	body: unknown
+}
+
+// ============================================================================
+// Response types
+// ============================================================================
+
+/**
+ * Response z LLM inference
+ */
+export interface InferenceResponse {
+	content: string | null
+	toolCalls: ToolCall[]
+	finishReason: 'stop' | 'tool_calls' | 'length' | 'error'
+	metrics: LLMMetrics
+	/** Provider-specific request ID (e.g., OpenRouter generation ID for fetching stats) */
+	providerRequestId?: string
+	/** Extended thinking / reasoning content (for models that support it) */
+	reasoning?: string
+}
+
+/**
+ * Metriky z LLM volání
+ */
+export interface LLMMetrics {
+	promptTokens: number
+	completionTokens: number
+	totalTokens: number
+	latencyMs: number
+	model: string
+	/** Provider name (e.g. "anthropic", "openrouter") */
+	provider?: string
+	cost?: number
+	/** Tokens served from prompt cache */
+	cachedTokens?: number
+	/** Tokens written to prompt cache */
+	cacheWriteTokens?: number
+	/** Reasoning/thinking tokens */
+	reasoningTokens?: number
+}
+
+// ============================================================================
+// Error types
+// ============================================================================
+
+/**
+ * LLM error types
+ */
+export interface LLMError {
+	type:
+		| 'rate_limit'
+		| 'invalid_request'
+		| 'context_length'
+		| 'server_error'
+		| 'network_error'
+		| 'timeout'
+		| 'aborted'
+	message: string
+	retryAfterMs?: number
+	/** HTTP status code from the provider response */
+	statusCode?: number
+	/** Raw HTTP response body from the provider */
+	responseBody?: string
+	cause?: unknown
+}
+
+// ============================================================================
+// Provider interface
+// ============================================================================
+
+/**
+ * LLMProvider interface
+ */
+export interface LLMProvider {
+	/**
+	 * Provede inference a vrátí response.
+	 */
+	inference(
+		request: InferenceRequest,
+		context?: InferenceContext,
+	): Promise<Result<InferenceResponse, LLMError>>
+
+	/**
+	 * Build the raw HTTP request that would be sent to the provider API.
+	 * Returns URL, headers (without auth), and body with resolved images.
+	 * Used for curl export from debug UI.
+	 */
+	buildHttpRequest?(
+		request: RawInferenceRequest,
+		context?: InferenceContext,
+	): Promise<ProviderHttpRequest>
+
+	readonly name: string
+}
+
+// ============================================================================
+// Message helpers
+// ============================================================================
+
+export const LLMMessageFactory = {
+	user: (content: string): UserLLMMessage => ({ role: 'user', content }),
+
+	assistant: (content: string, toolCalls?: ToolCall[]): AssistantLLMMessage => ({
+		role: 'assistant',
+		content,
+		toolCalls,
+	}),
+
+	toolResult: (toolCallId: ToolCallId, content: ToolResultContent): ToolLLMMessage => ({
+		role: 'tool',
+		content,
+		toolCallId,
+	}),
+
+	system: (content: string): SystemLLMMessage => ({ role: 'system', content }),
+}