@roj-ai/sdk 0.0.2

package/src/core/llm/cache-live.test.ts

@@ -0,0 +1,155 @@
+/**
+ * Live cache verification tests for Anthropic + OpenRouter providers.
+ *
+ * Opt-in: run with `LIVE_TESTS=1 ANTHROPIC_API_KEY=… OPENROUTER_API_KEY=…`.
+ * Skipped otherwise so the default `bun test` run stays hermetic.
+ *
+ * These tests bypass snapshot-fetch entirely (use real `globalThis.fetch`) so
+ * two sequential identical calls actually land on the live API — proving that
+ * the new message-level `cacheControl` flag → last-block placement is both
+ * accepted and honored as a prompt cache checkpoint.
+ *
+ * The system prompt is padded to comfortably exceed Anthropic's 1024-token
+ * minimum cacheable prefix for Haiku / Sonnet.
+ */
+
+import { describe, expect, test } from 'bun:test'
+import z4 from 'zod/v4'
+import type { LLMMessage } from '~/core/agents/state.js'
+import type { ImageProcessor } from '~/core/image/types.js'
+import type { ToolDefinition } from '~/core/tools/definition.js'
+import { ToolCallId } from '~/core/tools/schema.js'
+import { AnthropicProvider } from './anthropic.js'
+import { applyCacheBreakpoint } from './cache-breakpoints.js'
+import { OpenRouterProvider } from './openrouter.js'
+import { ModelId } from './schema.js'
+
+const liveEnabled = process.env.LIVE_TESTS === '1'
+const anthropicApiKey = liveEnabled ? process.env.ANTHROPIC_API_KEY : undefined
+const openRouterApiKey = liveEnabled ? process.env.OPENROUTER_API_KEY : undefined
+
+const noopImageProcessor: ImageProcessor = {
+	resolveContent: async (content) => content,
+}
+
+const weatherTool: ToolDefinition<{ city: string }> = {
+	name: 'get_weather',
+	description: 'Get current weather for a city',
+	input: z4.object({ city: z4.string() }),
+	execute: async () => ({ ok: true, value: 'not implemented' }),
+}
+
+/**
+ * Large filler system prompt to push the cacheable prefix comfortably past
+ * Anthropic's 1024-token minimum. Content is deterministic so successive
+ * calls reuse the exact same prefix.
+ */
+const LARGE_SYSTEM_PROMPT = [
+	'You are a meticulous assistant helping with data analysis tasks.',
+	'Always respond concisely and accurately. Never speculate beyond the data.',
+	'When you use tools, interpret the results carefully and explain your reasoning.',
+	'Follow these rules strictly:',
+	...Array.from(
+		{ length: 120 },
+		(_, i) => `- Rule ${i + 1}: When asked about topic ${i + 1}, prefer factual sources and cite them when possible. Decline to speculate.`,
+	),
+	'End of instructions.',
+].join('\n')
+
+const describeLive = (name: string, apiKey: string | undefined, fn: () => void) => {
+	if (!apiKey) {
+		describe.skip(`${name} (skipped — API key missing)`, fn)
+		return
+	}
+	describe(name, fn)
+}
+
+describeLive('Anthropic live cache verification', anthropicApiKey, () => {
+	test('cacheControl on tool_result target causes cache write then cache read', async () => {
+		const provider = new AnthropicProvider({
+			apiKey: anthropicApiKey!,
+			imageProcessor: noopImageProcessor,
+			defaultModel: 'claude-haiku-4-5-20251001',
+		})
+
+		const history: LLMMessage[] = [
+			{ role: 'user', content: 'What is the weather in Prague?' },
+			{
+				role: 'assistant',
+				content: '',
+				toolCalls: [{ id: ToolCallId('call_1'), name: 'get_weather', input: { city: 'Prague' } }],
+			},
+			{
+				role: 'tool',
+				toolCallId: ToolCallId('call_1'),
+				content: 'Sunny, 22°C, light breeze from the west.',
+			},
+		]
+		const messages = applyCacheBreakpoint(history, 0)
+
+		const request = {
+			model: ModelId('claude-haiku-4-5-20251001'),
+			systemPrompt: LARGE_SYSTEM_PROMPT,
+			messages,
+			tools: [weatherTool],
+		}
+
+		// First call — primes the cache
+		const first = await provider.inference(request)
+		if (!first.ok) {
+			if (first.error.message?.includes('credit balance')) {
+				console.warn('⚠️  Anthropic live cache test skipped: credit balance too low')
+				return
+			}
+			throw new Error(`first call failed: ${JSON.stringify(first.error)}`)
+		}
+		expect(first.value.metrics.promptTokens).toBeGreaterThan(1024)
+
+		// Second call — cache read proves the write on call 1 was honored
+		const second = await provider.inference(request)
+		if (!second.ok) throw new Error(`second call failed: ${JSON.stringify(second.error)}`)
+		expect(second.value.metrics.cachedTokens ?? 0).toBeGreaterThan(1024)
+	}, 60_000)
+})
+
+describeLive('OpenRouter live cache verification', openRouterApiKey, () => {
+	test('cacheControl on tool_result target produces cache hit on second call', async () => {
+		const provider = new OpenRouterProvider({
+			apiKey: openRouterApiKey!,
+			imageProcessor: noopImageProcessor,
+			defaultModel: 'anthropic/claude-haiku-4.5',
+		})
+
+		const history: LLMMessage[] = [
+			{ role: 'user', content: 'What is the weather in Prague?' },
+			{
+				role: 'assistant',
+				content: '',
+				toolCalls: [{ id: ToolCallId('call_1'), name: 'get_weather', input: { city: 'Prague' } }],
+			},
+			{
+				role: 'tool',
+				toolCallId: ToolCallId('call_1'),
+				content: 'Sunny, 22°C, light breeze from the west.',
+			},
+		]
+		const messages = applyCacheBreakpoint(history, 0)
+
+		const request = {
+			model: ModelId('anthropic/claude-haiku-4.5'),
+			systemPrompt: LARGE_SYSTEM_PROMPT,
+			messages,
+			tools: [weatherTool],
+		}
+
+		// First call primes the cache
+		const first = await provider.inference(request)
+		if (!first.ok) throw new Error(`first call failed: ${JSON.stringify(first.error)}`)
+		expect(first.value.metrics.promptTokens).toBeGreaterThan(1024)
+
+		// Second call — same prefix → cache read proves the write on call 1 was honored
+		const second = await provider.inference(request)
+		if (!second.ok) throw new Error(`second call failed: ${JSON.stringify(second.error)}`)
+		expect(second.value.metrics.cachedTokens ?? 0).toBeGreaterThan(1024)
+	}, 60_000)
+})

package/src/core/llm/index.ts

@@ -0,0 +1,30 @@
+export { AnthropicProvider } from './anthropic.js'
+export type { AnthropicConfig } from './anthropic.js'
+
+export { OpenRouterProvider } from './openrouter.js'
+export type { OpenRouterConfig } from './openrouter.js'
+
+export { RoutingLLMProvider } from './routing-provider.js'
+export type { RoutableLLMProvider } from './routing-provider.js'
+
+export { LoggingLLMProvider } from './logging-provider.js'
+
+export { MockLLMProvider, RequestMatchers } from './mock.js'
+export type { MockInferenceHandler } from './mock.js'
+
+export { LLMLogger } from './logger.js'
+export type { LLMLoggerConfig } from './logger.js'
+
+export { applyMiddleware, useProvider, withAnthropic, withMaxTokens, withOpenRouter, withTemperature } from './middleware.js'
+export type { InferenceNext, LLMMiddleware } from './middleware.js'
+
+export {
+	composeStrippers,
+	createSnapshotLLMMiddleware,
+	normalizeStripRuntime,
+	normalizeStripUuids,
+	normalizeWith,
+	stripEphemeralPorts,
+	stripUuids,
+} from './snapshot-middleware.js'
+export type { SnapshotLLMMiddlewareOptions } from './snapshot-middleware.js'

package/src/core/llm/llm-log-types.ts

@@ -0,0 +1,167 @@
+import type { AgentId } from '~/core/agents/schema.js'
+import type { AnthropicRequestOptions, OpenRouterRequestOptions } from '~/core/llm/provider.js'
+import type { LLMCallId } from '~/core/llm/schema.js'
+import type { SessionId } from '~/core/sessions/schema.js'
+import type { ToolCallId } from '~/core/tools/schema.js'
+import type { LLMToolCall } from './state.js'
+
+export type ChatMessageContentItemText = {
+	type: 'text'
+	text: string
+	cacheControl?: { type: 'ephemeral' }
+}
+
+export type ChatMessageContentItemImageDetail = 'auto' | 'low' | 'high'
+
+export type ChatMessageContentItemImage = {
+	type: 'image_url'
+	imageUrl: {
+		url: string
+		detail?: ChatMessageContentItemImageDetail
+	}
+}
+
+/**
+ * Union of all content item types.
+ * Start with text and image, extend with audio/video as needed.
+ */
+export type ChatMessageContentItem =
+	| ChatMessageContentItemText
+	| ChatMessageContentItemImage
+
+/**
+ * Tool result content - string or array of content items.
+ * Matches OpenRouter's ToolResponseMessageContent type.
+ */
+export type ToolResultContent = string | ChatMessageContentItem[]
+
+/**
+ * Helper to normalize content to string (for display/logging).
+ */
+export const contentToString = (content: ToolResultContent): string => {
+	if (typeof content === 'string') return content
+	return content
+		.filter((c): c is ChatMessageContentItemText => c.type === 'text')
+		.map((c) => c.text)
+		.join('\n')
+}
+
+/**
+ * Represents an LLM message in the request.
+ */
+export type LLMCallMessage = {
+	role: 'user' | 'assistant' | 'tool' | 'system'
+	content: string | ChatMessageContentItem[]
+	toolCalls?: LLMToolCall[]
+	toolCallId?: ToolCallId
+	/** Assistant reasoning (for models that support it) */
+	reasoning?: string
+	/** Prompt cache breakpoint marker (ephemeral cache checkpoint after this message) */
+	cacheControl?: { type: 'ephemeral' }
+}
+
+/**
+ * Tool definition for logging (includes full JSON schema).
+ */
+export type LLMCallToolDefinition = {
+	name: string
+	description: string
+	/** JSON Schema for tool parameters */
+	parameters?: Record<string, unknown>
+}
+
+/**
+ * Request data stored in the log entry.
+ */
+export type LLMCallRequest = {
+	model: string
+	systemPrompt: string
+	messages: LLMCallMessage[]
+	tools?: LLMCallToolDefinition[]
+	toolsCount: number
+	maxTokens?: number
+	temperature?: number
+	/** Provider-specific options set by middleware */
+	providerOptions?: {
+		openrouter?: OpenRouterRequestOptions
+		anthropic?: AnthropicRequestOptions
+	}
+}
+
+/**
+ * Response data stored in the log entry.
+ */
+export type LLMCallResponse = {
+	content: string | null
+	toolCalls: LLMToolCall[]
+	finishReason: 'stop' | 'tool_calls' | 'length' | 'error'
+	/** Assistant reasoning (for models that support extended thinking) */
+	reasoning?: string
+}
+
+/**
+ * Extended metrics with detailed token breakdown.
+ */
+export type LLMCallMetrics = {
+	promptTokens: number
+	completionTokens: number
+	totalTokens: number
+	/** Reasoning/thinking tokens (for models like o1, Claude with extended thinking) */
+	reasoningTokens?: number
+	/** Cached prompt tokens (for providers with prompt caching) */
+	cachedTokens?: number
+	/** Tokens written to prompt cache */
+	cacheWriteTokens?: number
+	latencyMs: number
+	/** Generation time (actual model inference time) */
+	generationTimeMs?: number
+	model: string
+	/** Provider name (e.g., "anthropic", "openai") */
+	provider?: string
+	cost?: number
+	/** Cache status */
+	cacheStatus?: 'hit' | 'miss' | 'none'
+}
+
+/**
+ * Error data stored in the log entry.
+ */
+export type LLMCallError = {
+	type: string
+	message: string
+	retryAfterMs?: number
+	/** HTTP status code from the provider response */
+	statusCode?: number
+	/** Raw HTTP response body from the provider */
+	responseBody?: string
+}
+
+/**
+ * Complete LLM call log entry.
+ * Each entry represents a single LLM API call with full request/response details.
+ */
+export type LLMCallLogEntry = {
+	id: LLMCallId
+	sessionId: SessionId
+	agentId: AgentId
+	createdAt: number
+	completedAt?: number
+	durationMs?: number
+	status: 'running' | 'success' | 'error'
+
+	request: LLMCallRequest
+	response?: LLMCallResponse
+	metrics?: LLMCallMetrics
+	error?: LLMCallError
+
+	/** Provider-specific request ID (e.g., OpenRouter generation ID) */
+	providerRequestId?: string
+}
+
+/**
+ * Response for listing LLM calls.
+ */
+export type GetLLMCallsResponse = {
+	calls: LLMCallLogEntry[]
+	total: number
+}

package/src/core/llm/logger.test.ts

@@ -0,0 +1,270 @@
+/**
+ * Tests for LLM Logger
+ *
+ * Tests individual LLM call file storage in session folders.
+ */
+
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test'
+import { mkdir, readdir, readFile, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import z from 'zod/v4'
+import { AgentId } from '~/core/agents/schema.js'
+import { LLMCallId, ModelId } from '~/core/llm/schema.js'
+import { SessionId } from '~/core/sessions/schema.js'
+import { createNodeFileSystem } from '~/testing/node-platform.js'
+import { LLMLogger } from './logger.js'
+import type { LLMLoggerConfig } from './logger.js'
+import type { InferenceRequest, InferenceResponse, LLMError } from './provider.js'
+
+// ============================================================================
+// Test helpers
+// ============================================================================
+
+const createTestConfig = (overrides: Partial<LLMLoggerConfig> = {}): LLMLoggerConfig => ({
+	basePath: join(tmpdir(), `llm-logger-test-${Date.now()}-${Math.random().toString(36).slice(2)}`),
+	enabled: true,
+	fs: createNodeFileSystem(),
+	...overrides,
+})
+
+const createTestRequest = (): InferenceRequest => {
+	return {
+		model: ModelId('test-model'),
+		systemPrompt: 'You are a test assistant.',
+		messages: [
+			{ role: 'user', content: 'Hello' },
+			{ role: 'assistant', content: 'Hi there!' },
+		],
+		tools: [
+			{ name: 'test_tool', description: 'A test tool', input: z.unknown(), execute: async () => ({ ok: true, value: '' }) },
+		],
+		maxTokens: 100,
+		temperature: 0.7,
+	}
+}
+
+const createTestResponse = (): InferenceResponse => ({
+	content: 'Test response',
+	toolCalls: [
+		{ id: 'tc-1' as any, name: 'test_tool', input: { foo: 'bar' } },
+	],
+	finishReason: 'stop',
+	metrics: {
+		promptTokens: 10,
+		completionTokens: 5,
+		totalTokens: 15,
+		latencyMs: 100,
+		model: 'test-model',
+		cost: 0.001,
+	},
+})
+
+const createTestError = (): LLMError => ({
+	type: 'rate_limit',
+	message: 'Rate limit exceeded',
+	retryAfterMs: 5000,
+})
+
+// ============================================================================
+// Tests
+// ============================================================================
+
+describe('LLMLogger', () => {
+	let testBasePath: string
+
+	beforeEach(async () => {
+		testBasePath = join(tmpdir(), `llm-logger-test-${Date.now()}-${Math.random().toString(36).slice(2)}`)
+	})
+
+	afterEach(async () => {
+		try {
+			await rm(testBasePath, { recursive: true, force: true })
+		} catch {
+			// Ignore cleanup errors
+		}
+	})
+
+	describe('createCall', () => {
+		it('creates a call file with request data', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId = SessionId('test-session-1')
+			const agentId = AgentId('test-agent-1')
+
+			const callId = await logger.createCall(sessionId, agentId, createTestRequest())
+
+			expect(callId).toBeTruthy()
+
+			const call = await logger.getCall(sessionId, callId)
+			expect(call).not.toBeNull()
+			expect(call!.id).toBe(callId)
+			expect(call!.sessionId).toBe(sessionId)
+			expect(call!.agentId).toBe(agentId)
+			expect(call!.status).toBe('running')
+			expect(call!.request.model).toBe('test-model')
+			expect(call!.request.systemPrompt).toBe('You are a test assistant.')
+			expect(call!.request.messages.length).toBe(2)
+			expect(call!.request.toolsCount).toBe(1)
+		})
+
+		it('stores call in session folder structure', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId = SessionId('session-abc')
+			const agentId = AgentId('agent-1')
+
+			const callId = await logger.createCall(sessionId, agentId, createTestRequest())
+
+			// Verify file exists in correct location
+			const callPath = join(testBasePath, 'sessions', sessionId, 'calls', `${callId}.json`)
+			const content = await readFile(callPath, 'utf-8')
+			const parsed = JSON.parse(content)
+			expect(parsed.id).toBe(callId)
+		})
+	})
+
+	describe('completeCall', () => {
+		it('updates call with response data', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId = SessionId('test-session-2')
+			const agentId = AgentId('test-agent-1')
+
+			const callId = await logger.createCall(sessionId, agentId, createTestRequest())
+			await logger.completeCall(sessionId, callId, createTestResponse(), 150)
+
+			const call = await logger.getCall(sessionId, callId)
+			expect(call!.status).toBe('success')
+			expect(call!.durationMs).toBe(150)
+			expect(call!.response).toBeDefined()
+			expect(call!.response!.content).toBe('Test response')
+			expect(call!.response!.finishReason).toBe('stop')
+			expect(call!.metrics).toBeDefined()
+			expect(call!.metrics!.promptTokens).toBe(10)
+			expect(call!.metrics!.completionTokens).toBe(5)
+			expect(call!.metrics!.cost).toBe(0.001)
+		})
+
+		it('handles non-existent call gracefully', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId = SessionId('test-session-3')
+
+			// Should not throw
+			await logger.completeCall(sessionId, LLMCallId('non-existent'), createTestResponse(), 100)
+		})
+	})
+
+	describe('failCall', () => {
+		it('updates call with error data', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId = SessionId('test-session-4')
+			const agentId = AgentId('test-agent-1')
+
+			const callId = await logger.createCall(sessionId, agentId, createTestRequest())
+			await logger.failCall(sessionId, callId, createTestError(), 50)
+
+			const call = await logger.getCall(sessionId, callId)
+			expect(call!.status).toBe('error')
+			expect(call!.durationMs).toBe(50)
+			expect(call!.error).toBeDefined()
+			expect(call!.error!.type).toBe('rate_limit')
+			expect(call!.error!.message).toBe('Rate limit exceeded')
+			expect(call!.error!.retryAfterMs).toBe(5000)
+		})
+	})
+
+	describe('getCall', () => {
+		it('returns null for non-existent call', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId = SessionId('test-session-6')
+
+			const call = await logger.getCall(sessionId, LLMCallId('non-existent'))
+			expect(call).toBeNull()
+		})
+	})
+
+	describe('listCalls', () => {
+		it('lists all calls for a session', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId = SessionId('test-session-7')
+			const agentId = AgentId('test-agent-1')
+
+			// Create multiple calls
+			await logger.createCall(sessionId, agentId, createTestRequest())
+			await logger.createCall(sessionId, agentId, createTestRequest())
+			await logger.createCall(sessionId, agentId, createTestRequest())
+
+			const result = await logger.listCalls(sessionId)
+			expect(result.total).toBe(3)
+			expect(result.calls.length).toBe(3)
+		})
+
+		it('returns empty list for session without calls', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId = SessionId('test-session-8')
+
+			const result = await logger.listCalls(sessionId)
+			expect(result.total).toBe(0)
+			expect(result.calls.length).toBe(0)
+		})
+
+		it('supports pagination', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId = SessionId('test-session-9')
+			const agentId = AgentId('test-agent-1')
+
+			// Create 5 calls
+			for (let i = 0; i < 5; i++) {
+				await logger.createCall(sessionId, agentId, createTestRequest())
+			}
+
+			const result = await logger.listCalls(sessionId, { limit: 2, offset: 1 })
+			expect(result.total).toBe(5)
+			expect(result.calls.length).toBe(2)
+		})
+
+		it('orders calls by most recent first', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId = SessionId('test-session-10')
+			const agentId = AgentId('test-agent-1')
+
+			const callId1 = await logger.createCall(sessionId, agentId, createTestRequest())
+			// Small delay to ensure different UUIDv7 timestamps
+			await new Promise(resolve => setTimeout(resolve, 2))
+			const callId2 = await logger.createCall(sessionId, agentId, createTestRequest())
+
+			const result = await logger.listCalls(sessionId)
+			// Most recent (callId2) should be first
+			expect(result.calls[0].id).toBe(callId2)
+			expect(result.calls[1].id).toBe(callId1)
+		})
+	})
+
+	describe('multiple sessions', () => {
+		it('isolates calls between sessions', async () => {
+			const config = createTestConfig({ basePath: testBasePath })
+			const logger = new LLMLogger(config)
+			const sessionId1 = SessionId('session-a')
+			const sessionId2 = SessionId('session-b')
+			const agentId = AgentId('test-agent-1')
+
+			await logger.createCall(sessionId1, agentId, createTestRequest())
+			await logger.createCall(sessionId1, agentId, createTestRequest())
+			await logger.createCall(sessionId2, agentId, createTestRequest())
+
+			const result1 = await logger.listCalls(sessionId1)
+			const result2 = await logger.listCalls(sessionId2)
+
+			expect(result1.total).toBe(2)
+			expect(result2.total).toBe(1)
+		})
+	})
+})