@roj-ai/sdk 0.1.31 → 0.1.32

package/src/core/llm/cache-breakpoints.ts

@@ -1,5 +1,44 @@
 import type { LLMMessage, LLMMessageCacheControl } from '~/core/agents/state.js'
 
+/** Anthropic's two ephemeral cache tiers: 5-minute (write 1.25×) and 1-hour (write 2×). Reads are 0.1× on both. */
+export type CacheTtl = '5m' | '1h'
+
+/**
+ * Per-breakpoint prompt cache TTL.
+ *
+ * A bare tier applies to both breakpoints. The object form sets them
+ * independently, which is usually what you want: the stable prefix and the
+ * churning conversation tail have opposite cost profiles.
+ *
+ * Choosing between them is an economic question about a specific agent's call
+ * pattern, so the SDK does not pick for you:
+ *
+ * - A long TTL on the **prefix** pays off when calls are spread further apart
+ *   than the short tier survives — an orchestrator waiting on a human.
+ * - A long TTL on the **tail** costs 2× on every write while the tail is
+ *   rewritten turn-to-turn anyway, so it only pays when consecutive calls are
+ *   themselves minutes apart.
+ *
+ * Worked example, from a measured 5-week production session: 95% of inferences
+ * followed a sub-5-minute gap, so `'1h'` uniformly came out ~2.5% *worse* than
+ * the default, while `{ prefix: '1h' }` targeted the 29 rewrites that actually
+ * expired. A batch agent called in a tight loop would want the opposite.
+ *
+ * Constraint: Anthropic requires a longer TTL to precede a shorter one, and the
+ * prefix breakpoint always sits before the tail. So `{ prefix: '5m', tail: '1h' }`
+ * is invalid at the API and will be rejected — the tail TTL must not exceed the
+ * prefix TTL.
+ */
+export type CacheTtlConfig = CacheTtl | { prefix?: CacheTtl; tail?: CacheTtl }
+
+const resolveTtls = (config: CacheTtlConfig | undefined): { prefix?: CacheTtl; tail?: CacheTtl } => {
+	if (config === undefined) return {}
+	if (typeof config === 'string') return { prefix: config, tail: config }
+	return config
+}
+
+const cacheControl = (ttl: CacheTtl | undefined): LLMMessageCacheControl => ttl ? { type: 'ephemeral', ttl } : { type: 'ephemeral' }
+
 /**
  * Mark the prompt cache breakpoints on a message list.
  *
@@ -26,22 +65,26 @@ import type { LLMMessage, LLMMessageCacheControl } from '~/core/agents/state.js'
  *    every inference AND compaction call. Anthropic allows up to 4 breakpoints
  *    and matches the longest cached prefix, so the two coexist.
  *
- * If the prefix and tail indices coincide, only one breakpoint is set.
+ * If the prefix and tail indices coincide, only one breakpoint is set — it
+ * covers the preamble, so it takes the prefix TTL.
  *
- * `ttl` opts into Anthropic's 1-hour cache tier (write cost 2× input, read
- * still 0.1×). Useful for long-lived agents where the default 5-minute TTL
- * would expire between user turns. Omit for the default 5-minute tier.
+ * `ttl` sets the cache tier per breakpoint; see {@link CacheTtlConfig} for how
+ * to choose. Providers must additionally give the system block the longest TTL
+ * in use, since it precedes every message and Anthropic rejects a longer entry
+ * that follows a shorter one.
  */
 export function applyCacheBreakpoint(
 	messages: LLMMessage[],
 	uncachedSuffixCount: number,
-	ttl?: '5m' | '1h',
+	ttl?: CacheTtlConfig,
 	cachedPrefixCount = 0,
 ): LLMMessage[] {
-	const cacheControl: LLMMessageCacheControl = ttl ? { type: 'ephemeral', ttl } : { type: 'ephemeral' }
+	const ttls = resolveTtls(ttl)
+	const prefixCacheControl = cacheControl(ttls.prefix)
+	const tailCacheControl = cacheControl(ttls.tail)
 	const result = [...messages]
 
-	const mark = (idx: number) => {
+	const mark = (idx: number, cacheControl: LLMMessageCacheControl) => {
 		if (idx < 0 || idx >= result.length) return
 		const target = result[idx]
 		switch (target.role) {
@@ -59,7 +102,9 @@ export function applyCacheBreakpoint(
 	// summary, sliding recent window), so the immutable preamble is read at 0.1×
 	// on every inference AND compaction call instead of being re-billed each turn.
 	const prefixIdx = cachedPrefixCount - 1
-	if (prefixIdx !== tailIdx) mark(prefixIdx)
-	mark(tailIdx)
+	if (prefixIdx !== tailIdx) mark(prefixIdx, prefixCacheControl)
+	// When the two coincide there is only one entry to pin, and it covers the
+	// preamble — so it takes the prefix TTL.
+	mark(tailIdx, prefixIdx === tailIdx ? prefixCacheControl : tailCacheControl)
 	return result
 }

package/src/core/llm/openrouter-mapping.test.ts

@@ -21,7 +21,7 @@ const createProvider = () =>
 		defaultModel: 'anthropic/claude-haiku-4.5',
 	})
 
-type AnyBlock = { type: string; cache_control?: { type: 'ephemeral' }; [key: string]: unknown }
+type AnyBlock = { type: string; cache_control?: { type: 'ephemeral'; ttl?: '5m' | '1h' }; [key: string]: unknown }
 type OpenRouterMessage = {
 	role: string
 	content: string | AnyBlock[]
@@ -152,6 +152,42 @@ describe('OpenRouterProvider buildHttpRequest', () => {
 		expect(last.content).toBe('<session-context>plugin status</session-context>')
 	})
 
+	test('ttl reaches the wire and the system block matches it', async () => {
+		// The whole point of the fix: OpenRouter dropped `ttl` on the floor, so a
+		// preset asking for the 1-hour tier silently ran on the 5-minute default.
+		const messages = applyCacheBreakpoint([
+			{ role: 'user', content: 'preamble' },
+			{ role: 'assistant', content: 'ok' },
+			{ role: 'user', content: 'work' },
+		], 0, { prefix: '1h' }, 1)
+
+		const http = await createProvider().buildHttpRequest(buildRequest(messages))
+		const msgs = getBodyMessages(http.body)
+
+		// System precedes every message, so it must carry the longest TTL in use —
+		// Anthropic rejects a longer entry that follows a shorter one.
+		const systemBlocks = asBlocks(msgs[0].content)
+		expect(systemBlocks[0].cache_control).toEqual({ type: 'ephemeral', ttl: '1h' })
+
+		// prefix breakpoint (message index 0 → body index 1) carries 1h
+		expect(asBlocks(msgs[1].content)[0].cache_control).toEqual({ type: 'ephemeral', ttl: '1h' })
+		// tail breakpoint stays on the cheap short tier
+		expect(asBlocks(msgs[3].content)[0].cache_control).toEqual({ type: 'ephemeral' })
+	})
+
+	test('no ttl requested: system block stays on the default tier', async () => {
+		const messages = applyCacheBreakpoint([
+			{ role: 'user', content: 'preamble' },
+			{ role: 'user', content: 'work' },
+		], 0, undefined, 1)
+
+		const http = await createProvider().buildHttpRequest(buildRequest(messages))
+		const msgs = getBodyMessages(http.body)
+
+		expect(asBlocks(msgs[0].content)[0].cache_control).toEqual({ type: 'ephemeral' })
+		expect(asBlocks(msgs[1].content)[0].cache_control).toEqual({ type: 'ephemeral' })
+	})
+
 	test('no flag on any message: only system has cache_control', async () => {
 		const http = await createProvider().buildHttpRequest(buildRequest([
 			{ role: 'user', content: 'What is 2+2?' },

package/src/core/llm/openrouter.ts

@@ -89,11 +89,17 @@ interface OpenRouterErrorResponse {
 // Request body types
 // ============================================================================
 
+interface OpenRouterCacheControl {
+	type: 'ephemeral'
+	/** Omitted = the provider default 5-minute tier. */
+	ttl?: '5m' | '1h'
+}
+
 interface OpenRouterContentItem {
 	type: string
 	text?: string
 	image_url?: { url: string; detail?: string }
-	cache_control?: { type: 'ephemeral' }
+	cache_control?: OpenRouterCacheControl
 }
 
 interface OpenRouterMessage {
@@ -104,18 +110,34 @@ interface OpenRouterMessage {
 }
 
 /**
- * Add `cache_control: { type: 'ephemeral' }` to the LAST content block of an
- * OpenRouterMessage, converting string content to an array text block first
- * so the mark has a place to live.
+ * Add `cache_control` to the LAST content block of an OpenRouterMessage,
+ * converting string content to an array text block first so the mark has a
+ * place to live.
+ *
+ * `ttl` reaches Anthropic through the Chat Completions surface. It is NOT
+ * expressible on the Responses API, where a breakpoint is downgraded to the
+ * default 5-minute tier — so this must stay on `/chat/completions`.
  */
-function applyCacheControlToLastBlock(msg: OpenRouterMessage): void {
+function applyCacheControlToLastBlock(msg: OpenRouterMessage, cacheControl: OpenRouterCacheControl): void {
 	if (typeof msg.content === 'string') {
-		msg.content = [{ type: 'text', text: msg.content, cache_control: { type: 'ephemeral' } }]
+		msg.content = [{ type: 'text', text: msg.content, cache_control: cacheControl }]
 		return
 	}
 	if (msg.content.length === 0) return
 	const lastIdx = msg.content.length - 1
-	msg.content[lastIdx] = { ...msg.content[lastIdx], cache_control: { type: 'ephemeral' } }
+	msg.content[lastIdx] = { ...msg.content[lastIdx], cache_control: cacheControl }
+}
+
+/**
+ * Longest TTL requested anywhere in the message list, or undefined when none.
+ *
+ * The system prompt precedes every message, and Anthropic requires a longer TTL
+ * to come BEFORE a shorter one. So when any breakpoint asks for 1h, the system
+ * block has to carry 1h too — otherwise the order is 5m → 1h → 5m and the
+ * longer entry is invalid.
+ */
+function longestCacheTtl(messages: LLMMessage[]): '1h' | undefined {
+	return messages.some((m) => m.cacheControl?.ttl === '1h') ? '1h' : undefined
 }
 
 interface OpenRouterToolDefinition {
@@ -261,8 +283,10 @@ export class OpenRouterProvider implements LLMProvider {
 		const sanitizedMessages = sanitizeProviderMessages(request.messages, this.name, this.logger)
 		const mappedMessages = await Promise.all(sanitizedMessages.map((m) => this.mapMessage(m, context)))
 
+		const systemTtl = longestCacheTtl(sanitizedMessages)
+		const systemCacheControl: OpenRouterCacheControl = systemTtl ? { type: 'ephemeral', ttl: systemTtl } : { type: 'ephemeral' }
 		const messages: OpenRouterMessage[] = [
-			{ role: 'system', content: [{ type: 'text', text: request.systemPrompt, cache_control: { type: 'ephemeral' } }] },
+			{ role: 'system', content: [{ type: 'text', text: request.systemPrompt, cache_control: systemCacheControl }] },
 			...mappedMessages,
 		]
 
@@ -317,7 +341,9 @@ export class OpenRouterProvider implements LLMProvider {
 	private async mapMessage(msg: LLMMessage, context?: InferenceContext): Promise<OpenRouterMessage> {
 		const mapped = await this.mapMessageContent(msg, context)
 		if (msg.cacheControl) {
-			applyCacheControlToLastBlock(mapped)
+			const cacheControl: OpenRouterCacheControl = { type: 'ephemeral' }
+			if (msg.cacheControl.ttl) cacheControl.ttl = msg.cacheControl.ttl
+			applyCacheControlToLastBlock(mapped, cacheControl)
 		}
 		return mapped
 	}

package/src/index.ts

@@ -53,6 +53,7 @@ export type { AgentPauseReason, AgentState, LLMMessage } from '~/core/agents/sta
 export type { DomainEvent } from '~/core/events/types.js'
 export type { FactoryEventType } from '~/core/events/types.js'
 export { isDomainEvent } from '~/core/events/types.js'
+export type { CacheTtl, CacheTtlConfig } from '~/core/llm/cache-breakpoints.js'
 export type { ChatMessageContentItem, LLMCallLogEntry, LLMCallMessage, ToolResultContent } from '~/core/llm/llm-log-types.js'
 export { contentToString } from '~/core/llm/llm-log-types.js'
 export { applyMiddleware, useProvider, withAnthropic, withMaxTokens, withOpenRouter, withTemperature } from '~/core/llm/middleware.js'

package/src/plugins/result-eviction/plugin.ts

@@ -6,6 +6,7 @@
  * head + tail + file path is returned instead.
  */
 
+import { type ChatMessageContentItem, contentToString, type ToolResultContent } from '~/core/llm/llm-log-types.js'
 import { truncateByTokens } from '~/core/llm/tokens.js'
 import { definePlugin } from '~/core/plugins/plugin-builder.js'
 
@@ -23,6 +24,21 @@ export interface EvictionAgentConfig {
 
 const DEFAULT_MAX_TOKENS = 20_000
 
+const isImage = (item: ChatMessageContentItem): boolean => item.type === 'image_url'
+
+/**
+ * Rebuild a multi-part result with its text collapsed to `preview` and every
+ * image part preserved in order.
+ *
+ * Images are kept rather than truncated because there is no partial image — the
+ * alternative to keeping it is dropping the observation entirely. Text is the
+ * part that both dominates the payload and survives being summarised.
+ */
+const withTextReplaced = (content: ChatMessageContentItem[], preview: string): ChatMessageContentItem[] => [
+	{ type: 'text', text: preview },
+	...content.filter(isImage),
+]
+
 export const resultEvictionPlugin = definePlugin('result-eviction')
 	.agentConfig<EvictionAgentConfig>()
 	.hook('afterToolCall', async (ctx) => {
@@ -34,31 +50,30 @@ export const resultEvictionPlugin = definePlugin('result-eviction')
 		}
 
 		const { content } = ctx.result
-
-		// Only evict string content
-		if (typeof content !== 'string') {
-			return null
-		}
-
 		const maxTokens = agentConfig?.config?.maxTokens ?? DEFAULT_MAX_TOKENS
 
-		const truncation = truncateByTokens(content, maxTokens)
+		// Multi-part results (e.g. a page snapshot returning JSON + a screenshot)
+		// used to bail out here, so no threshold could ever evict them however
+		// large the text grew. Measure the text, evict the text, keep the images.
+		const text = contentToString(content)
+		const truncation = truncateByTokens(text, maxTokens)
 		if (!truncation) {
 			return null
 		}
 
-		// Write full content to file via FileStore
+		// Write full text to file via FileStore
 		const fileName = `${ctx.toolCall.id}.txt`
 		const filePath = `.results/${fileName}`
-		await ctx.files.session.write(filePath, content)
+		await ctx.files.session.write(filePath, text)
 
-		const truncatedContent = `${truncation.content}\n\n[Full output saved to: ${filePath}]`
+		const preview = `${truncation.content}\n\n[Full output saved to: ${filePath}]`
+		const evicted: ToolResultContent = typeof content === 'string' ? preview : withTextReplaced(content, preview)
 
 		return {
 			action: 'modify',
 			result: {
 				isError: false,
-				content: truncatedContent,
+				content: evicted,
 			},
 		}
 	})

package/src/plugins/result-eviction/result-eviction.integration.test.ts

@@ -19,6 +19,18 @@ const bigOutputPlugin = definePlugin('big-output')
 			input: z.object({ size: z.number() }),
 			execute: async (input) => Ok('x'.repeat(input.size)),
 		}),
+		// Mirrors a page-snapshot style result: a large JSON/text block plus an
+		// image part. These used to bypass eviction entirely.
+		createTool({
+			name: 'generate_snapshot',
+			description: 'Generate a text + image result of a given text size',
+			input: z.object({ size: z.number() }),
+			execute: async (input) =>
+				Ok([
+					{ type: 'text' as const, text: 'x'.repeat(input.size) },
+					{ type: 'image_url' as const, imageUrl: { url: 'data:image/png;base64,AAAA' } },
+				]),
+		}),
 	])
 	.build()
 
@@ -100,6 +112,82 @@ describe('result-eviction plugin', () => {
 		await harness.shutdown()
 	})
 
+	it('large multi-part output → text evicted, image part kept', async () => {
+		let toolContent: unknown = null
+		const harness = createEvictionHarness({
+			presets: [createEvictionPreset()],
+			mockHandler: (request) => {
+				const callCount = harness.llmProvider.getCallCount()
+				if (callCount === 1) {
+					return {
+						content: null,
+						toolCalls: [{ id: ToolCallId('tc1'), name: 'generate_snapshot', input: { size: 250_000 } }],
+						finishReason: 'stop',
+						metrics: MockLLMProvider.defaultMetrics(),
+					}
+				}
+				for (const msg of request.messages) {
+					if (msg.role === 'tool' && Array.isArray(msg.content)) {
+						toolContent = msg.content
+					}
+				}
+				return { content: 'Done', toolCalls: [], finishReason: 'stop', metrics: MockLLMProvider.defaultMetrics() }
+			},
+		})
+
+		const session = await harness.createSession('test')
+		await session.sendAndWaitForIdle('Generate large snapshot')
+
+		expect(Array.isArray(toolContent)).toBe(true)
+		const parts = toolContent as Array<{ type: string; text?: string }>
+
+		// Text collapsed to a preview that names the spill file...
+		const textParts = parts.filter((p) => p.type === 'text')
+		expect(textParts).toHaveLength(1)
+		expect(textParts[0].text).toContain('truncated')
+		expect(textParts[0].text).toContain('.results/')
+		expect(textParts[0].text!.length).toBeLessThan(250_000)
+
+		// ...and the image survives, because there is no partial image: the only
+		// alternative to keeping it is losing the observation.
+		expect(parts.filter((p) => p.type === 'image_url')).toHaveLength(1)
+
+		await harness.shutdown()
+	})
+
+	it('small multi-part output → returned unchanged', async () => {
+		let toolContent: unknown = null
+		const harness = createEvictionHarness({
+			presets: [createEvictionPreset()],
+			mockHandler: (request) => {
+				const callCount = harness.llmProvider.getCallCount()
+				if (callCount === 1) {
+					return {
+						content: null,
+						toolCalls: [{ id: ToolCallId('tc1'), name: 'generate_snapshot', input: { size: 100 } }],
+						finishReason: 'stop',
+						metrics: MockLLMProvider.defaultMetrics(),
+					}
+				}
+				for (const msg of request.messages) {
+					if (msg.role === 'tool' && Array.isArray(msg.content)) {
+						toolContent = msg.content
+					}
+				}
+				return { content: 'Done', toolCalls: [], finishReason: 'stop', metrics: MockLLMProvider.defaultMetrics() }
+			},
+		})
+
+		const session = await harness.createSession('test')
+		await session.sendAndWaitForIdle('Generate small snapshot')
+
+		const parts = toolContent as Array<{ type: string; text?: string }>
+		expect(parts.filter((p) => p.type === 'text')[0].text).toBe('x'.repeat(100))
+		expect(parts.filter((p) => p.type === 'image_url')).toHaveLength(1)
+
+		await harness.shutdown()
+	})
+
 	it('full output saved to .results/<toolCallId>.txt', async () => {
 		const harness = createEvictionHarness({
 			presets: [createEvictionPreset()],