@ultimat3/ai 27.8.1 → 27.9.0

package/src/openai-messages.ts

@@ -8,6 +8,8 @@
 import type { AiMediaBlock } from './content-blocks';
 import { isMediaBlock } from './content-blocks';
 import { AiContentUnsupportedError } from './content-errors';
+import type { SystemPrompt } from './prompt-cache';
+import { CACHE_MARK, systemParts, systemText } from './prompt-cache';
 import type { AiContentBlock, AiMessage } from './provider';
 import type { JsonSchema, LlmTool } from './tools';
 
@@ -20,7 +22,12 @@ export interface OpenAiToolCall {
 
 /** A user message's parts, when it carries more than prose. Field names are the format's own. */
 export type OpenAiContentPart =
-  | { readonly type: 'text'; readonly text: string }
+  | {
+      readonly type: 'text';
+      readonly text: string;
+      /** A prompt-cache breakpoint — sent ONLY where `PROMPT_CACHE_RULES` says `parts`. */
+      readonly cache_control?: typeof CACHE_MARK;
+    }
   | { readonly type: 'image_url'; readonly image_url: { readonly url: string } }
   | {
       readonly type: 'file';
@@ -28,14 +35,18 @@ export type OpenAiContentPart =
     };
 
 export type OpenAiMessage =
-  | { readonly role: 'system'; readonly content: string }
+  | { readonly role: 'system'; readonly content: string | readonly OpenAiContentPart[] }
   | { readonly role: 'user'; readonly content: string | readonly OpenAiContentPart[] }
   | {
       readonly role: 'assistant';
       readonly content?: string | undefined;
       readonly tool_calls?: readonly OpenAiToolCall[] | undefined;
     }
-  | { readonly role: 'tool'; readonly tool_call_id: string; readonly content: string };
+  | {
+      readonly role: 'tool';
+      readonly tool_call_id: string;
+      readonly content: string | readonly OpenAiContentPart[];
+    };
 
 export interface OpenAiFunctionTool {
   readonly type: 'function';
@@ -68,12 +79,17 @@ export interface OpenAiToolChoice {
  * speaking this format — vLLM, Ollama, LiteLLM, Together — knows only `system`. OpenAI accepts it.
  */
 export function toOpenAiMessages(
-  system: string | undefined,
+  system: SystemPrompt | undefined,
   messages: readonly AiMessage[],
   target: MediaTarget,
+  /** Whether this endpoint and model take explicit `cache_control` parts (`./prompt-cache`). */
+  marked = false,
 ): readonly OpenAiMessage[] {
   const out: OpenAiMessage[] = [];
-  if (system !== undefined && system !== '') out.push({ role: 'system', content: system });
+  // One string, stable half first, wherever nothing is marked: every server in the family takes
+  // it, and an implicit cache is a prefix match.
+  const text = systemText(system);
+  if (text !== '') out.push({ role: 'system', content: marked ? systemPartsOf(system) : text });
   messages.forEach((message, m) => {
     if (typeof message.content === 'string') {
       out.push({ role: message.role, content: message.content });
@@ -84,9 +100,45 @@ export function toOpenAiMessages(
       out.push(...userTurn(message.content, m, target));
     }
   });
+  if (marked) markTail(out);
   return out;
 }
 
+/** The system message as parts: the stable one marked, the per-run one after it and unmarked. */
+function systemPartsOf(system: SystemPrompt | undefined): readonly OpenAiContentPart[] {
+  const { stable, run } = systemParts(system);
+  const parts: OpenAiContentPart[] = [];
+  // A whitespace-only part is sent as it stands and never marked.
+  if (stable !== '') {
+    parts.push({
+      type: 'text',
+      text: stable,
+      ...(stable.trim() === '' ? {} : { cache_control: CACHE_MARK }),
+    });
+  }
+  if (run !== '') parts.push({ type: 'text', text: run });
+  return parts;
+}
+
+/**
+ * Mark the LAST message's last part — the breakpoint the next turn reads back. A `user` or `tool`
+ * message only: those are the roles whose content this wire takes as parts, and a transcript sent
+ * to a model always ends on one of them.
+ */
+function markTail(out: OpenAiMessage[]): void {
+  const tail = out.at(-1);
+  if (tail === undefined || (tail.role !== 'user' && tail.role !== 'tool')) return;
+  const parts: readonly OpenAiContentPart[] =
+    typeof tail.content === 'string' ? [{ type: 'text', text: tail.content }] : tail.content;
+  const last = parts.at(-1);
+  // Only a text part carries the mark here, and never an empty or whitespace-only one.
+  if (last === undefined || last.type !== 'text' || last.text.trim() === '') return;
+  out[out.length - 1] = {
+    ...tail,
+    content: [...parts.slice(0, -1), { ...last, cache_control: CACHE_MARK }],
+  };
+}
+
 /** Who a media refusal names: this provider, and the model the request is for. */
 export interface MediaTarget {
   readonly provider: string;

package/src/openai-provider.ts

@@ -14,9 +14,10 @@ import { AiKeyMissingError, AiRequestInvalidError, AiTransportError } from './er
 import type { AiFetch } from './fetch-seam';
 import { resolveModel } from './model-resolve';
 import type { ModelId } from './models';
-import { chatCompletionBody } from './openai-body';
+import { chatCompletionBody, sessionHeaders } from './openai-body';
 import type { ChatAnswer } from './openai-wire';
 import { ChatCompletionStream, parseChatCompletion } from './openai-wire';
+import { promptCacheRule } from './prompt-cache';
 import type {
   GenerateRequest,
   GenerateResult,
@@ -105,6 +106,10 @@ class OpenAiProvider implements Provider {
     this.config = config;
   }
 
+  marksPromptCache(model: ModelId): boolean {
+    return promptCacheRule('openai', this.config.baseUrl, model).marks === 'parts';
+  }
+
   /**
    * Above `STREAM_ONLY_MAX_TOKENS` this runs the streaming transport and assembles the result,
    * for the reason the Anthropic provider does: a non-streaming request that large sits on an
@@ -115,10 +120,12 @@ class OpenAiProvider implements Provider {
     // this provider's first listed model — the app's three places are the only ones.
     const model = resolveModel('provider', request.model);
     if (requiresStreaming({ ...request, model })) return this.assemble(request);
+    const { baseUrl } = this.config;
     const response = await this.send(
-      chatCompletionBody({ request, model, stream: false, provider: this.name }),
+      chatCompletionBody({ request, model, stream: false, provider: this.name, baseUrl }),
       false,
       request.signal,
+      sessionHeaders(request, model, baseUrl),
     );
     const answer = parseChatCompletion((await response.json()) as unknown, this.name);
     return this.result(request, model, answer);
@@ -126,10 +133,12 @@ class OpenAiProvider implements Provider {
 
   async *stream(request: GenerateRequest): AsyncIterable<StreamChunk> {
     const model = resolveModel('provider', request.model);
+    const { baseUrl } = this.config;
     const response = await this.send(
-      chatCompletionBody({ request, model, stream: true, provider: this.name }),
+      chatCompletionBody({ request, model, stream: true, provider: this.name, baseUrl }),
       true,
       request.signal,
+      sessionHeaders(request, model, baseUrl),
     );
     if (response.body === null) {
       throw new AiTransportError({
@@ -187,6 +196,7 @@ class OpenAiProvider implements Provider {
     body: Record<string, unknown>,
     streaming: boolean,
     signal: AbortSignal | undefined,
+    extra: Readonly<Record<string, string>>,
   ): Promise<Response> {
     const apiKey = this.apiKey();
     const doFetch: AiFetch = this.config.fetch ?? fetch;
@@ -198,6 +208,8 @@ class OpenAiProvider implements Provider {
           : { authorization: `Bearer ${apiKey}` }),
         'content-type': 'application/json',
         accept: streaming ? 'text/event-stream' : 'application/json',
+        // The prompt-cache session, when the request carries one and this host routes by it.
+        ...extra,
         // Last, so a caller can replace the credential header outright for a gateway that wants
         // its own scheme. Nothing here is ever logged.
         ...this.config.headers,

package/src/openai-wire.ts

@@ -75,9 +75,14 @@ export function parseFinishReason(raw: unknown): StopReason | undefined {
 /**
  * `usage`, or `undefined` when the payload carried none.
  *
- * `prompt_tokens` INCLUDES the cached prefix on this wire, where Anthropic's `input_tokens`
- * excludes it — so the cached half is subtracted out before it is reported as `cacheReadTokens`.
- * Left in place it would be billed twice: once at the full input rate and again at the cache rate.
+ * `prompt_tokens` is the WHOLE prompt on this wire — the part read from a cache and the part
+ * written to one included — where Anthropic's `input_tokens` excludes both. So both are subtracted
+ * out before they are reported under their own counters. Left in place a cached token would be
+ * billed twice: once at the full input rate and again at the cache rate.
+ *
+ * `cache_write_tokens` is OpenRouter's (`prompt_tokens_details.cache_write_tokens`, "Number of
+ * tokens written to the cache"), reported for a model whose vendor bills a write. An endpoint that
+ * never sends it — OpenAI's own, where a write is free — reads as zero.
  */
 export function parseOpenAiUsage(raw: unknown): TokenUsage | undefined {
   const record = asRecord(raw);
@@ -85,14 +90,18 @@ export function parseOpenAiUsage(raw: unknown): TokenUsage | undefined {
   const prompt = countOf(record['prompt_tokens']);
   const completion = countOf(record['completion_tokens']);
   if (prompt === undefined && completion === undefined) return undefined;
-  const cached = countOf(asRecord(record['prompt_tokens_details'])?.['cached_tokens']) ?? 0;
+  const details = asRecord(record['prompt_tokens_details']);
+  const total = prompt ?? 0;
+  // Each clamped to what is left of the prompt: a counter larger than the prompt it is part of is
+  // the endpoint's mistake, and an unclamped one bills tokens that were never sent.
+  const cached = Math.min(countOf(details?.['cached_tokens']) ?? 0, total);
+  const written = Math.min(countOf(details?.['cache_write_tokens']) ?? 0, total - cached);
   return {
-    inputTokens: Math.max((prompt ?? 0) - cached, 0),
+    inputTokens: total - cached - written,
     // `completion_tokens` already contains `reasoning_tokens`; adding them is a double count.
     outputTokens: completion ?? 0,
     cacheReadTokens: cached,
-    // Caching is automatic here and carries no write surcharge, so there is nothing to report.
-    cacheWriteTokens: 0,
+    cacheWriteTokens: written,
   };
 }
 

package/src/prompt-cache-blocks.ts

@@ -0,0 +1,114 @@
+// Single responsibility: the Anthropic Messages wire's half of prompt caching — `system` as text
+// blocks, and `cache_control` on the blocks ./prompt-cache's `blocks` rule names. Pure, so the body
+// a test asserts is the body the socket gets.
+
+import type { SystemPrompt } from './prompt-cache';
+import { CACHE_MARK, systemParts } from './prompt-cache';
+import type { AiMessage, GenerateRequest } from './provider';
+
+type Marked<T> = T & { readonly cache_control?: typeof CACHE_MARK };
+type TextBlock = Marked<{ readonly type: 'text'; readonly text: string }>;
+type WireMessage = { readonly role: AiMessage['role']; readonly content: unknown };
+
+/** The vendor's ceiling on breakpoints in one request; a fifth is a 400. */
+const MAX_BREAKPOINTS = 4;
+
+/** An empty or whitespace-only text block cannot carry a mark (and is a 400 with one). */
+const blank = (text: string): boolean => text.trim() === '';
+
+export interface AnthropicPrompt {
+  readonly system: string | readonly TextBlock[] | undefined;
+  readonly messages: readonly WireMessage[];
+}
+
+/**
+ * `system` and `messages` as the wire takes them. Unmarked, a string system stays the string every
+ * existing request sent and the messages pass through untouched.
+ *
+ * Marked, at most three breakpoints, spent in this order: the transcript's TAIL (what the next
+ * request reads back), the stable system block (what other runs share), then the message before
+ * the tail (a second start for the 20-block lookback). Marks the caller already placed — a raw
+ * block or tool carrying its own `cache_control` — count against the four first, so this never
+ * turns a legal request into a 400; ours are the ones dropped.
+ */
+export function anthropicPrompt(request: GenerateRequest, marked: boolean): AnthropicPrompt {
+  const { messages } = request;
+  if (!marked) {
+    return { system: systemOf(request.system, false), messages: messages.map(plain) };
+  }
+  let left = MAX_BREAKPOINTS - placedBy(request);
+  const take = (wanted: boolean): boolean => {
+    if (!wanted || left <= 0) return false;
+    left -= 1;
+    return true;
+  };
+  const tail = messages.length - 1;
+  const markTail = take(tail >= 0 && markableAt(messages[tail]) !== undefined);
+  const markSystem = take(!blank(systemParts(request.system).stable));
+  const markPrevious = take(tail >= 1 && markableAt(messages[tail - 1]) !== undefined);
+  return {
+    system: systemOf(request.system, markSystem),
+    messages: messages.map((message, index) =>
+      (index === tail && markTail) || (index === tail - 1 && markPrevious)
+        ? { role: message.role, content: markedContent(message.content) }
+        : plain(message),
+    ),
+  };
+}
+
+const plain = (message: AiMessage): WireMessage => ({
+  role: message.role,
+  content: message.content,
+});
+
+/** The stable block first and the only one marked; an empty part is dropped, never sent. */
+function systemOf(
+  system: SystemPrompt | undefined,
+  marked: boolean,
+): string | readonly TextBlock[] | undefined {
+  if (system === undefined) return undefined;
+  if (typeof system === 'string' && !marked) return system;
+  const { stable, run } = systemParts(system);
+  const blocks: TextBlock[] = [];
+  if (stable !== '') {
+    blocks.push({ type: 'text', text: stable, ...(marked ? { cache_control: CACHE_MARK } : {}) });
+  }
+  if (run !== '') blocks.push({ type: 'text', text: run });
+  return blocks.length === 0 ? undefined : blocks;
+}
+
+/**
+ * Where one message's mark goes: its LAST block that is not blank text, so the mark moves back
+ * past a trailing empty block rather than landing on it. `undefined` when no block can carry one.
+ */
+function markableAt(message: AiMessage | undefined): number | undefined {
+  if (message === undefined) return undefined;
+  const { content } = message;
+  if (typeof content === 'string') return blank(content) ? undefined : 0;
+  for (let index = content.length - 1; index >= 0; index -= 1) {
+    const block = content[index];
+    if (block !== undefined && !(block.type === 'text' && blank(block.text))) return index;
+  }
+  return undefined;
+}
+
+function markedContent(content: AiMessage['content']): unknown {
+  if (typeof content === 'string') {
+    return [{ type: 'text', text: content, cache_control: CACHE_MARK }];
+  }
+  const at = markableAt({ role: 'user', content });
+  return content.map((block, index) =>
+    index === at ? { ...block, cache_control: CACHE_MARK } : block,
+  );
+}
+
+/** Breakpoints the caller placed itself: the types do not offer the field, a cast still sends it. */
+function placedBy(request: GenerateRequest): number {
+  const carries = (value: unknown): boolean =>
+    typeof value === 'object' && value !== null && Object.hasOwn(value, 'cache_control');
+  let count = (request.tools ?? []).filter(carries).length;
+  for (const { content } of request.messages) {
+    if (typeof content !== 'string') count += content.filter(carries).length;
+  }
+  return count;
+}

package/src/prompt-cache.ts

@@ -0,0 +1,189 @@
+// Single responsibility: PROVIDER prompt caching — the neutral request shapes (a system prompt in
+// a stable and a per-run part, the hint that asks for the prefix to be cached) and the table that
+// says how each wire and host is told. Not the gateway's response cache, not `llm()`'s semantic one.
+//
+// THE RULE, and where it comes from (both pages read 2026-10-11):
+//
+//   Anthropic — https://platform.claude.com/docs/en/docs/build-with-claude/prompt-caching
+//     "Place `cache_control` directly on individual content blocks"; the shape is
+//     `"cache_control": { "type": "ephemeral" }` (5-minute TTL, the default).
+//     "Prompt caching references the entire prompt: `tools`, `system`, and `messages` (in that
+//     order), up to and including the block designated with `cache_control`."
+//     "You can define up to 4 cache breakpoints" — a fifth is a 400.
+//     "The system checks at most 20 positions per breakpoint, counting the breakpoint itself."
+//     "Shorter prompts cannot be cached, even if marked with `cache_control`. Any requests to cache
+//     fewer than this number of tokens will be processed without caching, and no error is
+//     returned." Minimums are per model (512 to 4,096 tokens) — `ModelSpec.cacheMinimumTokens`.
+//     "5-minute cache write tokens are 1.25 times the base input tokens price … Cache read tokens
+//     are 0.1 times the base input tokens price" (0.05x and 0.025x on named models).
+//
+//   OpenRouter — https://openrouter.ai/docs/features/prompt-caching
+//     Anthropic models: `cache_control` on a `text` part inside a message's `content` array;
+//     "There is a limit of four explicit breakpoints."
+//     Gemini: "Gemini 2.5 series models and newer support implicit caching" with no write cost; an
+//     explicit breakpoint is billed as a write (input price plus storage). So: never mark.
+//     OpenAI: automatic, no configuration. So: never mark.
+//     Stickiness: `session_id` in the body or the `x-session-id` header, at most 256 characters;
+//     "Sticky routing is skipped when you set `provider.order`" — this package sends no `provider`
+//     object at all, and must not start to.
+//     Usage: `prompt_tokens_details.cached_tokens` and `.cache_write_tokens`.
+//     https://openrouter.ai/docs/use-cases/usage-accounting: "The `usage: { include: true }` …
+//     parameters are deprecated and have no effect" — so it is not sent.
+//
+//   OpenAI — automatic above 1,024 tokens (the OpenRouter page's summary of it); the body field
+//     `prompt_cache_key` groups requests that share a prefix. OpenRouter reads it as a session id
+//     when neither `session_id` nor the header is given.
+//
+//   Checked against the live service, 2026-10-11, `anthropic/claude-sonnet-5.5` on OpenRouter, a
+//   three-turn `agent()` run: a mark on a `role: 'tool'` message's text part IS honoured (turn 2
+//   read turn 1's 19,914-token prefix and wrote 72), and `prompt_tokens` INCLUDES both the read and
+//   the written tokens — which is why ./openai-wire subtracts both.
+//
+// Two limits of the rows below, stated rather than hidden:
+//   - the `anthropic` row matches ANY host an `anthropicProvider({ baseUrl })` is pointed at — a
+//     proxy speaking that wire is assumed to speak its `cache_control` too;
+//   - the `parts` row marks the tail only, so the next turn finds this turn's entry through the
+//     20-position lookback alone. A turn that appends more blocks than that — more than about 9
+//     tool calls in ONE turn, each a `tool_use` and a `tool_result` — misses it and is billed as a
+//     fresh write of the whole transcript. The stable system block still reads.
+//
+// The table is DATA: a new host or model family is a row. Unknown endpoints get NOTHING — a field
+// a self-hosted server has never heard of is a 400 on some of them, and a mark is never worth that.
+
+/**
+ * A system prompt in two parts. `stable` is byte-identical across runs — rules, a knowledge-base
+ * index — and is what a cache shares BETWEEN runs; `run` is whatever differs per run and comes
+ * after it, so it never invalidates the stable half. A cache is a prefix match: one changed byte in
+ * `stable` (a timestamp, an unsorted list) is a full-price write on every call.
+ */
+export interface SystemParts {
+  readonly stable: string;
+  readonly run?: string;
+}
+
+/** A plain string is the whole prompt as one stable part. */
+export type SystemPrompt = string | SystemParts;
+
+/**
+ * "Cache this request's prefix": present on a `GenerateRequest`, each provider marks the stable
+ * system part and the transcript's tail the way its wire wants — or not at all, where the host
+ * caches implicitly. Absent, nothing is marked and nothing is sent.
+ */
+export interface PromptCacheHint {
+  /**
+   * A key naming the runs that share a prefix, for hosts that route or shard by one (an
+   * `x-session-id` header on OpenRouter, `prompt_cache_key` on OpenAI). Never a secret, never
+   * personal data: it leaves the process in a header.
+   */
+  readonly session?: string;
+}
+
+/** OpenRouter's documented ceiling on a session id ("Maximum length is 256 characters"). */
+export const SESSION_KEY_MAX = 256;
+
+/**
+ * The longest `prompt_cache_key` sent to OpenAI verbatim. OpenAI's prompt-caching guide and its
+ * SDK types (read 2026-10-11) state NO maximum, and its API reference could not be read from
+ * here; 64 is the conservative figure a reviewer recalled. A longer key is not refused — it is a
+ * grouping key, so it is replaced by a deterministic hash of itself: same key, same group.
+ */
+export const OPENAI_CACHE_KEY_MAX = 64;
+
+/** The one mark this package sends: the 5-minute TTL, which is the rate `costOf` prices. */
+export const CACHE_MARK = { type: 'ephemeral' } as const;
+
+/** Both parts as one text, stable first — for an estimate, and for a wire with one system string. */
+export function systemText(system: SystemPrompt | undefined): string {
+  if (system === undefined) return '';
+  if (typeof system === 'string') return system;
+  return [system.stable, system.run ?? ''].filter((part) => part !== '').join('\n\n');
+}
+
+/** The parts, normalised: `''` for an absent half, so a wire never sends an empty block. */
+export function systemParts(system: SystemPrompt | undefined): Required<SystemParts> {
+  if (system === undefined) return { stable: '', run: '' };
+  if (typeof system === 'string') return { stable: system, run: '' };
+  return { stable: system.stable, run: system.run ?? '' };
+}
+
+export type CacheWire = 'anthropic' | 'openai';
+
+export interface PromptCacheRule {
+  readonly wire: CacheWire;
+  /** A hostname and every subdomain of it; `'*'` is any endpoint on this wire. */
+  readonly host: string;
+  /** A model-id prefix; `''` is every model. */
+  readonly model: string;
+  /**
+   * `blocks`: `cache_control` on Anthropic content blocks — the stable system block and the last
+   * block of the last two messages (three of the four breakpoints; the fourth is left free).
+   * `parts`: the same mark on chat-completions content PARTS — the stable system part and the tail
+   * message's last part (two breakpoints).
+   * `none`: the host caches by itself, or is unknown.
+   */
+  readonly marks: 'blocks' | 'parts' | 'none';
+  /** Where a `PromptCacheHint.session` goes. */
+  readonly session: 'x-session-id' | 'prompt_cache_key' | 'none';
+}
+
+/** First match wins. */
+export const PROMPT_CACHE_RULES: readonly PromptCacheRule[] = [
+  { wire: 'anthropic', host: '*', model: '', marks: 'blocks', session: 'none' },
+  {
+    wire: 'openai',
+    host: 'openrouter.ai',
+    model: 'anthropic/',
+    marks: 'parts',
+    session: 'x-session-id',
+  },
+  // Gemini, OpenAI, and everything else OpenRouter fronts: implicit or automatic. An explicit mark
+  // on Gemini turns a free implicit cache into a paid write.
+  { wire: 'openai', host: 'openrouter.ai', model: '', marks: 'none', session: 'x-session-id' },
+  { wire: 'openai', host: 'api.openai.com', model: '', marks: 'none', session: 'prompt_cache_key' },
+];
+
+const NO_RULE: Pick<PromptCacheRule, 'marks' | 'session'> = { marks: 'none', session: 'none' };
+
+/** OpenAI's own endpoint, which is what an `openAiProvider()` with no `baseUrl` talks to. */
+const DEFAULT_OPENAI_HOST = 'api.openai.com';
+
+/** The rule for one request. Total: a base URL that does not parse matches no host. */
+export function promptCacheRule(
+  wire: CacheWire,
+  baseUrl: string | undefined,
+  model: string,
+): Pick<PromptCacheRule, 'marks' | 'session'> {
+  const host = baseUrl === undefined ? DEFAULT_OPENAI_HOST : hostOf(baseUrl);
+  const found = PROMPT_CACHE_RULES.find(
+    (rule) =>
+      rule.wire === wire &&
+      (rule.host === '*' || host === rule.host || host.endsWith(`.${rule.host}`)) &&
+      model.startsWith(rule.model),
+  );
+  return found ?? NO_RULE;
+}
+
+function hostOf(baseUrl: string): string {
+  return URL.canParse(baseUrl) ? new URL(baseUrl).hostname.toLowerCase() : '';
+}
+
+/**
+ * What a declaration writes: `'auto'` marks every request of the call, `'off'` marks none, and the
+ * object form is `'auto'` plus a session key — a constant, or derived per call (per org, say).
+ */
+export type PromptCacheOption<A> =
+  | 'auto'
+  | 'off'
+  | { readonly session: string | ((args: A) => string) };
+
+/** The hint one call sends on every request, or `undefined` for none. */
+export function promptCacheHint<A>(
+  option: PromptCacheOption<A> | undefined,
+  fallback: 'auto' | 'off',
+  args: A,
+): PromptCacheHint | undefined {
+  const chosen = option ?? fallback;
+  if (chosen === 'off') return undefined;
+  if (chosen === 'auto') return {};
+  return { session: typeof chosen.session === 'string' ? chosen.session : chosen.session(args) };
+}

package/src/provider.ts

@@ -12,6 +12,9 @@ import type { AiFetch } from './fetch-seam';
 import { resolveModel } from './model-resolve';
 import type { Effort, ModelId, ThinkingMode } from './models';
 import { modelSpec, reasoningBody } from './models';
+import type { PromptCacheHint, SystemPrompt } from './prompt-cache';
+import { promptCacheRule, systemText } from './prompt-cache';
+import { anthropicPrompt } from './prompt-cache-blocks';
 import { readSse } from './sse';
 import type { LlmTool, LlmToolCall } from './tools';
 import {
@@ -65,7 +68,14 @@ export function messageText(message: AiMessage): string {
 
 export interface GenerateRequest {
   readonly model?: ModelId;
-  readonly system?: string;
+  /** One string, or a stable part and a per-run part (`./prompt-cache`) — two blocks on the wire. */
+  readonly system?: SystemPrompt;
+  /**
+   * Ask the provider to cache this request's prefix: the stable system part and the transcript's
+   * tail, marked per wire and host by `PROMPT_CACHE_RULES`. Absent sends no mark. Not part of
+   * `promptCacheKey` — it changes what a call costs, never what it answers.
+   */
+  readonly promptCache?: PromptCacheHint;
   readonly messages: readonly AiMessage[];
   /** Enforced output ceiling. The model never sees it, so it can be cut off mid-answer. */
   readonly maxTokens: number;
@@ -154,6 +164,12 @@ export interface Provider {
   readonly models: readonly ModelId[];
   generate(request: GenerateRequest): Promise<GenerateResult>;
   stream(request: GenerateRequest): AsyncIterable<StreamChunk>;
+  /**
+   * Whether a request carrying `promptCache` for this model would be sent with an explicit cache
+   * mark — billed as a cache WRITE. Absent is no: the gateway prices the write only where one
+   * can happen.
+   */
+  marksPromptCache?(model: ModelId): boolean;
 }
 
 /**
@@ -323,18 +339,26 @@ export class AnthropicProvider implements Provider {
     };
   }
 
+  marksPromptCache(model: ModelId): boolean {
+    return promptCacheRule('anthropic', this.config.baseUrl, model).marks === 'blocks';
+  }
+
   /** The request body. Pure and side-effect free so a test can assert it directly. */
   body(request: GenerateRequest): Record<string, unknown> {
     const model = this.modelOf(request);
     // The blocks pass through untouched below — they ARE this wire's shapes — so this is the screen.
     assertMediaContent(request.messages, { provider: this.name, model });
+    const prompt = anthropicPrompt(
+      request,
+      request.promptCache !== undefined && this.marksPromptCache(model),
+    );
     const body: Record<string, unknown> = {
       model,
       max_tokens: Math.min(request.maxTokens, modelSpec(model).maxOutput),
-      messages: request.messages.map((m) => ({ role: m.role, content: m.content })),
+      messages: prompt.messages,
       ...reasoningBody(model, request.effort, request.thinking),
     };
-    if (request.system !== undefined) body['system'] = request.system;
+    if (prompt.system !== undefined) body['system'] = prompt.system;
     if (request.tools !== undefined && request.tools.length > 0) body['tools'] = request.tools;
     if (request.stopSequences !== undefined) body['stop_sequences'] = request.stopSequences;
     return body;
@@ -439,7 +463,7 @@ const completionCeiling = (request: GenerateRequest): number =>
 /** The prompt half alone — what the provider bills at the input rate. */
 export function estimateInputTokens(request: GenerateRequest): number {
   const body = request.messages.map(messageText).join(' ');
-  const text = estimateTextTokens(body) + estimateTextTokens(request.system ?? '');
+  const text = estimateTextTokens(body) + estimateTextTokens(systemText(request.system));
   return text + mediaTokenEstimate(request.messages);
 }
 
@@ -448,13 +472,17 @@ export function estimateInputTokens(request: GenerateRequest): number {
  * `maxTokens` at the output rate. Deliberately pessimistic — a ceiling checked against an
  * optimistic estimate is a ceiling one long completion walks through.
  */
-export function estimateCost(request: GenerateRequest): Money {
-  return costOf(resolveModel('provider', request.model), {
-    inputTokens: estimateInputTokens(request),
-    outputTokens: completionCeiling(request),
-    cacheReadTokens: 0,
-    cacheWriteTokens: 0,
-  });
+export function estimateCost(request: GenerateRequest, marked = false): Money {
+  const model = resolveModel('provider', request.model);
+  const prompt = estimateInputTokens(request);
+  const usage = { outputTokens: completionCeiling(request), cacheReadTokens: 0 };
+  const asInput = costOf(model, { ...usage, inputTokens: prompt, cacheWriteTokens: 0 });
+  // A request a provider WILL mark (`marked`: the gateway asks the providers serving the model) can
+  // be billed as a cache write end to end. The DEARER of the two: a row whose write is cheaper
+  // than input must not lower the ceiling's estimate below what an unmarked call costs.
+  if (!marked || request.promptCache === undefined) return asInput;
+  const asWrite = costOf(model, { ...usage, inputTokens: 0, cacheWriteTokens: prompt });
+  return asWrite.minor > asInput.minor ? asWrite : asInput;
 }
 
 export function estimateTextTokens(text: string): number {