@ultimat3/ai 27.8.1 → 27.10.0

package/src/llm.ts

@@ -21,36 +21,36 @@
  * same rules.
  */
 
-import type { Action, ActionMcp, ActionPolicy, InvokeOptions } from '@ultimat3/action';
+import type { Action, InvokeOptions } from '@ultimat3/action';
 import { action } from '@ultimat3/action';
 import type { Ctx, Span, SpanAttributes } from '@ultimat3/core';
 import { finiteCount, withSpan } from '@ultimat3/core';
-import type { Money } from '@ultimat3/money';
 import type { InferInput, InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
 import { formatIssues, validateAsync } from '@ultimat3/schema';
+import { runTotal } from './agent-usage';
+import { LlmTruncatedError } from './answer-errors';
 import type { BudgetLimits } from './budget';
 import { budgetKeysFor, currentBudget, screenedMoney, withBudget } from './budget';
-import {
-  LlmOutputInvalidError,
-  LlmRefusedError,
-  LlmStreamInvalidError,
-  LlmTruncatedError,
-} from './errors';
+import { LlmOutputInvalidError, LlmRefusedError, LlmStreamInvalidError } from './errors';
 import type { Gateway } from './gateway';
-import type { LlmCache } from './llm-cache';
 import { openCache } from './llm-cache';
+import type { LlmBudget, LlmDef } from './llm-def';
 import type { LlmSink, LlmStreamChunk } from './llm-stream';
 import { currentLlmSink, llmStream, streamOneTurn, withLlmSink } from './llm-stream';
 import { modelForCall } from './model-resolve';
-import type { ModelId } from './models';
 import { moreCapableThan } from './models';
-import type { Prompt, PromptVars } from './prompt';
+import type { PromptVars } from './prompt';
+import { promptCacheHint } from './prompt-cache';
 import type { AiMessage, GenerateRequest, GenerateResult } from './provider';
 import { isTruncated } from './provider';
 import { assertNoSecrets } from './redaction';
 import type { Respond } from './respond';
 import { parseJsonish, RESPOND, respondFor } from './respond';
-import { aiGateway, aiRedactor } from './runtime';
+import type { RunMeter } from './run-meter';
+import { runMeter } from './run-meter';
+import { aiGateway, aiRedactor, installedGateway } from './runtime';
+import type { Sampling } from './sampling';
+import { ignoredSampling, screenSampling } from './sampling';
 
 /** Two attempts total: the answer, then one repair turn. See `LlmOutputInvalidError`. */
 const ATTEMPTS = 2;
@@ -61,45 +61,7 @@ const ATTEMPTS = 2;
  */
 const DEFAULT_MAX_TOKENS = 4_096;
 
-/** Per-call ceilings, checked before the provider is reached. Never truncates — refuses. */
-export interface LlmBudget {
-  /** Prompt tokens. */
-  readonly tokensIn?: number;
-  /** Worst-case price of one call, integer minor units. */
-  readonly costPerCall?: Money;
-}
-
-export interface LlmVarsArgs<TInput extends StandardSchemaV1> {
-  readonly input: InferOutput<TInput>;
-  readonly ctx: Ctx;
-}
-
-export interface LlmDef<
-  TInput extends StandardSchemaV1,
-  TOutput extends StandardSchemaV1,
-  V extends PromptVars,
-> {
-  /**
-   * Overrides the prompt artifact's own model. The prompt's is part of its content hash and
-   * travels with it; this one is the deployment decision, so it wins.
-   */
-  readonly model?: ModelId;
-  readonly input: TInput;
-  readonly output: TOutput;
-  readonly prompt: Prompt<V>;
-  /**
-   * Everything the prompt is allowed to see, derived from the parsed input. Required, and
-   * async, because the input is usually an id and the prompt needs the row behind it — this
-   * is the one declared place a model call loads data, so a reader can see what was sent.
-   */
-  vars(args: LlmVarsArgs<TInput>): V | Promise<V>;
-  readonly cache?: LlmCache<InferOutput<TInput>>;
-  readonly budget?: LlmBudget;
-  readonly policy: ActionPolicy;
-  readonly mcp?: ActionMcp;
-  /** Enforced completion ceiling. The model never sees it, so it can be cut off mid-answer. */
-  readonly maxTokens?: number;
-}
+export type { LlmBudget, LlmCall, LlmDef, LlmVarsArgs } from './llm-def';
 
 /**
  * An `action` with one extra way to be called. Every projection an action has, it has — its MCP tool,
@@ -143,12 +105,13 @@ export function llm<
   // off for the life of the process. A floor of 1 because a completion ceiling of zero tokens is
   // a call that cannot answer.
   finiteCount('llm', 'maxTokens', def.maxTokens ?? DEFAULT_MAX_TOKENS, 1);
+  const sampling = screenSampling('llm', def);
   const built = action<TInput, TOutput>({
     input: def.input,
     output: def.output,
     policy: def.policy,
     ...(def.mcp === undefined ? {} : { mcp: def.mcp }),
-    handle: (args) => generate(def, respond, args),
+    handle: (args) => generate(def, respond, sampling, args),
   });
   return streamable(built);
 }
@@ -186,6 +149,7 @@ async function generate<
 >(
   def: LlmDef<TInput, TOutput, V>,
   respond: Respond,
+  sampling: Sampling,
   args: { readonly input: InferOutput<TInput>; readonly ctx: Ctx },
 ): Promise<InferOutput<TOutput>> {
   const { prompt } = def;
@@ -207,6 +171,7 @@ async function generate<
   const rendered = redact(rawPrompt);
   const system = prompt.system === undefined ? undefined : redact(prompt.system);
   const redacted = rendered !== rawPrompt || system !== prompt.system;
+  const promptCache = promptCacheHint(def.promptCache, 'off', args);
 
   return withSpan('ai.llm', async (span) => {
     span.setAttributes({
@@ -217,6 +182,11 @@ async function generate<
       // an audit asks evidence for, and a redactor that silently stopped matching looks identical
       // to one that had nothing to remove until this attribute separates them.
       'llm.redacted': redacted,
+      'llm.sampling.ignored': ignoredSampling(
+        model,
+        sampling,
+        installedGateway()?.samplingWires?.(model),
+      ).join(','),
     });
 
     // A cached answer is still data of unknown provenance, so it goes through the schema like
@@ -239,10 +209,12 @@ async function generate<
     const base: GenerateRequest = {
       model,
       ...(system === undefined ? {} : { system }),
+      ...(promptCache === undefined ? {} : { promptCache }),
       messages: [{ role: 'user', content: rendered }],
       maxTokens: def.maxTokens ?? DEFAULT_MAX_TOKENS,
       ...(prompt.effort === undefined ? {} : { effort: prompt.effort }),
       ...(prompt.thinking === undefined ? {} : { thinking: prompt.thinking }),
+      ...sampling,
       // The caller's own signal, forwarded to the transport exactly as `agent()` does — and
       // inherited by `streamedAnswer` from this same `base`. Without it a disconnected caller left
       // the provider call in flight, billed and unread, and the repair turn bought a SECOND one.
@@ -261,17 +233,22 @@ async function generate<
     );
     const sink = currentLlmSink();
 
-    return withBudget(ledger, async () => {
+    // Every provider call goes through the meter, so a failure AFTER one answered — a refusal, a
+    // truncation, a schema no repair satisfied — is thrown carrying what it was billed.
+    const meter = runMeter();
+    const answer = async (): Promise<InferOutput<TOutput>> => {
       if (sink !== undefined) {
-        const value = await streamedAnswer(def.output, name, gateway, base, sink, span);
+        const value = await streamedAnswer(def, name, gateway, base, sink, span, meter);
         await cache?.remember(value);
         return value;
       }
       let messages: readonly AiMessage[] = request.messages;
       let issues = 'no output';
+      let rejected: readonly string[] | undefined;
       for (let attempt = 1; attempt <= ATTEMPTS; attempt += 1) {
-        const result = await gateway.generate({ ...request, messages });
+        const result = await meter.call(gateway.generate({ ...request, messages }));
         span.setAttributes({ 'llm.attempts': attempt, ...answerAttributes(result) });
+        await observeCall(def, meter, result, attempt, rejected);
         // Branch on the stop reason BEFORE reading the answer. A refusal carries empty or partial
         // content, so parsing it first reports a schema disagreement — a cause that is wrong, a
         // fix that does not apply, and a repair turn spent buying the same refusal again.
@@ -287,25 +264,38 @@ async function generate<
             explanation: result.stopDetails?.explanation,
           });
         }
+        // Cut off, so there is no answer to read — and it is read BEFORE the schema for the reason
+        // the refusal is: a `respond` call stopped at the ceiling arrives with no arguments, both
+        // wires read that as `{}`, and `{}` SATISFIES an `output` whose keys are optional. The
+        // ceiling is the same on the next attempt, so the repair turn is a second truncation.
+        if (isTruncated(result.stopReason)) {
+          throw new LlmTruncatedError({ prompt: name, maxTokens: request.maxTokens });
+        }
         const parsed = await validateAsync(def.output, respond.read(result));
         if (parsed.issues === undefined) {
           await cache?.remember(parsed.value);
           return parsed.value;
         }
-        // A cut-off answer that also fails its schema is not a disagreement about the shape: the
-        // ceiling is the same on the next attempt, so the repair turn is a second truncation.
-        if (isTruncated(result.stopReason)) {
-          throw new LlmTruncatedError({ prompt: name, maxTokens: request.maxTokens });
-        }
-        issues = formatIssues(parsed.issues).join('; ');
+        rejected = formatIssues(parsed.issues);
+        issues = rejected.join('; ');
         const echo = assistantEcho(result);
         messages =
           echo === undefined
             ? [...messages, repair(issues)]
             : [...messages, { role: 'assistant', content: echo }, repair(issues)];
       }
-      throw new LlmOutputInvalidError({ prompt: name, attempts: ATTEMPTS, issues });
-    });
+      throw new LlmOutputInvalidError({
+        prompt: name,
+        attempts: ATTEMPTS,
+        issues,
+        ...(rejected === undefined ? {} : { list: rejected }),
+      });
+    };
+    return withBudget(ledger, () =>
+      answer().catch((error: unknown) => {
+        throw meter.fail(error);
+      }),
+    );
   });
 }
 
@@ -334,16 +324,23 @@ export function answerAttributes(result: GenerateResult): SpanAttributes {
  * carries empty or partial content and parsing it first reports a schema disagreement that is not
  * one. Truncation is the same call it is on the non-streaming path.
  */
-async function streamedAnswer<TOutput extends StandardSchemaV1>(
-  output: TOutput,
+async function streamedAnswer<
+  TInput extends StandardSchemaV1,
+  TOutput extends StandardSchemaV1,
+  V extends PromptVars,
+>(
+  def: LlmDef<TInput, TOutput, V>,
   name: string,
   gateway: Gateway,
   request: GenerateRequest,
   sink: LlmSink,
   span: Span,
+  meter: RunMeter,
 ): Promise<InferOutput<TOutput>> {
-  const result = await streamOneTurn(gateway, request, sink);
+  const { output } = def;
+  const result = await meter.call(streamOneTurn(gateway, request, sink));
   span.setAttributes({ 'llm.attempts': 1, ...answerAttributes(result) });
+  await observeCall(def, meter, result, 1, undefined);
   if (result.stopReason === 'refusal') {
     throw new LlmRefusedError({
       prompt: name,
@@ -369,6 +366,32 @@ async function streamedAnswer<TOutput extends StandardSchemaV1>(
   });
 }
 
+/** One provider call, reported to the declaration's `onCall`. Awaited; a throw fails the call. */
+async function observeCall<
+  TInput extends StandardSchemaV1,
+  TOutput extends StandardSchemaV1,
+  V extends PromptVars,
+>(
+  def: LlmDef<TInput, TOutput, V>,
+  meter: RunMeter,
+  result: GenerateResult,
+  attempt: number,
+  repairOf: readonly string[] | undefined,
+): Promise<void> {
+  if (def.onCall === undefined) return;
+  await def.onCall({
+    attempt,
+    model: result.model,
+    provider: result.provider,
+    stopReason: result.stopReason,
+    usage: result.usage,
+    cost: result.cost,
+    ...(result.costExact === undefined ? {} : { costExact: result.costExact }),
+    ...(repairOf === undefined ? {} : { repairOf }),
+    run: runTotal(result.model, undefined, meter.total()),
+  });
+}
+
 /** `undefined` for "does not fit". Wrapped so a legitimately falsy value is still a hit. */
 async function accept<TOutput extends StandardSchemaV1>(
   schema: TOutput,

package/src/models.ts

@@ -5,6 +5,7 @@
 
 import { assert, finiteCount, renderCauseValue, renderFixLiteral } from '@ultimat3/core';
 import type { Money } from '@ultimat3/money';
+import { exponentOf } from '@ultimat3/money';
 import { AiModelUnknownError, AiRequestInvalidError } from './errors';
 
 /**
@@ -59,11 +60,18 @@ export interface ModelReasoning {
   readonly disabledThinking?: 'disabled' | 'between_tools';
 }
 
+/** Digits a price — and the cost computed from it — may carry beyond the currency's minor unit. */
+export const PRICE_SCALE_DIGITS = 6;
+
 export interface ModelSpec {
   readonly id: ModelId;
   readonly contextWindow: number;
   readonly maxOutput: number;
-  /** Cost of one million input tokens, in minor units. */
+  /**
+   * Cost of one million input tokens: minor units, or — for a price below one minor unit per
+   * million — `Money` at a finer `scale`: `{ minor: 75, currency: 'USD', scale: 3 }` is $0.075.
+   * The same holds for the three prices below.
+   */
   readonly inputPerMillion: Money;
   /** Cost of one million output tokens, in minor units. */
   readonly outputPerMillion: Money;
@@ -82,6 +90,18 @@ export interface ModelSpec {
   /** Minimum cacheable prefix; a shorter prefix silently does not cache. */
   readonly cacheMinimumTokens: number;
   readonly reasoning: ModelReasoning;
+  /**
+   * The sampling knobs this model TAKES (`./sampling`). Absent is none: a `temperature` or `seed` a
+   * request asks for is then not sent, because a reasoning model answers one with a 400. A row for
+   * a model that does take them says `{ temperature: true, seed: true }`.
+   */
+  readonly sampling?: { readonly temperature?: boolean; readonly seed?: boolean };
+  /**
+   * `false` for a model that rejects a FORCED tool choice (a 400 naming `tool_choice`, which
+   * always-thinking models answer). `llm()`'s one `respond` tool is then offered under the
+   * endpoint's default, and a prose answer takes the repair turn. Absent is forced, as before.
+   */
+  readonly forcedToolChoice?: boolean;
   /**
    * The non-text blocks a message to this model may carry. Absent is TEXT ONLY, deliberately: an
    * app's own row for a text-only endpoint must not have an image sent to it, and the refusal
@@ -147,10 +167,34 @@ export function registerModel(spec: ModelSpec): ModelSpec {
   if (spec.cacheWritePerMillion !== undefined) {
     finiteCount(SUBJECT, `${spec.id} cacheWritePerMillion.minor`, spec.cacheWritePerMillion.minor);
   }
+  assertPriceScales(spec);
   registry.set(spec.id, spec);
   return spec;
 }
 
+/**
+ * A price may state a `scale` finer than the currency's minor unit — `{ minor: 75, currency:
+ * 'USD', scale: 3 }` is $0.075 per million — up to the scale a COST is carried at. Finer than that
+ * cannot be priced exactly, so the row is refused at boot instead of rounded at every call.
+ */
+function assertPriceScales(spec: ModelSpec): void {
+  const prices = {
+    inputPerMillion: spec.inputPerMillion,
+    outputPerMillion: spec.outputPerMillion,
+    cacheReadPerMillion: spec.cacheReadPerMillion,
+    cacheWritePerMillion: spec.cacheWritePerMillion,
+  };
+  for (const [field, price] of Object.entries(prices)) {
+    if (price?.scale === undefined) continue;
+    const finest = exponentOf(price.currency) + PRICE_SCALE_DIGITS;
+    assert(
+      Number.isInteger(price.scale) && price.scale >= 0 && price.scale <= finest,
+      `${SUBJECT} row "${spec.id}" states ${field} at scale ${renderCauseValue(price.scale)}; a price in ${renderCauseValue(price.currency)} may be stated at scale 0 to ${finest}`,
+      `registerModel({ ...spec, ${field}: { minor, currency: ${renderFixLiteral(price.currency, '<currency>')}, scale: ${finest} } })   # at most ${PRICE_SCALE_DIGITS} digits finer than the minor unit`,
+    );
+  }
+}
+
 /**
  * Every price on a row in ONE currency. `costOf` sums them into a single `Money` stamped with the
  * input price's currency, so a cache rate in another one would be added as if it were the same.

package/src/openai-body.ts

@@ -9,7 +9,9 @@ import { AiRequestInvalidError } from './errors';
 import type { Effort, ModelId, ThinkingMode } from './models';
 import { modelSpec } from './models';
 import { toOpenAiMessages, toOpenAiTools, toolChoiceFor } from './openai-messages';
+import { OPENAI_CACHE_KEY_MAX, promptCacheRule, SESSION_KEY_MAX } from './prompt-cache';
 import type { GenerateRequest } from './provider';
+import { samplingBody } from './sampling';
 
 export interface ChatCompletionBodyInput {
   readonly request: GenerateRequest;
@@ -18,12 +20,14 @@ export interface ChatCompletionBodyInput {
   /** The provider's `name`, so a content refusal names the endpoint that cannot take the block. */
   readonly provider: string;
   readonly stream: boolean;
+  /** The endpoint, for `PROMPT_CACHE_RULES` — which host this is decides what a cache hint sends. */
+  readonly baseUrl?: string | undefined;
 }
 
 /**
  * The body. What is deliberately absent is as load-bearing as what is present:
- *   - no `temperature` / `top_p` / `presence_penalty`. Steering is the prompt's job, and a sampling
- *     knob on a reasoning model in this family is a 400.
+ *   - no `top_p` / `presence_penalty`, and `temperature` / `seed` only for a model whose row takes
+ *     them (`ModelSpec.sampling`): a sampling knob on a reasoning model in this family is a 400.
  *   - no `response_format`. The output schema is projected into the `respond` tool by `llm()`, and
  *     that projection is the framework's ONE structured-output path — see the README.
  *   - `max_completion_tokens`, never the deprecated `max_tokens`, which current reasoning models
@@ -33,20 +37,30 @@ export function chatCompletionBody(input: ChatCompletionBodyInput): Record<strin
   const { request, model, stream, provider } = input;
   const spec = modelSpec(model);
   assertMediaContent(request.messages, { provider, model });
+  const hint = request.promptCache;
+  const rule = promptCacheRule('openai', input.baseUrl, model);
+  const marked = hint !== undefined && rule.marks === 'parts';
   const body: Record<string, unknown> = {
     model,
-    messages: toOpenAiMessages(request.system, request.messages, { provider, model }),
+    messages: toOpenAiMessages(request.system, request.messages, { provider, model }, marked),
     max_completion_tokens: Math.min(request.maxTokens, spec.maxOutput),
     ...reasoningFields(model, request.effort, request.thinking),
+    ...samplingBody('openai', model, request),
   };
   if (request.tools !== undefined && request.tools.length > 0) {
     body['tools'] = toOpenAiTools(request.tools);
-    const choice = toolChoiceFor(request.tools);
+    // Forced unless the row says this model rejects a forced choice (`forcedToolChoice: false`).
+    const choice = spec.forcedToolChoice === false ? undefined : toolChoiceFor(request.tools);
     if (choice !== undefined) body['tool_choice'] = choice;
   }
   if (request.stopSequences !== undefined && request.stopSequences.length > 0) {
     body['stop'] = request.stopSequences;
   }
+  // OpenAI's own grouping key. OpenRouter's session rides a header instead (`sessionHeaders`), so
+  // the body never carries two spellings of one key.
+  if (hint?.session !== undefined && rule.session === 'prompt_cache_key') {
+    body['prompt_cache_key'] = cacheKey(hint.session);
+  }
   if (stream) {
     body['stream'] = true;
     // Without this the final chunk carries no `usage` and the budget reconciles against nothing —
@@ -57,6 +71,44 @@ export function chatCompletionBody(input: ChatCompletionBodyInput): Record<strin
   return body;
 }
 
+/** The headers a cache hint adds for this endpoint and model: OpenRouter's stickiness key, or none. */
+export function sessionHeaders(
+  request: GenerateRequest,
+  model: ModelId,
+  baseUrl: string | undefined,
+): Readonly<Record<string, string>> {
+  const session = request.promptCache?.session;
+  if (session === undefined) return {};
+  if (promptCacheRule('openai', baseUrl, model).session !== 'x-session-id') return {};
+  return { 'x-session-id': sessionKey(session) };
+}
+
+/**
+ * OpenAI's grouping key. Never cut — a truncated key is a different key two tenants can share —
+ * and never refused for length: over `OPENAI_CACHE_KEY_MAX` it is sent as a hash of itself, which
+ * groups the same requests. `h:` plus 62 hex characters of its SHA-256.
+ */
+function cacheKey(session: string): string {
+  if (session === '') return sessionKey(session);
+  if (session.length <= OPENAI_CACHE_KEY_MAX) return session;
+  const digest = new Bun.CryptoHasher('sha256').update(session).digest('hex');
+  return `h:${digest.slice(0, OPENAI_CACHE_KEY_MAX - 2)}`;
+}
+
+/**
+ * OpenRouter's header. Refused, never cut: a truncated key is a DIFFERENT key that two tenants can
+ * share, and a control character in a header value is a request the runtime refuses with no code.
+ */
+function sessionKey(session: string): string {
+  if (session !== '' && session.length <= SESSION_KEY_MAX && !/[^\x20-\x7e]/.test(session)) {
+    return session;
+  }
+  throw new AiRequestInvalidError({
+    detail: `the prompt-cache session key is ${session.length} characters; it must be 1 to ${SESSION_KEY_MAX} printable ASCII characters to travel as a header`,
+    fix: `promptCache: { session: 'kb-agent' }   # at most ${SESSION_KEY_MAX} printable ASCII characters (the x-session-id header's limit): a short, stable, non-secret name`,
+  });
+}
+
 /**
  * The reasoning half, shaped for one model. Refused LOCALLY when the model's spec says the endpoint
  * has no such control, for the reason models.ts states: a round trip to learn a rule the registry

package/src/openai-messages.ts

@@ -8,6 +8,8 @@
 import type { AiMediaBlock } from './content-blocks';
 import { isMediaBlock } from './content-blocks';
 import { AiContentUnsupportedError } from './content-errors';
+import type { SystemPrompt } from './prompt-cache';
+import { CACHE_MARK, systemParts, systemText } from './prompt-cache';
 import type { AiContentBlock, AiMessage } from './provider';
 import type { JsonSchema, LlmTool } from './tools';
 
@@ -20,7 +22,12 @@ export interface OpenAiToolCall {
 
 /** A user message's parts, when it carries more than prose. Field names are the format's own. */
 export type OpenAiContentPart =
-  | { readonly type: 'text'; readonly text: string }
+  | {
+      readonly type: 'text';
+      readonly text: string;
+      /** A prompt-cache breakpoint — sent ONLY where `PROMPT_CACHE_RULES` says `parts`. */
+      readonly cache_control?: typeof CACHE_MARK;
+    }
   | { readonly type: 'image_url'; readonly image_url: { readonly url: string } }
   | {
       readonly type: 'file';
@@ -28,14 +35,18 @@ export type OpenAiContentPart =
     };
 
 export type OpenAiMessage =
-  | { readonly role: 'system'; readonly content: string }
+  | { readonly role: 'system'; readonly content: string | readonly OpenAiContentPart[] }
   | { readonly role: 'user'; readonly content: string | readonly OpenAiContentPart[] }
   | {
       readonly role: 'assistant';
       readonly content?: string | undefined;
       readonly tool_calls?: readonly OpenAiToolCall[] | undefined;
     }
-  | { readonly role: 'tool'; readonly tool_call_id: string; readonly content: string };
+  | {
+      readonly role: 'tool';
+      readonly tool_call_id: string;
+      readonly content: string | readonly OpenAiContentPart[];
+    };
 
 export interface OpenAiFunctionTool {
   readonly type: 'function';
@@ -68,12 +79,17 @@ export interface OpenAiToolChoice {
  * speaking this format — vLLM, Ollama, LiteLLM, Together — knows only `system`. OpenAI accepts it.
  */
 export function toOpenAiMessages(
-  system: string | undefined,
+  system: SystemPrompt | undefined,
   messages: readonly AiMessage[],
   target: MediaTarget,
+  /** Whether this endpoint and model take explicit `cache_control` parts (`./prompt-cache`). */
+  marked = false,
 ): readonly OpenAiMessage[] {
   const out: OpenAiMessage[] = [];
-  if (system !== undefined && system !== '') out.push({ role: 'system', content: system });
+  // One string, stable half first, wherever nothing is marked: every server in the family takes
+  // it, and an implicit cache is a prefix match.
+  const text = systemText(system);
+  if (text !== '') out.push({ role: 'system', content: marked ? systemPartsOf(system) : text });
   messages.forEach((message, m) => {
     if (typeof message.content === 'string') {
       out.push({ role: message.role, content: message.content });
@@ -84,9 +100,45 @@ export function toOpenAiMessages(
       out.push(...userTurn(message.content, m, target));
     }
   });
+  if (marked) markTail(out);
   return out;
 }
 
+/** The system message as parts: the stable one marked, the per-run one after it and unmarked. */
+function systemPartsOf(system: SystemPrompt | undefined): readonly OpenAiContentPart[] {
+  const { stable, run } = systemParts(system);
+  const parts: OpenAiContentPart[] = [];
+  // A whitespace-only part is sent as it stands and never marked.
+  if (stable !== '') {
+    parts.push({
+      type: 'text',
+      text: stable,
+      ...(stable.trim() === '' ? {} : { cache_control: CACHE_MARK }),
+    });
+  }
+  if (run !== '') parts.push({ type: 'text', text: run });
+  return parts;
+}
+
+/**
+ * Mark the LAST message's last part — the breakpoint the next turn reads back. A `user` or `tool`
+ * message only: those are the roles whose content this wire takes as parts, and a transcript sent
+ * to a model always ends on one of them.
+ */
+function markTail(out: OpenAiMessage[]): void {
+  const tail = out.at(-1);
+  if (tail === undefined || (tail.role !== 'user' && tail.role !== 'tool')) return;
+  const parts: readonly OpenAiContentPart[] =
+    typeof tail.content === 'string' ? [{ type: 'text', text: tail.content }] : tail.content;
+  const last = parts.at(-1);
+  // Only a text part carries the mark here, and never an empty or whitespace-only one.
+  if (last === undefined || last.type !== 'text' || last.text.trim() === '') return;
+  out[out.length - 1] = {
+    ...tail,
+    content: [...parts.slice(0, -1), { ...last, cache_control: CACHE_MARK }],
+  };
+}
+
 /** Who a media refusal names: this provider, and the model the request is for. */
 export interface MediaTarget {
   readonly provider: string;