@cjhyy/code-shell-core 0.9.2 → 0.9.4

package/dist/llm/prompt-cache.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Provider-neutral prompt-cache planning.
+ *
+ * This module owns the semantic cache policy (session affinity and reusable
+ * prefix boundaries). Provider clients only translate the plan to their wire
+ * format: OpenAI `prompt_cache_*` fields or Anthropic `cache_control` blocks.
+ */
+export type PromptCacheStrategy = "openai-explicit" | "openai-implicit" | "anthropic-explicit" | "provider-managed";
+export type PromptCacheBreakpoint = "system" | "tools" | "stable-history" | "rolling-history";
+export interface PromptCacheRequestContext {
+    /** Stable run/session namespace used for provider cache affinity. */
+    scopeId?: string;
+    /**
+     * Number of source messages before the first volatile context message.
+     * Providers use this to retain a reusable durable-history breakpoint while
+     * also advancing a rolling breakpoint over the append-only in-run tail.
+     */
+    stablePrefixMessageCount?: number;
+}
+export interface PromptCachePolicy {
+    strategy: PromptCacheStrategy;
+    layoutVersion: string;
+    breakpoints: readonly PromptCacheBreakpoint[];
+    /** Opaque and <=64 chars, as required by OpenAI's prompt_cache_key. */
+    cacheKey?: string;
+    /** GPT-5.6+ explicit-cache request mode. */
+    promptCacheOptions?: {
+        mode: "explicit";
+        ttl: "30m";
+    };
+}
+export interface ResolvePromptCachePolicyInput {
+    provider: string;
+    providerKind?: string;
+    model: string;
+    request?: PromptCacheRequestContext;
+    /** Sticky compatibility fallback after an endpoint rejects explicit fields. */
+    explicitDisabled?: boolean;
+}
+/**
+ * Produce a privacy-preserving stable affinity key without leaking a raw
+ * session id to the provider. The prefix plus 48 hex chars is 51 characters.
+ */
+export declare function createPromptCacheKey(scopeId: string, namespace: string): string;
+/** Resolve one cache policy from the actual provider route and model family. */
+export declare function resolvePromptCachePolicy(input: ResolvePromptCachePolicyInput): PromptCachePolicy;
+/** Deduplicate semantic boundaries while preserving their left-to-right order. */
+export declare function uniquePromptCacheBreakpointIndexes(indexes: readonly (number | undefined)[]): number[];

package/dist/llm/prompt-cache.js

@@ -0,0 +1,100 @@
+/**
+ * Provider-neutral prompt-cache planning.
+ *
+ * This module owns the semantic cache policy (session affinity and reusable
+ * prefix boundaries). Provider clients only translate the plan to their wire
+ * format: OpenAI `prompt_cache_*` fields or Anthropic `cache_control` blocks.
+ */
+import { createHash } from "node:crypto";
+const OPENAI_EXPLICIT_BREAKPOINTS = ["system", "stable-history", "rolling-history"];
+const ANTHROPIC_BREAKPOINTS = ["system", "tools", "stable-history", "rolling-history"];
+const OPENROUTER_ANTHROPIC_BREAKPOINTS = ["system", "stable-history", "rolling-history"];
+function normalizedModel(model) {
+    return model.replace(/^~/, "");
+}
+function isAnthropicModel(model) {
+    return /^anthropic\/claude-/i.test(normalizedModel(model));
+}
+function isOpenAIModel(model) {
+    const normalized = normalizedModel(model);
+    return /^openai\//i.test(normalized) || /^(?:gpt-|o\d)/i.test(normalized);
+}
+/** GPT-5.6 and later 5.x releases support explicit prompt-cache breakpoints. */
+function supportsOpenAIExplicitCaching(model) {
+    const normalized = normalizedModel(model).replace(/^openai\//i, "");
+    const match = /^gpt-5\.(\d+)(?:[-.]|$)/i.exec(normalized);
+    return match !== null && Number(match[1]) >= 6;
+}
+/**
+ * Produce a privacy-preserving stable affinity key without leaking a raw
+ * session id to the provider. The prefix plus 48 hex chars is 51 characters.
+ */
+export function createPromptCacheKey(scopeId, namespace) {
+    const digest = createHash("sha256")
+        .update("codeshell-prompt-cache-v1\0")
+        .update(namespace)
+        .update("\0")
+        .update(scopeId)
+        .digest("hex")
+        .slice(0, 48);
+    return `cs:${digest}`;
+}
+/** Resolve one cache policy from the actual provider route and model family. */
+export function resolvePromptCachePolicy(input) {
+    const kind = (input.providerKind ?? input.provider).toLowerCase();
+    const model = normalizedModel(input.model);
+    const key = input.request?.scopeId
+        ? createPromptCacheKey(input.request.scopeId, `${kind}:${model}`)
+        : undefined;
+    if (input.provider === "anthropic" || kind === "anthropic") {
+        return {
+            strategy: "anthropic-explicit",
+            layoutVersion: "system-tools-stable-rolling-v2",
+            breakpoints: ANTHROPIC_BREAKPOINTS,
+        };
+    }
+    if (kind === "openrouter" && isAnthropicModel(model)) {
+        return {
+            strategy: "anthropic-explicit",
+            // OpenRouter/Anthropic includes tools in the system-prefix cache entry,
+            // so a separate tool marker is unnecessary and preserves one slot.
+            layoutVersion: "system-stable-rolling-v2",
+            breakpoints: OPENROUTER_ANTHROPIC_BREAKPOINTS,
+        };
+    }
+    const openAIRoute = kind === "openai" || (kind === "openrouter" && isOpenAIModel(model));
+    if (openAIRoute && supportsOpenAIExplicitCaching(model) && input.explicitDisabled !== true) {
+        return {
+            strategy: "openai-explicit",
+            layoutVersion: "system-stable-rolling-v1",
+            breakpoints: OPENAI_EXPLICIT_BREAKPOINTS,
+            ...(key ? { cacheKey: key } : {}),
+            promptCacheOptions: { mode: "explicit", ttl: "30m" },
+        };
+    }
+    if (openAIRoute) {
+        return {
+            strategy: "openai-implicit",
+            layoutVersion: "implicit-affinity-v1",
+            breakpoints: [],
+            ...(key ? { cacheKey: key } : {}),
+        };
+    }
+    return {
+        strategy: "provider-managed",
+        layoutVersion: "append-only-v1",
+        breakpoints: [],
+    };
+}
+/** Deduplicate semantic boundaries while preserving their left-to-right order. */
+export function uniquePromptCacheBreakpointIndexes(indexes) {
+    const seen = new Set();
+    const result = [];
+    for (const index of indexes) {
+        if (index === undefined || index < 0 || seen.has(index))
+            continue;
+        seen.add(index);
+        result.push(index);
+    }
+    return result;
+}

package/dist/llm/providers/anthropic.d.ts

@@ -16,6 +16,7 @@ export declare class AnthropicClient extends LLMClientBase {
      */
     private _capability;
     private get capability();
+    private promptCachePolicy;
     getPromptCacheConfigIdentity(): Readonly<Record<string, unknown>>;
     /**
      * Translate the resolved ReasoningSetting into Anthropic's `thinking` field,
@@ -44,7 +45,9 @@ export declare class AnthropicClient extends LLMClientBase {
     private nonStreamMessage;
     private streamMessage;
     private processResponse;
+    private buildSystem;
     private buildMessages;
+    private markAnthropicCacheBreakpoint;
     private convertTools;
     private handleApiError;
 }

package/dist/llm/providers/anthropic.js

@@ -9,6 +9,7 @@ import { countTokens } from "../token-counter.js";
 import { capabilitiesFor } from "../capabilities/index.js";
 import { resolveApiKey, resolveHeaders } from "../provider-auth.js";
 import { stripVisionFromHistory } from "../strip-vision.js";
+import { resolvePromptCachePolicy, uniquePromptCacheBreakpointIndexes, } from "../prompt-cache.js";
 /**
  * Anthropic's `max_tokens` is required, so unlike OpenAI we can't omit it when
  * the model's ceiling is unknown. Use a conservative floor in that rare case
@@ -75,12 +76,22 @@ export class AnthropicClient extends LLMClientBase {
         }
         return this._capability;
     }
+    promptCachePolicy(request) {
+        return resolvePromptCachePolicy({
+            provider: this.provider,
+            providerKind: this.config.providerKind,
+            model: this.model,
+            request,
+        });
+    }
     getPromptCacheConfigIdentity() {
+        const cachePolicy = this.promptCachePolicy();
         return {
             ...super.getPromptCacheConfigIdentity(),
-            cacheStrategy: "anthropic-explicit",
-            cacheLayoutVersion: "system-tools-history-v1",
-            breakpointCount: 3,
+            cacheStrategy: cachePolicy.strategy,
+            cacheLayoutVersion: cachePolicy.layoutVersion,
+            cacheBreakpoints: cachePolicy.breakpoints,
+            breakpointCount: cachePolicy.breakpoints.length,
             reasoningShape: this.capability.reasoning,
         };
     }
@@ -133,8 +144,9 @@ export class AnthropicClient extends LLMClientBase {
     }
     async createMessage(options) {
         return this.withRetry(async (requestSignal) => {
-            const messages = this.buildMessages(options.messages);
-            const tools = options.tools ? this.convertTools(options.tools) : undefined;
+            const cachePolicy = this.promptCachePolicy(options.promptCache);
+            const messages = this.buildMessages(options.messages, options.promptCache, cachePolicy);
+            const tools = options.tools ? this.convertTools(options.tools, cachePolicy) : undefined;
             // One span per outbound LLM request. Begin emits debug-level so the
             // info log isn't spammed during normal operation; end emits info with
             // duration_ms + usage so `--debug=llm` already gives a quick latency
@@ -146,6 +158,7 @@ export class AnthropicClient extends LLMClientBase {
                 stream: !!(options.stream && options.onChunk),
                 messageCount: messages.length,
                 toolCount: tools?.length ?? 0,
+                cacheStrategy: cachePolicy.strategy,
             });
             try {
                 const response = options.stream && options.onChunk
@@ -176,13 +189,7 @@ export class AnthropicClient extends LLMClientBase {
             const response = await this.client.messages.create({
                 model: this.model,
                 max_tokens: maxTokens,
-                system: [
-                    {
-                        type: "text",
-                        text: options.systemPrompt,
-                        cache_control: { type: "ephemeral" },
-                    },
-                ],
+                system: this.buildSystem(options.systemPrompt, this.promptCachePolicy(options.promptCache)),
                 messages,
                 ...(tools?.length ? { tools } : {}),
                 ...(thinking ? { thinking } : {}),
@@ -207,13 +214,7 @@ export class AnthropicClient extends LLMClientBase {
             const stream = this.client.messages.stream({
                 model: this.model,
                 max_tokens: maxTokens,
-                system: [
-                    {
-                        type: "text",
-                        text: options.systemPrompt,
-                        cache_control: { type: "ephemeral" },
-                    },
-                ],
+                system: this.buildSystem(options.systemPrompt, this.promptCachePolicy(options.promptCache)),
                 messages,
                 ...(tools?.length ? { tools } : {}),
                 ...(thinking ? { thinking } : {}),
@@ -316,10 +317,27 @@ export class AnthropicClient extends LLMClientBase {
             stopReason: response.stop_reason ?? undefined,
         };
     }
-    buildMessages(messages) {
+    buildSystem(systemPrompt, policy) {
+        return [
+            {
+                type: "text",
+                text: systemPrompt,
+                ...(policy.breakpoints.includes("system")
+                    ? { cache_control: { type: "ephemeral" } }
+                    : {}),
+            },
+        ];
+    }
+    buildMessages(messages, promptCache, policy) {
         messages = stripVisionFromHistory(messages, this.capability.supportsVision);
         const result = [];
-        for (const msg of messages) {
+        const stablePrefixMessageCount = Math.max(0, Math.min(messages.length, promptCache?.stablePrefixMessageCount ?? messages.length));
+        let stablePrefixEndMessage;
+        for (let sourceIndex = 0; sourceIndex < messages.length; sourceIndex++) {
+            if (sourceIndex === stablePrefixMessageCount) {
+                stablePrefixEndMessage = result[result.length - 1];
+            }
+            const msg = messages[sourceIndex];
             if (msg.role === "system")
                 continue;
             const role = msg.role === "tool" ? "user" : msg.role;
@@ -390,40 +408,45 @@ export class AnthropicClient extends LLMClientBase {
                 }
             }
         }
-        // Prompt-cache breakpoint on the history: mark the LAST content block of
-        // the LAST message. The API caches everything up to the marker, so the
-        // stable prefix (all prior turns) is reused; only the growing tail is
-        // re-billed. CC does exactly this (one marker at messages.length - 1) and
-        // warns a second history marker causes KV page eviction. We only ANNOTATE
-        // the tail block — never reorder — so the tool_use/tool_result adjacency
-        // invariant is untouched. A string-content message is lifted to a single
-        // text block so it can carry cache_control. See
-        // docs/todo/prompt-cache-optimization.md.
-        const lastMsg = result[result.length - 1];
-        if (lastMsg) {
-            if (typeof lastMsg.content === "string") {
-                lastMsg.content = [
-                    {
-                        type: "text",
-                        text: lastMsg.content,
-                        cache_control: { type: "ephemeral" },
-                    },
-                ];
-            }
-            else {
-                const lastBlock = lastMsg.content[lastMsg.content.length - 1];
-                // Skip thinking/redacted_thinking blocks: they reject cache_control and
-                // marking them would 400. buildMessages never emits them today (thinking
-                // is a top-level request field, not history content), but guard anyway
-                // so a future block type can't silently break the request.
-                if (lastBlock && lastBlock.type !== "thinking" && lastBlock.type !== "redacted_thinking") {
-                    lastBlock.cache_control = { type: "ephemeral" };
-                }
-            }
+        stablePrefixEndMessage ??= result[result.length - 1];
+        const stablePrefixEndIndex = stablePrefixEndMessage
+            ? result.indexOf(stablePrefixEndMessage)
+            : undefined;
+        const breakpointIndexes = uniquePromptCacheBreakpointIndexes([
+            policy.breakpoints.includes("stable-history") && stablePrefixEndIndex !== undefined
+                ? stablePrefixEndIndex
+                : undefined,
+            policy.breakpoints.includes("rolling-history") ? result.length - 1 : undefined,
+        ]);
+        for (const index of breakpointIndexes) {
+            this.markAnthropicCacheBreakpoint(result[index]);
         }
         return result;
     }
-    convertTools(tools) {
+    markAnthropicCacheBreakpoint(message) {
+        if (!message)
+            return;
+        if (typeof message.content === "string") {
+            message.content = [
+                {
+                    type: "text",
+                    text: message.content,
+                    cache_control: { type: "ephemeral" },
+                },
+            ];
+            return;
+        }
+        for (let index = message.content.length - 1; index >= 0; index--) {
+            const block = message.content[index];
+            // Thinking blocks reject cache_control. Walk backward so a preceding
+            // cacheable text/tool block can still anchor the prefix.
+            if (block.type === "thinking" || block.type === "redacted_thinking")
+                continue;
+            block.cache_control = { type: "ephemeral" };
+            return;
+        }
+    }
+    convertTools(tools, policy) {
         const converted = tools.map((t) => ({
             name: t.name,
             description: t.description,
@@ -436,8 +459,9 @@ export class AnthropicClient extends LLMClientBase {
         // not per-tool); a second marker here would waste a scarce cache_control
         // slot (max 4) and risk KV eviction. See docs/todo/prompt-cache-optimization.md.
         const last = converted[converted.length - 1];
-        if (last)
+        if (last && policy.breakpoints.includes("tools")) {
             last.cache_control = { type: "ephemeral" };
+        }
         return converted;
     }
     handleApiError(err) {

package/dist/llm/providers/openai.d.ts

@@ -52,6 +52,9 @@ export declare class OpenAIClient extends LLMClientBase {
     private readonly dangerouslyAllowBrowser;
     private _forceMaxCompletionTokens;
     private _dropReasoningEffort;
+    /** Compatibility fallbacks for OpenAI-compatible gateways that lag the API. */
+    private _disableExplicitPromptCache;
+    private _disablePromptCacheKey;
     constructor(config: LLMConfig, defaults?: ClientDefaults, runtimeOptions?: {
         dangerouslyAllowBrowser?: boolean;
     });
@@ -66,20 +69,7 @@ export declare class OpenAIClient extends LLMClientBase {
      */
     private _capability;
     private get capability();
-    /**
-     * True when this client routes an Anthropic-family model through OpenRouter's
-     * OpenAI-compatible endpoint. Anthropic caching is EXPLICIT — nothing is
-     * cached unless the request carries `cache_control` breakpoints (verified live
-     * 2026-07-02: plain requests to anthropic/claude-opus-4.7-fast via OpenRouter
-     * report cached_tokens 0 on every repeat; a single system-block breakpoint
-     * turns the whole stable prefix — tools + system — into a cache hit, ~89%
-     * cheaper on the follow-up). OpenAI and other OpenRouter models cache
-     * automatically, so they must NOT get breakpoints. The slug arrives resolved
-     * (e.g. "anthropic/claude-opus-4.7-fast") or as the router alias
-     * ("~anthropic/claude-opus-latest") — both start with an optional "~" then
-     * "anthropic/".
-     */
-    private get isOpenRouterAnthropic();
+    private promptCachePolicy;
     getPromptCacheConfigIdentity(): Readonly<Record<string, unknown>>;
     createMessage(options: CreateMessageOptions): Promise<LLMResponse>;
     /**
@@ -93,20 +83,10 @@ export declare class OpenAIClient extends LLMClientBase {
     private processChoice;
     private buildMessages;
     /**
-     * In-place: add prompt-cache breakpoints for Anthropic-over-OpenRouter.
-     * Mirrors the native anthropic provider (≤4 breakpoints):
-     *   1. System block  — the stable prefix. Anthropic sees tools BEFORE the
-     *      system prompt, so one marker on the system block caches tools too
-     *      (verified live: system-only marker cached 3511/3952 prompt tokens
-     *      including tool defs).
-     *   2. Last message  — one rolling breakpoint so the growing conversation
-     *      history becomes a cached prefix. Not scrolled: as history grows the
-     *      "last message" naturally advances and its tail is the next write.
-     * A string `content` is lifted to a single-element `[{type:"text",...}]`
-     * array so it can carry `cache_control`; OpenRouter accepts this OpenAI
-     * multimodal wire form for text.
+     * Translate semantic prefix boundaries to the active wire format. Both
+     * formats annotate content blocks without reordering messages.
      */
-    private applyAnthropicCacheBreakpoints;
+    private applyPromptCacheBreakpoints;
     private convertTools;
     private handleApiError;
 }