@jaypie/llm 1.3.14 → 1.3.15

package/dist/esm/index.js

@@ -498,6 +498,12 @@ function sumUsageByProviderModel(usage) {
         totals[key].output += item.output;
         totals[key].reasoning += item.reasoning;
         totals[key].total += item.total;
+        if (item.cacheRead !== undefined) {
+            totals[key].cacheRead = (totals[key].cacheRead ?? 0) + item.cacheRead;
+        }
+        if (item.cacheWrite !== undefined) {
+            totals[key].cacheWrite = (totals[key].cacheWrite ?? 0) + item.cacheWrite;
+        }
     }
     return totals;
 }
@@ -523,6 +529,7 @@ function buildExchangeEnvelope({ duration, initialHistoryLength, input, options,
     return {
         ids: extractResponseIds(response.responses),
         request: {
+            cache: options.cache,
             data: options.data,
             effort: options.effort,
             explain: options.explain,
@@ -560,6 +567,38 @@ function buildExchangeEnvelope({ duration, initialHistoryLength, input, options,
     };
 }
 
+const CACHE_TTL_DEFAULT = "5m";
+/**
+ * Normalize the caller-facing `cache` option into a concrete decision.
+ * - `undefined` / `true` → enabled at the default TTL
+ * - `false` / `0` → disabled
+ * - `"5m"` / `"1h"` → enabled at that TTL
+ */
+function resolveCache(cache) {
+    if (cache === false || cache === 0) {
+        return { enabled: false, ttl: CACHE_TTL_DEFAULT };
+    }
+    if (cache === "5m" || cache === "1h") {
+        return { enabled: true, ttl: cache };
+    }
+    // undefined or true
+    return { enabled: true, ttl: CACHE_TTL_DEFAULT };
+}
+/**
+ * Deterministic short key for providers with automatic, prefix-based caching
+ * (e.g. OpenAI `prompt_cache_key`). Derived from the stable prefix so the same
+ * system prompt + tools + model always routes to the same cache. Dependency-
+ * free FNV-1a; not cryptographic.
+ */
+function promptCacheKey(seed) {
+    let hash = 0x811c9dc5;
+    for (let i = 0; i < seed.length; i += 1) {
+        hash ^= seed.charCodeAt(i);
+        hash = Math.imul(hash, 0x01000193);
+    }
+    return `jaypie-${(hash >>> 0).toString(16)}`;
+}
+
 /**
  * Type guard to check if an item is a dedicated reasoning item (OpenAI)
  */
@@ -1411,6 +1450,14 @@ function tallyOperate({ toolCallNames = [], turns, usage = [], }) {
             usageByModel[key].output += item.output;
             usageByModel[key].reasoning += item.reasoning;
             usageByModel[key].total += item.total;
+            if (item.cacheRead !== undefined) {
+                usageByModel[key].cacheRead =
+                    (usageByModel[key].cacheRead ?? 0) + item.cacheRead;
+            }
+            if (item.cacheWrite !== undefined) {
+                usageByModel[key].cacheWrite =
+                    (usageByModel[key].cacheWrite ?? 0) + item.cacheWrite;
+            }
         }
         llm.usage = usageByModel;
     }
@@ -1517,27 +1564,26 @@ const MODULE$1 = {
 //
 // Helpers
 //
-// CJS/ESM compatible require - handles bundling to CJS where import.meta.url
-// becomes undefined (mirrors observability/llmobs.ts).
-// eslint-disable-next-line @typescript-eslint/ban-ts-comment
-// @ts-ignore - __filename exists in CJS context when ESM is bundled to CJS
-const requireModule$1 = typeof __filename !== "undefined"
-    ? createRequire(pathToFileURL(__filename).href)
-    : createRequire(import.meta.url);
+// Native dynamic import that neither rollup nor tsc rewrites to require(), so
+// a CJS-bundled build still loads @jaypie/dynamodb's ESM entry and shares the
+// host's initialized module instance. A require()-based resolution would load
+// the CJS build, whose module-level client state is separate from the ESM
+// instance an ESM host initializes (dual-package hazard, issue #429).
+const dynamicImport = new Function("s", "return import(s)");
 let resolved$1 = false;
 let cachedSdk$2 = null;
 /**
  * Lazily resolve @jaypie/dynamodb's storeExchange. Returns null (and never
  * throws) when the peer is absent. Cached after the first attempt.
  */
-function resolveExchangeStore() {
+async function resolveExchangeStore() {
     if (resolved$1) {
         return cachedSdk$2;
     }
     resolved$1 = true;
     try {
-        const dynamodb = requireModule$1(MODULE$1.JAYPIE_DYNAMODB);
-        const sdk = dynamodb?.default ?? dynamodb;
+        const dynamodb = await dynamicImport(MODULE$1.JAYPIE_DYNAMODB);
+        const sdk = (dynamodb?.default ?? dynamodb);
         if (sdk && typeof sdk.storeExchange === "function") {
             cachedSdk$2 = sdk;
         }
@@ -1560,7 +1606,7 @@ async function persistExchange(envelope) {
     if (!isExchangeStoreEnabled()) {
         return;
     }
-    const sdk = resolveExchangeStore();
+    const sdk = (await resolveExchangeStore());
     if (!sdk) {
         return;
     }
@@ -2348,8 +2394,19 @@ class AnthropicAdapter extends BaseProviderAdapter {
                 PROVIDER.ANTHROPIC.MAX_TOKENS.DEFAULT,
             stream: false,
         };
+        const cache = resolveCache(request.cache);
+        const cacheControl = cache.enabled
+            ? {
+                type: "ephemeral",
+                ...(cache.ttl === "1h" ? { ttl: "1h" } : {}),
+            }
+            : undefined;
         if (request.system) {
-            anthropicRequest.system = request.system;
+            // A cache breakpoint on the system block caches tools+system together
+            // (render order is tools -> system -> messages).
+            anthropicRequest.system = cacheControl
+                ? [{ type: "text", text: request.system, cache_control: cacheControl }]
+                : request.system;
         }
         const useFallbackStructuredOutput = Boolean(request.format) &&
             !this.supportsStructuredOutput(anthropicRequest.model);
@@ -2366,7 +2423,7 @@ class AnthropicAdapter extends BaseProviderAdapter {
             });
         }
         if (allTools.length > 0) {
-            anthropicRequest.tools = allTools.map((tool) => ({
+            anthropicRequest.tools = allTools.map((tool, index) => ({
                 name: tool.name,
                 description: tool.description,
                 input_schema: {
@@ -2374,6 +2431,11 @@ class AnthropicAdapter extends BaseProviderAdapter {
                     type: "object",
                 },
                 type: "custom",
+                // Breakpoint on the last tool caches the tool list (which renders
+                // before system) when there is no system prompt to anchor it.
+                ...(cacheControl && index === allTools.length - 1
+                    ? { cache_control: cacheControl }
+                    : {}),
             }));
             anthropicRequest.tool_choice = useFallbackStructuredOutput
                 ? { type: "any" }
@@ -2676,6 +2738,12 @@ class AnthropicAdapter extends BaseProviderAdapter {
             output: usage.output_tokens,
             reasoning: usage.thinking_tokens || 0,
             total: usage.input_tokens + usage.output_tokens,
+            ...(usage.cache_read_input_tokens !== undefined
+                ? { cacheRead: usage.cache_read_input_tokens }
+                : {}),
+            ...(usage.cache_creation_input_tokens !== undefined
+                ? { cacheWrite: usage.cache_creation_input_tokens }
+                : {}),
             provider: this.name,
             model,
         };
@@ -2928,6 +2996,14 @@ function isTemperatureDeprecationError$3(error) {
     const msg = error?.message ?? "";
     return /temperature.*deprecated|deprecated.*temperature/i.test(msg);
 }
+function isCachePointUnsupportedError(error) {
+    const name = error?.constructor?.name ?? "";
+    const msg = error?.message ?? "";
+    // A model that cannot cache rejects the cachePoint block with a
+    // ValidationException naming caching / cachePoint.
+    return (/ValidationException|invalid|not support/i.test(name + " " + msg) &&
+        /cachePoint|cache_point|prompt caching|caching/i.test(msg));
+}
 function extractJson(text) {
     // Try direct parse first
     try {
@@ -2963,6 +3039,13 @@ class BedrockAdapter extends BaseProviderAdapter {
         this.defaultModel = PROVIDER.BEDROCK.DEFAULT;
         this._modelsFallbackToStructuredOutputTool = new Set();
         this._modelsWithoutTemperature = new Set();
+        this._modelsWithoutCachePoint = new Set();
+    }
+    rememberModelRejectsCachePoint(model) {
+        this._modelsWithoutCachePoint.add(model);
+    }
+    supportsCachePoint(model) {
+        return !this._modelsWithoutCachePoint.has(model);
     }
     rememberModelRejectsOutputConfig(model) {
         this._modelsFallbackToStructuredOutputTool.add(model);
@@ -3099,11 +3182,51 @@ class BedrockAdapter extends BaseProviderAdapter {
                 };
             }
         }
+        // Prompt caching via Converse cachePoint blocks (5-min default TTL; the
+        // `ttl` option does not apply). Gated per model — cachePoint 400s on
+        // unsupported models, which executeRequest catches and denylists.
+        if (resolveCache(request.cache).enabled && this.supportsCachePoint(model)) {
+            const cachePoint = { cachePoint: { type: "default" } };
+            if (bedrockRequest.system && bedrockRequest.system.length > 0) {
+                bedrockRequest.system = [
+                    ...bedrockRequest.system,
+                    cachePoint,
+                ];
+            }
+            if (bedrockRequest.toolConfig?.tools?.length) {
+                bedrockRequest.toolConfig = {
+                    ...bedrockRequest.toolConfig,
+                    tools: [
+                        ...bedrockRequest.toolConfig.tools,
+                        cachePoint,
+                    ],
+                };
+            }
+        }
         if (request.providerOptions) {
             Object.assign(bedrockRequest, request.providerOptions);
         }
         return bedrockRequest;
     }
+    /** Remove any cachePoint blocks so the request can be retried unsupported. */
+    stripCachePoints(request) {
+        const stripped = { ...request };
+        if (stripped.system) {
+            stripped.system = stripped.system.filter((block) => !("cachePoint" in block));
+        }
+        if (stripped.toolConfig?.tools) {
+            stripped.toolConfig = {
+                ...stripped.toolConfig,
+                tools: stripped.toolConfig.tools.filter((tool) => !("cachePoint" in tool)),
+            };
+        }
+        return stripped;
+    }
+    requestHasCachePoints(request) {
+        const inSystem = (request.system ?? []).some((block) => "cachePoint" in block);
+        const inTools = (request.toolConfig?.tools ?? []).some((tool) => "cachePoint" in tool);
+        return inSystem || inTools;
+    }
     formatTools(toolkit) {
         return toolkit.tools.map((tool) => ({
             name: tool.name,
@@ -3158,6 +3281,15 @@ class BedrockAdapter extends BaseProviderAdapter {
                 const fallbackRequest = this.toFallbackStructuredOutputRequest(bedrockRequest);
                 return (await bedrockClient.send(new ConverseCommand(fallbackRequest), signal ? { abortSignal: signal } : undefined));
             }
+            if (this.requestHasCachePoints(bedrockRequest) &&
+                isCachePointUnsupportedError(error)) {
+                this.rememberModelRejectsCachePoint(bedrockRequest.modelId || this.defaultModel);
+                const retryRequest = this.stripCachePoints(bedrockRequest);
+                const response = (await bedrockClient.send(new ConverseCommand(retryRequest), signal ? { abortSignal: signal } : undefined));
+                if (wantsStructuredOutput)
+                    response.__jaypieStructuredOutput = true;
+                return response;
+            }
             throw error;
         }
     }
@@ -3301,6 +3433,12 @@ class BedrockAdapter extends BaseProviderAdapter {
             reasoning: 0,
             total: usage?.totalTokens ??
                 (usage?.inputTokens ?? 0) + (usage?.outputTokens ?? 0),
+            ...(usage?.cacheReadInputTokens !== undefined
+                ? { cacheRead: usage.cacheReadInputTokens }
+                : {}),
+            ...(usage?.cacheWriteInputTokens !== undefined
+                ? { cacheWrite: usage.cacheWriteInputTokens }
+                : {}),
             provider: this.name,
             model,
         };
@@ -5472,6 +5610,13 @@ class OpenAiAdapter extends BaseProviderAdapter {
     supportsReasoningEffort(model) {
         return isReasoningModel(model);
     }
+    /**
+     * Whether to emit `prompt_cache_key` (OpenAI Responses API). Overridable by
+     * OpenAI-compatible subclasses whose backend rejects the field.
+     */
+    supportsPromptCacheKey() {
+        return true;
+    }
     /** Translate a normalized effort to this provider's `reasoning.effort` value. */
     mapReasoningEffort(effort, model) {
         return toOpenAiEffort(effort, { model });
@@ -5511,6 +5656,18 @@ class OpenAiAdapter extends BaseProviderAdapter {
                 summary: "auto",
             };
         }
+        // OpenAI prompt caching is automatic (prefix-based). Setting a stable
+        // prompt_cache_key routes repeat traffic to the same cache across
+        // instances. Keyed on the stable prefix only (system + instructions +
+        // tools + model), never the volatile user turn.
+        if (this.supportsPromptCacheKey() && resolveCache(request.cache).enabled) {
+            openaiRequest.prompt_cache_key = promptCacheKey(JSON.stringify([
+                model,
+                request.system ?? "",
+                request.instructions ?? "",
+                request.tools ?? [],
+            ]));
+        }
         if (request.providerOptions) {
             Object.assign(openaiRequest, request.providerOptions);
         }
@@ -5769,11 +5926,13 @@ class OpenAiAdapter extends BaseProviderAdapter {
                 model,
             };
         }
+        const cachedTokens = openaiResponse.usage.input_tokens_details?.cached_tokens;
         return {
             input: openaiResponse.usage.input_tokens || 0,
             output: openaiResponse.usage.output_tokens || 0,
             reasoning: openaiResponse.usage.output_tokens_details?.reasoning_tokens || 0,
             total: openaiResponse.usage.total_tokens || 0,
+            ...(cachedTokens !== undefined ? { cacheRead: cachedTokens } : {}),
             provider: this.name,
             model,
         };
@@ -6165,7 +6324,13 @@ class OpenRouterAdapter extends BaseProviderAdapter {
     //
     buildRequest(request) {
         // Convert messages to OpenRouter format (OpenAI-compatible)
-        const messages = this.convertMessagesToOpenRouter(request.messages, request.system);
+        const cache = resolveCache(request.cache);
+        const messages = this.convertMessagesToOpenRouter(request.messages, request.system, cache.enabled
+            ? {
+                type: "ephemeral",
+                ...(cache.ttl === "1h" ? { ttl: "1h" } : {}),
+            }
+            : undefined);
         // Append instructions to last message if provided
         if (request.instructions && messages.length > 0) {
             const lastMsg = messages[messages.length - 1];
@@ -6529,11 +6694,15 @@ class OpenRouterAdapter extends BaseProviderAdapter {
         }
         // SDK returns camelCase, but support snake_case as fallback
         const usage = openRouterResponse.usage;
+        const cachedTokens = usage.promptTokensDetails?.cachedTokens ??
+            usage.promptTokensDetails?.cached_tokens ??
+            usage.prompt_tokens_details?.cached_tokens;
         return {
             input: usage.promptTokens || usage.prompt_tokens || 0,
             output: usage.completionTokens || usage.completion_tokens || 0,
             reasoning: usage.completionTokensDetails?.reasoningTokens || 0,
             total: usage.totalTokens || usage.total_tokens || 0,
+            ...(cachedTokens !== undefined ? { cacheRead: cachedTokens } : {}),
             provider: this.name,
             model,
         };
@@ -6734,13 +6903,17 @@ class OpenRouterAdapter extends BaseProviderAdapter {
         const choice = response.choices[0];
         return choice?.message?.content ?? undefined;
     }
-    convertMessagesToOpenRouter(messages, system) {
+    convertMessagesToOpenRouter(messages, system, cacheControl) {
         const openRouterMessages = [];
-        // Add system message if provided
+        // Add system message if provided. A cache_control breakpoint on the system
+        // content is forwarded by OpenRouter to Anthropic/Gemini backends (and
+        // ignored by others), caching the stable system prefix.
         if (system) {
             openRouterMessages.push({
                 role: "system",
-                content: system,
+                content: cacheControl
+                    ? [{ type: "text", text: system, cache_control: cacheControl }]
+                    : system,
             });
         }
         for (const msg of messages) {
@@ -8452,6 +8625,7 @@ class OperateLoop {
     }
     buildInitialRequest(state, options) {
         return {
+            cache: options.cache,
             effort: options.effort,
             format: state.formattedFormat,
             instructions: options.instructions,
@@ -9085,6 +9259,7 @@ class StreamLoop {
     }
     buildInitialRequest(state, options) {
         return {
+            cache: options.cache,
             effort: options.effort,
             format: state.formattedFormat,
             instructions: options.instructions,