@doclo/providers-llm 0.1.10 → 0.1.11

package/dist/index.d.ts

@@ -32,6 +32,26 @@ interface ResourceLimits {
      */
     maxJsonDepth?: number;
 }
+/** Caching configuration for prompt caching */
+interface CachingConfig {
+    /**
+     * Enable/disable prompt caching.
+     * Default varies by provider:
+     * - Anthropic: false (cache writes cost 1.25x-2x, opt-in)
+     * - OpenAI/Google/XAI/DeepSeek: true (automatic, free)
+     */
+    enabled?: boolean;
+    /**
+     * Cache TTL for providers that support it (Anthropic only).
+     * - '5m': 5-minute TTL, cache writes cost 1.25x (default)
+     * - '1h': 1-hour TTL, cache writes cost 2x
+     *
+     * Break-even: ~1.4 reads/write (5m) or ~2.2 reads/write (1h).
+     * For high-frequency flows (100+ docs/hr with same schema), caching
+     * is almost always cost-effective despite the write cost.
+     */
+    ttl?: '5m' | '1h';
+}
 /** Provider configuration */
 interface ProviderConfig {
     provider: ProviderType;
@@ -40,6 +60,8 @@ interface ProviderConfig {
     apiKey: string;
     baseUrl?: string;
     limits?: ResourceLimits;
+    /** Optional caching configuration for prompt caching */
+    caching?: CachingConfig;
 }
 /** Fallback configuration */
 interface FallbackConfig {
@@ -79,8 +101,12 @@ interface ResponseMetrics {
     attemptNumber: number;
     provider: string;
     model: string;
+    /** Tokens written to cache (Anthropic only - costs 1.25x-2x) */
     cacheCreationInputTokens?: number;
+    /** Tokens read from cache (all providers - significant cost savings) */
     cacheReadInputTokens?: number;
+    /** Calculated cache savings percentage (0-100) based on provider discount rates */
+    cacheSavingsPercent?: number;
     httpStatusCode?: number;
     httpMethod?: string;
     httpUrl?: string;
@@ -250,6 +276,19 @@ interface CircuitBreakerState {
     lastFailureTime?: number;
     isOpen: boolean;
 }
+/**
+ * Calculate the cache savings percentage based on provider discount rates.
+ *
+ * @param provider - The provider name (e.g., 'anthropic', 'openai', 'google')
+ * @param inputTokens - Total input tokens in the request
+ * @param cacheReadTokens - Tokens read from cache
+ * @returns Savings percentage (0-100) or undefined if not calculable
+ *
+ * @example
+ * // 1000 input tokens, 800 from cache, using Anthropic (90% discount)
+ * calculateCacheSavings('anthropic', 1000, 800) // => 72 (72% savings)
+ */
+declare function calculateCacheSavings(provider: string, inputTokens: number | undefined, cacheReadTokens: number | undefined): number | undefined;
 
 /**
  * Internal JSON Schema representation for schema translation.
@@ -1690,4 +1729,4 @@ declare function createVLMProvider(config: {
  */
 declare function buildLLMProvider(config: FallbackConfig): VLMProvider;
 
-export { type AccessMethod, AnthropicProvider, BLOCK_TYPES, type BlockType, type CircuitBreakerState, type DocumentBlock, type FallbackConfig, FallbackManager, GEMINI_BBOX_EXTRACTION_PROMPT, type GeminiBoundingBoxBlock, GoogleProvider, type ImageInput, type JsonMode, type LLMDerivedOptions, type LLMExtractedMetadata, type LLMModelMetadata, type LLMProvider, type LLMProviderMetadata, type LLMProviderType, type LLMResponse, type MultimodalInput, type NodeType, type NormalizedBBox, OpenAIProvider, type PDFInput, PROVIDER_METADATA, type ProviderCapabilities, type ProviderConfig, type ProviderFactory, type ProviderInputType, type ProviderType, type ReasoningConfig, type ReasoningDetail, type ResourceLimits, type ResponseMetrics, SUPPORTED_IMAGE_TYPES, SchemaTranslator, type SupportedImageMimeType, type UnifiedSchema, XAIProvider, adaptToCoreLLMProvider, buildBlockClassificationPrompt, buildConfidencePrompt, buildLLMDerivedFeaturesPrompt, buildLLMProvider, buildLanguageHintsPrompt, buildOutputFormatPrompt, buildSchemaPromptSection, buildSourcesPrompt, combineSchemaAndUserPrompt, combineSchemaUserAndDerivedPrompts, compareNativeVsOpenRouter, convertGeminiBlocksToDocumentBlocks, createProviderFromRegistry, createVLMProvider, estimateCost, extractMetadataFromResponse, formatSchemaForPrompt, geminiBoundingBoxSchema, getCheapestProvider, getProvidersForNode, isImageTypeSupported, isProviderCompatibleWithNode, normalizeGeminiBBox, providerRegistry, registerProvider, shouldExtractMetadata, supportsPDFsInline, toGeminiBBox };
+export { type AccessMethod, AnthropicProvider, BLOCK_TYPES, type BlockType, type CachingConfig, type CircuitBreakerState, type DocumentBlock, type FallbackConfig, FallbackManager, GEMINI_BBOX_EXTRACTION_PROMPT, type GeminiBoundingBoxBlock, GoogleProvider, type ImageInput, type JsonMode, type LLMDerivedOptions, type LLMExtractedMetadata, type LLMModelMetadata, type LLMProvider, type LLMProviderMetadata, type LLMProviderType, type LLMResponse, type MultimodalInput, type NodeType, type NormalizedBBox, OpenAIProvider, type PDFInput, PROVIDER_METADATA, type ProviderCapabilities, type ProviderConfig, type ProviderFactory, type ProviderInputType, type ProviderType, type ReasoningConfig, type ReasoningDetail, type ResourceLimits, type ResponseMetrics, SUPPORTED_IMAGE_TYPES, SchemaTranslator, type SupportedImageMimeType, type UnifiedSchema, XAIProvider, adaptToCoreLLMProvider, buildBlockClassificationPrompt, buildConfidencePrompt, buildLLMDerivedFeaturesPrompt, buildLLMProvider, buildLanguageHintsPrompt, buildOutputFormatPrompt, buildSchemaPromptSection, buildSourcesPrompt, calculateCacheSavings, combineSchemaAndUserPrompt, combineSchemaUserAndDerivedPrompts, compareNativeVsOpenRouter, convertGeminiBlocksToDocumentBlocks, createProviderFromRegistry, createVLMProvider, estimateCost, extractMetadataFromResponse, formatSchemaForPrompt, geminiBoundingBoxSchema, getCheapestProvider, getProvidersForNode, isImageTypeSupported, isProviderCompatibleWithNode, normalizeGeminiBBox, providerRegistry, registerProvider, shouldExtractMetadata, supportsPDFsInline, toGeminiBBox };

package/dist/index.js

@@ -11,6 +11,31 @@ import {
   formatSchemaForPrompt
 } from "./chunk-7YPJIWRM.js";
 
+// src/types.ts
+var CACHE_DISCOUNT_RATES = {
+  anthropic: 0.9,
+  // 90% discount on cached reads (0.1x price)
+  openai: 0.5,
+  // 50% discount
+  google: 0.75,
+  // 75% discount (0.25x price)
+  "x-ai": 0.75,
+  // 75% discount (Grok)
+  xai: 0.75,
+  // alias
+  deepseek: 0.9
+  // 90% discount
+};
+function calculateCacheSavings(provider, inputTokens, cacheReadTokens) {
+  if (!inputTokens || !cacheReadTokens || inputTokens === 0) {
+    return void 0;
+  }
+  const normalizedProvider = provider.includes("/") ? provider.split("/")[0] : provider;
+  const discountRate = CACHE_DISCOUNT_RATES[normalizedProvider.toLowerCase()] ?? 0.5;
+  const savingsPercent = Math.round(cacheReadTokens / inputTokens * discountRate * 100);
+  return Math.min(savingsPercent, 100);
+}
+
 // src/schema-translator.ts
 import { zodToJsonSchema } from "@alcyone-labs/zod-to-json-schema";
 var SchemaTranslator = class {
@@ -470,18 +495,23 @@ var OpenAIProvider = class {
       costUSD = this.calculateCost(data.usage);
     }
     const baseProvider = extractProviderFromModel(this.config.model, "openai");
+    const cacheReadInputTokens = data.usage?.prompt_tokens_details?.cached_tokens;
+    const inputTokens = data.usage?.prompt_tokens;
+    const cacheSavingsPercent = calculateCacheSavings(baseProvider, inputTokens, cacheReadInputTokens);
     return {
       json: parsed,
       rawText: content,
       metrics: {
         costUSD,
-        inputTokens: data.usage?.prompt_tokens,
+        inputTokens,
         outputTokens: data.usage?.completion_tokens,
         latencyMs,
         attemptNumber: 1,
         provider: baseProvider,
         // Base provider (e.g., "openai" from "openai/gpt-4...")
-        model: this.config.model
+        model: this.config.model,
+        cacheReadInputTokens,
+        cacheSavingsPercent
       },
       reasoning,
       reasoning_details,
@@ -747,8 +777,8 @@ var AnthropicProvider = class {
       inputTokens = data.usage?.prompt_tokens;
       outputTokens = data.usage?.completion_tokens;
       costUSD = data.usage?.total_cost ?? data.usage?.cost;
-      const cacheCreationInputTokens = data.usage?.cache_creation_input_tokens;
-      const cacheReadInputTokens = data.usage?.cache_read_input_tokens;
+      const cacheCreationInputTokens = data.usage?.cache_creation_input_tokens ?? data.usage?.prompt_tokens_details?.cache_write_tokens;
+      const cacheReadInputTokens = data.usage?.cache_read_input_tokens ?? data.usage?.prompt_tokens_details?.cached_tokens;
       if (process.env.DEBUG_PROVIDERS) {
         console.log("[AnthropicProvider] OpenRouter usage response:", JSON.stringify(data.usage, null, 2));
         console.log("[AnthropicProvider] Extracted costUSD:", costUSD);
@@ -757,6 +787,7 @@ var AnthropicProvider = class {
       }
       const latencyMs = Date.now() - startTime;
       const baseProvider = extractProviderFromModel2(this.config.model, "anthropic");
+      const cacheSavingsPercent = calculateCacheSavings(baseProvider, inputTokens, cacheReadInputTokens);
       const { json: cleanJson, metadata } = extractMetadata ? extractMetadataFromResponse(parsed) : { json: parsed, metadata: void 0 };
       return {
         json: cleanJson,
@@ -771,7 +802,8 @@ var AnthropicProvider = class {
           // Base provider (e.g., "anthropic" from "anthropic/claude-...")
           model: this.config.model,
           cacheCreationInputTokens,
-          cacheReadInputTokens
+          cacheReadInputTokens,
+          cacheSavingsPercent
         },
         reasoning,
         reasoning_details,
@@ -882,13 +914,22 @@ var AnthropicProvider = class {
   }
   translateToOpenRouterFormat(messages, schema, mode, max_tokens, reasoning, systemPrompt) {
     const useNewStructuredOutputs = this.supportsNewStructuredOutputs();
+    const cachingEnabled = this.config.caching?.enabled === true;
+    const cacheTTL = this.config.caching?.ttl || "5m";
     const jsonInstructions = mode === "strict" ? "You must respond ONLY with valid JSON that matches the provided schema. Do not include any markdown formatting, explanations, or additional text." : "You must respond ONLY with valid JSON. Do not include any markdown formatting, explanations, or additional text.";
     const systemContent = systemPrompt ? `${systemPrompt}
 
 ${jsonInstructions}` : `You are a data extraction assistant. ${jsonInstructions}`;
     const systemMessage = {
       role: "system",
-      content: systemContent
+      content: cachingEnabled ? [{
+        type: "text",
+        text: systemContent,
+        cache_control: {
+          type: "ephemeral",
+          ...cacheTTL === "1h" && { ttl: "1h" }
+        }
+      }] : systemContent
     };
     const messageArray = [systemMessage, ...messages];
     const requestBody = {
@@ -1055,22 +1096,38 @@ ${jsonInstructions}` : `You are a data extraction assistant. ${jsonInstructions}
           });
         }
       }
+      const cachingEnabled = this.config.caching?.enabled === true;
+      const cacheTTL = this.config.caching?.ttl || "5m";
       if (hasMedia) {
         const textContent = input.text || "Extract the requested information from the document.";
         if (process.env.DEBUG_PROVIDERS) {
-          console.log("[AnthropicProvider.buildMessages] Adding text block with cache_control");
+          console.log("[AnthropicProvider.buildMessages] Adding text block" + (cachingEnabled ? " with cache_control" : ""));
           console.log("  textContent:", textContent);
+          console.log("  cachingEnabled:", cachingEnabled);
         }
-        content.push({
+        const textBlock = {
           type: "text",
-          text: textContent,
-          cache_control: { type: "ephemeral" }
-        });
+          text: textContent
+        };
+        if (cachingEnabled) {
+          textBlock.cache_control = {
+            type: "ephemeral",
+            ...cacheTTL === "1h" && { ttl: "1h" }
+          };
+        }
+        content.push(textBlock);
       } else if (input.text) {
-        content.push({
+        const textBlock = {
           type: "text",
           text: input.text
-        });
+        };
+        if (cachingEnabled) {
+          textBlock.cache_control = {
+            type: "ephemeral",
+            ...cacheTTL === "1h" && { ttl: "1h" }
+          };
+        }
+        content.push(textBlock);
       }
     } else {
       if (input.text) {
@@ -1364,10 +1421,12 @@ var GoogleProvider = class {
       costUSD = data.usage?.total_cost ?? data.usage?.cost;
       const reasoning = message?.reasoning;
       const reasoning_details = message?.reasoning_details;
+      const cacheReadInputTokens = data.usage?.cached_tokens;
       content = content.replace(/^```json\s*\n?/, "").replace(/\n?```\s*$/, "").trim();
       const rawParsed = safeJsonParse3(content);
       const { json: parsed, metadata } = extractMetadata ? extractMetadataFromResponse(rawParsed) : { json: rawParsed, metadata: void 0 };
       const baseProvider = extractProviderFromModel3(this.config.model, "google");
+      const cacheSavingsPercent = calculateCacheSavings(baseProvider, inputTokens, cacheReadInputTokens);
       return {
         json: parsed,
         rawText: content,
@@ -1379,7 +1438,9 @@ var GoogleProvider = class {
           attemptNumber: 1,
           provider: baseProvider,
           // Base provider (e.g., "google" from "google/gemini-...")
-          model: this.config.model
+          model: this.config.model,
+          cacheReadInputTokens,
+          cacheSavingsPercent
         },
         reasoning,
         reasoning_details,
@@ -1391,11 +1452,13 @@ var GoogleProvider = class {
       inputTokens = data.usageMetadata?.promptTokenCount;
       outputTokens = data.usageMetadata?.candidatesTokenCount;
       costUSD = this.calculateCost(data.usageMetadata);
+      const cacheReadInputTokens = data.usageMetadata?.cachedContentTokenCount;
       const thinkingPart = candidate?.content?.parts?.find((part) => part.thought === true);
       const reasoning = thinkingPart?.text;
       const rawParsed = safeJsonParse3(content);
       const { json: parsed, metadata } = extractMetadata ? extractMetadataFromResponse(rawParsed) : { json: rawParsed, metadata: void 0 };
       const baseProvider = extractProviderFromModel3(this.config.model, "google");
+      const cacheSavingsPercent = calculateCacheSavings(baseProvider, inputTokens, cacheReadInputTokens);
       return {
         json: parsed,
         rawText: content,
@@ -1407,7 +1470,9 @@ var GoogleProvider = class {
           attemptNumber: 1,
           provider: baseProvider,
           // Base provider (e.g., "google" from "google/gemini-...")
-          model: this.config.model
+          model: this.config.model,
+          cacheReadInputTokens,
+          cacheSavingsPercent
         },
         reasoning,
         reasoning_details: reasoning ? [{
@@ -1750,18 +1815,23 @@ var XAIProvider = class {
       costUSD = this.calculateCost(data.usage);
     }
     const baseProvider = extractProviderFromModel4(this.config.model, "xai");
+    const cacheReadInputTokens = data.usage?.prompt_tokens_details?.cached_tokens;
+    const inputTokens = data.usage?.prompt_tokens;
+    const cacheSavingsPercent = calculateCacheSavings(baseProvider, inputTokens, cacheReadInputTokens);
     return {
       json: parsed,
       rawText: content,
       metrics: {
         costUSD,
-        inputTokens: data.usage?.prompt_tokens,
+        inputTokens,
         outputTokens: data.usage?.completion_tokens,
         latencyMs,
         attemptNumber: 1,
         provider: baseProvider,
         // Base provider (e.g., "x-ai" from "x-ai/grok-...")
-        model: this.config.model
+        model: this.config.model,
+        cacheReadInputTokens,
+        cacheSavingsPercent
       },
       reasoning,
       reasoning_details,
@@ -2900,6 +2970,7 @@ export {
   buildOutputFormatPrompt,
   buildSchemaPromptSection,
   buildSourcesPrompt,
+  calculateCacheSavings,
   combineSchemaAndUserPrompt,
   combineSchemaUserAndDerivedPrompts,
   compareNativeVsOpenRouter,