@startsimpli/llm 0.4.23 → 0.4.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@startsimpli/llm",
-  "version": "0.4.23",
+  "version": "0.4.24",
   "description": "Generic LLM service infrastructure — provider abstraction, retry logic, JSON extraction",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/__tests__/openai-provider.test.ts

@@ -33,6 +33,8 @@ interface Sent {
   max_tokens?: number;
   max_completion_tokens?: number;
   reasoning_effort?: string;
+  prompt_cache_key?: string;
+  prompt_cache_retention?: string;
 }
 
 const OK_BODY = {
@@ -280,3 +282,39 @@ describe('images', () => {
     expect(plain.messages[1]!.content).toBe('plain');
   });
 });
+
+describe('prompt caching', () => {
+  // Measured against api.openai.com 2026-10-15 (gpt-6-astra, gpt-6-luna): both
+  // parameters accepted; a repeat of a 2,020-token prompt came back with
+  // prompt_tokens_details.cached_tokens 2,017.
+  it('sends the cache key and retention when asked, and nothing when not', async () => {
+    const p = provider('gpt-6-astra');
+    await p.generate('u', 's', { cacheKey: 'crochet-decompose', cacheRetention: '24h' });
+    await p.generate('u', 's');
+    expect(sent[0]!.prompt_cache_key).toBe('crochet-decompose');
+    expect(sent[0]!.prompt_cache_retention).toBe('24h');
+    expect(sent[1]!.prompt_cache_key).toBeUndefined();
+    expect(sent[1]!.prompt_cache_retention).toBeUndefined();
+  });
+
+  it('reports cached prompt tokens', async () => {
+    fetchMock.mockImplementationOnce(async (_url: unknown, init: RequestInit) => {
+      sent.push(JSON.parse(String(init.body)) as Sent);
+      return { ok: true, status: 200, json: async () => ({ ...OK_BODY, usage: { prompt_tokens: 2020, completion_tokens: 4, total_tokens: 2024, prompt_tokens_details: { cached_tokens: 2017 } } }) } as unknown as Response;
+    });
+    const res = await provider('gpt-6-astra').generate('u', 's', { cacheKey: 'k' });
+    expect(res.usage.cachedTokens).toBe(2017);
+  });
+
+  it('drops the cache parameters for a model that rejects them, and remembers', async () => {
+    const p = provider('some-old-model');
+    reject({ code: 'unknown_parameter', param: 'prompt_cache_retention', message: "Unrecognized request argument supplied: prompt_cache_retention" });
+    const res = await p.generate('u', 's', { cacheKey: 'k', cacheRetention: '24h' });
+    expect(res.content).toBe('{"ok":true}');
+    expect(sent[1]!.prompt_cache_retention).toBeUndefined();
+    expect(sent[1]!.prompt_cache_key).toBeUndefined();
+    await p.generate('u', 's', { cacheKey: 'k', cacheRetention: '24h' });
+    expect(sent).toHaveLength(3);
+    expect(sent[2]!.prompt_cache_retention).toBeUndefined();
+  });
+});

package/src/__tests__/usage-tally.test.ts

@@ -0,0 +1,22 @@
+import { describe, it, expect } from 'vitest';
+import { newUsageTally, tallyingService } from '../utils/usage-tally';
+
+describe('tallyingService', () => {
+  it('adds up every call per model, cached tokens included, and passes other members through', async () => {
+    const svc = {
+      name: 'svc',
+      async generateChatResponse(_u: string, _s: string, opts?: { model?: string }) {
+        return { content: '{}', model: opts?.model ?? 'a', usage: { promptTokens: 100, completionTokens: 40, totalTokens: 140, cachedTokens: 60 } };
+      },
+    };
+    const tally = newUsageTally();
+    const counted = tallyingService(svc, tally);
+    await counted.generateChatResponse('u', 's', { model: 'astra' });
+    await counted.generateChatResponse('u', 's', { model: 'astra' });
+    await counted.generateChatResponse('u', 's', { model: 'luna' });
+    expect(counted.name).toBe('svc');
+    expect(tally).toMatchObject({ calls: 3, promptTokens: 300, completionTokens: 120, cachedTokens: 180 });
+    expect(tally.byModel.astra).toEqual({ calls: 2, promptTokens: 200, completionTokens: 80, cachedTokens: 120 });
+    expect(tally.byModel.luna.calls).toBe(1);
+  });
+});

package/src/index.ts

@@ -36,6 +36,8 @@ export type { MockBehavior } from './providers/mock';
 // Utilities
 export { extractJson } from './utils/json-extraction'
 export { llmErrorToHttpStatus } from './utils/http-status';
+export { newUsageTally, addUsage, tallyingService } from './utils/usage-tally';
+export type { UsageTally, UsageCounts } from './utils/usage-tally';
 export {
   buildSchemaErrorCorrectionPrompt,
   buildParseErrorCorrectionPrompt,

package/src/providers/openai.ts

@@ -19,6 +19,7 @@ import type {
   NativeToolResponse,
   ReasoningEffort,
   LLMImage,
+  TokenUsage,
 } from '../types';
 import { LLM_DEFAULTS } from '../types';
 import { LLMProviderError } from '../errors';
@@ -51,6 +52,8 @@ interface OpenAIRequest {
   /** Reasoning models only; dropped automatically for models that reject it. */
   reasoning_effort?: ReasoningEffort;
   response_format?: { type: 'json_object' | 'text' };
+  prompt_cache_key?: string;
+  prompt_cache_retention?: 'in_memory' | '24h';
 }
 
 /**
@@ -85,6 +88,23 @@ interface ModelQuirks {
   efforts?: ReasoningEffort[];
   /** Largest completion-token cap this model accepts. */
   maxOutput?: number;
+  /** The model rejected prompt-cache parameters; send none. */
+  noPromptCache?: boolean;
+}
+
+/** Prompt-cache parameters for a request, unless this model has rejected them. */
+function cacheParams(q: ModelQuirks, options?: GenerateOptions): Pick<OpenAIRequest, 'prompt_cache_key' | 'prompt_cache_retention'> {
+  if (q.noPromptCache) return {};
+  return {
+    ...(options?.cacheKey ? { prompt_cache_key: options.cacheKey } : {}),
+    ...(options?.cacheRetention ? { prompt_cache_retention: options.cacheRetention } : {}),
+  };
+}
+
+/** Cached prompt tokens from an OpenAI usage block, when it reports them. */
+function cachedTokens(usage: { prompt_tokens_details?: { cached_tokens?: number } } | undefined): { cachedTokens?: number } {
+  const n = usage?.prompt_tokens_details?.cached_tokens;
+  return typeof n === 'number' ? { cachedTokens: n } : {};
 }
 
 const EFFORT_ORDER: ReasoningEffort[] = ['none', 'minimal', 'low', 'medium', 'high', 'xhigh'];
@@ -103,6 +123,11 @@ function nearestEffort(want: ReasoningEffort, supported: ReasoningEffort[]): Rea
 function learnFromError(q: ModelQuirks, err: OpenAIErrorResponse['error'] | undefined, sent: { effort?: ReasoningEffort; maxTokens: number }): boolean {
   if (!err?.message) return false;
   const msg = err.message;
+  if (err.param === 'prompt_cache_key' || err.param === 'prompt_cache_retention' || /prompt_cache_(key|retention)/.test(msg)) {
+    if (q.noPromptCache) return false;
+    q.noPromptCache = true;
+    return true;
+  }
   if (err.param === 'temperature' || /'temperature'/.test(msg)) {
     if (q.noTemperature) return false;
     q.noTemperature = true;
@@ -183,6 +208,7 @@ interface OpenAIResponse {
     prompt_tokens: number;
     completion_tokens: number;
     total_tokens: number;
+    prompt_tokens_details?: { cached_tokens?: number };
   };
 }
 
@@ -216,7 +242,7 @@ interface OpenAIToolsResponse {
     message: { role: string; content: string | null; tool_calls?: OpenAIToolCall[] };
     finish_reason: string;
   }>;
-  usage: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
+  usage: { prompt_tokens: number; completion_tokens: number; total_tokens: number; prompt_tokens_details?: { cached_tokens?: number } };
 }
 
 const OPENAI_MODELS = [
@@ -351,13 +377,14 @@ export class OpenAIProvider implements ILLMProvider {
       const response = await this.postAdaptivelyRaw(model, (q) => ({
         ...base,
         ...samplingParams(q, temperature, maxTokens, effort),
+        ...cacheParams(q, options),
       }), { effort, maxTokens }, controller.signal);
 
       let content = '';
       let reportedModel = model;
       let finishReason: string | undefined;
       let responseId: string | undefined;
-      const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
+      const usage: TokenUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
       const reader = response.body!.getReader();
       const decoder = new TextDecoder();
       let buffer = '';
@@ -376,7 +403,7 @@ export class OpenAIProvider implements ILLMProvider {
             id?: string;
             model?: string;
             choices?: Array<{ delta?: { content?: string | null }; finish_reason?: string | null }>;
-            usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
+            usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number; prompt_tokens_details?: { cached_tokens?: number } };
           };
           if (chunk.id) responseId = chunk.id;
           if (chunk.model) reportedModel = chunk.model;
@@ -391,6 +418,7 @@ export class OpenAIProvider implements ILLMProvider {
             usage.promptTokens = chunk.usage.prompt_tokens;
             usage.completionTokens = chunk.usage.completion_tokens;
             usage.totalTokens = chunk.usage.total_tokens;
+            Object.assign(usage, cachedTokens(chunk.usage));
           }
         }
       }
@@ -450,6 +478,7 @@ export class OpenAIProvider implements ILLMProvider {
       const data = await this.postAdaptively<OpenAIResponse>(model, (q) => ({
         ...base,
         ...samplingParams(q, temperature, maxTokens, effort),
+        ...cacheParams(q, options),
       }), { effort, maxTokens }, controller.signal);
 
       clearTimeout(timeoutId);
@@ -470,6 +499,7 @@ export class OpenAIProvider implements ILLMProvider {
           promptTokens: data.usage.prompt_tokens,
           completionTokens: data.usage.completion_tokens,
           totalTokens: data.usage.total_tokens,
+          ...cachedTokens(data.usage),
         },
         model: data.model,
         provider: this.name,
@@ -568,6 +598,7 @@ export class OpenAIProvider implements ILLMProvider {
       const data = await this.postAdaptively<OpenAIToolsResponse>(model, (q) => ({
         ...base,
         ...samplingParams(q, temperature, maxTokens, effort),
+        ...cacheParams(q, options),
       }), { effort, maxTokens }, controller.signal);
       clearTimeout(timeoutId);
       const choice = data.choices?.[0];

package/src/types.ts

@@ -7,6 +7,8 @@ export interface TokenUsage {
   promptTokens: number;
   completionTokens: number;
   totalTokens: number;
+  /** Of promptTokens, how many were served from the provider's prompt cache (billed at the cached rate). */
+  cachedTokens?: number;
 }
 
 /**
@@ -56,6 +58,15 @@ export interface GenerateOptions {
    * a model look at a rendering of what it designed).
    */
   images?: LLMImage[];
+  /**
+   * Prompt caching (OpenAI). Requests that share a long identical opening
+   * (instructions first, per-request data last) are billed at the cached rate
+   * when they reach the same cache: `cacheKey` routes them there, and
+   * `cacheRetention: '24h'` keeps the cache for a day instead of minutes, which
+   * is what makes it pay off for low-traffic features.
+   */
+  cacheKey?: string;
+  cacheRetention?: 'in_memory' | '24h';
   /** Arbitrary extra data the caller wants to thread through (e.g. domain preferences) */
   extra?: Record<string, unknown>;
 }

package/src/utils/usage-tally.ts

@@ -0,0 +1,56 @@
+/**
+ * Token usage added up across every call a feature makes, per model, so what a
+ * request cost can be recorded and priced (a crochet design makes 3-7 calls on
+ * two models). Wrap any service whose methods resolve to `{ model, usage }`.
+ */
+import type { TokenUsage } from '../types';
+
+export interface UsageCounts {
+  calls: number;
+  promptTokens: number;
+  completionTokens: number;
+  /** Of promptTokens, how many were served from the prompt cache. */
+  cachedTokens: number;
+}
+
+export interface UsageTally extends UsageCounts {
+  byModel: Record<string, UsageCounts>;
+}
+
+const zero = (): UsageCounts => ({ calls: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0 });
+
+export function newUsageTally(): UsageTally {
+  return { ...zero(), byModel: {} };
+}
+
+/** Add one response's usage to the tally. Returns the response unchanged. */
+export function addUsage<R extends { model?: string; usage?: TokenUsage }>(tally: UsageTally, response: R): R {
+  const u = response.usage;
+  const m = (tally.byModel[response.model ?? 'unknown'] ??= zero());
+  for (const c of [tally, m]) {
+    c.calls++;
+    c.promptTokens += u?.promptTokens ?? 0;
+    c.completionTokens += u?.completionTokens ?? 0;
+    c.cachedTokens += u?.cachedTokens ?? 0;
+  }
+  return response;
+}
+
+/**
+ * The same service, with every call's usage added to `tally`. Methods in
+ * `methods` that return a promise of a response are counted; everything else
+ * passes through untouched.
+ */
+export function tallyingService<S extends object>(
+  service: S,
+  tally: UsageTally,
+  methods: readonly string[] = ['generateChatResponse', 'generateChatResponseStream', 'generate'],
+): S {
+  return new Proxy(service, {
+    get(target, prop, receiver) {
+      const value = Reflect.get(target, prop, receiver);
+      if (typeof prop !== 'string' || !methods.includes(prop) || typeof value !== 'function') return value;
+      return (...args: unknown[]) => (value as (...a: unknown[]) => Promise<{ model?: string; usage?: TokenUsage }>).apply(target, args).then((r) => addUsage(tally, r));
+    },
+  });
+}