@startsimpli/llm 0.4.22 → 0.4.24

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@startsimpli/llm",
-  "version": "0.4.22",
+  "version": "0.4.24",
   "description": "Generic LLM service infrastructure — provider abstraction, retry logic, JSON extraction",
   "main": "./src/index.ts",
   "types": "./src/index.ts",

package/src/__tests__/anthropic-provider.test.ts

@@ -169,3 +169,14 @@ describe('AnthropicProvider — recovering from a parameter the model rejects',
     expect(sent).toHaveLength(1);
   });
 });
+
+describe('images', () => {
+  it('sends images as base64 image blocks before the text', async () => {
+    await provider('claude-opus-5-5').generate('what is this?', 's', { images: [{ data: 'AAAA', mediaType: 'image/png' }] });
+    const body = sent[0] as unknown as { messages: Array<{ content: Array<Record<string, unknown>> }> };
+    const content = body.messages[0]!.content;
+    expect(content[0]).toEqual({ type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'AAAA' } });
+    expect(content[1]!.type).toBe('text');
+    expect(String(content[1]!.text)).toContain('what is this?');
+  });
+});

package/src/__tests__/openai-provider.test.ts

@@ -33,6 +33,8 @@ interface Sent {
   max_tokens?: number;
   max_completion_tokens?: number;
   reasoning_effort?: string;
+  prompt_cache_key?: string;
+  prompt_cache_retention?: string;
 }
 
 const OK_BODY = {
@@ -265,3 +267,54 @@ describe('generateStream', () => {
     await expect(provider('gpt-6-astra').generateStream!('u', 's', () => {})).rejects.toThrow(/Empty response/);
   });
 });
+
+describe('images', () => {
+  it('sends images as image_url parts after the text, and plain text without them', async () => {
+    const p = provider('gpt-6-astra');
+    await p.generate('what is this?', 's', { images: [{ data: 'AAAA', mediaType: 'image/png' }] });
+    await p.generate('plain', 's');
+    const withImage = sent[0] as unknown as { messages: Array<{ role: string; content: unknown }> };
+    expect(withImage.messages[1]!.content).toEqual([
+      { type: 'text', text: 'what is this?' },
+      { type: 'image_url', image_url: { url: 'data:image/png;base64,AAAA' } },
+    ]);
+    const plain = sent[1] as unknown as { messages: Array<{ role: string; content: unknown }> };
+    expect(plain.messages[1]!.content).toBe('plain');
+  });
+});
+
+describe('prompt caching', () => {
+  // Measured against api.openai.com 2026-10-15 (gpt-6-astra, gpt-6-luna): both
+  // parameters accepted; a repeat of a 2,020-token prompt came back with
+  // prompt_tokens_details.cached_tokens 2,017.
+  it('sends the cache key and retention when asked, and nothing when not', async () => {
+    const p = provider('gpt-6-astra');
+    await p.generate('u', 's', { cacheKey: 'crochet-decompose', cacheRetention: '24h' });
+    await p.generate('u', 's');
+    expect(sent[0]!.prompt_cache_key).toBe('crochet-decompose');
+    expect(sent[0]!.prompt_cache_retention).toBe('24h');
+    expect(sent[1]!.prompt_cache_key).toBeUndefined();
+    expect(sent[1]!.prompt_cache_retention).toBeUndefined();
+  });
+
+  it('reports cached prompt tokens', async () => {
+    fetchMock.mockImplementationOnce(async (_url: unknown, init: RequestInit) => {
+      sent.push(JSON.parse(String(init.body)) as Sent);
+      return { ok: true, status: 200, json: async () => ({ ...OK_BODY, usage: { prompt_tokens: 2020, completion_tokens: 4, total_tokens: 2024, prompt_tokens_details: { cached_tokens: 2017 } } }) } as unknown as Response;
+    });
+    const res = await provider('gpt-6-astra').generate('u', 's', { cacheKey: 'k' });
+    expect(res.usage.cachedTokens).toBe(2017);
+  });
+
+  it('drops the cache parameters for a model that rejects them, and remembers', async () => {
+    const p = provider('some-old-model');
+    reject({ code: 'unknown_parameter', param: 'prompt_cache_retention', message: "Unrecognized request argument supplied: prompt_cache_retention" });
+    const res = await p.generate('u', 's', { cacheKey: 'k', cacheRetention: '24h' });
+    expect(res.content).toBe('{"ok":true}');
+    expect(sent[1]!.prompt_cache_retention).toBeUndefined();
+    expect(sent[1]!.prompt_cache_key).toBeUndefined();
+    await p.generate('u', 's', { cacheKey: 'k', cacheRetention: '24h' });
+    expect(sent).toHaveLength(3);
+    expect(sent[2]!.prompt_cache_retention).toBeUndefined();
+  });
+});

package/src/__tests__/usage-tally.test.ts

@@ -0,0 +1,22 @@
+import { describe, it, expect } from 'vitest';
+import { newUsageTally, tallyingService } from '../utils/usage-tally';
+
+describe('tallyingService', () => {
+  it('adds up every call per model, cached tokens included, and passes other members through', async () => {
+    const svc = {
+      name: 'svc',
+      async generateChatResponse(_u: string, _s: string, opts?: { model?: string }) {
+        return { content: '{}', model: opts?.model ?? 'a', usage: { promptTokens: 100, completionTokens: 40, totalTokens: 140, cachedTokens: 60 } };
+      },
+    };
+    const tally = newUsageTally();
+    const counted = tallyingService(svc, tally);
+    await counted.generateChatResponse('u', 's', { model: 'astra' });
+    await counted.generateChatResponse('u', 's', { model: 'astra' });
+    await counted.generateChatResponse('u', 's', { model: 'luna' });
+    expect(counted.name).toBe('svc');
+    expect(tally).toMatchObject({ calls: 3, promptTokens: 300, completionTokens: 120, cachedTokens: 180 });
+    expect(tally.byModel.astra).toEqual({ calls: 2, promptTokens: 200, completionTokens: 80, cachedTokens: 120 });
+    expect(tally.byModel.luna.calls).toBe(1);
+  });
+});

package/src/index.ts

@@ -8,6 +8,7 @@ export type {
   LLMResponse,
   GenerateOptions,
   ReasoningEffort,
+  LLMImage,
   ProviderConfig,
   JsonExtractionResult,
   ValidationResult,
@@ -35,6 +36,8 @@ export type { MockBehavior } from './providers/mock';
 // Utilities
 export { extractJson } from './utils/json-extraction'
 export { llmErrorToHttpStatus } from './utils/http-status';
+export { newUsageTally, addUsage, tallyingService } from './utils/usage-tally';
+export type { UsageTally, UsageCounts } from './utils/usage-tally';
 export {
   buildSchemaErrorCorrectionPrompt,
   buildParseErrorCorrectionPrompt,

package/src/providers/anthropic.ts

@@ -11,9 +11,13 @@ import { LLM_DEFAULTS } from '../types';
 import { LLMProviderError } from '../errors';
 import type { LLMErrorCode } from '../errors';
 
+type AnthropicContentBlock =
+  | { type: 'text'; text: string }
+  | { type: 'image'; source: { type: 'base64'; media_type: string; data: string } };
+
 interface AnthropicMessage {
   role: 'user' | 'assistant';
-  content: string;
+  content: string | AnthropicContentBlock[];
 }
 
 interface AnthropicRequest {
@@ -199,7 +203,18 @@ Please respond with a valid JSON object only. Do not include any text before or
     const requestBody: AnthropicRequest = {
       model,
       max_tokens: maxTokens,
-      messages: [{ role: 'user', content: jsonWrappedPrompt }],
+      messages: [{
+        role: 'user',
+        content: options?.images?.length
+          ? [
+              ...options.images.map((img) => ({
+                type: 'image' as const,
+                source: { type: 'base64' as const, media_type: img.mediaType, data: img.data },
+              })),
+              { type: 'text' as const, text: jsonWrappedPrompt },
+            ]
+          : jsonWrappedPrompt,
+      }],
       system: systemPrompt,
     };
     // Only for models known to take it. The Claude 5 family answers a request

package/src/providers/openai.ts

@@ -18,14 +18,29 @@ import type {
   NativeChatMessage,
   NativeToolResponse,
   ReasoningEffort,
+  LLMImage,
+  TokenUsage,
 } from '../types';
 import { LLM_DEFAULTS } from '../types';
 import { LLMProviderError } from '../errors';
 import type { LLMErrorCode } from '../errors';
 
+type OpenAIContentPart =
+  | { type: 'text'; text: string }
+  | { type: 'image_url'; image_url: { url: string } };
+
 interface OpenAIMessage {
   role: 'system' | 'user' | 'assistant';
-  content: string;
+  content: string | OpenAIContentPart[];
+}
+
+/** The user message: plain text, or text plus images for vision models. */
+function userContent(prompt: string, images?: LLMImage[]): string | OpenAIContentPart[] {
+  if (!images?.length) return prompt;
+  return [
+    { type: 'text', text: prompt },
+    ...images.map((img) => ({ type: 'image_url' as const, image_url: { url: `data:${img.mediaType};base64,${img.data}` } })),
+  ];
 }
 
 interface OpenAIRequest {
@@ -37,6 +52,8 @@ interface OpenAIRequest {
   /** Reasoning models only; dropped automatically for models that reject it. */
   reasoning_effort?: ReasoningEffort;
   response_format?: { type: 'json_object' | 'text' };
+  prompt_cache_key?: string;
+  prompt_cache_retention?: 'in_memory' | '24h';
 }
 
 /**
@@ -71,6 +88,23 @@ interface ModelQuirks {
   efforts?: ReasoningEffort[];
   /** Largest completion-token cap this model accepts. */
   maxOutput?: number;
+  /** The model rejected prompt-cache parameters; send none. */
+  noPromptCache?: boolean;
+}
+
+/** Prompt-cache parameters for a request, unless this model has rejected them. */
+function cacheParams(q: ModelQuirks, options?: GenerateOptions): Pick<OpenAIRequest, 'prompt_cache_key' | 'prompt_cache_retention'> {
+  if (q.noPromptCache) return {};
+  return {
+    ...(options?.cacheKey ? { prompt_cache_key: options.cacheKey } : {}),
+    ...(options?.cacheRetention ? { prompt_cache_retention: options.cacheRetention } : {}),
+  };
+}
+
+/** Cached prompt tokens from an OpenAI usage block, when it reports them. */
+function cachedTokens(usage: { prompt_tokens_details?: { cached_tokens?: number } } | undefined): { cachedTokens?: number } {
+  const n = usage?.prompt_tokens_details?.cached_tokens;
+  return typeof n === 'number' ? { cachedTokens: n } : {};
 }
 
 const EFFORT_ORDER: ReasoningEffort[] = ['none', 'minimal', 'low', 'medium', 'high', 'xhigh'];
@@ -89,6 +123,11 @@ function nearestEffort(want: ReasoningEffort, supported: ReasoningEffort[]): Rea
 function learnFromError(q: ModelQuirks, err: OpenAIErrorResponse['error'] | undefined, sent: { effort?: ReasoningEffort; maxTokens: number }): boolean {
   if (!err?.message) return false;
   const msg = err.message;
+  if (err.param === 'prompt_cache_key' || err.param === 'prompt_cache_retention' || /prompt_cache_(key|retention)/.test(msg)) {
+    if (q.noPromptCache) return false;
+    q.noPromptCache = true;
+    return true;
+  }
   if (err.param === 'temperature' || /'temperature'/.test(msg)) {
     if (q.noTemperature) return false;
     q.noTemperature = true;
@@ -169,6 +208,7 @@ interface OpenAIResponse {
     prompt_tokens: number;
     completion_tokens: number;
     total_tokens: number;
+    prompt_tokens_details?: { cached_tokens?: number };
   };
 }
 
@@ -202,7 +242,7 @@ interface OpenAIToolsResponse {
     message: { role: string; content: string | null; tool_calls?: OpenAIToolCall[] };
     finish_reason: string;
   }>;
-  usage: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
+  usage: { prompt_tokens: number; completion_tokens: number; total_tokens: number; prompt_tokens_details?: { cached_tokens?: number } };
 }
 
 const OPENAI_MODELS = [
@@ -324,7 +364,7 @@ export class OpenAIProvider implements ILLMProvider {
       model,
       messages: [
         { role: 'system' as const, content: systemPrompt },
-        { role: 'user' as const, content: userPrompt },
+        { role: 'user' as const, content: userContent(userPrompt, options?.images) },
       ],
       response_format: { type: 'json_object' as const },
       stream: true,
@@ -337,13 +377,14 @@ export class OpenAIProvider implements ILLMProvider {
       const response = await this.postAdaptivelyRaw(model, (q) => ({
         ...base,
         ...samplingParams(q, temperature, maxTokens, effort),
+        ...cacheParams(q, options),
       }), { effort, maxTokens }, controller.signal);
 
       let content = '';
       let reportedModel = model;
       let finishReason: string | undefined;
       let responseId: string | undefined;
-      const usage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
+      const usage: TokenUsage = { promptTokens: 0, completionTokens: 0, totalTokens: 0 };
       const reader = response.body!.getReader();
       const decoder = new TextDecoder();
       let buffer = '';
@@ -362,7 +403,7 @@ export class OpenAIProvider implements ILLMProvider {
             id?: string;
             model?: string;
             choices?: Array<{ delta?: { content?: string | null }; finish_reason?: string | null }>;
-            usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number };
+            usage?: { prompt_tokens: number; completion_tokens: number; total_tokens: number; prompt_tokens_details?: { cached_tokens?: number } };
           };
           if (chunk.id) responseId = chunk.id;
           if (chunk.model) reportedModel = chunk.model;
@@ -377,6 +418,7 @@ export class OpenAIProvider implements ILLMProvider {
             usage.promptTokens = chunk.usage.prompt_tokens;
             usage.completionTokens = chunk.usage.completion_tokens;
             usage.totalTokens = chunk.usage.total_tokens;
+            Object.assign(usage, cachedTokens(chunk.usage));
           }
         }
       }
@@ -424,7 +466,7 @@ export class OpenAIProvider implements ILLMProvider {
       model,
       messages: [
         { role: 'system' as const, content: systemPrompt },
-        { role: 'user' as const, content: userPrompt },
+        { role: 'user' as const, content: userContent(userPrompt, options?.images) },
       ],
       response_format: { type: 'json_object' as const },
     };
@@ -436,6 +478,7 @@ export class OpenAIProvider implements ILLMProvider {
       const data = await this.postAdaptively<OpenAIResponse>(model, (q) => ({
         ...base,
         ...samplingParams(q, temperature, maxTokens, effort),
+        ...cacheParams(q, options),
       }), { effort, maxTokens }, controller.signal);
 
       clearTimeout(timeoutId);
@@ -456,6 +499,7 @@ export class OpenAIProvider implements ILLMProvider {
           promptTokens: data.usage.prompt_tokens,
           completionTokens: data.usage.completion_tokens,
           totalTokens: data.usage.total_tokens,
+          ...cachedTokens(data.usage),
         },
         model: data.model,
         provider: this.name,
@@ -554,6 +598,7 @@ export class OpenAIProvider implements ILLMProvider {
       const data = await this.postAdaptively<OpenAIToolsResponse>(model, (q) => ({
         ...base,
         ...samplingParams(q, temperature, maxTokens, effort),
+        ...cacheParams(q, options),
       }), { effort, maxTokens }, controller.signal);
       clearTimeout(timeoutId);
       const choice = data.choices?.[0];

package/src/types.ts

@@ -7,6 +7,8 @@ export interface TokenUsage {
   promptTokens: number;
   completionTokens: number;
   totalTokens: number;
+  /** Of promptTokens, how many were served from the provider's prompt cache (billed at the cached rate). */
+  cachedTokens?: number;
 }
 
 /**
@@ -30,6 +32,12 @@ export interface LLMResponse {
 /**
  * Options passed through to the provider for a single generation call.
  */
+/** An image attached to a prompt: base64 data (no data: prefix) and its media type. */
+export interface LLMImage {
+  data: string;
+  mediaType: 'image/png' | 'image/jpeg' | 'image/gif' | 'image/webp';
+}
+
 export interface GenerateOptions {
   /** Model to use (provider-specific) */
   model?: string;
@@ -45,6 +53,20 @@ export interface GenerateOptions {
    * provider adds headroom for the reasoning on top.
    */
   reasoningEffort?: ReasoningEffort;
+  /**
+   * Images sent with the user prompt, for vision-capable models (e.g. to have
+   * a model look at a rendering of what it designed).
+   */
+  images?: LLMImage[];
+  /**
+   * Prompt caching (OpenAI). Requests that share a long identical opening
+   * (instructions first, per-request data last) are billed at the cached rate
+   * when they reach the same cache: `cacheKey` routes them there, and
+   * `cacheRetention: '24h'` keeps the cache for a day instead of minutes, which
+   * is what makes it pay off for low-traffic features.
+   */
+  cacheKey?: string;
+  cacheRetention?: 'in_memory' | '24h';
   /** Arbitrary extra data the caller wants to thread through (e.g. domain preferences) */
   extra?: Record<string, unknown>;
 }

package/src/utils/usage-tally.ts

@@ -0,0 +1,56 @@
+/**
+ * Token usage added up across every call a feature makes, per model, so what a
+ * request cost can be recorded and priced (a crochet design makes 3-7 calls on
+ * two models). Wrap any service whose methods resolve to `{ model, usage }`.
+ */
+import type { TokenUsage } from '../types';
+
+export interface UsageCounts {
+  calls: number;
+  promptTokens: number;
+  completionTokens: number;
+  /** Of promptTokens, how many were served from the prompt cache. */
+  cachedTokens: number;
+}
+
+export interface UsageTally extends UsageCounts {
+  byModel: Record<string, UsageCounts>;
+}
+
+const zero = (): UsageCounts => ({ calls: 0, promptTokens: 0, completionTokens: 0, cachedTokens: 0 });
+
+export function newUsageTally(): UsageTally {
+  return { ...zero(), byModel: {} };
+}
+
+/** Add one response's usage to the tally. Returns the response unchanged. */
+export function addUsage<R extends { model?: string; usage?: TokenUsage }>(tally: UsageTally, response: R): R {
+  const u = response.usage;
+  const m = (tally.byModel[response.model ?? 'unknown'] ??= zero());
+  for (const c of [tally, m]) {
+    c.calls++;
+    c.promptTokens += u?.promptTokens ?? 0;
+    c.completionTokens += u?.completionTokens ?? 0;
+    c.cachedTokens += u?.cachedTokens ?? 0;
+  }
+  return response;
+}
+
+/**
+ * The same service, with every call's usage added to `tally`. Methods in
+ * `methods` that return a promise of a response are counted; everything else
+ * passes through untouched.
+ */
+export function tallyingService<S extends object>(
+  service: S,
+  tally: UsageTally,
+  methods: readonly string[] = ['generateChatResponse', 'generateChatResponseStream', 'generate'],
+): S {
+  return new Proxy(service, {
+    get(target, prop, receiver) {
+      const value = Reflect.get(target, prop, receiver);
+      if (typeof prop !== 'string' || !methods.includes(prop) || typeof value !== 'function') return value;
+      return (...args: unknown[]) => (value as (...a: unknown[]) => Promise<{ model?: string; usage?: TokenUsage }>).apply(target, args).then((r) => addUsage(tally, r));
+    },
+  });
+}