llm-fns 1.0.26 → 1.0.27

package/dist/createCachedFetcher.d.ts

@@ -8,23 +8,19 @@ export type FetcherOptions = RequestInit & {
 };
 export type Fetcher = (url: string | URL | Request, options?: FetcherOptions) => Promise<Response>;
 export interface CreateFetcherDependencies {
-    /** The cache instance (e.g., from cache-manager). */
+    /** Cache instance, for example a cache-manager Cache. */
     cache?: CacheLike;
-    /** A prefix for all cache keys to avoid collisions. Defaults to 'http-cache'. */
+    /** Prefix for all cache keys. Defaults to `http-cache`. */
     prefix?: string;
-    /** Time-to-live for cache entries, in milliseconds. */
+    /** Time-to-live in milliseconds. */
     ttl?: number;
-    /** Request timeout in milliseconds. If not provided, no timeout is applied. */
+    /** Request timeout in milliseconds. */
     timeout?: number;
-    /** User-Agent string for requests. */
+    /** User-Agent header to add to requests. */
     userAgent?: string;
-    /** Optional custom fetch implementation. Defaults to global fetch. */
+    /** Fetch implementation. Defaults to global fetch. */
     fetch?: (url: string | URL | Request, init?: RequestInit) => Promise<Response>;
-    /**
-     * Optional callback to determine if a response should be cached.
-     * It receives a cloned response that can be read (e.g. .json()).
-     * If it returns false, the response is not cached.
-     */
+    /** Return false to prevent a successful response from being cached. */
     shouldCache?: (response: Response) => Promise<boolean> | boolean;
 }
 export declare class CachedResponse extends Response {
@@ -32,9 +28,5 @@ export declare class CachedResponse extends Response {
     constructor(body: BodyInit | null, init: ResponseInit, finalUrl: string);
     get url(): string;
 }
-/**
- * Factory function that creates a `fetch` replacement with a caching layer.
- * @param deps - Dependencies including the cache instance, prefix, TTL, and timeout.
- * @returns A function with the same signature as native `fetch`.
- */
+/** Creates a fetch-compatible function backed by deterministic response caching. */
 export declare function createCachedFetcher(deps: CreateFetcherDependencies): Fetcher;

package/dist/createCachedFetcher.js

@@ -1,4 +1,4 @@
-import crypto from 'crypto';
+import crypto from 'node:crypto';
 export class CachedResponse extends Response {
     #finalUrl;
     constructor(body, init, finalUrl) {
@@ -9,177 +9,166 @@ export class CachedResponse extends Response {
         return this.#finalUrl;
     }
 }
-/**
- * Creates a deterministic hash of headers for cache key generation.
- * Headers are sorted alphabetically to ensure consistency.
- */
-function hashHeaders(headers) {
-    if (!headers)
-        return '';
-    let headerEntries;
-    if (headers instanceof Headers) {
-        headerEntries = Array.from(headers.entries());
-    }
-    else if (Array.isArray(headers)) {
-        headerEntries = headers;
+function hashBytes(value) {
+    return crypto.createHash('sha256').update(typeof value === 'string' ? value : Buffer.from(value instanceof ArrayBuffer
+        ? value
+        : value.buffer, value instanceof ArrayBuffer ? undefined : value.byteOffset, value instanceof ArrayBuffer ? undefined : value.byteLength)).digest('hex');
+}
+function normalizedHeaders(headers) {
+    return [...headers.entries()]
+        .map(([name, value]) => {
+        const normalizedName = name.toLowerCase();
+        const normalizedValue = normalizedName === 'content-type' && /^multipart\/form-data\b/iu.test(value)
+            ? 'multipart/form-data'
+            : value.trim();
+        return [normalizedName, normalizedValue];
+    })
+        // Content length and multipart boundaries are transport details, not request semantics.
+        .filter(([name]) => name !== 'content-length')
+        .sort(([left], [right]) => left.localeCompare(right));
+}
+async function hashFormData(formData) {
+    const hash = crypto.createHash('sha256');
+    for (const [name, value] of formData.entries()) {
+        const nameBytes = Buffer.from(name);
+        hash.update(`name:${nameBytes.byteLength}:`);
+        hash.update(nameBytes);
+        if (typeof value === 'string') {
+            const valueBytes = Buffer.from(value);
+            hash.update(`:text:${valueBytes.byteLength}:`);
+            hash.update(valueBytes);
+            continue;
+        }
+        const fileName = 'name' in value && typeof value.name === 'string' ? value.name : '';
+        const metadata = Buffer.from(`${fileName}\0${value.type}`);
+        const bytes = await value.arrayBuffer();
+        hash.update(`:blob:${metadata.byteLength}:`);
+        hash.update(metadata);
+        hash.update(`:${bytes.byteLength}:`);
+        hash.update(Buffer.from(bytes));
     }
-    else {
-        headerEntries = Object.entries(headers);
+    return hash.digest('hex');
+}
+async function hashBody(body) {
+    if (body instanceof FormData)
+        return `form-data:${await hashFormData(body)}`;
+    if (body instanceof URLSearchParams)
+        return `url-search-params:${hashBytes(body.toString())}`;
+    if (typeof body === 'string')
+        return `text:${hashBytes(body)}`;
+    if (body instanceof Blob)
+        return `blob:${body.type}:${hashBytes(await body.arrayBuffer())}`;
+    if (body instanceof ArrayBuffer || ArrayBuffer.isView(body))
+        return `bytes:${hashBytes(body)}`;
+    return `stream:${hashBytes(await new Response(body).arrayBuffer())}`;
+}
+async function hashRequestBody(request) {
+    if (!request.body)
+        return 'none';
+    const clone = request.clone();
+    const contentType = clone.headers.get('content-type') ?? '';
+    if (/^multipart\/form-data\b/iu.test(contentType)) {
+        return `form-data:${await hashFormData(await clone.formData())}`;
     }
-    if (headerEntries.length === 0)
-        return '';
-    // Sort alphabetically by key for deterministic ordering
-    headerEntries.sort((a, b) => a[0].localeCompare(b[0]));
-    const headerString = headerEntries
-        .map(([key, value]) => `${key}:${value}`)
-        .join('|');
-    return crypto.createHash('md5').update(headerString).digest('hex');
+    return `bytes:${hashBytes(await clone.arrayBuffer())}`;
+}
+async function createRequestFingerprint(input, method, headers, body) {
+    const bodyHash = body !== undefined && body !== null
+        ? await hashBody(body)
+        : input instanceof Request
+            ? await hashRequestBody(input)
+            : 'none';
+    return hashBytes(JSON.stringify({
+        version: 2,
+        method: method.toUpperCase(),
+        url: input instanceof Request ? input.url : input.toString(),
+        headers: normalizedHeaders(headers),
+        body: bodyHash,
+    }));
 }
-/**
- * Factory function that creates a `fetch` replacement with a caching layer.
- * @param deps - Dependencies including the cache instance, prefix, TTL, and timeout.
- * @returns A function with the same signature as native `fetch`.
- */
+/** Creates a fetch-compatible function backed by deterministic response caching. */
 export function createCachedFetcher(deps) {
     const { cache, prefix = 'http-cache', ttl, timeout, userAgent, fetch: customFetch, shouldCache } = deps;
     const fetchImpl = customFetch ?? fetch;
-    const fetchWithTimeout = async (url, options) => {
-        const headers = new Headers(options?.headers);
-        if (userAgent) {
-            headers.set('User-Agent', userAgent);
+    return async (input, options) => {
+        const { ttl: requestTtl, ...requestOptions } = options ?? {};
+        const method = requestOptions.method ?? (input instanceof Request ? input.method : 'GET');
+        const headers = new Headers(requestOptions.headers ?? (input instanceof Request ? input.headers : undefined));
+        if (userAgent)
+            headers.set('user-agent', userAgent);
+        let body = requestOptions.body;
+        // Tee streaming bodies so fingerprinting never consumes the copy sent over the network.
+        if (body instanceof ReadableStream) {
+            const [transportBody, fingerprintBody] = body.tee();
+            body = fingerprintBody;
+            requestOptions.body = transportBody;
         }
-        const finalOptions = {
-            ...options,
-            headers,
-        };
-        if (!timeout) {
+        const finalOptions = { ...requestOptions, headers };
+        const urlString = input instanceof Request ? input.url : input.toString();
+        if (!cache)
+            return fetchWithTimeout(fetchImpl, input, finalOptions, timeout, urlString);
+        const fingerprint = await createRequestFingerprint(input, method, headers, body);
+        const cacheKey = `${prefix}:v2:${fingerprint}`;
+        const cachedItem = await cache.get(cacheKey);
+        if (cachedItem) {
+            return new CachedResponse(Buffer.from(cachedItem.bodyBase64, 'base64'), {
+                status: cachedItem.status,
+                headers: cachedItem.headers,
+            }, cachedItem.finalUrl);
+        }
+        const response = await fetchWithTimeout(fetchImpl, input, finalOptions, timeout, urlString);
+        if (!response.ok)
+            return response;
+        let isCacheable = true;
+        if (shouldCache) {
             try {
-                return await fetchImpl(url, finalOptions);
+                isCacheable = await shouldCache(response.clone());
             }
             catch (error) {
-                throw error;
+                console.warn('[Cache Check Error] shouldCache threw an error, skipping cache', error);
+                isCacheable = false;
             }
         }
-        const controller = new AbortController();
-        const timeoutId = setTimeout(() => {
-            const urlString = typeof url === 'string' ? url : url.toString();
-            console.log(`[Fetch Timeout] Request timed out after ${timeout}ms for: ${urlString}`);
-            controller.abort();
-        }, timeout);
-        finalOptions.signal = controller.signal;
-        try {
-            const response = await fetchImpl(url, finalOptions);
-            return response;
-        }
-        catch (error) {
-            if (error instanceof Error && error.name === 'AbortError') {
-                const urlString = typeof url === 'string' ? url : url.toString();
-                throw new Error(`Request to ${urlString} timed out after ${timeout}ms`);
-            }
-            throw error;
-        }
-        finally {
-            clearTimeout(timeoutId);
-        }
-    };
-    return async (url, options) => {
-        let method = 'GET';
-        if (options?.method) {
-            method = options.method;
-        }
-        else if (url instanceof Request) {
-            method = url.method;
-        }
-        const urlString = typeof url === 'string' ? url : url.toString();
-        if (!cache) {
-            console.log(`[Cache SKIP] Cache not configured for request to: ${urlString}`);
-            return fetchWithTimeout(url, options);
-        }
-        let cacheKey = `${prefix}:${urlString}`;
-        // Hash body for POST requests
-        if (method.toUpperCase() === 'POST' && options?.body) {
-            let bodyStr = '';
-            if (typeof options.body === 'string') {
-                bodyStr = options.body;
-            }
-            else if (options.body instanceof URLSearchParams) {
-                bodyStr = options.body.toString();
+        else if (response.headers.get('content-type')?.includes('application/json')) {
+            try {
+                const responseBody = await response.clone().json();
+                if (responseBody && typeof responseBody === 'object' && 'error' in responseBody)
+                    isCacheable = false;
             }
-            else {
-                try {
-                    bodyStr = JSON.stringify(options.body);
-                }
-                catch (e) {
-                    bodyStr = 'unserializable';
-                }
+            catch {
+                // A malformed JSON content type does not make an otherwise successful response uncacheable.
             }
-            const bodyHash = crypto.createHash('md5').update(bodyStr).digest('hex');
-            cacheKey += `:body:${bodyHash}`;
         }
-        // Hash all request headers into cache key
-        const headersHash = hashHeaders(options?.headers);
-        if (headersHash) {
-            cacheKey += `:headers:${headersHash}`;
+        if (isCacheable) {
+            const bodyBase64 = Buffer.from(await response.clone().arrayBuffer()).toString('base64');
+            await cache.set(cacheKey, {
+                bodyBase64,
+                headers: Object.fromEntries(response.headers.entries()),
+                status: response.status,
+                finalUrl: response.url,
+            }, requestTtl ?? ttl);
         }
-        // 1. Check the cache
-        const cachedItem = await cache.get(cacheKey);
-        if (cachedItem) {
-            const body = Buffer.from(cachedItem.bodyBase64, 'base64');
-            return new CachedResponse(body, {
-                status: cachedItem.status,
-                headers: cachedItem.headers,
-            }, cachedItem.finalUrl);
-        }
-        // 2. Perform the actual fetch if not in cache
-        const fetchAndCache = async () => {
-            const response = await fetchWithTimeout(url, options);
-            // 3. Store in cache on success
-            if (response.ok) {
-                let isCacheable = true;
-                if (shouldCache) {
-                    const checkClone = response.clone();
-                    try {
-                        isCacheable = await shouldCache(checkClone);
-                    }
-                    catch (e) {
-                        console.warn('[Cache Check Error] shouldCache threw an error, skipping cache', e);
-                        isCacheable = false;
-                    }
-                }
-                else {
-                    const contentType = response.headers.get('content-type');
-                    if (contentType && contentType.includes('application/json')) {
-                        const checkClone = response.clone();
-                        try {
-                            const body = await checkClone.json();
-                            if (body && typeof body === 'object' && 'error' in body) {
-                                console.log(`[Cache SKIP] JSON response contains .error property for: ${urlString}`);
-                                isCacheable = false;
-                            }
-                        }
-                        catch (e) {
-                            // Ignore JSON parse errors, assume cacheable if status is OK
-                        }
-                    }
-                }
-                if (isCacheable) {
-                    const responseClone = response.clone();
-                    const bodyBuffer = await responseClone.arrayBuffer();
-                    const bodyBase64 = Buffer.from(bodyBuffer).toString('base64');
-                    const headers = Object.fromEntries(response.headers.entries());
-                    const itemToCache = {
-                        bodyBase64,
-                        headers,
-                        status: response.status,
-                        finalUrl: response.url,
-                    };
-                    await cache.set(cacheKey, itemToCache, options?.ttl ?? ttl);
-                    console.log(`[Cache SET] for: ${cacheKey}`);
-                }
-            }
-            // 4. Return the original response
-            return response;
-        };
-        return fetchAndCache();
+        return response;
     };
 }
+async function fetchWithTimeout(fetchImpl, input, options, timeout, urlString) {
+    if (!timeout)
+        return fetchImpl(input, options);
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), timeout);
+    const signal = options.signal
+        ? AbortSignal.any([options.signal, controller.signal])
+        : controller.signal;
+    try {
+        return await fetchImpl(input, { ...options, signal });
+    }
+    catch (error) {
+        if (controller.signal.aborted && !options.signal?.aborted) {
+            throw new Error(`Request to ${urlString} timed out after ${timeout}ms`, { cause: error });
+        }
+        throw error;
+    }
+    finally {
+        clearTimeout(timeoutId);
+    }
+}

package/dist/createCachedFetcher.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/createCachedFetcher.spec.js

@@ -0,0 +1,83 @@
+import { describe, expect, it, vi } from 'vitest';
+import OpenAI, { toFile } from 'openai';
+import { createCachedFetcher } from './createCachedFetcher.js';
+function createMemoryCache() {
+    const values = new Map();
+    const cache = {
+        get: async (key) => values.get(key),
+        set: async (key, value) => { values.set(key, value); },
+    };
+    return { cache, values };
+}
+describe('createCachedFetcher', () => {
+    it('caches equivalent multipart requests and distinguishes file contents', async () => {
+        const { cache } = createMemoryCache();
+        const transport = vi.fn(async (_input, init) => {
+            const data = init?.body;
+            return Response.json({ file: await data.get('file').text() });
+        });
+        const cachedFetch = createCachedFetcher({ cache, fetch: transport });
+        const request = (contents) => {
+            const data = new FormData();
+            data.append('model', 'scribe');
+            data.append('file', new Blob([contents], { type: 'audio/flac' }), 'sample.flac');
+            return cachedFetch('https://speech.example/transcriptions', { method: 'POST', body: data });
+        };
+        expect(await (await request('first audio')).json()).toEqual({ file: 'first audio' });
+        expect(await (await request('first audio')).json()).toEqual({ file: 'first audio' });
+        expect(await (await request('different audio')).json()).toEqual({ file: 'different audio' });
+        expect(transport).toHaveBeenCalledTimes(2);
+    });
+    it('fingerprints bodies supplied by Request objects without consuming them', async () => {
+        const { cache } = createMemoryCache();
+        const transport = vi.fn(async (input) => Response.json({ body: await input.text() }));
+        const cachedFetch = createCachedFetcher({ cache, fetch: transport });
+        const makeRequest = (body) => new Request('https://example.test/items', {
+            method: 'PUT', body, headers: { authorization: 'Bearer secret' },
+        });
+        expect(await (await cachedFetch(makeRequest('one'))).json()).toEqual({ body: 'one' });
+        expect(await (await cachedFetch(makeRequest('one'))).json()).toEqual({ body: 'one' });
+        expect(await (await cachedFetch(makeRequest('two'))).json()).toEqual({ body: 'two' });
+        expect(transport).toHaveBeenCalledTimes(2);
+    });
+    it('keeps credentials opaque while separating requests with different headers', async () => {
+        const { cache, values } = createMemoryCache();
+        const transport = vi.fn(async () => Response.json({ ok: true }));
+        const cachedFetch = createCachedFetcher({ cache, prefix: 'test', fetch: transport });
+        await cachedFetch('https://example.test', { headers: { authorization: 'Bearer first-secret' } });
+        await cachedFetch('https://example.test', { headers: { authorization: 'Bearer second-secret' } });
+        expect(transport).toHaveBeenCalledTimes(2);
+        expect([...values.keys()].join('\n')).not.toContain('secret');
+    });
+    it('preserves a streaming request body after hashing it', async () => {
+        const { cache } = createMemoryCache();
+        const transport = vi.fn(async (_input, init) => Response.json({ body: await new Response(init?.body).text() }));
+        const cachedFetch = createCachedFetcher({ cache, fetch: transport });
+        const stream = new Blob(['stream contents']).stream();
+        expect(await (await cachedFetch('https://example.test/upload', {
+            method: 'POST', body: stream, duplex: 'half',
+        })).json()).toEqual({ body: 'stream contents' });
+    });
+    it('caches multipart uploads produced by the OpenAI SDK', async () => {
+        const { cache } = createMemoryCache();
+        const transport = vi.fn(async (input) => {
+            if (input.toString() === 'data:,')
+                return new Response();
+            return Response.json({ words: [] });
+        });
+        const client = new OpenAI({
+            apiKey: 'test-key',
+            fetch: createCachedFetcher({ cache, fetch: transport }),
+        });
+        const transcribe = async (contents) => client.audio.transcriptions.create({
+            file: await toFile(Buffer.from(contents), 'sample.flac'),
+            model: 'whisper-1',
+            response_format: 'verbose_json',
+        });
+        await transcribe('same audio');
+        await transcribe('same audio');
+        await transcribe('different audio');
+        const apiCalls = transport.mock.calls.filter(([input]) => input.toString().startsWith('https://'));
+        expect(apiCalls).toHaveLength(2);
+    });
+});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "llm-fns",
-  "version": "1.0.26",
+  "version": "1.0.27",
   "description": "",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/readme.md

@@ -10,6 +10,28 @@ Designed for power users who need to switch between simple string prompts and co
 npm install openai zod cache-manager p-queue ajv
 ```
 
+## Cached Fetch
+
+`createCachedFetcher` is a fetch-compatible response cache that can be injected into SDKs such as OpenAI:
+
+```typescript
+import OpenAI from 'openai';
+import { createCachedFetcher } from 'llm-fns';
+
+const cachedFetch = createCachedFetcher({
+    cache,
+    fetch: globalThis.fetch,
+    prefix: 'openai',
+    ttl: 24 * 60 * 60 * 1000,
+});
+
+const openai = new OpenAI({ apiKey: process.env.OPENAI_API_KEY, fetch: cachedFetch });
+```
+
+Cache keys fingerprint the effective method, URL, normalized headers, and body. Standard Fetch bodies are supported, including strings, binary data, blobs, streams, `Request` bodies, URL-encoded forms, and multipart `FormData` with file contents. Multipart boundaries and content length are ignored because they are transport details; file bytes, filenames, media types, and form fields remain part of the fingerprint. Header values, including credentials, affect the fingerprint but are never written into the cache key in plain text.
+
+The cache-key format is versioned. Version 1.0.27 intentionally does not read entries created by older releases.
+
 ## Quick Start (Factory)
 
 The `createLlm` factory bundles all functionality (Basic, Retry, Zod) into a single client.