ai-functions 2.1.3 → 2.4.0

package/dist/middleware/cache.js

@@ -0,0 +1,228 @@
+/**
+ * cacheMiddleware — content-addressable cache for `wrapLanguageModel`
+ *
+ * Implements the AI SDK cookbook's local-caching-middleware pattern
+ * (https://ai-sdk.dev/cookbook/node/local-caching-middleware) on top of the
+ * AI SDK 6 `LanguageModelV3Middleware` shape:
+ *
+ * - **Hit derivation:** content-hash of `{ prompt, modelId, responseFormat }`
+ *   so a schema change (responseFormat.type === 'json' carries a `schema`
+ *   JSONSchema7) invalidates the entry. Generation parameters (temperature,
+ *   topP, etc.) are deliberately *not* part of the key for the eval-fixture
+ *   use case — flipping temperature shouldn't blow up a 5x verify-time win.
+ *   Callers who want strict keying should pass a custom `keyHash`.
+ *
+ * - **Stream support:** cached entries store the `LanguageModelV3StreamPart[]`
+ *   array; `wrapStream` replays them via `simulateReadableStream` so consumers
+ *   see the same chunked event sequence on a hit. (`wrapGenerate` is the
+ *   common path; both share the same cache map.)
+ *
+ * - **TTL:** 24h default, configurable via `ttlMs`. Entries past TTL are
+ *   evicted on access (lazy expiry — no background timer).
+ *
+ * - **Pluggable store:** in-memory default (Map-backed); `'disk'` writes to
+ *   a JSON file at `.cache/v3-eval-cache.json` for cross-process fixture
+ *   sharing. Disk reads/writes are best-effort — IO failures fall through
+ *   to the wrapped model.
+ *
+ * - **Env gate:** honors `process.env.V3_EVAL_CACHE`. When unset/empty, the
+ *   middleware short-circuits to a passthrough — useful for production where
+ *   cache hits would be incorrect but the operator wants the same wrap chain.
+ *   Set to `'1'` (or any truthy non-empty string) to enable.
+ *
+ * @packageDocumentation
+ */
+import { simulateReadableStream } from 'ai';
+import { hashKey } from '../cache.js';
+// ============================================================================
+// Stores
+// ============================================================================
+class MemoryStore {
+    map = new Map();
+    get(key) {
+        return this.map.get(key);
+    }
+    set(key, value) {
+        this.map.set(key, value);
+    }
+    delete(key) {
+        this.map.delete(key);
+    }
+}
+/**
+ * Disk-backed store. Best-effort — JSON parse / write errors fall through
+ * silently so a corrupt cache file never blocks an LLM call. The whole map
+ * is rewritten on each `set` (cheap for the eval-fixture use case which is
+ * dominated by reads).
+ */
+class DiskStore {
+    path;
+    cache = null;
+    constructor(path) {
+        this.path = path;
+    }
+    load() {
+        if (this.cache !== null)
+            return this.cache;
+        this.cache = new Map();
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-require-imports
+            const fs = require('fs');
+            if (fs.existsSync(this.path)) {
+                const raw = fs.readFileSync(this.path, 'utf-8');
+                const parsed = JSON.parse(raw);
+                for (const [k, v] of Object.entries(parsed)) {
+                    this.cache.set(k, v);
+                }
+            }
+        }
+        catch {
+            // best-effort
+        }
+        return this.cache;
+    }
+    flush() {
+        if (this.cache === null)
+            return;
+        try {
+            // eslint-disable-next-line @typescript-eslint/no-require-imports
+            const fs = require('fs');
+            // eslint-disable-next-line @typescript-eslint/no-require-imports
+            const path = require('path');
+            const dir = path.dirname(this.path);
+            if (!fs.existsSync(dir)) {
+                fs.mkdirSync(dir, { recursive: true });
+            }
+            const obj = Object.fromEntries(this.cache);
+            fs.writeFileSync(this.path, JSON.stringify(obj), 'utf-8');
+        }
+        catch {
+            // best-effort
+        }
+    }
+    get(key) {
+        return this.load().get(key);
+    }
+    set(key, value) {
+        this.load().set(key, value);
+        this.flush();
+    }
+    delete(key) {
+        this.load().delete(key);
+        this.flush();
+    }
+}
+// ============================================================================
+// Helpers
+// ============================================================================
+const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
+function defaultKeyHash(params, modelId) {
+    // Stable hash of prompt + model + responseFormat (which carries the
+    // schema for object generation). Generation knobs are deliberately
+    // excluded so the eval-fixture cache survives temperature tweaks.
+    return hashKey({
+        prompt: params.prompt,
+        modelId,
+        responseFormat: params.responseFormat,
+    });
+}
+function envGateEnabled() {
+    const v = process.env['V3_EVAL_CACHE'];
+    return typeof v === 'string' && v.length > 0;
+}
+function isExpired(entry, ttlMs) {
+    return Date.now() - entry.createdAt > ttlMs;
+}
+// ============================================================================
+// Middleware
+// ============================================================================
+/**
+ * Build a cache middleware for `wrapLanguageModel`. Wraps `doGenerate` and
+ * `doStream`; on a hit replays the cached payload, on a miss invokes the
+ * downstream model and stores the result.
+ *
+ * Composition note: install **before** budget/trace so cache hits don't
+ * pay the downstream model cost (the trace/budget middleware still see the
+ * payload via the wrapped result they observe in their own `wrapGenerate`).
+ *
+ * @example
+ * ```ts
+ * import { wrapLanguageModel } from 'ai'
+ * import { cacheMiddleware } from 'ai-functions'
+ *
+ * const model = wrapLanguageModel({
+ *   model: openai('gpt-4o'),
+ *   middleware: cacheMiddleware({ store: 'disk', ttlMs: 86_400_000 }),
+ * })
+ * ```
+ */
+export function cacheMiddleware(options = {}) {
+    const ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
+    const keyHash = options.keyHash ?? defaultKeyHash;
+    const store = options.store === undefined || options.store === 'memory'
+        ? new MemoryStore()
+        : options.store === 'disk'
+            ? new DiskStore(options.diskPath ?? '.cache/v3-eval-cache.json')
+            : options.store;
+    const enabled = options.enabled ?? envGateEnabled();
+    return {
+        specificationVersion: 'v3',
+        async wrapGenerate({ doGenerate, params, model }) {
+            if (!enabled)
+                return doGenerate();
+            const key = keyHash(params, model.modelId);
+            const cached = store.get(key);
+            if (cached !== undefined) {
+                if (isExpired(cached, ttlMs)) {
+                    store.delete(key);
+                }
+                else if (cached.generateResult !== undefined) {
+                    return cached.generateResult;
+                }
+            }
+            const result = await doGenerate();
+            store.set(key, { generateResult: result, createdAt: Date.now() });
+            return result;
+        },
+        async wrapStream({ doStream, params, model }) {
+            if (!enabled)
+                return doStream();
+            const key = keyHash(params, model.modelId);
+            const cached = store.get(key);
+            if (cached !== undefined) {
+                if (isExpired(cached, ttlMs)) {
+                    store.delete(key);
+                }
+                else if (cached.streamChunks !== undefined) {
+                    // Replay cached chunks via simulateReadableStream so consumers
+                    // see the same async iteration shape as a fresh call.
+                    const replay = {
+                        stream: simulateReadableStream({
+                            chunks: cached.streamChunks,
+                            initialDelayInMs: 0,
+                            chunkDelayInMs: 0,
+                        }),
+                    };
+                    return replay;
+                }
+            }
+            const result = await doStream();
+            // Tee the stream: forward to caller, accumulate for cache.
+            const chunks = [];
+            const transformedStream = result.stream.pipeThrough(new TransformStream({
+                transform(chunk, controller) {
+                    chunks.push(chunk);
+                    controller.enqueue(chunk);
+                },
+                flush() {
+                    store.set(key, { streamChunks: chunks, createdAt: Date.now() });
+                },
+            }));
+            return {
+                ...result,
+                stream: transformedStream,
+            };
+        },
+    };
+}
+//# sourceMappingURL=cache.js.map

package/dist/middleware/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/middleware/cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAiCG;AAEH,OAAO,EAAE,sBAAsB,EAAE,MAAM,IAAI,CAAA;AAQ3C,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AAsDrC,+EAA+E;AAC/E,SAAS;AACT,+EAA+E;AAE/E,MAAM,WAAW;IACE,GAAG,GAA4B,IAAI,GAAG,EAAE,CAAA;IACzD,GAAG,CAAC,GAAW;QACb,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAC1B,CAAC;IACD,GAAG,CAAC,GAAW,EAAE,KAAiB;QAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IAC1B,CAAC;IACD,MAAM,CAAC,GAAW;QAChB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC;CACF;AAED;;;;;GAKG;AACH,MAAM,SAAS;IACI,IAAI,CAAQ;IACrB,KAAK,GAAmC,IAAI,CAAA;IAEpD,YAAY,IAAY;QACtB,IAAI,CAAC,IAAI,GAAG,IAAI,CAAA;IAClB,CAAC;IAEO,IAAI;QACV,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC,KAAK,CAAA;QAC1C,IAAI,CAAC,KAAK,GAAG,IAAI,GAAG,EAAE,CAAA;QACtB,IAAI,CAAC;YACH,iEAAiE;YACjE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAwB,CAAA;YAC/C,IAAI,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;gBAC/C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAA+B,CAAA;gBAC5D,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBAC5C,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;gBACtB,CAAC;YACH,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,cAAc;QAChB,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAA;IACnB,CAAC;IAEO,KAAK;QACX,IAAI,IAAI,CAAC,KAAK,KAAK,IAAI;YAAE,OAAM;QAC/B,IAAI,CAAC;YACH,iEAAiE;YACjE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAwB,CAAA;YAC/C,iEAAiE;YACjE,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAA0B,CAAA;YACrD,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACnC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACxB,EAAE,CAAC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;YACxC,CAAC;YACD,MAAM,GAAG,GAAG,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC1C,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,OAAO,CAAC,CAAA;QAC3D,CAAC;QAAC,MAAM,CAAC;YACP,cAAc;QAChB,CAAC;IACH,CAAC;IAED,GAAG,CAAC,GAAW;QACb,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAC7B,CAAC;IAED,GAAG,CAAC,GAAW,EAAE,KAAiB;QAChC,IAAI,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QAC3B,IAAI,CAAC,KAAK,EAAE,CAAA;IACd,CAAC;IAED,MAAM,CAAC,GAAW;QAChB,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACvB,IAAI,CAAC,KAAK,EAAE,CAAA;IACd,CAAC;CACF;AAED,+EAA+E;AAC/E,UAAU;AACV,+EAA+E;AAE/E,MAAM,cAAc,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAE1C,SAAS,cAAc,CAAC,MAAkC,EAAE,OAAe;IACzE,oEAAoE;IACpE,mEAAmE;IACnE,kEAAkE;IAClE,OAAO,OAAO,CAAC;QACb,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO;QACP,cAAc,EAAE,MAAM,CAAC,cAAc;KACtC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,cAAc;IACrB,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IACtC,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;AAC9C,CAAC;AAED,SAAS,SAAS,CAAC,KAAiB,EAAE,KAAa;IACjD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,GAAG,KAAK,CAAA;AAC7C,CAAC;AAED,+EAA+E;AAC/E,aAAa;AACb,+EAA+E;AAE/E;;;;;;;;;;;;;;;;;;;GAmBG;AACH,MAAM,UAAU,eAAe,CAAC,UAAkC,EAAE;IAClE,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,cAAc,CAAA;IAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,cAAc,CAAA;IACjD,MAAM,KAAK,GACT,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ;QACvD,CAAC,CAAC,IAAI,WAAW,EAAE;QACnB,CAAC,CAAC,OAAO,CAAC,KAAK,KAAK,MAAM;YAC1B,CAAC,CAAC,IAAI,SAAS,CAAC,OAAO,CAAC,QAAQ,IAAI,2BAA2B,CAAC;YAChE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAA;IACnB,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,cAAc,EAAE,CAAA;IAEnD,OAAO;QACL,oBAAoB,EAAE,IAAI;QAC1B,KAAK,CAAC,YAAY,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,KAAK,EAAE;YAC9C,IAAI,CAAC,OAAO;gBAAE,OAAO,UAAU,EAAE,CAAA;YACjC,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;YAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAC7B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;oBAC7B,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBACnB,CAAC;qBAAM,IAAI,MAAM,CAAC,cAAc,KAAK,SAAS,EAAE,CAAC;oBAC/C,OAAO,MAAM,CAAC,cAAc,CAAA;gBAC9B,CAAC;YACH,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,UAAU,EAAE,CAAA;YACjC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,cAAc,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;YACjE,OAAO,MAAM,CAAA;QACf,CAAC;QACD,KAAK,CAAC,UAAU,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE;YAC1C,IAAI,CAAC,OAAO;gBAAE,OAAO,QAAQ,EAAE,CAAA;YAC/B,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;YAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAC7B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;oBAC7B,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBACnB,CAAC;qBAAM,IAAI,MAAM,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;oBAC7C,+DAA+D;oBAC/D,sDAAsD;oBACtD,MAAM,MAAM,GAAgC;wBAC1C,MAAM,EAAE,sBAAsB,CAA4B;4BACxD,MAAM,EAAE,MAAM,CAAC,YAAY;4BAC3B,gBAAgB,EAAE,CAAC;4BACnB,cAAc,EAAE,CAAC;yBAClB,CAAC;qBACH,CAAA;oBACD,OAAO,MAAM,CAAA;gBACf,CAAC;YACH,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,QAAQ,EAAE,CAAA;YAC/B,2DAA2D;YAC3D,MAAM,MAAM,GAAgC,EAAE,CAAA;YAC9C,MAAM,iBAAiB,GAAG,MAAM,CAAC,MAAM,CAAC,WAAW,CACjD,IAAI,eAAe,CAAuD;gBACxE,SAAS,CAAC,KAAK,EAAE,UAAU;oBACzB,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;oBAClB,UAAU,CAAC,OAAO,CAAC,KAAK,CAAC,CAAA;gBAC3B,CAAC;gBACD,KAAK;oBACH,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,YAAY,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;gBACjE,CAAC;aACF,CAAC,CACH,CAAA;YACD,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,iBAAiB;aAC1B,CAAA;QACH,CAAC;KACF,CAAA;AACH,CAAC"}

package/dist/middleware/embed-cache.d.ts

@@ -0,0 +1,99 @@
+/**
+ * embeddingCacheMiddleware — content-addressable cache for `wrapEmbeddingModel`
+ *
+ * Embedding-side analogue of {@link cacheMiddleware}. Wraps `doEmbed` and
+ * caches the resulting embeddings keyed on
+ * `{ values, modelId, providerOptions }` so a re-embed of the same value
+ * batch with the same model returns the cached vectors without hitting the
+ * provider.
+ *
+ * **Why a separate middleware instead of reusing `cacheMiddleware`?**
+ * AI SDK 6 splits language-model and embedding-model surfaces:
+ * `LanguageModelV3Middleware` exposes `wrapGenerate` / `wrapStream` against
+ * `LanguageModelV3CallOptions`, while `EmbeddingModelV3Middleware` exposes
+ * `wrapEmbed` against `EmbeddingModelV3CallOptions`. The cache shape
+ * (per-value vector vs. per-prompt completion payload) is also different —
+ * embeddings cache batched arrays, generations cache single result objects.
+ *
+ * - **Hit derivation:** stable hash of `{ values, modelId, providerOptions }`.
+ *   `values` is the array as-passed (caller can pre-normalise if they want
+ *   case/whitespace insensitivity). Generation knobs don't apply.
+ *
+ * - **Batch semantics:** the cache key is the *whole* batch. A subset hit
+ *   doesn't trigger a partial-fill — that's a more invasive shape change
+ *   (the legacy `EmbeddingCache.getMany` did per-text caching, but it was
+ *   only used in the example and added 100+ LOC of bookkeeping). Callers
+ *   that want per-text caching should use stable per-text batches.
+ *
+ * - **TTL:** 24h default, configurable. Lazy expiry on access.
+ *
+ * - **Pluggable store:** in-memory default (Map-backed); custom store
+ *   honored as-is. Disk persistence is intentionally not provided here —
+ *   embedding payloads (large `number[][]`) make on-disk JSON a bad fit;
+ *   callers who want it should pass a custom store.
+ *
+ * - **Env gate:** honors `process.env.V3_EVAL_CACHE` for parity with
+ *   `cacheMiddleware`. Override via the `enabled` option.
+ *
+ * @packageDocumentation
+ */
+import type { EmbeddingModelV3CallOptions, EmbeddingModelV3Embedding, EmbeddingModelV3Middleware, SharedV3Warning } from '@ai-sdk/provider';
+/** Cached embedding payload. */
+interface EmbedCacheEntry {
+    /** The embedding vectors returned for the cached batch. */
+    embeddings: Array<EmbeddingModelV3Embedding>;
+    /** Provider warnings carried alongside the cached batch. */
+    warnings: Array<SharedV3Warning>;
+    /** Insert epoch ms — drives TTL eviction. */
+    createdAt: number;
+}
+/** Pluggable cache store for embedding results. */
+export interface EmbedCacheMiddlewareStore {
+    get(key: string): EmbedCacheEntry | undefined;
+    set(key: string, value: EmbedCacheEntry): void;
+    delete(key: string): void;
+}
+/** Options for {@link embeddingCacheMiddleware}. */
+export interface EmbedCacheMiddlewareOptions {
+    /**
+     * Cache backend. `'memory'` uses a process-local Map. A custom
+     * {@link EmbedCacheMiddlewareStore} can be passed instead.
+     *
+     * @default 'memory'
+     */
+    store?: 'memory' | EmbedCacheMiddlewareStore;
+    /**
+     * TTL in milliseconds. Entries older than `ttlMs` are evicted on access.
+     *
+     * @default 86_400_000 (24h)
+     */
+    ttlMs?: number;
+    /**
+     * Custom hash function for cache keys. Defaults to a stable hash of
+     * `{ values, modelId, providerOptions }`.
+     */
+    keyHash?: (params: EmbeddingModelV3CallOptions, modelId: string) => string;
+    /**
+     * Optional override for the env gate. When `false`, the middleware acts
+     * as a passthrough regardless of `V3_EVAL_CACHE`. When `true`, always
+     * caches. Defaults to `process.env.V3_EVAL_CACHE` truthy-check.
+     */
+    enabled?: boolean;
+}
+/**
+ * Build an embedding-cache middleware for `wrapEmbeddingModel`.
+ *
+ * @example
+ * ```ts
+ * import { wrapEmbeddingModel } from 'ai'
+ * import { embeddingCacheMiddleware } from 'ai-functions'
+ *
+ * const model = wrapEmbeddingModel({
+ *   model: openai.embedding('text-embedding-3-small'),
+ *   middleware: embeddingCacheMiddleware({ ttlMs: 86_400_000 }),
+ * })
+ * ```
+ */
+export declare function embeddingCacheMiddleware(options?: EmbedCacheMiddlewareOptions): EmbeddingModelV3Middleware;
+export {};
+//# sourceMappingURL=embed-cache.d.ts.map

package/dist/middleware/embed-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embed-cache.d.ts","sourceRoot":"","sources":["../../src/middleware/embed-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AAEH,OAAO,KAAK,EACV,2BAA2B,EAC3B,yBAAyB,EACzB,0BAA0B,EAE1B,eAAe,EAChB,MAAM,kBAAkB,CAAA;AAOzB,gCAAgC;AAChC,UAAU,eAAe;IACvB,2DAA2D;IAC3D,UAAU,EAAE,KAAK,CAAC,yBAAyB,CAAC,CAAA;IAC5C,4DAA4D;IAC5D,QAAQ,EAAE,KAAK,CAAC,eAAe,CAAC,CAAA;IAChC,6CAA6C;IAC7C,SAAS,EAAE,MAAM,CAAA;CAClB;AAED,mDAAmD;AACnD,MAAM,WAAW,yBAAyB;IACxC,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,GAAG,SAAS,CAAA;IAC7C,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,eAAe,GAAG,IAAI,CAAA;IAC9C,MAAM,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,CAAA;CAC1B;AAED,oDAAoD;AACpD,MAAM,WAAW,2BAA2B;IAC1C;;;;;OAKG;IACH,KAAK,CAAC,EAAE,QAAQ,GAAG,yBAAyB,CAAA;IAC5C;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;IACd;;;OAGG;IACH,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,2BAA2B,EAAE,OAAO,EAAE,MAAM,KAAK,MAAM,CAAA;IAC1E;;;;OAIG;IACH,OAAO,CAAC,EAAE,OAAO,CAAA;CAClB;AA8CD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,wBAAwB,CACtC,OAAO,GAAE,2BAAgC,GACxC,0BAA0B,CAoC5B"}

package/dist/middleware/embed-cache.js

@@ -0,0 +1,128 @@
+/**
+ * embeddingCacheMiddleware — content-addressable cache for `wrapEmbeddingModel`
+ *
+ * Embedding-side analogue of {@link cacheMiddleware}. Wraps `doEmbed` and
+ * caches the resulting embeddings keyed on
+ * `{ values, modelId, providerOptions }` so a re-embed of the same value
+ * batch with the same model returns the cached vectors without hitting the
+ * provider.
+ *
+ * **Why a separate middleware instead of reusing `cacheMiddleware`?**
+ * AI SDK 6 splits language-model and embedding-model surfaces:
+ * `LanguageModelV3Middleware` exposes `wrapGenerate` / `wrapStream` against
+ * `LanguageModelV3CallOptions`, while `EmbeddingModelV3Middleware` exposes
+ * `wrapEmbed` against `EmbeddingModelV3CallOptions`. The cache shape
+ * (per-value vector vs. per-prompt completion payload) is also different —
+ * embeddings cache batched arrays, generations cache single result objects.
+ *
+ * - **Hit derivation:** stable hash of `{ values, modelId, providerOptions }`.
+ *   `values` is the array as-passed (caller can pre-normalise if they want
+ *   case/whitespace insensitivity). Generation knobs don't apply.
+ *
+ * - **Batch semantics:** the cache key is the *whole* batch. A subset hit
+ *   doesn't trigger a partial-fill — that's a more invasive shape change
+ *   (the legacy `EmbeddingCache.getMany` did per-text caching, but it was
+ *   only used in the example and added 100+ LOC of bookkeeping). Callers
+ *   that want per-text caching should use stable per-text batches.
+ *
+ * - **TTL:** 24h default, configurable. Lazy expiry on access.
+ *
+ * - **Pluggable store:** in-memory default (Map-backed); custom store
+ *   honored as-is. Disk persistence is intentionally not provided here —
+ *   embedding payloads (large `number[][]`) make on-disk JSON a bad fit;
+ *   callers who want it should pass a custom store.
+ *
+ * - **Env gate:** honors `process.env.V3_EVAL_CACHE` for parity with
+ *   `cacheMiddleware`. Override via the `enabled` option.
+ *
+ * @packageDocumentation
+ */
+import { hashKey } from '../cache.js';
+// ============================================================================
+// Stores
+// ============================================================================
+class MemoryStore {
+    map = new Map();
+    get(key) {
+        return this.map.get(key);
+    }
+    set(key, value) {
+        this.map.set(key, value);
+    }
+    delete(key) {
+        this.map.delete(key);
+    }
+}
+// ============================================================================
+// Helpers
+// ============================================================================
+const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
+function defaultKeyHash(params, modelId) {
+    return hashKey({
+        values: params.values,
+        modelId,
+        providerOptions: params.providerOptions,
+    });
+}
+function envGateEnabled() {
+    const v = process.env['V3_EVAL_CACHE'];
+    return typeof v === 'string' && v.length > 0;
+}
+function isExpired(entry, ttlMs) {
+    return Date.now() - entry.createdAt > ttlMs;
+}
+// ============================================================================
+// Middleware
+// ============================================================================
+/**
+ * Build an embedding-cache middleware for `wrapEmbeddingModel`.
+ *
+ * @example
+ * ```ts
+ * import { wrapEmbeddingModel } from 'ai'
+ * import { embeddingCacheMiddleware } from 'ai-functions'
+ *
+ * const model = wrapEmbeddingModel({
+ *   model: openai.embedding('text-embedding-3-small'),
+ *   middleware: embeddingCacheMiddleware({ ttlMs: 86_400_000 }),
+ * })
+ * ```
+ */
+export function embeddingCacheMiddleware(options = {}) {
+    const ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
+    const keyHash = options.keyHash ?? defaultKeyHash;
+    const store = options.store === undefined || options.store === 'memory' ? new MemoryStore() : options.store;
+    const enabled = options.enabled ?? envGateEnabled();
+    return {
+        specificationVersion: 'v3',
+        async wrapEmbed({ doEmbed, params, model }) {
+            if (!enabled)
+                return doEmbed();
+            const key = keyHash(params, model.modelId);
+            const cached = store.get(key);
+            if (cached !== undefined) {
+                if (isExpired(cached, ttlMs)) {
+                    store.delete(key);
+                }
+                else {
+                    // Replay shape matches EmbeddingModelV3Result. Provider-side
+                    // metadata (response headers, body, usage) is intentionally absent
+                    // on a hit — callers reading those should disable the cache.
+                    const replay = {
+                        embeddings: cached.embeddings,
+                        warnings: cached.warnings,
+                    };
+                    return replay;
+                }
+            }
+            const result = await doEmbed();
+            store.set(key, {
+                embeddings: result.embeddings,
+                warnings: result.warnings,
+                createdAt: Date.now(),
+            });
+            return result;
+        },
+    };
+}
+//# sourceMappingURL=embed-cache.js.map

package/dist/middleware/embed-cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embed-cache.js","sourceRoot":"","sources":["../../src/middleware/embed-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsCG;AASH,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAA;AAmDrC,+EAA+E;AAC/E,SAAS;AACT,+EAA+E;AAE/E,MAAM,WAAW;IACE,GAAG,GAAiC,IAAI,GAAG,EAAE,CAAA;IAC9D,GAAG,CAAC,GAAW;QACb,OAAO,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;IAC1B,CAAC;IACD,GAAG,CAAC,GAAW,EAAE,KAAsB;QACrC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;IAC1B,CAAC;IACD,MAAM,CAAC,GAAW;QAChB,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC;CACF;AAED,+EAA+E;AAC/E,UAAU;AACV,+EAA+E;AAE/E,MAAM,cAAc,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAE1C,SAAS,cAAc,CAAC,MAAmC,EAAE,OAAe;IAC1E,OAAO,OAAO,CAAC;QACb,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,OAAO;QACP,eAAe,EAAE,MAAM,CAAC,eAAe;KACxC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,cAAc;IACrB,MAAM,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAA;IACtC,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAA;AAC9C,CAAC;AAED,SAAS,SAAS,CAAC,KAAsB,EAAE,KAAa;IACtD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,SAAS,GAAG,KAAK,CAAA;AAC7C,CAAC;AAED,+EAA+E;AAC/E,aAAa;AACb,+EAA+E;AAE/E;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,wBAAwB,CACtC,UAAuC,EAAE;IAEzC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,cAAc,CAAA;IAC7C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,cAAc,CAAA;IACjD,MAAM,KAAK,GACT,OAAO,CAAC,KAAK,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,IAAI,WAAW,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAA;IAC/F,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,cAAc,EAAE,CAAA;IAEnD,OAAO;QACL,oBAAoB,EAAE,IAAI;QAC1B,KAAK,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE;YACxC,IAAI,CAAC,OAAO;gBAAE,OAAO,OAAO,EAAE,CAAA;YAC9B,MAAM,GAAG,GAAG,OAAO,CAAC,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,CAAA;YAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YAC7B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,SAAS,CAAC,MAAM,EAAE,KAAK,CAAC,EAAE,CAAC;oBAC7B,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBACnB,CAAC;qBAAM,CAAC;oBACN,6DAA6D;oBAC7D,mEAAmE;oBACnE,6DAA6D;oBAC7D,MAAM,MAAM,GAA2B;wBACrC,UAAU,EAAE,MAAM,CAAC,UAAU;wBAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;qBAC1B,CAAA;oBACD,OAAO,MAAM,CAAA;gBACf,CAAC;YACH,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,OAAO,EAAE,CAAA;YAC9B,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE;gBACb,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC,CAAA;YACF,OAAO,MAAM,CAAA;QACf,CAAC;KACF,CAAA;AACH,CAAC"}

package/dist/middleware/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Middleware barrel — composable AI SDK 6 `LanguageModelV3Middleware`
+ * primitives for `wrapLanguageModel`.
+ *
+ * @packageDocumentation
+ */
+export { cacheMiddleware, type CacheMiddlewareOptions, type CacheMiddlewareStore } from './cache.js';
+export { embeddingCacheMiddleware, type EmbedCacheMiddlewareOptions, type EmbedCacheMiddlewareStore, } from './embed-cache.js';
+export { budgetMiddleware, type BudgetMiddlewareOptions, type PricingOverlay } from './budget.js';
+export { traceMiddleware, type TraceEvent, type TraceEventKind, type TraceMiddlewareOptions, } from './trace.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/middleware/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/middleware/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAAE,KAAK,sBAAsB,EAAE,KAAK,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAEpG,OAAO,EACL,wBAAwB,EACxB,KAAK,2BAA2B,EAChC,KAAK,yBAAyB,GAC/B,MAAM,kBAAkB,CAAA;AAEzB,OAAO,EAAE,gBAAgB,EAAE,KAAK,uBAAuB,EAAE,KAAK,cAAc,EAAE,MAAM,aAAa,CAAA;AAEjG,OAAO,EACL,eAAe,EACf,KAAK,UAAU,EACf,KAAK,cAAc,EACnB,KAAK,sBAAsB,GAC5B,MAAM,YAAY,CAAA"}

package/dist/middleware/index.js

@@ -0,0 +1,11 @@
+/**
+ * Middleware barrel — composable AI SDK 6 `LanguageModelV3Middleware`
+ * primitives for `wrapLanguageModel`.
+ *
+ * @packageDocumentation
+ */
+export { cacheMiddleware } from './cache.js';
+export { embeddingCacheMiddleware, } from './embed-cache.js';
+export { budgetMiddleware } from './budget.js';
+export { traceMiddleware, } from './trace.js';
+//# sourceMappingURL=index.js.map

package/dist/middleware/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/middleware/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,eAAe,EAA0D,MAAM,YAAY,CAAA;AAEpG,OAAO,EACL,wBAAwB,GAGzB,MAAM,kBAAkB,CAAA;AAEzB,OAAO,EAAE,gBAAgB,EAAqD,MAAM,aAAa,CAAA;AAEjG,OAAO,EACL,eAAe,GAIhB,MAAM,YAAY,CAAA"}

package/dist/middleware/trace.d.ts

@@ -0,0 +1,103 @@
+/**
+ * traceMiddleware — emit per-call trace events for `wrapLanguageModel`
+ *
+ * Wraps `doGenerate` / `doStream` and emits a {@link TraceEvent} on every
+ * completion. The sink is opaque (caller supplies `emit`) so this primitive
+ * works equally well piping into:
+ *
+ *   - the v3 cascade-walker InvocationEvent stream (round 16+ work to add
+ *     `'persona-trace'` / `'cascade-trace'` to the union),
+ *   - an {@link import('../eval-log/index.js').EvalLogStore} for fixture
+ *     replay,
+ *   - OpenTelemetry / Datadog / Honeycomb adapters that map the event into
+ *     a span.
+ *
+ * **Emit-error tolerance:** if the supplied `emit` throws, we *swallow* the
+ * error (with a one-time `console.warn`) so a flaky trace sink can never
+ * break the wrapped LLM call. This matches the Evalite v0.19 trace
+ * middleware behaviour.
+ *
+ * Composition note: install **last** so the event sees the final outcome
+ * (post-cache, post-budget). The event's `costUsd` field is best-effort —
+ * the trace middleware doesn't have direct access to the budget tracker, so
+ * the caller can pass a `getCostUsd` resolver if they want costs in the
+ * event payload.
+ *
+ * @packageDocumentation
+ */
+import type { LanguageModelV3Middleware, LanguageModelV3Usage } from '@ai-sdk/provider';
+/**
+ * Discriminator for the originating call site. Callers inject this via the
+ * `kind` option so a single sink can fan events into different downstream
+ * streams (persona panel vs. cascade walker vs. ad-hoc test).
+ */
+export type TraceEventKind = 'persona-trace' | 'cascade-trace' | 'eval-trace' | string;
+/**
+ * Trace event payload emitted on every wrapped call completion.
+ *
+ * Field design notes:
+ *   - `prompt` / `response` are stringified for cheap downstream storage
+ *     (the structured `LanguageModelV3Prompt` / `LanguageModelV3Content[]`
+ *     shapes are intentionally flattened).
+ *   - `usage` is the raw V3 shape (with the cache breakdown) — the
+ *     EvalLogStore consumer flattens it into total counts.
+ *   - `costUsd` is optional because the trace middleware doesn't compute
+ *     cost itself; callers either pass a resolver or compute downstream
+ *     from `usage`.
+ */
+export interface TraceEvent {
+    kind: TraceEventKind;
+    model: string;
+    prompt: string;
+    response: string;
+    usage: LanguageModelV3Usage | undefined;
+    costUsd?: number;
+    durationMs: number;
+    /** Optional caller-supplied tags for downstream filtering. */
+    tags?: Record<string, string>;
+}
+/** Options for {@link traceMiddleware}. */
+export interface TraceMiddlewareOptions {
+    /**
+     * Opaque sink. Errors thrown from `emit` are swallowed (with a one-time
+     * `console.warn`) so a flaky sink never breaks the wrapped LLM call.
+     */
+    emit: (event: TraceEvent) => void | Promise<void>;
+    /**
+     * Discriminator threaded into the event's `kind` field. Defaults to
+     * `'eval-trace'`.
+     */
+    kind?: TraceEventKind;
+    /**
+     * Optional cost resolver. When supplied, called with the V3 usage shape
+     * and the modelId; result is set on `event.costUsd`. Useful when the
+     * caller has a side-channel pricing table (the budgetMiddleware's
+     * tracker) and wants costs in the trace event itself.
+     */
+    getCostUsd?: (modelId: string, usage: LanguageModelV3Usage | undefined) => number;
+    /** Optional caller-supplied tags merged into every emitted event. */
+    tags?: Record<string, string>;
+}
+/**
+ * Build a trace middleware for `wrapLanguageModel`. Emits a
+ * {@link TraceEvent} on every successful `doGenerate` / `doStream`
+ * completion. Errors from `emit` are swallowed (one-time warn) so a flaky
+ * trace sink can never break the wrapped LLM call.
+ *
+ * @example
+ * ```ts
+ * import { wrapLanguageModel } from 'ai'
+ * import { traceMiddleware, getEvalLogStore } from 'ai-functions'
+ *
+ * const store = getEvalLogStore()
+ * const model = wrapLanguageModel({
+ *   model: openai('gpt-4o'),
+ *   middleware: traceMiddleware({
+ *     kind: 'cascade-trace',
+ *     emit: (event) => store.record({ ...event, costUsd: event.costUsd ?? 0 }),
+ *   }),
+ * })
+ * ```
+ */
+export declare function traceMiddleware(options: TraceMiddlewareOptions): LanguageModelV3Middleware;
+//# sourceMappingURL=trace.d.ts.map

package/dist/middleware/trace.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"trace.d.ts","sourceRoot":"","sources":["../../src/middleware/trace.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAGV,yBAAyB,EAGzB,oBAAoB,EACrB,MAAM,kBAAkB,CAAA;AAMzB;;;;GAIG;AACH,MAAM,MAAM,cAAc,GAAG,eAAe,GAAG,eAAe,GAAG,YAAY,GAAG,MAAM,CAAA;AAEtF;;;;;;;;;;;;GAYG;AACH,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,cAAc,CAAA;IACpB,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,CAAA;IACd,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,oBAAoB,GAAG,SAAS,CAAA;IACvC,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,8DAA8D;IAC9D,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC9B;AAED,2CAA2C;AAC3C,MAAM,WAAW,sBAAsB;IACrC;;;OAGG;IACH,IAAI,EAAE,CAAC,KAAK,EAAE,UAAU,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;IACjD;;;OAGG;IACH,IAAI,CAAC,EAAE,cAAc,CAAA;IACrB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,oBAAoB,GAAG,SAAS,KAAK,MAAM,CAAA;IACjF,qEAAqE;IACrE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CAC9B;AAwED;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,sBAAsB,GAAG,yBAAyB,CA4D1F"}