better-mastra 0.0.0-stage → 0.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Hans Kristoffer Hansen
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,46 @@
-# Temporary Holding Version
+# better-mastra
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+A small layer on Mastra and the AI SDK: model registry, chat turns, resumable streams and chat clients for React and Vue.
+
+Work in progress.
+
+## Resumable streams (`better-mastra/resumable`)
+
+Opt-in. A dropped connection no longer aborts the turn: the client reconnects with AI SDK `reconnectToStream` (`useChat({ resume: true })`) and gets a full replay of the current assistant message plus the live tail (`resumeStreamIsReplay: true`, the `DefaultChatTransport` default).
+
+```ts
+import { RedisClient } from 'bun'
+import { bunRedis, createResumableStreams } from 'better-mastra/resumable'
+
+const resumable = createResumableStreams({ redis: bunRedis(new RedisClient(process.env.REDIS_URL)) })
+
+// POST /chat/:threadId — do not tie the run to request.signal
+const abort = new AbortController()
+const stream = await resumable.start({ threadKey: threadId, stream: uiMessageStream, abort })
+
+// GET /chat/:threadId/stream
+const resumed = await resumable.reconnect(threadId, request.signal)
+if (!resumed) return new Response(null, { status: 204 })
+
+// POST /chat/:threadId/stop — works from any replica
+await resumable.stop(threadId)
+
+// shutdown: waits for active turns (drainTimeoutMs)
+await resumable.close()
+```
+
+Use `ioRedis(client)` for ioredis. Both adapters open extra connections with `duplicate()`: one subscriber per replica, and one per cross-replica reader (for `XREAD BLOCK`).
+
+How it works: chunks are buffered in memory (consecutive `text-delta`s merged) and fanned out to subscribers. A per-thread locator `{ streamId, owner }` in Redis says which replica owns the turn. Reconnecting to the owner is served from memory. Reconnecting to another replica publishes a spill request on the owner's control channel; the owner writes its buffer in one `XADD`, then mirrors new chunks in 50 ms batches (first one immediate) until a terminal marker, and the reader follows with `XREAD BLOCK`.
+
+Redis ops for a ~1,500-chunk turn:
+
+| Design | Redis ops |
+|---|---|
+| Mastra durable agents (`CachingPubSub`: cache write + publish per chunk) | ~3,000–4,500 |
+| Coalesced SETEX + PUBLISH (hellu support-turn) | ~600 |
+| This, no reconnect | 2 (locator SET, final PEXPIRE) + 1 PEXPIRE per 10 s of turn |
+| This, same-replica reconnect | +0 for a running turn, +1 GET for a finished one |
+| This, cross-replica reconnect | +1 PUBLISH, +1 XADD and EXPIRE per 50 ms batch, reader XREADs |
+
+Failure behaviour: a buffer over `maxBufferBytes` (2 MB) keeps streaming but new reconnects get 204. If the owner crashes, its locator expires and the cross-replica reader ends with an `error` chunk after `idleTimeoutMs`; the client reloads the thread from Mastra memory. If Redis is down, turns still run and same-replica reconnects work; cross-replica reconnects return 204. Finished turns stay resumable for `graceMs` (30 s).

package/dist/embeddings/bounded-promise-cache.d.ts

@@ -0,0 +1,54 @@
+type BoundedPromiseCacheOptions<Key> = {
+    /** How long a value is served without triggering any reload. */
+    freshMs: number;
+    /**
+     * Maximum age (from load) a value may be served at all. Between `freshMs`
+     * and `staleMs` the stale value is returned while a rebuild runs in the
+     * background; past `staleMs` the caller pays the reload inline. Omitted (or
+     * `<= freshMs`) makes this a plain TTL cache.
+     */
+    staleMs?: number;
+    maxEntries: number;
+    now?: () => number;
+    /**
+     * Fired when a caller has to wait for a reload because the entry is past
+     * `staleMs` - the case a voice first-audio path must never hit.
+     */
+    onBlockingReload?: (key: Key) => void;
+};
+/**
+ * Small in-process promise cache: bounded, shares work already in flight,
+ * evicts a rejected first load so the next caller retries, and - when
+ * `staleMs > 0` - serves the stale value while it revalidates in the
+ * background. A failed background rebuild keeps serving the stale value
+ * until the stale window closes.
+ */
+export declare class BoundedPromiseCache<Key, Value> {
+    private readonly options;
+    private readonly entries;
+    private now;
+    constructor(options: BoundedPromiseCacheOptions<Key>);
+    /** Test seam: pin the clock. `null` restores `Date.now`. */
+    setNowForTests(now: (() => number) | null): void;
+    /** Reports a fresh OR stale entry - anything `getOrLoad` would return without a blocking reload. */
+    has(key: Key): boolean;
+    /** Drop every entry. Test seam for module-level caches. */
+    clear(): void;
+    delete(key: Key): void;
+    deleteWhere(predicate: (key: Key) => boolean): void;
+    getOrLoad(key: Key, load: () => Promise<Value>, 
+    /**
+     * `serveStale: false` takes the reload inline once the fresh window has
+     * passed. Serve-stale exists for voice first-audio; on a written channel
+     * it only meant an owner's settings change took up to `staleMs` to show
+     * up, because invalidation never leaves the process that served it.
+     */
+    options?: {
+        serveStale?: boolean;
+    }): Promise<Value>;
+    private scheduleRefresh;
+    private entryFor;
+    private evictIfNeeded;
+}
+export {};
+//# sourceMappingURL=bounded-promise-cache.d.ts.map

package/dist/embeddings/bounded-promise-cache.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bounded-promise-cache.d.ts","sourceRoot":"","sources":["../../src/embeddings/bounded-promise-cache.ts"],"names":[],"mappings":"AAQA,KAAK,0BAA0B,CAAC,GAAG,IAAI;IACtC,gEAAgE;IAChE,OAAO,EAAE,MAAM,CAAA;IACf;;;;;OAKG;IACH,OAAO,CAAC,EAAE,MAAM,CAAA;IAChB,UAAU,EAAE,MAAM,CAAA;IAClB,GAAG,CAAC,EAAE,MAAM,MAAM,CAAA;IAClB;;;OAGG;IACH,gBAAgB,CAAC,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,IAAI,CAAA;CACrC,CAAA;AAED;;;;;;GAMG;AACH,qBAAa,mBAAmB,CAAC,GAAG,EAAE,KAAK;IAI9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAHpC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2C;IACnE,OAAO,CAAC,GAAG,CAAc;gBAEI,OAAO,EAAE,0BAA0B,CAAC,GAAG,CAAC;IAIrE,4DAA4D;IAC5D,cAAc,CAAC,GAAG,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,IAAI;IAIzC,oGAAoG;IACpG,GAAG,CAAC,GAAG,EAAE,GAAG;IAKZ,2DAA2D;IAC3D,KAAK;IAIL,MAAM,CAAC,GAAG,EAAE,GAAG;IAIf,WAAW,CAAC,SAAS,EAAE,CAAC,GAAG,EAAE,GAAG,KAAK,OAAO;IAM5C,SAAS,CACR,GAAG,EAAE,GAAG,EACR,IAAI,EAAE,MAAM,OAAO,CAAC,KAAK,CAAC;IAC1B;;;;;OAKG;IACH,OAAO,CAAC,EAAE;QAAE,UAAU,CAAC,EAAE,OAAO,CAAA;KAAE,GAChC,OAAO,CAAC,KAAK,CAAC;IA4BjB,OAAO,CAAC,eAAe;IAmBvB,OAAO,CAAC,QAAQ;IAahB,OAAO,CAAC,aAAa;CASrB"}

package/dist/embeddings/bounded-promise-cache.js

@@ -0,0 +1,113 @@
+/**
+ * Small in-process promise cache: bounded, shares work already in flight,
+ * evicts a rejected first load so the next caller retries, and - when
+ * `staleMs > 0` - serves the stale value while it revalidates in the
+ * background. A failed background rebuild keeps serving the stale value
+ * until the stale window closes.
+ */
+export class BoundedPromiseCache {
+    options;
+    entries = new Map();
+    now;
+    constructor(options) {
+        this.options = options;
+        this.now = options.now ?? Date.now;
+    }
+    /** Test seam: pin the clock. `null` restores `Date.now`. */
+    setNowForTests(now) {
+        this.now = now ?? this.options.now ?? Date.now;
+    }
+    /** Reports a fresh OR stale entry - anything `getOrLoad` would return without a blocking reload. */
+    has(key) {
+        const cached = this.entries.get(key);
+        return Boolean(cached && cached.staleUntil > this.now());
+    }
+    /** Drop every entry. Test seam for module-level caches. */
+    clear() {
+        this.entries.clear();
+    }
+    delete(key) {
+        this.entries.delete(key);
+    }
+    deleteWhere(predicate) {
+        for (const key of this.entries.keys()) {
+            if (predicate(key))
+                this.entries.delete(key);
+        }
+    }
+    getOrLoad(key, load, 
+    /**
+     * `serveStale: false` takes the reload inline once the fresh window has
+     * passed. Serve-stale exists for voice first-audio; on a written channel
+     * it only meant an owner's settings change took up to `staleMs` to show
+     * up, because invalidation never leaves the process that served it.
+     */
+    options) {
+        const now = this.now();
+        const cached = this.entries.get(key);
+        if (cached) {
+            if (cached.freshUntil > now)
+                return cached.value;
+            if (cached.staleUntil > now) {
+                if (options?.serveStale !== false) {
+                    this.scheduleRefresh(key, cached, load);
+                    return cached.value;
+                }
+            }
+            else {
+                // Only a caller that WANTED the stale value is surprised by this.
+                this.options.onBlockingReload?.(key);
+            }
+            this.entries.delete(key);
+        }
+        this.evictIfNeeded(now);
+        const value = load();
+        const entry = this.entryFor(now, value);
+        this.entries.set(key, entry);
+        value.catch(() => {
+            if (this.entries.get(key) === entry)
+                this.entries.delete(key);
+        });
+        return value;
+    }
+    scheduleRefresh(key, cached, load) {
+        if (cached.refresh)
+            return;
+        const refresh = load();
+        cached.refresh = refresh;
+        refresh
+            .then((value) => {
+            if (this.entries.get(key) !== cached)
+                return;
+            this.entries.set(key, this.entryFor(this.now(), Promise.resolve(value)));
+        })
+            .catch(() => {
+            // Keep serving the stale value; the next read past `freshMs` retries.
+            if (this.entries.get(key) === cached)
+                cached.refresh = null;
+        });
+    }
+    entryFor(now, value) {
+        const { freshMs, staleMs = 0 } = this.options;
+        return {
+            freshUntil: now + freshMs,
+            staleUntil: now + Math.max(freshMs, staleMs),
+            value,
+            refresh: null
+        };
+    }
+    evictIfNeeded(now) {
+        if (this.entries.size < this.options.maxEntries)
+            return;
+        for (const [key, entry] of this.entries) {
+            if (entry.staleUntil <= now)
+                this.entries.delete(key);
+        }
+        if (this.entries.size < this.options.maxEntries)
+            return;
+        const oldest = this.entries.keys().next();
+        if (!oldest.done)
+            this.entries.delete(oldest.value);
+    }
+}
+//# sourceMappingURL=bounded-promise-cache.js.map

package/dist/embeddings/bounded-promise-cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bounded-promise-cache.js","sourceRoot":"","sources":["../../src/embeddings/bounded-promise-cache.ts"],"names":[],"mappings":"AA2BA;;;;;;GAMG;AACH,MAAM,OAAO,mBAAmB;IAIF;IAHZ,OAAO,GAAG,IAAI,GAAG,EAAiC,CAAA;IAC3D,GAAG,CAAc;IAEzB,YAA6B,OAAwC;QAAxC,YAAO,GAAP,OAAO,CAAiC;QACpE,IAAI,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAA;IACnC,CAAC;IAED,4DAA4D;IAC5D,cAAc,CAAC,GAA0B;QACxC,IAAI,CAAC,GAAG,GAAG,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,IAAI,CAAC,GAAG,CAAA;IAC/C,CAAC;IAED,oGAAoG;IACpG,GAAG,CAAC,GAAQ;QACX,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACpC,OAAO,OAAO,CAAC,MAAM,IAAI,MAAM,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAA;IACzD,CAAC;IAED,2DAA2D;IAC3D,KAAK;QACJ,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAA;IACrB,CAAC;IAED,MAAM,CAAC,GAAQ;QACd,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;IACzB,CAAC;IAED,WAAW,CAAC,SAAgC;QAC3C,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC;YACvC,IAAI,SAAS,CAAC,GAAG,CAAC;gBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAC7C,CAAC;IACF,CAAC;IAED,SAAS,CACR,GAAQ,EACR,IAA0B;IAC1B;;;;;OAKG;IACH,OAAkC;QAElC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QACpC,IAAI,MAAM,EAAE,CAAC;YACZ,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG;gBAAE,OAAO,MAAM,CAAC,KAAK,CAAA;YAChD,IAAI,MAAM,CAAC,UAAU,GAAG,GAAG,EAAE,CAAC;gBAC7B,IAAI,OAAO,EAAE,UAAU,KAAK,KAAK,EAAE,CAAC;oBACnC,IAAI,CAAC,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,IAAI,CAAC,CAAA;oBACvC,OAAO,MAAM,CAAC,KAAK,CAAA;gBACpB,CAAC;YACF,CAAC;iBAAM,CAAC;gBACP,kEAAkE;gBAClE,IAAI,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC,GAAG,CAAC,CAAA;YACrC,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACzB,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,CAAA;QAEvB,MAAM,KAAK,GAAG,IAAI,EAAE,CAAA;QACpB,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QAC5B,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;YAChB,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,KAAK;gBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QAC9D,CAAC,CAAC,CAAA;QACF,OAAO,KAAK,CAAA;IACb,CAAC;IAEO,eAAe,CACtB,GAAQ,EACR,MAAgC,EAChC,IAA0B;QAE1B,IAAI,MAAM,CAAC,OAAO;YAAE,OAAM;QAC1B,MAAM,OAAO,GAAG,IAAI,EAAE,CAAA;QACtB,MAAM,CAAC,OAAO,GAAG,OAAO,CAAA;QACxB,OAAO;aACL,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE;YACf,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,MAAM;gBAAE,OAAM;YAC5C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,EAAE,EAAE,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;QACzE,CAAC,CAAC;aACD,KAAK,CAAC,GAAG,EAAE;YACX,sEAAsE;YACtE,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,KAAK,MAAM;gBAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAA;QAC5D,CAAC,CAAC,CAAA;IACJ,CAAC;IAEO,QAAQ,CACf,GAAW,EACX,KAAqB;QAErB,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,CAAC,EAAE,GAAG,IAAI,CAAC,OAAO,CAAA;QAC7C,OAAO;YACN,UAAU,EAAE,GAAG,GAAG,OAAO;YACzB,UAAU,EAAE,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,OAAO,CAAC;YAC5C,KAAK;YACL,OAAO,EAAE,IAAI;SACb,CAAA;IACF,CAAC;IAEO,aAAa,CAAC,GAAW;QAChC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU;YAAE,OAAM;QACvD,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACzC,IAAI,KAAK,CAAC,UAAU,IAAI,GAAG;gBAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;QACtD,CAAC;QACD,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,UAAU;YAAE,OAAM;QACvD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAA;QACzC,IAAI,CAAC,MAAM,CAAC,IAAI;YAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;IACpD,CAAC;CACD","sourcesContent":["type PromiseCacheEntry<Value> = {\n\tfreshUntil: number\n\tstaleUntil: number\n\tvalue: Promise<Value>\n\t/** The background rebuild running for a stale entry, if any. */\n\trefresh: Promise<Value> | null\n}\n\ntype BoundedPromiseCacheOptions<Key> = {\n\t/** How long a value is served without triggering any reload. */\n\tfreshMs: number\n\t/**\n\t * Maximum age (from load) a value may be served at all. Between `freshMs`\n\t * and `staleMs` the stale value is returned while a rebuild runs in the\n\t * background; past `staleMs` the caller pays the reload inline. Omitted (or\n\t * `<= freshMs`) makes this a plain TTL cache.\n\t */\n\tstaleMs?: number\n\tmaxEntries: number\n\tnow?: () => number\n\t/**\n\t * Fired when a caller has to wait for a reload because the entry is past\n\t * `staleMs` - the case a voice first-audio path must never hit.\n\t */\n\tonBlockingReload?: (key: Key) => void\n}\n\n/**\n * Small in-process promise cache: bounded, shares work already in flight,\n * evicts a rejected first load so the next caller retries, and - when\n * `staleMs > 0` - serves the stale value while it revalidates in the\n * background. A failed background rebuild keeps serving the stale value\n * until the stale window closes.\n */\nexport class BoundedPromiseCache<Key, Value> {\n\tprivate readonly entries = new Map<Key, PromiseCacheEntry<Value>>()\n\tprivate now: () => number\n\n\tconstructor(private readonly options: BoundedPromiseCacheOptions<Key>) {\n\t\tthis.now = options.now ?? Date.now\n\t}\n\n\t/** Test seam: pin the clock. `null` restores `Date.now`. */\n\tsetNowForTests(now: (() => number) | null) {\n\t\tthis.now = now ?? this.options.now ?? Date.now\n\t}\n\n\t/** Reports a fresh OR stale entry - anything `getOrLoad` would return without a blocking reload. */\n\thas(key: Key) {\n\t\tconst cached = this.entries.get(key)\n\t\treturn Boolean(cached && cached.staleUntil > this.now())\n\t}\n\n\t/** Drop every entry. Test seam for module-level caches. */\n\tclear() {\n\t\tthis.entries.clear()\n\t}\n\n\tdelete(key: Key) {\n\t\tthis.entries.delete(key)\n\t}\n\n\tdeleteWhere(predicate: (key: Key) => boolean) {\n\t\tfor (const key of this.entries.keys()) {\n\t\t\tif (predicate(key)) this.entries.delete(key)\n\t\t}\n\t}\n\n\tgetOrLoad(\n\t\tkey: Key,\n\t\tload: () => Promise<Value>,\n\t\t/**\n\t\t * `serveStale: false` takes the reload inline once the fresh window has\n\t\t * passed. Serve-stale exists for voice first-audio; on a written channel\n\t\t * it only meant an owner's settings change took up to `staleMs` to show\n\t\t * up, because invalidation never leaves the process that served it.\n\t\t */\n\t\toptions?: { serveStale?: boolean }\n\t): Promise<Value> {\n\t\tconst now = this.now()\n\t\tconst cached = this.entries.get(key)\n\t\tif (cached) {\n\t\t\tif (cached.freshUntil > now) return cached.value\n\t\t\tif (cached.staleUntil > now) {\n\t\t\t\tif (options?.serveStale !== false) {\n\t\t\t\t\tthis.scheduleRefresh(key, cached, load)\n\t\t\t\t\treturn cached.value\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\t// Only a caller that WANTED the stale value is surprised by this.\n\t\t\t\tthis.options.onBlockingReload?.(key)\n\t\t\t}\n\t\t\tthis.entries.delete(key)\n\t\t}\n\n\t\tthis.evictIfNeeded(now)\n\n\t\tconst value = load()\n\t\tconst entry = this.entryFor(now, value)\n\t\tthis.entries.set(key, entry)\n\t\tvalue.catch(() => {\n\t\t\tif (this.entries.get(key) === entry) this.entries.delete(key)\n\t\t})\n\t\treturn value\n\t}\n\n\tprivate scheduleRefresh(\n\t\tkey: Key,\n\t\tcached: PromiseCacheEntry<Value>,\n\t\tload: () => Promise<Value>\n\t) {\n\t\tif (cached.refresh) return\n\t\tconst refresh = load()\n\t\tcached.refresh = refresh\n\t\trefresh\n\t\t\t.then((value) => {\n\t\t\t\tif (this.entries.get(key) !== cached) return\n\t\t\t\tthis.entries.set(key, this.entryFor(this.now(), Promise.resolve(value)))\n\t\t\t})\n\t\t\t.catch(() => {\n\t\t\t\t// Keep serving the stale value; the next read past `freshMs` retries.\n\t\t\t\tif (this.entries.get(key) === cached) cached.refresh = null\n\t\t\t})\n\t}\n\n\tprivate entryFor(\n\t\tnow: number,\n\t\tvalue: Promise<Value>\n\t): PromiseCacheEntry<Value> {\n\t\tconst { freshMs, staleMs = 0 } = this.options\n\t\treturn {\n\t\t\tfreshUntil: now + freshMs,\n\t\t\tstaleUntil: now + Math.max(freshMs, staleMs),\n\t\t\tvalue,\n\t\t\trefresh: null\n\t\t}\n\t}\n\n\tprivate evictIfNeeded(now: number) {\n\t\tif (this.entries.size < this.options.maxEntries) return\n\t\tfor (const [key, entry] of this.entries) {\n\t\t\tif (entry.staleUntil <= now) this.entries.delete(key)\n\t\t}\n\t\tif (this.entries.size < this.options.maxEntries) return\n\t\tconst oldest = this.entries.keys().next()\n\t\tif (!oldest.done) this.entries.delete(oldest.value)\n\t}\n}\n"]}

package/dist/embeddings/embedder.d.ts

@@ -0,0 +1,50 @@
+import { type EmbeddingModel } from 'ai';
+import { type Logger } from '../models/route.js';
+/** A shared cache for query embeddings; Redis `get`/`setex` fit as they are. */
+export type EmbeddingCache = {
+    get: (key: string) => Promise<string | null>;
+    set: (key: string, value: string, ttlSeconds: number) => Promise<unknown>;
+};
+/** Tokens an embedding call used, with the caller's context. */
+export type EmbeddingUsage<TContext> = {
+    tokens: number;
+    modelId: string;
+    context: TContext | undefined;
+};
+/** Case and whitespace do not change what a search query means. */
+export declare function queryEmbeddingCacheKey(modelId: string, query: string, prefix?: string): string;
+/**
+ * Query and document embeddings on one model. `embedQuery` is for the hot
+ * path: an in-process cache in front of the shared one, and concurrent calls
+ * for one query share a single request. `embedMany` is for indexing: batches
+ * of `batchSize` with `concurrency` requests at once, and every vector checked
+ * before it is returned. Reuse one embedder per model, so every call reuses
+ * the provider's connection.
+ */
+export declare function createEmbedder<TContext = undefined>(input: {
+    model: EmbeddingModel;
+    /** Shared query cache (Redis), kept 24 hours. */
+    cache?: EmbeddingCache;
+    /** Prefix of the shared cache keys. */
+    cachePrefix?: string;
+    /** Tokens per call that reached the provider; cache hits cost none. */
+    usage?: (usage: EmbeddingUsage<TContext>) => void;
+    /** Every vector must have this length. */
+    dimensions?: number;
+    batchSize?: number;
+    concurrency?: number;
+    logger?: Logger;
+}): {
+    modelId: string;
+    embedQuery: (value: string, context?: TContext) => Promise<{
+        embedding: number[];
+        tokens: number;
+        cached: boolean;
+    }>;
+    embedMany: (values: readonly string[], context?: TContext) => Promise<number[][]>;
+    prewarm: () => Promise<void>;
+    /** Drops the in-process query cache. */
+    clearCache: () => void;
+};
+export type Embedder<TContext = undefined> = ReturnType<typeof createEmbedder<TContext>>;
+//# sourceMappingURL=embedder.d.ts.map

package/dist/embeddings/embedder.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/embeddings/embedder.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,cAAc,EAAmC,MAAM,IAAI,CAAA;AACzE,OAAO,EAAmB,KAAK,MAAM,EAAgB,MAAM,oBAAoB,CAAA;AAG/E,gFAAgF;AAChF,MAAM,MAAM,cAAc,GAAG;IAC5B,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAAA;IAC5C,GAAG,EAAE,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,OAAO,CAAC,OAAO,CAAC,CAAA;CACzE,CAAA;AAED,gEAAgE;AAChE,MAAM,MAAM,cAAc,CAAC,QAAQ,IAAI;IACtC,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,OAAO,EAAE,QAAQ,GAAG,SAAS,CAAA;CAC7B,CAAA;AAWD,mEAAmE;AACnE,wBAAgB,sBAAsB,CACrC,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EACb,MAAM,SAAS,UAOf;AAED;;;;;;;GAOG;AACH,wBAAgB,cAAc,CAAC,QAAQ,GAAG,SAAS,EAAE,KAAK,EAAE;IAC3D,KAAK,EAAE,cAAc,CAAA;IACrB,iDAAiD;IACjD,KAAK,CAAC,EAAE,cAAc,CAAA;IACtB,uCAAuC;IACvC,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,uEAAuE;IACvE,KAAK,CAAC,EAAE,CAAC,KAAK,EAAE,cAAc,CAAC,QAAQ,CAAC,KAAK,IAAI,CAAA;IACjD,0CAA0C;IAC1C,UAAU,CAAC,EAAE,MAAM,CAAA;IACnB,SAAS,CAAC,EAAE,MAAM,CAAA;IAClB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,MAAM,CAAC,EAAE,MAAM,CAAA;CACf;;wBA2CiC,MAAM,YAAY,QAAQ;;;;;wBAoB1B,SAAS,MAAM,EAAE,YAAY,QAAQ;;IAyCrE,wCAAwC;;EAGzC;AAED,MAAM,MAAM,QAAQ,CAAC,QAAQ,GAAG,SAAS,IAAI,UAAU,CACtD,OAAO,cAAc,CAAC,QAAQ,CAAC,CAC/B,CAAA"}

package/dist/embeddings/embedder.js

@@ -0,0 +1,127 @@
+import { createHash } from 'node:crypto';
+import { embed, embedMany as aiEmbedMany } from 'ai';
+import { errorAttributes, silentLogger } from '../models/route.js';
+import { BoundedPromiseCache } from './bounded-promise-cache.js';
+const SHARED_TTL_SECONDS = 24 * 60 * 60;
+const MEMORY_TTL_MS = 5 * 60 * 1000;
+const MEMORY_MAX_ENTRIES = 500;
+const PREWARM_QUERY = 'warmup';
+function modelIdOf(model) {
+    return typeof model === 'string' ? model : model.modelId;
+}
+/** Case and whitespace do not change what a search query means. */
+export function queryEmbeddingCacheKey(modelId, query, prefix = 'qemb') {
+    const normalized = query.trim().toLowerCase().replace(/\s+/g, ' ');
+    const digest = createHash('sha256')
+        .update(`${modelId}:${normalized}`)
+        .digest('hex');
+    return `${prefix}:${digest}`;
+}
+/**
+ * Query and document embeddings on one model. `embedQuery` is for the hot
+ * path: an in-process cache in front of the shared one, and concurrent calls
+ * for one query share a single request. `embedMany` is for indexing: batches
+ * of `batchSize` with `concurrency` requests at once, and every vector checked
+ * before it is returned. Reuse one embedder per model, so every call reuses
+ * the provider's connection.
+ */
+export function createEmbedder(input) {
+    const modelId = modelIdOf(input.model);
+    const batchSize = input.batchSize ?? 100;
+    const concurrency = input.concurrency ?? 3;
+    const logger = input.logger ?? silentLogger;
+    const memory = new BoundedPromiseCache({
+        freshMs: MEMORY_TTL_MS,
+        maxEntries: MEMORY_MAX_ENTRIES
+    });
+    function checked(vector) {
+        if (vector.length === 0 ||
+            (input.dimensions !== undefined && vector.length !== input.dimensions)) {
+            throw new Error(`Embedding has ${vector.length} dimensions, expected ${input.dimensions ?? 'more than 0'}`);
+        }
+        return vector;
+    }
+    function report(tokens, context) {
+        if (tokens > 0)
+            input.usage?.({ tokens, modelId, context });
+    }
+    async function readShared(key) {
+        if (!input.cache)
+            return null;
+        try {
+            const raw = await input.cache.get(key);
+            if (!raw)
+                return null;
+            const parsed = JSON.parse(raw);
+            return Array.isArray(parsed) &&
+                parsed.length > 0 &&
+                parsed.every((value) => typeof value === 'number')
+                ? parsed
+                : null;
+        }
+        catch (error) {
+            logger.warn('Query embedding cache read failed', errorAttributes(error));
+            return null;
+        }
+    }
+    async function embedQuery(value, context) {
+        const key = queryEmbeddingCacheKey(modelId, value, input.cachePrefix);
+        let tokens = 0;
+        let cached = true;
+        const embedding = await memory.getOrLoad(key, async () => {
+            const shared = await readShared(key);
+            if (shared)
+                return shared;
+            cached = false;
+            const result = await embed({ model: input.model, value });
+            tokens = result.usage.tokens ?? 0;
+            const vector = checked(result.embedding);
+            void input.cache
+                ?.set(key, JSON.stringify(vector), SHARED_TTL_SECONDS)
+                .catch(() => undefined);
+            return vector;
+        });
+        report(tokens, context);
+        return { embedding, tokens, cached };
+    }
+    async function embedMany(values, context) {
+        const batches = [];
+        for (let start = 0; start < values.length; start += batchSize) {
+            batches.push(values.slice(start, start + batchSize));
+        }
+        const results = new Array(batches.length);
+        let next = 0;
+        const worker = async () => {
+            while (next < batches.length) {
+                const index = next++;
+                const batch = batches[index];
+                const result = await aiEmbedMany({ model: input.model, values: batch });
+                if (result.embeddings.length !== batch.length) {
+                    throw new Error(`Incomplete embedding batch: ${result.embeddings.length} of ${batch.length}`);
+                }
+                results[index] = result.embeddings.map(checked);
+                report(result.usage.tokens ?? 0, context);
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(concurrency, batches.length) }, worker));
+        return results.flat();
+    }
+    /** Best-effort connection warm, so the first real query skips the handshake. */
+    async function prewarm() {
+        try {
+            await embedQuery(PREWARM_QUERY);
+        }
+        catch (error) {
+            logger.warn('Embedding connection prewarm failed', errorAttributes(error));
+        }
+    }
+    return {
+        modelId,
+        embedQuery,
+        embedMany,
+        prewarm,
+        /** Drops the in-process query cache. */
+        clearCache: () => memory.clear()
+    };
+}
+//# sourceMappingURL=embedder.js.map

package/dist/embeddings/embedder.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/embeddings/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAuB,KAAK,EAAE,SAAS,IAAI,WAAW,EAAE,MAAM,IAAI,CAAA;AACzE,OAAO,EAAE,eAAe,EAAe,YAAY,EAAE,MAAM,oBAAoB,CAAA;AAC/E,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAehE,MAAM,kBAAkB,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAA;AACvC,MAAM,aAAa,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAA;AACnC,MAAM,kBAAkB,GAAG,GAAG,CAAA;AAC9B,MAAM,aAAa,GAAG,QAAQ,CAAA;AAE9B,SAAS,SAAS,CAAC,KAAqB;IACvC,OAAO,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAA;AACzD,CAAC;AAED,mEAAmE;AACnE,MAAM,UAAU,sBAAsB,CACrC,OAAe,EACf,KAAa,EACb,MAAM,GAAG,MAAM;IAEf,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAA;IAClE,MAAM,MAAM,GAAG,UAAU,CAAC,QAAQ,CAAC;SACjC,MAAM,CAAC,GAAG,OAAO,IAAI,UAAU,EAAE,CAAC;SAClC,MAAM,CAAC,KAAK,CAAC,CAAA;IACf,OAAO,GAAG,MAAM,IAAI,MAAM,EAAE,CAAA;AAC7B,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,cAAc,CAAuB,KAapD;IACA,MAAM,OAAO,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;IACtC,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,IAAI,GAAG,CAAA;IACxC,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,CAAC,CAAA;IAC1C,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,YAAY,CAAA;IAC3C,MAAM,MAAM,GAAG,IAAI,mBAAmB,CAAmB;QACxD,OAAO,EAAE,aAAa;QACtB,UAAU,EAAE,kBAAkB;KAC9B,CAAC,CAAA;IAEF,SAAS,OAAO,CAAC,MAAgB;QAChC,IACC,MAAM,CAAC,MAAM,KAAK,CAAC;YACnB,CAAC,KAAK,CAAC,UAAU,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,CAAC,UAAU,CAAC,EACrE,CAAC;YACF,MAAM,IAAI,KAAK,CACd,iBAAiB,MAAM,CAAC,MAAM,yBAAyB,KAAK,CAAC,UAAU,IAAI,aAAa,EAAE,CAC1F,CAAA;QACF,CAAC;QACD,OAAO,MAAM,CAAA;IACd,CAAC;IAED,SAAS,MAAM,CAAC,MAAc,EAAE,OAA6B;QAC5D,IAAI,MAAM,GAAG,CAAC;YAAE,KAAK,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC,CAAA;IAC5D,CAAC;IAED,KAAK,UAAU,UAAU,CAAC,GAAW;QACpC,IAAI,CAAC,KAAK,CAAC,KAAK;YAAE,OAAO,IAAI,CAAA;QAC7B,IAAI,CAAC;YACJ,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;YACtC,IAAI,CAAC,GAAG;gBAAE,OAAO,IAAI,CAAA;YACrB,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;YACvC,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAC3B,MAAM,CAAC,MAAM,GAAG,CAAC;gBACjB,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,OAAO,KAAK,KAAK,QAAQ,CAAC;gBAClD,CAAC,CAAE,MAAmB;gBACtB,CAAC,CAAC,IAAI,CAAA;QACR,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE,eAAe,CAAC,KAAK,CAAC,CAAC,CAAA;YACxE,OAAO,IAAI,CAAA;QACZ,CAAC;IACF,CAAC;IAED,KAAK,UAAU,UAAU,CAAC,KAAa,EAAE,OAAkB;QAC1D,MAAM,GAAG,GAAG,sBAAsB,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,CAAC,WAAW,CAAC,CAAA;QACrE,IAAI,MAAM,GAAG,CAAC,CAAA;QACd,IAAI,MAAM,GAAG,IAAI,CAAA;QACjB,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,GAAG,CAAC,CAAA;YACpC,IAAI,MAAM;gBAAE,OAAO,MAAM,CAAA;YACzB,MAAM,GAAG,KAAK,CAAA;YACd,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAA;YACzD,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,CAAA;YACjC,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACxC,KAAK,KAAK,CAAC,KAAK;gBACf,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,kBAAkB,CAAC;iBACrD,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAA;YACxB,OAAO,MAAM,CAAA;QACd,CAAC,CAAC,CAAA;QACF,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACvB,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;IACrC,CAAC;IAED,KAAK,UAAU,SAAS,CAAC,MAAyB,EAAE,OAAkB;QACrE,MAAM,OAAO,GAAe,EAAE,CAAA;QAC9B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,SAAS,EAAE,CAAC;YAC/D,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,SAAS,CAAC,CAAC,CAAA;QACrD,CAAC;QACD,MAAM,OAAO,GAAiB,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAA;QACvD,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,MAAM,MAAM,GAAG,KAAK,IAAI,EAAE;YACzB,OAAO,IAAI,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;gBAC9B,MAAM,KAAK,GAAG,IAAI,EAAE,CAAA;gBACpB,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAE,CAAA;gBAC7B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAA;gBACvE,IAAI,MAAM,CAAC,UAAU,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;oBAC/C,MAAM,IAAI,KAAK,CACd,+BAA+B,MAAM,CAAC,UAAU,CAAC,MAAM,OAAO,KAAK,CAAC,MAAM,EAAE,CAC5E,CAAA;gBACF,CAAC;gBACD,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAA;gBAC/C,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,OAAO,CAAC,CAAA;YAC1C,CAAC;QACF,CAAC,CAAA;QACD,MAAM,OAAO,CAAC,GAAG,CAChB,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,MAAM,CAAC,CACrE,CAAA;QACD,OAAO,OAAO,CAAC,IAAI,EAAE,CAAA;IACtB,CAAC;IAED,gFAAgF;IAChF,KAAK,UAAU,OAAO;QACrB,IAAI,CAAC;YACJ,MAAM,UAAU,CAAC,aAAa,CAAC,CAAA;QAChC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,qCAAqC,EAAE,eAAe,CAAC,KAAK,CAAC,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,OAAO;QACN,OAAO;QACP,UAAU;QACV,SAAS;QACT,OAAO;QACP,wCAAwC;QACxC,UAAU,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE;KAChC,CAAA;AACF,CAAC","sourcesContent":["import { createHash } from 'node:crypto'\nimport { type EmbeddingModel, embed, embedMany as aiEmbedMany } from 'ai'\nimport { errorAttributes, type Logger, silentLogger } from '../models/route.js'\nimport { BoundedPromiseCache } from './bounded-promise-cache.js'\n\n/** A shared cache for query embeddings; Redis `get`/`setex` fit as they are. */\nexport type EmbeddingCache = {\n\tget: (key: string) => Promise<string | null>\n\tset: (key: string, value: string, ttlSeconds: number) => Promise<unknown>\n}\n\n/** Tokens an embedding call used, with the caller's context. */\nexport type EmbeddingUsage<TContext> = {\n\ttokens: number\n\tmodelId: string\n\tcontext: TContext | undefined\n}\n\nconst SHARED_TTL_SECONDS = 24 * 60 * 60\nconst MEMORY_TTL_MS = 5 * 60 * 1000\nconst MEMORY_MAX_ENTRIES = 500\nconst PREWARM_QUERY = 'warmup'\n\nfunction modelIdOf(model: EmbeddingModel) {\n\treturn typeof model === 'string' ? model : model.modelId\n}\n\n/** Case and whitespace do not change what a search query means. */\nexport function queryEmbeddingCacheKey(\n\tmodelId: string,\n\tquery: string,\n\tprefix = 'qemb'\n) {\n\tconst normalized = query.trim().toLowerCase().replace(/\\s+/g, ' ')\n\tconst digest = createHash('sha256')\n\t\t.update(`${modelId}:${normalized}`)\n\t\t.digest('hex')\n\treturn `${prefix}:${digest}`\n}\n\n/**\n * Query and document embeddings on one model. `embedQuery` is for the hot\n * path: an in-process cache in front of the shared one, and concurrent calls\n * for one query share a single request. `embedMany` is for indexing: batches\n * of `batchSize` with `concurrency` requests at once, and every vector checked\n * before it is returned. Reuse one embedder per model, so every call reuses\n * the provider's connection.\n */\nexport function createEmbedder<TContext = undefined>(input: {\n\tmodel: EmbeddingModel\n\t/** Shared query cache (Redis), kept 24 hours. */\n\tcache?: EmbeddingCache\n\t/** Prefix of the shared cache keys. */\n\tcachePrefix?: string\n\t/** Tokens per call that reached the provider; cache hits cost none. */\n\tusage?: (usage: EmbeddingUsage<TContext>) => void\n\t/** Every vector must have this length. */\n\tdimensions?: number\n\tbatchSize?: number\n\tconcurrency?: number\n\tlogger?: Logger\n}) {\n\tconst modelId = modelIdOf(input.model)\n\tconst batchSize = input.batchSize ?? 100\n\tconst concurrency = input.concurrency ?? 3\n\tconst logger = input.logger ?? silentLogger\n\tconst memory = new BoundedPromiseCache<string, number[]>({\n\t\tfreshMs: MEMORY_TTL_MS,\n\t\tmaxEntries: MEMORY_MAX_ENTRIES\n\t})\n\n\tfunction checked(vector: number[]) {\n\t\tif (\n\t\t\tvector.length === 0 ||\n\t\t\t(input.dimensions !== undefined && vector.length !== input.dimensions)\n\t\t) {\n\t\t\tthrow new Error(\n\t\t\t\t`Embedding has ${vector.length} dimensions, expected ${input.dimensions ?? 'more than 0'}`\n\t\t\t)\n\t\t}\n\t\treturn vector\n\t}\n\n\tfunction report(tokens: number, context: TContext | undefined) {\n\t\tif (tokens > 0) input.usage?.({ tokens, modelId, context })\n\t}\n\n\tasync function readShared(key: string) {\n\t\tif (!input.cache) return null\n\t\ttry {\n\t\t\tconst raw = await input.cache.get(key)\n\t\t\tif (!raw) return null\n\t\t\tconst parsed: unknown = JSON.parse(raw)\n\t\t\treturn Array.isArray(parsed) &&\n\t\t\t\tparsed.length > 0 &&\n\t\t\t\tparsed.every((value) => typeof value === 'number')\n\t\t\t\t? (parsed as number[])\n\t\t\t\t: null\n\t\t} catch (error) {\n\t\t\tlogger.warn('Query embedding cache read failed', errorAttributes(error))\n\t\t\treturn null\n\t\t}\n\t}\n\n\tasync function embedQuery(value: string, context?: TContext) {\n\t\tconst key = queryEmbeddingCacheKey(modelId, value, input.cachePrefix)\n\t\tlet tokens = 0\n\t\tlet cached = true\n\t\tconst embedding = await memory.getOrLoad(key, async () => {\n\t\t\tconst shared = await readShared(key)\n\t\t\tif (shared) return shared\n\t\t\tcached = false\n\t\t\tconst result = await embed({ model: input.model, value })\n\t\t\ttokens = result.usage.tokens ?? 0\n\t\t\tconst vector = checked(result.embedding)\n\t\t\tvoid input.cache\n\t\t\t\t?.set(key, JSON.stringify(vector), SHARED_TTL_SECONDS)\n\t\t\t\t.catch(() => undefined)\n\t\t\treturn vector\n\t\t})\n\t\treport(tokens, context)\n\t\treturn { embedding, tokens, cached }\n\t}\n\n\tasync function embedMany(values: readonly string[], context?: TContext) {\n\t\tconst batches: string[][] = []\n\t\tfor (let start = 0; start < values.length; start += batchSize) {\n\t\t\tbatches.push(values.slice(start, start + batchSize))\n\t\t}\n\t\tconst results: number[][][] = new Array(batches.length)\n\t\tlet next = 0\n\t\tconst worker = async () => {\n\t\t\twhile (next < batches.length) {\n\t\t\t\tconst index = next++\n\t\t\t\tconst batch = batches[index]!\n\t\t\t\tconst result = await aiEmbedMany({ model: input.model, values: batch })\n\t\t\t\tif (result.embeddings.length !== batch.length) {\n\t\t\t\t\tthrow new Error(\n\t\t\t\t\t\t`Incomplete embedding batch: ${result.embeddings.length} of ${batch.length}`\n\t\t\t\t\t)\n\t\t\t\t}\n\t\t\t\tresults[index] = result.embeddings.map(checked)\n\t\t\t\treport(result.usage.tokens ?? 0, context)\n\t\t\t}\n\t\t}\n\t\tawait Promise.all(\n\t\t\tArray.from({ length: Math.min(concurrency, batches.length) }, worker)\n\t\t)\n\t\treturn results.flat()\n\t}\n\n\t/** Best-effort connection warm, so the first real query skips the handshake. */\n\tasync function prewarm() {\n\t\ttry {\n\t\t\tawait embedQuery(PREWARM_QUERY)\n\t\t} catch (error) {\n\t\t\tlogger.warn('Embedding connection prewarm failed', errorAttributes(error))\n\t\t}\n\t}\n\n\treturn {\n\t\tmodelId,\n\t\tembedQuery,\n\t\tembedMany,\n\t\tprewarm,\n\t\t/** Drops the in-process query cache. */\n\t\tclearCache: () => memory.clear()\n\t}\n}\n\nexport type Embedder<TContext = undefined> = ReturnType<\n\ttypeof createEmbedder<TContext>\n>\n"]}

package/dist/embeddings/index.d.ts

@@ -0,0 +1,3 @@
+export { BoundedPromiseCache } from './bounded-promise-cache.js';
+export { createEmbedder, type Embedder, type EmbeddingCache, type EmbeddingUsage, queryEmbeddingCacheKey } from './embedder.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/embeddings/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EACN,cAAc,EACd,KAAK,QAAQ,EACb,KAAK,cAAc,EACnB,KAAK,cAAc,EACnB,sBAAsB,EACtB,MAAM,eAAe,CAAA"}

package/dist/embeddings/index.js

@@ -0,0 +1,3 @@
+export { BoundedPromiseCache } from './bounded-promise-cache.js';
+export { createEmbedder, queryEmbeddingCacheKey } from './embedder.js';
+//# sourceMappingURL=index.js.map

package/dist/embeddings/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAA;AAChE,OAAO,EACN,cAAc,EAId,sBAAsB,EACtB,MAAM,eAAe,CAAA","sourcesContent":["export { BoundedPromiseCache } from './bounded-promise-cache.js'\nexport {\n\tcreateEmbedder,\n\ttype Embedder,\n\ttype EmbeddingCache,\n\ttype EmbeddingUsage,\n\tqueryEmbeddingCacheKey\n} from './embedder.js'\n"]}

package/dist/evals/case.d.ts

@@ -0,0 +1,85 @@
+/**
+ * An eval case: one conversation the assistant must handle, sent turn by turn
+ * as a persona the app seeds, with what a good answer does. Every bucket
+ * should hold cases, so "always acts" and "always asks" both score badly.
+ */
+export type EvalCase<Page = unknown, Persona extends string = string, Bucket extends string = string, Suite extends string = string> = {
+    id: string;
+    suites: readonly Suite[];
+    bucket: Bucket;
+    /** Who asks: a key into the app's seeded personas. */
+    persona: Persona;
+    /** The language the user writes in, which the reply must follow. */
+    language: string;
+    /** Where the conversation starts; the app's turn reads it. */
+    page?: Page;
+    /**
+     * The user's messages, one per turn, in order. A turn with its own page
+     * was sent from there: the user moved, or the page's data changed.
+     */
+    turns: readonly (string | {
+        text: string;
+        page: Page;
+    })[];
+    expect: EvalExpectation;
+    tags?: readonly string[];
+};
+/** Tool names are matched after the checks' `toolName` normalizer. */
+export type EvalExpectation = {
+    /** Every one of these is called. */
+    tools?: readonly string[];
+    /** At least one of these is called. */
+    anyTool?: readonly string[];
+    /** None of these is called. */
+    forbidTools?: readonly string[];
+    /** No write and no approval card at all. */
+    noWrites?: true;
+    /** The reply names at least one of these (case-insensitive). */
+    replyMentionsAny?: readonly string[];
+    /** A call to `tool` whose input holds at least one of `anyOf` (case-insensitive). */
+    toolInputMentions?: {
+        tool: string;
+        anyOf: readonly string[];
+    };
+    /** What a good answer does, in words, for the judge. */
+    expectations?: readonly string[];
+};
+/** Identity function that keeps the case's literal types. */
+export declare function defineEvalCase<const C extends EvalCase<unknown>>(testCase: C): C;
+/** The text and page of one case turn; a plain string keeps the case's page. */
+export declare function caseTurn<Page>(testCase: Pick<EvalCase<Page>, 'page'>, turn: EvalCase<Page>['turns'][number]): {
+    text: string;
+    page: Page | undefined;
+};
+export type EvalToolCall = {
+    name: string;
+    state: string;
+    input: unknown;
+    output: unknown;
+    errorText: string | null;
+};
+export type EvalTurn = {
+    userText: string;
+    replyText: string;
+    toolCalls: EvalToolCall[];
+    /** The turn stopped on a card waiting for the user's Confirm. */
+    heldForApproval: string[];
+    error: string | null;
+    /** Wait until the first word of the first answer, as the user feels it. */
+    firstTextMs: number | null;
+    durationMs: number;
+    steps: number;
+    inputTokens: number;
+    cacheReadTokens: number;
+    cacheWriteTokens: number;
+    outputTokens: number;
+    costUsd: number | null;
+};
+export type EvalTrace = {
+    caseId: string;
+    modelId: string;
+    /** Facts the turn's instructions held (profile, status line), so the judge does not call them invented. */
+    instructionFacts?: unknown;
+    turns: EvalTurn[];
+};
+//# sourceMappingURL=case.d.ts.map

package/dist/evals/case.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"case.d.ts","sourceRoot":"","sources":["../../src/evals/case.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,MAAM,QAAQ,CACnB,IAAI,GAAG,OAAO,EACd,OAAO,SAAS,MAAM,GAAG,MAAM,EAC/B,MAAM,SAAS,MAAM,GAAG,MAAM,EAC9B,KAAK,SAAS,MAAM,GAAG,MAAM,IAC1B;IACH,EAAE,EAAE,MAAM,CAAA;IACV,MAAM,EAAE,SAAS,KAAK,EAAE,CAAA;IACxB,MAAM,EAAE,MAAM,CAAA;IACd,sDAAsD;IACtD,OAAO,EAAE,OAAO,CAAA;IAChB,oEAAoE;IACpE,QAAQ,EAAE,MAAM,CAAA;IAChB,8DAA8D;IAC9D,IAAI,CAAC,EAAE,IAAI,CAAA;IACX;;;OAGG;IACH,KAAK,EAAE,SAAS,CAAC,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,IAAI,CAAA;KAAE,CAAC,EAAE,CAAA;IACzD,MAAM,EAAE,eAAe,CAAA;IACvB,IAAI,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CACxB,CAAA;AAED,sEAAsE;AACtE,MAAM,MAAM,eAAe,GAAG;IAC7B,oCAAoC;IACpC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACzB,uCAAuC;IACvC,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC3B,+BAA+B;IAC/B,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IAC/B,4CAA4C;IAC5C,QAAQ,CAAC,EAAE,IAAI,CAAA;IACf,gEAAgE;IAChE,gBAAgB,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;IACpC,qFAAqF;IACrF,iBAAiB,CAAC,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,SAAS,MAAM,EAAE,CAAA;KAAE,CAAA;IAC9D,wDAAwD;IACxD,YAAY,CAAC,EAAE,SAAS,MAAM,EAAE,CAAA;CAChC,CAAA;AAED,6DAA6D;AAC7D,wBAAgB,cAAc,CAAC,KAAK,CAAC,CAAC,SAAS,QAAQ,CAAC,OAAO,CAAC,EAC/D,QAAQ,EAAE,CAAC,GACT,CAAC,CAEH;AAED,gFAAgF;AAChF,wBAAgB,QAAQ,CAAC,IAAI,EAC5B,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,MAAM,CAAC,EACtC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,GACnC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,IAAI,GAAG,SAAS,CAAA;CAAE,CAI1C;AAED,MAAM,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;IACb,KAAK,EAAE,OAAO,CAAA;IACd,MAAM,EAAE,OAAO,CAAA;IACf,SAAS,EAAE,MAAM,GAAG,IAAI,CAAA;CACxB,CAAA;AAED,MAAM,MAAM,QAAQ,GAAG;IACtB,QAAQ,EAAE,MAAM,CAAA;IAChB,SAAS,EAAE,MAAM,CAAA;IACjB,SAAS,EAAE,YAAY,EAAE,CAAA;IACzB,iEAAiE;IACjE,eAAe,EAAE,MAAM,EAAE,CAAA;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAA;IACpB,2EAA2E;IAC3E,WAAW,EAAE,MAAM,GAAG,IAAI,CAAA;IAC1B,UAAU,EAAE,MAAM,CAAA;IAClB,KAAK,EAAE,MAAM,CAAA;IACb,WAAW,EAAE,MAAM,CAAA;IACnB,eAAe,EAAE,MAAM,CAAA;IACvB,gBAAgB,EAAE,MAAM,CAAA;IACxB,YAAY,EAAE,MAAM,CAAA;IACpB,OAAO,EAAE,MAAM,GAAG,IAAI,CAAA;CACtB,CAAA;AAED,MAAM,MAAM,SAAS,GAAG;IACvB,MAAM,EAAE,MAAM,CAAA;IACd,OAAO,EAAE,MAAM,CAAA;IACf,2GAA2G;IAC3G,gBAAgB,CAAC,EAAE,OAAO,CAAA;IAC1B,KAAK,EAAE,QAAQ,EAAE,CAAA;CACjB,CAAA"}

package/dist/evals/case.js

@@ -0,0 +1,11 @@
+/** Identity function that keeps the case's literal types. */
+export function defineEvalCase(testCase) {
+    return testCase;
+}
+/** The text and page of one case turn; a plain string keeps the case's page. */
+export function caseTurn(testCase, turn) {
+    return typeof turn === 'string'
+        ? { text: turn, page: testCase.page }
+        : { text: turn.text, page: turn.page };
+}
+//# sourceMappingURL=case.js.map

package/dist/evals/case.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"case.js","sourceRoot":"","sources":["../../src/evals/case.ts"],"names":[],"mappings":"AA+CA,6DAA6D;AAC7D,MAAM,UAAU,cAAc,CAC7B,QAAW;IAEX,OAAO,QAAQ,CAAA;AAChB,CAAC;AAED,gFAAgF;AAChF,MAAM,UAAU,QAAQ,CACvB,QAAsC,EACtC,IAAqC;IAErC,OAAO,OAAO,IAAI,KAAK,QAAQ;QAC9B,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE;QACrC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAA;AACxC,CAAC","sourcesContent":["/**\n * An eval case: one conversation the assistant must handle, sent turn by turn\n * as a persona the app seeds, with what a good answer does. Every bucket\n * should hold cases, so \"always acts\" and \"always asks\" both score badly.\n */\nexport type EvalCase<\n\tPage = unknown,\n\tPersona extends string = string,\n\tBucket extends string = string,\n\tSuite extends string = string\n> = {\n\tid: string\n\tsuites: readonly Suite[]\n\tbucket: Bucket\n\t/** Who asks: a key into the app's seeded personas. */\n\tpersona: Persona\n\t/** The language the user writes in, which the reply must follow. */\n\tlanguage: string\n\t/** Where the conversation starts; the app's turn reads it. */\n\tpage?: Page\n\t/**\n\t * The user's messages, one per turn, in order. A turn with its own page\n\t * was sent from there: the user moved, or the page's data changed.\n\t */\n\tturns: readonly (string | { text: string; page: Page })[]\n\texpect: EvalExpectation\n\ttags?: readonly string[]\n}\n\n/** Tool names are matched after the checks' `toolName` normalizer. */\nexport type EvalExpectation = {\n\t/** Every one of these is called. */\n\ttools?: readonly string[]\n\t/** At least one of these is called. */\n\tanyTool?: readonly string[]\n\t/** None of these is called. */\n\tforbidTools?: readonly string[]\n\t/** No write and no approval card at all. */\n\tnoWrites?: true\n\t/** The reply names at least one of these (case-insensitive). */\n\treplyMentionsAny?: readonly string[]\n\t/** A call to `tool` whose input holds at least one of `anyOf` (case-insensitive). */\n\ttoolInputMentions?: { tool: string; anyOf: readonly string[] }\n\t/** What a good answer does, in words, for the judge. */\n\texpectations?: readonly string[]\n}\n\n/** Identity function that keeps the case's literal types. */\nexport function defineEvalCase<const C extends EvalCase<unknown>>(\n\ttestCase: C\n): C {\n\treturn testCase\n}\n\n/** The text and page of one case turn; a plain string keeps the case's page. */\nexport function caseTurn<Page>(\n\ttestCase: Pick<EvalCase<Page>, 'page'>,\n\tturn: EvalCase<Page>['turns'][number]\n): { text: string; page: Page | undefined } {\n\treturn typeof turn === 'string'\n\t\t? { text: turn, page: testCase.page }\n\t\t: { text: turn.text, page: turn.page }\n}\n\nexport type EvalToolCall = {\n\tname: string\n\tstate: string\n\tinput: unknown\n\toutput: unknown\n\terrorText: string | null\n}\n\nexport type EvalTurn = {\n\tuserText: string\n\treplyText: string\n\ttoolCalls: EvalToolCall[]\n\t/** The turn stopped on a card waiting for the user's Confirm. */\n\theldForApproval: string[]\n\terror: string | null\n\t/** Wait until the first word of the first answer, as the user feels it. */\n\tfirstTextMs: number | null\n\tdurationMs: number\n\tsteps: number\n\tinputTokens: number\n\tcacheReadTokens: number\n\tcacheWriteTokens: number\n\toutputTokens: number\n\tcostUsd: number | null\n}\n\nexport type EvalTrace = {\n\tcaseId: string\n\tmodelId: string\n\t/** Facts the turn's instructions held (profile, status line), so the judge does not call them invented. */\n\tinstructionFacts?: unknown\n\tturns: EvalTurn[]\n}\n"]}