@ryanstark24/sfgraph-core 1.2.0 → 1.6.0

package/dist/embedding/cache.js

@@ -0,0 +1,69 @@
+import { mkdirSync } from "node:fs";
+import path from "node:path";
+import Database from "better-sqlite3";
+/** `<dataDir>/embedding-cache/<safe key>.sqlite`. Every character outside
+ *  `[A-Za-z0-9._@-]` becomes `_` and leading dots are stripped, so a model id
+ *  such as `Xenova/all-MiniLM-L6-v2` maps to one file in the cache dir and no
+ *  id can escape it. */
+export function embeddingCachePath(dataDir, modelId) {
+    const safe = modelId.replace(/[^A-Za-z0-9._@-]/g, "_").replace(/^\.+/, "") || "model";
+    return path.join(dataDir, "embedding-cache", `${safe}.sqlite`);
+}
+export class EmbeddingCache {
+    db;
+    getStmt;
+    putTx;
+    constructor(db) {
+        this.db = db;
+        // busy_timeout first, so switching to WAL waits out another process's
+        // lock instead of failing. (better-sqlite3 already defaults to a 5 s
+        // timeout at construction; open() also passes it explicitly.)
+        this.db.pragma("busy_timeout = 5000");
+        this.db.pragma("journal_mode = WAL");
+        this.db.pragma("synchronous = NORMAL");
+        this.db.exec(`CREATE TABLE IF NOT EXISTS vectors (
+         text_sha256 TEXT PRIMARY KEY,
+         dim INTEGER NOT NULL,
+         vector BLOB NOT NULL
+       ) WITHOUT ROWID`);
+        this.getStmt = this.db.prepare("SELECT dim, vector FROM vectors WHERE text_sha256 = ?");
+        const put = this.db.prepare("INSERT OR IGNORE INTO vectors(text_sha256, dim, vector) VALUES (?, ?, ?)");
+        this.putTx = this.db.transaction((entries) => {
+            for (const e of entries) {
+                put.run(e.hash, e.vector.length, Buffer.from(e.vector.buffer, e.vector.byteOffset, e.vector.byteLength));
+            }
+        });
+    }
+    /** Open (creating the directory and table if needed). Throws on failure. */
+    static open(file) {
+        mkdirSync(path.dirname(file), { recursive: true });
+        return new EmbeddingCache(new Database(file, { timeout: 5000 }));
+    }
+    /** Cached vectors for the given hashes; entries of another dim are misses.
+     *  Each returned vector owns a fresh copy of its bytes. */
+    getMany(hashes, dim) {
+        const out = new Map();
+        for (const h of hashes) {
+            if (out.has(h))
+                continue;
+            const row = this.getStmt.get(h);
+            if (!row || row.dim !== dim || row.vector.byteLength !== dim * 4)
+                continue;
+            const v = new Float32Array(dim);
+            new Uint8Array(v.buffer).set(row.vector);
+            out.set(h, v);
+        }
+        return out;
+    }
+    putMany(entries) {
+        if (entries.length > 0)
+            this.putTx(entries);
+    }
+    count() {
+        return this.db.prepare("SELECT COUNT(*) AS c FROM vectors").get().c;
+    }
+    close() {
+        this.db.close();
+    }
+}
+//# sourceMappingURL=cache.js.map

package/dist/embedding/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cache.js","sourceRoot":"","sources":["../../src/embedding/cache.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AACpC,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAmBtC;;;wBAGwB;AACxB,MAAM,UAAU,kBAAkB,CAAC,OAAe,EAAE,OAAe;IACjE,MAAM,IAAI,GAAG,OAAO,CAAC,OAAO,CAAC,mBAAmB,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,IAAI,OAAO,CAAC;IACtF,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,IAAI,SAAS,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,OAAO,cAAc;IAIY;IAHpB,OAAO,CAAC;IACR,KAAK,CAA2E;IAEjG,YAAqC,EAAyB;QAAzB,OAAE,GAAF,EAAE,CAAuB;QAC5D,sEAAsE;QACtE,qEAAqE;QACrE,8DAA8D;QAC9D,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC;QACtC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,sBAAsB,CAAC,CAAC;QACvC,IAAI,CAAC,EAAE,CAAC,IAAI,CACV;;;;uBAIiB,CAClB,CAAC;QACF,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,uDAAuD,CAAC,CAAC;QACxF,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CACzB,0EAA0E,CAC3E,CAAC;QACF,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAC9B,CAAC,OAA8D,EAAE,EAAE;YACjE,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;gBACxB,GAAG,CAAC,GAAG,CACL,CAAC,CAAC,IAAI,EACN,CAAC,CAAC,MAAM,CAAC,MAAM,EACf,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,CACvE,CAAC;YACJ,CAAC;QACH,CAAC,CACF,CAAC;IACJ,CAAC;IAED,4EAA4E;IAC5E,MAAM,CAAC,IAAI,CAAC,IAAY;QACtB,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACnD,OAAO,IAAI,cAAc,CACvB,IAAI,QAAQ,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAqC,CAC1E,CAAC;IACJ,CAAC;IAED;+DAC2D;IAC3D,OAAO,CAAC,MAAyB,EAAE,GAAW;QAC5C,MAAM,GAAG,GAAG,IAAI,GAAG,EAAwB,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;gBAAE,SAAS;YACzB,MAAM,GAAG,GAAG,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAgD,CAAC;YAC/E,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,GAAG,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,KAAK,GAAG,GAAG,CAAC;gBAAE,SAAS;YAC3E,MAAM,CAAC,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;YAChC,IAAI,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAChB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IAED,OAAO,CAAC,OAA8D;QACpE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;YAAE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,KAAK;QACH,OAAQ,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,mCAAmC,CAAC,CAAC,GAAG,EAAoB,CAAC,CAAC,CAAC;IACzF,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;CACF"}

package/dist/embedding/embed.d.ts

@@ -22,10 +22,69 @@ export interface EmbedOptions {
     /** Override embedding dimension. */
     dim?: number;
 }
+/** Minimal view of the @xenova/transformers feature-extraction pipeline we use. */
+interface TensorView {
+    data: ArrayLike<number | bigint>;
+    dims: number[];
+}
+export interface EmbedPipeline {
+    (texts: string[], opts?: Record<string, unknown>): Promise<unknown>;
+    tokenizer: (texts: string[], opts: {
+        padding: boolean;
+        truncation: boolean;
+        max_length?: number;
+    }) => {
+        attention_mask: TensorView;
+    } & Record<string, unknown>;
+    model: (inputs: Record<string, unknown>) => Promise<{
+        last_hidden_state?: TensorView;
+        logits?: TensorView;
+        token_embeddings?: TensorView;
+    }>;
+}
+/**
+ * Model input cap, in tokens. all-MiniLM-L6-v2 was trained with a 256-token
+ * window (sentence-transformers `max_seq_length`), but its tokenizer config
+ * says 512, so `truncation: true` alone let long texts through at 512. With
+ * 64-text sub-batches that made the attention tensors 4x larger
+ * (64 x 12 heads x 512^2 x 4 B ~ 805 MB each) — the main peak-RSS suspect.
+ * `SFGRAPH_EMBED_MAX_TOKENS` overrides it (the cache identity includes a
+ * non-default value).
+ */
+export declare const EMBED_MAX_TOKENS = 256;
+/** Effective token cap: SFGRAPH_EMBED_MAX_TOKENS when a positive integer, else 256. */
+export declare function embedMaxTokens(): number;
+/** The model id embeddings are computed with (opts > env > default). */
+export declare function resolveEmbedModelId(opts?: EmbedOptions): string;
+/** The ONNX weights file the pipeline loads (`quantized: true`):
+ *  `<model dir>/<modelId>/onnx/model_quantized.onnx`, or null without a model dir. */
+export declare function resolveEmbedModelFile(opts?: EmbedOptions): Promise<string | null>;
+/**
+ * SHA-256 (hex) of the model weights file, or null when it does not exist.
+ * Part of the embedding-cache key, so new weights under the same model id
+ * never reuse vectors from the old ones. Memoized per file, size and mtime.
+ */
+export declare function embedModelFingerprint(opts?: EmbedOptions): Promise<string | null>;
+/** The loaded pipeline, or null when the runtime/model is unavailable. For
+ *  tests and diagnostics (e.g. comparing against the library's own pooling). */
+export declare function getEmbedPipeline(opts?: EmbedOptions): Promise<EmbedPipeline | null>;
+/**
+ * Attention-masked mean pooling followed by L2 normalisation, straight over the
+ * model's raw `[batch, seq, dim]` Float32Array. Replaces the library's
+ * `pooling: "mean", normalize: true` path, which walks the tensor with strided,
+ * Proxy-mediated element access (~395 s on a 40k-node rebuild).
+ *
+ * The arithmetic mirrors the library step for step so results match to well
+ * under 1e-6: the per-component sum and token count are accumulated in float64
+ * in sequence order, the mean is rounded to float32, the squared norm is
+ * accumulated in float32, and its square root is taken as `x ** 0.5`.
+ */
+export declare function meanPoolNormalize(hidden: ArrayLike<number>, dims: readonly number[], mask: ArrayLike<number | bigint>): Float32Array[];
 export declare function embedTexts(texts: string[], opts?: EmbedOptions): Promise<Float32Array[]>;
 /** Convenience: embed a single string. Returns null when transformers/model
  *  unavailable — distinct from a zero-vector so callers can distinguish
  *  "the runtime can't embed" from "I embedded but the result happens to be
  *  all zeros." */
 export declare function embedSingle(text: string, opts?: EmbedOptions): Promise<Float32Array | null>;
+export {};
 //# sourceMappingURL=embed.d.ts.map

package/dist/embedding/embed.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"embed.d.ts","sourceRoot":"","sources":["../../src/embedding/embed.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,YAAY;IAC3B,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAKD,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EAAE,EACf,IAAI,GAAE,YAAiB,GACtB,OAAO,CAAC,YAAY,EAAE,CAAC,CA8CzB;AAED;;;kBAGkB;AAClB,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,IAAI,GAAE,YAAiB,GACtB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAoB9B"}
+{"version":3,"file":"embed.d.ts","sourceRoot":"","sources":["../../src/embedding/embed.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;GAeG;AACH,MAAM,WAAW,YAAY;IAC3B,+EAA+E;IAC/E,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yBAAyB;IACzB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,oCAAoC;IACpC,GAAG,CAAC,EAAE,MAAM,CAAC;CACd;AAmBD,mFAAmF;AACnF,UAAU,UAAU;IAClB,IAAI,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,CAAC;IACjC,IAAI,EAAE,MAAM,EAAE,CAAC;CAChB;AACD,MAAM,WAAW,aAAa;IAC5B,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;IACpE,SAAS,EAAE,CACT,KAAK,EAAE,MAAM,EAAE,EACf,IAAI,EAAE;QAAE,OAAO,EAAE,OAAO,CAAC;QAAC,UAAU,EAAE,OAAO,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAA;KAAE,KACjE;QAAE,cAAc,EAAE,UAAU,CAAA;KAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9D,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC;QAClD,iBAAiB,CAAC,EAAE,UAAU,CAAC;QAC/B,MAAM,CAAC,EAAE,UAAU,CAAC;QACpB,gBAAgB,CAAC,EAAE,UAAU,CAAC;KAC/B,CAAC,CAAC;CACJ;AAcD;;;;;;;;GAQG;AACH,eAAO,MAAM,gBAAgB,MAAM,CAAC;AAEpC,uFAAuF;AACvF,wBAAgB,cAAc,IAAI,MAAM,CAGvC;AAED,wEAAwE;AACxE,wBAAgB,mBAAmB,CAAC,IAAI,GAAE,YAAiB,GAAG,MAAM,CAEnE;AAkBD;sFACsF;AACtF,wBAAsB,qBAAqB,CAAC,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAI3F;AAID;;;;GAIG;AACH,wBAAsB,qBAAqB,CAAC,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAoB3F;AA+BD;gFACgF;AAChF,wBAAsB,gBAAgB,CAAC,IAAI,GAAE,YAAiB,GAAG,OAAO,CAAC,aAAa,GAAG,IAAI,CAAC,CAM7F;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,EACzB,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,IAAI,EAAE,SAAS,CAAC,MAAM,GAAG,MAAM,CAAC,GAC/B,YAAY,EAAE,CAiChB;AAED,wBAAsB,UAAU,CAC9B,KAAK,EAAE,MAAM,EAAE,EACf,IAAI,GAAE,YAAiB,GACtB,OAAO,CAAC,YAAY,EAAE,CAAC,CAmCzB;AAED;;;kBAGkB;AAClB,wBAAsB,WAAW,CAC/B,IAAI,EAAE,MAAM,EACZ,IAAI,GAAE,YAAiB,GACtB,OAAO,CAAC,YAAY,GAAG,IAAI,CAAC,CAoB9B"}

package/dist/embedding/embed.js

@@ -1,42 +1,220 @@
+import { createHash } from "node:crypto";
+import { createReadStream } from "node:fs";
+import { stat } from "node:fs/promises";
+import path from "node:path";
 const DEFAULT_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
 const DEFAULT_DIM = 384;
-export async function embedTexts(texts, opts = {}) {
+/** Warn at most once per process when embedding falls back to zero-vectors.
+ *  A silent fallback means semantic search (find_similar) is dead but ingest
+ *  still "succeeds" — exactly how a missing `sharp`/transformers native build
+ *  went unnoticed until a real-org validation. Loud once, not per-batch. */
+let warnedZeroVectorFallback = false;
+function warnZeroVectorFallback(err) {
+    if (warnedZeroVectorFallback)
+        return;
+    warnedZeroVectorFallback = true;
+    const msg = err instanceof Error ? err.message : String(err);
+    console.warn(`sfgraph: embedding model unavailable — falling back to ZERO VECTORS (semantic search will not work). Most common cause: the 'sharp' native dependency wasn't built (@xenova/transformers needs it). Ensure 'sharp' is in pnpm onlyBuiltDependencies and reinstall. Underlying error: ${msg}`);
+}
+/** Memoized feature-extraction pipeline, keyed by modelId+localPath. Loading
+ *  the ONNX model is expensive (~seconds); without this cache, embedTexts
+ *  rebuilt it on EVERY call — i.e. once per 16-text batch during ingest
+ *  (~1,900 reloads on a 30k-node org) and on every ad-hoc find_similar. The
+ *  promise is cached (not just the resolved pipe) so concurrent callers share
+ *  one load. */
+let cachedPipe = null;
+/** Texts per model call inside embedTexts. Inputs are length-sorted first, so
+ *  each sub-batch pads only to its own longest text. */
+const EMBED_SUB_BATCH = 64;
+/**
+ * Model input cap, in tokens. all-MiniLM-L6-v2 was trained with a 256-token
+ * window (sentence-transformers `max_seq_length`), but its tokenizer config
+ * says 512, so `truncation: true` alone let long texts through at 512. With
+ * 64-text sub-batches that made the attention tensors 4x larger
+ * (64 x 12 heads x 512^2 x 4 B ~ 805 MB each) — the main peak-RSS suspect.
+ * `SFGRAPH_EMBED_MAX_TOKENS` overrides it (the cache identity includes a
+ * non-default value).
+ */
+export const EMBED_MAX_TOKENS = 256;
+/** Effective token cap: SFGRAPH_EMBED_MAX_TOKENS when a positive integer, else 256. */
+export function embedMaxTokens() {
+    const n = Number.parseInt(process.env.SFGRAPH_EMBED_MAX_TOKENS ?? "", 10);
+    return Number.isInteger(n) && n > 0 ? n : EMBED_MAX_TOKENS;
+}
+/** The model id embeddings are computed with (opts > env > default). */
+export function resolveEmbedModelId(opts = {}) {
+    return opts.modelId ?? process.env.SFGRAPH_EMBED_MODEL_ID ?? DEFAULT_MODEL_ID;
+}
+/** The directory models are loaded from (transformers' `env.localModelPath`):
+ *  opts.modelPath > SFGRAPH_EMBED_MODEL_PATH > the vendored
+ *  @ryanstark24/sfgraph-models data dir. Undefined when none is available. */
+async function resolveEmbedModelDir(opts) {
+    const envPath = opts.modelPath ?? process.env.SFGRAPH_EMBED_MODEL_PATH;
+    if (envPath)
+        return envPath;
     try {
-        const mod = (await import("@xenova/transformers"));
-        const { pipeline, env } = mod;
-        env.allowRemoteModels = false;
-        const envPath = opts.modelPath ?? process.env.SFGRAPH_EMBED_MODEL_PATH;
-        const modelId = opts.modelId ?? process.env.SFGRAPH_EMBED_MODEL_ID ?? DEFAULT_MODEL_ID;
-        const dim = opts.dim ??
-            (process.env.SFGRAPH_EMBED_MODEL_DIM
-                ? Number.parseInt(process.env.SFGRAPH_EMBED_MODEL_DIM, 10)
-                : DEFAULT_DIM);
-        if (envPath) {
-            env.localModelPath = envPath;
-        }
-        else {
-            try {
-                const models = (await import("@ryanstark24/sfgraph-models"));
-                if (models?.MODEL_DATA_DIR)
-                    env.localModelPath = models.MODEL_DATA_DIR;
+        const models = (await import("@ryanstark24/sfgraph-models"));
+        return models?.MODEL_DATA_DIR || undefined;
+    }
+    catch {
+        return undefined; // models package optional
+    }
+}
+/** The ONNX weights file the pipeline loads (`quantized: true`):
+ *  `<model dir>/<modelId>/onnx/model_quantized.onnx`, or null without a model dir. */
+export async function resolveEmbedModelFile(opts = {}) {
+    const dir = await resolveEmbedModelDir(opts);
+    if (!dir)
+        return null;
+    return path.join(dir, ...resolveEmbedModelId(opts).split("/"), "onnx", "model_quantized.onnx");
+}
+let fingerprintMemo = null;
+/**
+ * SHA-256 (hex) of the model weights file, or null when it does not exist.
+ * Part of the embedding-cache key, so new weights under the same model id
+ * never reuse vectors from the old ones. Memoized per file, size and mtime.
+ */
+export async function embedModelFingerprint(opts = {}) {
+    const file = await resolveEmbedModelFile(opts);
+    if (!file)
+        return null;
+    let st;
+    try {
+        st = await stat(file, { bigint: true });
+    }
+    catch {
+        return null;
+    }
+    const key = `${file}\u0000${st.size}\u0000${st.mtimeNs}`;
+    if (fingerprintMemo?.key === key)
+        return fingerprintMemo.sha;
+    const h = createHash("sha256");
+    try {
+        for await (const chunk of createReadStream(file))
+            h.update(chunk);
+    }
+    catch {
+        return null;
+    }
+    const sha = h.digest("hex");
+    fingerprintMemo = { key, sha };
+    return sha;
+}
+/** Load (memoized) the feature-extraction pipeline. Throws when the runtime or
+ *  model is unavailable. */
+async function loadPipeline(opts) {
+    const mod = (await import("@xenova/transformers"));
+    const { pipeline, env } = mod;
+    env.allowRemoteModels = false;
+    const modelId = resolveEmbedModelId(opts);
+    const modelDir = await resolveEmbedModelDir(opts);
+    if (modelDir)
+        env.localModelPath = modelDir;
+    const key = `${modelId} ${env.localModelPath ?? ""}`;
+    if (!cachedPipe || cachedPipe.key !== key) {
+        cachedPipe = {
+            key,
+            pipe: pipeline("feature-extraction", modelId, { quantized: true, local_files_only: true }),
+        };
+    }
+    try {
+        return await cachedPipe.pipe;
+    }
+    catch (loadErr) {
+        cachedPipe = null; // don't cache a failed load — allow retry next call
+        throw loadErr;
+    }
+}
+/** The loaded pipeline, or null when the runtime/model is unavailable. For
+ *  tests and diagnostics (e.g. comparing against the library's own pooling). */
+export async function getEmbedPipeline(opts = {}) {
+    try {
+        return await loadPipeline(opts);
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Attention-masked mean pooling followed by L2 normalisation, straight over the
+ * model's raw `[batch, seq, dim]` Float32Array. Replaces the library's
+ * `pooling: "mean", normalize: true` path, which walks the tensor with strided,
+ * Proxy-mediated element access (~395 s on a 40k-node rebuild).
+ *
+ * The arithmetic mirrors the library step for step so results match to well
+ * under 1e-6: the per-component sum and token count are accumulated in float64
+ * in sequence order, the mean is rounded to float32, the squared norm is
+ * accumulated in float32, and its square root is taken as `x ** 0.5`.
+ */
+export function meanPoolNormalize(hidden, dims, mask) {
+    const [batch = 0, seq = 0, dim = 0] = dims;
+    const out = [];
+    const sum = new Float64Array(dim);
+    for (let b = 0; b < batch; b++) {
+        sum.fill(0);
+        let count = 0;
+        const rowBase = b * seq * dim;
+        const maskBase = b * seq;
+        for (let j = 0; j < seq; j++) {
+            const attn = Number(mask[maskBase + j]);
+            if (attn === 0)
+                continue; // adds exactly 0 to every component
+            count += attn;
+            const base = rowBase + j * dim;
+            if (attn === 1) {
+                for (let k = 0; k < dim; k++)
+                    sum[k] = sum[k] + hidden[base + k];
             }
-            catch {
-                /* models package optional */
+            else {
+                for (let k = 0; k < dim; k++)
+                    sum[k] = sum[k] + hidden[base + k] * attn;
             }
         }
-        const pipe = await pipeline("feature-extraction", modelId, {
-            quantized: true,
-            local_files_only: true,
+        const v = new Float32Array(dim);
+        let sq = 0;
+        for (let k = 0; k < dim; k++) {
+            const m = Math.fround(sum[k] / count);
+            v[k] = m;
+            sq = Math.fround(sq + m * m);
+        }
+        const norm = Math.fround(sq ** (1 / 2));
+        for (let k = 0; k < dim; k++)
+            v[k] = v[k] / norm;
+        out.push(v);
+    }
+    return out;
+}
+export async function embedTexts(texts, opts = {}) {
+    if (texts.length === 0)
+        return [];
+    try {
+        const pipe = await loadPipeline(opts);
+        // Length-sort (stable) so each sub-batch pads to a similar length, then
+        // scatter results back to input order.
+        const order = texts
+            .map((_, i) => i)
+            .sort((a, b) => {
+            const d = texts[a].length - texts[b].length;
+            return d !== 0 ? d : a - b;
         });
-        const out = await pipe(texts, { pooling: "mean", normalize: true });
-        const results = [];
-        for (let i = 0; i < texts.length; i++) {
-            results.push(new Float32Array(out.data.slice(i * dim, (i + 1) * dim)));
+        const results = new Array(texts.length);
+        const maxLength = embedMaxTokens();
+        for (let s = 0; s < order.length; s += EMBED_SUB_BATCH) {
+            const idx = order.slice(s, s + EMBED_SUB_BATCH);
+            const inputs = pipe.tokenizer(idx.map((i) => texts[i]), { padding: true, truncation: true, max_length: maxLength });
+            const outputs = await pipe.model(inputs);
+            const hidden = outputs.last_hidden_state ?? outputs.logits ?? outputs.token_embeddings;
+            if (!hidden)
+                throw new Error("embedTexts: model returned no hidden state");
+            const vecs = meanPoolNormalize(hidden.data, hidden.dims, inputs.attention_mask.data);
+            for (let k = 0; k < idx.length; k++)
+                results[idx[k]] = vecs[k];
         }
         return results;
     }
-    catch {
-        return texts.map(() => new Float32Array(DEFAULT_DIM));
+    catch (err) {
+        warnZeroVectorFallback(err);
+        return texts.map(() => new Float32Array(opts.dim ?? DEFAULT_DIM));
     }
 }
 /** Convenience: embed a single string. Returns null when transformers/model

package/dist/embedding/embed.js.map

@@ -1 +1 @@
-{"version":3,"file":"embed.js","sourceRoot":"","sources":["../../src/embedding/embed.ts"],"names":[],"mappings":"AAyBA,MAAM,gBAAgB,GAAG,yBAAyB,CAAC;AACnD,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,OAAqB,EAAE;IAEvB,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAKhD,CAAC;QACF,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC;QAC9B,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC;QAE9B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;QACvE,MAAM,OAAO,GACX,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,gBAAgB,CAAC;QACzE,MAAM,GAAG,GACP,IAAI,CAAC,GAAG;YACR,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB;gBAClC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,EAAE,CAAC;gBAC1D,CAAC,CAAC,WAAW,CAAC,CAAC;QAEnB,IAAI,OAAO,EAAE,CAAC;YACZ,GAAG,CAAC,cAAc,GAAG,OAAO,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAC1B,6BAAkD,CACnD,CAAgC,CAAC;gBAClC,IAAI,MAAM,EAAE,cAAc;oBAAE,GAAG,CAAC,cAAc,GAAG,MAAM,CAAC,cAAc,CAAC;YACzE,CAAC;YAAC,MAAM,CAAC;gBACP,6BAA6B;YAC/B,CAAC;QACH,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,OAAO,EAAE;YACzD,SAAS,EAAE,IAAI;YACf,gBAAgB,EAAE,IAAI;SACvB,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpE,MAAM,OAAO,GAAmB,EAAE,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,OAAO,CAAC,IAAI,CAAC,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;QACzE,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,YAAY,CAAC,WAAW,CAAC,CAAC,CAAC;IACxD,CAAC;AACH,CAAC;AAED;;;kBAGkB;AAClB,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAY,EACZ,OAAqB,EAAE;IAEvB,IAAI,CAAC;QACH,kEAAkE;QAClE,uDAAuD;QACvD,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,2EAA2E;IAC3E,yEAAyE;IACzE,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,GAAG,KAAK,CAAC;YAChB,MAAM;QACR,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9B,CAAC"}
+{"version":3,"file":"embed.js","sourceRoot":"","sources":["../../src/embedding/embed.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,SAAS,CAAC;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AACxC,OAAO,IAAI,MAAM,WAAW,CAAC;AA0B7B,MAAM,gBAAgB,GAAG,yBAAyB,CAAC;AACnD,MAAM,WAAW,GAAG,GAAG,CAAC;AAExB;;;4EAG4E;AAC5E,IAAI,wBAAwB,GAAG,KAAK,CAAC;AACrC,SAAS,sBAAsB,CAAC,GAAY;IAC1C,IAAI,wBAAwB;QAAE,OAAO;IACrC,wBAAwB,GAAG,IAAI,CAAC;IAChC,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7D,OAAO,CAAC,IAAI,CACV,wRAAwR,GAAG,EAAE,CAC9R,CAAC;AACJ,CAAC;AAoBD;;;;;gBAKgB;AAChB,IAAI,UAAU,GAAyD,IAAI,CAAC;AAE5E;wDACwD;AACxD,MAAM,eAAe,GAAG,EAAE,CAAC;AAE3B;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAEpC,uFAAuF;AACvF,MAAM,UAAU,cAAc;IAC5B,MAAM,CAAC,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAwB,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;IAC1E,OAAO,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC;AAC7D,CAAC;AAED,wEAAwE;AACxE,MAAM,UAAU,mBAAmB,CAAC,OAAqB,EAAE;IACzD,OAAO,IAAI,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,sBAAsB,IAAI,gBAAgB,CAAC;AAChF,CAAC;AAED;;8EAE8E;AAC9E,KAAK,UAAU,oBAAoB,CAAC,IAAkB;IACpD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC;IACvE,IAAI,OAAO;QAAE,OAAO,OAAO,CAAC;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,CAAC,MAAM,MAAM,CAAC,6BAAkD,CAAC,CAE/E,CAAC;QACF,OAAO,MAAM,EAAE,cAAc,IAAI,SAAS,CAAC;IAC7C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,SAAS,CAAC,CAAC,0BAA0B;IAC9C,CAAC;AACH,CAAC;AAED;sFACsF;AACtF,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAAqB,EAAE;IACjE,MAAM,GAAG,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,MAAM,EAAE,sBAAsB,CAAC,CAAC;AACjG,CAAC;AAED,IAAI,eAAe,GAAwC,IAAI,CAAC;AAEhE;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,qBAAqB,CAAC,OAAqB,EAAE;IACjE,MAAM,IAAI,GAAG,MAAM,qBAAqB,CAAC,IAAI,CAAC,CAAC;IAC/C,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,IAAI,EAAqC,CAAC;IAC1C,IAAI,CAAC;QACH,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,GAAG,IAAI,SAAS,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC,OAAO,EAAE,CAAC;IACzD,IAAI,eAAe,EAAE,GAAG,KAAK,GAAG;QAAE,OAAO,eAAe,CAAC,GAAG,CAAC;IAC7D,MAAM,CAAC,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC/B,IAAI,CAAC;QACH,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,gBAAgB,CAAC,IAAI,CAAC;YAAE,CAAC,CAAC,MAAM,CAAC,KAAe,CAAC,CAAC;IAC9E,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAC5B,eAAe,GAAG,EAAE,GAAG,EAAE,GAAG,EAAE,CAAC;IAC/B,OAAO,GAAG,CAAC;AACb,CAAC;AAED;4BAC4B;AAC5B,KAAK,UAAU,YAAY,CAAC,IAAkB;IAC5C,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAGhD,CAAC;IACF,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,GAAG,GAAG,CAAC;IAC9B,GAAG,CAAC,iBAAiB,GAAG,KAAK,CAAC;IAE9B,MAAM,OAAO,GAAG,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAC1C,MAAM,QAAQ,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,CAAC;IAClD,IAAI,QAAQ;QAAE,GAAG,CAAC,cAAc,GAAG,QAAQ,CAAC;IAE5C,MAAM,GAAG,GAAG,GAAG,OAAO,IAAI,GAAG,CAAC,cAAc,IAAI,EAAE,EAAE,CAAC;IACrD,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,GAAG,KAAK,GAAG,EAAE,CAAC;QAC1C,UAAU,GAAG;YACX,GAAG;YACH,IAAI,EAAE,QAAQ,CAAC,oBAAoB,EAAE,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,gBAAgB,EAAE,IAAI,EAAE,CAAC;SAC3F,CAAC;IACJ,CAAC;IACD,IAAI,CAAC;QACH,OAAO,MAAM,UAAU,CAAC,IAAI,CAAC;IAC/B,CAAC;IAAC,OAAO,OAAO,EAAE,CAAC;QACjB,UAAU,GAAG,IAAI,CAAC,CAAC,oDAAoD;QACvE,MAAM,OAAO,CAAC;IAChB,CAAC;AACH,CAAC;AAED;gFACgF;AAChF,MAAM,CAAC,KAAK,UAAU,gBAAgB,CAAC,OAAqB,EAAE;IAC5D,IAAI,CAAC;QACH,OAAO,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,iBAAiB,CAC/B,MAAyB,EACzB,IAAuB,EACvB,IAAgC;IAEhC,MAAM,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,EAAE,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC;IAC3C,MAAM,GAAG,GAAmB,EAAE,CAAC;IAC/B,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IAClC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;QAC/B,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACZ,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,GAAG,GAAG,CAAC;QAC9B,MAAM,QAAQ,GAAG,CAAC,GAAG,GAAG,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC;YACxC,IAAI,IAAI,KAAK,CAAC;gBAAE,SAAS,CAAC,oCAAoC;YAC9D,KAAK,IAAI,IAAI,CAAC;YACd,MAAM,IAAI,GAAG,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC;YAC/B,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;gBACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;oBAAE,GAAG,CAAC,CAAC,CAAC,GAAI,GAAG,CAAC,CAAC,CAAY,GAAI,MAAM,CAAC,IAAI,GAAG,CAAC,CAAY,CAAC;YAC3F,CAAC;iBAAM,CAAC;gBACN,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;oBAC1B,GAAG,CAAC,CAAC,CAAC,GAAI,GAAG,CAAC,CAAC,CAAY,GAAI,MAAM,CAAC,IAAI,GAAG,CAAC,CAAY,GAAG,IAAI,CAAC;YACtE,CAAC;QACH,CAAC;QACD,MAAM,CAAC,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;QAChC,IAAI,EAAE,GAAG,CAAC,CAAC;QACX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,MAAM,CAAC,GAAG,IAAI,CAAC,MAAM,CAAE,GAAG,CAAC,CAAC,CAAY,GAAG,KAAK,CAAC,CAAC;YAClD,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;YACT,EAAE,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;QAC/B,CAAC;QACD,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACxC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;YAAE,CAAC,CAAC,CAAC,CAAC,GAAI,CAAC,CAAC,CAAC,CAAY,GAAG,IAAI,CAAC;QAC7D,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACd,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAe,EACf,OAAqB,EAAE;IAEvB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAC;QACtC,wEAAwE;QACxE,uCAAuC;QACvC,MAAM,KAAK,GAAG,KAAK;aAChB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;aAChB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACb,MAAM,CAAC,GAAI,KAAK,CAAC,CAAC,CAAY,CAAC,MAAM,GAAI,KAAK,CAAC,CAAC,CAAY,CAAC,MAAM,CAAC;YACpE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QACL,MAAM,OAAO,GAAmB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACxD,MAAM,SAAS,GAAG,cAAc,EAAE,CAAC;QACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,eAAe,EAAE,CAAC;YACvD,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,eAAe,CAAC,CAAC;YAChD,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAC3B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,CAAW,CAAC,EAClC,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,CAC3D,CAAC;YACF,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;YACzC,MAAM,MAAM,GAAG,OAAO,CAAC,iBAAiB,IAAI,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,gBAAgB,CAAC;YACvF,IAAI,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAC3E,MAAM,IAAI,GAAG,iBAAiB,CAC5B,MAAM,CAAC,IAAyB,EAChC,MAAM,CAAC,IAAI,EACX,MAAM,CAAC,cAAc,CAAC,IAAI,CAC3B,CAAC;YACF,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE;gBAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAW,CAAC,GAAG,IAAI,CAAC,CAAC,CAAiB,CAAC;QAC3F,CAAC;QACD,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,sBAAsB,CAAC,GAAG,CAAC,CAAC;QAC5B,OAAO,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,GAAG,IAAI,WAAW,CAAC,CAAC,CAAC;IACpE,CAAC;AACH,CAAC;AAED;;;kBAGkB;AAClB,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAY,EACZ,OAAqB,EAAE;IAEvB,IAAI,CAAC;QACH,kEAAkE;QAClE,uDAAuD;QACvD,MAAM,MAAM,CAAC,sBAAsB,CAAC,CAAC;IACvC,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,UAAU,CAAC,CAAC,IAAI,CAAC,EAAE,IAAI,CAAC,CAAC;IAC7C,IAAI,CAAC,GAAG;QAAE,OAAO,IAAI,CAAC;IACtB,2EAA2E;IAC3E,yEAAyE;IACzE,IAAI,OAAO,GAAG,IAAI,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YACjB,OAAO,GAAG,KAAK,CAAC;YAChB,MAAM;QACR,CAAC;IACH,CAAC;IACD,OAAO,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;AAC9B,CAAC"}

package/dist/embedding/index.d.ts

@@ -3,4 +3,11 @@ export type { EmbeddingItem, EmbeddingQueueOpts, VectorSink } from "./queue.js";
 export { EMBED_DIM } from "./pool.js";
 export { embedTexts, embedSingle } from "./embed.js";
 export type { EmbedOptions } from "./embed.js";
+export { hashEmbedText } from "./queue.js";
+export type { EmbeddingQueueStats } from "./queue.js";
+export { EmbeddingPool, defaultEmbeddingWorkerUrl } from "./pool.js";
+export type { EmbeddingPoolOpts } from "./pool.js";
+export { EmbeddingCache, embeddingCachePath } from "./cache.js";
+export type { EmbeddingCacheLike } from "./cache.js";
+export { EMBED_MAX_TOKENS, embedMaxTokens, embedModelFingerprint, resolveEmbedModelFile, resolveEmbedModelId, } from "./embed.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/embedding/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAChF,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACrD,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAC5C,YAAY,EAAE,aAAa,EAAE,kBAAkB,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAChF,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACrD,YAAY,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAC3C,YAAY,EAAE,mBAAmB,EAAE,MAAM,YAAY,CAAC;AACtD,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,WAAW,CAAC;AACrE,YAAY,EAAE,iBAAiB,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAChE,YAAY,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,qBAAqB,EACrB,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,YAAY,CAAC"}

package/dist/embedding/index.js

@@ -1,4 +1,8 @@
 export { EmbeddingQueue } from "./queue.js";
 export { EMBED_DIM } from "./pool.js";
 export { embedTexts, embedSingle } from "./embed.js";
+export { hashEmbedText } from "./queue.js";
+export { EmbeddingPool, defaultEmbeddingWorkerUrl } from "./pool.js";
+export { EmbeddingCache, embeddingCachePath } from "./cache.js";
+export { EMBED_MAX_TOKENS, embedMaxTokens, embedModelFingerprint, resolveEmbedModelFile, resolveEmbedModelId, } from "./embed.js";
 //# sourceMappingURL=index.js.map

package/dist/embedding/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedding/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAE5C,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAErD,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAE3C,OAAO,EAAE,aAAa,EAAE,yBAAyB,EAAE,MAAM,WAAW,CAAC;AAErE,OAAO,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhE,OAAO,EACL,gBAAgB,EAChB,cAAc,EACd,qBAAqB,EACrB,qBAAqB,EACrB,mBAAmB,GACpB,MAAM,YAAY,CAAC"}

package/dist/embedding/pool.d.ts

@@ -1,10 +1,70 @@
+export declare const EMBED_DIM = 384;
+/** Request posted to the embedding worker. */
+export interface EmbedRequest {
+    id: number;
+    texts: string[];
+}
+/** Reply from the embedding worker: vectors in request order, or an error. */
+export type EmbedReply = {
+    id: number;
+    vectors: Float32Array[];
+} | {
+    id: number;
+    error: string;
+};
+export interface EmbeddingPoolOpts {
+    /** Worker script (defaults to the compiled `./worker.js`). */
+    workerUrl?: URL | string;
+    workerData?: unknown;
+    /**
+     * Liveness bound: if requests are outstanding and the worker sends no reply
+     * for this long, it is treated as hung (terminated; every pending and later
+     * call rejects). It measures lack of progress, so queueing behind earlier
+     * requests does not count. Default SFGRAPH_EMBED_TIMEOUT_MS or 10 minutes
+     * (one batch normally takes well under a second, the first one also loads
+     * the model).
+     */
+    requestTimeoutMs?: number;
+}
+/**
+ * The compiled worker entry next to this module, or null when it is not there
+ * (running from TypeScript sources, e.g. under vitest) or when
+ * `SFGRAPH_EMBED_WORKER=0`. Callers fall back to in-process embedding.
+ */
+export declare function defaultEmbeddingWorkerUrl(): URL | null;
 /**
- * Placeholder for a future piscina-backed embedding pool.
+ * Embedding inference in a worker_thread, so ONNX, tokenisation and pooling run
+ * alongside the ingest fan-out instead of on the main thread.
  *
- * For Commit C we ship an in-process queue (see `./queue.ts`) since
- * worker-thread loading of transformers.js has its own bootstrapping cost
- * that exceeds the budget for a single-batch run. A worker-thread pool is
- * a v1.1 optimization.
+ * One worker: onnxruntime already parallelises each run across cores, so more
+ * workers would only oversubscribe the CPU. Requests are answered in FIFO order.
+ * The worker is spawned on the first call (a warm-cache run never pays the
+ * model load) and is unref'd while idle so it never holds the process open.
+ *
+ * A crash (uncaught error, non-zero exit) or a hang (no reply within
+ * `requestTimeoutMs` while requests are outstanding) rejects every outstanding
+ * request and every later one with that error — callers see an error, never
+ * a hang.
  */
-export declare const EMBED_DIM = 384;
+export declare class EmbeddingPool {
+    private readonly opts;
+    private worker;
+    private nextId;
+    private readonly pending;
+    private crash;
+    private closed;
+    private stallTimer;
+    private readonly timeoutMs;
+    constructor(opts?: EmbeddingPoolOpts);
+    /** Requests posted and not yet answered. */
+    get inFlight(): number;
+    embed(texts: string[]): Promise<Float32Array[]>;
+    close(): Promise<void>;
+    private ensureWorker;
+    private onReply;
+    private onCrash;
+    private armStallTimer;
+    private clearStallTimer;
+    private failAll;
+}
 //# sourceMappingURL=pool.d.ts.map

package/dist/embedding/pool.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"pool.d.ts","sourceRoot":"","sources":["../../src/embedding/pool.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,eAAO,MAAM,SAAS,MAAM,CAAC"}
+{"version":3,"file":"pool.d.ts","sourceRoot":"","sources":["../../src/embedding/pool.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,SAAS,MAAM,CAAC;AAE7B,8CAA8C;AAC9C,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,MAAM,EAAE,CAAC;CACjB;AAED,8EAA8E;AAC9E,MAAM,MAAM,UAAU,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,YAAY,EAAE,CAAA;CAAE,GAAG;IAAE,EAAE,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC;AAEjG,MAAM,WAAW,iBAAiB;IAChC,8DAA8D;IAC9D,SAAS,CAAC,EAAE,GAAG,GAAG,MAAM,CAAC;IACzB,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB;;;;;;;OAOG;IACH,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AASD;;;;GAIG;AACH,wBAAgB,yBAAyB,IAAI,GAAG,GAAG,IAAI,CAQtD;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,aAAa;IASZ,OAAO,CAAC,QAAQ,CAAC,IAAI;IARjC,OAAO,CAAC,MAAM,CAAuB;IACrC,OAAO,CAAC,MAAM,CAAK;IACnB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA8B;IACtD,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,UAAU,CAA8C;IAChE,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;gBAEN,IAAI,GAAE,iBAAsB;IAKzD,4CAA4C;IAC5C,IAAI,QAAQ,IAAI,MAAM,CAErB;IAED,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAoBzC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAU5B,OAAO,CAAC,YAAY;IAqBpB,OAAO,CAAC,OAAO;IAYf,OAAO,CAAC,OAAO;IAOf,OAAO,CAAC,aAAa;IAgBrB,OAAO,CAAC,eAAe;IAKvB,OAAO,CAAC,OAAO;CAKhB"}