@zvec/zvec-grep 0.1.6 → 0.1.7

package/dist/engine/models/backends/model2vec-runtime.d.ts

@@ -0,0 +1,63 @@
+import type { EmbeddingResult } from "../embeddings.js";
+export type TokenTensor = {
+    data: ArrayLike<number | bigint>;
+};
+export type TokenizerOutput = {
+    input_ids: TokenTensor;
+};
+export type TokenizerLike = {
+    (text: string, options: {
+        add_special_tokens: false;
+        truncation: true;
+        max_length: number;
+    }): TokenizerOutput | Promise<TokenizerOutput>;
+    unk_token_id?: number | null;
+};
+export type StaticEmbeddingTable = {
+    data: Float32Array | Uint16Array;
+    dimension: number;
+    dtype: "F16" | "F32";
+    rows: number;
+};
+export type Model2VecWorkerData = {
+    tokenizerSource: string;
+    maxInputTokens: number;
+    normalize: boolean;
+    tableBuffer: SharedArrayBuffer;
+    dimension: number;
+    dtype: "F16" | "F32";
+    rows: number;
+};
+export type Model2VecWorkerRequest = {
+    id: number;
+    texts: string[];
+};
+export type SerializedWorkerError = {
+    name?: string;
+    message: string;
+    stack?: string;
+};
+export type Model2VecWorkerResponse = {
+    type: "ready";
+} | {
+    type: "result";
+    id: number;
+    vectors: ArrayBuffer;
+    vectorCount: number;
+    truncated: number[];
+} | {
+    type: "error";
+    id: number;
+    error: SerializedWorkerError;
+};
+export type TokenizedModel2VecTexts = {
+    tokenLists: number[][];
+    truncated: number[];
+};
+export declare function embedModel2VecTexts(texts: readonly string[], tokenizer: TokenizerLike, table: StaticEmbeddingTable, maxInputTokens: number, normalize: boolean): Promise<EmbeddingResult>;
+export declare function tokenizeModel2VecTexts(texts: readonly string[], tokenizer: TokenizerLike, maxInputTokens: number): Promise<TokenizedModel2VecTexts>;
+export declare function embedModel2VecTokenLists(tokenLists: readonly (readonly number[])[], table: StaticEmbeddingTable, normalize: boolean): number[][];
+export declare function sharedStaticEmbeddingTable(table: StaticEmbeddingTable): StaticEmbeddingTable & {
+    data: Float32Array | Uint16Array;
+};
+export declare function staticEmbeddingTableFromWorkerData(data: Model2VecWorkerData): StaticEmbeddingTable;

package/dist/engine/models/backends/model2vec-runtime.js

@@ -0,0 +1,112 @@
+export async function embedModel2VecTexts(texts, tokenizer, table, maxInputTokens, normalize) {
+    const tokenized = await tokenizeModel2VecTexts(texts, tokenizer, maxInputTokens);
+    return {
+        vectors: embedModel2VecTokenLists(tokenized.tokenLists, table, normalize),
+        truncated: tokenized.truncated,
+    };
+}
+export async function tokenizeModel2VecTexts(texts, tokenizer, maxInputTokens) {
+    const truncatedInputIndexes = [];
+    const tokenLists = await Promise.all(texts.map(async (text, index) => {
+        const encoded = await tokenizer(text, {
+            add_special_tokens: false,
+            truncation: true,
+            max_length: maxInputTokens + 1,
+        });
+        const encodedTokenIds = Array.from(encoded.input_ids.data, Number);
+        if (encodedTokenIds.length > maxInputTokens) {
+            truncatedInputIndexes.push(index);
+        }
+        const unknownTokenId = tokenizer.unk_token_id;
+        return encodedTokenIds
+            .slice(0, maxInputTokens)
+            .filter((id) => unknownTokenId === undefined ||
+            unknownTokenId === null ||
+            id !== unknownTokenId);
+    }));
+    return {
+        tokenLists,
+        truncated: truncatedInputIndexes.sort((left, right) => left - right),
+    };
+}
+export function embedModel2VecTokenLists(tokenLists, table, normalize) {
+    return tokenLists.map((tokenIds) => embedStaticTokenList(tokenIds, table, normalize));
+}
+export function sharedStaticEmbeddingTable(table) {
+    if (table.data.buffer instanceof SharedArrayBuffer) {
+        return table;
+    }
+    const sharedBuffer = new SharedArrayBuffer(table.data.byteLength);
+    new Uint8Array(sharedBuffer).set(new Uint8Array(table.data.buffer, table.data.byteOffset, table.data.byteLength));
+    return {
+        ...table,
+        data: table.dtype === "F16"
+            ? new Uint16Array(sharedBuffer)
+            : new Float32Array(sharedBuffer),
+    };
+}
+export function staticEmbeddingTableFromWorkerData(data) {
+    return {
+        data: data.dtype === "F16"
+            ? new Uint16Array(data.tableBuffer)
+            : new Float32Array(data.tableBuffer),
+        dimension: data.dimension,
+        dtype: data.dtype,
+        rows: data.rows,
+    };
+}
+function embedStaticTokenList(tokenIds, table, normalize) {
+    const dimension = table.dimension;
+    const halfValues = table.dtype === "F16" ? halfFloatValues() : null;
+    const vector = Array.from({ length: dimension }, () => 0);
+    if (tokenIds.length === 0) {
+        return vector;
+    }
+    for (let index = 0; index < tokenIds.length; index++) {
+        const tokenId = tokenIds[index];
+        if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= table.rows) {
+            throw new Error(`Tokenizer returned out-of-range token id: id=${tokenId} rows=${table.rows}`);
+        }
+        const start = tokenId * dimension;
+        for (let column = 0; column < dimension; column++) {
+            const value = table.data[start + column];
+            vector[column] += halfValues ? halfValues[value] : value;
+        }
+    }
+    let squaredNorm = 0;
+    for (let column = 0; column < dimension; column++) {
+        vector[column] /= tokenIds.length;
+        squaredNorm += vector[column] * vector[column];
+    }
+    if (normalize && squaredNorm > 0) {
+        const inverseNorm = 1 / Math.sqrt(squaredNorm);
+        for (let column = 0; column < dimension; column++) {
+            vector[column] *= inverseNorm;
+        }
+    }
+    return vector;
+}
+let cachedHalfFloatValues = null;
+function halfFloatValues() {
+    if (cachedHalfFloatValues) {
+        return cachedHalfFloatValues;
+    }
+    const values = new Float32Array(65_536);
+    for (let bits = 0; bits < values.length; bits++) {
+        const sign = bits & 0x8000 ? -1 : 1;
+        const exponent = (bits >> 10) & 0x1f;
+        const fraction = bits & 0x03ff;
+        if (exponent === 0) {
+            values[bits] = sign * 2 ** -14 * (fraction / 1024);
+        }
+        else if (exponent === 0x1f) {
+            values[bits] = fraction === 0 ? sign * Infinity : Number.NaN;
+        }
+        else {
+            values[bits] = sign * 2 ** (exponent - 15) * (1 + fraction / 1024);
+        }
+    }
+    cachedHalfFloatValues = values;
+    return values;
+}
+//# sourceMappingURL=model2vec-runtime.js.map

package/dist/engine/models/backends/model2vec-runtime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model2vec-runtime.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec-runtime.ts"],"names":[],"mappings":"AAsEA,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAwB,EACxB,SAAwB,EACxB,KAA2B,EAC3B,cAAsB,EACtB,SAAkB;IAElB,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAC5C,KAAK,EACL,SAAS,EACT,cAAc,CACf,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,wBAAwB,CAAC,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC;QACzE,SAAS,EAAE,SAAS,CAAC,SAAS;KAC/B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAwB,EACxB,SAAwB,EACxB,cAAsB;IAEtB,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;QAC9B,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE;YACpC,kBAAkB,EAAE,KAAK;YACzB,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,cAAc,GAAG,CAAC;SAC/B,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,eAAe,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;YAC5C,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;QACD,MAAM,cAAc,GAAG,SAAS,CAAC,YAAY,CAAC;QAC9C,OAAO,eAAe;aACnB,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC;aACxB,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CACL,cAAc,KAAK,SAAS;YAC5B,cAAc,KAAK,IAAI;YACvB,EAAE,KAAK,cAAc,CACxB,CAAC;IACN,CAAC,CAAC,CACH,CAAC;IAEF,OAAO;QACL,UAAU;QACV,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;KACrE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,UAA0C,EAC1C,KAA2B,EAC3B,SAAkB;IAElB,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CACjD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAA2B;IAE3B,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,YAAY,iBAAiB,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClE,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC,GAAG,CAC9B,IAAI,UAAU,CACZ,KAAK,CAAC,IAAI,CAAC,MAAM,EACjB,KAAK,CAAC,IAAI,CAAC,UAAU,EACrB,KAAK,CAAC,IAAI,CAAC,UAAU,CACtB,CACF,CAAC;IACF,OAAO;QACL,GAAG,KAAK;QACR,IAAI,EACF,KAAK,CAAC,KAAK,KAAK,KAAK;YACnB,CAAC,CAAC,IAAI,WAAW,CAAC,YAAY,CAAC;YAC/B,CAAC,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC;KACrC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kCAAkC,CAChD,IAAyB;IAEzB,OAAO;QACL,IAAI,EACF,IAAI,CAAC,KAAK,KAAK,KAAK;YAClB,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC;YACnC,CAAC,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC;QACxC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI,EAAE,IAAI,CAAC,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAA2B,EAC3B,KAA2B,EAC3B,SAAkB;IAElB,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;IAClC,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CACb,gDAAgD,OAAO,SAAS,KAAK,CAAC,IAAI,EAAE,CAC7E,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,GAAG,SAAS,CAAC;QAClC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC;QAClC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,SAAS,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACjC,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/C,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC;QAChC,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,IAAI,qBAAqB,GAAwB,IAAI,CAAC;AAEtD,SAAS,eAAe;IACtB,IAAI,qBAAqB,EAAE,CAAC;QAC1B,OAAO,qBAAqB,CAAC;IAC/B,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IACxC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;QACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,MAAM,CAAC;QAC/B,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;QACrD,CAAC;aAAM,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,qBAAqB,GAAG,MAAM,CAAC;IAC/B,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/engine/models/backends/model2vec-tokenizer.d.ts

@@ -0,0 +1,2 @@
+import type { TokenizerLike } from "./model2vec-runtime.js";
+export declare function loadModel2VecTokenizer(source: string): Promise<TokenizerLike>;

package/dist/engine/models/backends/model2vec-tokenizer.js

@@ -0,0 +1,42 @@
+import { Tokenizer } from "@huggingface/tokenizers";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+export async function loadModel2VecTokenizer(source) {
+    const [tokenizerJson, tokenizerConfig] = await Promise.all([
+        readJson(join(source, "tokenizer.json")),
+        readJson(join(source, "tokenizer_config.json")),
+    ]);
+    const tokenizer = new Tokenizer(tokenizerJson, tokenizerConfig);
+    const unknownTokenId = resolveUnknownTokenId(tokenizer, tokenizerJson);
+    const tokenize = (text, options) => {
+        const ids = tokenizer.encode(text, {
+            add_special_tokens: options.add_special_tokens,
+        }).ids;
+        return {
+            input_ids: {
+                data: ids.length > options.max_length
+                    ? ids.slice(0, options.max_length)
+                    : ids,
+            },
+        };
+    };
+    return Object.assign(tokenize, { unk_token_id: unknownTokenId });
+}
+async function readJson(path) {
+    return JSON.parse(await readFile(path, "utf8"));
+}
+function resolveUnknownTokenId(tokenizer, tokenizerJson) {
+    const configuredId = tokenizerJson.model?.unk_id;
+    if (typeof configuredId === "number" && Number.isInteger(configuredId)) {
+        return configuredId;
+    }
+    const configuredToken = tokenizerJson.model?.unk_token;
+    if (typeof configuredToken !== "string") {
+        return null;
+    }
+    const tokenId = tokenizer.token_to_id(configuredToken);
+    return typeof tokenId === "number" && Number.isInteger(tokenId)
+        ? tokenId
+        : null;
+}
+//# sourceMappingURL=model2vec-tokenizer.js.map

package/dist/engine/models/backends/model2vec-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model2vec-tokenizer.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec-tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAUjC,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAc;IAEd,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACzD,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QACxC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;KAChD,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IAChE,MAAM,cAAc,GAAG,qBAAqB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,CACf,IAAY,EACZ,OAIC,EACD,EAAE;QACF,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE;YACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;SAC/C,CAAC,CAAC,GAAG,CAAC;QACP,OAAO;YACL,SAAS,EAAE;gBACT,IAAI,EACF,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU;oBAC7B,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC;oBAClC,CAAC,CAAC,GAAG;aACV;SACF,CAAC;IACJ,CAAC,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAA4B,CAAC;AAC7E,CAAC;AAED,SAAS,qBAAqB,CAC5B,SAAoB,EACpB,aAA4B;IAE5B,MAAM,YAAY,GAAG,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC;IACjD,IAAI,OAAO,YAAY,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC;QACvE,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,eAAe,GAAG,aAAa,CAAC,KAAK,EAAE,SAAS,CAAC;IACvD,IAAI,OAAO,eAAe,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAY,SAAS,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChE,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;QAC7D,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,IAAI,CAAC;AACX,CAAC"}

package/dist/engine/models/backends/model2vec-worker-pool.d.ts

@@ -0,0 +1,25 @@
+import type { EmbeddingResult } from "../embeddings.js";
+import type { Model2VecWorkerData } from "./model2vec-runtime.js";
+export declare class Model2VecWorkerPool {
+    private readonly data;
+    private readonly workerUrl;
+    private disposed;
+    private nextJobId;
+    private readonly maxWorkers;
+    private readonly queue;
+    private readonly slots;
+    constructor(data: Model2VecWorkerData, maxWorkers?: number, workerUrl?: URL);
+    start(): Promise<void>;
+    run(texts: readonly string[], signal?: AbortSignal): Promise<EmbeddingResult>;
+    dispose(): Promise<void>;
+    private spawnWorker;
+    private handleMessage;
+    private dispatch;
+    private postJob;
+    private drain;
+    private nextQueuedJob;
+    private failSlot;
+    private resolveJob;
+    private rejectJob;
+    private ensureNotDisposed;
+}

package/dist/engine/models/backends/model2vec-worker-pool.js

@@ -0,0 +1,264 @@
+import { availableParallelism } from "node:os";
+import { Worker } from "node:worker_threads";
+export class Model2VecWorkerPool {
+    data;
+    workerUrl;
+    disposed = false;
+    nextJobId = 1;
+    maxWorkers;
+    queue = [];
+    slots = [];
+    constructor(data, maxWorkers = availableParallelism(), workerUrl = new URL("./model2vec-worker.js", import.meta.url)) {
+        this.data = data;
+        this.workerUrl = workerUrl;
+        this.maxWorkers = Math.max(1, Math.floor(maxWorkers));
+    }
+    async start() {
+        this.ensureNotDisposed();
+        if (this.slots.length === 0) {
+            await this.spawnWorker().readyPromise;
+        }
+    }
+    async run(texts, signal) {
+        this.ensureNotDisposed();
+        throwIfAborted(signal);
+        return await new Promise((resolve, reject) => {
+            const job = {
+                id: this.nextJobId++,
+                texts: [...texts],
+                signal,
+                resolve,
+                reject,
+                settled: false,
+            };
+            job.abort = () => {
+                if (job.settled) {
+                    return;
+                }
+                job.settled = true;
+                const queuedIndex = this.queue.indexOf(job);
+                if (queuedIndex >= 0) {
+                    this.queue.splice(queuedIndex, 1);
+                }
+                reject(abortError(signal));
+            };
+            signal?.addEventListener("abort", job.abort, { once: true });
+            const idle = this.slots.find((slot) => slot.ready && !slot.closed && !slot.job);
+            if (idle) {
+                this.dispatch(idle, job);
+            }
+            else if (this.slots.length < this.maxWorkers) {
+                const slot = this.spawnWorker(job);
+                void slot.readyPromise.catch(() => undefined);
+            }
+            else {
+                this.queue.push(job);
+            }
+        });
+    }
+    async dispose() {
+        if (this.disposed) {
+            return;
+        }
+        this.disposed = true;
+        const error = new Error("Model2Vec worker pool is disposed");
+        for (const job of this.queue.splice(0)) {
+            this.rejectJob(job, error);
+        }
+        for (const slot of this.slots) {
+            if (slot.job) {
+                this.rejectJob(slot.job, error);
+                slot.job = undefined;
+            }
+            slot.closed = true;
+        }
+        await Promise.allSettled(this.slots.map((slot) => slot.worker.terminate()));
+        this.slots.length = 0;
+    }
+    spawnWorker(job) {
+        const worker = new Worker(this.workerUrl, { workerData: this.data });
+        let resolveReady;
+        let rejectReady;
+        const readyPromise = new Promise((resolve, reject) => {
+            resolveReady = resolve;
+            rejectReady = reject;
+        });
+        const slot = {
+            worker,
+            ready: false,
+            closed: false,
+            job,
+            resolveReady,
+            rejectReady,
+            readyPromise,
+        };
+        this.slots.push(slot);
+        worker.on("message", (response) => {
+            this.handleMessage(slot, response);
+        });
+        worker.on("error", (error) => {
+            this.failSlot(slot, error);
+        });
+        worker.on("exit", (code) => {
+            if (!slot.closed) {
+                this.failSlot(slot, new Error(`Model2Vec worker exited unexpectedly with code ${code}`));
+            }
+        });
+        return slot;
+    }
+    handleMessage(slot, response) {
+        if (slot.closed) {
+            return;
+        }
+        if (response.type === "ready") {
+            slot.ready = true;
+            slot.resolveReady();
+            if (slot.job) {
+                this.postJob(slot, slot.job);
+            }
+            else {
+                slot.worker.unref();
+                this.drain();
+            }
+            return;
+        }
+        const job = slot.job;
+        if (!job || response.id !== job.id) {
+            this.failSlot(slot, new Error("Model2Vec worker response is out of order"));
+            return;
+        }
+        slot.job = undefined;
+        slot.worker.unref();
+        if (response.type === "error") {
+            this.rejectJob(job, workerError(response.error));
+        }
+        else {
+            try {
+                this.resolveJob(job, workerEmbeddingResult(response, this.data.dimension));
+            }
+            catch (error) {
+                this.rejectJob(job, error);
+                this.failSlot(slot, error);
+                return;
+            }
+        }
+        this.drain();
+    }
+    dispatch(slot, job) {
+        slot.worker.ref();
+        slot.job = job;
+        this.postJob(slot, job);
+    }
+    postJob(slot, job) {
+        if (job.settled) {
+            slot.job = undefined;
+            slot.worker.unref();
+            this.drain();
+            return;
+        }
+        slot.worker.postMessage({
+            id: job.id,
+            texts: job.texts,
+        });
+    }
+    drain() {
+        if (this.disposed) {
+            return;
+        }
+        for (const slot of this.slots) {
+            if (!slot.ready || slot.closed || slot.job) {
+                continue;
+            }
+            const job = this.nextQueuedJob();
+            if (!job) {
+                return;
+            }
+            this.dispatch(slot, job);
+        }
+        while (this.queue.length > 0 && this.slots.length < this.maxWorkers) {
+            const job = this.nextQueuedJob();
+            if (!job) {
+                return;
+            }
+            const slot = this.spawnWorker(job);
+            void slot.readyPromise.catch(() => undefined);
+        }
+    }
+    nextQueuedJob() {
+        while (this.queue.length > 0) {
+            const job = this.queue.shift();
+            if (job && !job.settled) {
+                return job;
+            }
+        }
+        return undefined;
+    }
+    failSlot(slot, error) {
+        if (slot.closed) {
+            return;
+        }
+        slot.closed = true;
+        slot.rejectReady(error);
+        if (slot.job) {
+            this.rejectJob(slot.job, error);
+            slot.job = undefined;
+        }
+        const index = this.slots.indexOf(slot);
+        if (index >= 0) {
+            this.slots.splice(index, 1);
+        }
+        void slot.worker.terminate().catch(() => undefined);
+        this.drain();
+    }
+    resolveJob(job, result) {
+        if (job.settled) {
+            return;
+        }
+        job.settled = true;
+        job.signal?.removeEventListener("abort", job.abort);
+        job.resolve(result);
+    }
+    rejectJob(job, error) {
+        if (job.settled) {
+            return;
+        }
+        job.settled = true;
+        job.signal?.removeEventListener("abort", job.abort);
+        job.reject(error);
+    }
+    ensureNotDisposed() {
+        if (this.disposed) {
+            throw new Error("Model2Vec worker pool is disposed");
+        }
+    }
+}
+function workerEmbeddingResult(response, dimension) {
+    const flat = new Float32Array(response.vectors);
+    if (flat.length !== response.vectorCount * dimension) {
+        throw new Error("Model2Vec worker returned an invalid vector buffer");
+    }
+    const vectors = [];
+    for (let index = 0; index < response.vectorCount; index++) {
+        vectors.push(Array.from(flat.subarray(index * dimension, (index + 1) * dimension)));
+    }
+    return { vectors, truncated: response.truncated };
+}
+function workerError(serialized) {
+    const error = new Error(serialized.message);
+    error.name = serialized.name ?? "Error";
+    if (serialized.stack) {
+        error.stack = serialized.stack;
+    }
+    return error;
+}
+function throwIfAborted(signal) {
+    if (signal?.aborted) {
+        throw abortError(signal);
+    }
+}
+function abortError(signal) {
+    return signal?.reason instanceof Error
+        ? signal.reason
+        : new Error("Model2Vec embedding was cancelled");
+}
+//# sourceMappingURL=model2vec-worker-pool.js.map

package/dist/engine/models/backends/model2vec-worker-pool.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model2vec-worker-pool.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec-worker-pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AA6B7C,MAAM,OAAO,mBAAmB;IAQX;IAEA;IATX,QAAQ,GAAG,KAAK,CAAC;IACjB,SAAS,GAAG,CAAC,CAAC;IACL,UAAU,CAAS;IACnB,KAAK,GAAgB,EAAE,CAAC;IACxB,KAAK,GAAiB,EAAE,CAAC;IAE1C,YACmB,IAAyB,EAC1C,UAAU,GAAG,oBAAoB,EAAE,EAClB,YAAY,IAAI,GAAG,CAClC,uBAAuB,EACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB;QALgB,SAAI,GAAJ,IAAI,CAAqB;QAEzB,cAAS,GAAT,SAAS,CAGzB;QAED,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,YAAY,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CACP,KAAwB,EACxB,MAAoB;QAEpB,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,cAAc,CAAC,MAAM,CAAC,CAAC;QAEvB,OAAO,MAAM,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5D,MAAM,GAAG,GAAc;gBACrB,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE;gBACpB,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC;gBACjB,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,OAAO,EAAE,KAAK;aACf,CAAC;YACF,GAAG,CAAC,KAAK,GAAG,GAAG,EAAE;gBACf,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;oBAChB,OAAO;gBACT,CAAC;gBACD,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;gBACnB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC5C,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;oBACrB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBACpC,CAAC;gBACD,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7B,CAAC,CAAC;YACF,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAE7D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAC1B,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAClD,CAAC;YACF,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACnC,KAAK,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC7D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7B,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;YACvB,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACrB,CAAC;QACD,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IACxB,CAAC;IAEO,WAAW,CAAC,GAAe;QACjC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,IAAI,YAAyB,CAAC;QAC9B,IAAI,WAAsC,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACzD,YAAY,GAAG,OAAO,CAAC;YACvB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAe;YACvB,MAAM;YACN,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,KAAK;YACb,GAAG;YACH,YAAY;YACZ,WAAW;YACX,YAAY;SACb,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,QAAiC,EAAE,EAAE;YACzD,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC3B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,IAAI,CAAC,QAAQ,CACX,IAAI,EACJ,IAAI,KAAK,CAAC,kDAAkD,IAAI,EAAE,CAAC,CACpE,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,aAAa,CACnB,IAAgB,EAChB,QAAiC;QAEjC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAClB,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACpB,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACrB,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CACX,IAAI,EACJ,IAAI,KAAK,CAAC,2CAA2C,CAAC,CACvD,CAAC;YACF,OAAO;QACT,CAAC;QACD,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,IAAI,CAAC,UAAU,CACb,GAAG,EACH,qBAAqB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CACrD,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC3B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC3B,OAAO;YACT,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAEO,QAAQ,CAAC,IAAgB,EAAE,GAAc;QAC/C,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEO,OAAO,CAAC,IAAgB,EAAE,GAAc;QAC9C,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO;QACT,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;YACtB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,KAAK,EAAE,GAAG,CAAC,KAAK;SACgB,CAAC,CAAC;IACtC,CAAC;IAEO,KAAK;QACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,OAAO;YACT,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACpE,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,OAAO;YACT,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACnC,KAAK,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAEO,aAAa;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACxB,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,QAAQ,CAAC,IAAgB,EAAE,KAAc;QAC/C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACxB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;QACvB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9B,CAAC;QACD,KAAK,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACpD,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAEO,UAAU,CAAC,GAAc,EAAE,MAAuB;QACxD,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QACD,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;QACnB,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,KAAM,CAAC,CAAC;QACrD,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IAEO,SAAS,CAAC,GAAc,EAAE,KAAc;QAC9C,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QACD,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;QACnB,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,KAAM,CAAC,CAAC;QACrD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;IAEO,iBAAiB;QACvB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;CACF;AAED,SAAS,qBAAqB,CAC5B,QAA8D,EAC9D,SAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,WAAW,GAAG,SAAS,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC;QAC1D,OAAO,CAAC,IAAI,CACV,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CACtE,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,WAAW,CAAC,UAAiC;IACpD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC5C,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC;IACxC,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;IACjC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,MAAoB;IAC1C,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,MAAoB;IACtC,OAAO,MAAM,EAAE,MAAM,YAAY,KAAK;QACpC,CAAC,CAAC,MAAM,CAAC,MAAM;QACf,CAAC,CAAC,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;AACrD,CAAC"}

package/dist/engine/models/backends/model2vec-worker.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/engine/models/backends/model2vec-worker.js

@@ -0,0 +1,41 @@
+import { parentPort, workerData } from "node:worker_threads";
+import { loadModel2VecTokenizer } from "./model2vec-tokenizer.js";
+import { embedModel2VecTexts, staticEmbeddingTableFromWorkerData, } from "./model2vec-runtime.js";
+const port = parentPort;
+if (!port) {
+    throw new Error("Model2Vec worker requires a parent port");
+}
+const data = workerData;
+const tokenizer = await loadModel2VecTokenizer(data.tokenizerSource);
+const table = staticEmbeddingTableFromWorkerData(data);
+port.postMessage({ type: "ready" });
+port.on("message", async (request) => {
+    try {
+        const result = await embedModel2VecTexts(request.texts, tokenizer, table, data.maxInputTokens, data.normalize);
+        const flatVectors = new Float32Array(result.vectors.length * data.dimension);
+        for (const [index, vector] of result.vectors.entries()) {
+            flatVectors.set(vector, index * data.dimension);
+        }
+        const response = {
+            type: "result",
+            id: request.id,
+            vectors: flatVectors.buffer,
+            vectorCount: result.vectors.length,
+            truncated: result.truncated,
+        };
+        port.postMessage(response, [flatVectors.buffer]);
+    }
+    catch (error) {
+        const cause = error instanceof Error ? error : new Error(String(error));
+        port.postMessage({
+            type: "error",
+            id: request.id,
+            error: {
+                name: cause.name,
+                message: cause.message,
+                stack: cause.stack,
+            },
+        });
+    }
+});
+//# sourceMappingURL=model2vec-worker.js.map

package/dist/engine/models/backends/model2vec-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model2vec-worker.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACL,mBAAmB,EACnB,kCAAkC,GAInC,MAAM,wBAAwB,CAAC;AAEhC,MAAM,IAAI,GAAG,UAAU,CAAC;AACxB,IAAI,CAAC,IAAI,EAAE,CAAC;IACV,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,IAAI,GAAG,UAAiC,CAAC;AAC/C,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;AACrE,MAAM,KAAK,GAAG,kCAAkC,CAAC,IAAI,CAAC,CAAC;AAEvD,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAoC,CAAC,CAAC;AACtE,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,OAA+B,EAAE,EAAE;IAC3D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,mBAAmB,CACtC,OAAO,CAAC,KAAK,EACb,SAAS,EACT,KAAK,EACL,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,SAAS,CACf,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,YAAY,CAClC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CACvC,CAAC;QACF,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,QAAQ,GAA4B;YACxC,IAAI,EAAE,QAAQ;YACd,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,OAAO,EAAE,WAAW,CAAC,MAAM;YAC3B,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;YAClC,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,WAAW,CAAC;YACf,IAAI,EAAE,OAAO;YACb,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,KAAK,EAAE;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB;SACgC,CAAC,CAAC;IACvC,CAAC;AACH,CAAC,CAAC,CAAC"}

package/dist/engine/models/backends/model2vec.d.ts

@@ -1,26 +1,7 @@
 import type { Content } from "../../types.js";
 import { BaseEmbeddingModel, type CreateEmbeddingModelOptions, type EmbeddingModelInfo, type EmbeddingResult, type NormalizedEmbeddingOptions } from "../embeddings.js";
 import type { Model2VecEmbeddingCatalogEntry } from "../catalog.js";
-type TokenTensor = {
-    data: ArrayLike<number | bigint>;
-};
-type TokenizerOutput = {
-    input_ids: TokenTensor;
-};
-type TokenizerLike = {
-    (text: string, options: {
-        add_special_tokens: false;
-        truncation: true;
-        max_length: number;
-    }): TokenizerOutput | Promise<TokenizerOutput>;
-    unk_token_id?: number | null;
-};
-type StaticEmbeddingTable = {
-    data: Float32Array | Uint16Array;
-    dimension: number;
-    dtype: "F16" | "F32";
-    rows: number;
-};
+import { type StaticEmbeddingTable, type TokenizerLike } from "./model2vec-runtime.js";
 type Model2VecDependencies = {
     loadTokenizer(repo: string, options: {
         cache_dir: string;
@@ -40,7 +21,9 @@ export declare class Model2VecEmbeddingModel extends BaseEmbeddingModel {
     private readonly dependencies;
     private tokenizer;
     private staticTable;
+    private workerPool;
     private loadPromise;
+    private readonly useWorkerPool;
     private disposed;
     constructor(entry: Model2VecEmbeddingCatalogEntry, options: CreateEmbeddingModelOptions, dependencies?: Partial<Model2VecDependencies>);
     protected doEmbed(contents: readonly Content[], options: NormalizedEmbeddingOptions): Promise<EmbeddingResult>;