@zvec/zvec-grep 0.2.2 → 0.3.0-rc.2

package/dist/engine/models/backends/model2vec-runtime.js

@@ -1,112 +0,0 @@
-export async function embedModel2VecTexts(texts, tokenizer, table, maxInputTokens, normalize) {
-    const tokenized = await tokenizeModel2VecTexts(texts, tokenizer, maxInputTokens);
-    return {
-        vectors: embedModel2VecTokenLists(tokenized.tokenLists, table, normalize),
-        truncated: tokenized.truncated,
-    };
-}
-export async function tokenizeModel2VecTexts(texts, tokenizer, maxInputTokens) {
-    const truncatedInputIndexes = [];
-    const tokenLists = await Promise.all(texts.map(async (text, index) => {
-        const encoded = await tokenizer(text, {
-            add_special_tokens: false,
-            truncation: true,
-            max_length: maxInputTokens + 1,
-        });
-        const encodedTokenIds = Array.from(encoded.input_ids.data, Number);
-        if (encodedTokenIds.length > maxInputTokens) {
-            truncatedInputIndexes.push(index);
-        }
-        const unknownTokenId = tokenizer.unk_token_id;
-        return encodedTokenIds
-            .slice(0, maxInputTokens)
-            .filter((id) => unknownTokenId === undefined ||
-            unknownTokenId === null ||
-            id !== unknownTokenId);
-    }));
-    return {
-        tokenLists,
-        truncated: truncatedInputIndexes.sort((left, right) => left - right),
-    };
-}
-export function embedModel2VecTokenLists(tokenLists, table, normalize) {
-    return tokenLists.map((tokenIds) => embedStaticTokenList(tokenIds, table, normalize));
-}
-export function sharedStaticEmbeddingTable(table) {
-    if (table.data.buffer instanceof SharedArrayBuffer) {
-        return table;
-    }
-    const sharedBuffer = new SharedArrayBuffer(table.data.byteLength);
-    new Uint8Array(sharedBuffer).set(new Uint8Array(table.data.buffer, table.data.byteOffset, table.data.byteLength));
-    return {
-        ...table,
-        data: table.dtype === "F16"
-            ? new Uint16Array(sharedBuffer)
-            : new Float32Array(sharedBuffer),
-    };
-}
-export function staticEmbeddingTableFromWorkerData(data) {
-    return {
-        data: data.dtype === "F16"
-            ? new Uint16Array(data.tableBuffer)
-            : new Float32Array(data.tableBuffer),
-        dimension: data.dimension,
-        dtype: data.dtype,
-        rows: data.rows,
-    };
-}
-function embedStaticTokenList(tokenIds, table, normalize) {
-    const dimension = table.dimension;
-    const halfValues = table.dtype === "F16" ? halfFloatValues() : null;
-    const vector = Array.from({ length: dimension }, () => 0);
-    if (tokenIds.length === 0) {
-        return vector;
-    }
-    for (let index = 0; index < tokenIds.length; index++) {
-        const tokenId = tokenIds[index];
-        if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= table.rows) {
-            throw new Error(`Tokenizer returned out-of-range token id: id=${tokenId} rows=${table.rows}`);
-        }
-        const start = tokenId * dimension;
-        for (let column = 0; column < dimension; column++) {
-            const value = table.data[start + column];
-            vector[column] += halfValues ? halfValues[value] : value;
-        }
-    }
-    let squaredNorm = 0;
-    for (let column = 0; column < dimension; column++) {
-        vector[column] /= tokenIds.length;
-        squaredNorm += vector[column] * vector[column];
-    }
-    if (normalize && squaredNorm > 0) {
-        const inverseNorm = 1 / Math.sqrt(squaredNorm);
-        for (let column = 0; column < dimension; column++) {
-            vector[column] *= inverseNorm;
-        }
-    }
-    return vector;
-}
-let cachedHalfFloatValues = null;
-function halfFloatValues() {
-    if (cachedHalfFloatValues) {
-        return cachedHalfFloatValues;
-    }
-    const values = new Float32Array(65_536);
-    for (let bits = 0; bits < values.length; bits++) {
-        const sign = bits & 0x8000 ? -1 : 1;
-        const exponent = (bits >> 10) & 0x1f;
-        const fraction = bits & 0x03ff;
-        if (exponent === 0) {
-            values[bits] = sign * 2 ** -14 * (fraction / 1024);
-        }
-        else if (exponent === 0x1f) {
-            values[bits] = fraction === 0 ? sign * Infinity : Number.NaN;
-        }
-        else {
-            values[bits] = sign * 2 ** (exponent - 15) * (1 + fraction / 1024);
-        }
-    }
-    cachedHalfFloatValues = values;
-    return values;
-}
-//# sourceMappingURL=model2vec-runtime.js.map

package/dist/engine/models/backends/model2vec-runtime.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"model2vec-runtime.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec-runtime.ts"],"names":[],"mappings":"AAsEA,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAwB,EACxB,SAAwB,EACxB,KAA2B,EAC3B,cAAsB,EACtB,SAAkB;IAElB,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAC5C,KAAK,EACL,SAAS,EACT,cAAc,CACf,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,wBAAwB,CAAC,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC;QACzE,SAAS,EAAE,SAAS,CAAC,SAAS;KAC/B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAwB,EACxB,SAAwB,EACxB,cAAsB;IAEtB,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;QAC9B,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE;YACpC,kBAAkB,EAAE,KAAK;YACzB,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,cAAc,GAAG,CAAC;SAC/B,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,eAAe,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;YAC5C,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;QACD,MAAM,cAAc,GAAG,SAAS,CAAC,YAAY,CAAC;QAC9C,OAAO,eAAe;aACnB,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC;aACxB,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CACL,cAAc,KAAK,SAAS;YAC5B,cAAc,KAAK,IAAI;YACvB,EAAE,KAAK,cAAc,CACxB,CAAC;IACN,CAAC,CAAC,CACH,CAAC;IAEF,OAAO;QACL,UAAU;QACV,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;KACrE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,UAA0C,EAC1C,KAA2B,EAC3B,SAAkB;IAElB,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CACjD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAA2B;IAE3B,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,YAAY,iBAAiB,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClE,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC,GAAG,CAC9B,IAAI,UAAU,CACZ,KAAK,CAAC,IAAI,CAAC,MAAM,EACjB,KAAK,CAAC,IAAI,CAAC,UAAU,EACrB,KAAK,CAAC,IAAI,CAAC,UAAU,CACtB,CACF,CAAC;IACF,OAAO;QACL,GAAG,KAAK;QACR,IAAI,EACF,KAAK,CAAC,KAAK,KAAK,KAAK;YACnB,CAAC,CAAC,IAAI,WAAW,CAAC,YAAY,CAAC;YAC/B,CAAC,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC;KACrC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kCAAkC,CAChD,IAAyB;IAEzB,OAAO;QACL,IAAI,EACF,IAAI,CAAC,KAAK,KAAK,KAAK;YAClB,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC;YACnC,CAAC,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC;QACxC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI,EAAE,IAAI,CAAC,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAA2B,EAC3B,KAA2B,EAC3B,SAAkB;IAElB,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;IAClC,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CACb,gDAAgD,OAAO,SAAS,KAAK,CAAC,IAAI,EAAE,CAC7E,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,GAAG,SAAS,CAAC;QAClC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC;QAClC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,SAAS,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACjC,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/C,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC;QAChC,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,IAAI,qBAAqB,GAAwB,IAAI,CAAC;AAEtD,SAAS,eAAe;IACtB,IAAI,qBAAqB,EAAE,CAAC;QAC1B,OAAO,qBAAqB,CAAC;IAC/B,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IACxC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;QACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,MAAM,CAAC;QAC/B,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;QACrD,CAAC;aAAM,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,qBAAqB,GAAG,MAAM,CAAC;IAC/B,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/engine/models/backends/model2vec-tokenizer.d.ts

@@ -1,2 +0,0 @@
-import type { TokenizerLike } from "./model2vec-runtime.js";
-export declare function loadModel2VecTokenizer(source: string): Promise<TokenizerLike>;

package/dist/engine/models/backends/model2vec-tokenizer.js

@@ -1,42 +0,0 @@
-import { Tokenizer } from "@huggingface/tokenizers";
-import { readFile } from "node:fs/promises";
-import { join } from "node:path";
-export async function loadModel2VecTokenizer(source) {
-    const [tokenizerJson, tokenizerConfig] = await Promise.all([
-        readJson(join(source, "tokenizer.json")),
-        readJson(join(source, "tokenizer_config.json")),
-    ]);
-    const tokenizer = new Tokenizer(tokenizerJson, tokenizerConfig);
-    const unknownTokenId = resolveUnknownTokenId(tokenizer, tokenizerJson);
-    const tokenize = (text, options) => {
-        const ids = tokenizer.encode(text, {
-            add_special_tokens: options.add_special_tokens,
-        }).ids;
-        return {
-            input_ids: {
-                data: ids.length > options.max_length
-                    ? ids.slice(0, options.max_length)
-                    : ids,
-            },
-        };
-    };
-    return Object.assign(tokenize, { unk_token_id: unknownTokenId });
-}
-async function readJson(path) {
-    return JSON.parse(await readFile(path, "utf8"));
-}
-function resolveUnknownTokenId(tokenizer, tokenizerJson) {
-    const configuredId = tokenizerJson.model?.unk_id;
-    if (typeof configuredId === "number" && Number.isInteger(configuredId)) {
-        return configuredId;
-    }
-    const configuredToken = tokenizerJson.model?.unk_token;
-    if (typeof configuredToken !== "string") {
-        return null;
-    }
-    const tokenId = tokenizer.token_to_id(configuredToken);
-    return typeof tokenId === "number" && Number.isInteger(tokenId)
-        ? tokenId
-        : null;
-}
-//# sourceMappingURL=model2vec-tokenizer.js.map

package/dist/engine/models/backends/model2vec-tokenizer.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"model2vec-tokenizer.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec-tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAUjC,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAc;IAEd,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACzD,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QACxC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;KAChD,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IAChE,MAAM,cAAc,GAAG,qBAAqB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,CACf,IAAY,EACZ,OAIC,EACD,EAAE;QACF,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE;YACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;SAC/C,CAAC,CAAC,GAAG,CAAC;QACP,OAAO;YACL,SAAS,EAAE;gBACT,IAAI,EACF,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU;oBAC7B,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC;oBAClC,CAAC,CAAC,GAAG;aACV;SACF,CAAC;IACJ,CAAC,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAA4B,CAAC;AAC7E,CAAC;AAED,SAAS,qBAAqB,CAC5B,SAAoB,EACpB,aAA4B;IAE5B,MAAM,YAAY,GAAG,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC;IACjD,IAAI,OAAO,YAAY,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC;QACvE,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,eAAe,GAAG,aAAa,CAAC,KAAK,EAAE,SAAS,CAAC;IACvD,IAAI,OAAO,eAAe,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAY,SAAS,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChE,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;QAC7D,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,IAAI,CAAC;AACX,CAAC"}

package/dist/engine/models/backends/model2vec-worker-pool.d.ts

@@ -1,25 +0,0 @@
-import type { EmbeddingResult } from "../embeddings.js";
-import type { Model2VecWorkerData } from "./model2vec-runtime.js";
-export declare class Model2VecWorkerPool {
-    private readonly data;
-    private readonly workerUrl;
-    private disposed;
-    private nextJobId;
-    private readonly maxWorkers;
-    private readonly queue;
-    private readonly slots;
-    constructor(data: Model2VecWorkerData, maxWorkers?: number, workerUrl?: URL);
-    start(): Promise<void>;
-    run(texts: readonly string[], signal?: AbortSignal): Promise<EmbeddingResult>;
-    dispose(): Promise<void>;
-    private spawnWorker;
-    private handleMessage;
-    private dispatch;
-    private postJob;
-    private drain;
-    private nextQueuedJob;
-    private failSlot;
-    private resolveJob;
-    private rejectJob;
-    private ensureNotDisposed;
-}

package/dist/engine/models/backends/model2vec-worker-pool.js

@@ -1,264 +0,0 @@
-import { availableParallelism } from "node:os";
-import { Worker } from "node:worker_threads";
-export class Model2VecWorkerPool {
-    data;
-    workerUrl;
-    disposed = false;
-    nextJobId = 1;
-    maxWorkers;
-    queue = [];
-    slots = [];
-    constructor(data, maxWorkers = availableParallelism(), workerUrl = new URL("./model2vec-worker.js", import.meta.url)) {
-        this.data = data;
-        this.workerUrl = workerUrl;
-        this.maxWorkers = Math.max(1, Math.floor(maxWorkers));
-    }
-    async start() {
-        this.ensureNotDisposed();
-        if (this.slots.length === 0) {
-            await this.spawnWorker().readyPromise;
-        }
-    }
-    async run(texts, signal) {
-        this.ensureNotDisposed();
-        throwIfAborted(signal);
-        return await new Promise((resolve, reject) => {
-            const job = {
-                id: this.nextJobId++,
-                texts: [...texts],
-                signal,
-                resolve,
-                reject,
-                settled: false,
-            };
-            job.abort = () => {
-                if (job.settled) {
-                    return;
-                }
-                job.settled = true;
-                const queuedIndex = this.queue.indexOf(job);
-                if (queuedIndex >= 0) {
-                    this.queue.splice(queuedIndex, 1);
-                }
-                reject(abortError(signal));
-            };
-            signal?.addEventListener("abort", job.abort, { once: true });
-            const idle = this.slots.find((slot) => slot.ready && !slot.closed && !slot.job);
-            if (idle) {
-                this.dispatch(idle, job);
-            }
-            else if (this.slots.length < this.maxWorkers) {
-                const slot = this.spawnWorker(job);
-                void slot.readyPromise.catch(() => undefined);
-            }
-            else {
-                this.queue.push(job);
-            }
-        });
-    }
-    async dispose() {
-        if (this.disposed) {
-            return;
-        }
-        this.disposed = true;
-        const error = new Error("Model2Vec worker pool is disposed");
-        for (const job of this.queue.splice(0)) {
-            this.rejectJob(job, error);
-        }
-        for (const slot of this.slots) {
-            if (slot.job) {
-                this.rejectJob(slot.job, error);
-                slot.job = undefined;
-            }
-            slot.closed = true;
-        }
-        await Promise.allSettled(this.slots.map((slot) => slot.worker.terminate()));
-        this.slots.length = 0;
-    }
-    spawnWorker(job) {
-        const worker = new Worker(this.workerUrl, { workerData: this.data });
-        let resolveReady;
-        let rejectReady;
-        const readyPromise = new Promise((resolve, reject) => {
-            resolveReady = resolve;
-            rejectReady = reject;
-        });
-        const slot = {
-            worker,
-            ready: false,
-            closed: false,
-            job,
-            resolveReady,
-            rejectReady,
-            readyPromise,
-        };
-        this.slots.push(slot);
-        worker.on("message", (response) => {
-            this.handleMessage(slot, response);
-        });
-        worker.on("error", (error) => {
-            this.failSlot(slot, error);
-        });
-        worker.on("exit", (code) => {
-            if (!slot.closed) {
-                this.failSlot(slot, new Error(`Model2Vec worker exited unexpectedly with code ${code}`));
-            }
-        });
-        return slot;
-    }
-    handleMessage(slot, response) {
-        if (slot.closed) {
-            return;
-        }
-        if (response.type === "ready") {
-            slot.ready = true;
-            slot.resolveReady();
-            if (slot.job) {
-                this.postJob(slot, slot.job);
-            }
-            else {
-                slot.worker.unref();
-                this.drain();
-            }
-            return;
-        }
-        const job = slot.job;
-        if (!job || response.id !== job.id) {
-            this.failSlot(slot, new Error("Model2Vec worker response is out of order"));
-            return;
-        }
-        slot.job = undefined;
-        slot.worker.unref();
-        if (response.type === "error") {
-            this.rejectJob(job, workerError(response.error));
-        }
-        else {
-            try {
-                this.resolveJob(job, workerEmbeddingResult(response, this.data.dimension));
-            }
-            catch (error) {
-                this.rejectJob(job, error);
-                this.failSlot(slot, error);
-                return;
-            }
-        }
-        this.drain();
-    }
-    dispatch(slot, job) {
-        slot.worker.ref();
-        slot.job = job;
-        this.postJob(slot, job);
-    }
-    postJob(slot, job) {
-        if (job.settled) {
-            slot.job = undefined;
-            slot.worker.unref();
-            this.drain();
-            return;
-        }
-        slot.worker.postMessage({
-            id: job.id,
-            texts: job.texts,
-        });
-    }
-    drain() {
-        if (this.disposed) {
-            return;
-        }
-        for (const slot of this.slots) {
-            if (!slot.ready || slot.closed || slot.job) {
-                continue;
-            }
-            const job = this.nextQueuedJob();
-            if (!job) {
-                return;
-            }
-            this.dispatch(slot, job);
-        }
-        while (this.queue.length > 0 && this.slots.length < this.maxWorkers) {
-            const job = this.nextQueuedJob();
-            if (!job) {
-                return;
-            }
-            const slot = this.spawnWorker(job);
-            void slot.readyPromise.catch(() => undefined);
-        }
-    }
-    nextQueuedJob() {
-        while (this.queue.length > 0) {
-            const job = this.queue.shift();
-            if (job && !job.settled) {
-                return job;
-            }
-        }
-        return undefined;
-    }
-    failSlot(slot, error) {
-        if (slot.closed) {
-            return;
-        }
-        slot.closed = true;
-        slot.rejectReady(error);
-        if (slot.job) {
-            this.rejectJob(slot.job, error);
-            slot.job = undefined;
-        }
-        const index = this.slots.indexOf(slot);
-        if (index >= 0) {
-            this.slots.splice(index, 1);
-        }
-        void slot.worker.terminate().catch(() => undefined);
-        this.drain();
-    }
-    resolveJob(job, result) {
-        if (job.settled) {
-            return;
-        }
-        job.settled = true;
-        job.signal?.removeEventListener("abort", job.abort);
-        job.resolve(result);
-    }
-    rejectJob(job, error) {
-        if (job.settled) {
-            return;
-        }
-        job.settled = true;
-        job.signal?.removeEventListener("abort", job.abort);
-        job.reject(error);
-    }
-    ensureNotDisposed() {
-        if (this.disposed) {
-            throw new Error("Model2Vec worker pool is disposed");
-        }
-    }
-}
-function workerEmbeddingResult(response, dimension) {
-    const flat = new Float32Array(response.vectors);
-    if (flat.length !== response.vectorCount * dimension) {
-        throw new Error("Model2Vec worker returned an invalid vector buffer");
-    }
-    const vectors = [];
-    for (let index = 0; index < response.vectorCount; index++) {
-        vectors.push(Array.from(flat.subarray(index * dimension, (index + 1) * dimension)));
-    }
-    return { vectors, truncated: response.truncated };
-}
-function workerError(serialized) {
-    const error = new Error(serialized.message);
-    error.name = serialized.name ?? "Error";
-    if (serialized.stack) {
-        error.stack = serialized.stack;
-    }
-    return error;
-}
-function throwIfAborted(signal) {
-    if (signal?.aborted) {
-        throw abortError(signal);
-    }
-}
-function abortError(signal) {
-    return signal?.reason instanceof Error
-        ? signal.reason
-        : new Error("Model2Vec embedding was cancelled");
-}
-//# sourceMappingURL=model2vec-worker-pool.js.map

package/dist/engine/models/backends/model2vec-worker-pool.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"model2vec-worker-pool.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec-worker-pool.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,SAAS,CAAC;AAC/C,OAAO,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AA6B7C,MAAM,OAAO,mBAAmB;IAQX;IAEA;IATX,QAAQ,GAAG,KAAK,CAAC;IACjB,SAAS,GAAG,CAAC,CAAC;IACL,UAAU,CAAS;IACnB,KAAK,GAAgB,EAAE,CAAC;IACxB,KAAK,GAAiB,EAAE,CAAC;IAE1C,YACmB,IAAyB,EAC1C,UAAU,GAAG,oBAAoB,EAAE,EAClB,YAAY,IAAI,GAAG,CAClC,uBAAuB,EACvB,MAAM,CAAC,IAAI,CAAC,GAAG,CAChB;QALgB,SAAI,GAAJ,IAAI,CAAqB;QAEzB,cAAS,GAAT,SAAS,CAGzB;QAED,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC;IACxD,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC,YAAY,CAAC;QACxC,CAAC;IACH,CAAC;IAED,KAAK,CAAC,GAAG,CACP,KAAwB,EACxB,MAAoB;QAEpB,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,cAAc,CAAC,MAAM,CAAC,CAAC;QAEvB,OAAO,MAAM,IAAI,OAAO,CAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YAC5D,MAAM,GAAG,GAAc;gBACrB,EAAE,EAAE,IAAI,CAAC,SAAS,EAAE;gBACpB,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC;gBACjB,MAAM;gBACN,OAAO;gBACP,MAAM;gBACN,OAAO,EAAE,KAAK;aACf,CAAC;YACF,GAAG,CAAC,KAAK,GAAG,GAAG,EAAE;gBACf,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;oBAChB,OAAO;gBACT,CAAC;gBACD,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;gBACnB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;gBAC5C,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;oBACrB,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;gBACpC,CAAC;gBACD,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC;YAC7B,CAAC,CAAC;YACF,MAAM,EAAE,gBAAgB,CAAC,OAAO,EAAE,GAAG,CAAC,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAE7D,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAC1B,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,IAAI,CAAC,GAAG,CAClD,CAAC;YACF,IAAI,IAAI,EAAE,CAAC;gBACT,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;gBACnC,KAAK,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACvB,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QAC7D,KAAK,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;YACvC,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QAC7B,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAChC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;YACvB,CAAC;YACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACrB,CAAC;QACD,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;QAC5E,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;IACxB,CAAC;IAEO,WAAW,CAAC,GAAe;QACjC,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QACrE,IAAI,YAAyB,CAAC;QAC9B,IAAI,WAAsC,CAAC;QAC3C,MAAM,YAAY,GAAG,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACzD,YAAY,GAAG,OAAO,CAAC;YACvB,WAAW,GAAG,MAAM,CAAC;QACvB,CAAC,CAAC,CAAC;QACH,MAAM,IAAI,GAAe;YACvB,MAAM;YACN,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,KAAK;YACb,GAAG;YACH,YAAY;YACZ,WAAW;YACX,YAAY;SACb,CAAC;QACF,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,QAAiC,EAAE,EAAE;YACzD,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC3B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;QAC7B,CAAC,CAAC,CAAC;QACH,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACzB,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC;gBACjB,IAAI,CAAC,QAAQ,CACX,IAAI,EACJ,IAAI,KAAK,CAAC,kDAAkD,IAAI,EAAE,CAAC,CACpE,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,aAAa,CACnB,IAAgB,EAChB,QAAiC;QAEjC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;YAClB,IAAI,CAAC,YAAY,EAAE,CAAC;YACpB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBACb,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC;YAC/B,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;gBACpB,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,CAAC;YACD,OAAO;QACT,CAAC;QAED,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC;QACrB,IAAI,CAAC,GAAG,IAAI,QAAQ,CAAC,EAAE,KAAK,GAAG,CAAC,EAAE,EAAE,CAAC;YACnC,IAAI,CAAC,QAAQ,CACX,IAAI,EACJ,IAAI,KAAK,CAAC,2CAA2C,CAAC,CACvD,CAAC;YACF,OAAO;QACT,CAAC;QACD,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;QACrB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,QAAQ,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC9B,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,WAAW,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QACnD,CAAC;aAAM,CAAC;YACN,IAAI,CAAC;gBACH,IAAI,CAAC,UAAU,CACb,GAAG,EACH,qBAAqB,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CACrD,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,IAAI,CAAC,SAAS,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;gBAC3B,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAC3B,OAAO;YACT,CAAC;QACH,CAAC;QACD,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAEO,QAAQ,CAAC,IAAgB,EAAE,GAAc;QAC/C,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAClB,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;QACf,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAC1B,CAAC;IAEO,OAAO,CAAC,IAAgB,EAAE,GAAc;QAC9C,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;YACpB,IAAI,CAAC,KAAK,EAAE,CAAC;YACb,OAAO;QACT,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC;YACtB,EAAE,EAAE,GAAG,CAAC,EAAE;YACV,KAAK,EAAE,GAAG,CAAC,KAAK;SACgB,CAAC,CAAC;IACtC,CAAC;IAEO,KAAK;QACX,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YAC9B,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC3C,SAAS;YACX,CAAC;YACD,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,OAAO;YACT,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QACD,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;YACpE,MAAM,GAAG,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YACjC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,OAAO;YACT,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;YACnC,KAAK,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IAEO,aAAa;QACnB,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;YAC/B,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;gBACxB,OAAO,GAAG,CAAC;YACb,CAAC;QACH,CAAC;QACD,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,QAAQ,CAAC,IAAgB,EAAE,KAAc;QAC/C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC;QACnB,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,CAAC;QACxB,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC;YACb,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG,GAAG,SAAS,CAAC;QACvB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,KAAK,IAAI,CAAC,EAAE,CAAC;YACf,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;QAC9B,CAAC;QACD,KAAK,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;QACpD,IAAI,CAAC,KAAK,EAAE,CAAC;IACf,CAAC;IAEO,UAAU,CAAC,GAAc,EAAE,MAAuB;QACxD,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QACD,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;QACnB,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,KAAM,CAAC,CAAC;QACrD,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IAEO,SAAS,CAAC,GAAc,EAAE,KAAc;QAC9C,IAAI,GAAG,CAAC,OAAO,EAAE,CAAC;YAChB,OAAO;QACT,CAAC;QACD,GAAG,CAAC,OAAO,GAAG,IAAI,CAAC;QACnB,GAAG,CAAC,MAAM,EAAE,mBAAmB,CAAC,OAAO,EAAE,GAAG,CAAC,KAAM,CAAC,CAAC;QACrD,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpB,CAAC;IAEO,iBAAiB;QACvB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACvD,CAAC;IACH,CAAC;CACF;AAED,SAAS,qBAAqB,CAC5B,QAA8D,EAC9D,SAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,YAAY,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IAChD,IAAI,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,WAAW,GAAG,SAAS,EAAE,CAAC;QACrD,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAC;IACxE,CAAC;IACD,MAAM,OAAO,GAAe,EAAE,CAAC;IAC/B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,KAAK,EAAE,EAAE,CAAC;QAC1D,OAAO,CAAC,IAAI,CACV,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,SAAS,EAAE,CAAC,KAAK,GAAG,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC,CACtE,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,QAAQ,CAAC,SAAS,EAAE,CAAC;AACpD,CAAC;AAED,SAAS,WAAW,CAAC,UAAiC;IACpD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC5C,KAAK,CAAC,IAAI,GAAG,UAAU,CAAC,IAAI,IAAI,OAAO,CAAC;IACxC,IAAI,UAAU,CAAC,KAAK,EAAE,CAAC;QACrB,KAAK,CAAC,KAAK,GAAG,UAAU,CAAC,KAAK,CAAC;IACjC,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CAAC,MAAoB;IAC1C,IAAI,MAAM,EAAE,OAAO,EAAE,CAAC;QACpB,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,MAAoB;IACtC,OAAO,MAAM,EAAE,MAAM,YAAY,KAAK;QACpC,CAAC,CAAC,MAAM,CAAC,MAAM;QACf,CAAC,CAAC,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC;AACrD,CAAC"}

package/dist/engine/models/backends/model2vec-worker.d.ts

@@ -1 +0,0 @@
-export {};

package/dist/engine/models/backends/model2vec-worker.js

@@ -1,41 +0,0 @@
-import { parentPort, workerData } from "node:worker_threads";
-import { loadModel2VecTokenizer } from "./model2vec-tokenizer.js";
-import { embedModel2VecTexts, staticEmbeddingTableFromWorkerData, } from "./model2vec-runtime.js";
-const port = parentPort;
-if (!port) {
-    throw new Error("Model2Vec worker requires a parent port");
-}
-const data = workerData;
-const tokenizer = await loadModel2VecTokenizer(data.tokenizerSource);
-const table = staticEmbeddingTableFromWorkerData(data);
-port.postMessage({ type: "ready" });
-port.on("message", async (request) => {
-    try {
-        const result = await embedModel2VecTexts(request.texts, tokenizer, table, data.maxInputTokens, data.normalize);
-        const flatVectors = new Float32Array(result.vectors.length * data.dimension);
-        for (const [index, vector] of result.vectors.entries()) {
-            flatVectors.set(vector, index * data.dimension);
-        }
-        const response = {
-            type: "result",
-            id: request.id,
-            vectors: flatVectors.buffer,
-            vectorCount: result.vectors.length,
-            truncated: result.truncated,
-        };
-        port.postMessage(response, [flatVectors.buffer]);
-    }
-    catch (error) {
-        const cause = error instanceof Error ? error : new Error(String(error));
-        port.postMessage({
-            type: "error",
-            id: request.id,
-            error: {
-                name: cause.name,
-                message: cause.message,
-                stack: cause.stack,
-            },
-        });
-    }
-});
-//# sourceMappingURL=model2vec-worker.js.map

package/dist/engine/models/backends/model2vec-worker.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"model2vec-worker.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec-worker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AAC7D,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EACL,mBAAmB,EACnB,kCAAkC,GAInC,MAAM,wBAAwB,CAAC;AAEhC,MAAM,IAAI,GAAG,UAAU,CAAC;AACxB,IAAI,CAAC,IAAI,EAAE,CAAC;IACV,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;AAC7D,CAAC;AAED,MAAM,IAAI,GAAG,UAAiC,CAAC;AAC/C,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;AACrE,MAAM,KAAK,GAAG,kCAAkC,CAAC,IAAI,CAAC,CAAC;AAEvD,IAAI,CAAC,WAAW,CAAC,EAAE,IAAI,EAAE,OAAO,EAAoC,CAAC,CAAC;AACtE,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,EAAE,OAA+B,EAAE,EAAE;IAC3D,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,mBAAmB,CACtC,OAAO,CAAC,KAAK,EACb,SAAS,EACT,KAAK,EACL,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,SAAS,CACf,CAAC;QACF,MAAM,WAAW,GAAG,IAAI,YAAY,CAClC,MAAM,CAAC,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,SAAS,CACvC,CAAC;QACF,KAAK,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YACvD,WAAW,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAClD,CAAC;QACD,MAAM,QAAQ,GAA4B;YACxC,IAAI,EAAE,QAAQ;YACd,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,OAAO,EAAE,WAAW,CAAC,MAAM;YAC3B,WAAW,EAAE,MAAM,CAAC,OAAO,CAAC,MAAM;YAClC,SAAS,EAAE,MAAM,CAAC,SAAS;SAC5B,CAAC;QACF,IAAI,CAAC,WAAW,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;IACnD,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,KAAK,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACxE,IAAI,CAAC,WAAW,CAAC;YACf,IAAI,EAAE,OAAO;YACb,EAAE,EAAE,OAAO,CAAC,EAAE;YACd,KAAK,EAAE;gBACL,IAAI,EAAE,KAAK,CAAC,IAAI;gBAChB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;aACnB;SACgC,CAAC,CAAC;IACvC,CAAC;AACH,CAAC,CAAC,CAAC"}

package/dist/engine/models/backends/model2vec.d.ts

@@ -1,38 +0,0 @@
-import type { Content } from "../../types.js";
-import { BaseEmbeddingModel, type CreateEmbeddingModelOptions, type EmbeddingModelInfo, type EmbeddingOptions, type EmbeddingResult, type NormalizedEmbeddingOptions } from "../embeddings.js";
-import type { Model2VecEmbeddingCatalogEntry } from "../catalog.js";
-import { resolveModelArtifacts } from "../artifact-downloader.js";
-import { type StaticEmbeddingTable, type TokenizerLike } from "./model2vec-runtime.js";
-type Model2VecDependencies = {
-    loadTokenizer(repo: string, options: {
-        cache_dir: string;
-        revision: string;
-        local_files_only?: boolean;
-    }): Promise<TokenizerLike>;
-    loadSafetensors(path: string, tensorName: string, dimension: number): Promise<StaticEmbeddingTable>;
-    resolveArtifacts: typeof resolveModelArtifacts;
-};
-export declare class Model2VecEmbeddingModel extends BaseEmbeddingModel {
-    private readonly entry;
-    readonly info: EmbeddingModelInfo;
-    private readonly modelCacheDir;
-    private readonly dependencies;
-    private tokenizer;
-    private staticTable;
-    private workerPool;
-    private loadPromise;
-    private readonly useWorkerPool;
-    private sourceFallbackWarningReported;
-    private disposed;
-    constructor(entry: Model2VecEmbeddingCatalogEntry, options: CreateEmbeddingModelOptions, dependencies?: Partial<Model2VecDependencies>);
-    protected doEmbed(contents: readonly Content[], options: NormalizedEmbeddingOptions): Promise<EmbeddingResult>;
-    prepare(options?: Pick<EmbeddingOptions, "signal" | "onProgress">): Promise<void>;
-    private embedBatch;
-    dispose(): Promise<void>;
-    private ensureLoaded;
-    private loadModel;
-    private createArtifactSources;
-    private embedTexts;
-    private ensureNotDisposed;
-}
-export {};