@zvec/zvec-grep 0.2.1 → 0.3.0-rc.1

package/dist/engine/models/backends/transformers-js.js

@@ -1,316 +0,0 @@
-import { join } from "node:path";
-import { EngineError } from "../../errors.js";
-import { defaultHome } from "../../utils/path.js";
-import { BaseEmbeddingModel, } from "../embeddings.js";
-import { createModelDownloadProgressReporter, } from "../download-progress.js";
-const DEFAULT_MODEL_CACHE_DIR = join(defaultHome(), "models");
-async function defaultTransformersJsLoader() {
-    try {
-        return (await import("@huggingface/transformers"));
-    }
-    catch (cause) {
-        throw new EngineError("Transformers.js is required for this local embedding model", {
-            code: "ZVEC_GREP.ENGINE.MODELS.TRANSFORMERS_JS_MISSING_DEPENDENCY",
-            context: "Reinstall zvec-grep to restore @huggingface/transformers",
-            cause,
-        });
-    }
-}
-let defaultRuntimeImport = null;
-const defaultDependencies = {
-    loadRuntime() {
-        defaultRuntimeImport ??= defaultTransformersJsLoader();
-        return defaultRuntimeImport;
-    },
-};
-export class TransformersJsEmbeddingModel extends BaseEmbeddingModel {
-    entry;
-    info;
-    modelCacheDir;
-    executionProvider;
-    dependencies;
-    pipeline = null;
-    pipelineLoadPromise = null;
-    usingCpuFallback = false;
-    disposed = false;
-    constructor(entry, options, dependencies = {}) {
-        super();
-        this.entry = entry;
-        this.info = {
-            reference: entry.reference,
-            provider: entry.provider,
-            name: entry.model,
-            dimension: entry.dimension,
-            metric: entry.metric,
-            inputKinds: ["text"],
-            limits: {
-                maxBatchSize: entry.maxBatchSize,
-                maxInputTokens: entry.maxInputTokens,
-            },
-        };
-        this.modelCacheDir =
-            options.modelCacheDir ??
-                process.env.ZVEC_GREP_MODEL_CACHE ??
-                DEFAULT_MODEL_CACHE_DIR;
-        this.executionProvider = resolveExecutionProvider(options.device);
-        this.dependencies = { ...defaultDependencies, ...dependencies };
-    }
-    async doEmbed(contents, options) {
-        return await this.embedBatch(contents, options);
-    }
-    async embedBatch(contents, options) {
-        this.ensureNotDisposed();
-        const texts = contents.map((content) => formatText(content.text, options.purpose, this.entry));
-        let pipeline;
-        try {
-            pipeline = await this.ensurePipeline(options.onProgress);
-        }
-        catch (cause) {
-            throw new EngineError("Transformers.js embedding failed", {
-                code: "ZVEC_GREP.ENGINE.MODELS.TRANSFORMERS_JS_EMBED_FAILED",
-                context: `model=${this.entry.reference} repo=${this.entry.repo}`,
-                cause,
-            });
-        }
-        let truncatedInputIndexes;
-        try {
-            truncatedInputIndexes = await findTruncatedInputIndexes(pipeline.tokenizer, texts, this.entry.maxInputTokens);
-        }
-        catch (cause) {
-            throw new EngineError("Transformers.js tokenization failed", {
-                code: "ZVEC_GREP.ENGINE.MODELS.TRANSFORMERS_JS_TOKENIZATION_FAILED",
-                context: `model=${this.entry.reference} repo=${this.entry.repo}`,
-                cause,
-            });
-        }
-        let failure;
-        try {
-            return await this.embedTexts(pipeline, texts, truncatedInputIndexes);
-        }
-        catch (cause) {
-            failure = cause;
-        }
-        if (await this.fallbackToCpu(failure, options.onProgress)) {
-            try {
-                return await this.embedTexts(await this.ensurePipeline(options.onProgress), texts, truncatedInputIndexes);
-            }
-            catch (cause) {
-                failure = cause;
-            }
-        }
-        throw new EngineError("Transformers.js embedding failed", {
-            code: "ZVEC_GREP.ENGINE.MODELS.TRANSFORMERS_JS_EMBED_FAILED",
-            context: `model=${this.entry.reference} repo=${this.entry.repo}`,
-            cause: failure,
-        });
-    }
-    async dispose() {
-        if (this.disposed) {
-            return;
-        }
-        this.disposed = true;
-        const pipeline = this.pipeline;
-        this.pipeline = null;
-        this.pipelineLoadPromise = null;
-        await pipeline?.dispose();
-    }
-    async ensurePipeline(onProgress) {
-        if (this.pipeline) {
-            return this.pipeline;
-        }
-        if (this.pipelineLoadPromise) {
-            return await this.pipelineLoadPromise;
-        }
-        this.pipelineLoadPromise = this.loadPipeline(onProgress);
-        try {
-            this.pipeline = await this.pipelineLoadPromise;
-            return this.pipeline;
-        }
-        finally {
-            this.pipelineLoadPromise = null;
-        }
-    }
-    async loadPipeline(onProgress) {
-        const downloadProgress = createModelDownloadProgressReporter(this.entry.reference, onProgress);
-        downloadProgress.start();
-        const runtime = await this.dependencies.loadRuntime();
-        let pipeline;
-        const executionProvider = this.usingCpuFallback
-            ? "cpu"
-            : this.executionProvider;
-        try {
-            pipeline = await this.createPipeline(runtime, executionProvider, downloadProgress);
-        }
-        catch (cause) {
-            if (!executionProvider || executionProvider === "cpu") {
-                throw cause;
-            }
-            const warning = `Transformers.js ${executionProvider} embedding initialization failed (${formatErrorMessage(cause)}), falling back to CPU.`;
-            if (!downloadProgress.warning(warning)) {
-                process.stderr.write(`zvec-grep warning: ${warning}\n`);
-            }
-            this.usingCpuFallback = true;
-            pipeline = await this.createPipeline(runtime, "cpu", downloadProgress);
-        }
-        pipeline.tokenizer.model_max_length = this.entry.maxInputTokens;
-        downloadProgress.finish();
-        return pipeline;
-    }
-    async embedTexts(pipeline, texts, truncatedInputIndexes) {
-        const tensor = await pipeline(texts, {
-            pooling: this.entry.pooling,
-            normalize: this.entry.normalize,
-            truncation: true,
-            max_length: this.entry.maxInputTokens,
-        });
-        return {
-            vectors: tensorToVectors(tensor, texts.length, this.entry.dimension),
-            truncated: truncatedInputIndexes,
-        };
-    }
-    async fallbackToCpu(cause, onProgress) {
-        if (this.usingCpuFallback ||
-            !this.executionProvider ||
-            this.executionProvider === "cpu") {
-            return false;
-        }
-        const warning = `Transformers.js ${this.executionProvider} embedding inference failed (${formatErrorMessage(cause)}), falling back to CPU.`;
-        if (onProgress) {
-            onProgress({
-                stage: "warning",
-                model: this.entry.reference,
-                message: warning,
-            });
-        }
-        else {
-            process.stderr.write(`zvec-grep warning: ${warning}\n`);
-        }
-        this.usingCpuFallback = true;
-        const pipeline = this.pipeline;
-        this.pipeline = null;
-        this.pipelineLoadPromise = null;
-        await pipeline?.dispose();
-        return true;
-    }
-    async createPipeline(runtime, executionProvider, downloadProgress) {
-        return await runtime.pipeline("feature-extraction", this.entry.repo, {
-            cache_dir: this.modelCacheDir,
-            revision: this.entry.revision,
-            dtype: this.entry.dtype,
-            progress_callback: (progress) => {
-                if (progress.status === "initiate" &&
-                    typeof progress.file === "string") {
-                    downloadProgress.register(progress.file);
-                    return;
-                }
-                if (progress.status !== "progress" ||
-                    typeof progress.file !== "string" ||
-                    typeof progress.loaded !== "number") {
-                    return;
-                }
-                downloadProgress.report({
-                    artifact: progress.file,
-                    downloadedBytes: progress.loaded,
-                    totalBytes: progress.total,
-                });
-            },
-            ...(executionProvider
-                ? { session_options: { executionProviders: [executionProvider] } }
-                : {}),
-        });
-    }
-    ensureNotDisposed() {
-        if (this.disposed) {
-            throw new EngineError("Transformers.js embedding model is disposed", {
-                code: "ZVEC_GREP.ENGINE.MODELS.TRANSFORMERS_JS_DISPOSED",
-                context: `model=${this.entry.reference}`,
-            });
-        }
-    }
-}
-async function findTruncatedInputIndexes(tokenizer, texts, maxInputTokens) {
-    const encoded = await tokenizer(texts, {
-        truncation: true,
-        max_length: maxInputTokens + 1,
-        padding: true,
-    });
-    const mask = encoded.attention_mask;
-    const [batchSize, sequenceLength] = mask.dims;
-    if (mask.dims.length !== 2 ||
-        batchSize !== texts.length ||
-        !Number.isInteger(sequenceLength) ||
-        sequenceLength < 0 ||
-        mask.data.length !== batchSize * sequenceLength) {
-        throw new Error("Transformers.js tokenizer returned an unexpected mask");
-    }
-    const truncatedInputIndexes = [];
-    for (let inputIndex = 0; inputIndex < batchSize; inputIndex++) {
-        let tokenCount = 0;
-        const offset = inputIndex * sequenceLength;
-        for (let tokenIndex = 0; tokenIndex < sequenceLength; tokenIndex++) {
-            if (Number(mask.data[offset + tokenIndex]) !== 0) {
-                tokenCount++;
-            }
-        }
-        if (tokenCount > maxInputTokens) {
-            truncatedInputIndexes.push(inputIndex);
-        }
-    }
-    return truncatedInputIndexes;
-}
-function resolveExecutionProvider(device) {
-    if (device === undefined) {
-        return null;
-    }
-    if (device === "cpu") {
-        return "cpu";
-    }
-    if (device === "metal" || device === "vulkan") {
-        return "webgpu";
-    }
-    if (device === "cuda") {
-        return "cuda";
-    }
-    if (process.platform === "win32") {
-        return "dml";
-    }
-    if (process.platform === "linux" && process.arch === "x64") {
-        return "cuda";
-    }
-    return "webgpu";
-}
-function formatErrorMessage(error) {
-    return error instanceof Error ? error.message : String(error);
-}
-function formatText(text, purpose, entry) {
-    const prefix = purpose === "query"
-        ? "queryPrefix" in entry
-            ? entry.queryPrefix
-            : undefined
-        : "documentPrefix" in entry
-            ? entry.documentPrefix
-            : undefined;
-    return prefix ? `${prefix}${text}` : text;
-}
-function tensorToVectors(tensor, count, dimension) {
-    if (tensor.dims.length !== 2 ||
-        tensor.dims[0] !== count ||
-        tensor.dims[1] !== dimension ||
-        tensor.data.length !== count * dimension) {
-        throw new EngineError("Transformers.js returned an unexpected tensor", {
-            code: "ZVEC_GREP.ENGINE.MODELS.TRANSFORMERS_JS_INVALID_TENSOR",
-            context: `expected=${count}x${dimension} actual=${tensor.dims.join("x")}`,
-        });
-    }
-    return Array.from({ length: count }, (_, index) => Array.from({ length: dimension }, (__, offset) => {
-        const value = tensor.data[index * dimension + offset];
-        if (!Number.isFinite(value)) {
-            throw new EngineError("Transformers.js returned a non-finite tensor value", {
-                code: "ZVEC_GREP.ENGINE.MODELS.TRANSFORMERS_JS_INVALID_TENSOR",
-                context: `index=${index} offset=${offset}`,
-            });
-        }
-        return value;
-    }));
-}
-//# sourceMappingURL=transformers-js.js.map

package/dist/engine/models/backends/transformers-js.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"transformers-js.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/transformers-js.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EACL,kBAAkB,GAMnB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EACL,mCAAmC,GAEpC,MAAM,yBAAyB,CAAC;AA8EjC,MAAM,uBAAuB,GAAG,IAAI,CAAC,WAAW,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE9D,KAAK,UAAU,2BAA2B;IACxC,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAyB,CAAC;IAC7E,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,IAAI,WAAW,CACnB,4DAA4D,EAC5D;YACE,IAAI,EAAE,4DAA4D;YAClE,OAAO,EAAE,0DAA0D;YACnE,KAAK;SACN,CACF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,IAAI,oBAAoB,GAAyC,IAAI,CAAC;AAEtE,MAAM,mBAAmB,GAA+B;IACtD,WAAW;QACT,oBAAoB,KAAK,2BAA2B,EAAE,CAAC;QACvD,OAAO,oBAAoB,CAAC;IAC9B,CAAC;CACF,CAAC;AAEF,MAAM,OAAO,4BAA6B,SAAQ,kBAAkB;IAY/C;IAXV,IAAI,CAAqB;IAEjB,aAAa,CAAS;IACtB,iBAAiB,CAAyC;IAC1D,YAAY,CAA6B;IAClD,QAAQ,GAAqC,IAAI,CAAC;IAClD,mBAAmB,GAA8C,IAAI,CAAC;IACtE,gBAAgB,GAAG,KAAK,CAAC;IACzB,QAAQ,GAAG,KAAK,CAAC;IAEzB,YACmB,KAA0C,EAC3D,OAAoC,EACpC,eAAoD,EAAE;QAEtD,KAAK,EAAE,CAAC;QAJS,UAAK,GAAL,KAAK,CAAqC;QAK3D,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,KAAK,CAAC,KAAK;YACjB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,UAAU,EAAE,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE;gBACN,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,cAAc,EAAE,KAAK,CAAC,cAAc;aACrC;SACF,CAAC;QACF,IAAI,CAAC,aAAa;YAChB,OAAO,CAAC,aAAa;gBACrB,OAAO,CAAC,GAAG,CAAC,qBAAqB;gBACjC,uBAAuB,CAAC;QAC1B,IAAI,CAAC,iBAAiB,GAAG,wBAAwB,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAClE,IAAI,CAAC,YAAY,GAAG,EAAE,GAAG,mBAAmB,EAAE,GAAG,YAAY,EAAE,CAAC;IAClE,CAAC;IAES,KAAK,CAAC,OAAO,CACrB,QAA4B,EAC5B,OAAmC;QAEnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,QAA4B,EAC5B,OAAmC;QAEnC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,MAAM,KAAK,GAAI,QAAmC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACjE,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CACtD,CAAC;QACF,IAAI,QAAmC,CAAC;QACxC,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC3D,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,WAAW,CAAC,kCAAkC,EAAE;gBACxD,IAAI,EAAE,sDAAsD;gBAC5D,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;gBAChE,KAAK;aACN,CAAC,CAAC;QACL,CAAC;QACD,IAAI,qBAA+B,CAAC;QACpC,IAAI,CAAC;YACH,qBAAqB,GAAG,MAAM,yBAAyB,CACrD,QAAQ,CAAC,SAAS,EAClB,KAAK,EACL,IAAI,CAAC,KAAK,CAAC,cAAc,CAC1B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,WAAW,CAAC,qCAAqC,EAAE;gBAC3D,IAAI,EAAE,6DAA6D;gBACnE,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;gBAChE,KAAK;aACN,CAAC,CAAC;QACL,CAAC;QAED,IAAI,OAAgB,CAAC;QACrB,IAAI,CAAC;YACH,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,KAAK,EAAE,qBAAqB,CAAC,CAAC;QACvE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,GAAG,KAAK,CAAC;QAClB,CAAC;QAED,IAAI,MAAM,IAAI,CAAC,aAAa,CAAC,OAAO,EAAE,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;YAC1D,IAAI,CAAC;gBACH,OAAO,MAAM,IAAI,CAAC,UAAU,CAC1B,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,UAAU,CAAC,EAC7C,KAAK,EACL,qBAAqB,CACtB,CAAC;YACJ,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,OAAO,GAAG,KAAK,CAAC;YAClB,CAAC;QACH,CAAC;QAED,MAAM,IAAI,WAAW,CAAC,kCAAkC,EAAE;YACxD,IAAI,EAAE,sDAAsD;YAC5D,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;YAChE,KAAK,EAAE,OAAO;SACf,CAAC,CAAC;IACL,CAAC;IAEQ,KAAK,CAAC,OAAO;QACpB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,MAAM,QAAQ,EAAE,OAAO,EAAE,CAAC;IAC5B,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,UAAuD;QAEvD,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;QACD,IAAI,IAAI,CAAC,mBAAmB,EAAE,CAAC;YAC7B,OAAO,MAAM,IAAI,CAAC,mBAAmB,CAAC;QACxC,CAAC;QAED,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC,YAAY,CAAC,UAAU,CAAC,CAAC;QACzD,IAAI,CAAC;YACH,IAAI,CAAC,QAAQ,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC;YAC/C,OAAO,IAAI,CAAC,QAAQ,CAAC;QACvB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAClC,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,UAAuD;QAEvD,MAAM,gBAAgB,GAAG,mCAAmC,CAC1D,IAAI,CAAC,KAAK,CAAC,SAAS,EACpB,UAAU,CACX,CAAC;QACF,gBAAgB,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,WAAW,EAAE,CAAC;QACtD,IAAI,QAAmC,CAAC;QACxC,MAAM,iBAAiB,GAAG,IAAI,CAAC,gBAAgB;YAC7C,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC;QAE3B,IAAI,CAAC;YACH,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAClC,OAAO,EACP,iBAAiB,EACjB,gBAAgB,CACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,CAAC,iBAAiB,IAAI,iBAAiB,KAAK,KAAK,EAAE,CAAC;gBACtD,MAAM,KAAK,CAAC;YACd,CAAC;YAED,MAAM,OAAO,GAAG,mBAAmB,iBAAiB,qCAAqC,kBAAkB,CAAC,KAAK,CAAC,yBAAyB,CAAC;YAC5I,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,OAAO,IAAI,CAAC,CAAC;YAC1D,CAAC;YACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC7B,QAAQ,GAAG,MAAM,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,KAAK,EAAE,gBAAgB,CAAC,CAAC;QACzE,CAAC;QAED,QAAQ,CAAC,SAAS,CAAC,gBAAgB,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC;QAChE,gBAAgB,CAAC,MAAM,EAAE,CAAC;QAC1B,OAAO,QAAQ,CAAC;IAClB,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,QAAmC,EACnC,KAAe,EACf,qBAA+B;QAE/B,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,EAAE;YACnC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,OAAO;YAC3B,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS;YAC/B,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc;SACtC,CAAC,CAAC;QACH,OAAO;YACL,OAAO,EAAE,eAAe,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;YACpE,SAAS,EAAE,qBAAqB;SACjC,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,KAAc,EACd,UAAuD;QAEvD,IACE,IAAI,CAAC,gBAAgB;YACrB,CAAC,IAAI,CAAC,iBAAiB;YACvB,IAAI,CAAC,iBAAiB,KAAK,KAAK,EAChC,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAED,MAAM,OAAO,GAAG,mBAAmB,IAAI,CAAC,iBAAiB,gCAAgC,kBAAkB,CAAC,KAAK,CAAC,yBAAyB,CAAC;QAC5I,IAAI,UAAU,EAAE,CAAC;YACf,UAAU,CAAC;gBACT,KAAK,EAAE,SAAS;gBAChB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS;gBAC3B,OAAO,EAAE,OAAO;aACjB,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,OAAO,IAAI,CAAC,CAAC;QAC1D,CAAC;QACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC7B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,IAAI,CAAC,mBAAmB,GAAG,IAAI,CAAC;QAChC,MAAM,QAAQ,EAAE,OAAO,EAAE,CAAC;QAC1B,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,KAAK,CAAC,cAAc,CAC1B,OAA6B,EAC7B,iBAAyD,EACzD,gBAA+C;QAE/C,OAAO,MAAM,OAAO,CAAC,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;YACnE,SAAS,EAAE,IAAI,CAAC,aAAa;YAC7B,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ;YAC7B,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK;YACvB,iBAAiB,EAAE,CAAC,QAAQ,EAAE,EAAE;gBAC9B,IACE,QAAQ,CAAC,MAAM,KAAK,UAAU;oBAC9B,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ,EACjC,CAAC;oBACD,gBAAgB,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;oBACzC,OAAO;gBACT,CAAC;gBACD,IACE,QAAQ,CAAC,MAAM,KAAK,UAAU;oBAC9B,OAAO,QAAQ,CAAC,IAAI,KAAK,QAAQ;oBACjC,OAAO,QAAQ,CAAC,MAAM,KAAK,QAAQ,EACnC,CAAC;oBACD,OAAO;gBACT,CAAC;gBACD,gBAAgB,CAAC,MAAM,CAAC;oBACtB,QAAQ,EAAE,QAAQ,CAAC,IAAI;oBACvB,eAAe,EAAE,QAAQ,CAAC,MAAM;oBAChC,UAAU,EAAE,QAAQ,CAAC,KAAK;iBAC3B,CAAC,CAAC;YACL,CAAC;YACD,GAAG,CAAC,iBAAiB;gBACnB,CAAC,CAAC,EAAE,eAAe,EAAE,EAAE,kBAAkB,EAAE,CAAC,iBAAiB,CAAC,EAAE,EAAE;gBAClE,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;IACL,CAAC;IAEO,iBAAiB;QACvB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,WAAW,CAAC,6CAA6C,EAAE;gBACnE,IAAI,EAAE,kDAAkD;gBACxD,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;aACzC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF;AAED,KAAK,UAAU,yBAAyB,CACtC,SAAwB,EACxB,KAAe,EACf,cAAsB;IAEtB,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,KAAK,EAAE;QACrC,UAAU,EAAE,IAAI;QAChB,UAAU,EAAE,cAAc,GAAG,CAAC;QAC9B,OAAO,EAAE,IAAI;KACd,CAAC,CAAC;IACH,MAAM,IAAI,GAAG,OAAO,CAAC,cAAc,CAAC;IACpC,MAAM,CAAC,SAAS,EAAE,cAAc,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC;IAC9C,IACE,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;QACtB,SAAS,KAAK,KAAK,CAAC,MAAM;QAC1B,CAAC,MAAM,CAAC,SAAS,CAAC,cAAc,CAAC;QACjC,cAAc,GAAG,CAAC;QAClB,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,GAAG,cAAc,EAC/C,CAAC;QACD,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,SAAS,EAAE,UAAU,EAAE,EAAE,CAAC;QAC9D,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,MAAM,GAAG,UAAU,GAAG,cAAc,CAAC;QAC3C,KAAK,IAAI,UAAU,GAAG,CAAC,EAAE,UAAU,GAAG,cAAc,EAAE,UAAU,EAAE,EAAE,CAAC;YACnE,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;gBACjD,UAAU,EAAE,CAAC;YACf,CAAC;QACH,CAAC;QACD,IAAI,UAAU,GAAG,cAAc,EAAE,CAAC;YAChC,qBAAqB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACzC,CAAC;IACH,CAAC;IACD,OAAO,qBAAqB,CAAC;AAC/B,CAAC;AAED,SAAS,wBAAwB,CAC/B,MAA6C;IAE7C,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;QACzB,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,MAAM,KAAK,KAAK,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,MAAM,KAAK,OAAO,IAAI,MAAM,KAAK,QAAQ,EAAE,CAAC;QAC9C,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,MAAM,KAAK,MAAM,EAAE,CAAC;QACtB,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QACjC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,OAAO,CAAC,IAAI,KAAK,KAAK,EAAE,CAAC;QAC3D,OAAO,MAAM,CAAC;IAChB,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAChE,CAAC;AAED,SAAS,UAAU,CACjB,IAAY,EACZ,OAA8C,EAC9C,KAA0C;IAE1C,MAAM,MAAM,GACV,OAAO,KAAK,OAAO;QACjB,CAAC,CAAC,aAAa,IAAI,KAAK;YACtB,CAAC,CAAC,KAAK,CAAC,WAAW;YACnB,CAAC,CAAC,SAAS;QACb,CAAC,CAAC,gBAAgB,IAAI,KAAK;YACzB,CAAC,CAAC,KAAK,CAAC,cAAc;YACtB,CAAC,CAAC,SAAS,CAAC;IAClB,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5C,CAAC;AAED,SAAS,eAAe,CACtB,MAAkB,EAClB,KAAa,EACb,SAAiB;IAEjB,IACE,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,KAAK;QACxB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS;QAC5B,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,GAAG,SAAS,EACxC,CAAC;QACD,MAAM,IAAI,WAAW,CAAC,+CAA+C,EAAE;YACrE,IAAI,EAAE,wDAAwD;YAC9D,OAAO,EAAE,YAAY,KAAK,IAAI,SAAS,WAAW,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE;SAC1E,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAChD,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,EAAE;QAC/C,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,GAAG,SAAS,GAAG,MAAM,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,IAAI,WAAW,CACnB,oDAAoD,EACpD;gBACE,IAAI,EAAE,wDAAwD;gBAC9D,OAAO,EAAE,SAAS,KAAK,WAAW,MAAM,EAAE;aAC3C,CACF,CAAC;QACJ,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC,CAAC,CACH,CAAC;AACJ,CAAC"}

package/dist/engine/models/catalog.d.ts

@@ -1,231 +0,0 @@
-export declare const EMBEDDING_MODEL_CATALOG: {
-    readonly "local/embeddinggemma-300m": {
-        readonly backend: "llama-cpp";
-        readonly reference: "local/embeddinggemma-300m";
-        readonly provider: "local";
-        readonly model: "embeddinggemma-300m";
-        readonly uri: "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
-        readonly dimension: 768;
-        readonly metric: "cosine";
-        readonly format: "embeddinggemma";
-        readonly contextSize: 2048;
-        readonly maxBatchSize: 16;
-    };
-    readonly "local/qwen3-embedding-0.6b": {
-        readonly backend: "llama-cpp";
-        readonly reference: "local/qwen3-embedding-0.6b";
-        readonly provider: "local";
-        readonly model: "qwen3-embedding-0.6b";
-        readonly uri: "hf:Qwen/Qwen3-Embedding-0.6B-GGUF/Qwen3-Embedding-0.6B-Q8_0.gguf";
-        readonly dimension: 1024;
-        readonly metric: "cosine";
-        readonly format: "qwen3";
-        readonly contextSize: 8192;
-        readonly maxBatchSize: 8;
-    };
-    readonly "qwen/text-embedding-v4": {
-        readonly backend: "qwen";
-        readonly kind: "text";
-        readonly reference: "qwen/text-embedding-v4";
-        readonly provider: "qwen";
-        readonly model: "text-embedding-v4";
-        readonly dimension: 1024;
-        readonly metric: "cosine";
-        readonly defaultEndpoint: "https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings";
-        readonly maxBatchSize: 10;
-        readonly maxInputTokens: 8192;
-    };
-    readonly "qwen/qwen3.7-text-embedding": {
-        readonly backend: "qwen";
-        readonly kind: "text";
-        readonly reference: "qwen/qwen3.7-text-embedding";
-        readonly provider: "qwen";
-        readonly model: "qwen3.7-text-embedding";
-        readonly dimension: 1024;
-        readonly metric: "cosine";
-        readonly defaultEndpoint: "https://dashscope.aliyuncs.com/compatible-mode/v1/embeddings";
-        readonly maxBatchSize: 20;
-        readonly maxInputTokens: 128000;
-    };
-    readonly "qwen/qwen3-vl-embedding": {
-        readonly backend: "qwen";
-        readonly kind: "multimodal";
-        readonly reference: "qwen/qwen3-vl-embedding";
-        readonly provider: "qwen";
-        readonly model: "qwen3-vl-embedding";
-        readonly dimension: 2560;
-        readonly metric: "cosine";
-        readonly defaultEndpoint: "https://dashscope.aliyuncs.com/api/v1/services/embeddings/multimodal-embedding/multimodal-embedding";
-        readonly maxBatchSize: 20;
-        readonly maxInputTokens: 32000;
-        readonly maxImageBytes: number;
-    };
-    readonly "local/bge-small-en-v1.5": {
-        readonly backend: "transformers-js";
-        readonly reference: "local/bge-small-en-v1.5";
-        readonly provider: "local";
-        readonly model: "bge-small-en-v1.5";
-        readonly repo: "onnx-community/bge-small-en-v1.5-ONNX";
-        readonly revision: "4a9a46c7b88fa408e650a571a1800243f26309bd";
-        readonly dtype: "q4";
-        readonly dimension: 384;
-        readonly metric: "cosine";
-        readonly pooling: "cls";
-        readonly normalize: true;
-        readonly queryPrefix: "Represent this sentence for searching relevant passages: ";
-        readonly maxInputTokens: 512;
-        readonly maxBatchSize: 4;
-    };
-    readonly "local/all-minilm-l6-v2": {
-        readonly backend: "transformers-js";
-        readonly reference: "local/all-minilm-l6-v2";
-        readonly provider: "local";
-        readonly model: "all-minilm-l6-v2";
-        readonly repo: "onnx-community/all-MiniLM-L6-v2-ONNX";
-        readonly revision: "aff7a1dc4e8a1ea593e6ea21e95c22ef0a25966f";
-        readonly dtype: "q4";
-        readonly dimension: 384;
-        readonly metric: "cosine";
-        readonly pooling: "mean";
-        readonly normalize: true;
-        readonly maxInputTokens: 256;
-        readonly maxBatchSize: 4;
-    };
-    readonly "local/potion-retrieval-32m": {
-        readonly backend: "model2vec";
-        readonly reference: "local/potion-retrieval-32m";
-        readonly provider: "local";
-        readonly model: "potion-retrieval-32m";
-        readonly repo: "minishlab/potion-retrieval-32M";
-        readonly revision: "6fc8051fab2a1e0ee76689cf08c853792ac285e7";
-        readonly modelFile: "model.safetensors";
-        readonly embeddingTensor: "embeddings";
-        readonly tokenizerFile: "tokenizer.json";
-        readonly dimension: 512;
-        readonly metric: "cosine";
-        readonly normalize: true;
-        readonly maxInputTokens: 1024;
-        readonly maxBatchSize: 256;
-        readonly defaultConcurrency: 2;
-    };
-    readonly "local/potion-multilingual-128m": {
-        readonly backend: "model2vec";
-        readonly reference: "local/potion-multilingual-128m";
-        readonly provider: "local";
-        readonly model: "potion-multilingual-128m";
-        readonly repo: "minishlab/potion-multilingual-128M";
-        readonly revision: "73908c3438cf03b6a01bcb9611d62b23d0726f08";
-        readonly modelFile: "model.safetensors";
-        readonly embeddingTensor: "embeddings";
-        readonly tokenizerFile: "tokenizer.json";
-        readonly dimension: 256;
-        readonly metric: "cosine";
-        readonly normalize: true;
-        readonly maxInputTokens: 1024;
-        readonly maxBatchSize: 256;
-        readonly defaultConcurrency: 2;
-    };
-    readonly "local/potion-code-16m-v2": {
-        readonly backend: "model2vec";
-        readonly reference: "local/potion-code-16m-v2";
-        readonly provider: "local";
-        readonly model: "potion-code-16m-v2";
-        readonly repo: "minishlab/potion-code-16M-v2";
-        readonly revision: "e9d2a44ca6a05ac6685f3b23709ea57eb7352d5b";
-        readonly modelFile: "model.safetensors";
-        readonly embeddingTensor: "embeddings";
-        readonly tokenizerFile: "tokenizer.json";
-        readonly dimension: 256;
-        readonly metric: "cosine";
-        readonly normalize: true;
-        readonly maxInputTokens: 1024;
-        readonly maxBatchSize: 256;
-        readonly defaultConcurrency: 2;
-    };
-    readonly "local/multilingual-e5-small": {
-        readonly backend: "transformers-js";
-        readonly reference: "local/multilingual-e5-small";
-        readonly provider: "local";
-        readonly model: "multilingual-e5-small";
-        readonly repo: "Xenova/multilingual-e5-small";
-        readonly revision: "761b726dd34fb83930e26aab4e9ac3899aa1fa78";
-        readonly dtype: "q8";
-        readonly dimension: 384;
-        readonly metric: "cosine";
-        readonly pooling: "mean";
-        readonly normalize: true;
-        readonly queryPrefix: "query: ";
-        readonly documentPrefix: "passage: ";
-        readonly maxInputTokens: 512;
-        readonly maxBatchSize: 4;
-    };
-    readonly "local/jina-embeddings-v2-base-code": {
-        readonly backend: "transformers-js";
-        readonly reference: "local/jina-embeddings-v2-base-code";
-        readonly provider: "local";
-        readonly model: "jina-embeddings-v2-base-code";
-        readonly repo: "jinaai/jina-embeddings-v2-base-code";
-        readonly revision: "516f4baf13dec4ddddda8631e019b5737c8bc250";
-        readonly dtype: "q8";
-        readonly dimension: 768;
-        readonly metric: "cosine";
-        readonly pooling: "mean";
-        readonly normalize: true;
-        readonly maxInputTokens: 8192;
-        readonly maxBatchSize: 2;
-    };
-    readonly "local/gte-modernbert-base": {
-        readonly backend: "transformers-js";
-        readonly reference: "local/gte-modernbert-base";
-        readonly provider: "local";
-        readonly model: "gte-modernbert-base";
-        readonly repo: "Alibaba-NLP/gte-modernbert-base";
-        readonly revision: "e7f32e3c00f91d699e8c43b53106206bcc72bb22";
-        readonly dtype: "q4";
-        readonly dimension: 768;
-        readonly metric: "cosine";
-        readonly pooling: "cls";
-        readonly normalize: true;
-        readonly maxInputTokens: 8192;
-        readonly maxBatchSize: 2;
-    };
-    readonly "local/nomic-embed-text-v1.5": {
-        readonly backend: "transformers-js";
-        readonly reference: "local/nomic-embed-text-v1.5";
-        readonly provider: "local";
-        readonly model: "nomic-embed-text-v1.5";
-        readonly repo: "nomic-ai/nomic-embed-text-v1.5";
-        readonly revision: "e9b6763023c676ca8431644204f50c2b100d9aab";
-        readonly dtype: "q4";
-        readonly dimension: 768;
-        readonly metric: "cosine";
-        readonly pooling: "mean";
-        readonly normalize: true;
-        readonly queryPrefix: "search_query: ";
-        readonly documentPrefix: "search_document: ";
-        readonly maxInputTokens: 8192;
-        readonly maxBatchSize: 2;
-    };
-};
-export type EmbeddingCatalogEntry = (typeof EMBEDDING_MODEL_CATALOG)[keyof typeof EMBEDDING_MODEL_CATALOG];
-export type LlamaCppEmbeddingCatalogEntry = Extract<EmbeddingCatalogEntry, {
-    backend: "llama-cpp";
-}>;
-export type TransformersJsEmbeddingCatalogEntry = Extract<EmbeddingCatalogEntry, {
-    backend: "transformers-js";
-}>;
-export type Model2VecEmbeddingCatalogEntry = Extract<EmbeddingCatalogEntry, {
-    backend: "model2vec";
-}>;
-export type QwenEmbeddingCatalogEntry = Extract<EmbeddingCatalogEntry, {
-    backend: "qwen";
-}>;
-export type QwenTextEmbeddingCatalogEntry = Extract<QwenEmbeddingCatalogEntry, {
-    kind: "text";
-}>;
-export type QwenMultimodalEmbeddingCatalogEntry = Extract<QwenEmbeddingCatalogEntry, {
-    kind: "multimodal";
-}>;
-export type EmbeddingModelCatalogId = keyof typeof EMBEDDING_MODEL_CATALOG;
-export declare function listEmbeddingModels(): EmbeddingCatalogEntry[];
-export declare function getEmbeddingModelCatalogEntry(reference: string): EmbeddingCatalogEntry | undefined;