@zvec/zvec-grep 0.2.2 → 0.3.0-rc.1

package/dist/engine/models/backends/model2vec.js

@@ -1,344 +0,0 @@
-import { open, readFile } from "node:fs/promises";
-import { basename, dirname, join, resolve } from "node:path";
-import { EngineError } from "../../errors.js";
-import { writeJsonFile } from "../../utils/json.js";
-import { defaultHome } from "../../utils/path.js";
-import { BaseEmbeddingModel, } from "../embeddings.js";
-import { resolveModelArtifacts, } from "../artifact-downloader.js";
-import { createModelDownloadProgressReporter } from "../download-progress.js";
-import { loadModel2VecTokenizer } from "./model2vec-tokenizer.js";
-import { Model2VecWorkerPool } from "./model2vec-worker-pool.js";
-import { embedModel2VecTexts, sharedStaticEmbeddingTable, } from "./model2vec-runtime.js";
-const DEFAULT_MODEL_CACHE_DIR = join(defaultHome(), "models");
-const defaultDependencies = {
-    async loadTokenizer(source) {
-        return await loadModel2VecTokenizer(source);
-    },
-    async loadSafetensors(path, tensorName, dimension) {
-        return await readStaticEmbeddingTable(path, tensorName, dimension);
-    },
-    resolveArtifacts: resolveModelArtifacts,
-};
-export class Model2VecEmbeddingModel extends BaseEmbeddingModel {
-    entry;
-    info;
-    modelCacheDir;
-    dependencies;
-    tokenizer = null;
-    staticTable = null;
-    workerPool = null;
-    loadPromise = null;
-    useWorkerPool;
-    sourceFallbackWarningReported = false;
-    disposed = false;
-    constructor(entry, options, dependencies = {}) {
-        super();
-        this.entry = entry;
-        this.info = {
-            reference: entry.reference,
-            provider: entry.provider,
-            name: entry.model,
-            dimension: entry.dimension,
-            metric: entry.metric,
-            defaultConcurrency: entry.defaultConcurrency,
-            inputKinds: ["text"],
-            limits: {
-                maxBatchSize: entry.maxBatchSize,
-                maxInputTokens: entry.maxInputTokens,
-            },
-        };
-        this.modelCacheDir = resolve(options.modelCacheDir ??
-            process.env.ZVEC_GREP_MODEL_CACHE ??
-            DEFAULT_MODEL_CACHE_DIR);
-        this.dependencies = { ...defaultDependencies, ...dependencies };
-        this.useWorkerPool = Object.keys(dependencies).length === 0;
-    }
-    async doEmbed(contents, options) {
-        return await this.embedBatch(contents, options);
-    }
-    async prepare(options = {}) {
-        options.signal?.throwIfAborted();
-        this.ensureNotDisposed();
-        await this.ensureLoaded(options.onProgress);
-        options.signal?.throwIfAborted();
-        this.ensureNotDisposed();
-    }
-    async embedBatch(contents, options) {
-        this.ensureNotDisposed();
-        await this.ensureLoaded(options.onProgress);
-        try {
-            const texts = contents.map((content) => formatText(content.text, options.purpose, this.entry));
-            if (this.workerPool) {
-                return await this.workerPool.run(texts, options.signal);
-            }
-            return await this.embedTexts(texts);
-        }
-        catch (cause) {
-            throw new EngineError("Model2Vec embedding failed", {
-                code: "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_EMBED_FAILED",
-                context: `model=${this.entry.reference} repo=${this.entry.repo}`,
-                cause,
-            });
-        }
-    }
-    async dispose() {
-        if (this.disposed) {
-            return;
-        }
-        this.disposed = true;
-        await this.workerPool?.dispose();
-        this.workerPool = null;
-        this.staticTable = null;
-        this.tokenizer = null;
-        this.loadPromise = null;
-    }
-    async ensureLoaded(onProgress) {
-        if (this.workerPool || (this.tokenizer && this.staticTable)) {
-            return;
-        }
-        if (this.loadPromise) {
-            return await this.loadPromise;
-        }
-        this.loadPromise = this.loadModel(onProgress);
-        try {
-            await this.loadPromise;
-        }
-        finally {
-            this.loadPromise = null;
-        }
-    }
-    async loadModel(onProgress) {
-        const downloadProgress = createModelDownloadProgressReporter(this.entry.reference, onProgress);
-        downloadProgress.start();
-        try {
-            const resolved = await this.dependencies
-                .resolveArtifacts({
-                model: this.entry.reference,
-                sources: this.createArtifactSources(),
-                artifacts: this.entry.artifacts,
-                onDownloadPlan: (artifacts) => {
-                    downloadProgress.setDownloadPlan(artifacts);
-                },
-                onProgress: (progress) => {
-                    downloadProgress.report({
-                        artifact: progress.artifact,
-                        downloadedBytes: progress.downloadedBytes,
-                    });
-                },
-                onFallback: (warning) => {
-                    if (this.sourceFallbackWarningReported) {
-                        return;
-                    }
-                    this.sourceFallbackWarningReported = true;
-                    if (!downloadProgress.warning(warning)) {
-                        process.stderr.write(`zvec-grep warning: ${warning}\n`);
-                    }
-                },
-            })
-                .catch((cause) => {
-                throw new EngineError("Unable to download Model2Vec model artifacts", {
-                    code: "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_DOWNLOAD_FAILED",
-                    context: `model=${this.entry.reference} repo=${this.entry.repo} revision=${this.entry.revision}`,
-                    cause,
-                });
-            });
-            const modelPath = resolved.paths[this.entry.modelFile];
-            const tokenizerPath = resolved.paths[this.entry.tokenizerFile];
-            if (!modelPath || !tokenizerPath) {
-                throw new EngineError("Resolved Model2Vec snapshot is incomplete", {
-                    code: "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_DOWNLOAD_FAILED",
-                    context: `model=${this.entry.reference} source=${resolved.source.kind}`,
-                });
-            }
-            const tokenizerSource = dirname(tokenizerPath);
-            const configPath = join(tokenizerSource, "tokenizer_config.json");
-            await ensureTokenizerConfig(configPath);
-            const staticTable = await this.dependencies.loadSafetensors(modelPath, this.entry.embeddingTensor, this.info.dimension);
-            this.ensureNotDisposed();
-            if (this.useWorkerPool) {
-                const sharedTable = sharedStaticEmbeddingTable(staticTable);
-                const workerPool = new Model2VecWorkerPool({
-                    tokenizerSource,
-                    maxInputTokens: this.entry.maxInputTokens,
-                    normalize: this.entry.normalize,
-                    tableBuffer: sharedTable.data.buffer,
-                    dimension: sharedTable.dimension,
-                    dtype: sharedTable.dtype,
-                    rows: sharedTable.rows,
-                });
-                try {
-                    await workerPool.start();
-                    this.ensureNotDisposed();
-                    this.workerPool = workerPool;
-                }
-                catch (error) {
-                    await workerPool.dispose();
-                    throw error;
-                }
-            }
-            else {
-                this.tokenizer = await this.dependencies.loadTokenizer(tokenizerSource, {
-                    cache_dir: this.modelCacheDir,
-                    revision: resolved.source.revision,
-                    local_files_only: true,
-                });
-                this.ensureNotDisposed();
-                this.staticTable = staticTable;
-            }
-            downloadProgress.finish();
-        }
-        catch (cause) {
-            downloadProgress.warning("Unable to prepare the local embedding model. Check network access and the model cache.");
-            if (cause instanceof EngineError &&
-                (cause.code === "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_DOWNLOAD_FAILED" ||
-                    cause.code === "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_DISPOSED")) {
-                throw cause;
-            }
-            throw new EngineError("Unable to load Model2Vec model", {
-                code: "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_LOAD_FAILED",
-                context: `model=${this.entry.reference} repo=${this.entry.repo} revision=${this.entry.revision}`,
-                cause,
-            });
-        }
-    }
-    createArtifactSources() {
-        const huggingFace = this.entry.sources.huggingFace;
-        const modelScope = this.entry.sources.modelScope;
-        const localPaths = {
-            [this.entry.modelFile]: basename(this.entry.modelFile),
-            [this.entry.tokenizerFile]: "tokenizer/tokenizer.json",
-        };
-        return [
-            {
-                kind: "huggingface",
-                repo: huggingFace.repo,
-                revision: huggingFace.revision,
-                cacheDirectory: join(this.modelCacheDir, "model2vec", huggingFace.repo.replaceAll("/", "--"), huggingFace.revision),
-                localPaths,
-            },
-            {
-                kind: "modelscope",
-                repo: modelScope.repo,
-                revision: modelScope.revision,
-                cacheDirectory: join(this.modelCacheDir, "modelscope", "model2vec", modelScope.repo.replaceAll("/", "--"), modelScope.revision),
-                localPaths,
-            },
-        ];
-    }
-    async embedTexts(texts) {
-        const tokenizer = this.tokenizer;
-        const staticTable = this.staticTable;
-        if (!tokenizer || !staticTable) {
-            throw new Error("Model2Vec model is not loaded");
-        }
-        return await embedModel2VecTexts(texts, tokenizer, staticTable, this.entry.maxInputTokens, this.entry.normalize);
-    }
-    ensureNotDisposed() {
-        if (this.disposed) {
-            throw new EngineError("Model2Vec embedding model is disposed", {
-                code: "ZVEC_GREP.ENGINE.MODELS.MODEL2VEC_DISPOSED",
-                context: `model=${this.entry.reference}`,
-            });
-        }
-    }
-}
-async function ensureTokenizerConfig(path) {
-    try {
-        const value = JSON.parse(await readFile(path, "utf8"));
-        if (typeof value === "object" &&
-            value !== null &&
-            "tokenizer_class" in value &&
-            value.tokenizer_class === "PreTrainedTokenizer") {
-            return;
-        }
-    }
-    catch (error) {
-        if (!(error instanceof SyntaxError) &&
-            !(typeof error === "object" &&
-                error !== null &&
-                "code" in error &&
-                error.code === "ENOENT")) {
-            throw error;
-        }
-    }
-    await writeJsonFile(path, { tokenizer_class: "PreTrainedTokenizer" });
-}
-function formatText(text, purpose, entry) {
-    const prefix = purpose === "query"
-        ? "queryPrefix" in entry && typeof entry.queryPrefix === "string"
-            ? entry.queryPrefix
-            : undefined
-        : "documentPrefix" in entry && typeof entry.documentPrefix === "string"
-            ? entry.documentPrefix
-            : undefined;
-    return prefix ? `${prefix}${text}` : text;
-}
-async function readStaticEmbeddingTable(path, tensorName, expectedDimension) {
-    const file = await open(path, "r");
-    try {
-        const stats = await file.stat();
-        if (stats.size < 9) {
-            throw new Error("Safetensors file is too small");
-        }
-        const prefix = Buffer.alloc(8);
-        await readExactly(file, prefix, 0);
-        const headerLength = Number(prefix.readBigUInt64LE(0));
-        const dataStart = 8 + headerLength;
-        if (!Number.isSafeInteger(headerLength) || dataStart > stats.size) {
-            throw new Error("Safetensors header length is invalid");
-        }
-        const headerBytes = Buffer.alloc(headerLength);
-        await readExactly(file, headerBytes, 8);
-        let header;
-        try {
-            header = JSON.parse(headerBytes.toString("utf8"));
-        }
-        catch (cause) {
-            throw new Error("Safetensors header is invalid JSON", { cause });
-        }
-        const tensor = header[tensorName];
-        if (!tensor ||
-            tensor.shape?.length !== 2 ||
-            tensor.shape[1] !== expectedDimension ||
-            tensor.data_offsets?.length !== 2 ||
-            (tensor.dtype !== "F16" && tensor.dtype !== "F32")) {
-            throw new Error(`Safetensors tensor '${tensorName}' is missing or incompatible`);
-        }
-        const [relativeStart, relativeEnd] = tensor.data_offsets;
-        const rows = tensor.shape[0];
-        const valueCount = rows * expectedDimension;
-        const bytesPerValue = tensor.dtype === "F16" ? 2 : 4;
-        const tensorByteLength = valueCount * bytesPerValue;
-        if (!Number.isSafeInteger(valueCount) ||
-            !Number.isSafeInteger(tensorByteLength) ||
-            relativeStart < 0 ||
-            relativeEnd - relativeStart !== tensorByteLength ||
-            dataStart + relativeEnd > stats.size) {
-            throw new Error(`Safetensors tensor '${tensorName}' has invalid offsets`);
-        }
-        const sharedBuffer = new SharedArrayBuffer(tensorByteLength);
-        await readExactly(file, Buffer.from(sharedBuffer), dataStart + relativeStart);
-        return {
-            data: tensor.dtype === "F16"
-                ? new Uint16Array(sharedBuffer)
-                : new Float32Array(sharedBuffer),
-            dimension: expectedDimension,
-            dtype: tensor.dtype,
-            rows,
-        };
-    }
-    finally {
-        await file.close();
-    }
-}
-async function readExactly(file, buffer, position) {
-    let offset = 0;
-    while (offset < buffer.byteLength) {
-        const { bytesRead } = await file.read(buffer, offset, buffer.byteLength - offset, position + offset);
-        if (bytesRead === 0) {
-            throw new Error("Safetensors file ended unexpectedly");
-        }
-        offset += bytesRead;
-    }
-}
-//# sourceMappingURL=model2vec.js.map

package/dist/engine/models/backends/model2vec.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"model2vec.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAElD,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAC7D,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EACL,kBAAkB,GAOnB,MAAM,kBAAkB,CAAC;AAE1B,OAAO,EACL,qBAAqB,GAEtB,MAAM,2BAA2B,CAAC;AACnC,OAAO,EAAE,mCAAmC,EAAE,MAAM,yBAAyB,CAAC;AAC9E,OAAO,EAAE,sBAAsB,EAAE,MAAM,0BAA0B,CAAC;AAClE,OAAO,EAAE,mBAAmB,EAAE,MAAM,4BAA4B,CAAC;AACjE,OAAO,EACL,mBAAmB,EACnB,0BAA0B,GAG3B,MAAM,wBAAwB,CAAC;AAmBhC,MAAM,uBAAuB,GAAG,IAAI,CAAC,WAAW,EAAE,EAAE,QAAQ,CAAC,CAAC;AAE9D,MAAM,mBAAmB,GAA0B;IACjD,KAAK,CAAC,aAAa,CAAC,MAAM;QACxB,OAAO,MAAM,sBAAsB,CAAC,MAAM,CAAC,CAAC;IAC9C,CAAC;IACD,KAAK,CAAC,eAAe,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS;QAC/C,OAAO,MAAM,wBAAwB,CAAC,IAAI,EAAE,UAAU,EAAE,SAAS,CAAC,CAAC;IACrE,CAAC;IACD,gBAAgB,EAAE,qBAAqB;CACxC,CAAC;AAEF,MAAM,OAAO,uBAAwB,SAAQ,kBAAkB;IAc1C;IAbV,IAAI,CAAqB;IAEjB,aAAa,CAAS;IACtB,YAAY,CAAwB;IAC7C,SAAS,GAAyB,IAAI,CAAC;IACvC,WAAW,GAAgC,IAAI,CAAC;IAChD,UAAU,GAA+B,IAAI,CAAC;IAC9C,WAAW,GAAyB,IAAI,CAAC;IAChC,aAAa,CAAU;IAChC,6BAA6B,GAAG,KAAK,CAAC;IACtC,QAAQ,GAAG,KAAK,CAAC;IAEzB,YACmB,KAAqC,EACtD,OAAoC,EACpC,eAA+C,EAAE;QAEjD,KAAK,EAAE,CAAC;QAJS,UAAK,GAAL,KAAK,CAAgC;QAKtD,IAAI,CAAC,IAAI,GAAG;YACV,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,IAAI,EAAE,KAAK,CAAC,KAAK;YACjB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;YAC5C,UAAU,EAAE,CAAC,MAAM,CAAC;YACpB,MAAM,EAAE;gBACN,YAAY,EAAE,KAAK,CAAC,YAAY;gBAChC,cAAc,EAAE,KAAK,CAAC,cAAc;aACrC;SACF,CAAC;QACF,IAAI,CAAC,aAAa,GAAG,OAAO,CAC1B,OAAO,CAAC,aAAa;YACnB,OAAO,CAAC,GAAG,CAAC,qBAAqB;YACjC,uBAAuB,CAC1B,CAAC;QACF,IAAI,CAAC,YAAY,GAAG,EAAE,GAAG,mBAAmB,EAAE,GAAG,YAAY,EAAE,CAAC;QAChE,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC;IAC9D,CAAC;IAES,KAAK,CAAC,OAAO,CACrB,QAA4B,EAC5B,OAAmC;QAEnC,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,OAAO,CACX,UAA2D,EAAE;QAE7D,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAC5C,OAAO,CAAC,MAAM,EAAE,cAAc,EAAE,CAAC;QACjC,IAAI,CAAC,iBAAiB,EAAE,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,UAAU,CACtB,QAA4B,EAC5B,OAAmC;QAEnC,IAAI,CAAC,iBAAiB,EAAE,CAAC;QACzB,MAAM,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;QAE5C,IAAI,CAAC;YACH,MAAM,KAAK,GAAI,QAAmC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CACjE,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,CACtD,CAAC;YACF,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpB,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAC1D,CAAC;YACD,OAAO,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QACtC,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,WAAW,CAAC,4BAA4B,EAAE;gBAClD,IAAI,EAAE,gDAAgD;gBACtD,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE;gBAChE,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEQ,KAAK,CAAC,OAAO;QACpB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,OAAO;QACT,CAAC;QACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;QACrB,MAAM,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,CAAC;QACjC,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QACxB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QACtB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;IAC1B,CAAC;IAEO,KAAK,CAAC,YAAY,CACxB,UAAuD;QAEvD,IAAI,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,CAAC,SAAS,IAAI,IAAI,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5D,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,OAAO,MAAM,IAAI,CAAC,WAAW,CAAC;QAChC,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC9C,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,WAAW,CAAC;QACzB,CAAC;gBAAS,CAAC;YACT,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;QAC1B,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,SAAS,CACrB,UAAuD;QAEvD,MAAM,gBAAgB,GAAG,mCAAmC,CAC1D,IAAI,CAAC,KAAK,CAAC,SAAS,EACpB,UAAU,CACX,CAAC;QACF,gBAAgB,CAAC,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,YAAY;iBACrC,gBAAgB,CAAC;gBAChB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS;gBAC3B,OAAO,EAAE,IAAI,CAAC,qBAAqB,EAAE;gBACrC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS;gBAC/B,cAAc,EAAE,CAAC,SAAS,EAAE,EAAE;oBAC5B,gBAAgB,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;gBAC9C,CAAC;gBACD,UAAU,EAAE,CAAC,QAAQ,EAAE,EAAE;oBACvB,gBAAgB,CAAC,MAAM,CAAC;wBACtB,QAAQ,EAAE,QAAQ,CAAC,QAAQ;wBAC3B,eAAe,EAAE,QAAQ,CAAC,eAAe;qBAC1C,CAAC,CAAC;gBACL,CAAC;gBACD,UAAU,EAAE,CAAC,OAAO,EAAE,EAAE;oBACtB,IAAI,IAAI,CAAC,6BAA6B,EAAE,CAAC;wBACvC,OAAO;oBACT,CAAC;oBACD,IAAI,CAAC,6BAA6B,GAAG,IAAI,CAAC;oBAC1C,IAAI,CAAC,gBAAgB,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;wBACvC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,sBAAsB,OAAO,IAAI,CAAC,CAAC;oBAC1D,CAAC;gBACH,CAAC;aACF,CAAC;iBACD,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBACxB,MAAM,IAAI,WAAW,CACnB,8CAA8C,EAC9C;oBACE,IAAI,EAAE,mDAAmD;oBACzD,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,aAAa,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE;oBAChG,KAAK;iBACN,CACF,CAAC;YACJ,CAAC,CAAC,CAAC;YACL,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;YACvD,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;YAC/D,IAAI,CAAC,SAAS,IAAI,CAAC,aAAa,EAAE,CAAC;gBACjC,MAAM,IAAI,WAAW,CAAC,2CAA2C,EAAE;oBACjE,IAAI,EAAE,mDAAmD;oBACzD,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,WAAW,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE;iBACxE,CAAC,CAAC;YACL,CAAC;YACD,MAAM,eAAe,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;YAC/C,MAAM,UAAU,GAAG,IAAI,CAAC,eAAe,EAAE,uBAAuB,CAAC,CAAC;YAClE,MAAM,qBAAqB,CAAC,UAAU,CAAC,CAAC;YACxC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,eAAe,CACzD,SAAS,EACT,IAAI,CAAC,KAAK,CAAC,eAAe,EAC1B,IAAI,CAAC,IAAI,CAAC,SAAS,CACpB,CAAC;YACF,IAAI,CAAC,iBAAiB,EAAE,CAAC;YACzB,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;gBACvB,MAAM,WAAW,GAAG,0BAA0B,CAAC,WAAW,CAAC,CAAC;gBAC5D,MAAM,UAAU,GAAG,IAAI,mBAAmB,CAAC;oBACzC,eAAe;oBACf,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,cAAc;oBACzC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS;oBAC/B,WAAW,EAAE,WAAW,CAAC,IAAI,CAAC,MAA2B;oBACzD,SAAS,EAAE,WAAW,CAAC,SAAS;oBAChC,KAAK,EAAE,WAAW,CAAC,KAAK;oBACxB,IAAI,EAAE,WAAW,CAAC,IAAI;iBACvB,CAAC,CAAC;gBACH,IAAI,CAAC;oBACH,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC;oBACzB,IAAI,CAAC,iBAAiB,EAAE,CAAC;oBACzB,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;gBAC/B,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,UAAU,CAAC,OAAO,EAAE,CAAC;oBAC3B,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,SAAS,GAAG,MAAM,IAAI,CAAC,YAAY,CAAC,aAAa,CACpD,eAAe,EACf;oBACE,SAAS,EAAE,IAAI,CAAC,aAAa;oBAC7B,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,QAAQ;oBAClC,gBAAgB,EAAE,IAAI;iBACvB,CACF,CAAC;gBACF,IAAI,CAAC,iBAAiB,EAAE,CAAC;gBACzB,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;YACjC,CAAC;YACD,gBAAgB,CAAC,MAAM,EAAE,CAAC;QAC5B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,gBAAgB,CAAC,OAAO,CACtB,wFAAwF,CACzF,CAAC;YACF,IACE,KAAK,YAAY,WAAW;gBAC5B,CAAC,KAAK,CAAC,IAAI,KAAK,mDAAmD;oBACjE,KAAK,CAAC,IAAI,KAAK,4CAA4C,CAAC,EAC9D,CAAC;gBACD,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,IAAI,WAAW,CAAC,gCAAgC,EAAE;gBACtD,IAAI,EAAE,+CAA+C;gBACrD,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,SAAS,IAAI,CAAC,KAAK,CAAC,IAAI,aAAa,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE;gBAChG,KAAK;aACN,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAEO,qBAAqB;QAC3B,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,CAAC;QACnD,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC;QACjD,MAAM,UAAU,GAAG;YACjB,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC;YACtD,CAAC,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,0BAA0B;SACvD,CAAC;QACF,OAAO;YACL;gBACE,IAAI,EAAE,aAAa;gBACnB,IAAI,EAAE,WAAW,CAAC,IAAI;gBACtB,QAAQ,EAAE,WAAW,CAAC,QAAQ;gBAC9B,cAAc,EAAE,IAAI,CAClB,IAAI,CAAC,aAAa,EAClB,WAAW,EACX,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,EACtC,WAAW,CAAC,QAAQ,CACrB;gBACD,UAAU;aACX;YACD;gBACE,IAAI,EAAE,YAAY;gBAClB,IAAI,EAAE,UAAU,CAAC,IAAI;gBACrB,QAAQ,EAAE,UAAU,CAAC,QAAQ;gBAC7B,cAAc,EAAE,IAAI,CAClB,IAAI,CAAC,aAAa,EAClB,YAAY,EACZ,WAAW,EACX,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,EACrC,UAAU,CAAC,QAAQ,CACpB;gBACD,UAAU;aACX;SACF,CAAC;IACJ,CAAC;IAEO,KAAK,CAAC,UAAU,CAAC,KAAe;QACtC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACjC,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;QACrC,IAAI,CAAC,SAAS,IAAI,CAAC,WAAW,EAAE,CAAC;YAC/B,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,OAAO,MAAM,mBAAmB,CAC9B,KAAK,EACL,SAAS,EACT,WAAW,EACX,IAAI,CAAC,KAAK,CAAC,cAAc,EACzB,IAAI,CAAC,KAAK,CAAC,SAAS,CACrB,CAAC;IACJ,CAAC;IAEO,iBAAiB;QACvB,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,IAAI,WAAW,CAAC,uCAAuC,EAAE;gBAC7D,IAAI,EAAE,4CAA4C;gBAClD,OAAO,EAAE,SAAS,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE;aACzC,CAAC,CAAC;QACL,CAAC;IACH,CAAC;CACF;AAED,KAAK,UAAU,qBAAqB,CAAC,IAAY;IAC/C,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAY,CAAC;QAClE,IACE,OAAO,KAAK,KAAK,QAAQ;YACzB,KAAK,KAAK,IAAI;YACd,iBAAiB,IAAI,KAAK;YAC1B,KAAK,CAAC,eAAe,KAAK,qBAAqB,EAC/C,CAAC;YACD,OAAO;QACT,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IACE,CAAC,CAAC,KAAK,YAAY,WAAW,CAAC;YAC/B,CAAC,CACC,OAAO,KAAK,KAAK,QAAQ;gBACzB,KAAK,KAAK,IAAI;gBACd,MAAM,IAAI,KAAK;gBACf,KAAK,CAAC,IAAI,KAAK,QAAQ,CACxB,EACD,CAAC;YACD,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,MAAM,aAAa,CAAC,IAAI,EAAE,EAAE,eAAe,EAAE,qBAAqB,EAAE,CAAC,CAAC;AACxE,CAAC;AAED,SAAS,UAAU,CACjB,IAAY,EACZ,OAA8C,EAC9C,KAAqC;IAErC,MAAM,MAAM,GACV,OAAO,KAAK,OAAO;QACjB,CAAC,CAAC,aAAa,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;YAC/D,CAAC,CAAC,KAAK,CAAC,WAAW;YACnB,CAAC,CAAC,SAAS;QACb,CAAC,CAAC,gBAAgB,IAAI,KAAK,IAAI,OAAO,KAAK,CAAC,cAAc,KAAK,QAAQ;YACrE,CAAC,CAAC,KAAK,CAAC,cAAc;YACtB,CAAC,CAAC,SAAS,CAAC;IAClB,OAAO,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAC5C,CAAC;AAED,KAAK,UAAU,wBAAwB,CACrC,IAAY,EACZ,UAAkB,EAClB,iBAAyB;IAEzB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACnC,IAAI,CAAC;QACH,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACnB,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACnD,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC/B,MAAM,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QACnC,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,MAAM,SAAS,GAAG,CAAC,GAAG,YAAY,CAAC;QACnC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,YAAY,CAAC,IAAI,SAAS,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;YAClE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC1D,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QAC/C,MAAM,WAAW,CAAC,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;QACxC,IAAI,MAGH,CAAC;QACF,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAkB,CAAC;QACrE,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,oCAAoC,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;QACnE,CAAC;QACD,MAAM,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,CAAC;QAClC,IACE,CAAC,MAAM;YACP,MAAM,CAAC,KAAK,EAAE,MAAM,KAAK,CAAC;YAC1B,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,iBAAiB;YACrC,MAAM,CAAC,YAAY,EAAE,MAAM,KAAK,CAAC;YACjC,CAAC,MAAM,CAAC,KAAK,KAAK,KAAK,IAAI,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,EAClD,CAAC;YACD,MAAM,IAAI,KAAK,CACb,uBAAuB,UAAU,8BAA8B,CAChE,CAAC;QACJ,CAAC;QAED,MAAM,CAAC,aAAa,EAAE,WAAW,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;QACzD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QAC7B,MAAM,UAAU,GAAG,IAAI,GAAG,iBAAiB,CAAC;QAC5C,MAAM,aAAa,GAAG,MAAM,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACrD,MAAM,gBAAgB,GAAG,UAAU,GAAG,aAAa,CAAC;QACpD,IACE,CAAC,MAAM,CAAC,aAAa,CAAC,UAAU,CAAC;YACjC,CAAC,MAAM,CAAC,aAAa,CAAC,gBAAgB,CAAC;YACvC,aAAa,GAAG,CAAC;YACjB,WAAW,GAAG,aAAa,KAAK,gBAAgB;YAChD,SAAS,GAAG,WAAW,GAAG,KAAK,CAAC,IAAI,EACpC,CAAC;YACD,MAAM,IAAI,KAAK,CAAC,uBAAuB,UAAU,uBAAuB,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;QAC7D,MAAM,WAAW,CACf,IAAI,EACJ,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,EACzB,SAAS,GAAG,aAAa,CAC1B,CAAC;QACF,OAAO;YACL,IAAI,EACF,MAAM,CAAC,KAAK,KAAK,KAAK;gBACpB,CAAC,CAAC,IAAI,WAAW,CAAC,YAAY,CAAC;gBAC/B,CAAC,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC;YACpC,SAAS,EAAE,iBAAiB;YAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,IAAI;SACL,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED,KAAK,UAAU,WAAW,CACxB,IAAgB,EAChB,MAAc,EACd,QAAgB;IAEhB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,OAAO,MAAM,GAAG,MAAM,CAAC,UAAU,EAAE,CAAC;QAClC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,IAAI,CAAC,IAAI,CACnC,MAAM,EACN,MAAM,EACN,MAAM,CAAC,UAAU,GAAG,MAAM,EAC1B,QAAQ,GAAG,MAAM,CAClB,CAAC;QACF,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,IAAI,SAAS,CAAC;IACtB,CAAC;AACH,CAAC"}

package/dist/engine/models/backends/qwen.d.ts

@@ -1,44 +0,0 @@
-import type { Content } from "../../types.js";
-import { BaseEmbeddingModel, type CreateEmbeddingModelOptions, type EmbeddingModelInfo, type EmbeddingResult, type NormalizedEmbeddingOptions } from "../embeddings.js";
-import type { QwenMultimodalEmbeddingCatalogEntry, QwenTextEmbeddingCatalogEntry } from "../catalog.js";
-type QwenDependencies = {
-    fetch: typeof globalThis.fetch;
-};
-type QwenTextEmbeddingSpec = {
-    displayName: string;
-    errorCodePrefix: string;
-};
-type QwenTextEmbeddingV4CatalogEntry = Extract<QwenTextEmbeddingCatalogEntry, {
-    model: "text-embedding-v4";
-}>;
-type Qwen37TextEmbeddingCatalogEntry = Extract<QwenTextEmbeddingCatalogEntry, {
-    model: "qwen3.7-text-embedding";
-}>;
-declare abstract class QwenTextEmbeddingModel extends BaseEmbeddingModel {
-    readonly info: EmbeddingModelInfo;
-    private readonly entry;
-    private readonly apiKey;
-    private readonly endpoint;
-    private readonly displayName;
-    private readonly errorCodePrefix;
-    private readonly dependencies;
-    constructor(entry: QwenTextEmbeddingCatalogEntry, spec: QwenTextEmbeddingSpec, options: CreateEmbeddingModelOptions, dependencies: Partial<QwenDependencies>);
-    protected doEmbed(contents: readonly Content[], options: NormalizedEmbeddingOptions): Promise<EmbeddingResult>;
-    private errorCode;
-}
-export declare class QwenTextEmbeddingV4Model extends QwenTextEmbeddingModel {
-    constructor(entry: QwenTextEmbeddingV4CatalogEntry, options: CreateEmbeddingModelOptions, dependencies?: Partial<QwenDependencies>);
-}
-export declare class Qwen37TextEmbeddingModel extends QwenTextEmbeddingModel {
-    constructor(entry: Qwen37TextEmbeddingCatalogEntry, options: CreateEmbeddingModelOptions, dependencies?: Partial<QwenDependencies>);
-}
-export declare class Qwen3VlEmbeddingModel extends BaseEmbeddingModel {
-    readonly info: EmbeddingModelInfo;
-    private readonly entry;
-    private readonly apiKey;
-    private readonly endpoint;
-    private readonly dependencies;
-    constructor(entry: QwenMultimodalEmbeddingCatalogEntry, options: CreateEmbeddingModelOptions, dependencies?: Partial<QwenDependencies>);
-    protected doEmbed(contents: readonly Content[], options: NormalizedEmbeddingOptions): Promise<EmbeddingResult>;
-}
-export {};