@angri450/zvec-grep 0.2.2

package/dist/engine/models/embeddings.js

@@ -0,0 +1,125 @@
+import { EngineError } from "../errors.js";
+export const EmbeddingPurpose = {
+    Document: "document",
+    Query: "query",
+};
+export class BaseEmbeddingModel {
+    async embed(contents, options = {}) {
+        this.validateContents(contents);
+        const result = await this.doEmbed(contents, {
+            purpose: options.purpose ?? "document",
+            signal: options.signal,
+            onProgress: options.onProgress,
+        });
+        this.validateResult(contents, result);
+        return result;
+    }
+    async dispose() {
+        // Most embedding backends do not hold local resources.
+    }
+    validateContents(contents) {
+        if (contents.length === 0) {
+            throw new EngineError("Embedding requires at least one content item", {
+                code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_EMPTY_INPUT",
+            });
+        }
+        if (contents.length > this.info.limits.maxBatchSize) {
+            throw new EngineError("Embedding batch size exceeds model limit", {
+                code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_BATCH_TOO_LARGE",
+                context: `model=${this.info.reference} batchSize=${contents.length} maxBatchSize=${this.info.limits.maxBatchSize}`,
+            });
+        }
+        for (const [index, content] of contents.entries()) {
+            if (!this.info.inputKinds.includes(content.kind)) {
+                throw new EngineError("Embedding model does not support content kind", {
+                    code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_UNSUPPORTED_CONTENT",
+                    context: `model=${this.info.reference} index=${index} kind=${content.kind}`,
+                });
+            }
+            if (content.kind === "text" && content.text.trim().length === 0) {
+                throw new EngineError("Embedding text content must not be empty", {
+                    code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_EMPTY_TEXT",
+                    context: `model=${this.info.reference} index=${index}`,
+                });
+            }
+            if (content.kind === "image") {
+                if (content.data.byteLength === 0) {
+                    throw new EngineError("Embedding image content must not be empty", {
+                        code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_EMPTY_IMAGE",
+                        context: `model=${this.info.reference} index=${index}`,
+                    });
+                }
+                if (content.format.trim().length === 0) {
+                    throw new EngineError("Embedding image content must include a format", {
+                        code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_MISSING_IMAGE_FORMAT",
+                        context: `model=${this.info.reference} index=${index}`,
+                    });
+                }
+                const maxImageBytes = this.info.limits.maxImageBytes;
+                if (maxImageBytes !== undefined &&
+                    content.data.byteLength > maxImageBytes) {
+                    throw new EngineError("Embedding image content exceeds model limit", {
+                        code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_IMAGE_TOO_LARGE",
+                        context: `model=${this.info.reference} index=${index} imageBytes=${content.data.byteLength} maxImageBytes=${maxImageBytes}`,
+                    });
+                }
+            }
+        }
+    }
+    validateResult(contents, result) {
+        if (!result || !Array.isArray(result.vectors)) {
+            throw new EngineError("Embedding model returned a non-array response", {
+                code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_INVALID_RESPONSE",
+                context: `model=${this.info.reference}`,
+            });
+        }
+        if (result.vectors.length !== contents.length) {
+            throw new EngineError("Embedding model returned the wrong number of vectors", {
+                code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_VECTOR_COUNT_MISMATCH",
+                context: `model=${this.info.reference} contentCount=${contents.length} vectorCount=${result.vectors.length}`,
+            });
+        }
+        for (const [vectorIndex, vector] of result.vectors.entries()) {
+            if (!Array.isArray(vector)) {
+                throw new EngineError("Embedding model returned a non-array vector", {
+                    code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_INVALID_VECTOR",
+                    context: `model=${this.info.reference} vectorIndex=${vectorIndex}`,
+                });
+            }
+            if (vector.length !== this.info.dimension) {
+                throw new EngineError("Embedding model returned a vector with the wrong dimension", {
+                    code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_DIMENSION_MISMATCH",
+                    context: `model=${this.info.reference} vectorIndex=${vectorIndex} expectedDimension=${this.info.dimension} actualDimension=${vector.length}`,
+                });
+            }
+            for (const [valueIndex, value] of vector.entries()) {
+                if (!Number.isFinite(value)) {
+                    throw new EngineError("Embedding model returned a non-finite vector value", {
+                        code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_NON_FINITE_VECTOR_VALUE",
+                        context: `model=${this.info.reference} vectorIndex=${vectorIndex} valueIndex=${valueIndex}`,
+                    });
+                }
+            }
+        }
+        if (!Array.isArray(result.truncated)) {
+            throw new EngineError("Embedding model returned invalid truncation information", {
+                code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_INVALID_TRUNCATION",
+                context: `model=${this.info.reference}`,
+            });
+        }
+        const seen = new Set();
+        for (const index of result.truncated) {
+            if (!Number.isInteger(index) ||
+                index < 0 ||
+                index >= contents.length ||
+                seen.has(index)) {
+                throw new EngineError("Embedding model returned an invalid truncated input index", {
+                    code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_INVALID_TRUNCATED_INPUT_INDEX",
+                    context: `model=${this.info.reference} index=${index} inputCount=${contents.length}`,
+                });
+            }
+            seen.add(index);
+        }
+    }
+}
+//# sourceMappingURL=embeddings.js.map

package/dist/engine/models/embeddings.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embeddings.js","sourceRoot":"","sources":["../../../src/engine/models/embeddings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAY3C,MAAM,CAAC,MAAM,gBAAgB,GAAG;IAC9B,QAAQ,EAAE,UAAU;IACpB,KAAK,EAAE,OAAO;CACN,CAAC;AAyEX,MAAM,OAAgB,kBAAkB;IAGtC,KAAK,CAAC,KAAK,CACT,QAA4B,EAC5B,UAA4B,EAAE;QAE9B,IAAI,CAAC,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAChC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE;YAC1C,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,UAAU;YACtC,MAAM,EAAE,OAAO,CAAC,MAAM;YACtB,UAAU,EAAE,OAAO,CAAC,UAAU;SAC/B,CAAC,CAAC;QACH,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACtC,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO;QACX,uDAAuD;IACzD,CAAC;IAOO,gBAAgB,CAAC,QAA4B;QACnD,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,WAAW,CAAC,8CAA8C,EAAE;gBACpE,IAAI,EAAE,+CAA+C;aACtD,CAAC,CAAC;QACL,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE,CAAC;YACpD,MAAM,IAAI,WAAW,CAAC,0CAA0C,EAAE;gBAChE,IAAI,EAAE,mDAAmD;gBACzD,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,cAAc,QAAQ,CAAC,MAAM,iBAAiB,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,YAAY,EAAE;aACnH,CAAC,CAAC;QACL,CAAC;QAED,KAAK,MAAM,CAAC,KAAK,EAAE,OAAO,CAAC,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;YAClD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;gBACjD,MAAM,IAAI,WAAW,CAAC,+CAA+C,EAAE;oBACrE,IAAI,EAAE,uDAAuD;oBAC7D,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,UAAU,KAAK,SAAS,OAAO,CAAC,IAAI,EAAE;iBAC5E,CAAC,CAAC;YACL,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBAChE,MAAM,IAAI,WAAW,CAAC,0CAA0C,EAAE;oBAChE,IAAI,EAAE,8CAA8C;oBACpD,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,UAAU,KAAK,EAAE;iBACvD,CAAC,CAAC;YACL,CAAC;YAED,IAAI,OAAO,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBAC7B,IAAI,OAAO,CAAC,IAAI,CAAC,UAAU,KAAK,CAAC,EAAE,CAAC;oBAClC,MAAM,IAAI,WAAW,CAAC,2CAA2C,EAAE;wBACjE,IAAI,EAAE,+CAA+C;wBACrD,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,UAAU,KAAK,EAAE;qBACvD,CAAC,CAAC;gBACL,CAAC;gBAED,IAAI,OAAO,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACvC,MAAM,IAAI,WAAW,CACnB,+CAA+C,EAC/C;wBACE,IAAI,EAAE,wDAAwD;wBAC9D,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,UAAU,KAAK,EAAE;qBACvD,CACF,CAAC;gBACJ,CAAC;gBAED,MAAM,aAAa,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC;gBACrD,IACE,aAAa,KAAK,SAAS;oBAC3B,OAAO,CAAC,IAAI,CAAC,UAAU,GAAG,aAAa,EACvC,CAAC;oBACD,MAAM,IAAI,WAAW,CAAC,6CAA6C,EAAE;wBACnE,IAAI,EAAE,mDAAmD;wBACzD,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,UAAU,KAAK,eAAe,OAAO,CAAC,IAAI,CAAC,UAAU,kBAAkB,aAAa,EAAE;qBAC5H,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAEO,cAAc,CACpB,QAA4B,EAC5B,MAAuB;QAEvB,IAAI,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9C,MAAM,IAAI,WAAW,CAAC,+CAA+C,EAAE;gBACrE,IAAI,EAAE,oDAAoD;gBAC1D,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;aACxC,CAAC,CAAC;QACL,CAAC;QAED,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,QAAQ,CAAC,MAAM,EAAE,CAAC;YAC9C,MAAM,IAAI,WAAW,CACnB,sDAAsD,EACtD;gBACE,IAAI,EAAE,yDAAyD;gBAC/D,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,iBAAiB,QAAQ,CAAC,MAAM,gBAAgB,MAAM,CAAC,OAAO,CAAC,MAAM,EAAE;aAC7G,CACF,CAAC;QACJ,CAAC;QAED,KAAK,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7D,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,WAAW,CAAC,6CAA6C,EAAE;oBACnE,IAAI,EAAE,kDAAkD;oBACxD,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,gBAAgB,WAAW,EAAE;iBACnE,CAAC,CAAC;YACL,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC1C,MAAM,IAAI,WAAW,CACnB,4DAA4D,EAC5D;oBACE,IAAI,EAAE,sDAAsD;oBAC5D,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,gBAAgB,WAAW,sBAAsB,IAAI,CAAC,IAAI,CAAC,SAAS,oBAAoB,MAAM,CAAC,MAAM,EAAE;iBAC7I,CACF,CAAC;YACJ,CAAC;YAED,KAAK,MAAM,CAAC,UAAU,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,EAAE,EAAE,CAAC;gBACnD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;oBAC5B,MAAM,IAAI,WAAW,CACnB,oDAAoD,EACpD;wBACE,IAAI,EAAE,2DAA2D;wBACjE,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,gBAAgB,WAAW,eAAe,UAAU,EAAE;qBAC5F,CACF,CAAC;gBACJ,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,CAAC,EAAE,CAAC;YACrC,MAAM,IAAI,WAAW,CACnB,yDAAyD,EACzD;gBACE,IAAI,EAAE,sDAAsD;gBAC5D,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE;aACxC,CACF,CAAC;QACJ,CAAC;QAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrC,IACE,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;gBACxB,KAAK,GAAG,CAAC;gBACT,KAAK,IAAI,QAAQ,CAAC,MAAM;gBACxB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,EACf,CAAC;gBACD,MAAM,IAAI,WAAW,CACnB,2DAA2D,EAC3D;oBACE,IAAI,EAAE,iEAAiE;oBACvE,OAAO,EAAE,SAAS,IAAI,CAAC,IAAI,CAAC,SAAS,UAAU,KAAK,eAAe,QAAQ,CAAC,MAAM,EAAE;iBACrF,CACF,CAAC;YACJ,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAClB,CAAC;IACH,CAAC;CACF"}

package/dist/engine/models/factory.d.ts

@@ -0,0 +1,2 @@
+import type { CreateEmbeddingModelOptions, EmbeddingModel } from "./embeddings.js";
+export declare function createEmbeddingModel(reference: string, options?: CreateEmbeddingModelOptions): EmbeddingModel;

package/dist/engine/models/factory.js

@@ -0,0 +1,51 @@
+import { EngineError } from "../errors.js";
+import { getEmbeddingModelCatalogEntry, } from "./catalog.js";
+import { LlamaCppEmbeddingModel } from "./backends/llama-cpp.js";
+import { Model2VecEmbeddingModel } from "./backends/model2vec.js";
+import { Qwen37TextEmbeddingModel, Qwen3VlEmbeddingModel, QwenTextEmbeddingV4Model, } from "./backends/qwen.js";
+import { TransformersJsEmbeddingModel } from "./backends/transformers-js.js";
+export function createEmbeddingModel(reference, options = {}) {
+    const catalogEntry = getEmbeddingModelCatalogEntry(reference);
+    if (!catalogEntry) {
+        throw new EngineError("Embedding model is not in the zvec-grep catalog", {
+            code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_CATALOG_MODEL_NOT_FOUND",
+            context: `embedding=${reference}`,
+        });
+    }
+    switch (catalogEntry.backend) {
+        case "llama-cpp":
+            return new LlamaCppEmbeddingModel(catalogEntry, options);
+        case "model2vec":
+            return new Model2VecEmbeddingModel(catalogEntry, options);
+        case "transformers-js":
+            return new TransformersJsEmbeddingModel(catalogEntry, options);
+        case "qwen":
+            return createQwenEmbeddingModel(catalogEntry, options);
+        default:
+            return unsupportedCatalogEntry(catalogEntry);
+    }
+}
+function createQwenEmbeddingModel(entry, options) {
+    switch (entry.kind) {
+        case "multimodal":
+            return new Qwen3VlEmbeddingModel(entry, options);
+        case "text":
+            switch (entry.model) {
+                case "text-embedding-v4":
+                    return new QwenTextEmbeddingV4Model(entry, options);
+                case "qwen3.7-text-embedding":
+                    return new Qwen37TextEmbeddingModel(entry, options);
+                default:
+                    return unsupportedCatalogEntry(entry);
+            }
+        default:
+            return unsupportedCatalogEntry(entry);
+    }
+}
+function unsupportedCatalogEntry(entry) {
+    throw new EngineError("Embedding catalog entry is not implemented", {
+        code: "ZVEC_GREP.ENGINE.MODELS.EMBEDDING_MODEL_NOT_IMPLEMENTED",
+        context: JSON.stringify(entry),
+    });
+}
+//# sourceMappingURL=factory.js.map

package/dist/engine/models/factory.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"factory.js","sourceRoot":"","sources":["../../../src/engine/models/factory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EACL,6BAA6B,GAE9B,MAAM,cAAc,CAAC;AAKtB,OAAO,EAAE,sBAAsB,EAAE,MAAM,yBAAyB,CAAC;AACjE,OAAO,EAAE,uBAAuB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,EACL,wBAAwB,EACxB,qBAAqB,EACrB,wBAAwB,GACzB,MAAM,oBAAoB,CAAC;AAC5B,OAAO,EAAE,4BAA4B,EAAE,MAAM,+BAA+B,CAAC;AAE7E,MAAM,UAAU,oBAAoB,CAClC,SAAiB,EACjB,UAAuC,EAAE;IAEzC,MAAM,YAAY,GAAG,6BAA6B,CAAC,SAAS,CAAC,CAAC;IAC9D,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,MAAM,IAAI,WAAW,CAAC,iDAAiD,EAAE;YACvE,IAAI,EAAE,2DAA2D;YACjE,OAAO,EAAE,aAAa,SAAS,EAAE;SAClC,CAAC,CAAC;IACL,CAAC;IAED,QAAQ,YAAY,CAAC,OAAO,EAAE,CAAC;QAC7B,KAAK,WAAW;YACd,OAAO,IAAI,sBAAsB,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAC3D,KAAK,WAAW;YACd,OAAO,IAAI,uBAAuB,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QAC5D,KAAK,iBAAiB;YACpB,OAAO,IAAI,4BAA4B,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACjE,KAAK,MAAM;YACT,OAAO,wBAAwB,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzD;YACE,OAAO,uBAAuB,CAAC,YAAY,CAAC,CAAC;IACjD,CAAC;AACH,CAAC;AAED,SAAS,wBAAwB,CAC/B,KAAgC,EAChC,OAAoC;IAEpC,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,YAAY;YACf,OAAO,IAAI,qBAAqB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;QACnD,KAAK,MAAM;YACT,QAAQ,KAAK,CAAC,KAAK,EAAE,CAAC;gBACpB,KAAK,mBAAmB;oBACtB,OAAO,IAAI,wBAAwB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;gBACtD,KAAK,wBAAwB;oBAC3B,OAAO,IAAI,wBAAwB,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;gBACtD;oBACE,OAAO,uBAAuB,CAAC,KAAK,CAAC,CAAC;YAC1C,CAAC;QACH;YACE,OAAO,uBAAuB,CAAC,KAAK,CAAC,CAAC;IAC1C,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAY;IAC3C,MAAM,IAAI,WAAW,CAAC,4CAA4C,EAAE;QAClE,IAAI,EAAE,yDAAyD;QAC/D,OAAO,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;KAC/B,CAAC,CAAC;AACL,CAAC"}

package/dist/engine/models/index.d.ts

@@ -0,0 +1,5 @@
+export { EmbeddingPurpose, type CreateEmbeddingModelOptions, type EmbeddingModel, type EmbeddingModelProgress, type EmbeddingModelInfo, type EmbeddingOptions, type EmbeddingResult, } from "./embeddings.js";
+export { createEmbeddingModel } from "./factory.js";
+export { normalizeLocalEmbeddingConcurrency, resolveIndexEmbeddingConcurrencyOverride, } from "./local-embedding-parallelism.js";
+export { getEmbeddingModelCatalogEntry, listEmbeddingModels, } from "./catalog.js";
+export { resolveEmbeddingReference, type ResolveEmbeddingReferenceOptions, } from "./resolution.js";

package/dist/engine/models/index.js

@@ -0,0 +1,6 @@
+export { EmbeddingPurpose, } from "./embeddings.js";
+export { createEmbeddingModel } from "./factory.js";
+export { normalizeLocalEmbeddingConcurrency, resolveIndexEmbeddingConcurrencyOverride, } from "./local-embedding-parallelism.js";
+export { getEmbeddingModelCatalogEntry, listEmbeddingModels, } from "./catalog.js";
+export { resolveEmbeddingReference, } from "./resolution.js";
+//# sourceMappingURL=index.js.map

package/dist/engine/models/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/engine/models/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,gBAAgB,GAOjB,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,oBAAoB,EAAE,MAAM,cAAc,CAAC;AACpD,OAAO,EACL,kCAAkC,EAClC,wCAAwC,GACzC,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EACL,6BAA6B,EAC7B,mBAAmB,GACpB,MAAM,cAAc,CAAC;AACtB,OAAO,EACL,yBAAyB,GAE1B,MAAM,iBAAiB,CAAC"}

package/dist/engine/models/local-embedding-parallelism.d.ts

@@ -0,0 +1,15 @@
+export declare const INDEX_EMBEDDING_CONCURRENCY_ENV = "ZVEC_GREP_INDEX_EMBEDDING_CONCURRENCY";
+/** Validate an explicit backend limit without consulting the environment. */
+export declare function normalizeLocalEmbeddingConcurrency(value?: number): number | undefined;
+/** Resolve index-stage configuration; individual backends apply their own cap. */
+export declare function resolveIndexEmbeddingConcurrencyOverride(options?: {
+    embeddingConcurrency?: number;
+    legacyLlama?: boolean;
+}): number | undefined;
+export declare function resolveLocalEmbeddingParallelism(options: {
+    override?: number;
+    gpu: boolean;
+    getVramState?: () => Promise<{
+        free: number;
+    }>;
+}): Promise<number>;

package/dist/engine/models/local-embedding-parallelism.js

@@ -0,0 +1,63 @@
+import { EngineError } from "../errors.js";
+export const INDEX_EMBEDDING_CONCURRENCY_ENV = "ZVEC_GREP_INDEX_EMBEDDING_CONCURRENCY";
+const LEGACY_LLAMA_PARALLELISM_ENV = "ZVEC_GREP_LLAMA_CONTEXT_PARALLELISM";
+const DEFAULT_PARALLELISM_CAP = 8;
+const BYTES_PER_MIB = 1024 * 1024;
+const CONTEXT_VRAM_MB = 150;
+const GPU_VRAM_BUDGET_RATIO = 0.25;
+/** Validate an explicit backend limit without consulting the environment. */
+export function normalizeLocalEmbeddingConcurrency(value) {
+    const concurrency = validateEmbeddingConcurrency(value);
+    return concurrency === undefined
+        ? undefined
+        : Math.min(DEFAULT_PARALLELISM_CAP, concurrency);
+}
+/** Resolve index-stage configuration; individual backends apply their own cap. */
+export function resolveIndexEmbeddingConcurrencyOverride(options = {}) {
+    if (options.embeddingConcurrency !== undefined) {
+        return validateEmbeddingConcurrency(options.embeddingConcurrency);
+    }
+    let name = INDEX_EMBEDDING_CONCURRENCY_ENV;
+    let value = process.env[name]?.trim() ?? "";
+    if (!value && options.legacyLlama) {
+        name = LEGACY_LLAMA_PARALLELISM_ENV;
+        value = process.env[name]?.trim() ?? "";
+    }
+    if (!value) {
+        return undefined;
+    }
+    const concurrency = Number(value);
+    if (!/^0*[1-9]\d*$/.test(value) || !Number.isSafeInteger(concurrency)) {
+        process.stderr.write(`zvec-grep warning: invalid ${name}="${value}", using automatic parallelism.\n`);
+        return undefined;
+    }
+    return concurrency;
+}
+function validateEmbeddingConcurrency(value) {
+    if (value !== undefined && (!Number.isSafeInteger(value) || value < 1)) {
+        throw new EngineError("Embedding concurrency requires a positive integer", {
+            code: "ZVEC_GREP.ENGINE.MODELS.INVALID_EMBEDDING_CONCURRENCY",
+        });
+    }
+    return value;
+}
+export async function resolveLocalEmbeddingParallelism(options) {
+    if (options.override !== undefined) {
+        return options.override;
+    }
+    if (!options.gpu || !options.getVramState) {
+        return 1;
+    }
+    try {
+        const { free } = await options.getVramState();
+        if (!Number.isFinite(free) || free < 0) {
+            return 2;
+        }
+        const freeMb = free / BYTES_PER_MIB;
+        return Math.max(1, Math.min(DEFAULT_PARALLELISM_CAP, Math.floor((freeMb * GPU_VRAM_BUDGET_RATIO) / CONTEXT_VRAM_MB)));
+    }
+    catch {
+        return 2;
+    }
+}
+//# sourceMappingURL=local-embedding-parallelism.js.map

package/dist/engine/models/local-embedding-parallelism.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-embedding-parallelism.js","sourceRoot":"","sources":["../../../src/engine/models/local-embedding-parallelism.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAE3C,MAAM,CAAC,MAAM,+BAA+B,GAC1C,uCAAuC,CAAC;AAE1C,MAAM,4BAA4B,GAAG,qCAAqC,CAAC;AAC3E,MAAM,uBAAuB,GAAG,CAAC,CAAC;AAClC,MAAM,aAAa,GAAG,IAAI,GAAG,IAAI,CAAC;AAClC,MAAM,eAAe,GAAG,GAAG,CAAC;AAC5B,MAAM,qBAAqB,GAAG,IAAI,CAAC;AAEnC,6EAA6E;AAC7E,MAAM,UAAU,kCAAkC,CAChD,KAAc;IAEd,MAAM,WAAW,GAAG,4BAA4B,CAAC,KAAK,CAAC,CAAC;IACxD,OAAO,WAAW,KAAK,SAAS;QAC9B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,uBAAuB,EAAE,WAAW,CAAC,CAAC;AACrD,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,wCAAwC,CACtD,UAAoE,EAAE;IAEtE,IAAI,OAAO,CAAC,oBAAoB,KAAK,SAAS,EAAE,CAAC;QAC/C,OAAO,4BAA4B,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;IACpE,CAAC;IACD,IAAI,IAAI,GAAG,+BAA+B,CAAC;IAC3C,IAAI,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC5C,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,WAAW,EAAE,CAAC;QAClC,IAAI,GAAG,4BAA4B,CAAC;QACpC,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC1C,CAAC;IACD,IAAI,CAAC,KAAK,EAAE,CAAC;QACX,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,MAAM,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAClC,IAAI,CAAC,cAAc,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,WAAW,CAAC,EAAE,CAAC;QACtE,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,8BAA8B,IAAI,KAAK,KAAK,mCAAmC,CAChF,CAAC;QACF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,OAAO,WAAW,CAAC;AACrB,CAAC;AAED,SAAS,4BAA4B,CAAC,KAAc;IAClD,IAAI,KAAK,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,WAAW,CAAC,mDAAmD,EAAE;YACzE,IAAI,EAAE,uDAAuD;SAC9D,CAAC,CAAC;IACL,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,gCAAgC,CAAC,OAItD;IACC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,CAAC;QACnC,OAAO,OAAO,CAAC,QAAQ,CAAC;IAC1B,CAAC;IACD,IAAI,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,YAAY,EAAE,CAAC;QAC1C,OAAO,CAAC,CAAC;IACX,CAAC;IAED,IAAI,CAAC;QACH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,OAAO,CAAC,YAAY,EAAE,CAAC;QAC9C,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC;YACvC,OAAO,CAAC,CAAC;QACX,CAAC;QACD,MAAM,MAAM,GAAG,IAAI,GAAG,aAAa,CAAC;QACpC,OAAO,IAAI,CAAC,GAAG,CACb,CAAC,EACD,IAAI,CAAC,GAAG,CACN,uBAAuB,EACvB,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,qBAAqB,CAAC,GAAG,eAAe,CAAC,CAC/D,CACF,CAAC;IACJ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC"}

package/dist/engine/models/local-embedding-queue.d.ts

@@ -0,0 +1,12 @@
+/** A FIFO inference queue with barriers for runtime replacement and disposal. */
+export declare class LocalEmbeddingQueue {
+    private readonly resolveLimit;
+    private readonly pending;
+    private limit;
+    private resolving;
+    private active;
+    private exclusive;
+    constructor(resolveLimit: () => Promise<number>);
+    run<T>(operation: () => Promise<T>, exclusive?: boolean): Promise<T>;
+    private pump;
+}

package/dist/engine/models/local-embedding-queue.js

@@ -0,0 +1,65 @@
+/** A FIFO inference queue with barriers for runtime replacement and disposal. */
+export class LocalEmbeddingQueue {
+    resolveLimit;
+    pending = [];
+    limit;
+    resolving = false;
+    active = 0;
+    exclusive = false;
+    constructor(resolveLimit) {
+        this.resolveLimit = resolveLimit;
+    }
+    run(operation, exclusive = false) {
+        return new Promise((resolve, reject) => {
+            this.pending.push({
+                exclusive,
+                reject,
+                start: () => {
+                    this.active++;
+                    this.exclusive = exclusive;
+                    void Promise.resolve()
+                        .then(operation)
+                        .then(resolve, reject)
+                        .finally(() => {
+                        this.active--;
+                        if (exclusive)
+                            this.exclusive = false;
+                        this.pump();
+                    });
+                },
+            });
+            this.pump();
+        });
+    }
+    pump() {
+        if (this.pending.length === 0 || this.exclusive)
+            return;
+        if (this.limit === undefined) {
+            if (this.resolving)
+                return;
+            this.resolving = true;
+            void Promise.resolve()
+                .then(this.resolveLimit)
+                .then((limit) => {
+                this.limit = limit;
+                this.resolving = false;
+                this.pump();
+            }, (cause) => {
+                this.resolving = false;
+                for (const operation of this.pending.splice(0)) {
+                    operation.reject(cause);
+                }
+            });
+            return;
+        }
+        while (this.pending.length > 0 && this.active < this.limit) {
+            if (this.pending[0].exclusive && this.active > 0)
+                return;
+            const operation = this.pending.shift();
+            operation.start();
+            if (operation.exclusive)
+                return;
+        }
+    }
+}
+//# sourceMappingURL=local-embedding-queue.js.map

package/dist/engine/models/local-embedding-queue.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"local-embedding-queue.js","sourceRoot":"","sources":["../../../src/engine/models/local-embedding-queue.ts"],"names":[],"mappings":"AAMA,iFAAiF;AACjF,MAAM,OAAO,mBAAmB;IAOD;IANZ,OAAO,GAAuB,EAAE,CAAC;IAC1C,KAAK,CAAqB;IAC1B,SAAS,GAAG,KAAK,CAAC;IAClB,MAAM,GAAG,CAAC,CAAC;IACX,SAAS,GAAG,KAAK,CAAC;IAE1B,YAA6B,YAAmC;QAAnC,iBAAY,GAAZ,YAAY,CAAuB;IAAG,CAAC;IAEpE,GAAG,CAAI,SAA2B,EAAE,SAAS,GAAG,KAAK;QACnD,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACxC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;gBAChB,SAAS;gBACT,MAAM;gBACN,KAAK,EAAE,GAAG,EAAE;oBACV,IAAI,CAAC,MAAM,EAAE,CAAC;oBACd,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;oBAC3B,KAAK,OAAO,CAAC,OAAO,EAAE;yBACnB,IAAI,CAAC,SAAS,CAAC;yBACf,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC;yBACrB,OAAO,CAAC,GAAG,EAAE;wBACZ,IAAI,CAAC,MAAM,EAAE,CAAC;wBACd,IAAI,SAAS;4BAAE,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;wBACtC,IAAI,CAAC,IAAI,EAAE,CAAC;oBACd,CAAC,CAAC,CAAC;gBACP,CAAC;aACF,CAAC,CAAC;YACH,IAAI,CAAC,IAAI,EAAE,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,IAAI;QACV,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO;QACxD,IAAI,IAAI,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAC7B,IAAI,IAAI,CAAC,SAAS;gBAAE,OAAO;YAC3B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;YACtB,KAAK,OAAO,CAAC,OAAO,EAAE;iBACnB,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC;iBACvB,IAAI,CACH,CAAC,KAAK,EAAE,EAAE;gBACR,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;gBACnB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBACvB,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,CAAC,EACD,CAAC,KAAc,EAAE,EAAE;gBACjB,IAAI,CAAC,SAAS,GAAG,KAAK,CAAC;gBACvB,KAAK,MAAM,SAAS,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;oBAC/C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC1B,CAAC;YACH,CAAC,CACF,CAAC;YACJ,OAAO;QACT,CAAC;QAED,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;YAC3D,IAAI,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,OAAO;YACzD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,KAAK,EAAG,CAAC;YACxC,SAAS,CAAC,KAAK,EAAE,CAAC;YAClB,IAAI,SAAS,CAAC,SAAS;gBAAE,OAAO;QAClC,CAAC;IACH,CAAC;CACF"}

package/dist/engine/models/ranking.d.ts

@@ -0,0 +1,19 @@
+import type { Content, ContentKind } from "../types.js";
+export type RankingCandidate = {
+    id: string;
+    content: Content;
+};
+export type RankingScore = {
+    id: string;
+    score: number;
+};
+export interface RankingModel {
+    readonly info: Readonly<{
+        reference: string;
+        provider: string;
+        name: string;
+        inputKinds: readonly ContentKind[];
+    }>;
+    rank(query: Content, candidates: readonly RankingCandidate[]): Promise<RankingScore[]>;
+    dispose(): Promise<void>;
+}

package/dist/engine/models/ranking.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=ranking.js.map

package/dist/engine/models/ranking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ranking.js","sourceRoot":"","sources":["../../../src/engine/models/ranking.ts"],"names":[],"mappings":""}

package/dist/engine/models/resolution.d.ts

@@ -0,0 +1,8 @@
+export type ResolveEmbeddingReferenceOptions = {
+    explicit?: string;
+    existing?: string;
+    globalDefault?: string;
+    environment?: NodeJS.ProcessEnv;
+    fallback?: string;
+};
+export declare function resolveEmbeddingReference(options: ResolveEmbeddingReferenceOptions): string | undefined;

package/dist/engine/models/resolution.js

@@ -0,0 +1,22 @@
+import { EngineError } from "../errors.js";
+import { getEmbeddingModelCatalogEntry } from "./catalog.js";
+export function resolveEmbeddingReference(options) {
+    if (options.explicit !== undefined)
+        return options.explicit;
+    if (options.existing !== undefined)
+        return options.existing;
+    const environmentReference = nonEmptyEnvironmentValue((options.environment ?? process.env).ZVEC_GREP_EMBEDDING);
+    if (environmentReference !== undefined &&
+        !getEmbeddingModelCatalogEntry(environmentReference)) {
+        throw new EngineError(`Invalid ZVEC_GREP_EMBEDDING: unsupported model ${environmentReference}. Run \`zg --help models\` to list supported models.`, {
+            code: "ZVEC_GREP.ENGINE.CONFIG.EMBEDDING_ENVIRONMENT_INVALID",
+            context: "source=ZVEC_GREP_EMBEDDING",
+        });
+    }
+    return environmentReference ?? options.globalDefault ?? options.fallback;
+}
+function nonEmptyEnvironmentValue(value) {
+    const normalized = value?.trim();
+    return normalized ? normalized : undefined;
+}
+//# sourceMappingURL=resolution.js.map

package/dist/engine/models/resolution.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolution.js","sourceRoot":"","sources":["../../../src/engine/models/resolution.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,cAAc,CAAC;AAC3C,OAAO,EAAE,6BAA6B,EAAE,MAAM,cAAc,CAAC;AAU7D,MAAM,UAAU,yBAAyB,CACvC,OAAyC;IAEzC,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,QAAQ,CAAC;IAC5D,IAAI,OAAO,CAAC,QAAQ,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC,QAAQ,CAAC;IAE5D,MAAM,oBAAoB,GAAG,wBAAwB,CACnD,CAAC,OAAO,CAAC,WAAW,IAAI,OAAO,CAAC,GAAG,CAAC,CAAC,mBAAmB,CACzD,CAAC;IACF,IACE,oBAAoB,KAAK,SAAS;QAClC,CAAC,6BAA6B,CAAC,oBAAoB,CAAC,EACpD,CAAC;QACD,MAAM,IAAI,WAAW,CACnB,kDAAkD,oBAAoB,sDAAsD,EAC5H;YACE,IAAI,EAAE,uDAAuD;YAC7D,OAAO,EAAE,4BAA4B;SACtC,CACF,CAAC;IACJ,CAAC;IACD,OAAO,oBAAoB,IAAI,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,QAAQ,CAAC;AAC3E,CAAC;AAED,SAAS,wBAAwB,CAC/B,KAAyB;IAEzB,MAAM,UAAU,GAAG,KAAK,EAAE,IAAI,EAAE,CAAC;IACjC,OAAO,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,CAAC;AAC7C,CAAC"}

package/dist/engine/pipeline/indexing/index.d.ts

@@ -0,0 +1,14 @@
+import type { EmbeddingModel } from "../../models/index.js";
+import type { WorkspaceIndexStorage } from "../../storage/index.js";
+import type { WorkspaceIndexStatus, WorkspaceIndexInfo, FileInfo, IndexProgress, IndexResult } from "../../types.js";
+export type IndexContext = {
+    workspaceIndex: WorkspaceIndexInfo;
+    storage: WorkspaceIndexStorage;
+    embeddingModel: EmbeddingModel;
+    embeddingConcurrency?: number;
+    onProgress?: (progress: IndexProgress) => void;
+    signal?: AbortSignal;
+};
+export declare function indexWorkspace(ctx: IndexContext): Promise<IndexResult>;
+export declare function indexWorkspacePaths(ctx: IndexContext, changedPaths: readonly string[]): Promise<IndexResult>;
+export declare function getWorkspaceIndexStatus(workspaceIndex: WorkspaceIndexInfo, storedFiles: readonly FileInfo[]): Promise<WorkspaceIndexStatus>;