@zvec/zvec-grep 0.1.6 → 0.1.8

package/dist/engine/file-type.js

@@ -57,46 +57,79 @@ const IMAGE_FORMATS = {
     ".png": "png",
     ".webp": "webp",
 };
-const BINARY_EXTENSIONS = new Set([
-    ".zip",
-    ".tar",
-    ".gz",
-    ".bz2",
-    ".xz",
-    ".7z",
-    ".rar",
-    ".exe",
-    ".dll",
-    ".dylib",
-    ".so",
-    ".a",
-    ".o",
-    ".obj",
-    ".wasm",
-    ".class",
-    ".jar",
-    ".pdf",
-    ".doc",
-    ".docx",
-    ".ppt",
-    ".pptx",
-    ".xls",
-    ".xlsx",
-    ".mp3",
-    ".mp4",
-    ".mov",
-    ".avi",
-    ".mkv",
-    ".db",
-    ".sqlite",
-]);
+const NAMED_FILE_FORMATS = {
+    Dockerfile: { kind: "code", format: "dockerfile" },
+    Makefile: { kind: "code", format: "makefile" },
+};
+const FILE_FORMATS_BY_KIND = {
+    code: CODE_FORMATS,
+    data: DATA_FORMATS,
+    text: TEXT_FORMATS,
+    image: IMAGE_FORMATS,
+};
+const BINARY_EXTENSION_GROUPS = [
+    {
+        label: "Archives",
+        extensions: [".zip", ".tar", ".gz", ".bz2", ".xz", ".7z", ".rar"],
+    },
+    {
+        label: "Compiled",
+        extensions: [
+            ".exe",
+            ".dll",
+            ".dylib",
+            ".so",
+            ".a",
+            ".o",
+            ".obj",
+            ".wasm",
+            ".class",
+            ".jar",
+        ],
+    },
+    {
+        label: "Documents",
+        extensions: [".pdf", ".doc", ".docx", ".ppt", ".pptx", ".xls", ".xlsx"],
+    },
+    {
+        label: "Media",
+        extensions: [".mp3", ".mp4", ".mov", ".avi", ".mkv"],
+    },
+    { label: "Databases", extensions: [".db", ".sqlite"] },
+];
+const BINARY_EXTENSIONS = new Set(BINARY_EXTENSION_GROUPS.flatMap((group) => group.extensions));
+export function listRecognizedFileTypes() {
+    const recognized = new Map();
+    const append = (kind, format, pattern) => {
+        const key = `${kind}:${format}`;
+        const existing = recognized.get(key);
+        if (existing) {
+            existing.patterns.push(pattern);
+            return;
+        }
+        recognized.set(key, { kind, format, patterns: [pattern] });
+    };
+    for (const kind of Object.keys(FILE_FORMATS_BY_KIND)) {
+        for (const [extension, format] of Object.entries(FILE_FORMATS_BY_KIND[kind])) {
+            append(kind, format, extension);
+        }
+    }
+    for (const [name, type] of Object.entries(NAMED_FILE_FORMATS)) {
+        append(type.kind, type.format, name);
+    }
+    return [...recognized.values()];
+}
+export function listKnownBinaryExtensionGroups() {
+    return BINARY_EXTENSION_GROUPS.map((group) => ({
+        label: group.label,
+        extensions: [...group.extensions],
+    }));
+}
 export function detectFileType(path) {
     const name = basename(path);
-    if (name === "Dockerfile") {
-        return { kind: "code", format: "dockerfile" };
-    }
-    if (name === "Makefile") {
-        return { kind: "code", format: "makefile" };
+    const namedType = NAMED_FILE_FORMATS[name];
+    if (namedType) {
+        return { ...namedType };
     }
     const extension = extname(name).toLowerCase();
     if (extension in CODE_FORMATS) {

package/dist/engine/file-type.js.map

@@ -1 +1 @@
-{"version":3,"file":"file-type.js","sourceRoot":"","sources":["../../src/engine/file-type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAG9C,MAAM,YAAY,GAA2B;IAC3C,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,IAAI,EAAE,KAAK;IACX,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,YAAY;IACpB,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,MAAM;IACb,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,KAAK;IACb,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,QAAQ;IACf,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,MAAM;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,SAAS,EAAE,QAAQ;CACpB,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,MAAM;IAChB,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;CACf,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,KAAK,EAAE,UAAU;IACjB,MAAM,EAAE,UAAU;IAClB,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,MAAM;IACd,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;CACd,CAAC;AAEF,MAAM,aAAa,GAAgC;IACjD,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;CAChB,CAAC;AAEF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IAChC,MAAM;IACN,MAAM;IACN,KAAK;IACL,MAAM;IACN,KAAK;IACL,KAAK;IACL,MAAM;IACN,MAAM;IACN,MAAM;IACN,QAAQ;IACR,KAAK;IACL,IAAI;IACJ,IAAI;IACJ,MAAM;IACN,OAAO;IACP,QAAQ;IACR,MAAM;IACN,MAAM;IACN,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,OAAO;IACP,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,MAAM;IACN,KAAK;IACL,SAAS;CACV,CAAC,CAAC;AAEH,MAAM,UAAU,cAAc,CAC5B,IAAY;IAEZ,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAE5B,IAAI,IAAI,KAAK,YAAY,EAAE,CAAC;QAC1B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC;IAChD,CAAC;IAED,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACxB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IAE9C,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,SAAS,IAAI,aAAa,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,IAAI,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;AAC3E,CAAC"}
+{"version":3,"file":"file-type.js","sourceRoot":"","sources":["../../src/engine/file-type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAG9C,MAAM,YAAY,GAA2B;IAC3C,IAAI,EAAE,GAAG;IACT,KAAK,EAAE,KAAK;IACZ,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,IAAI,EAAE,KAAK;IACX,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,IAAI;IACX,OAAO,EAAE,MAAM;IACf,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,YAAY;IACpB,MAAM,EAAE,YAAY;IACpB,KAAK,EAAE,YAAY;IACnB,MAAM,EAAE,KAAK;IACb,KAAK,EAAE,QAAQ;IACf,KAAK,EAAE,MAAM;IACb,KAAK,EAAE,MAAM;IACb,MAAM,EAAE,KAAK;IACb,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,QAAQ;IACf,MAAM,EAAE,QAAQ;IAChB,KAAK,EAAE,QAAQ;IACf,QAAQ,EAAE,OAAO;IACjB,KAAK,EAAE,MAAM;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,KAAK;IACb,SAAS,EAAE,QAAQ;CACpB,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,QAAQ,EAAE,MAAM;IAChB,OAAO,EAAE,MAAM;IACf,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;CACf,CAAC;AAEF,MAAM,YAAY,GAA2B;IAC3C,KAAK,EAAE,UAAU;IACjB,MAAM,EAAE,UAAU;IAClB,MAAM,EAAE,KAAK;IACb,MAAM,EAAE,MAAM;IACd,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;CACd,CAAC;AAEF,MAAM,aAAa,GAAgC;IACjD,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;IACf,MAAM,EAAE,MAAM;IACd,MAAM,EAAE,KAAK;IACb,OAAO,EAAE,MAAM;CAChB,CAAC;AAEF,MAAM,kBAAkB,GAGpB;IACF,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE;IAClD,QAAQ,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE;CAC/C,CAAC;AAEF,MAAM,oBAAoB,GAAG;IAC3B,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,YAAY;IAClB,IAAI,EAAE,YAAY;IAClB,KAAK,EAAE,aAAa;CAC8B,CAAC;AAErD,MAAM,uBAAuB,GAAG;IAC9B;QACE,KAAK,EAAE,UAAU;QACjB,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC;KAClE;IACD;QACE,KAAK,EAAE,UAAU;QACjB,UAAU,EAAE;YACV,MAAM;YACN,MAAM;YACN,QAAQ;YACR,KAAK;YACL,IAAI;YACJ,IAAI;YACJ,MAAM;YACN,OAAO;YACP,QAAQ;YACR,MAAM;SACP;KACF;IACD;QACE,KAAK,EAAE,WAAW;QAClB,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC;KACxE;IACD;QACE,KAAK,EAAE,OAAO;QACd,UAAU,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;KACrD;IACD,EAAE,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,CAAC,KAAK,EAAE,SAAS,CAAC,EAAE;CAC9C,CAAC;AAEX,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAC/B,uBAAuB,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,UAAU,CAAC,CAC7D,CAAC;AAQF,MAAM,UAAU,uBAAuB;IACrC,MAAM,UAAU,GAAG,IAAI,GAAG,EAA8B,CAAC;IACzD,MAAM,MAAM,GAAG,CACb,IAAc,EACd,MAAkB,EAClB,OAAe,EACT,EAAE;QACR,MAAM,GAAG,GAAG,GAAG,IAAI,IAAI,MAAM,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACrC,IAAI,QAAQ,EAAE,CAAC;YACb,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAChC,OAAO;QACT,CAAC;QACD,UAAU,CAAC,GAAG,CAAC,GAAG,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC7D,CAAC,CAAC;IAEF,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,oBAAoB,CAAe,EAAE,CAAC;QACnE,KAAK,MAAM,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAC9C,oBAAoB,CAAC,IAAI,CAAC,CAC3B,EAAE,CAAC;YACF,MAAM,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;QAClC,CAAC;IACH,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAED,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,8BAA8B;IAI5C,OAAO,uBAAuB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;QAC7C,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,UAAU,EAAE,CAAC,GAAG,KAAK,CAAC,UAAU,CAAC;KAClC,CAAC,CAAC,CAAC;AACN,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,IAAY;IAEZ,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;IAC5B,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;IAC3C,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,EAAE,GAAG,SAAS,EAAE,CAAC;IAC1B,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IAE9C,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,SAAS,IAAI,YAAY,EAAE,CAAC;QAC9B,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,SAAS,CAAC,EAAE,CAAC;IAC3D,CAAC;IAED,IAAI,SAAS,IAAI,aAAa,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;IAC7D,CAAC;IAED,IAAI,iBAAiB,CAAC,GAAG,CAAC,SAAS,CAAC,EAAE,CAAC;QACrC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;AAC3E,CAAC"}

package/dist/engine/manifest.d.ts

@@ -9,4 +9,5 @@ export type WorkspaceManifest = WorkspaceIndexInfo & {
 export declare function workspaceManifestPath(home: string): string;
 export declare function readWorkspaceManifest(home: string): WorkspaceManifest | null;
 export declare function writeWorkspaceManifest(home: string, manifest: WorkspaceManifest): void;
+export declare function deleteWorkspaceManifest(home: string): void;
 export declare function workspaceIndexInfoFromManifest(manifest: WorkspaceManifest): WorkspaceIndexInfo;

package/dist/engine/manifest.js

@@ -1,3 +1,4 @@
+import { rmSync } from "node:fs";
 import { join } from "node:path";
 import { EngineError } from "./errors.js";
 import { readJsonFileSync, writeJsonFileSync } from "./utils/json.js";
@@ -28,6 +29,9 @@ export function writeWorkspaceManifest(home, manifest) {
         fileMode: WORKSPACE_MANIFEST_MODE,
     });
 }
+export function deleteWorkspaceManifest(home) {
+    rmSync(workspaceManifestPath(home), { force: true });
+}
 export function workspaceIndexInfoFromManifest(manifest) {
     return {
         id: manifest.id,

package/dist/engine/manifest.js.map

@@ -1 +1 @@
-{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/engine/manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEtE,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAC;AACvD,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AAE1C,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,MAAM,uBAAuB,GAAG,KAAK,CAAC;AAOtC,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,OAAO,IAAI,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,IAAI,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,gBAAgB,CAAU,IAAI,EAAE,IAAI,CAAC,CAAC;IACpD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,WAAW,CAAC,qCAAqC,EAAE;YAC3D,IAAI,EAAE,mCAAmC;YACzC,OAAO,EAAE,QAAQ,IAAI,EAAE;SACxB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,IAAY,EACZ,QAA2B;IAE3B,iBAAiB,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE;QACvD,aAAa,EAAE,wBAAwB;QACvC,QAAQ,EAAE,uBAAuB;KAClC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC5C,QAA2B;IAE3B,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,WAAW,EAAE,QAAQ,CAAC,WAAW;KAClC,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc;IACzC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,eAAe,KAAK,wBAAwB,EAAE,CAAC;QAC3E,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,CACL,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC;QAC5B,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC;QAC5B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC;QAC9B,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;QAC1B,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC;QACjC,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,IAAI,KAAK,CAAC,WAAW,KAAK,UAAU,CAAC;QACrE,CAAC,KAAK,CAAC,SAAS,KAAK,IAAI,IAAI,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChE,CAAC,KAAK,CAAC,YAAY,KAAK,IAAI,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACrE,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;QACrC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC;QAClC,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;QACrC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC;QAClC,kBAAkB,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAC3C,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,gBAAgB,CAAC,KAAK,CAAC,YAAY,CAAC;QACpC,OAAO,KAAK,CAAC,SAAS,KAAK,SAAS;QACpC,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC;QACpC,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC;QACpC,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC;QAClC,qBAAqB,CAAC,KAAK,CAAC,gBAAgB,CAAC;QAC7C,qBAAqB,CAAC,KAAK,CAAC,SAAS,CAAC;QACtC,qBAAqB,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAC9C,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC;QAC/B,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC;QACjC,qBAAqB,CAAC,KAAK,CAAC,WAAW,CAAC;QACxC,4BAA4B,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,4BAA4B,CAAC,KAAK,CAAC,gBAAgB,CAAC;QACpD,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC,CAChC,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAChC,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC;QAC7B,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC;QACjC,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,KAAK,CAAC,SAAS,GAAG,CAAC;QACnB,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ;YACxB,KAAK,CAAC,MAAM,KAAK,KAAK;YACtB,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAChC,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,CACL,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC;QAChE,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC;QACpE,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS;YACzB,KAAK,CAAC,MAAM,KAAK,MAAM;YACvB,KAAK,CAAC,MAAM,KAAK,KAAK;YACtB,KAAK,CAAC,MAAM,KAAK,OAAO;YACxB,KAAK,CAAC,MAAM,KAAK,QAAQ;YACzB,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAC3B,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,OAAO,CACL,KAAK,KAAK,SAAS;QACnB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAC1E,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,SAAS,CAAC;AAC3D,CAAC;AAED,SAAS,4BAA4B,CAAC,KAAc;IAClD,OAAO,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AACvD,CAAC"}
+{"version":3,"file":"manifest.js","sourceRoot":"","sources":["../../src/engine/manifest.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAEjC,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE1C,OAAO,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEtE,MAAM,CAAC,MAAM,uBAAuB,GAAG,eAAe,CAAC;AACvD,MAAM,CAAC,MAAM,wBAAwB,GAAG,CAAC,CAAC;AAE1C,MAAM,wBAAwB,GAAG,KAAK,CAAC;AACvC,MAAM,uBAAuB,GAAG,KAAK,CAAC;AAOtC,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,OAAO,IAAI,CAAC,IAAI,EAAE,uBAAuB,CAAC,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,IAAY;IAChD,MAAM,IAAI,GAAG,qBAAqB,CAAC,IAAI,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,gBAAgB,CAAU,IAAI,EAAE,IAAI,CAAC,CAAC;IACpD,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QACnB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,CAAC,mBAAmB,CAAC,KAAK,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,WAAW,CAAC,qCAAqC,EAAE;YAC3D,IAAI,EAAE,mCAAmC;YACzC,OAAO,EAAE,QAAQ,IAAI,EAAE;SACxB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,IAAY,EACZ,QAA2B;IAE3B,iBAAiB,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE;QACvD,aAAa,EAAE,wBAAwB;QACvC,QAAQ,EAAE,uBAAuB;KAClC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,IAAY;IAClD,MAAM,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACvD,CAAC;AAED,MAAM,UAAU,8BAA8B,CAC5C,QAA2B;IAE3B,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;QACnB,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,SAAS,EAAE,QAAQ,CAAC,SAAS;QAC7B,YAAY,EAAE,QAAQ,CAAC,YAAY;QACnC,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,WAAW,EAAE,QAAQ,CAAC,WAAW;KAClC,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc;IACzC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,eAAe,KAAK,wBAAwB,EAAE,CAAC;QAC3E,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,CACL,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC;QAC5B,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC;QAC5B,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC;QAC9B,KAAK,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC;QAC1B,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,UAAU,CAAC;QACjC,CAAC,KAAK,CAAC,WAAW,KAAK,SAAS,IAAI,KAAK,CAAC,WAAW,KAAK,UAAU,CAAC;QACrE,CAAC,KAAK,CAAC,SAAS,KAAK,IAAI,IAAI,iBAAiB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QAChE,CAAC,KAAK,CAAC,YAAY,KAAK,IAAI,IAAI,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;QACrE,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;QACrC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC;QAClC,OAAO,KAAK,CAAC,WAAW,KAAK,QAAQ;QACrC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC;QAClC,kBAAkB,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAC3C,CAAC;AACJ,CAAC;AAED,SAAS,UAAU,CAAC,KAAc;IAChC,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,gBAAgB,CAAC,KAAK,CAAC,YAAY,CAAC;QACpC,OAAO,KAAK,CAAC,SAAS,KAAK,SAAS;QACpC,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC;QACpC,qBAAqB,CAAC,KAAK,CAAC,OAAO,CAAC;QACpC,qBAAqB,CAAC,KAAK,CAAC,KAAK,CAAC;QAClC,qBAAqB,CAAC,KAAK,CAAC,gBAAgB,CAAC;QAC7C,qBAAqB,CAAC,KAAK,CAAC,SAAS,CAAC;QACtC,qBAAqB,CAAC,KAAK,CAAC,iBAAiB,CAAC;QAC9C,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC;QAC/B,iBAAiB,CAAC,KAAK,CAAC,QAAQ,CAAC;QACjC,qBAAqB,CAAC,KAAK,CAAC,WAAW,CAAC;QACxC,4BAA4B,CAAC,KAAK,CAAC,QAAQ,CAAC;QAC5C,4BAA4B,CAAC,KAAK,CAAC,gBAAgB,CAAC;QACpD,iBAAiB,CAAC,KAAK,CAAC,MAAM,CAAC,CAChC,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,CACL,QAAQ,CAAC,KAAK,CAAC;QACf,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC;QAChC,gBAAgB,CAAC,KAAK,CAAC,KAAK,CAAC;QAC7B,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC;QACjC,OAAO,KAAK,CAAC,SAAS,KAAK,QAAQ;QACnC,KAAK,CAAC,SAAS,GAAG,CAAC;QACnB,CAAC,KAAK,CAAC,MAAM,KAAK,QAAQ;YACxB,KAAK,CAAC,MAAM,KAAK,KAAK;YACtB,KAAK,CAAC,MAAM,KAAK,WAAW,CAAC,CAChC,CAAC;AACJ,CAAC;AAED,SAAS,kBAAkB,CAAC,KAAc;IACxC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACrB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO,CACL,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,MAAM,KAAK,QAAQ,CAAC;QAChE,CAAC,KAAK,CAAC,QAAQ,KAAK,SAAS,IAAI,OAAO,KAAK,CAAC,QAAQ,KAAK,QAAQ,CAAC;QACpE,CAAC,KAAK,CAAC,MAAM,KAAK,SAAS;YACzB,KAAK,CAAC,MAAM,KAAK,MAAM;YACvB,KAAK,CAAC,MAAM,KAAK,KAAK;YACtB,KAAK,CAAC,MAAM,KAAK,OAAO;YACxB,KAAK,CAAC,MAAM,KAAK,QAAQ;YACzB,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,CAC3B,CAAC;AACJ,CAAC;AAED,SAAS,qBAAqB,CAAC,KAAc;IAC3C,OAAO,CACL,KAAK,KAAK,SAAS;QACnB,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,CAAC,CAAC,CAC1E,CAAC;AACJ,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc;IACvC,OAAO,KAAK,KAAK,SAAS,IAAI,OAAO,KAAK,KAAK,SAAS,CAAC;AAC3D,CAAC;AAED,SAAS,4BAA4B,CAAC,KAAc;IAClD,OAAO,KAAK,KAAK,SAAS,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC;AAChF,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;AACvD,CAAC"}

package/dist/engine/models/backends/model2vec-runtime.d.ts

@@ -0,0 +1,63 @@
+import type { EmbeddingResult } from "../embeddings.js";
+export type TokenTensor = {
+    data: ArrayLike<number | bigint>;
+};
+export type TokenizerOutput = {
+    input_ids: TokenTensor;
+};
+export type TokenizerLike = {
+    (text: string, options: {
+        add_special_tokens: false;
+        truncation: true;
+        max_length: number;
+    }): TokenizerOutput | Promise<TokenizerOutput>;
+    unk_token_id?: number | null;
+};
+export type StaticEmbeddingTable = {
+    data: Float32Array | Uint16Array;
+    dimension: number;
+    dtype: "F16" | "F32";
+    rows: number;
+};
+export type Model2VecWorkerData = {
+    tokenizerSource: string;
+    maxInputTokens: number;
+    normalize: boolean;
+    tableBuffer: SharedArrayBuffer;
+    dimension: number;
+    dtype: "F16" | "F32";
+    rows: number;
+};
+export type Model2VecWorkerRequest = {
+    id: number;
+    texts: string[];
+};
+export type SerializedWorkerError = {
+    name?: string;
+    message: string;
+    stack?: string;
+};
+export type Model2VecWorkerResponse = {
+    type: "ready";
+} | {
+    type: "result";
+    id: number;
+    vectors: ArrayBuffer;
+    vectorCount: number;
+    truncated: number[];
+} | {
+    type: "error";
+    id: number;
+    error: SerializedWorkerError;
+};
+export type TokenizedModel2VecTexts = {
+    tokenLists: number[][];
+    truncated: number[];
+};
+export declare function embedModel2VecTexts(texts: readonly string[], tokenizer: TokenizerLike, table: StaticEmbeddingTable, maxInputTokens: number, normalize: boolean): Promise<EmbeddingResult>;
+export declare function tokenizeModel2VecTexts(texts: readonly string[], tokenizer: TokenizerLike, maxInputTokens: number): Promise<TokenizedModel2VecTexts>;
+export declare function embedModel2VecTokenLists(tokenLists: readonly (readonly number[])[], table: StaticEmbeddingTable, normalize: boolean): number[][];
+export declare function sharedStaticEmbeddingTable(table: StaticEmbeddingTable): StaticEmbeddingTable & {
+    data: Float32Array | Uint16Array;
+};
+export declare function staticEmbeddingTableFromWorkerData(data: Model2VecWorkerData): StaticEmbeddingTable;

package/dist/engine/models/backends/model2vec-runtime.js

@@ -0,0 +1,112 @@
+export async function embedModel2VecTexts(texts, tokenizer, table, maxInputTokens, normalize) {
+    const tokenized = await tokenizeModel2VecTexts(texts, tokenizer, maxInputTokens);
+    return {
+        vectors: embedModel2VecTokenLists(tokenized.tokenLists, table, normalize),
+        truncated: tokenized.truncated,
+    };
+}
+export async function tokenizeModel2VecTexts(texts, tokenizer, maxInputTokens) {
+    const truncatedInputIndexes = [];
+    const tokenLists = await Promise.all(texts.map(async (text, index) => {
+        const encoded = await tokenizer(text, {
+            add_special_tokens: false,
+            truncation: true,
+            max_length: maxInputTokens + 1,
+        });
+        const encodedTokenIds = Array.from(encoded.input_ids.data, Number);
+        if (encodedTokenIds.length > maxInputTokens) {
+            truncatedInputIndexes.push(index);
+        }
+        const unknownTokenId = tokenizer.unk_token_id;
+        return encodedTokenIds
+            .slice(0, maxInputTokens)
+            .filter((id) => unknownTokenId === undefined ||
+            unknownTokenId === null ||
+            id !== unknownTokenId);
+    }));
+    return {
+        tokenLists,
+        truncated: truncatedInputIndexes.sort((left, right) => left - right),
+    };
+}
+export function embedModel2VecTokenLists(tokenLists, table, normalize) {
+    return tokenLists.map((tokenIds) => embedStaticTokenList(tokenIds, table, normalize));
+}
+export function sharedStaticEmbeddingTable(table) {
+    if (table.data.buffer instanceof SharedArrayBuffer) {
+        return table;
+    }
+    const sharedBuffer = new SharedArrayBuffer(table.data.byteLength);
+    new Uint8Array(sharedBuffer).set(new Uint8Array(table.data.buffer, table.data.byteOffset, table.data.byteLength));
+    return {
+        ...table,
+        data: table.dtype === "F16"
+            ? new Uint16Array(sharedBuffer)
+            : new Float32Array(sharedBuffer),
+    };
+}
+export function staticEmbeddingTableFromWorkerData(data) {
+    return {
+        data: data.dtype === "F16"
+            ? new Uint16Array(data.tableBuffer)
+            : new Float32Array(data.tableBuffer),
+        dimension: data.dimension,
+        dtype: data.dtype,
+        rows: data.rows,
+    };
+}
+function embedStaticTokenList(tokenIds, table, normalize) {
+    const dimension = table.dimension;
+    const halfValues = table.dtype === "F16" ? halfFloatValues() : null;
+    const vector = Array.from({ length: dimension }, () => 0);
+    if (tokenIds.length === 0) {
+        return vector;
+    }
+    for (let index = 0; index < tokenIds.length; index++) {
+        const tokenId = tokenIds[index];
+        if (!Number.isInteger(tokenId) || tokenId < 0 || tokenId >= table.rows) {
+            throw new Error(`Tokenizer returned out-of-range token id: id=${tokenId} rows=${table.rows}`);
+        }
+        const start = tokenId * dimension;
+        for (let column = 0; column < dimension; column++) {
+            const value = table.data[start + column];
+            vector[column] += halfValues ? halfValues[value] : value;
+        }
+    }
+    let squaredNorm = 0;
+    for (let column = 0; column < dimension; column++) {
+        vector[column] /= tokenIds.length;
+        squaredNorm += vector[column] * vector[column];
+    }
+    if (normalize && squaredNorm > 0) {
+        const inverseNorm = 1 / Math.sqrt(squaredNorm);
+        for (let column = 0; column < dimension; column++) {
+            vector[column] *= inverseNorm;
+        }
+    }
+    return vector;
+}
+let cachedHalfFloatValues = null;
+function halfFloatValues() {
+    if (cachedHalfFloatValues) {
+        return cachedHalfFloatValues;
+    }
+    const values = new Float32Array(65_536);
+    for (let bits = 0; bits < values.length; bits++) {
+        const sign = bits & 0x8000 ? -1 : 1;
+        const exponent = (bits >> 10) & 0x1f;
+        const fraction = bits & 0x03ff;
+        if (exponent === 0) {
+            values[bits] = sign * 2 ** -14 * (fraction / 1024);
+        }
+        else if (exponent === 0x1f) {
+            values[bits] = fraction === 0 ? sign * Infinity : Number.NaN;
+        }
+        else {
+            values[bits] = sign * 2 ** (exponent - 15) * (1 + fraction / 1024);
+        }
+    }
+    cachedHalfFloatValues = values;
+    return values;
+}
+//# sourceMappingURL=model2vec-runtime.js.map

package/dist/engine/models/backends/model2vec-runtime.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model2vec-runtime.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec-runtime.ts"],"names":[],"mappings":"AAsEA,MAAM,CAAC,KAAK,UAAU,mBAAmB,CACvC,KAAwB,EACxB,SAAwB,EACxB,KAA2B,EAC3B,cAAsB,EACtB,SAAkB;IAElB,MAAM,SAAS,GAAG,MAAM,sBAAsB,CAC5C,KAAK,EACL,SAAS,EACT,cAAc,CACf,CAAC;IAEF,OAAO;QACL,OAAO,EAAE,wBAAwB,CAAC,SAAS,CAAC,UAAU,EAAE,KAAK,EAAE,SAAS,CAAC;QACzE,SAAS,EAAE,SAAS,CAAC,SAAS;KAC/B,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,KAAwB,EACxB,SAAwB,EACxB,cAAsB;IAEtB,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAClC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE;QAC9B,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE;YACpC,kBAAkB,EAAE,KAAK;YACzB,UAAU,EAAE,IAAI;YAChB,UAAU,EAAE,cAAc,GAAG,CAAC;SAC/B,CAAC,CAAC;QACH,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QACnE,IAAI,eAAe,CAAC,MAAM,GAAG,cAAc,EAAE,CAAC;YAC5C,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,CAAC;QACD,MAAM,cAAc,GAAG,SAAS,CAAC,YAAY,CAAC;QAC9C,OAAO,eAAe;aACnB,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC;aACxB,MAAM,CACL,CAAC,EAAE,EAAE,EAAE,CACL,cAAc,KAAK,SAAS;YAC5B,cAAc,KAAK,IAAI;YACvB,EAAE,KAAK,cAAc,CACxB,CAAC;IACN,CAAC,CAAC,CACH,CAAC;IAEF,OAAO;QACL,UAAU;QACV,SAAS,EAAE,qBAAqB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC;KACrE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,wBAAwB,CACtC,UAA0C,EAC1C,KAA2B,EAC3B,SAAkB;IAElB,OAAO,UAAU,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CACjC,oBAAoB,CAAC,QAAQ,EAAE,KAAK,EAAE,SAAS,CAAC,CACjD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAA2B;IAE3B,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,YAAY,iBAAiB,EAAE,CAAC;QACnD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClE,IAAI,UAAU,CAAC,YAAY,CAAC,CAAC,GAAG,CAC9B,IAAI,UAAU,CACZ,KAAK,CAAC,IAAI,CAAC,MAAM,EACjB,KAAK,CAAC,IAAI,CAAC,UAAU,EACrB,KAAK,CAAC,IAAI,CAAC,UAAU,CACtB,CACF,CAAC;IACF,OAAO;QACL,GAAG,KAAK;QACR,IAAI,EACF,KAAK,CAAC,KAAK,KAAK,KAAK;YACnB,CAAC,CAAC,IAAI,WAAW,CAAC,YAAY,CAAC;YAC/B,CAAC,CAAC,IAAI,YAAY,CAAC,YAAY,CAAC;KACrC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,kCAAkC,CAChD,IAAyB;IAEzB,OAAO;QACL,IAAI,EACF,IAAI,CAAC,KAAK,KAAK,KAAK;YAClB,CAAC,CAAC,IAAI,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC;YACnC,CAAC,CAAC,IAAI,YAAY,CAAC,IAAI,CAAC,WAAW,CAAC;QACxC,SAAS,EAAE,IAAI,CAAC,SAAS;QACzB,KAAK,EAAE,IAAI,CAAC,KAAK;QACjB,IAAI,EAAE,IAAI,CAAC,IAAI;KAChB,CAAC;AACJ,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAA2B,EAC3B,KAA2B,EAC3B,SAAkB;IAElB,MAAM,SAAS,GAAG,KAAK,CAAC,SAAS,CAAC;IAClC,MAAM,UAAU,GAAG,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IAC1D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACrD,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,GAAG,CAAC,IAAI,OAAO,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACvE,MAAM,IAAI,KAAK,CACb,gDAAgD,OAAO,SAAS,KAAK,CAAC,IAAI,EAAE,CAC7E,CAAC;QACJ,CAAC;QACD,MAAM,KAAK,GAAG,OAAO,GAAG,SAAS,CAAC;QAClC,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,MAAM,CAAC,CAAC;YACzC,MAAM,CAAC,MAAM,CAAC,IAAI,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC;QAC3D,CAAC;IACH,CAAC;IAED,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC;QAClD,MAAM,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC;QAClC,WAAW,IAAI,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IACjD,CAAC;IACD,IAAI,SAAS,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACjC,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;QAC/C,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,EAAE,MAAM,EAAE,EAAE,CAAC;YAClD,MAAM,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC;QAChC,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,IAAI,qBAAqB,GAAwB,IAAI,CAAC;AAEtD,SAAS,eAAe;IACtB,IAAI,qBAAqB,EAAE,CAAC;QAC1B,OAAO,qBAAqB,CAAC;IAC/B,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;IACxC,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,IAAI,GAAG,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,QAAQ,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC;QACrC,MAAM,QAAQ,GAAG,IAAI,GAAG,MAAM,CAAC;QAC/B,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;YACnB,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,CAAC;QACrD,CAAC;aAAM,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;QAC/D,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,IAAI,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,qBAAqB,GAAG,MAAM,CAAC;IAC/B,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/engine/models/backends/model2vec-tokenizer.d.ts

@@ -0,0 +1,2 @@
+import type { TokenizerLike } from "./model2vec-runtime.js";
+export declare function loadModel2VecTokenizer(source: string): Promise<TokenizerLike>;

package/dist/engine/models/backends/model2vec-tokenizer.js

@@ -0,0 +1,42 @@
+import { Tokenizer } from "@huggingface/tokenizers";
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+export async function loadModel2VecTokenizer(source) {
+    const [tokenizerJson, tokenizerConfig] = await Promise.all([
+        readJson(join(source, "tokenizer.json")),
+        readJson(join(source, "tokenizer_config.json")),
+    ]);
+    const tokenizer = new Tokenizer(tokenizerJson, tokenizerConfig);
+    const unknownTokenId = resolveUnknownTokenId(tokenizer, tokenizerJson);
+    const tokenize = (text, options) => {
+        const ids = tokenizer.encode(text, {
+            add_special_tokens: options.add_special_tokens,
+        }).ids;
+        return {
+            input_ids: {
+                data: ids.length > options.max_length
+                    ? ids.slice(0, options.max_length)
+                    : ids,
+            },
+        };
+    };
+    return Object.assign(tokenize, { unk_token_id: unknownTokenId });
+}
+async function readJson(path) {
+    return JSON.parse(await readFile(path, "utf8"));
+}
+function resolveUnknownTokenId(tokenizer, tokenizerJson) {
+    const configuredId = tokenizerJson.model?.unk_id;
+    if (typeof configuredId === "number" && Number.isInteger(configuredId)) {
+        return configuredId;
+    }
+    const configuredToken = tokenizerJson.model?.unk_token;
+    if (typeof configuredToken !== "string") {
+        return null;
+    }
+    const tokenId = tokenizer.token_to_id(configuredToken);
+    return typeof tokenId === "number" && Number.isInteger(tokenId)
+        ? tokenId
+        : null;
+}
+//# sourceMappingURL=model2vec-tokenizer.js.map

package/dist/engine/models/backends/model2vec-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model2vec-tokenizer.js","sourceRoot":"","sources":["../../../../src/engine/models/backends/model2vec-tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AACpD,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAUjC,MAAM,CAAC,KAAK,UAAU,sBAAsB,CAC1C,MAAc;IAEd,MAAM,CAAC,aAAa,EAAE,eAAe,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC;QACzD,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;QACxC,QAAQ,CAAC,IAAI,CAAC,MAAM,EAAE,uBAAuB,CAAC,CAAC;KAChD,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,aAAa,EAAE,eAAe,CAAC,CAAC;IAChE,MAAM,cAAc,GAAG,qBAAqB,CAAC,SAAS,EAAE,aAAa,CAAC,CAAC;IACvE,MAAM,QAAQ,GAAG,CACf,IAAY,EACZ,OAIC,EACD,EAAE;QACF,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,EAAE;YACjC,kBAAkB,EAAE,OAAO,CAAC,kBAAkB;SAC/C,CAAC,CAAC,GAAG,CAAC;QACP,OAAO;YACL,SAAS,EAAE;gBACT,IAAI,EACF,GAAG,CAAC,MAAM,GAAG,OAAO,CAAC,UAAU;oBAC7B,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,OAAO,CAAC,UAAU,CAAC;oBAClC,CAAC,CAAC,GAAG;aACV;SACF,CAAC;IACJ,CAAC,CAAC;IAEF,OAAO,MAAM,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,YAAY,EAAE,cAAc,EAAE,CAAC,CAAC;AACnE,CAAC;AAED,KAAK,UAAU,QAAQ,CAAC,IAAY;IAClC,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAA4B,CAAC;AAC7E,CAAC;AAED,SAAS,qBAAqB,CAC5B,SAAoB,EACpB,aAA4B;IAE5B,MAAM,YAAY,GAAG,aAAa,CAAC,KAAK,EAAE,MAAM,CAAC;IACjD,IAAI,OAAO,YAAY,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC;QACvE,OAAO,YAAY,CAAC;IACtB,CAAC;IAED,MAAM,eAAe,GAAG,aAAa,CAAC,KAAK,EAAE,SAAS,CAAC;IACvD,IAAI,OAAO,eAAe,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,IAAI,CAAC;IACd,CAAC;IACD,MAAM,OAAO,GAAY,SAAS,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC;IAChE,OAAO,OAAO,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC;QAC7D,CAAC,CAAC,OAAO;QACT,CAAC,CAAC,IAAI,CAAC;AACX,CAAC"}

package/dist/engine/models/backends/model2vec-worker-pool.d.ts

@@ -0,0 +1,25 @@
+import type { EmbeddingResult } from "../embeddings.js";
+import type { Model2VecWorkerData } from "./model2vec-runtime.js";
+export declare class Model2VecWorkerPool {
+    private readonly data;
+    private readonly workerUrl;
+    private disposed;
+    private nextJobId;
+    private readonly maxWorkers;
+    private readonly queue;
+    private readonly slots;
+    constructor(data: Model2VecWorkerData, maxWorkers?: number, workerUrl?: URL);
+    start(): Promise<void>;
+    run(texts: readonly string[], signal?: AbortSignal): Promise<EmbeddingResult>;
+    dispose(): Promise<void>;
+    private spawnWorker;
+    private handleMessage;
+    private dispatch;
+    private postJob;
+    private drain;
+    private nextQueuedJob;
+    private failSlot;
+    private resolveJob;
+    private rejectJob;
+    private ensureNotDisposed;
+}