@knowledgine/core 0.0.1 → 0.2.0

package/dist/config/config-loader.d.ts

@@ -0,0 +1,15 @@
+import type { KnowledgineConfig } from "../config.js";
+export interface RcConfig {
+    semantic?: boolean;
+    [key: string]: unknown;
+}
+/**
+ * Load knowledgine configuration from RC file and environment variables.
+ * Priority: env var > RC file > defaults (embedding.enabled = false)
+ */
+export declare function loadConfig(rootPath: string): KnowledgineConfig;
+/**
+ * Write a .knowledginerc.json config file to the project root.
+ */
+export declare function writeRcConfig(rootPath: string, config: RcConfig): void;
+//# sourceMappingURL=config-loader.d.ts.map

package/dist/config/config-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config-loader.d.ts","sourceRoot":"","sources":["../../src/config/config-loader.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,cAAc,CAAC;AAEtD,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,EAAE,OAAO,CAAC;IACnB,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC;CACxB;AAED;;;GAGG;AACH,wBAAgB,UAAU,CAAC,QAAQ,EAAE,MAAM,GAAG,iBAAiB,CAgB9D;AAqBD;;GAEG;AACH,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,GAAG,IAAI,CAGtE"}

package/dist/config/config-loader.js

@@ -0,0 +1,46 @@
+import { readFileSync, existsSync, writeFileSync } from "fs";
+import { resolve } from "path";
+import { parse as parseYaml } from "yaml";
+import { defineConfig } from "../config.js";
+/**
+ * Load knowledgine configuration from RC file and environment variables.
+ * Priority: env var > RC file > defaults (embedding.enabled = false)
+ */
+export function loadConfig(rootPath) {
+    const rcConfig = loadRcFile(rootPath);
+    // Environment variable override
+    const envSemantic = process.env["KNOWLEDGINE_SEMANTIC"];
+    const semanticEnabled = envSemantic === "true" || envSemantic === "1" || rcConfig?.semantic === true;
+    return defineConfig({
+        rootPath,
+        embedding: {
+            modelName: "all-MiniLM-L6-v2",
+            dimensions: 384,
+            enabled: semanticEnabled,
+        },
+    });
+}
+function loadRcFile(rootPath) {
+    const jsonPath = resolve(rootPath, ".knowledginerc.json");
+    const ymlPath = resolve(rootPath, ".knowledginerc.yml");
+    try {
+        if (existsSync(jsonPath)) {
+            return JSON.parse(readFileSync(jsonPath, "utf-8"));
+        }
+        if (existsSync(ymlPath)) {
+            return parseYaml(readFileSync(ymlPath, "utf-8"));
+        }
+    }
+    catch (error) {
+        console.error(`Warning: Failed to parse config file: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    return null;
+}
+/**
+ * Write a .knowledginerc.json config file to the project root.
+ */
+export function writeRcConfig(rootPath, config) {
+    const rcPath = resolve(rootPath, ".knowledginerc.json");
+    writeFileSync(rcPath, JSON.stringify(config, null, 2) + "\n");
+}
+//# sourceMappingURL=config-loader.js.map

package/dist/config/config-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"config-loader.js","sourceRoot":"","sources":["../../src/config/config-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,IAAI,CAAC;AAC7D,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAC/B,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAQ5C;;;GAGG;AACH,MAAM,UAAU,UAAU,CAAC,QAAgB;IACzC,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC;IAEtC,gCAAgC;IAChC,MAAM,WAAW,GAAG,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACxD,MAAM,eAAe,GACnB,WAAW,KAAK,MAAM,IAAI,WAAW,KAAK,GAAG,IAAI,QAAQ,EAAE,QAAQ,KAAK,IAAI,CAAC;IAE/E,OAAO,YAAY,CAAC;QAClB,QAAQ;QACR,SAAS,EAAE;YACT,SAAS,EAAE,kBAAkB;YAC7B,UAAU,EAAE,GAAG;YACf,OAAO,EAAE,eAAe;SACzB;KACF,CAAC,CAAC;AACL,CAAC;AAED,SAAS,UAAU,CAAC,QAAgB;IAClC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,qBAAqB,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAG,OAAO,CAAC,QAAQ,EAAE,oBAAoB,CAAC,CAAC;IAExD,IAAI,CAAC;QACH,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAa,CAAC;QACjE,CAAC;QACD,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACxB,OAAO,SAAS,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAa,CAAC;QAC/D,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CACX,yCAAyC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAClG,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,aAAa,CAAC,QAAgB,EAAE,MAAgB;IAC9D,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,EAAE,qBAAqB,CAAC,CAAC;IACxD,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;AAChE,CAAC"}

package/dist/config.d.ts

@@ -1,4 +1,18 @@
 export type PatternCategory = "problem" | "solution" | "learning" | "time";
+export interface EmbeddingConfig {
+    /** 使用するモデル名 */
+    modelName: string;
+    /** 埋め込み次元数 */
+    dimensions: number;
+    /** 埋め込み生成を有効にするか */
+    enabled: boolean;
+}
+export interface SearchConfig {
+    /** デフォルト検索モード */
+    defaultMode: "keyword" | "semantic" | "hybrid";
+    /** ハイブリッド検索のFTSスコア重み (0-1) */
+    hybridAlpha: number;
+}
 export interface KnowledgineConfig {
     rootPath: string;
     dbPath: string;
@@ -8,6 +22,8 @@ export interface KnowledgineConfig {
     frontmatter: {
         requiredFields: string[];
     };
+    embedding: EmbeddingConfig;
+    search: SearchConfig;
 }
 export declare function defineConfig(partial?: Partial<KnowledgineConfig>): KnowledgineConfig;
 //# sourceMappingURL=config.d.ts.map

package/dist/config.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;AAE3E,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE;QACR,OAAO,EAAE,eAAe,EAAE,CAAC;KAC5B,CAAC;IACF,WAAW,EAAE;QACX,cAAc,EAAE,MAAM,EAAE,CAAC;KAC1B,CAAC;CACH;AAaD,wBAAgB,YAAY,CAAC,OAAO,GAAE,OAAO,CAAC,iBAAiB,CAAM,GAAG,iBAAiB,CAkBxF"}
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAEA,MAAM,MAAM,eAAe,GAAG,SAAS,GAAG,UAAU,GAAG,UAAU,GAAG,MAAM,CAAC;AAE3E,MAAM,WAAW,eAAe;IAC9B,eAAe;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,oBAAoB;IACpB,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,MAAM,WAAW,YAAY;IAC3B,iBAAiB;IACjB,WAAW,EAAE,SAAS,GAAG,UAAU,GAAG,QAAQ,CAAC;IAC/C,8BAA8B;IAC9B,WAAW,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE;QACR,OAAO,EAAE,eAAe,EAAE,CAAC;KAC5B,CAAC;IACF,WAAW,EAAE;QACX,cAAc,EAAE,MAAM,EAAE,CAAC;KAC1B,CAAC;IACF,SAAS,EAAE,eAAe,CAAC;IAC3B,MAAM,EAAE,YAAY,CAAC;CACtB;AAsBD,wBAAgB,YAAY,CAAC,OAAO,GAAE,OAAO,CAAC,iBAAiB,CAAM,GAAG,iBAAiB,CA0BxF"}

package/dist/config.js

@@ -8,6 +8,15 @@ const DEFAULT_CONFIG = {
     frontmatter: {
         requiredFields: [],
     },
+    embedding: {
+        modelName: "all-MiniLM-L6-v2",
+        dimensions: 384,
+        enabled: false,
+    },
+    search: {
+        defaultMode: "keyword",
+        hybridAlpha: 0.3,
+    },
 };
 export function defineConfig(partial = {}) {
     const rootPath = partial.rootPath ?? DEFAULT_CONFIG.rootPath;
@@ -25,6 +34,14 @@ export function defineConfig(partial = {}) {
             ...DEFAULT_CONFIG.frontmatter,
             ...partial.frontmatter,
         },
+        embedding: {
+            ...DEFAULT_CONFIG.embedding,
+            ...partial.embedding,
+        },
+        search: {
+            ...DEFAULT_CONFIG.search,
+            ...partial.search,
+        },
     };
 }
 //# sourceMappingURL=config.js.map

package/dist/config.js.map

@@ -1 +1 @@
-{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAe/B,MAAM,cAAc,GAAsB;IACxC,QAAQ,EAAE,GAAG;IACb,MAAM,EAAE,EAAE;IACV,QAAQ,EAAE;QACR,OAAO,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC;KACrD;IACD,WAAW,EAAE;QACX,cAAc,EAAE,EAAE;KACnB;CACF,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,UAAsC,EAAE;IACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,cAAc,CAAC,QAAQ,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,cAAc,CAAC,CAAC;IAEnF,OAAO;QACL,GAAG,cAAc;QACjB,GAAG,OAAO;QACV,QAAQ;QACR,MAAM;QACN,QAAQ,EAAE;YACR,GAAG,cAAc,CAAC,QAAQ;YAC1B,GAAG,OAAO,CAAC,QAAQ;SACpB;QACD,WAAW,EAAE;YACX,GAAG,cAAc,CAAC,WAAW;YAC7B,GAAG,OAAO,CAAC,WAAW;SACvB;KACF,CAAC;AACJ,CAAC"}
+{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AAiC/B,MAAM,cAAc,GAAsB;IACxC,QAAQ,EAAE,GAAG;IACb,MAAM,EAAE,EAAE;IACV,QAAQ,EAAE;QACR,OAAO,EAAE,CAAC,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,MAAM,CAAC;KACrD;IACD,WAAW,EAAE;QACX,cAAc,EAAE,EAAE;KACnB;IACD,SAAS,EAAE;QACT,SAAS,EAAE,kBAAkB;QAC7B,UAAU,EAAE,GAAG;QACf,OAAO,EAAE,KAAK;KACf;IACD,MAAM,EAAE;QACN,WAAW,EAAE,SAAS;QACtB,WAAW,EAAE,GAAG;KACjB;CACF,CAAC;AAEF,MAAM,UAAU,YAAY,CAAC,UAAsC,EAAE;IACnE,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,cAAc,CAAC,QAAQ,CAAC;IAC7D,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,OAAO,CAAC,QAAQ,EAAE,cAAc,EAAE,cAAc,CAAC,CAAC;IAEnF,OAAO;QACL,GAAG,cAAc;QACjB,GAAG,OAAO;QACV,QAAQ;QACR,MAAM;QACN,QAAQ,EAAE;YACR,GAAG,cAAc,CAAC,QAAQ;YAC1B,GAAG,OAAO,CAAC,QAAQ;SACpB;QACD,WAAW,EAAE;YACX,GAAG,cAAc,CAAC,WAAW;YAC7B,GAAG,OAAO,CAAC,WAAW;SACvB;QACD,SAAS,EAAE;YACT,GAAG,cAAc,CAAC,SAAS;YAC3B,GAAG,OAAO,CAAC,SAAS;SACrB;QACD,MAAM,EAAE;YACN,GAAG,cAAc,CAAC,MAAM;YACxB,GAAG,OAAO,CAAC,MAAM;SAClB;KACF,CAAC;AACJ,CAAC"}

package/dist/embedding/embedding-provider.d.ts

@@ -0,0 +1,9 @@
+export interface EmbeddingProvider {
+    /** テキストを埋め込みベクトルに変換する */
+    embed(text: string): Promise<Float32Array>;
+    /** 複数テキストをバッチ処理する */
+    embedBatch(texts: string[]): Promise<Float32Array[]>;
+    /** 埋め込みの次元数を返す */
+    getDimensions(): number;
+}
+//# sourceMappingURL=embedding-provider.d.ts.map

package/dist/embedding/embedding-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-provider.d.ts","sourceRoot":"","sources":["../../src/embedding/embedding-provider.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,iBAAiB;IAChC,yBAAyB;IACzB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;IAC3C,qBAAqB;IACrB,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IACrD,kBAAkB;IAClB,aAAa,IAAI,MAAM,CAAC;CACzB"}

package/dist/embedding/embedding-provider.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=embedding-provider.js.map

package/dist/embedding/embedding-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"embedding-provider.js","sourceRoot":"","sources":["../../src/embedding/embedding-provider.ts"],"names":[],"mappings":""}

package/dist/embedding/model-downloader.d.ts

@@ -0,0 +1,34 @@
+/**
+ * Embedding model downloader.
+ * Downloads all-MiniLM-L6-v2 ONNX model files from HuggingFace.
+ *
+ * Features:
+ * - Atomic download (.tmp + rename)
+ * - Redirect limit (max 5)
+ * - Timeout (5 minutes)
+ * - Progress callback
+ * - Skip existing files (0-byte treated as corrupt)
+ * - SIGINT cleanup
+ */
+import type { ModelManager } from "./model-manager.js";
+export interface DownloadProgress {
+    file: string;
+    downloaded: number;
+    total: number | null;
+}
+export interface DownloadOptions {
+    onProgress?: (progress: DownloadProgress) => void;
+    onFileComplete?: (file: string) => void;
+    timeoutMs?: number;
+    maxRedirects?: number;
+}
+export interface ModelFile {
+    url: string;
+    dest: string;
+}
+export declare const MODEL_FILES: ModelFile[];
+export declare function downloadModel(modelManager: ModelManager, options?: DownloadOptions): Promise<{
+    downloaded: string[];
+    skipped: string[];
+}>;
+//# sourceMappingURL=model-downloader.d.ts.map

package/dist/embedding/model-downloader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-downloader.d.ts","sourceRoot":"","sources":["../../src/embedding/model-downloader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAMH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAEvD,MAAM,WAAW,gBAAgB;IAC/B,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,gBAAgB,KAAK,IAAI,CAAC;IAClD,cAAc,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,CAAC;IACxC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,EAAE,MAAM,CAAC;IACZ,IAAI,EAAE,MAAM,CAAC;CACd;AAED,eAAO,MAAM,WAAW,EAAE,SAAS,EAalC,CAAC;AAyHF,wBAAsB,aAAa,CACjC,YAAY,EAAE,YAAY,EAC1B,OAAO,GAAE,eAAoB,GAC5B,OAAO,CAAC;IAAE,UAAU,EAAE,MAAM,EAAE,CAAC;IAAC,OAAO,EAAE,MAAM,EAAE,CAAA;CAAE,CAAC,CAqBtD"}

package/dist/embedding/model-downloader.js

@@ -0,0 +1,149 @@
+/**
+ * Embedding model downloader.
+ * Downloads all-MiniLM-L6-v2 ONNX model files from HuggingFace.
+ *
+ * Features:
+ * - Atomic download (.tmp + rename)
+ * - Redirect limit (max 5)
+ * - Timeout (5 minutes)
+ * - Progress callback
+ * - Skip existing files (0-byte treated as corrupt)
+ * - SIGINT cleanup
+ */
+import { mkdirSync, renameSync, unlinkSync, existsSync, statSync, createWriteStream } from "fs";
+import { get as httpsGet } from "https";
+import { get as httpGet } from "http";
+import { pipeline } from "stream/promises";
+export const MODEL_FILES = [
+    {
+        url: "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/tokenizer.json",
+        dest: "tokenizer.json",
+    },
+    {
+        url: "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/config.json",
+        dest: "config.json",
+    },
+    {
+        url: "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2/resolve/main/onnx/model_quantized.onnx",
+        dest: "model.onnx",
+    },
+];
+function isExistingAndValid(filePath) {
+    if (!existsSync(filePath))
+        return false;
+    const stat = statSync(filePath);
+    return stat.size > 0;
+}
+function downloadFile(url, destPath, options, fileName, redirectCount = 0) {
+    const maxRedirects = options.maxRedirects ?? 5;
+    const timeoutMs = options.timeoutMs ?? 300_000; // 5 minutes
+    if (redirectCount > maxRedirects) {
+        return Promise.reject(new Error(`Too many redirects (${maxRedirects}) for ${url}`));
+    }
+    return new Promise((resolve, reject) => {
+        const tmpPath = destPath + ".tmp";
+        let aborted = false;
+        const cleanup = () => {
+            try {
+                if (existsSync(tmpPath))
+                    unlinkSync(tmpPath);
+            }
+            catch {
+                // ignore cleanup errors
+            }
+        };
+        const onSigint = () => {
+            aborted = true;
+            cleanup();
+        };
+        process.on("SIGINT", onSigint);
+        const getFunc = url.startsWith("http://") ? httpGet : httpsGet;
+        const timer = setTimeout(() => {
+            aborted = true;
+            req.destroy();
+            cleanup();
+            reject(new Error(`Download timeout (${timeoutMs}ms) for ${fileName}`));
+        }, timeoutMs);
+        const req = getFunc(url, (response) => {
+            if (aborted)
+                return;
+            // Handle redirects (301, 302, 307, 308)
+            if (response.statusCode === 301 || response.statusCode === 302 || response.statusCode === 307 || response.statusCode === 308) {
+                clearTimeout(timer);
+                process.removeListener("SIGINT", onSigint);
+                const location = response.headers.location;
+                if (!location) {
+                    reject(new Error(`Redirect without location header for ${url}`));
+                    return;
+                }
+                downloadFile(location, destPath, options, fileName, redirectCount + 1)
+                    .then(resolve)
+                    .catch(reject);
+                return;
+            }
+            if (response.statusCode !== 200) {
+                clearTimeout(timer);
+                process.removeListener("SIGINT", onSigint);
+                reject(new Error(`HTTP ${response.statusCode} for ${fileName}`));
+                return;
+            }
+            const totalSize = response.headers["content-length"]
+                ? parseInt(response.headers["content-length"], 10)
+                : null;
+            let downloaded = 0;
+            response.on("data", (chunk) => {
+                downloaded += chunk.length;
+                options.onProgress?.({
+                    file: fileName,
+                    downloaded,
+                    total: totalSize,
+                });
+            });
+            const fileStream = createWriteStream(tmpPath);
+            pipeline(response, fileStream)
+                .then(() => {
+                clearTimeout(timer);
+                process.removeListener("SIGINT", onSigint);
+                if (aborted) {
+                    cleanup();
+                    reject(new Error("Download aborted"));
+                    return;
+                }
+                // Atomic rename
+                renameSync(tmpPath, destPath);
+                options.onFileComplete?.(fileName);
+                resolve();
+            })
+                .catch((err) => {
+                clearTimeout(timer);
+                process.removeListener("SIGINT", onSigint);
+                cleanup();
+                reject(err);
+            });
+        });
+        req.on("error", (err) => {
+            clearTimeout(timer);
+            process.removeListener("SIGINT", onSigint);
+            cleanup();
+            reject(err);
+        });
+    });
+}
+export async function downloadModel(modelManager, options = {}) {
+    const modelDir = modelManager.getModelDir();
+    mkdirSync(modelDir, { recursive: true });
+    const downloaded = [];
+    const skipped = [];
+    for (const file of MODEL_FILES) {
+        const destPath = `${modelDir}/${file.dest}`;
+        if (isExistingAndValid(destPath)) {
+            skipped.push(file.dest);
+            options.onFileComplete?.(file.dest);
+            continue;
+        }
+        await downloadFile(file.url, destPath, options, file.dest);
+        downloaded.push(file.dest);
+    }
+    return { downloaded, skipped };
+}
+//# sourceMappingURL=model-downloader.js.map

package/dist/embedding/model-downloader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-downloader.js","sourceRoot":"","sources":["../../src/embedding/model-downloader.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,SAAS,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,QAAQ,EAAE,iBAAiB,EAAE,MAAM,IAAI,CAAC;AAChG,OAAO,EAAE,GAAG,IAAI,QAAQ,EAAE,MAAM,OAAO,CAAC;AACxC,OAAO,EAAE,GAAG,IAAI,OAAO,EAAE,MAAM,MAAM,CAAC;AACtC,OAAO,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAqB3C,MAAM,CAAC,MAAM,WAAW,GAAgB;IACtC;QACE,GAAG,EAAE,2FAA2F;QAChG,IAAI,EAAE,gBAAgB;KACvB;IACD;QACE,GAAG,EAAE,wFAAwF;QAC7F,IAAI,EAAE,aAAa;KACpB;IACD;QACE,GAAG,EAAE,sGAAsG;QAC3G,IAAI,EAAE,YAAY;KACnB;CACF,CAAC;AAEF,SAAS,kBAAkB,CAAC,QAAgB;IAC1C,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,KAAK,CAAC;IACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAChC,OAAO,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;AACvB,CAAC;AAED,SAAS,YAAY,CACnB,GAAW,EACX,QAAgB,EAChB,OAAwB,EACxB,QAAgB,EAChB,gBAAwB,CAAC;IAEzB,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC;IAC/C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAC,CAAC,YAAY;IAE5D,IAAI,aAAa,GAAG,YAAY,EAAE,CAAC;QACjC,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,uBAAuB,YAAY,SAAS,GAAG,EAAE,CAAC,CAAC,CAAC;IACtF,CAAC;IAED,OAAO,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QAC3C,MAAM,OAAO,GAAG,QAAQ,GAAG,MAAM,CAAC;QAClC,IAAI,OAAO,GAAG,KAAK,CAAC;QAEpB,MAAM,OAAO,GAAG,GAAS,EAAE;YACzB,IAAI,CAAC;gBACH,IAAI,UAAU,CAAC,OAAO,CAAC;oBAAE,UAAU,CAAC,OAAO,CAAC,CAAC;YAC/C,CAAC;YAAC,MAAM,CAAC;gBACP,wBAAwB;YAC1B,CAAC;QACH,CAAC,CAAC;QAEF,MAAM,QAAQ,GAAG,GAAS,EAAE;YAC1B,OAAO,GAAG,IAAI,CAAC;YACf,OAAO,EAAE,CAAC;QACZ,CAAC,CAAC;QACF,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QAE/B,MAAM,OAAO,GAAG,GAAG,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;QAE/D,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YAC5B,OAAO,GAAG,IAAI,CAAC;YACf,GAAG,CAAC,OAAO,EAAE,CAAC;YACd,OAAO,EAAE,CAAC;YACV,MAAM,CAAC,IAAI,KAAK,CAAC,qBAAqB,SAAS,WAAW,QAAQ,EAAE,CAAC,CAAC,CAAC;QACzE,CAAC,EAAE,SAAS,CAAC,CAAC;QAEd,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC,QAAQ,EAAE,EAAE;YACpC,IAAI,OAAO;gBAAE,OAAO;YAEpB,wCAAwC;YACxC,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAC7H,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC3C,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC;gBAC3C,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,CAAC,IAAI,KAAK,CAAC,wCAAwC,GAAG,EAAE,CAAC,CAAC,CAAC;oBACjE,OAAO;gBACT,CAAC;gBACD,YAAY,CAAC,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,QAAQ,EAAE,aAAa,GAAG,CAAC,CAAC;qBACnE,IAAI,CAAC,OAAO,CAAC;qBACb,KAAK,CAAC,MAAM,CAAC,CAAC;gBACjB,OAAO;YACT,CAAC;YAED,IAAI,QAAQ,CAAC,UAAU,KAAK,GAAG,EAAE,CAAC;gBAChC,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC3C,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,QAAQ,CAAC,UAAU,QAAQ,QAAQ,EAAE,CAAC,CAAC,CAAC;gBACjE,OAAO;YACT,CAAC;YAED,MAAM,SAAS,GAAG,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC;gBAClD,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,gBAAgB,CAAC,EAAE,EAAE,CAAC;gBAClD,CAAC,CAAC,IAAI,CAAC;YAET,IAAI,UAAU,GAAG,CAAC,CAAC;YACnB,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBACpC,UAAU,IAAI,KAAK,CAAC,MAAM,CAAC;gBAC3B,OAAO,CAAC,UAAU,EAAE,CAAC;oBACnB,IAAI,EAAE,QAAQ;oBACd,UAAU;oBACV,KAAK,EAAE,SAAS;iBACjB,CAAC,CAAC;YACL,CAAC,CAAC,CAAC;YAEH,MAAM,UAAU,GAAG,iBAAiB,CAAC,OAAO,CAAC,CAAC;YAE9C,QAAQ,CAAC,QAAQ,EAAE,UAAU,CAAC;iBAC3B,IAAI,CAAC,GAAG,EAAE;gBACT,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC3C,IAAI,OAAO,EAAE,CAAC;oBACZ,OAAO,EAAE,CAAC;oBACV,MAAM,CAAC,IAAI,KAAK,CAAC,kBAAkB,CAAC,CAAC,CAAC;oBACtC,OAAO;gBACT,CAAC;gBACD,gBAAgB;gBAChB,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;gBAC9B,OAAO,CAAC,cAAc,EAAE,CAAC,QAAQ,CAAC,CAAC;gBACnC,OAAO,EAAE,CAAC;YACZ,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;gBACb,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;gBAC3C,OAAO,EAAE,CAAC;gBACV,MAAM,CAAC,GAAG,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;QACP,CAAC,CAAC,CAAC;QAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,GAAG,EAAE,EAAE;YACtB,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,cAAc,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;YAC3C,OAAO,EAAE,CAAC;YACV,MAAM,CAAC,GAAG,CAAC,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,YAA0B,EAC1B,UAA2B,EAAE;IAE7B,MAAM,QAAQ,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC;IAC5C,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAEzC,MAAM,UAAU,GAAa,EAAE,CAAC;IAChC,MAAM,OAAO,GAAa,EAAE,CAAC;IAE7B,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAG,GAAG,QAAQ,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAE5C,IAAI,kBAAkB,CAAC,QAAQ,CAAC,EAAE,CAAC;YACjC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACxB,OAAO,CAAC,cAAc,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpC,SAAS;QACX,CAAC;QAED,MAAM,YAAY,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;QAC3D,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC;AACjC,CAAC"}

package/dist/embedding/model-manager.d.ts

@@ -0,0 +1,10 @@
+export declare const DEFAULT_MODEL_NAME = "all-MiniLM-L6-v2";
+export declare class ModelManager {
+    private modelsDir;
+    constructor(modelsDir?: string);
+    getModelDir(modelName?: string): string;
+    getModelPath(modelName?: string): string;
+    getTokenizerPath(modelName?: string): string;
+    isModelAvailable(modelName?: string): boolean;
+}
+//# sourceMappingURL=model-manager.d.ts.map

package/dist/embedding/model-manager.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-manager.d.ts","sourceRoot":"","sources":["../../src/embedding/model-manager.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,kBAAkB,qBAAqB,CAAC;AAErD,qBAAa,YAAY;IACvB,OAAO,CAAC,SAAS,CAAS;gBAEd,SAAS,CAAC,EAAE,MAAM;IAY9B,WAAW,CAAC,SAAS,GAAE,MAA2B,GAAG,MAAM;IAI3D,YAAY,CAAC,SAAS,GAAE,MAA2B,GAAG,MAAM;IAI5D,gBAAgB,CAAC,SAAS,GAAE,MAA2B,GAAG,MAAM;IAIhE,gBAAgB,CAAC,SAAS,GAAE,MAA2B,GAAG,OAAO;CAGlE"}

package/dist/embedding/model-manager.js

@@ -0,0 +1,32 @@
+import { existsSync } from "fs";
+import { join, dirname } from "path";
+import { fileURLToPath } from "url";
+export const DEFAULT_MODEL_NAME = "all-MiniLM-L6-v2";
+export class ModelManager {
+    modelsDir;
+    constructor(modelsDir) {
+        if (modelsDir) {
+            this.modelsDir = modelsDir;
+        }
+        else {
+            // Resolve models/ relative to the package root (one level above src/embedding/)
+            const currentDir = dirname(fileURLToPath(import.meta.url));
+            // In dist: dist/embedding/ -> package root
+            // In src: src/embedding/ -> package root
+            this.modelsDir = join(currentDir, "..", "..", "models");
+        }
+    }
+    getModelDir(modelName = DEFAULT_MODEL_NAME) {
+        return join(this.modelsDir, modelName);
+    }
+    getModelPath(modelName = DEFAULT_MODEL_NAME) {
+        return join(this.getModelDir(modelName), "model.onnx");
+    }
+    getTokenizerPath(modelName = DEFAULT_MODEL_NAME) {
+        return join(this.getModelDir(modelName), "tokenizer.json");
+    }
+    isModelAvailable(modelName = DEFAULT_MODEL_NAME) {
+        return existsSync(this.getModelPath(modelName)) && existsSync(this.getTokenizerPath(modelName));
+    }
+}
+//# sourceMappingURL=model-manager.js.map

package/dist/embedding/model-manager.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"model-manager.js","sourceRoot":"","sources":["../../src/embedding/model-manager.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,MAAM,CAAC;AACrC,OAAO,EAAE,aAAa,EAAE,MAAM,KAAK,CAAC;AAEpC,MAAM,CAAC,MAAM,kBAAkB,GAAG,kBAAkB,CAAC;AAErD,MAAM,OAAO,YAAY;IACf,SAAS,CAAS;IAE1B,YAAY,SAAkB;QAC5B,IAAI,SAAS,EAAE,CAAC;YACd,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC7B,CAAC;aAAM,CAAC;YACN,gFAAgF;YAChF,MAAM,UAAU,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAC3D,2CAA2C;YAC3C,yCAAyC;YACzC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,UAAU,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;QAC1D,CAAC;IACH,CAAC;IAED,WAAW,CAAC,YAAoB,kBAAkB;QAChD,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;IACzC,CAAC;IAED,YAAY,CAAC,YAAoB,kBAAkB;QACjD,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,YAAY,CAAC,CAAC;IACzD,CAAC;IAED,gBAAgB,CAAC,YAAoB,kBAAkB;QACrD,OAAO,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,EAAE,gBAAgB,CAAC,CAAC;IAC7D,CAAC;IAED,gBAAgB,CAAC,YAAoB,kBAAkB;QACrD,OAAO,UAAU,CAAC,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC,CAAC;IAClG,CAAC;CACF"}

package/dist/embedding/onnx-embedding-provider.d.ts

@@ -0,0 +1,18 @@
+import type { EmbeddingProvider } from "./embedding-provider.js";
+import { ModelManager } from "./model-manager.js";
+export declare class OnnxEmbeddingProvider implements EmbeddingProvider {
+    private session;
+    private tokenizer;
+    private modelName;
+    private modelManager;
+    constructor(modelName?: string, modelManager?: ModelManager);
+    private getSession;
+    private getTokenizer;
+    embed(text: string): Promise<Float32Array>;
+    embedBatch(texts: string[]): Promise<Float32Array[]>;
+    getDimensions(): number;
+    close(): Promise<void>;
+    private meanPool;
+    private normalize;
+}
+//# sourceMappingURL=onnx-embedding-provider.d.ts.map

package/dist/embedding/onnx-embedding-provider.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"onnx-embedding-provider.d.ts","sourceRoot":"","sources":["../../src/embedding/onnx-embedding-provider.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAEjE,OAAO,EAAE,YAAY,EAAsB,MAAM,oBAAoB,CAAC;AAKtE,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,OAAO,CAAC,OAAO,CAAiC;IAChD,OAAO,CAAC,SAAS,CAAmC;IACpD,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,YAAY,CAAe;gBAEvB,SAAS,GAAE,MAA2B,EAAE,YAAY,CAAC,EAAE,YAAY;YAKjE,UAAU;IAwBxB,OAAO,CAAC,YAAY;IAWd,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAK1C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IAsD1D,aAAa,IAAI,MAAM;IAIjB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAO5B,OAAO,CAAC,QAAQ;IA0BhB,OAAO,CAAC,SAAS;CAalB"}

package/dist/embedding/onnx-embedding-provider.js

@@ -0,0 +1,121 @@
+import { WordPieceTokenizer } from "./tokenizer.js";
+import { ModelManager, DEFAULT_MODEL_NAME } from "./model-manager.js";
+import { EmbeddingNotAvailableError, EmbeddingError } from "../errors.js";
+const DIMENSIONS = 384;
+export class OnnxEmbeddingProvider {
+    session = null;
+    tokenizer = null;
+    modelName;
+    modelManager;
+    constructor(modelName = DEFAULT_MODEL_NAME, modelManager) {
+        this.modelName = modelName;
+        this.modelManager = modelManager ?? new ModelManager();
+    }
+    async getSession() {
+        if (this.session)
+            return this.session;
+        if (!this.modelManager.isModelAvailable(this.modelName)) {
+            throw new EmbeddingNotAvailableError(`Model "${this.modelName}" not found. Run 'knowledgine init' to download the model automatically.`);
+        }
+        try {
+            const ort = await import("onnxruntime-node");
+            this.session = await ort.InferenceSession.create(this.modelManager.getModelPath(this.modelName), { executionProviders: ["cpu"] });
+            return this.session;
+        }
+        catch (error) {
+            throw new EmbeddingError(`Failed to load ONNX model "${this.modelName}"`, error instanceof Error ? error : new Error(String(error)));
+        }
+    }
+    getTokenizer() {
+        if (this.tokenizer)
+            return this.tokenizer;
+        if (!this.modelManager.isModelAvailable(this.modelName)) {
+            throw new EmbeddingNotAvailableError(`Model "${this.modelName}" not found. Run 'knowledgine init' to download the model automatically.`);
+        }
+        this.tokenizer = new WordPieceTokenizer(this.modelManager.getTokenizerPath(this.modelName));
+        return this.tokenizer;
+    }
+    async embed(text) {
+        const [result] = await this.embedBatch([text]);
+        return result;
+    }
+    async embedBatch(texts) {
+        const session = await this.getSession();
+        const tokenizer = this.getTokenizer();
+        try {
+            const ort = await import("onnxruntime-node");
+            const results = [];
+            for (const text of texts) {
+                const encoded = tokenizer.encode(text);
+                const seqLen = encoded.inputIds.length;
+                const inputIds = new ort.Tensor("int64", BigInt64Array.from(encoded.inputIds.map(BigInt)), [
+                    1,
+                    seqLen,
+                ]);
+                const attentionMask = new ort.Tensor("int64", BigInt64Array.from(encoded.attentionMask.map(BigInt)), [1, seqLen]);
+                const tokenTypeIds = new ort.Tensor("int64", BigInt64Array.from(encoded.tokenTypeIds.map(BigInt)), [1, seqLen]);
+                const feeds = {
+                    input_ids: inputIds,
+                    attention_mask: attentionMask,
+                    token_type_ids: tokenTypeIds,
+                };
+                const output = await session.run(feeds);
+                // Mean pooling over token embeddings (last hidden state)
+                const outputKey = Object.keys(output)[0];
+                const lastHiddenState = output["last_hidden_state"] ?? output[outputKey];
+                const data = lastHiddenState.data;
+                const embedding = this.meanPool(data, encoded.attentionMask, seqLen, DIMENSIONS);
+                results.push(this.normalize(embedding));
+            }
+            return results;
+        }
+        catch (error) {
+            if (error instanceof EmbeddingNotAvailableError || error instanceof EmbeddingError) {
+                throw error;
+            }
+            throw new EmbeddingError("ONNX inference failed", error instanceof Error ? error : new Error(String(error)));
+        }
+    }
+    getDimensions() {
+        return DIMENSIONS;
+    }
+    async close() {
+        if (this.session) {
+            await this.session.release();
+            this.session = null;
+        }
+    }
+    meanPool(data, attentionMask, seqLen, hiddenSize) {
+        const pooled = new Float32Array(hiddenSize);
+        let count = 0;
+        for (let i = 0; i < seqLen; i++) {
+            if (attentionMask[i] === 0)
+                continue;
+            count++;
+            for (let j = 0; j < hiddenSize; j++) {
+                pooled[j] += data[i * hiddenSize + j];
+            }
+        }
+        if (count > 0) {
+            for (let j = 0; j < hiddenSize; j++) {
+                pooled[j] /= count;
+            }
+        }
+        return pooled;
+    }
+    normalize(vec) {
+        let norm = 0;
+        for (const v of vec) {
+            norm += v * v;
+        }
+        norm = Math.sqrt(norm);
+        if (norm === 0)
+            return vec;
+        const result = new Float32Array(vec.length);
+        for (let i = 0; i < vec.length; i++) {
+            result[i] = vec[i] / norm;
+        }
+        return result;
+    }
+}
+//# sourceMappingURL=onnx-embedding-provider.js.map

package/dist/embedding/onnx-embedding-provider.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"onnx-embedding-provider.js","sourceRoot":"","sources":["../../src/embedding/onnx-embedding-provider.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,YAAY,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACtE,OAAO,EAAE,0BAA0B,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAE1E,MAAM,UAAU,GAAG,GAAG,CAAC;AAEvB,MAAM,OAAO,qBAAqB;IACxB,OAAO,GAA4B,IAAI,CAAC;IACxC,SAAS,GAA8B,IAAI,CAAC;IAC5C,SAAS,CAAS;IAClB,YAAY,CAAe;IAEnC,YAAY,YAAoB,kBAAkB,EAAE,YAA2B;QAC7E,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;QAC3B,IAAI,CAAC,YAAY,GAAG,YAAY,IAAI,IAAI,YAAY,EAAE,CAAC;IACzD,CAAC;IAEO,KAAK,CAAC,UAAU;QACtB,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,OAAO,CAAC;QAEtC,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,0BAA0B,CAClC,UAAU,IAAI,CAAC,SAAS,0EAA0E,CACnG,CAAC;QACJ,CAAC;QAED,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAC7C,IAAI,CAAC,OAAO,GAAG,MAAM,GAAG,CAAC,gBAAgB,CAAC,MAAM,CAC9C,IAAI,CAAC,YAAY,CAAC,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,EAC9C,EAAE,kBAAkB,EAAE,CAAC,KAAK,CAAC,EAAE,CAChC,CAAC;YACF,OAAO,IAAI,CAAC,OAAO,CAAC;QACtB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,cAAc,CACtB,8BAA8B,IAAI,CAAC,SAAS,GAAG,EAC/C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC1D,CAAC;QACJ,CAAC;IACH,CAAC;IAEO,YAAY;QAClB,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC,SAAS,CAAC;QAC1C,IAAI,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;YACxD,MAAM,IAAI,0BAA0B,CAClC,UAAU,IAAI,CAAC,SAAS,0EAA0E,CACnG,CAAC;QACJ,CAAC;QACD,IAAI,CAAC,SAAS,GAAG,IAAI,kBAAkB,CAAC,IAAI,CAAC,YAAY,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5F,OAAO,IAAI,CAAC,SAAS,CAAC;IACxB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC/C,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,UAAU,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;QAEtC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAmB,EAAE,CAAC;YAEnC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,OAAO,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;gBACvC,MAAM,MAAM,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAEvC,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,EAAE,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EAAE;oBACzF,CAAC;oBACD,MAAM;iBACP,CAAC,CAAC;gBACH,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,MAAM,CAClC,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EACrD,CAAC,CAAC,EAAE,MAAM,CAAC,CACZ,CAAC;gBACF,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,MAAM,CACjC,OAAO,EACP,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,EACpD,CAAC,CAAC,EAAE,MAAM,CAAC,CACZ,CAAC;gBAEF,MAAM,KAAK,GAA2B;oBACpC,SAAS,EAAE,QAAQ;oBACnB,cAAc,EAAE,aAAa;oBAC7B,cAAc,EAAE,YAAY;iBAC7B,CAAC;gBACF,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAExC,yDAAyD;gBACzD,MAAM,SAAS,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;gBACzC,MAAM,eAAe,GAAG,MAAM,CAAC,mBAAmB,CAAC,IAAI,MAAM,CAAC,SAAS,CAAC,CAAC;gBACzE,MAAM,IAAI,GAAG,eAAe,CAAC,IAAoB,CAAC;gBAClD,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,aAAa,EAAE,MAAM,EAAE,UAAU,CAAC,CAAC;gBACjF,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YAC1C,CAAC;YAED,OAAO,OAAO,CAAC;QACjB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,0BAA0B,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACnF,MAAM,KAAK,CAAC;YACd,CAAC;YACD,MAAM,IAAI,cAAc,CACtB,uBAAuB,EACvB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAC1D,CAAC;QACJ,CAAC;IACH,CAAC;IAED,aAAa;QACX,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,MAAM,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,CAAC;YAC7B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAEO,QAAQ,CACd,IAAkB,EAClB,aAAuB,EACvB,MAAc,EACd,UAAkB;QAElB,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,UAAU,CAAC,CAAC;QAC5C,IAAI,KAAK,GAAG,CAAC,CAAC;QAEd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAChC,IAAI,aAAa,CAAC,CAAC,CAAC,KAAK,CAAC;gBAAE,SAAS;YACrC,KAAK,EAAE,CAAC;YACR,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,MAAM,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,UAAU,GAAG,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;YACd,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;gBACpC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC;YACrB,CAAC;QACH,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,SAAS,CAAC,GAAiB;QACjC,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,GAAG,EAAE,CAAC;YACpB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC;QAChB,CAAC;QACD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,IAAI,IAAI,KAAK,CAAC;YAAE,OAAO,GAAG,CAAC;QAC3B,MAAM,MAAM,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACpC,MAAM,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC;QAC5B,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC;CACF"}

package/dist/embedding/tokenizer.d.ts

@@ -0,0 +1,19 @@
+interface EncodingResult {
+    inputIds: number[];
+    attentionMask: number[];
+    tokenTypeIds: number[];
+}
+export declare class WordPieceTokenizer {
+    private vocab;
+    private unkId;
+    private clsId;
+    private sepId;
+    private padId;
+    constructor(tokenizerJsonPath: string);
+    encode(text: string): EncodingResult;
+    private lookupToken;
+    private tokenize;
+    private wordpieceTokenize;
+}
+export {};
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/embedding/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/embedding/tokenizer.ts"],"names":[],"mappings":"AASA,UAAU,cAAc;IACtB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,aAAa,EAAE,MAAM,EAAE,CAAC;IACxB,YAAY,EAAE,MAAM,EAAE,CAAC;CACxB;AAQD,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,KAAK,CAAsB;IACnC,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,KAAK,CAAS;gBAEV,iBAAiB,EAAE,MAAM;IAWrC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc;IAsBpC,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,QAAQ;IAUhB,OAAO,CAAC,iBAAiB;CAiC1B"}