mcp-local-rag 0.8.2 → 0.9.0

package/README.md

@@ -119,6 +119,30 @@ HTML is automatically cleaned—you get the article content, not the boilerplate
 
 > **Note:** The RAG server itself doesn't fetch web content—your AI assistant retrieves it and passes the HTML to `ingest_data`. This keeps the server fully local while letting you index any content your assistant can access. Please respect website terms of service and copyright when ingesting external content.
 
+### Bulk Ingestion (CLI)
+
+For ingesting multiple files or an entire directory, use the CLI command instead of calling `ingest_file` repeatedly:
+
+```bash
+npx mcp-local-rag ingest --db-path ./lancedb --base-dir ./docs ./docs/
+```
+
+Supports PDF, DOCX, TXT, and Markdown (same as MCP tools). HTML is only supported via `ingest_data`, not CLI.
+
+Key options:
+
+| Option | Default | Description |
+|--------|---------|-------------|
+| `--db-path <path>` | `./lancedb/` | LanceDB database path |
+| `--base-dir <path>` | cwd | Base directory for documents |
+| `--model-name <name>` | `Xenova/all-MiniLM-L6-v2` | Embedding model |
+| `--cache-dir <path>` | `./models/` | Model cache directory |
+| `--max-file-size <n>` | `104857600` | Max file size in bytes |
+
+This processes all supported files recursively and runs optimization once at the end — much faster than per-file ingestion for large batches.
+
+> ⚠️ **CLI options must match your MCP server config.** Especially `--model-name` — using a different embedding model against an existing database produces incompatible vectors, silently degrading search quality.
+
 ### Searching Documents
 
 ```
@@ -407,6 +431,7 @@ pnpm run check:all     # Full quality check
 src/
   index.ts      # Entry point
   server/       # MCP tool handlers
+  cli/          # CLI subcommands (ingest)
   parser/       # PDF, DOCX, TXT, MD parsing
   chunker/      # Text splitting
   embedder/     # Transformers.js embeddings

package/dist/cli/ingest.d.ts

@@ -0,0 +1,24 @@
+interface CliOptions {
+    dbPath?: string | undefined;
+    baseDir?: string | undefined;
+    cacheDir?: string | undefined;
+    modelName?: string | undefined;
+    maxFileSize?: number | undefined;
+}
+interface ParsedArgs {
+    positional: string | undefined;
+    options: CliOptions;
+    help: boolean;
+}
+/**
+ * Parse CLI arguments into options and a positional path.
+ * Flags: --db-path, --base-dir, --cache-dir, --model-name, --max-file-size, -h/--help
+ */
+export declare function parseArgs(args: string[]): ParsedArgs;
+/**
+ * Run the ingest CLI subcommand.
+ * @param args - Arguments after "ingest" (e.g., option flags and file/directory path)
+ */
+export declare function runIngest(args: string[]): Promise<void>;
+export {};
+//# sourceMappingURL=ingest.d.ts.map

package/dist/cli/ingest.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../../src/cli/ingest.ts"],"names":[],"mappings":"AA8BA,UAAU,UAAU;IAClB,MAAM,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC3B,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC5B,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC7B,SAAS,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;IAC9B,WAAW,CAAC,EAAE,MAAM,GAAG,SAAS,CAAA;CACjC;AAED,UAAU,UAAU;IAClB,UAAU,EAAE,MAAM,GAAG,SAAS,CAAA;IAC9B,OAAO,EAAE,UAAU,CAAA;IACnB,IAAI,EAAE,OAAO,CAAA;CACd;AAiCD;;;GAGG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,UAAU,CAiDpD;AAmJD;;;GAGG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAgG7D"}

package/dist/cli/ingest.js

@@ -0,0 +1,295 @@
+// CLI ingest subcommand — bulk file ingestion with single optimize() at end
+import { randomUUID } from 'node:crypto';
+import { readdir, stat } from 'node:fs/promises';
+import { extname, join, resolve } from 'node:path';
+import { SemanticChunker } from '../chunker/index.js';
+import { Embedder } from '../embedder/index.js';
+import { DocumentParser, SUPPORTED_EXTENSIONS } from '../parser/index.js';
+import { VectorStore } from '../vectordb/index.js';
+// ============================================
+// Defaults
+// ============================================
+const DEFAULTS = {
+    dbPath: './lancedb/',
+    cacheDir: './models/',
+    modelName: 'Xenova/all-MiniLM-L6-v2',
+    maxFileSize: 104857600,
+};
+// ============================================
+// Help
+// ============================================
+const HELP_TEXT = `Usage: mcp-local-rag ingest [options] <path>
+
+Ingest a single file or all supported files under a directory.
+
+Options:
+  --db-path <path>       LanceDB database path (default: ${DEFAULTS.dbPath})
+  --base-dir <path>      Base directory for documents (default: cwd)
+  --cache-dir <path>     Model cache directory (default: ${DEFAULTS.cacheDir})
+  --model-name <name>    Embedding model (default: ${DEFAULTS.modelName})
+  --max-file-size <n>    Max file size in bytes (default: ${DEFAULTS.maxFileSize})
+  -h, --help             Show this help`;
+// ============================================
+// Arg Parsing
+// ============================================
+/**
+ * Parse CLI arguments into options and a positional path.
+ * Flags: --db-path, --base-dir, --cache-dir, --model-name, --max-file-size, -h/--help
+ */
+export function parseArgs(args) {
+    const options = {};
+    let positional;
+    let help = false;
+    let i = 0;
+    while (i < args.length) {
+        const arg = args[i];
+        switch (arg) {
+            case '-h':
+            case '--help':
+                help = true;
+                i++;
+                break;
+            case '--db-path':
+                options.dbPath = args[++i];
+                i++;
+                break;
+            case '--base-dir':
+                options.baseDir = args[++i];
+                i++;
+                break;
+            case '--cache-dir':
+                options.cacheDir = args[++i];
+                i++;
+                break;
+            case '--model-name':
+                options.modelName = args[++i];
+                i++;
+                break;
+            case '--max-file-size': {
+                const parsed = Number.parseInt(args[++i] ?? '', 10);
+                options.maxFileSize = Number.isNaN(parsed) ? undefined : parsed;
+                i++;
+                break;
+            }
+            default:
+                if (arg.startsWith('-')) {
+                    console.error(`Unknown option: ${arg}`);
+                    console.error(HELP_TEXT);
+                    process.exit(1);
+                }
+                positional = arg;
+                i++;
+                break;
+        }
+    }
+    return { positional, options, help };
+}
+// ============================================
+// NaN Defense
+// ============================================
+/**
+ * Ensure maxFileSize is a valid number, falling back to default if NaN.
+ */
+function sanitizeMaxFileSize(value) {
+    return Number.isNaN(value) ? DEFAULTS.maxFileSize : value;
+}
+// ============================================
+// Config Resolution
+// ============================================
+/**
+ * Resolve config with priority: CLI flags > environment variables > defaults.
+ */
+function resolveConfig(cliOptions = {}) {
+    return {
+        baseDir: cliOptions.baseDir ?? process.env['BASE_DIR'] ?? process.cwd(),
+        dbPath: cliOptions.dbPath ?? process.env['DB_PATH'] ?? DEFAULTS.dbPath,
+        cacheDir: cliOptions.cacheDir ?? process.env['CACHE_DIR'] ?? DEFAULTS.cacheDir,
+        modelName: cliOptions.modelName ?? process.env['MODEL_NAME'] ?? DEFAULTS.modelName,
+        maxFileSize: sanitizeMaxFileSize(cliOptions.maxFileSize ??
+            (process.env['MAX_FILE_SIZE']
+                ? Number.parseInt(process.env['MAX_FILE_SIZE'], 10)
+                : DEFAULTS.maxFileSize)),
+    };
+}
+// ============================================
+// File Collection
+// ============================================
+/**
+ * Collect files to ingest from a path.
+ * - If path is a file with supported extension, return [path].
+ * - If path is a directory, recursively scan for supported files.
+ * - Exclude dbPath and cacheDir directories.
+ */
+async function collectFiles(targetPath, excludePaths) {
+    const resolved = resolve(targetPath);
+    const info = await stat(resolved);
+    if (info.isFile()) {
+        const ext = extname(resolved).toLowerCase();
+        if (!SUPPORTED_EXTENSIONS.has(ext)) {
+            console.error(`Unsupported file extension: ${ext} (supported: ${[...SUPPORTED_EXTENSIONS].join(', ')})`);
+            return [];
+        }
+        return [resolved];
+    }
+    if (info.isDirectory()) {
+        const entries = await readdir(resolved, { recursive: true, withFileTypes: true });
+        return entries
+            .filter((e) => e.isFile() && SUPPORTED_EXTENSIONS.has(extname(e.name).toLowerCase()))
+            .map((e) => join(e.parentPath, e.name))
+            .filter((filePath) => !excludePaths.some((ep) => filePath.startsWith(ep)))
+            .sort();
+    }
+    return [];
+}
+// ============================================
+// Per-file Ingestion
+// ============================================
+/**
+ * Ingest a single file: parse, chunk, embed, delete old chunks, insert new chunks.
+ * Returns the number of chunks inserted.
+ */
+async function ingestSingleFile(filePath, parser, chunker, embedder, vectorStore) {
+    // Parse file
+    const isPdf = filePath.toLowerCase().endsWith('.pdf');
+    let text;
+    let title = null;
+    if (isPdf) {
+        const result = await parser.parsePdf(filePath, embedder);
+        text = result.content;
+        title = result.title || null;
+    }
+    else {
+        const result = await parser.parseFile(filePath);
+        text = result.content;
+        title = result.title || null;
+    }
+    // Chunk text
+    const chunks = await chunker.chunkText(text, embedder);
+    if (chunks.length === 0) {
+        console.error(`  Warning: 0 chunks generated (file may be empty or too short)`);
+        return 0;
+    }
+    // Generate embeddings
+    const embeddings = await embedder.embedBatch(chunks.map((c) => c.text));
+    // Delete existing chunks for this file
+    await vectorStore.deleteChunks(filePath);
+    // Build vector chunks
+    const timestamp = new Date().toISOString();
+    const vectorChunks = chunks.map((chunk, index) => {
+        const embedding = embeddings[index];
+        if (!embedding) {
+            throw new Error(`Missing embedding for chunk ${index}`);
+        }
+        return {
+            id: randomUUID(),
+            filePath,
+            chunkIndex: chunk.index,
+            text: chunk.text,
+            vector: embedding,
+            metadata: {
+                fileName: filePath.split('/').pop() || filePath,
+                fileSize: text.length,
+                fileType: filePath.split('.').pop() || '',
+            },
+            fileTitle: title,
+            timestamp,
+        };
+    });
+    // Insert chunks
+    await vectorStore.insertChunks(vectorChunks);
+    return vectorChunks.length;
+}
+// ============================================
+// Main Entry Point
+// ============================================
+/**
+ * Run the ingest CLI subcommand.
+ * @param args - Arguments after "ingest" (e.g., option flags and file/directory path)
+ */
+export async function runIngest(args) {
+    // Parse CLI options
+    const { positional, options, help } = parseArgs(args);
+    // Handle --help
+    if (help) {
+        console.error(HELP_TEXT);
+        process.exit(0);
+    }
+    // Validate positional argument
+    if (!positional) {
+        console.error('Usage: mcp-local-rag ingest [options] <path>');
+        console.error('  Ingest a single file or all supported files under a directory.');
+        console.error('  Run with --help for all options.');
+        process.exit(1);
+    }
+    const targetPath = positional;
+    // Validate path exists
+    try {
+        await stat(targetPath);
+    }
+    catch {
+        console.error(`Error: path does not exist: ${targetPath}`);
+        process.exit(1);
+    }
+    // Resolve config: CLI flags > env vars > defaults
+    const config = resolveConfig(options);
+    const excludePaths = [`${resolve(config.dbPath)}/`, `${resolve(config.cacheDir)}/`];
+    // Collect files
+    const files = await collectFiles(targetPath, excludePaths);
+    if (files.length === 0) {
+        console.error('No supported files found.');
+        process.exit(1);
+    }
+    console.error(`Found ${files.length} file(s) to ingest.`);
+    // Initialize components (single instances reused across all files)
+    const parser = new DocumentParser({
+        baseDir: config.baseDir,
+        maxFileSize: config.maxFileSize,
+    });
+    const chunker = new SemanticChunker();
+    const embedder = new Embedder({
+        modelPath: config.modelName,
+        batchSize: 16,
+        cacheDir: config.cacheDir,
+    });
+    const vectorStore = new VectorStore({
+        dbPath: config.dbPath,
+        tableName: 'chunks',
+    });
+    await vectorStore.initialize();
+    // Process each file
+    const summary = { succeeded: 0, failed: 0, totalChunks: 0 };
+    for (let i = 0; i < files.length; i++) {
+        const filePath = files[i];
+        const label = `[${i + 1}/${files.length}]`;
+        try {
+            const chunkCount = await ingestSingleFile(filePath, parser, chunker, embedder, vectorStore);
+            if (chunkCount === 0) {
+                // 0 chunks is a skip/warning, not a failure
+                console.error(`${label} ${filePath} ... SKIPPED (0 chunks)`);
+                summary.succeeded++;
+            }
+            else {
+                console.error(`${label} ${filePath} ... OK (${chunkCount} chunks)`);
+                summary.succeeded++;
+                summary.totalChunks += chunkCount;
+            }
+        }
+        catch (error) {
+            const reason = error instanceof Error ? error.message : String(error);
+            console.error(`${label} ${filePath} ... FAILED: ${reason}`);
+            summary.failed++;
+        }
+    }
+    // Optimize once at end (not per-file)
+    await vectorStore.optimize();
+    // Print summary
+    console.error('');
+    console.error('--- Ingest Summary ---');
+    console.error(`Succeeded: ${summary.succeeded}`);
+    console.error(`Failed:    ${summary.failed}`);
+    console.error(`Total chunks: ${summary.totalChunks}`);
+    if (summary.failed > 0) {
+        process.exitCode = 1;
+    }
+}
+//# sourceMappingURL=ingest.js.map

package/dist/cli/ingest.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../../src/cli/ingest.ts"],"names":[],"mappings":"AAAA,4EAA4E;AAE5E,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AACxC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AAChD,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAElD,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AACrD,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAA;AAC/C,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAEzE,OAAO,EAAE,WAAW,EAAE,MAAM,sBAAsB,CAAA;AAkClD,+CAA+C;AAC/C,WAAW;AACX,+CAA+C;AAE/C,MAAM,QAAQ,GAAG;IACf,MAAM,EAAE,YAAY;IACpB,QAAQ,EAAE,WAAW;IACrB,SAAS,EAAE,yBAAyB;IACpC,WAAW,EAAE,SAAS;CACd,CAAA;AAEV,+CAA+C;AAC/C,OAAO;AACP,+CAA+C;AAE/C,MAAM,SAAS,GAAG;;;;;2DAKyC,QAAQ,CAAC,MAAM;;2DAEf,QAAQ,CAAC,QAAQ;qDACvB,QAAQ,CAAC,SAAS;4DACX,QAAQ,CAAC,WAAW;wCACxC,CAAA;AAExC,+CAA+C;AAC/C,cAAc;AACd,+CAA+C;AAE/C;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,MAAM,OAAO,GAAe,EAAE,CAAA;IAC9B,IAAI,UAA8B,CAAA;IAClC,IAAI,IAAI,GAAG,KAAK,CAAA;IAEhB,IAAI,CAAC,GAAG,CAAC,CAAA;IACT,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QACvB,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAA;QACpB,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ;gBACX,IAAI,GAAG,IAAI,CAAA;gBACX,CAAC,EAAE,CAAA;gBACH,MAAK;YACP,KAAK,WAAW;gBACd,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;gBAC1B,CAAC,EAAE,CAAA;gBACH,MAAK;YACP,KAAK,YAAY;gBACf,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;gBAC3B,CAAC,EAAE,CAAA;gBACH,MAAK;YACP,KAAK,aAAa;gBAChB,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;gBAC5B,CAAC,EAAE,CAAA;gBACH,MAAK;YACP,KAAK,cAAc;gBACjB,OAAO,CAAC,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,CAAC,CAAA;gBAC7B,CAAC,EAAE,CAAA;gBACH,MAAK;YACP,KAAK,iBAAiB,CAAC,CAAC,CAAC;gBACvB,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAA;gBACnD,OAAO,CAAC,WAAW,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,MAAM,CAAA;gBAC/D,CAAC,EAAE,CAAA;gBACH,MAAK;YACP,CAAC;YACD;gBACE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBACxB,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAA;oBACvC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;oBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,UAAU,GAAG,GAAG,CAAA;gBAChB,CAAC,EAAE,CAAA;gBACH,MAAK;QACT,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,CAAA;AACtC,CAAC;AAED,+CAA+C;AAC/C,cAAc;AACd,+CAA+C;AAE/C;;GAEG;AACH,SAAS,mBAAmB,CAAC,KAAa;IACxC,OAAO,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC,CAAC,KAAK,CAAA;AAC3D,CAAC;AAED,+CAA+C;AAC/C,oBAAoB;AACpB,+CAA+C;AAE/C;;GAEG;AACH,SAAS,aAAa,CAAC,aAAyB,EAAE;IAChD,OAAO;QACL,OAAO,EAAE,UAAU,CAAC,OAAO,IAAI,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,OAAO,CAAC,GAAG,EAAE;QACvE,MAAM,EAAE,UAAU,CAAC,MAAM,IAAI,OAAO,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,QAAQ,CAAC,MAAM;QACtE,QAAQ,EAAE,UAAU,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,IAAI,QAAQ,CAAC,QAAQ;QAC9E,SAAS,EAAE,UAAU,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,QAAQ,CAAC,SAAS;QAClF,WAAW,EAAE,mBAAmB,CAC9B,UAAU,CAAC,WAAW;YACpB,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;gBAC3B,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,EAAE,EAAE,CAAC;gBACnD,CAAC,CAAC,QAAQ,CAAC,WAAW,CAAC,CAC5B;KACF,CAAA;AACH,CAAC;AAED,+CAA+C;AAC/C,kBAAkB;AAClB,+CAA+C;AAE/C;;;;;GAKG;AACH,KAAK,UAAU,YAAY,CAAC,UAAkB,EAAE,YAAsB;IACpE,MAAM,QAAQ,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;IACpC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAA;IAEjC,IAAI,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAClB,MAAM,GAAG,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAA;QAC3C,IAAI,CAAC,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,CACX,+BAA+B,GAAG,gBAAgB,CAAC,GAAG,oBAAoB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAC1F,CAAA;YACD,OAAO,EAAE,CAAA;QACX,CAAC;QACD,OAAO,CAAC,QAAQ,CAAC,CAAA;IACnB,CAAC;IAED,IAAI,IAAI,CAAC,WAAW,EAAE,EAAE,CAAC;QACvB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,QAAQ,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAA;QACjF,OAAO,OAAO;aACX,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,EAAE,IAAI,oBAAoB,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;aACpF,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;aACtC,MAAM,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;aACzE,IAAI,EAAE,CAAA;IACX,CAAC;IAED,OAAO,EAAE,CAAA;AACX,CAAC;AAED,+CAA+C;AAC/C,qBAAqB;AACrB,+CAA+C;AAE/C;;;GAGG;AACH,KAAK,UAAU,gBAAgB,CAC7B,QAAgB,EAChB,MAAsB,EACtB,OAAwB,EACxB,QAAkB,EAClB,WAAwB;IAExB,aAAa;IACb,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAA;IACrD,IAAI,IAAY,CAAA;IAChB,IAAI,KAAK,GAAkB,IAAI,CAAA;IAC/B,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;QACxD,IAAI,GAAG,MAAM,CAAC,OAAO,CAAA;QACrB,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,IAAI,CAAA;IAC9B,CAAC;SAAM,CAAC;QACN,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;QAC/C,IAAI,GAAG,MAAM,CAAC,OAAO,CAAA;QACrB,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,IAAI,CAAA;IAC9B,CAAC;IAED,aAAa;IACb,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,SAAS,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAA;IACtD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,CAAC,KAAK,CAAC,gEAAgE,CAAC,CAAA;QAC/E,OAAO,CAAC,CAAA;IACV,CAAC;IAED,sBAAsB;IACtB,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAEvE,uCAAuC;IACvC,MAAM,WAAW,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;IAExC,sBAAsB;IACtB,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;IAC1C,MAAM,YAAY,GAAkB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,EAAE;QAC9D,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAC,CAAA;QACnC,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAA;QACzD,CAAC;QACD,OAAO;YACL,EAAE,EAAE,UAAU,EAAE;YAChB,QAAQ;YACR,UAAU,EAAE,KAAK,CAAC,KAAK;YACvB,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,MAAM,EAAE,SAAS;YACjB,QAAQ,EAAE;gBACR,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ;gBAC/C,QAAQ,EAAE,IAAI,CAAC,MAAM;gBACrB,QAAQ,EAAE,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE;aAC1C;YACD,SAAS,EAAE,KAAK;YAChB,SAAS;SACV,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,gBAAgB;IAChB,MAAM,WAAW,CAAC,YAAY,CAAC,YAAY,CAAC,CAAA;IAE5C,OAAO,YAAY,CAAC,MAAM,CAAA;AAC5B,CAAC;AAED,+CAA+C;AAC/C,mBAAmB;AACnB,+CAA+C;AAE/C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAc;IAC5C,oBAAoB;IACpB,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,IAAI,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IAErD,gBAAgB;IAChB,IAAI,IAAI,EAAE,CAAC;QACT,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,+BAA+B;IAC/B,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAA;QAC7D,OAAO,CAAC,KAAK,CAAC,kEAAkE,CAAC,CAAA;QACjF,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAA;QACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,MAAM,UAAU,GAAG,UAAU,CAAA;IAE7B,uBAAuB;IACvB,IAAI,CAAC;QACH,MAAM,IAAI,CAAC,UAAU,CAAC,CAAA;IACxB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,+BAA+B,UAAU,EAAE,CAAC,CAAA;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,kDAAkD;IAClD,MAAM,MAAM,GAAG,aAAa,CAAC,OAAO,CAAC,CAAA;IACrC,MAAM,YAAY,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAA;IAEnF,gBAAgB;IAChB,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,UAAU,EAAE,YAAY,CAAC,CAAA;IAC1D,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAA;QAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,OAAO,CAAC,KAAK,CAAC,SAAS,KAAK,CAAC,MAAM,qBAAqB,CAAC,CAAA;IAEzD,mEAAmE;IACnE,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC;QAChC,OAAO,EAAE,MAAM,CAAC,OAAO;QACvB,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,IAAI,eAAe,EAAE,CAAA;IACrC,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC;QAC5B,SAAS,EAAE,MAAM,CAAC,SAAS;QAC3B,SAAS,EAAE,EAAE;QACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;KAC1B,CAAC,CAAA;IACF,MAAM,WAAW,GAAG,IAAI,WAAW,CAAC;QAClC,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,SAAS,EAAE,QAAQ;KACpB,CAAC,CAAA;IACF,MAAM,WAAW,CAAC,UAAU,EAAE,CAAA;IAE9B,oBAAoB;IACpB,MAAM,OAAO,GAAkB,EAAE,SAAS,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAA;IAE1E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAE,CAAA;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAA;QAE1C,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC,CAAA;YAC3F,IAAI,UAAU,KAAK,CAAC,EAAE,CAAC;gBACrB,4CAA4C;gBAC5C,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,QAAQ,yBAAyB,CAAC,CAAA;gBAC5D,OAAO,CAAC,SAAS,EAAE,CAAA;YACrB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,QAAQ,YAAY,UAAU,UAAU,CAAC,CAAA;gBACnE,OAAO,CAAC,SAAS,EAAE,CAAA;gBACnB,OAAO,CAAC,WAAW,IAAI,UAAU,CAAA;YACnC,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,MAAM,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACrE,OAAO,CAAC,KAAK,CAAC,GAAG,KAAK,IAAI,QAAQ,gBAAgB,MAAM,EAAE,CAAC,CAAA;YAC3D,OAAO,CAAC,MAAM,EAAE,CAAA;QAClB,CAAC;IACH,CAAC;IAED,sCAAsC;IACtC,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAA;IAE5B,gBAAgB;IAChB,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;IACjB,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAA;IACvC,OAAO,CAAC,KAAK,CAAC,cAAc,OAAO,CAAC,SAAS,EAAE,CAAC,CAAA;IAChD,OAAO,CAAC,KAAK,CAAC,cAAc,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;IAC7C,OAAO,CAAC,KAAK,CAAC,iBAAiB,OAAO,CAAC,WAAW,EAAE,CAAC,CAAA;IAErD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;IACtB,CAAC;AACH,CAAC"}

package/dist/cli-main.d.ts

@@ -2,5 +2,5 @@
  * Handle CLI subcommands
  * @param args - Command line arguments (after the binary name)
  */
-export declare function handleCli(args: string[]): void;
+export declare function handleCli(args: string[]): Promise<void>;
 //# sourceMappingURL=cli-main.d.ts.map

package/dist/cli-main.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cli-main.d.ts","sourceRoot":"","sources":["../src/cli-main.ts"],"names":[],"mappings":"AAGA;;;GAGG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,IAAI,CAsB9C"}
+{"version":3,"file":"cli-main.d.ts","sourceRoot":"","sources":["../src/cli-main.ts"],"names":[],"mappings":"AAIA;;;GAGG;AACH,wBAAsB,SAAS,CAAC,IAAI,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA0B7D"}

package/dist/cli-main.js

@@ -1,10 +1,11 @@
 // CLI entry point for subcommands (skills install, etc.)
 import { run as runSkillsInstall } from './bin/install-skills.js';
+import { runIngest } from './cli/ingest.js';
 /**
  * Handle CLI subcommands
  * @param args - Command line arguments (after the binary name)
  */
-export function handleCli(args) {
+export async function handleCli(args) {
     const subcommand = args[0];
     switch (subcommand) {
         case 'skills':
@@ -18,9 +19,12 @@ export function handleCli(args) {
                 process.exit(1);
             }
             break;
+        case 'ingest':
+            await runIngest(args.slice(1));
+            break;
         default:
             console.error(`Unknown command: ${subcommand}`);
-            console.error('Available commands: skills');
+            console.error('Available commands: skills, ingest');
             process.exit(1);
     }
 }

package/dist/cli-main.js.map

@@ -1 +1 @@
-{"version":3,"file":"cli-main.js","sourceRoot":"","sources":["../src/cli-main.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,OAAO,EAAE,GAAG,IAAI,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAEjE;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,IAAc;IACtC,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IAE1B,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,QAAQ;YACX,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC1B,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CACX,8EAA8E,CAC/E,CAAA;gBACD,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAA;gBACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;YACD,MAAK;QAEP;YACE,OAAO,CAAC,KAAK,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAA;YAC/C,OAAO,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAA;YAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC;AACH,CAAC"}
+{"version":3,"file":"cli-main.js","sourceRoot":"","sources":["../src/cli-main.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,OAAO,EAAE,GAAG,IAAI,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAE3C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,IAAc;IAC5C,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;IAE1B,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,QAAQ;YACX,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC1B,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CACX,8EAA8E,CAC/E,CAAA;gBACD,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAA;gBACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;YACD,MAAK;QAEP,KAAK,QAAQ;YACX,MAAM,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;YAC9B,MAAK;QAEP;YACE,OAAO,CAAC,KAAK,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAA;YAC/C,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAA;YACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACnB,CAAC;AACH,CAAC"}

package/dist/embedder/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;CACjB;AAMD;;GAEG;AACH,qBAAa,cAAe,SAAQ,KAAK;aAGZ,KAAK,CAAC,EAAE,KAAK;gBADtC,OAAO,EAAE,MAAM,EACU,KAAK,CAAC,EAAE,KAAK,YAAA;CAKzC;AAMD;;;;;;;GAOG;AACH,qBAAa,QAAQ;IAEnB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,MAAM,EAAE,cAAc;IAIlC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAuBjC;;;OAGG;YACW,iBAAiB;IA+B/B;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAiC5C;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA0BvD"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AAQA;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;CACjB;AAMD;;GAEG;AACH,qBAAa,cAAe,SAAQ,KAAK;aAGZ,KAAK,CAAC,EAAE,KAAK;gBADtC,OAAO,EAAE,MAAM,EACU,KAAK,CAAC,EAAE,KAAK,YAAA;CAKzC;AAMD;;;;;;;GAOG;AACH,qBAAa,QAAQ;IAEnB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;gBAE3B,MAAM,EAAE,cAAc;IAIlC;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBjC;;;OAGG;YACW,iBAAiB;IA+B/B;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAiC5C;;;;;OAKG;IACG,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CA0BvD"}

package/dist/embedder/index.js

@@ -45,7 +45,9 @@ export class Embedder {
             console.error(`Embedder: Setting cache directory to "${this.config.cacheDir}"`);
             console.error(`Embedder: Loading model "${this.config.modelPath}"...`);
             // Use type assertion to avoid TS2590 (union type too complex with @types/jsdom)
-            this.model = await pipeline('feature-extraction', this.config.modelPath);
+            this.model = await pipeline('feature-extraction', this.config.modelPath, {
+                dtype: 'fp32',
+            });
             console.error('Embedder: Model loaded successfully');
         }
         catch (error) {

package/dist/embedder/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAE/C,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AAkBzD,+CAA+C;AAC/C,gBAAgB;AAChB,+CAA+C;AAE/C;;GAEG;AACH,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YACE,OAAe,EACU,KAAa;QAEtC,KAAK,CAAC,OAAO,CAAC,CAAA;QAFW,UAAK,GAAL,KAAK,CAAQ;QAGtC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAA;IAC9B,CAAC;CACF;AAED,+CAA+C;AAC/C,iBAAiB;AACjB,+CAA+C;AAE/C;;;;;;;GAOG;AACH,MAAM,OAAO,QAAQ;IAMnB,YAAY,MAAsB;QALlC,2EAA2E;QACnE,UAAK,GAAY,IAAI,CAAA;QACrB,gBAAW,GAAyB,IAAI,CAAA;QAI9C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,8BAA8B;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,IAAI,CAAC;YACH,+CAA+C;YAC/C,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAA;YAEnC,OAAO,CAAC,KAAK,CAAC,yCAAyC,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAA;YAC/E,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,CAAC,SAAS,MAAM,CAAC,CAAA;YACtE,gFAAgF;YAChF,IAAI,CAAC,KAAK,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;YACxE,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,cAAc,CACtB,kCAAmC,KAAe,CAAC,OAAO,EAAE,EAC5D,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB;QAC7B,sBAAsB;QACtB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,kDAAkD;QAClD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,WAAW,CAAA;YACtB,OAAM;QACR,CAAC;QAED,uBAAuB;QACvB,OAAO,CAAC,KAAK,CACX,+FAA+F,CAChG,CAAA;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACnD,8CAA8C;YAC9C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;YAEvB,+CAA+C;YAC/C,MAAM,IAAI,cAAc,CACtB,+CAAgD,KAAe,CAAC,OAAO,wTAAwT,IAAI,CAAC,MAAM,CAAC,QAAQ,gCAAgC,EACnb,KAAc,CACf,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC;YACH,mEAAmE;YACnE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,cAAc,CAAC,0CAA0C,CAAC,CAAA;YACtE,CAAC;YAED,uEAAuE;YACvE,8FAA8F;YAC9F,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;YACpD,MAAM,SAAS,GAAG,IAAI,CAAC,KAGa,CAAA;YACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YAE7C,qCAAqC;YACrC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACzC,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,cAAc,CACtB,iCAAkC,KAAe,CAAC,OAAO,EAAE,EAC3D,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAA;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAe,EAAE,CAAA;YAEjC,6CAA6C;YAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;gBACvD,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBAChF,UAAU,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAA;YACrC,CAAC;YAED,OAAO,UAAU,CAAA;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,cAAc,CACtB,wCAAyC,KAAe,CAAC,OAAO,EAAE,EAClE,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;CACF"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AAAA,+CAA+C;AAE/C,OAAO,EAAE,GAAG,EAAE,QAAQ,EAAE,MAAM,2BAA2B,CAAA;AAkBzD,+CAA+C;AAC/C,gBAAgB;AAChB,+CAA+C;AAE/C;;GAEG;AACH,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,YACE,OAAe,EACU,KAAa;QAEtC,KAAK,CAAC,OAAO,CAAC,CAAA;QAFW,UAAK,GAAL,KAAK,CAAQ;QAGtC,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAA;IAC9B,CAAC;CACF;AAED,+CAA+C;AAC/C,iBAAiB;AACjB,+CAA+C;AAE/C;;;;;;;GAOG;AACH,MAAM,OAAO,QAAQ;IAMnB,YAAY,MAAsB;QALlC,2EAA2E;QACnE,UAAK,GAAY,IAAI,CAAA;QACrB,gBAAW,GAAyB,IAAI,CAAA;QAI9C,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACd,8BAA8B;QAC9B,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,IAAI,CAAC;YACH,+CAA+C;YAC/C,GAAG,CAAC,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAA;YAEnC,OAAO,CAAC,KAAK,CAAC,yCAAyC,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAA;YAC/E,OAAO,CAAC,KAAK,CAAC,4BAA4B,IAAI,CAAC,MAAM,CAAC,SAAS,MAAM,CAAC,CAAA;YACtE,gFAAgF;YAChF,IAAI,CAAC,KAAK,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE;gBACvE,KAAK,EAAE,MAAM;aACd,CAAC,CAAA;YACF,OAAO,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAA;QACtD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,cAAc,CACtB,kCAAmC,KAAe,CAAC,OAAO,EAAE,EAC5D,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;OAGG;IACK,KAAK,CAAC,iBAAiB;QAC7B,sBAAsB;QACtB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAM;QACR,CAAC;QAED,kDAAkD;QAClD,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,MAAM,IAAI,CAAC,WAAW,CAAA;YACtB,OAAM;QACR,CAAC;QAED,uBAAuB;QACvB,OAAO,CAAC,KAAK,CACX,+FAA+F,CAChG,CAAA;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;YACnD,8CAA8C;YAC9C,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;YAEvB,+CAA+C;YAC/C,MAAM,IAAI,cAAc,CACtB,+CAAgD,KAAe,CAAC,OAAO,wTAAwT,IAAI,CAAC,MAAM,CAAC,QAAQ,gCAAgC,EACnb,KAAc,CACf,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,MAAM,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,CAAC;YACH,mEAAmE;YACnE,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,MAAM,IAAI,cAAc,CAAC,0CAA0C,CAAC,CAAA;YACtE,CAAC;YAED,uEAAuE;YACvE,8FAA8F;YAC9F,MAAM,OAAO,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;YACpD,MAAM,SAAS,GAAG,IAAI,CAAC,KAGa,CAAA;YACpC,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YAE7C,qCAAqC;YACrC,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACzC,OAAO,SAAS,CAAA;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,KAAK,YAAY,cAAc,EAAE,CAAC;gBACpC,MAAM,KAAK,CAAA;YACb,CAAC;YACD,MAAM,IAAI,cAAc,CACtB,iCAAkC,KAAe,CAAC,OAAO,EAAE,EAC3D,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;IAED;;;;;OAKG;IACH,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,0EAA0E;QAC1E,MAAM,IAAI,CAAC,iBAAiB,EAAE,CAAA;QAE9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,OAAO,EAAE,CAAA;QACX,CAAC;QAED,IAAI,CAAC;YACH,MAAM,UAAU,GAAe,EAAE,CAAA;YAEjC,6CAA6C;YAC7C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;gBAC7D,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAA;gBACvD,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBAChF,UAAU,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC,CAAA;YACrC,CAAC;YAED,OAAO,UAAU,CAAA;QACnB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,cAAc,CACtB,wCAAyC,KAAe,CAAC,OAAO,EAAE,EAClE,KAAc,CACf,CAAA;QACH,CAAC;IACH,CAAC;CACF"}

package/dist/index.js

@@ -6,12 +6,15 @@ import { startServer } from './server-main.js';
 // ============================================
 // Routing
 // ============================================
-const SUBCOMMANDS = new Set(['skills']);
+const SUBCOMMANDS = new Set(['skills', 'ingest']);
 const args = process.argv.slice(2);
 const firstArg = args[0];
 if (firstArg && SUBCOMMANDS.has(firstArg)) {
     // CLI subcommand
-    handleCli(args);
+    handleCli(args).catch((error) => {
+        console.error(error);
+        process.exit(1);
+    });
 }
 else {
     // Default: start MCP server

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,gCAAgC;AAChC,qDAAqD;AAErD,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAE9C,+CAA+C;AAC/C,UAAU;AACV,+CAA+C;AAE/C,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAA;AAEvC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;AAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;AAExB,IAAI,QAAQ,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC1C,iBAAiB;IACjB,SAAS,CAAC,IAAI,CAAC,CAAA;AACjB,CAAC;KAAM,CAAC;IACN,4BAA4B;IAC5B,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;QACnD,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAA;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,EAAE;QACxC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAA;QAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC,CAAC,CAAA;IAEF,WAAW,EAAE,CAAA;AACf,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AACA,gCAAgC;AAChC,qDAAqD;AAErD,OAAO,EAAE,SAAS,EAAE,MAAM,eAAe,CAAA;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,kBAAkB,CAAA;AAE9C,+CAA+C;AAC/C,UAAU;AACV,+CAA+C;AAE/C,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAA;AAEjD,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAA;AAClC,MAAM,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;AAExB,IAAI,QAAQ,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;IAC1C,iBAAiB;IACjB,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;QAC9B,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;QACpB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC,CAAC,CAAA;AACJ,CAAC;KAAM,CAAC;IACN,4BAA4B;IAC5B,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE;QACnD,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,CAAC,CAAA;QACpE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC,CAAC,CAAA;IAEF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,KAAK,EAAE,EAAE;QACxC,OAAO,CAAC,KAAK,CAAC,qBAAqB,EAAE,KAAK,CAAC,CAAA;QAC3C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC,CAAC,CAAA;IAEF,WAAW,EAAE,CAAA;AACf,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mcp-local-rag",
-  "version": "0.8.2",
+  "version": "0.9.0",
   "description": "Local RAG MCP Server - Easy-to-setup document search with minimal configuration",
   "type": "module",
   "main": "dist/index.js",

package/skills/mcp-local-rag/SKILL.md

@@ -1,6 +1,6 @@
 ---
 name: mcp-local-rag
-description: Provides score interpretation (< 0.3 good, > 0.5 skip), query optimization, and source naming for query_documents, ingest_file, ingest_data tools. Use this skill when working with RAG, searching documents, ingesting files, saving web content, or handling PDF, HTML, DOCX, TXT, Markdown.
+description: Use this skill when ingesting or querying documents with MCP local RAG, including `query_documents`, `ingest_file`, `ingest_data`, and CLI bulk ingestion. Covers query refinement, result score interpretation, and source metadata conventions for PDF, HTML, DOCX, TXT, and Markdown. Not for general file operations or SQL/database queries.
 ---
 
 # MCP Local RAG Skills
@@ -13,6 +13,7 @@ description: Provides score interpretation (< 0.3 good, > 0.5 skip), query optim
 | `ingest_data` | Raw content (HTML, text) with source URL |
 | `query_documents` | Semantic + keyword hybrid search |
 | `delete_file` / `list_files` / `status` | Management |
+| `npx mcp-local-rag ingest` | Multiple files or directory (shell) |
 
 ## Search: Core Rules
 
@@ -112,9 +113,34 @@ ingest_data({
 
 Re-ingest same source to update. Use same source in `delete_file` to remove.
 
+### CLI ingest
+
+For multiple files or directory ingestion. Prefer over repeated `ingest_file` calls.
+
+| Scenario | Use |
+|----------|-----|
+| Single file from user request | `ingest_file` |
+| Multiple files or a directory | `npx mcp-local-rag ingest <path>` |
+| Raw HTML/text content | `ingest_data` (CLI does not support stdin) |
+
+```bash
+npx mcp-local-rag ingest [options] <path>
+```
+
+- `<path>`: file or directory (recursively scans supported formats)
+- Use `--help` for all options and defaults
+- Options must match MCP server config — see [cli-ingest.md](references/cli-ingest.md)
+
+**Output interpretation:**
+
+- Exit code 0: all files succeeded
+- Exit code 1: one or more files failed — report failed files to user
+- `SKIPPED (0 chunks)`: file was empty or too short, counted as success
+
 ## References
 
 For edge cases and examples:
 - [html-ingestion.md](references/html-ingestion.md) - URL normalization, SPA handling
 - [query-optimization.md](references/query-optimization.md) - Query patterns by intent
 - [result-refinement.md](references/result-refinement.md) - Contradiction resolution, chunking
+- [cli-ingest.md](references/cli-ingest.md) - CLI options, config matching

package/skills/mcp-local-rag/references/cli-ingest.md

@@ -0,0 +1,45 @@
+# CLI Ingest Reference
+
+## Command
+
+```bash
+npx mcp-local-rag ingest [options] <path>
+```
+
+## Options
+
+| Option | Env Var | Default | Description |
+|--------|---------|---------|-------------|
+| `--db-path <path>` | `DB_PATH` | `./lancedb/` | LanceDB database path |
+| `--base-dir <path>` | `BASE_DIR` | cwd | Base directory for documents |
+| `--cache-dir <path>` | `CACHE_DIR` | `./models/` | Model cache directory |
+| `--model-name <name>` | `MODEL_NAME` | `Xenova/all-MiniLM-L6-v2` | Embedding model |
+| `--max-file-size <n>` | `MAX_FILE_SIZE` | `104857600` | Max file size in bytes |
+| `-h, --help` | — | — | Show usage with defaults |
+
+Priority: CLI flags > environment variables > defaults.
+
+## Config Matching
+
+When ingesting into an existing database, options **must match** the MCP server config — especially `--model-name`. Using a different embedding model against an existing database produces vectors in a different space, silently degrading search quality.
+
+## Output
+
+Output goes to stderr. Summary block:
+
+```
+--- Ingest Summary ---
+Succeeded: 12
+Failed:    1
+Total chunks: 247
+```
+
+- Exit code 0: all succeeded
+- Exit code 1: one or more failed
+- `SKIPPED (0 chunks)`: empty or too-short file, counted as success
+
+## Supported Formats
+
+Same as MCP tools: `.pdf`, `.docx`, `.txt`, `.md`, `.html`
+
+Directory mode recursively scans for these extensions, excluding `dbPath` and `cacheDir` paths.