@ssmeo/mcp-local-rag 0.18.6

package/dist/cli/sync.js

@@ -0,0 +1,271 @@
+// CLI sync subcommand — foreground incremental reconciliation.
+//
+// Composition root for `src/features/sync.ts`: supplies the real collaborators
+// and renders the result. Planning, prune eligibility, execution order and the
+// stop-on-first-error policy all live in the core, including the
+// "inside a configured root" check.
+//
+// The run stays attached to the process: backgrounding and polling belong to
+// the caller, so there is no daemon or watchdog here.
+import { readFile, stat } from 'node:fs/promises';
+import { resolve, sep } from 'node:path';
+import { SemanticChunker } from '../chunker/index.js';
+import { formatSyncError, runSync as runSyncCore, } from '../features/sync.js';
+import { computeContentHash } from '../ingest/compute.js';
+import { DocumentParser } from '../parser/index.js';
+import { MAX_SCAN_DEPTH } from '../utils/limits.js';
+import { bfsCollectSupportedFiles, canonicalizeRequestedPath, classifyRequestedPath, } from '../utils/scan.js';
+import { createEmbedder, createVectorStore, formatCliError } from './common.js';
+import { buildFileIngestOptions, ingestSingleFile, resolveConfig } from './ingest.js';
+import { consumeBaseDirArg, requireVisualQuality, resolveGlobalConfig } from './options.js';
+// ============================================
+// Help
+// ============================================
+const HELP_TEXT = `Usage: mcp-local-rag [global-options] sync [options] [path]
+
+Reconcile the index with the files on disk: ingest new and changed files, leave
+unchanged files alone, and remove index entries for files that are gone.
+
+Each changed PDF keeps the visual profile it was last indexed with; a new PDF,
+or one indexed before profiles were recorded, stays text-only. Use --visual to
+request VLM captioning for every PDF in scope instead, which also establishes a
+profile for those PDFs. A PDF is re-ingested when its bytes or its profile
+differ from what is indexed, and skipped otherwise. To turn visual mode off
+for a path, run "ingest <path>": a successful normal ingest clears the recorded
+profile. To retry a failed caption, run "ingest <path> --visual
+--visual-quality <profile>" with the profile you want, because a plain ingest
+clears it. Image storage is separate: --images is never recorded and never
+makes a file changed.
+
+Runs in the foreground until it finishes. Use your shell to run it in the
+background.
+
+Arguments:
+  <path>                 File or directory inside a configured base directory
+                         (default: every configured base directory)
+
+Options:
+  --base-dir <path>      Document root (repeatable; overrides environment roots)
+  --images               Store images for new/changed PDF and DOCX files
+  --visual               Request VLM captioning for every PDF in scope, overriding
+                         each PDF's recorded profile (PDFs only)
+  --visual-quality <profile>
+                         VLM profile when --visual is set: fast (default,
+                         lightweight) or quality (Qwen2.5-VL-3B, ~10x cache,
+                         ~2x inference)
+  -h, --help             Show this help
+
+Without --base-dir, roots come from BASE_DIRS / BASE_DIR (default: current directory).
+
+Global options (must appear before "sync"):
+  --db-path <path>       LanceDB database path
+  --cache-dir <path>     Model cache directory
+  --model-name <name>    Embedding model`;
+/**
+ * Parse sync-specific CLI arguments: repeatable roots, at most one positional
+ * path, and -h/--help. Unknown flags still print the usage and exit 1.
+ */
+function parseArgs(args) {
+    let help = false;
+    const baseDirs = [];
+    let path;
+    let images = false;
+    let visual = false;
+    let visualQuality;
+    let index = 0;
+    while (index < args.length) {
+        const arg = args[index] ?? '';
+        switch (arg) {
+            case '-h':
+            case '--help':
+                help = true;
+                index++;
+                break;
+            case '--base-dir': {
+                const valueIndex = consumeBaseDirArg(args, index, baseDirs);
+                index = valueIndex + 1;
+                break;
+            }
+            case '--images':
+                images = true;
+                index++;
+                break;
+            case '--visual':
+                visual = true;
+                index++;
+                break;
+            case '--visual-quality':
+                visualQuality = requireVisualQuality(args, index);
+                index += 2;
+                break;
+            default:
+                if (arg.startsWith('-')) {
+                    console.error(`Unknown option: ${arg}`);
+                    console.error(HELP_TEXT);
+                    process.exit(1);
+                }
+                if (path !== undefined) {
+                    console.error(`Unexpected argument: ${arg}`);
+                    console.error('Only one path is accepted. Omit it to sync every configured base directory.');
+                    process.exit(1);
+                }
+                path = arg;
+                index++;
+                break;
+        }
+    }
+    const parsed = { help, baseDirs, images, visual };
+    if (path !== undefined) {
+        parsed.path = path;
+    }
+    if (visualQuality !== undefined) {
+        parsed.visualQuality = visualQuality;
+    }
+    return parsed;
+}
+// ============================================
+// Reporting
+// ============================================
+/**
+ * Render the scanner's coverage facts as operator warnings. Each unobserved
+ * region is why prune was withheld there, so the path is named; the wording
+ * itself is not a contract.
+ */
+function coverageWarnings(coverage, maxFileSize) {
+    return [
+        ...coverage.unreadableDirs.map(({ dirPath, code }) => `Warning: cannot read directory (${code}), so its indexed files were kept: ${dirPath}`),
+        ...coverage.depthLimitedDirs.map((dirPath) => `Warning: not scanned because it exceeds the maximum depth (${MAX_SCAN_DEPTH}), so its indexed files were kept: ${dirPath}`),
+        ...coverage.skippedSymlinks.map((linkPath) => `Warning: symbolic link not followed, so its indexed files were kept: ${linkPath}`),
+        ...coverage.oversizedFiles.map((filePath) => `Warning: not read because it exceeds the maximum file size (${maxFileSize} bytes), so its indexed chunks were kept: ${filePath}`),
+    ];
+}
+// ============================================
+// Main Entry Point
+// ============================================
+/**
+ * Run the sync CLI subcommand.
+ */
+export async function runSync(args, globalOptions = {}) {
+    const parsed = parseArgs(args);
+    if (parsed.help) {
+        console.error(HELP_TEXT);
+        process.exit(0);
+    }
+    const globalConfig = resolveGlobalConfig(globalOptions);
+    // Shared with `ingest`: CLI roots replace BASE_DIRS / BASE_DIR / cwd; an empty
+    // array preserves that environment fallback. Size/chunk settings share the
+    // same resolution and validation path.
+    const config = await resolveConfig(globalConfig, { baseDirs: parsed.baseDirs });
+    for (const warning of config.baseDirsWarnings) {
+        console.error(warning.message);
+    }
+    const excludePaths = [`${resolve(config.dbPath)}${sep}`, `${resolve(config.cacheDir)}${sep}`];
+    const vectorStore = createVectorStore(globalConfig);
+    // The parser's realpath check is the security boundary, so it takes the
+    // realpath'd roots; scanning uses the resolve()-only `rawBaseDirs` so scanned
+    // paths match the resolve()-stored DB keys (same split as `list`).
+    const parser = new DocumentParser({
+        baseDirs: config.baseDirs.baseDirs,
+        maxFileSize: config.maxFileSize,
+    });
+    const chunker = new SemanticChunker(config.chunkMinLength !== undefined ? { minChunkLength: config.chunkMinLength } : {});
+    // Built on the first upsert only: a run with nothing to ingest must not pay
+    // for the embedding model.
+    let embedder;
+    const ensureEmbedder = () => {
+        embedder ??= createEmbedder(globalConfig);
+        return embedder;
+    };
+    const collaborators = {
+        // The containment boundary for a requested path: the core compares this
+        // canonical form against the realpath'd roots, which is the only way to see
+        // that an intermediate component is a symbolic link out of the root.
+        canonicalizeRequestedPath,
+        // The walker's own predicates, so an explicitly requested path is subject to
+        // the same rules as a discovered one and is refused before it is read.
+        classifyPath: async (path) => await classifyRequestedPath(path, excludePaths),
+        // No `scope` argument, on purpose: a scope-pruned directory appears in none
+        // of the coverage arrays, which would hide an unobserved region and make
+        // prune unsafe.
+        scanDir: async (rootPath) => await bfsCollectSupportedFiles(rootPath, excludePaths, { maxDepth: MAX_SCAN_DEPTH }),
+        // Size first, bytes second: `MAX_FILE_SIZE` is otherwise enforced inside
+        // the parser, long after the whole file is already in memory here.
+        // Declining (`null`) keeps the rest of the run usable rather than aborting
+        // every future sync of the root over one oversized file.
+        //
+        // The bound holds only against a non-racing filesystem: a writer that grows
+        // the file between the `stat` and the `readFile` restores the unbounded
+        // read. That actor already has local write access as this user and can
+        // reach the database directly, so this is a recorded limitation.
+        hashFile: async (filePath) => {
+            if ((await stat(filePath)).size > config.maxFileSize) {
+                return null;
+            }
+            return computeContentHash(await readFile(filePath));
+        },
+        loadDbManifest: async () => await vectorStore.listSyncManifest(),
+        // Named as it happens, so a long run shows which file it is on and the
+        // counters alone are not the only record of what changed. A zero-chunk file
+        // already reports itself from inside `ingestSingleFile`.
+        ingestFile: async (filePath, options) => {
+            const chunkCount = await ingestSingleFile(filePath, { parser, chunker, embedder: ensureEmbedder(), vectorStore }, buildFileIngestOptions(options, globalConfig));
+            if (chunkCount > 0) {
+                console.error(`upserted ${filePath} (${chunkCount} chunks)`);
+            }
+            return chunkCount;
+        },
+        deleteExactPath: async (filePath) => await vectorStore.deleteChunks(filePath),
+        optimize: async () => {
+            await vectorStore.optimize();
+        },
+    };
+    try {
+        await vectorStore.initialize();
+        const result = await runSyncCore({
+            roots: config.baseDirs.rawBaseDirs,
+            // The realpath'd counterpart of the same roots, which is what the core
+            // decides requested-path containment in (the parser's boundary domain).
+            canonicalRoots: config.baseDirs.baseDirs,
+            dbPath: config.dbPath,
+            excludePaths,
+            platform: process.platform,
+            // resolve() (never realpath) so the requested path is spelled like the
+            // stored DB keys; the core validates it against the configured roots.
+            ...(parsed.path === undefined ? {} : { requestedPath: resolve(parsed.path) }),
+            ...(parsed.images ? { images: true } : {}),
+            // Only `--visual` makes the request explicit. Without it the core lets
+            // each PDF inherit its recorded profile, so a bare `--visual-quality` is
+            // parsed (and validated) but has no effect.
+            ...(parsed.visual ? { visualProfile: parsed.visualQuality ?? 'fast' } : {}),
+            collaborators,
+        });
+        for (const prunedPath of result.prunedPaths) {
+            console.error(`pruned ${prunedPath}`);
+        }
+        for (const warning of coverageWarnings(result.coverage, config.maxFileSize)) {
+            console.error(warning);
+        }
+        if (result.error !== null) {
+            console.error(`Error: ${formatSyncError(result.error)}`);
+            process.exitCode = 1;
+            return;
+        }
+        process.stdout.write(JSON.stringify({
+            upserted: result.upserted,
+            skipped: result.skipped,
+            empty: result.empty,
+            pruned: result.pruned,
+        }));
+    }
+    catch (error) {
+        console.error(`Error: ${formatCliError(error)}`);
+        process.exitCode = 1;
+    }
+    finally {
+        if (embedder !== undefined) {
+            await embedder.dispose();
+        }
+        await vectorStore.close();
+    }
+}
+//# sourceMappingURL=sync.js.map

package/dist/cli/sync.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"sync.js","sourceRoot":"","sources":["../../src/cli/sync.ts"],"names":[],"mappings":"AAAA,+DAA+D;AAC/D,EAAE;AACF,+EAA+E;AAC/E,+EAA+E;AAC/E,iEAAiE;AACjE,oCAAoC;AACpC,EAAE;AACF,6EAA6E;AAC7E,sDAAsD;AAEtD,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AACjD,OAAO,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAA;AAExC,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAA;AAGrD,OAAO,EACL,eAAe,EACf,OAAO,IAAI,WAAW,GAIvB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAA;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAA;AACnD,OAAO,EACL,wBAAwB,EACxB,yBAAyB,EACzB,qBAAqB,GACtB,MAAM,kBAAkB,CAAA;AAEzB,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC/E,OAAO,EAAE,sBAAsB,EAAE,gBAAgB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAErF,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,cAAc,CAAA;AAE3F,+CAA+C;AAC/C,OAAO;AACP,+CAA+C;AAE/C,MAAM,SAAS,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yCAuCuB,CAAA;AAgBzC;;;GAGG;AACH,SAAS,SAAS,CAAC,IAAc;IAC/B,IAAI,IAAI,GAAG,KAAK,CAAA;IAChB,MAAM,QAAQ,GAAa,EAAE,CAAA;IAC7B,IAAI,IAAwB,CAAA;IAC5B,IAAI,MAAM,GAAG,KAAK,CAAA;IAClB,IAAI,MAAM,GAAG,KAAK,CAAA;IAClB,IAAI,aAAyC,CAAA;IAE7C,IAAI,KAAK,GAAG,CAAC,CAAA;IACb,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QAC7B,QAAQ,GAAG,EAAE,CAAC;YACZ,KAAK,IAAI,CAAC;YACV,KAAK,QAAQ;gBACX,IAAI,GAAG,IAAI,CAAA;gBACX,KAAK,EAAE,CAAA;gBACP,MAAK;YACP,KAAK,YAAY,EAAE,CAAC;gBAClB,MAAM,UAAU,GAAG,iBAAiB,CAAC,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAA;gBAC3D,KAAK,GAAG,UAAU,GAAG,CAAC,CAAA;gBACtB,MAAK;YACP,CAAC;YACD,KAAK,UAAU;gBACb,MAAM,GAAG,IAAI,CAAA;gBACb,KAAK,EAAE,CAAA;gBACP,MAAK;YACP,KAAK,UAAU;gBACb,MAAM,GAAG,IAAI,CAAA;gBACb,KAAK,EAAE,CAAA;gBACP,MAAK;YACP,KAAK,kBAAkB;gBACrB,aAAa,GAAG,oBAAoB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAA;gBACjD,KAAK,IAAI,CAAC,CAAA;gBACV,MAAK;YACP;gBACE,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;oBACxB,OAAO,CAAC,KAAK,CAAC,mBAAmB,GAAG,EAAE,CAAC,CAAA;oBACvC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;oBACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;oBACvB,OAAO,CAAC,KAAK,CAAC,wBAAwB,GAAG,EAAE,CAAC,CAAA;oBAC5C,OAAO,CAAC,KAAK,CACX,6EAA6E,CAC9E,CAAA;oBACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;gBACjB,CAAC;gBACD,IAAI,GAAG,GAAG,CAAA;gBACV,KAAK,EAAE,CAAA;gBACP,MAAK;QACT,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAa,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,CAAA;IAC3D,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,GAAG,IAAI,CAAA;IACpB,CAAC;IACD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,CAAC,aAAa,GAAG,aAAa,CAAA;IACtC,CAAC;IACD,OAAO,MAAM,CAAA;AACf,CAAC;AAED,+CAA+C;AAC/C,YAAY;AACZ,+CAA+C;AAE/C;;;;GAIG;AACH,SAAS,gBAAgB,CAAC,QAAsB,EAAE,WAAmB;IACnE,OAAO;QACL,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAC5B,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE,EAAE,CACpB,mCAAmC,IAAI,sCAAsC,OAAO,EAAE,CACzF;QACD,GAAG,QAAQ,CAAC,gBAAgB,CAAC,GAAG,CAC9B,CAAC,OAAO,EAAE,EAAE,CACV,8DAA8D,cAAc,sCAAsC,OAAO,EAAE,CAC9H;QACD,GAAG,QAAQ,CAAC,eAAe,CAAC,GAAG,CAC7B,CAAC,QAAQ,EAAE,EAAE,CACX,wEAAwE,QAAQ,EAAE,CACrF;QACD,GAAG,QAAQ,CAAC,cAAc,CAAC,GAAG,CAC5B,CAAC,QAAQ,EAAE,EAAE,CACX,+DAA+D,WAAW,6CAA6C,QAAQ,EAAE,CACpI;KACF,CAAA;AACH,CAAC;AAED,+CAA+C;AAC/C,mBAAmB;AACnB,+CAA+C;AAE/C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO,CAAC,IAAc,EAAE,aAAa,GAAkB,EAAE;IAC7E,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;IAE9B,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;QAChB,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;QACxB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACjB,CAAC;IAED,MAAM,YAAY,GAAG,mBAAmB,CAAC,aAAa,CAAC,CAAA;IACvD,+EAA+E;IAC/E,2EAA2E;IAC3E,uCAAuC;IACvC,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,YAAY,EAAE,EAAE,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAA;IAE/E,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,gBAAgB,EAAE,CAAC;QAC9C,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAA;IAChC,CAAC;IAED,MAAM,YAAY,GAAG,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,CAAC,CAAA;IAC7F,MAAM,WAAW,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IACnD,wEAAwE;IACxE,8EAA8E;IAC9E,mEAAmE;IACnE,MAAM,MAAM,GAAG,IAAI,cAAc,CAAC;QAChC,QAAQ,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ;QAClC,WAAW,EAAE,MAAM,CAAC,WAAW;KAChC,CAAC,CAAA;IACF,MAAM,OAAO,GAAG,IAAI,eAAe,CACjC,MAAM,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,MAAM,CAAC,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,CACrF,CAAA;IAED,4EAA4E;IAC5E,2BAA2B;IAC3B,IAAI,QAA+C,CAAA;IACnD,MAAM,cAAc,GAAG,GAA8B,EAAE;QACrD,QAAQ,KAAK,cAAc,CAAC,YAAY,CAAC,CAAA;QACzC,OAAO,QAAQ,CAAA;IACjB,CAAC,CAAA;IAED,MAAM,aAAa,GAAsB;QACvC,wEAAwE;QACxE,4EAA4E;QAC5E,qEAAqE;QACrE,yBAAyB;QACzB,6EAA6E;QAC7E,uEAAuE;QACvE,YAAY,EAAE,KAAK,EAAE,IAAY,EAAE,EAAE,CAAC,MAAM,qBAAqB,CAAC,IAAI,EAAE,YAAY,CAAC;QACrF,4EAA4E;QAC5E,yEAAyE;QACzE,gBAAgB;QAChB,OAAO,EAAE,KAAK,EAAE,QAAgB,EAAE,EAAE,CAClC,MAAM,wBAAwB,CAAC,QAAQ,EAAE,YAAY,EAAE,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC;QACtF,yEAAyE;QACzE,mEAAmE;QACnE,2EAA2E;QAC3E,yDAAyD;QACzD,EAAE;QACF,4EAA4E;QAC5E,wEAAwE;QACxE,uEAAuE;QACvE,iEAAiE;QACjE,QAAQ,EAAE,KAAK,EAAE,QAAgB,EAAE,EAAE;YACnC,IAAI,CAAC,MAAM,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,GAAG,MAAM,CAAC,WAAW,EAAE,CAAC;gBACrD,OAAO,IAAI,CAAA;YACb,CAAC;YACD,OAAO,kBAAkB,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAA;QACrD,CAAC;QACD,cAAc,EAAE,KAAK,IAAI,EAAE,CAAC,MAAM,WAAW,CAAC,gBAAgB,EAAE;QAChE,uEAAuE;QACvE,4EAA4E;QAC5E,yDAAyD;QACzD,UAAU,EAAE,KAAK,EAAE,QAAgB,EAAE,OAA0B,EAAE,EAAE;YACjE,MAAM,UAAU,GAAG,MAAM,gBAAgB,CACvC,QAAQ,EACR,EAAE,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,EAAE,WAAW,EAAE,EAC5D,sBAAsB,CAAC,OAAO,EAAE,YAAY,CAAC,CAC9C,CAAA;YACD,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;gBACnB,OAAO,CAAC,KAAK,CAAC,YAAY,QAAQ,KAAK,UAAU,UAAU,CAAC,CAAA;YAC9D,CAAC;YACD,OAAO,UAAU,CAAA;QACnB,CAAC;QACD,eAAe,EAAE,KAAK,EAAE,QAAgB,EAAE,EAAE,CAAC,MAAM,WAAW,CAAC,YAAY,CAAC,QAAQ,CAAC;QACrF,QAAQ,EAAE,KAAK,IAAI,EAAE;YACnB,MAAM,WAAW,CAAC,QAAQ,EAAE,CAAA;QAC9B,CAAC;KACF,CAAA;IAED,IAAI,CAAC;QACH,MAAM,WAAW,CAAC,UAAU,EAAE,CAAA;QAE9B,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;YAC/B,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,WAAW;YAClC,uEAAuE;YACvE,wEAAwE;YACxE,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,QAAQ;YACxC,MAAM,EAAE,MAAM,CAAC,MAAM;YACrB,YAAY;YACZ,QAAQ,EAAE,OAAO,CAAC,QAAQ;YAC1B,uEAAuE;YACvE,sEAAsE;YACtE,GAAG,CAAC,MAAM,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC;YAC7E,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1C,uEAAuE;YACvE,yEAAyE;YACzE,4CAA4C;YAC5C,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,aAAa,EAAE,MAAM,CAAC,aAAa,IAAI,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3E,aAAa;SACd,CAAC,CAAA;QAEF,KAAK,MAAM,UAAU,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YAC5C,OAAO,CAAC,KAAK,CAAC,UAAU,UAAU,EAAE,CAAC,CAAA;QACvC,CAAC;QAED,KAAK,MAAM,OAAO,IAAI,gBAAgB,CAAC,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC;YAC5E,OAAO,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;QACxB,CAAC;QAED,IAAI,MAAM,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,CAAC,UAAU,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;YACxD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;YACpB,OAAM;QACR,CAAC;QAED,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,IAAI,CAAC,SAAS,CAAC;YACb,QAAQ,EAAE,MAAM,CAAC,QAAQ;YACzB,OAAO,EAAE,MAAM,CAAC,OAAO;YACvB,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,MAAM,EAAE,MAAM,CAAC,MAAM;SACtB,CAAC,CACH,CAAA;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,OAAO,CAAC,KAAK,CAAC,UAAU,cAAc,CAAC,KAAK,CAAC,EAAE,CAAC,CAAA;QAChD,OAAO,CAAC,QAAQ,GAAG,CAAC,CAAA;IACtB,CAAC;YAAS,CAAC;QACT,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,QAAQ,CAAC,OAAO,EAAE,CAAA;QAC1B,CAAC;QACD,MAAM,WAAW,CAAC,KAAK,EAAE,CAAA;IAC3B,CAAC;AACH,CAAC"}

package/dist/cli-main.d.ts

@@ -0,0 +1,9 @@
+import type { GlobalOptions } from './cli/options.js';
+export declare const SUBCOMMANDS: readonly ['skills', 'ingest', 'list', 'query', 'status', 'delete', 'read-neighbors', 'sync'];
+export type Subcommand = (typeof SUBCOMMANDS)[number];
+/**
+ * Handle CLI subcommands. The caller is expected to have already validated
+ * `subcommand` against `SUBCOMMANDS`; the union type makes the switch exhaustive.
+ */
+export declare function handleCli(subcommand: Subcommand, args: string[], globalOptions?: GlobalOptions): Promise<void>;
+//# sourceMappingURL=cli-main.d.ts.map

package/dist/cli-main.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-main.d.ts","sourceRoot":"","sources":["../src/cli-main.ts"],"names":[],"mappings":"AAKA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAMrD,eAAO,MAAM,WAAW,YACtB,QAAQ,EACR,QAAQ,EACR,MAAM,EACN,OAAO,EACP,QAAQ,EACR,QAAQ,EACR,gBAAgB,EAChB,MAAM,CACE,CAAA;AAEV,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,WAAW,CAAC,CAAC,MAAM,CAAC,CAAA;AAErD;;;GAGG;AACH,wBAAsB,SAAS,CAC7B,UAAU,EAAE,UAAU,EACtB,IAAI,EAAE,MAAM,EAAE,EACd,aAAa,GAAE,aAAkB,GAChC,OAAO,CAAC,IAAI,CAAC,CA2Cf"}

package/dist/cli-main.js

@@ -0,0 +1,60 @@
+// CLI entry point for subcommands (skills install, etc.)
+import { run as runSkillsInstall } from './bin/install-skills.js';
+import { runDelete } from './cli/delete.js';
+import { runIngest } from './cli/ingest.js';
+import { runList } from './cli/list.js';
+import { runQuery } from './cli/query.js';
+import { runReadNeighbors } from './cli/read-neighbors.js';
+import { runStatus } from './cli/status.js';
+import { runSync } from './cli/sync.js';
+export const SUBCOMMANDS = [
+    'skills',
+    'ingest',
+    'list',
+    'query',
+    'status',
+    'delete',
+    'read-neighbors',
+    'sync',
+];
+/**
+ * Handle CLI subcommands. The caller is expected to have already validated
+ * `subcommand` against `SUBCOMMANDS`; the union type makes the switch exhaustive.
+ */
+export async function handleCli(subcommand, args, globalOptions = {}) {
+    switch (subcommand) {
+        case 'skills':
+            if (args[0] === 'install') {
+                runSkillsInstall(args.slice(1));
+                process.exit(0);
+            }
+            else {
+                console.error('Unknown skills subcommand. Usage: npx mcp-local-rag skills install [options]');
+                console.error('Run "npx mcp-local-rag skills install --help" for more information.');
+                process.exit(1);
+            }
+            break;
+        case 'ingest':
+            await runIngest(args, globalOptions);
+            break;
+        case 'list':
+            await runList(args, globalOptions);
+            break;
+        case 'query':
+            await runQuery(args, globalOptions);
+            break;
+        case 'status':
+            await runStatus(args, globalOptions);
+            break;
+        case 'delete':
+            await runDelete(args, globalOptions);
+            break;
+        case 'read-neighbors':
+            await runReadNeighbors(args, globalOptions);
+            break;
+        case 'sync':
+            await runSync(args, globalOptions);
+            break;
+    }
+}
+//# sourceMappingURL=cli-main.js.map

package/dist/cli-main.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"cli-main.js","sourceRoot":"","sources":["../src/cli-main.ts"],"names":[],"mappings":"AAAA,yDAAyD;AACzD,OAAO,EAAE,GAAG,IAAI,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AACjE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAEvC,OAAO,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AACzC,OAAO,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAA;AAC1D,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAC3C,OAAO,EAAE,OAAO,EAAE,MAAM,eAAe,CAAA;AAEvC,MAAM,CAAC,MAAM,WAAW,GAAG;IACzB,QAAQ;IACR,QAAQ;IACR,MAAM;IACN,OAAO;IACP,QAAQ;IACR,QAAQ;IACR,gBAAgB;IAChB,MAAM;CACE,CAAA;AAIV;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,UAAsB,EACtB,IAAc,EACd,aAAa,GAAkB,EAAE;IAEjC,QAAQ,UAAU,EAAE,CAAC;QACnB,KAAK,QAAQ;YACX,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,SAAS,EAAE,CAAC;gBAC1B,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;gBAC/B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,KAAK,CACX,8EAA8E,CAC/E,CAAA;gBACD,OAAO,CAAC,KAAK,CAAC,qEAAqE,CAAC,CAAA;gBACpF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YACjB,CAAC;YACD,MAAK;QAEP,KAAK,QAAQ;YACX,MAAM,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACpC,MAAK;QAEP,KAAK,MAAM;YACT,MAAM,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YAClC,MAAK;QAEP,KAAK,OAAO;YACV,MAAM,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACnC,MAAK;QAEP,KAAK,QAAQ;YACX,MAAM,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACpC,MAAK;QAEP,KAAK,QAAQ;YACX,MAAM,SAAS,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YACpC,MAAK;QAEP,KAAK,gBAAgB;YACnB,MAAM,gBAAgB,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YAC3C,MAAK;QAEP,KAAK,MAAM;YACT,MAAM,OAAO,CAAC,IAAI,EAAE,aAAa,CAAC,CAAA;YAClC,MAAK;IACT,CAAC;AACH,CAAC"}

package/dist/embedder/index.d.ts

@@ -0,0 +1,143 @@
+import { AppError } from '../utils/errors.js';
+/**
+ * Embedder configuration
+ */
+export interface EmbedderConfig {
+    /** HuggingFace model path */
+    modelPath: string;
+    /** Batch size */
+    batchSize: number;
+    /** Model cache directory */
+    cacheDir: string;
+    /** Device type */
+    device?: string;
+    /**
+     * Quantization dtype, passed through to transformers.js with no allowlist.
+     * `undefined` means unset, which `initialize()` resolves to fp32 — the
+     * distinction gates failure-path error enrichment, so keep it.
+     */
+    dtype?: string;
+}
+interface TokenizerOptions {
+    padding?: boolean;
+    truncation?: boolean;
+    return_tensor?: boolean;
+    max_length?: number;
+}
+export interface PipelineTokenizer {
+    (input: string[], options: TokenizerOptions): {
+        input_ids?: unknown;
+    } | null | undefined;
+    /** `unknown` because a model may omit it or report a sentinel; see {@link usableTokenLimit}. */
+    model_max_length?: unknown;
+}
+export interface TokenLimitClamp {
+    /** The effective token cap, or `null` for degraded mode (no cap, no clamp). */
+    tokenLimit: number | null;
+    /** The pre-clamp tokenizer, for measuring true lengths. */
+    measurementTokenizer: PipelineTokenizer | null;
+}
+/**
+ * Bound the pipeline's tokenization length to the model's position window, by
+ * installing a proxy on `candidate`'s tokenizer.
+ *
+ * The pipeline tokenizes with no `max_length`, which resolves to
+ * `model_max_length ?? Infinity` and then to the batch's longest sequence, so a
+ * model whose `tokenizer_config.json` omits a real limit sends more positions
+ * than it has position embeddings and onnxruntime fails in the
+ * position-embedding `Add` node (#202).
+ *
+ * The measurement tokenizer is captured before the proxy, so measurement
+ * reports true lengths. A `Proxy` is used because `model_max_length` is a getter
+ * with no setter. An unrecognized shape, or no usable limit, leaves `candidate`
+ * untouched.
+ */
+export declare function installTokenLimitClamp(candidate: unknown): TokenLimitClamp;
+/**
+ * Embedding generation error
+ */
+export declare class EmbeddingError extends AppError {
+    constructor(message: string, options?: {
+        cause?: Error;
+    });
+}
+/** Transformers.js wrapper: lazily loaded model, batched embedding. */
+export declare class Embedder {
+    private model;
+    private initPromise;
+    /** The resolved cap, or `null` before initialization and in degraded mode. */
+    private tokenLimit;
+    /** The pre-clamp tokenizer, so measurement never reports clamped lengths. */
+    private measurementTokenizer;
+    /**
+     * One-shot flags, per instance: each state belongs to this embedder's model.
+     * They survive `dispose()`; re-initialization is not new information.
+     */
+    private truncationWarned;
+    private degradedModeWarned;
+    private readonly config;
+    constructor(config: EmbedderConfig);
+    /**
+     * Release resources held by the Embedder pipeline
+     */
+    dispose(): Promise<void>;
+    /**
+     * Initialize Transformers.js model
+     */
+    initialize(): Promise<void>;
+    /**
+     * Best-effort failure-path enrichment for an explicit `RAG_DTYPE`: name the
+     * dtypes the model does provide when the requested one is absent.
+     *
+     * The enumeration is a Hub network call in its own try/catch, so an
+     * air-gapped run degrades to a generic dtype-aware message instead of
+     * surfacing a confusing secondary error. Never throws, and never converts
+     * the load failure into a fallback — the caller always re-throws.
+     */
+    private enrichDtypeFailureMessage;
+    /**
+     * Ensure model is initialized (lazy initialization)
+     * This method is called automatically by embed() and embedBatch()
+     */
+    private ensureInitialized;
+    /**
+     * The effective token cap, or `null` when none could be resolved.
+     *
+     * Self-initializing: the value exists only after the model loads, and a
+     * caller may need it before the first embedding.
+     */
+    getTokenLimit(): Promise<number | null>;
+    /**
+     * Report once that no cap was resolved. Warning here rather than in
+     * `initialize()` covers every consumer, which all read the cap through
+     * {@link getTokenLimit}.
+     */
+    private warnDegradedMode;
+    /**
+     * The lengths batch planning runs on: `min(trueLength, cap)`, the length the
+     * clamped pipeline actually feeds the model. Warns once when a true length
+     * exceeds the cap.
+     */
+    private planTokenLengths;
+    /**
+     * The true token length of each text, in input order. Tokenization only, no
+     * inference.
+     *
+     * Measured with `truncation: false` through the pre-clamp tokenizer, so a
+     * length above {@link getTokenLimit} means inference will truncate. Throws
+     * when no such tokenizer was captured, rather than returning a length
+     * measured through an unknown surface.
+     */
+    countTokens(texts: string[]): Promise<number[]>;
+    /**
+     * Single-text embedding; the vector dimension depends on the model.
+     *
+     * Delegates to {@link embedBatch} so this path shares its clamp, measurement
+     * and warning rather than reaching the pipeline unguarded.
+     */
+    embed(text: string): Promise<number[]>;
+    /** Batched embedding; the vector dimension depends on the model. */
+    embedBatch(texts: string[]): Promise<number[][]>;
+}
+export {};
+//# sourceMappingURL=index.d.ts.map

package/dist/embedder/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embedder/index.ts"],"names":[],"mappings":"AASA,OAAO,EAAE,QAAQ,EAAW,MAAM,oBAAoB,CAAA;AAOtD;;GAEG;AACH,MAAM,WAAW,cAAc;IAC7B,6BAA6B;IAC7B,SAAS,EAAE,MAAM,CAAA;IACjB,iBAAiB;IACjB,SAAS,EAAE,MAAM,CAAA;IACjB,4BAA4B;IAC5B,QAAQ,EAAE,MAAM,CAAA;IAChB,kBAAkB;IAClB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf;;;;OAIG;IACH,KAAK,CAAC,EAAE,MAAM,CAAA;CACf;AAQD,UAAU,gBAAgB;IACxB,OAAO,CAAC,EAAE,OAAO,CAAA;IACjB,UAAU,CAAC,EAAE,OAAO,CAAA;IACpB,aAAa,CAAC,EAAE,OAAO,CAAA;IACvB,UAAU,CAAC,EAAE,MAAM,CAAA;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,OAAO,EAAE,gBAAgB,GAAG;QAAE,SAAS,CAAC,EAAE,OAAO,CAAA;KAAE,GAAG,IAAI,GAAG,SAAS,CAAA;IACxF,gGAAgG;IAChG,gBAAgB,CAAC,EAAE,OAAO,CAAA;CAC3B;AA6CD,MAAM,WAAW,eAAe;IAC9B,+EAA+E;IAC/E,UAAU,EAAE,MAAM,GAAG,IAAI,CAAA;IACzB,2DAA2D;IAC3D,oBAAoB,EAAE,iBAAiB,GAAG,IAAI,CAAA;CAC/C;AA4BD;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,sBAAsB,CAAC,SAAS,EAAE,OAAO,GAAG,eAAe,CAyB1E;AAkED;;GAEG;AACH,qBAAa,cAAe,SAAQ,QAAQ;IAC1C,YAAY,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,CAAC,EAAE,KAAK,CAAA;KAAE,EAGvD;CACF;AAMD,uEAAuE;AACvE,qBAAa,QAAQ;IAEnB,OAAO,CAAC,KAAK,CAAgB;IAC7B,OAAO,CAAC,WAAW,CAA6B;IAChD,8EAA8E;IAC9E,OAAO,CAAC,UAAU,CAAsB;IACxC,6EAA6E;IAC7E,OAAO,CAAC,oBAAoB,CAAiC;IAC7D;;;OAGG;IACH,OAAO,CAAC,gBAAgB,CAAiB;IACzC,OAAO,CAAC,kBAAkB,CAAiB;IAC3C,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAgB;IAEvC,YAAY,MAAM,EAAE,cAAc,EAEjC;IAED;;OAEG;IACG,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAc7B;IAED;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAyChC;IAED;;;;;;;;OAQG;YACW,yBAAyB;IAqBvC;;;OAGG;YACW,iBAAiB;IAyB/B;;;;;OAKG;IACG,aAAa,IAAI,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,CAM5C;IAED;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAUxB;;;;OAIG;YACW,gBAAgB;IAoB9B;;;;;;;;OAQG;IACG,WAAW,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAkBpD;IAED;;;;;OAKG;IACG,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAO3C;IAED,oEAAoE;IAC9D,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAyFrD;CACF"}