opencode-rag-plugin 1.12.5 → 1.12.9

package/dist/indexer/pipeline.js

@@ -0,0 +1,179 @@
+import pLimit from "p-limit";
+import { scanWorkspaceFiles } from "../content/reader.js";
+import { loadManifest, saveManifest } from "../core/manifest.js";
+import { createIndexStats } from "./stats.js";
+import { processFile } from "./worker.js";
+export { createWatchPassScheduler, createWatchIgnore } from "./watch.js";
+function createLogger(logger) {
+    return {
+        info: logger?.info ?? (() => { }),
+        warn: logger?.warn ?? (() => { }),
+    };
+}
+export async function runIndexPass(options) {
+    const logger = createLogger(options.logger);
+    const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger);
+    const loadResult = await loadManifest(options.storePath);
+    const manifest = loadResult.manifest;
+    let manifestStatus = loadResult.status;
+    let rebuildPerformed = false;
+    const existingCount = await options.store.count();
+    if (options.force || (manifestStatus !== "ok" && existingCount > 0)) {
+        await options.store.clear();
+        options.keywordIndex?.clear();
+        for (const key of Object.keys(manifest.files)) {
+            delete manifest.files[key];
+        }
+        manifest.lastIndexedAt = undefined;
+        rebuildPerformed = existingCount > 0 || Boolean(options.force);
+        if (manifestStatus !== "ok" && existingCount > 0) {
+            logger.warn("Manifest missing or corrupt; rebuilding full index.");
+        }
+        manifestStatus = options.force ? "missing" : manifestStatus;
+    }
+    const stats = createIndexStats(workspaceFiles.length, manifestStatus);
+    stats.rebuildPerformed = rebuildPerformed;
+    for (const file of workspaceFiles) {
+        if (file.extractionStatus === "failed" && file.extractionError) {
+            stats.extractionFailures++;
+            stats.extractionErrors.push({
+                filePath: file.filePath,
+                error: file.extractionError,
+            });
+        }
+    }
+    const currentPaths = new Set(workspaceFiles.map((file) => file.normalizedPath));
+    for (const indexedPath of Object.keys(manifest.files)) {
+        if (!currentPaths.has(indexedPath)) {
+            await options.store.deleteByFilePath(indexedPath);
+            options.keywordIndex?.removeByFilePath(indexedPath);
+            delete manifest.files[indexedPath];
+            stats.deletedFiles++;
+        }
+    }
+    let newCount = 0;
+    let modifiedCount = 0;
+    let unchangedCount = 0;
+    for (const file of workspaceFiles) {
+        const previous = manifest.files[file.normalizedPath];
+        if (file.isEmpty || file.isTooSmall)
+            continue;
+        if (previous && previous.hash === file.hash) {
+            unchangedCount++;
+        }
+        else if (previous) {
+            modifiedCount++;
+        }
+        else {
+            newCount++;
+        }
+    }
+    logger.info(`Processing ${workspaceFiles.length} files (${newCount} new, ${modifiedCount} modified, ${unchangedCount} unchanged)...`);
+    const limit = pLimit(options.config.indexing.concurrency);
+    let completed = 0;
+    const total = workspaceFiles.length;
+    const workerResults = await Promise.all(workspaceFiles.map((file) => limit(async () => {
+        const result = await processFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.store, options.keywordIndex, options.embedder, options.descriptionProvider, logger);
+        completed++;
+        if (total > 20 && completed % 50 === 0) {
+            logger.info(`  Processing file ${completed}/${total}...`);
+        }
+        return result;
+    })));
+    aggregateStats(stats, workerResults, manifest);
+    manifest.lastIndexedAt = Date.now();
+    await saveManifest(options.storePath, manifest);
+    await options.keywordIndex?.save(options.storePath);
+    stats.finalCount = await options.store.count();
+    return stats;
+}
+function aggregateStats(stats, results, manifest) {
+    for (const result of results) {
+        if (result.isEmpty) {
+            stats.skippedEmptyFiles++;
+            if (result.isRemoved) {
+                delete manifest.files[result.normalizedPath];
+                stats.removedFiles++;
+            }
+            continue;
+        }
+        if (result.isTooSmall) {
+            stats.skippedSmallFiles++;
+            if (result.isRemoved) {
+                delete manifest.files[result.normalizedPath];
+                stats.removedFiles++;
+            }
+            continue;
+        }
+        if (result.isUnchanged) {
+            stats.unchangedFiles++;
+            continue;
+        }
+        if (result.isRemoved) {
+            delete manifest.files[result.normalizedPath];
+            stats.removedFiles++;
+            continue;
+        }
+        if (result.isModified) {
+            stats.modifiedFiles++;
+        }
+        else if (result.isNew) {
+            stats.newFiles++;
+        }
+        if (result.chunkCount > 0) {
+            manifest.files[result.normalizedPath] = {
+                hash: result.hash,
+                chunkCount: result.chunkCount,
+                indexedAt: Date.now(),
+            };
+            stats.totalChunks += result.chunkCount;
+            stats.batchesFlushed++;
+        }
+    }
+}
+export async function getIndexStatusSummary(cwd, storePath, config, store) {
+    const workspaceFiles = await scanWorkspaceFiles(cwd, config);
+    const loadResult = await loadManifest(storePath);
+    const storeCount = await store.count();
+    if (loadResult.status !== "ok") {
+        return {
+            manifestStatus: loadResult.status,
+            manifestEntries: 0,
+            upToDateFiles: 0,
+            pendingFiles: workspaceFiles.length,
+            rebuildRequired: storeCount > 0,
+        };
+    }
+    const manifest = loadResult.manifest;
+    const currentPaths = new Set(workspaceFiles.map((file) => file.normalizedPath));
+    let upToDateFiles = 0;
+    let pendingFiles = 0;
+    for (const file of workspaceFiles) {
+        const previous = manifest.files[file.normalizedPath];
+        if (file.isEmpty || file.isTooSmall) {
+            if (previous)
+                pendingFiles++;
+            continue;
+        }
+        if (previous && previous.hash === file.hash) {
+            upToDateFiles++;
+        }
+        else {
+            pendingFiles++;
+        }
+    }
+    for (const indexedPath of Object.keys(manifest.files)) {
+        if (!currentPaths.has(indexedPath)) {
+            pendingFiles++;
+        }
+    }
+    return {
+        manifestStatus: loadResult.status,
+        manifestEntries: Object.keys(manifest.files).length,
+        upToDateFiles,
+        pendingFiles,
+        lastIndexedAt: manifest.lastIndexedAt,
+        rebuildRequired: false,
+    };
+}
+//# sourceMappingURL=pipeline.js.map

package/dist/indexer/pipeline.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/indexer/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAOjE,OAAO,EAAE,gBAAgB,EAA+C,MAAM,YAAY,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAqB,MAAM,aAAa,CAAC;AAG7D,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAmBzE,SAAS,YAAY,CAAC,MAAwB;IAC5C,OAAO;QACL,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;QAChC,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;KACjC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrF,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;IACrC,IAAI,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IACvC,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAE7B,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAClD,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,cAAc,KAAK,IAAI,IAAI,aAAa,GAAG,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAC5B,OAAO,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC;QAC9B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9C,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QACD,QAAQ,CAAC,aAAa,GAAG,SAAS,CAAC;QACnC,gBAAgB,GAAG,aAAa,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC/D,IAAI,cAAc,KAAK,IAAI,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACrE,CAAC;QACD,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC;IAC9D,CAAC;IAED,MAAM,KAAK,GAAG,gBAAgB,CAAC,cAAc,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACtE,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAE1C,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,gBAAgB,KAAK,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAC/D,KAAK,CAAC,kBAAkB,EAAE,CAAC;YAC3B,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK,EAAE,IAAI,CAAC,eAAe;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;IAChF,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YAClD,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACpD,OAAO,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACnC,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU;YAAE,SAAS;QAC9C,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5C,cAAc,EAAE,CAAC;QACnB,CAAC;aAAM,IAAI,QAAQ,EAAE,CAAC;YACpB,aAAa,EAAE,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,QAAQ,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,cAAc,cAAc,CAAC,MAAM,WAAW,QAAQ,SAAS,aAAa,cAAc,cAAc,gBAAgB,CAAC,CAAC;IAEtI,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC1D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC;IAEpC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC1B,KAAK,CAAC,KAAK,IAAI,EAAE;QACf,MAAM,MAAM,GAAG,MAAM,WAAW,CAC9B,IAAI,EACJ,OAAO,CAAC,GAAG,EACX,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EACnC,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,YAAY,EACpB,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,mBAAmB,EAC3B,MAAM,CACP,CAAC;QACF,SAAS,EAAE,CAAC;QACZ,IAAI,KAAK,GAAG,EAAE,IAAI,SAAS,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,qBAAqB,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CACH,CACF,CAAC;IAEF,cAAc,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;IAE/C,QAAQ,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,YAAY,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACpD,KAAK,CAAC,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC/C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CACrB,KAAoB,EACpB,OAAuB,EACvB,QAA6F;IAE7F,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBAC7C,KAAK,CAAC,YAAY,EAAE,CAAC;YACvB,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBAC7C,KAAK,CAAC,YAAY,EAAE,CAAC;YACvB,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,KAAK,CAAC,cAAc,EAAE,CAAC;YACvB,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YAC7C,KAAK,CAAC,YAAY,EAAE,CAAC;YACrB,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,KAAK,CAAC,aAAa,EAAE,CAAC;QACxB,CAAC;aAAM,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACxB,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG;gBACtC,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC;YACF,KAAK,CAAC,WAAW,IAAI,MAAM,CAAC,UAAU,CAAC;YACvC,KAAK,CAAC,cAAc,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,GAAW,EACX,SAAiB,EACjB,MAAiB,EACjB,KAAkB;IAElB,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IAEvC,IAAI,UAAU,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QAC/B,OAAO;YACL,cAAc,EAAE,UAAU,CAAC,MAAM;YACjC,eAAe,EAAE,CAAC;YAClB,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,cAAc,CAAC,MAAM;YACnC,eAAe,EAAE,UAAU,GAAG,CAAC;SAChC,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;IACrC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;IAChF,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpC,IAAI,QAAQ;gBAAE,YAAY,EAAE,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5C,aAAa,EAAE,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,YAAY,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,YAAY,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,UAAU,CAAC,MAAM;QACjC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM;QACnD,aAAa;QACb,YAAY;QACZ,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,eAAe,EAAE,KAAK;KACvB,CAAC;AACJ,CAAC"}

package/dist/indexer/stats.d.ts

@@ -0,0 +1,29 @@
+export interface IndexRunStats {
+    totalFiles: number;
+    newFiles: number;
+    modifiedFiles: number;
+    unchangedFiles: number;
+    deletedFiles: number;
+    removedFiles: number;
+    skippedEmptyFiles: number;
+    skippedSmallFiles: number;
+    totalChunks: number;
+    finalCount: number;
+    manifestStatus: "ok" | "missing" | "corrupt";
+    rebuildPerformed: boolean;
+    batchesFlushed: number;
+    extractionFailures: number;
+    extractionErrors: Array<{
+        filePath: string;
+        error: string;
+    }>;
+}
+export interface IndexStatusSummary {
+    manifestStatus: "ok" | "missing" | "corrupt";
+    manifestEntries: number;
+    upToDateFiles: number;
+    pendingFiles: number;
+    lastIndexedAt?: number;
+    rebuildRequired: boolean;
+}
+export declare function createIndexStats(totalFiles: number, manifestStatus: IndexRunStats["manifestStatus"]): IndexRunStats;

package/dist/indexer/stats.js

@@ -0,0 +1,20 @@
+export function createIndexStats(totalFiles, manifestStatus) {
+    return {
+        totalFiles,
+        newFiles: 0,
+        modifiedFiles: 0,
+        unchangedFiles: 0,
+        deletedFiles: 0,
+        removedFiles: 0,
+        skippedEmptyFiles: 0,
+        skippedSmallFiles: 0,
+        totalChunks: 0,
+        finalCount: 0,
+        manifestStatus,
+        rebuildPerformed: false,
+        batchesFlushed: 0,
+        extractionFailures: 0,
+        extractionErrors: [],
+    };
+}
+//# sourceMappingURL=stats.js.map

package/dist/indexer/stats.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"stats.js","sourceRoot":"","sources":["../../src/indexer/stats.ts"],"names":[],"mappings":"AA2BA,MAAM,UAAU,gBAAgB,CAC9B,UAAkB,EAClB,cAA+C;IAE/C,OAAO;QACL,UAAU;QACV,QAAQ,EAAE,CAAC;QACX,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;QACjB,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;QACf,iBAAiB,EAAE,CAAC;QACpB,iBAAiB,EAAE,CAAC;QACpB,WAAW,EAAE,CAAC;QACd,UAAU,EAAE,CAAC;QACb,cAAc;QACd,gBAAgB,EAAE,KAAK;QACvB,cAAc,EAAE,CAAC;QACjB,kBAAkB,EAAE,CAAC;QACrB,gBAAgB,EAAE,EAAE;KACrB,CAAC;AACJ,CAAC"}

package/dist/indexer/watch.d.ts

@@ -0,0 +1,8 @@
+import type { RagConfig } from "../core/config.js";
+export interface WatchPassScheduler {
+    notifyChange(): void;
+    waitForIdle(): Promise<void>;
+    close(): void;
+}
+export declare function createWatchPassScheduler(runPass: () => Promise<void>, onError: (error: unknown) => void, debounceMs?: number): WatchPassScheduler;
+export declare function createWatchIgnore(cwd: string, config: RagConfig, storePath: string): (watchedPath: string) => boolean;

package/dist/indexer/watch.js

@@ -0,0 +1,95 @@
+import path from "node:path";
+import { manifestPathFor } from "../core/manifest.js";
+export function createWatchPassScheduler(runPass, onError, debounceMs = 300) {
+    let timer = null;
+    let running = false;
+    let rerunRequested = false;
+    let closed = false;
+    const waiters = [];
+    function resolveWaiters() {
+        if (running || timer || rerunRequested)
+            return;
+        while (waiters.length > 0) {
+            waiters.shift()?.();
+        }
+    }
+    function schedule() {
+        if (closed)
+            return;
+        if (timer)
+            clearTimeout(timer);
+        timer = setTimeout(() => {
+            timer = null;
+            void execute();
+        }, debounceMs);
+    }
+    async function execute() {
+        if (closed)
+            return;
+        if (running) {
+            rerunRequested = true;
+            return;
+        }
+        running = true;
+        try {
+            await runPass();
+        }
+        catch (error) {
+            onError(error);
+        }
+        finally {
+            running = false;
+            if (rerunRequested) {
+                rerunRequested = false;
+                schedule();
+            }
+            else {
+                resolveWaiters();
+            }
+        }
+    }
+    return {
+        notifyChange() {
+            if (closed)
+                return;
+            if (running) {
+                rerunRequested = true;
+                return;
+            }
+            schedule();
+        },
+        waitForIdle() {
+            if (!running && !timer && !rerunRequested) {
+                return Promise.resolve();
+            }
+            return new Promise((resolve) => {
+                waiters.push(resolve);
+            });
+        },
+        close() {
+            closed = true;
+            if (timer) {
+                clearTimeout(timer);
+                timer = null;
+            }
+            resolveWaiters();
+        },
+    };
+}
+export function createWatchIgnore(cwd, config, storePath) {
+    const manifestPath = manifestPathFor(storePath);
+    const excludeDirs = new Set(config.indexing.excludeDirs);
+    return (watchedPath) => {
+        const resolved = path.resolve(watchedPath);
+        if (resolved.startsWith(storePath))
+            return true;
+        if (resolved === manifestPath)
+            return true;
+        const relative = path.relative(cwd, resolved);
+        if (!relative || relative.startsWith(".."))
+            return false;
+        const segments = relative.split(path.sep);
+        return segments.some((segment) => excludeDirs.has(segment));
+    };
+}
+//# sourceMappingURL=watch.js.map

package/dist/indexer/watch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"watch.js","sourceRoot":"","sources":["../../src/indexer/watch.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAStD,MAAM,UAAU,wBAAwB,CACtC,OAA4B,EAC5B,OAAiC,EACjC,aAAqB,GAAG;IAExB,IAAI,KAAK,GAA0B,IAAI,CAAC;IACxC,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,IAAI,cAAc,GAAG,KAAK,CAAC;IAC3B,IAAI,MAAM,GAAG,KAAK,CAAC;IACnB,MAAM,OAAO,GAAsB,EAAE,CAAC;IAEtC,SAAS,cAAc;QACrB,IAAI,OAAO,IAAI,KAAK,IAAI,cAAc;YAAE,OAAO;QAC/C,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,EAAE,EAAE,EAAE,CAAC;QACtB,CAAC;IACH,CAAC;IAED,SAAS,QAAQ;QACf,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,KAAK;YAAE,YAAY,CAAC,KAAK,CAAC,CAAC;QAC/B,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;YACtB,KAAK,GAAG,IAAI,CAAC;YACb,KAAK,OAAO,EAAE,CAAC;QACjB,CAAC,EAAE,UAAU,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,UAAU,OAAO;QACpB,IAAI,MAAM;YAAE,OAAO;QACnB,IAAI,OAAO,EAAE,CAAC;YACZ,cAAc,GAAG,IAAI,CAAC;YACtB,OAAO;QACT,CAAC;QAED,OAAO,GAAG,IAAI,CAAC;QACf,IAAI,CAAC;YACH,MAAM,OAAO,EAAE,CAAC;QAClB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC;gBAAS,CAAC;YACT,OAAO,GAAG,KAAK,CAAC;YAChB,IAAI,cAAc,EAAE,CAAC;gBACnB,cAAc,GAAG,KAAK,CAAC;gBACvB,QAAQ,EAAE,CAAC;YACb,CAAC;iBAAM,CAAC;gBACN,cAAc,EAAE,CAAC;YACnB,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,YAAY;YACV,IAAI,MAAM;gBAAE,OAAO;YACnB,IAAI,OAAO,EAAE,CAAC;gBACZ,cAAc,GAAG,IAAI,CAAC;gBACtB,OAAO;YACT,CAAC;YACD,QAAQ,EAAE,CAAC;QACb,CAAC;QACD,WAAW;YACT,IAAI,CAAC,OAAO,IAAI,CAAC,KAAK,IAAI,CAAC,cAAc,EAAE,CAAC;gBAC1C,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;YAC3B,CAAC;YACD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;gBAC7B,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC,CAAC,CAAC;QACL,CAAC;QACD,KAAK;YACH,MAAM,GAAG,IAAI,CAAC;YACd,IAAI,KAAK,EAAE,CAAC;gBACV,YAAY,CAAC,KAAK,CAAC,CAAC;gBACpB,KAAK,GAAG,IAAI,CAAC;YACf,CAAC;YACD,cAAc,EAAE,CAAC;QACnB,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,GAAW,EACX,MAAiB,EACjB,SAAiB;IAEjB,MAAM,YAAY,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAEzD,OAAO,CAAC,WAAmB,EAAW,EAAE;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC;QAC3C,IAAI,QAAQ,CAAC,UAAU,CAAC,SAAS,CAAC;YAAE,OAAO,IAAI,CAAC;QAChD,IAAI,QAAQ,KAAK,YAAY;YAAE,OAAO,IAAI,CAAC;QAE3C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;QAC9C,IAAI,CAAC,QAAQ,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC;YAAE,OAAO,KAAK,CAAC;QACzD,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1C,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC;IAC9D,CAAC,CAAC;AACJ,CAAC"}

package/dist/indexer/worker.d.ts

@@ -0,0 +1,36 @@
+import type { WorkspaceFile } from "../content/reader.js";
+import type { DescriptionProvider, EmbeddingProvider, KeywordIndex, VectorStore } from "../core/interfaces.js";
+export interface WorkerResult {
+    normalizedPath: string;
+    hash: string;
+    chunkCount: number;
+    fileLabel: string;
+    isNew: boolean;
+    isModified: boolean;
+    isUnchanged: boolean;
+    isEmpty: boolean;
+    isTooSmall: boolean;
+    isRemoved: boolean;
+    hadChunks: boolean;
+}
+interface Logger {
+    info(message: string): void;
+    warn(message: string): void;
+}
+interface ManifestFile {
+    hash: string;
+    chunkCount: number;
+    indexedAt?: number;
+}
+export declare function processFile(file: WorkspaceFile, cwd: string, previous: ManifestFile | undefined, config: {
+    embedding: {
+        documentPrefix?: string;
+    };
+    indexing: {
+        embedBatchSize: number;
+    };
+    chunking?: {
+        nodeTypes?: Record<string, string[]>;
+    };
+}, store: VectorStore, keywordIndex: KeywordIndex | undefined, embedder: EmbeddingProvider, descriptionProvider: DescriptionProvider | undefined, logger: Logger): Promise<WorkerResult>;
+export {};

package/dist/indexer/worker.js

@@ -0,0 +1,151 @@
+import path from "node:path";
+import { chunkFile } from "../chunker/factory.js";
+import { uuid } from "../chunker/uuid.js";
+import { isImageFile } from "../content/image.js";
+import { buildFileMetadataHeader } from "./metadata.js";
+import { generateDescriptions, buildFallbackDescription } from "./description-stage.js";
+export async function processFile(file, cwd, previous, config, store, keywordIndex, embedder, descriptionProvider, logger) {
+    const fileLabel = path.relative(cwd, file.filePath);
+    if (file.isEmpty) {
+        if (previous) {
+            await store.deleteByFilePath(file.normalizedPath);
+            keywordIndex?.removeByFilePath(file.normalizedPath);
+            logger.info(`  ${fileLabel} (empty, removed from index)`);
+            return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: true, isTooSmall: false, isRemoved: true, hadChunks: false };
+        }
+        logger.info(`  ${fileLabel} (empty, skipped)`);
+        return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: true, isTooSmall: false, isRemoved: false, hadChunks: false };
+    }
+    if (file.isTooSmall) {
+        if (previous) {
+            await store.deleteByFilePath(file.normalizedPath);
+            keywordIndex?.removeByFilePath(file.normalizedPath);
+            logger.info(`  ${fileLabel} (too small, removed from index)`);
+            return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: true, isRemoved: true, hadChunks: false };
+        }
+        logger.info(`  ${fileLabel} (too small, skipped)`);
+        return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: true, isRemoved: false, hadChunks: false };
+    }
+    if (previous && previous.hash === file.hash) {
+        logger.info(`  ${fileLabel} (unchanged)`);
+        return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: false };
+    }
+    let isModified = false;
+    if (previous) {
+        await store.deleteByFilePath(file.normalizedPath);
+        keywordIndex?.removeByFilePath(file.normalizedPath);
+        isModified = true;
+    }
+    let chunks;
+    if (isImageFile(file.filePath) && file.content.trim().length > 0) {
+        const imgExt = path.extname(file.filePath).toLowerCase();
+        const imgRelPath = path.relative(cwd, file.filePath).replace(/\\/g, "/");
+        const metaHeader = `[image] [${imgExt.slice(1)}] [${imgRelPath}]`;
+        chunks = [{
+                id: uuid(),
+                content: metaHeader + " " + file.content,
+                metadata: {
+                    filePath: file.filePath,
+                    startLine: 1,
+                    endLine: 1,
+                    language: "image",
+                    contentType: "image",
+                },
+            }];
+    }
+    else {
+        chunks = await chunkFile(file.filePath, file.content, config.chunking?.nodeTypes).catch((err) => {
+            logger.warn(`  ${fileLabel} (chunking failed: ${err.message})`);
+            return null;
+        });
+    }
+    if (chunks === null || chunks.length === 0) {
+        if (chunks === null) {
+            if (previous) {
+                return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: previous.chunkCount, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: true };
+            }
+            return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
+        }
+        logger.info(`  ${fileLabel} (0 chunks, removed from index)`);
+        return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
+    }
+    keywordIndex?.addChunks(chunks);
+    try {
+        const docPrefix = config.embedding.documentPrefix ?? "";
+        const relPath = path.relative(cwd, file.filePath).replace(/\\/g, "/");
+        const isImage = isImageFile(file.filePath);
+        const metaHeader = isImage
+            ? ""
+            : buildFileMetadataHeader(file.filePath, cwd, file.content);
+        const textToEmbed = [];
+        if (descriptionProvider) {
+            const { descriptionMap } = await generateDescriptions(chunks, descriptionProvider, logger);
+            for (const chunk of chunks) {
+                if (isImage) {
+                    textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
+                }
+                else {
+                    const batchDesc = descriptionMap.get(chunk.id);
+                    if (batchDesc && batchDesc.trim().length > 0) {
+                        textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + batchDesc + "\n\n" + chunk.content);
+                    }
+                    else if (chunk.description) {
+                        textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.description + "\n\n" + chunk.content);
+                    }
+                    else {
+                        textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.content);
+                    }
+                }
+            }
+        }
+        else {
+            for (const chunk of chunks) {
+                if (isImage) {
+                    chunk.description = chunk.content;
+                    textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
+                }
+                else {
+                    chunk.description = buildFallbackDescription(chunk);
+                    textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.description + "\n\n" + chunk.content);
+                }
+            }
+        }
+        const { embedBatch } = await import("../embedder/factory.js");
+        const embeddings = await embedBatch(embedder, textToEmbed, config.indexing.embedBatchSize, "document");
+        for (let i = 0; i < chunks.length; i++) {
+            const emb = embeddings[i];
+            if (Array.isArray(emb) && emb.length > 0 && typeof emb[0] === "number") {
+                chunks[i].embedding = emb;
+            }
+            else {
+                chunks[i].embedding = undefined;
+            }
+        }
+        const validChunks = chunks.filter((c) => c.embedding && c.embedding.length > 0);
+        if (validChunks.length > 0) {
+            await store.addChunks(validChunks);
+        }
+        logger.info(`  ${fileLabel} (${chunks.length} chunks${isModified ? ", modified" : ", new"})`);
+        return {
+            normalizedPath: file.normalizedPath,
+            hash: file.hash,
+            chunkCount: chunks.length,
+            fileLabel,
+            isNew: !isModified && !previous,
+            isModified,
+            isUnchanged: false,
+            isEmpty: false,
+            isTooSmall: false,
+            isRemoved: false,
+            hadChunks: chunks.length > 0,
+        };
+    }
+    catch (err) {
+        logger.warn(`  ${fileLabel} (embed/store failed: ${err.message})`);
+        if (previous) {
+            return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: previous.chunkCount, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: true };
+        }
+        return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
+    }
+}
+//# sourceMappingURL=worker.js.map

package/dist/indexer/worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.js","sourceRoot":"","sources":["../../src/indexer/worker.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAQlD,OAAO,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAC;AACxD,OAAO,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,MAAM,wBAAwB,CAAC;AA2BxF,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,IAAmB,EACnB,GAAW,EACX,QAAkC,EAClC,MAIC,EACD,KAAkB,EAClB,YAAsC,EACtC,QAA2B,EAC3B,mBAAoD,EACpD,MAAc;IAEd,MAAM,SAAS,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC;IAEpD,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;QACjB,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClD,YAAY,EAAE,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,8BAA8B,CAAC,CAAC;YAC1D,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QACtN,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,mBAAmB,CAAC,CAAC;QAC/C,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACvN,CAAC;IAED,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;QACpB,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAClD,YAAY,EAAE,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YACpD,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,kCAAkC,CAAC,CAAC;YAC9D,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QACtN,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,uBAAuB,CAAC,CAAC;QACnD,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACvN,CAAC;IAED,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;QAC5C,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,cAAc,CAAC,CAAC;QAC1C,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACvN,CAAC;IAED,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAClD,YAAY,EAAE,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACpD,UAAU,GAAG,IAAI,CAAC;IACpB,CAAC;IAED,IAAI,MAAM,CAAC;IAEX,IAAI,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACjE,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,CAAC;QACzD,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACzE,MAAM,UAAU,GAAG,YAAY,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,UAAU,GAAG,CAAC;QAClE,MAAM,GAAG,CAAC;gBACR,EAAE,EAAE,IAAI,EAAE;gBACV,OAAO,EAAE,UAAU,GAAG,GAAG,GAAG,IAAI,CAAC,OAAO;gBACxC,QAAQ,EAAE;oBACR,QAAQ,EAAE,IAAI,CAAC,QAAQ;oBACvB,SAAS,EAAE,CAAC;oBACZ,OAAO,EAAE,CAAC;oBACV,QAAQ,EAAE,OAAO;oBACjB,WAAW,EAAE,OAAO;iBACrB;aACF,CAAC,CAAC;IACL,CAAC;SAAM,CAAC;QACN,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,CAAC,QAAQ,EAAE,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YAC9F,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,sBAAuB,GAAa,CAAC,OAAO,GAAG,CAAC,CAAC;YAC3E,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IAED,IAAI,MAAM,KAAK,IAAI,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;YACpB,IAAI,QAAQ,EAAE,CAAC;gBACb,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;YACxO,CAAC;YACD,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;QACvN,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,iCAAiC,CAAC,CAAC;QAC7D,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACvN,CAAC;IAED,YAAY,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;IAEhC,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC,cAAc,IAAI,EAAE,CAAC;QACxD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACtE,MAAM,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,UAAU,GAAG,OAAO;YACxB,CAAC,CAAC,EAAE;YACJ,CAAC,CAAC,uBAAuB,CAAC,IAAI,CAAC,QAAQ,EAAE,GAAG,EAAE,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9D,MAAM,WAAW,GAAa,EAAE,CAAC;QAEjC,IAAI,mBAAmB,EAAE,CAAC;YACxB,MAAM,EAAE,cAAc,EAAE,GAAG,MAAM,oBAAoB,CAAC,MAAM,EAAE,mBAAmB,EAAE,MAAM,CAAC,CAAC;YAE3F,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,OAAO,EAAE,CAAC;oBACZ,WAAW,CAAC,IAAI,CAAC,SAAS,GAAG,OAAO,GAAG,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;gBACrE,CAAC;qBAAM,CAAC;oBACN,MAAM,SAAS,GAAG,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBAC/C,IAAI,SAAS,IAAI,SAAS,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC7C,WAAW,CAAC,IAAI,CAAC,SAAS,GAAG,OAAO,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;oBAC5G,CAAC;yBAAM,IAAI,KAAK,CAAC,WAAW,EAAE,CAAC;wBAC7B,WAAW,CAAC,IAAI,CAAC,SAAS,GAAG,OAAO,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,CAAC,WAAW,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;oBACpH,CAAC;yBAAM,CAAC;wBACN,WAAW,CAAC,IAAI,CAAC,SAAS,GAAG,OAAO,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;oBACvF,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,OAAO,EAAE,CAAC;oBACZ,KAAK,CAAC,WAAW,GAAG,KAAK,CAAC,OAAO,CAAC;oBAClC,WAAW,CAAC,IAAI,CAAC,SAAS,GAAG,OAAO,GAAG,MAAM,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC;gBACrE,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,WAAW,GAAG,wBAAwB,CAAC,KAAK,CAAC,CAAC;oBACpD,WAAW,CAAC,IAAI,CAAC,SAAS,GAAG,OAAO,GAAG,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,KAAK,CAAC,WAAW,GAAG,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC;gBACpH,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;QAC9D,MAAM,UAAU,GAAG,MAAM,UAAU,CAAC,QAAQ,EAAE,WAAW,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc,EAAE,UAAU,CAAC,CAAC;QAEvG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACvC,MAAM,GAAG,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,GAAG,CAAC,CAAC,CAAC,KAAK,QAAQ,EAAE,CAAC;gBACvE,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,GAAG,GAAe,CAAC;YACzC,CAAC;iBAAM,CAAC;gBACN,MAAM,CAAC,CAAC,CAAE,CAAC,SAAS,GAAG,SAAS,CAAC;YACnC,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAChF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC;QACrC,CAAC;QAED,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,KAAK,MAAM,CAAC,MAAM,UAAU,UAAU,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,CAAC;QAE9F,OAAO;YACL,cAAc,EAAE,IAAI,CAAC,cAAc;YACnC,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,UAAU,EAAE,MAAM,CAAC,MAAM;YACzB,SAAS;YACT,KAAK,EAAE,CAAC,UAAU,IAAI,CAAC,QAAQ;YAC/B,UAAU;YACV,WAAW,EAAE,KAAK;YAClB,OAAO,EAAE,KAAK;YACd,UAAU,EAAE,KAAK;YACjB,SAAS,EAAE,KAAK;YAChB,SAAS,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,KAAK,SAAS,yBAA0B,GAAa,CAAC,OAAO,GAAG,CAAC,CAAC;QAC9E,IAAI,QAAQ,EAAE,CAAC;YACb,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,QAAQ,CAAC,UAAU,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;QACxO,CAAC;QACD,OAAO,EAAE,cAAc,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,KAAK,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;IACvN,CAAC;AACH,CAAC"}

package/dist/indexer.d.ts

@@ -1,60 +1,13 @@
+export type { IndexRunStats, IndexStatusSummary } from "./indexer/stats.js";
+export { createIndexStats } from "./indexer/stats.js";
+export type { WorkspaceFile } from "./content/reader.js";
+export { scanWorkspaceFiles, walkFiles } from "./content/reader.js";
+export type { RunIndexPassOptions, WatchPassScheduler } from "./indexer/pipeline.js";
+export { runIndexPass, getIndexStatusSummary, createWatchPassScheduler, createWatchIgnore, } from "./indexer/pipeline.js";
 import type { RagConfig } from "./core/config.js";
-import type { DescriptionProvider, EmbeddingProvider, KeywordIndex, VectorStore } from "./core/interfaces.js";
-export interface IndexRunStats {
-    totalFiles: number;
-    newFiles: number;
-    modifiedFiles: number;
-    unchangedFiles: number;
-    deletedFiles: number;
-    removedFiles: number;
-    skippedEmptyFiles: number;
-    skippedSmallFiles: number;
-    totalChunks: number;
-    finalCount: number;
-    manifestStatus: "ok" | "missing" | "corrupt";
-    rebuildPerformed: boolean;
-    batchesFlushed: number;
-}
-export interface IndexStatusSummary {
-    manifestStatus: "ok" | "missing" | "corrupt";
-    manifestEntries: number;
-    upToDateFiles: number;
-    pendingFiles: number;
-    lastIndexedAt?: number;
-    rebuildRequired: boolean;
-}
-export interface WorkspaceFile {
-    filePath: string;
-    normalizedPath: string;
-    content: string;
-    hash: string;
-    isEmpty: boolean;
-    isTooSmall: boolean;
-}
+import type { WorkspaceFile } from "./content/reader.js";
 interface Logger {
     info(message: string): void;
     warn(message: string): void;
 }
-export interface RunIndexPassOptions {
-    cwd: string;
-    storePath: string;
-    config: RagConfig;
-    store: VectorStore;
-    embedder: EmbeddingProvider;
-    force?: boolean;
-    logger?: Partial<Logger>;
-    keywordIndex?: KeywordIndex;
-    descriptionProvider?: DescriptionProvider;
-}
-export interface WatchPassScheduler {
-    notifyChange(): void;
-    waitForIdle(): Promise<void>;
-    close(): void;
-}
-export declare function walkFiles(dir: string, extensions: Set<string>, excludeDirs: Set<string>): Promise<string[]>;
 export declare function scanWorkspace(cwd: string, config: RagConfig, logger?: Logger): Promise<WorkspaceFile[]>;
-export declare function runIndexPass(options: RunIndexPassOptions): Promise<IndexRunStats>;
-export declare function getIndexStatusSummary(cwd: string, storePath: string, config: RagConfig, store: VectorStore): Promise<IndexStatusSummary>;
-export declare function createWatchPassScheduler(runPass: () => Promise<void>, onError: (error: unknown) => void, debounceMs?: number): WatchPassScheduler;
-export declare function createWatchIgnore(cwd: string, config: RagConfig, storePath: string): (watchedPath: string) => boolean;
-export {};