opencode-rag-plugin 1.12.10 → 1.12.12

package/dist/indexer/git-diff.js

@@ -0,0 +1,52 @@
+import { execSync } from "node:child_process";
+export function getRepoRoot(cwd) {
+    try {
+        return execSync("git rev-parse --show-toplevel", {
+            cwd,
+            encoding: "utf-8",
+            timeout: 5000,
+            stdio: ["ignore", "pipe", "ignore"],
+        }).trim();
+    }
+    catch {
+        return null;
+    }
+}
+export function getCurrentCommit(cwd) {
+    try {
+        return execSync("git rev-parse HEAD", {
+            cwd,
+            encoding: "utf-8",
+            timeout: 5000,
+            stdio: ["ignore", "pipe", "ignore"],
+        }).trim();
+    }
+    catch {
+        return null;
+    }
+}
+export function getChangedFilesSince(cwd, fromCommit) {
+    try {
+        const changedRaw = execSync(`git diff --name-only --diff-filter=ACMRT "${fromCommit}" HEAD`, { cwd, encoding: "utf-8", timeout: 10000, stdio: ["ignore", "pipe", "ignore"] }).trim();
+        const deletedRaw = execSync(`git diff --name-only --diff-filter=D "${fromCommit}" HEAD`, { cwd, encoding: "utf-8", timeout: 10000, stdio: ["ignore", "pipe", "ignore"] }).trim();
+        const changedFiles = changedRaw.length > 0 ? changedRaw.split("\n") : [];
+        const deletedFiles = deletedRaw.length > 0 ? deletedRaw.split("\n") : [];
+        const currentCommit = getCurrentCommit(cwd);
+        if (!currentCommit)
+            return null;
+        return { changedFiles, deletedFiles, currentCommit };
+    }
+    catch {
+        return null;
+    }
+}
+export function getUntrackedFiles(cwd) {
+    try {
+        const raw = execSync("git ls-files --others --exclude-standard", { cwd, encoding: "utf-8", timeout: 5000, stdio: ["ignore", "pipe", "ignore"] }).trim();
+        return raw.length > 0 ? raw.split("\n") : [];
+    }
+    catch {
+        return [];
+    }
+}
+//# sourceMappingURL=git-diff.js.map

package/dist/indexer/git-diff.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"git-diff.js","sourceRoot":"","sources":["../../src/indexer/git-diff.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAQ9C,MAAM,UAAU,WAAW,CAAC,GAAW;IACrC,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,+BAA+B,EAAE;YAC/C,GAAG;YACH,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;SACpC,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,GAAW;IAC1C,IAAI,CAAC;QACH,OAAO,QAAQ,CAAC,oBAAoB,EAAE;YACpC,GAAG;YACH,QAAQ,EAAE,OAAO;YACjB,OAAO,EAAE,IAAI;YACb,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC;SACpC,CAAC,CAAC,IAAI,EAAE,CAAC;IACZ,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,oBAAoB,CAClC,GAAW,EACX,UAAkB;IAElB,IAAI,CAAC;QACH,MAAM,UAAU,GAAG,QAAQ,CACzB,6CAA6C,UAAU,QAAQ,EAC/D,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAChF,CAAC,IAAI,EAAE,CAAC;QAET,MAAM,UAAU,GAAG,QAAQ,CACzB,yCAAyC,UAAU,QAAQ,EAC3D,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAChF,CAAC,IAAI,EAAE,CAAC;QAET,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACzE,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAEzE,MAAM,aAAa,GAAG,gBAAgB,CAAC,GAAG,CAAC,CAAC;QAC5C,IAAI,CAAC,aAAa;YAAE,OAAO,IAAI,CAAC;QAEhC,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,CAAC;IACvD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,MAAM,UAAU,iBAAiB,CAAC,GAAW;IAC3C,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,QAAQ,CAClB,0CAA0C,EAC1C,EAAE,GAAG,EAAE,QAAQ,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CAC/E,CAAC,IAAI,EAAE,CAAC;QACT,OAAO,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}

package/dist/indexer/pipeline.js

@@ -1,8 +1,10 @@
 import pLimit from "p-limit";
 import { scanWorkspaceFiles } from "../content/reader.js";
 import { loadManifest, saveManifest } from "../core/manifest.js";
+import { embedBatch } from "../embedder/factory.js";
 import { createIndexStats } from "./stats.js";
-import { processFile } from "./worker.js";
+import { prepareFile, storeFileChunks } from "./worker.js";
+import { getCurrentCommit, getRepoRoot } from "./git-diff.js";
 export { createWatchPassScheduler, createWatchIgnore } from "./watch.js";
 function createLogger(logger) {
     return {
@@ -11,13 +13,31 @@ function createLogger(logger) {
         debug: logger?.debug ?? (() => { }),
     };
 }
+function tryUpdateLastGitCommit(cwd, manifest) {
+    try {
+        const repoRoot = getRepoRoot(cwd);
+        if (!repoRoot)
+            return false;
+        const commit = getCurrentCommit(repoRoot);
+        if (!commit)
+            return false;
+        manifest.lastGitCommit = commit;
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
 export async function runIndexPass(options) {
     const logger = createLogger(options.logger);
-    const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger);
     const loadResult = await loadManifest(options.storePath);
     const manifest = loadResult.manifest;
     let manifestStatus = loadResult.status;
     let rebuildPerformed = false;
+    if (options.force) {
+        manifestStatus = "missing";
+    }
+    const workspaceFiles = await scanWorkspaceFiles(options.cwd, options.config, logger, options.force ? undefined : manifest);
     const existingCount = await options.store.count();
     if (options.force || (manifestStatus !== "ok" && existingCount > 0)) {
         await options.store.clear();
@@ -44,13 +64,15 @@ export async function runIndexPass(options) {
         }
     }
     const currentPaths = new Set(workspaceFiles.map((file) => file.normalizedPath));
-    for (const indexedPath of Object.keys(manifest.files)) {
-        if (!currentPaths.has(indexedPath)) {
-            await options.store.deleteByFilePath(indexedPath);
-            options.keywordIndex?.removeByFilePath(indexedPath);
-            delete manifest.files[indexedPath];
+    const stalePaths = Object.keys(manifest.files).filter((p) => !currentPaths.has(p));
+    if (stalePaths.length > 0) {
+        const deleteLimit = pLimit(options.config.indexing.concurrency);
+        await Promise.all(stalePaths.map((p) => deleteLimit(async () => {
+            await options.store.deleteByFilePath(p);
+            options.keywordIndex?.removeByFilePath(p);
+            delete manifest.files[p];
             stats.deletedFiles++;
-        }
+        })));
     }
     let newCount = 0;
     let modifiedCount = 0;
@@ -69,26 +91,81 @@ export async function runIndexPass(options) {
             newCount++;
         }
     }
+    if (stats.deletedFiles > 0) {
+        logger.info(`Removed ${stats.deletedFiles} deleted files from index.`);
+    }
     logger.info(`Processing ${workspaceFiles.length} files (${newCount} new, ${modifiedCount} modified, ${unchangedCount} unchanged)...`);
+    // ── Phase 1: prepare all files (chunk + description + keyword + textToEmbed) ──
     const limit = pLimit(options.config.indexing.concurrency);
     let completed = 0;
     const total = workspaceFiles.length;
-    const workerResults = await Promise.all(workspaceFiles.map((file) => limit(async () => {
-        const result = await processFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.store, options.keywordIndex, options.embedder, options.descriptionProvider, logger);
+    const prepared = await Promise.all(workspaceFiles.map((file) => limit(async () => {
+        const prep = await prepareFile(file, options.cwd, manifest.files[file.normalizedPath], options.config, options.keywordIndex, options.descriptionProvider, logger);
         completed++;
         if (total > 20 && completed % 50 === 0) {
-            logger.info(`  Processing file ${completed}/${total}...`);
+            logger.info(`  Preparing file ${completed}/${total}...`);
+        }
+        return prep;
+    })));
+    // ── Handle deletions from store for files being re-processed ──
+    const deletionLimit = pLimit(options.config.indexing.concurrency);
+    await Promise.all(prepared.map((prep) => deletionLimit(async () => {
+        if (prep.earlyResult?.isRemoved) {
+            await options.store.deleteByFilePath(prep.normalizedPath);
+            options.keywordIndex?.removeByFilePath(prep.normalizedPath);
+            return;
+        }
+        if (prep.isModified && prep.chunks && prep.chunks.length > 0) {
+            await options.store.deleteByFilePath(prep.normalizedPath);
+            options.keywordIndex?.removeByFilePath(prep.normalizedPath);
         }
-        return result;
     })));
-    aggregateStats(stats, workerResults, manifest);
+    // ── Phase 2: global embedding batch ──
+    const allTextToEmbed = [];
+    const textPrepIndex = [];
+    for (let i = 0; i < prepared.length; i++) {
+        const prep = prepared[i];
+        if (prep.chunks && prep.textToEmbed && prep.textToEmbed.length > 0) {
+            for (const text of prep.textToEmbed) {
+                allTextToEmbed.push(text);
+                textPrepIndex.push(i);
+            }
+        }
+    }
+    let allEmbeddings = [];
+    if (allTextToEmbed.length > 0) {
+        logger.info(`  Embedding ${allTextToEmbed.length} chunks in global batch...`);
+        allEmbeddings = await embedBatch(options.embedder, allTextToEmbed, options.config.indexing.embedBatchSize, "document");
+    }
+    // ── Distribute embeddings back to prepared files ──
+    const embedByPrep = new Map();
+    for (let i = 0; i < textPrepIndex.length; i++) {
+        const prepIdx = textPrepIndex[i];
+        const emb = allEmbeddings[i];
+        if (!embedByPrep.has(prepIdx)) {
+            embedByPrep.set(prepIdx, []);
+        }
+        if (emb) {
+            embedByPrep.get(prepIdx).push(emb);
+        }
+    }
+    // ── Phase 3: store all files with their embeddings ──
+    const storeLimit = pLimit(options.config.indexing.concurrency);
+    const workerResults = await Promise.all(prepared.map((prep, idx) => storeLimit(async () => {
+        const embeddings = embedByPrep.get(idx) ?? [];
+        return storeFileChunks(prep, embeddings, options.store, logger);
+    })));
+    // ── Aggregate stats and update manifest ──
+    aggregateStats(stats, workerResults, manifest, workspaceFiles);
     manifest.lastIndexedAt = Date.now();
+    tryUpdateLastGitCommit(options.cwd, manifest);
     await saveManifest(options.storePath, manifest);
     await options.keywordIndex?.save(options.storePath);
     stats.finalCount = await options.store.count();
     return stats;
 }
-function aggregateStats(stats, results, manifest) {
+function aggregateStats(stats, results, manifest, workspaceFiles) {
+    const fileMeta = new Map(workspaceFiles.map((f) => [f.normalizedPath, { mtime: f.mtime, size: f.size }]));
     for (const result of results) {
         if (result.isEmpty) {
             stats.skippedEmptyFiles++;
@@ -122,19 +199,30 @@ function aggregateStats(stats, results, manifest) {
             stats.newFiles++;
         }
         if (result.chunkCount > 0) {
+            const meta = fileMeta.get(result.normalizedPath);
             manifest.files[result.normalizedPath] = {
                 hash: result.hash,
                 chunkCount: result.chunkCount,
                 indexedAt: Date.now(),
+                mtime: meta?.mtime,
+                size: meta?.size,
             };
             stats.totalChunks += result.chunkCount;
             stats.batchesFlushed++;
         }
     }
+    for (const { normalizedPath, mtime, size } of workspaceFiles) {
+        const entry = manifest.files[normalizedPath];
+        if (entry && (mtime !== undefined || size !== undefined)) {
+            entry.mtime = mtime;
+            entry.size = size;
+        }
+    }
 }
 export async function getIndexStatusSummary(cwd, storePath, config, store) {
-    const workspaceFiles = await scanWorkspaceFiles(cwd, config);
     const loadResult = await loadManifest(storePath);
+    const manifest = loadResult.manifest;
+    const workspaceFiles = await scanWorkspaceFiles(cwd, config, undefined, manifest);
     const storeCount = await store.count();
     if (loadResult.status !== "ok") {
         return {
@@ -145,7 +233,6 @@ export async function getIndexStatusSummary(cwd, storePath, config, store) {
             rebuildRequired: storeCount > 0,
         };
     }
-    const manifest = loadResult.manifest;
     const currentPaths = new Set(workspaceFiles.map((file) => file.normalizedPath));
     let upToDateFiles = 0;
     let pendingFiles = 0;

package/dist/indexer/pipeline.js.map

@@ -1 +1 @@
-{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/indexer/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAOjE,OAAO,EAAE,gBAAgB,EAA+C,MAAM,YAAY,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAqB,MAAM,aAAa,CAAC;AAG7D,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAoBzE,SAAS,YAAY,CAAC,MAAwB;IAC5C,OAAO;QACL,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;QAChC,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;QAChC,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;KACnC,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,OAAO,CAAC,GAAG,EAAE,OAAO,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACrF,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;IACrC,IAAI,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IACvC,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAE7B,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAClD,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,cAAc,KAAK,IAAI,IAAI,aAAa,GAAG,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAC5B,OAAO,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC;QAC9B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9C,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QACD,QAAQ,CAAC,aAAa,GAAG,SAAS,CAAC;QACnC,gBAAgB,GAAG,aAAa,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC/D,IAAI,cAAc,KAAK,IAAI,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACrE,CAAC;QACD,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC;IAC9D,CAAC;IAED,MAAM,KAAK,GAAG,gBAAgB,CAAC,cAAc,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACtE,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAE1C,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,gBAAgB,KAAK,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAC/D,KAAK,CAAC,kBAAkB,EAAE,CAAC;YAC3B,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK,EAAE,IAAI,CAAC,eAAe;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;IAChF,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,MAAM,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,WAAW,CAAC,CAAC;YAClD,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,WAAW,CAAC,CAAC;YACpD,OAAO,QAAQ,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACnC,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU;YAAE,SAAS;QAC9C,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5C,cAAc,EAAE,CAAC;QACnB,CAAC;aAAM,IAAI,QAAQ,EAAE,CAAC;YACpB,aAAa,EAAE,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,QAAQ,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,cAAc,cAAc,CAAC,MAAM,WAAW,QAAQ,SAAS,aAAa,cAAc,cAAc,gBAAgB,CAAC,CAAC;IAEtI,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC1D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC;IAEpC,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC1B,KAAK,CAAC,KAAK,IAAI,EAAE;QACf,MAAM,MAAM,GAAG,MAAM,WAAW,CAC9B,IAAI,EACJ,OAAO,CAAC,GAAG,EACX,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EACnC,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,KAAK,EACb,OAAO,CAAC,YAAY,EACpB,OAAO,CAAC,QAAQ,EAChB,OAAO,CAAC,mBAAmB,EAC3B,MAAM,CACP,CAAC;QACF,SAAS,EAAE,CAAC;QACZ,IAAI,KAAK,GAAG,EAAE,IAAI,SAAS,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,qBAAqB,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CACH,CACF,CAAC;IAEF,cAAc,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,CAAC,CAAC;IAE/C,QAAQ,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,MAAM,YAAY,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACpD,KAAK,CAAC,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC/C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CACrB,KAAoB,EACpB,OAAuB,EACvB,QAA6F;IAE7F,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBAC7C,KAAK,CAAC,YAAY,EAAE,CAAC;YACvB,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBAC7C,KAAK,CAAC,YAAY,EAAE,CAAC;YACvB,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,KAAK,CAAC,cAAc,EAAE,CAAC;YACvB,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YAC7C,KAAK,CAAC,YAAY,EAAE,CAAC;YACrB,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,KAAK,CAAC,aAAa,EAAE,CAAC;QACxB,CAAC;aAAM,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACxB,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YAC1B,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG;gBACtC,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC;YACF,KAAK,CAAC,WAAW,IAAI,MAAM,CAAC,UAAU,CAAC;YACvC,KAAK,CAAC,cAAc,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,GAAW,EACX,SAAiB,EACjB,MAAiB,EACjB,KAAkB;IAElB,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;IAC7D,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;IACjD,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IAEvC,IAAI,UAAU,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QAC/B,OAAO;YACL,cAAc,EAAE,UAAU,CAAC,MAAM;YACjC,eAAe,EAAE,CAAC;YAClB,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,cAAc,CAAC,MAAM;YACnC,eAAe,EAAE,UAAU,GAAG,CAAC;SAChC,CAAC;IACJ,CAAC;IAED,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;IACrC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;IAChF,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpC,IAAI,QAAQ;gBAAE,YAAY,EAAE,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5C,aAAa,EAAE,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,YAAY,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,YAAY,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,UAAU,CAAC,MAAM;QACjC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM;QACnD,aAAa;QACb,YAAY;QACZ,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,eAAe,EAAE,KAAK;KACvB,CAAC;AACJ,CAAC"}
+{"version":3,"file":"pipeline.js","sourceRoot":"","sources":["../../src/indexer/pipeline.ts"],"names":[],"mappings":"AAAA,OAAO,MAAM,MAAM,SAAS,CAAC;AAC7B,OAAO,EAAE,kBAAkB,EAAE,MAAM,sBAAsB,CAAC;AAE1D,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAOjE,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,gBAAgB,EAA+C,MAAM,YAAY,CAAC;AAC3F,OAAO,EAAE,WAAW,EAAE,eAAe,EAAwC,MAAM,aAAa,CAAC;AACjG,OAAO,EAAE,gBAAgB,EAA2C,WAAW,EAAE,MAAM,eAAe,CAAC;AAGvG,OAAO,EAAE,wBAAwB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAoBzE,SAAS,YAAY,CAAC,MAAwB;IAC5C,OAAO;QACL,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;QAChC,IAAI,EAAE,MAAM,EAAE,IAAI,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;QAChC,KAAK,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;KACnC,CAAC;AACJ,CAAC;AAED,SAAS,sBAAsB,CAAC,GAAW,EAAE,QAAoC;IAC/E,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC;QAClC,IAAI,CAAC,QAAQ;YAAE,OAAO,KAAK,CAAC;QAC5B,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM;YAAE,OAAO,KAAK,CAAC;QAC1B,QAAQ,CAAC,aAAa,GAAG,MAAM,CAAC;QAChC,OAAO,IAAI,CAAC;IACd,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAA4B;IAC7D,MAAM,MAAM,GAAG,YAAY,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;IACrC,IAAI,cAAc,GAAG,UAAU,CAAC,MAAM,CAAC;IACvC,IAAI,gBAAgB,GAAG,KAAK,CAAC;IAE7B,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAClB,cAAc,GAAG,SAAS,CAAC;IAC7B,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAC7C,OAAO,CAAC,GAAG,EACX,OAAO,CAAC,MAAM,EACd,MAAM,EACN,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CACrC,CAAC;IAEF,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAClD,IAAI,OAAO,CAAC,KAAK,IAAI,CAAC,cAAc,KAAK,IAAI,IAAI,aAAa,GAAG,CAAC,CAAC,EAAE,CAAC;QACpE,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;QAC5B,OAAO,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC;QAC9B,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YAC9C,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC7B,CAAC;QACD,QAAQ,CAAC,aAAa,GAAG,SAAS,CAAC;QACnC,gBAAgB,GAAG,aAAa,GAAG,CAAC,IAAI,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC/D,IAAI,cAAc,KAAK,IAAI,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;YACjD,MAAM,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QACrE,CAAC;QACD,cAAc,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,cAAc,CAAC;IAC9D,CAAC;IAED,MAAM,KAAK,GAAG,gBAAgB,CAAC,cAAc,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACtE,KAAK,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;IAE1C,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,IAAI,IAAI,CAAC,gBAAgB,KAAK,QAAQ,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YAC/D,KAAK,CAAC,kBAAkB,EAAE,CAAC;YAC3B,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC;gBAC1B,QAAQ,EAAE,IAAI,CAAC,QAAQ;gBACvB,KAAK,EAAE,IAAI,CAAC,eAAe;aAC5B,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;IAChF,MAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACnF,IAAI,UAAU,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,WAAW,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;QAChE,MAAM,OAAO,CAAC,GAAG,CACf,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CACnB,WAAW,CAAC,KAAK,IAAI,EAAE;YACrB,MAAM,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC;YACxC,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,CAAC,CAAC,CAAC;YAC1C,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YACzB,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,CAAC,CAAC,CACH,CACF,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,cAAc,GAAG,CAAC,CAAC;IACvB,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU;YAAE,SAAS;QAC9C,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5C,cAAc,EAAE,CAAC;QACnB,CAAC;aAAM,IAAI,QAAQ,EAAE,CAAC;YACpB,aAAa,EAAE,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,QAAQ,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,YAAY,GAAG,CAAC,EAAE,CAAC;QAC3B,MAAM,CAAC,IAAI,CAAC,WAAW,KAAK,CAAC,YAAY,4BAA4B,CAAC,CAAC;IACzE,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,cAAc,cAAc,CAAC,MAAM,WAAW,QAAQ,SAAS,aAAa,cAAc,cAAc,gBAAgB,CAAC,CAAC;IAEtI,iFAAiF;IACjF,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC1D,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,MAAM,KAAK,GAAG,cAAc,CAAC,MAAM,CAAC;IAEpC,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAC1B,KAAK,CAAC,KAAK,IAAI,EAAE;QACf,MAAM,IAAI,GAAG,MAAM,WAAW,CAC5B,IAAI,EACJ,OAAO,CAAC,GAAG,EACX,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,EACnC,OAAO,CAAC,MAAM,EACd,OAAO,CAAC,YAAY,EACpB,OAAO,CAAC,mBAAmB,EAC3B,MAAM,CACP,CAAC;QACF,SAAS,EAAE,CAAC;QACZ,IAAI,KAAK,GAAG,EAAE,IAAI,SAAS,GAAG,EAAE,KAAK,CAAC,EAAE,CAAC;YACvC,MAAM,CAAC,IAAI,CAAC,oBAAoB,SAAS,IAAI,KAAK,KAAK,CAAC,CAAC;QAC3D,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CACH,CACF,CAAC;IAEF,iEAAiE;IACjE,MAAM,aAAa,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAClE,MAAM,OAAO,CAAC,GAAG,CACf,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CACpB,aAAa,CAAC,KAAK,IAAI,EAAE;QACvB,IAAI,IAAI,CAAC,WAAW,EAAE,SAAS,EAAE,CAAC;YAChC,MAAM,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC1D,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC5D,OAAO;QACT,CAAC;QACD,IAAI,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC7D,MAAM,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAC1D,OAAO,CAAC,YAAY,EAAE,gBAAgB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC,CAAC,CACH,CACF,CAAC;IAEF,wCAAwC;IACxC,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,MAAM,aAAa,GAAa,EAAE,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACzC,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAE,CAAC;QAC1B,IAAI,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,WAAW,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnE,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;gBACpC,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1B,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,aAAa,GAAe,EAAE,CAAC;IACnC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,eAAe,cAAc,CAAC,MAAM,4BAA4B,CAAC,CAAC;QAC9E,aAAa,GAAG,MAAM,UAAU,CAC9B,OAAO,CAAC,QAAQ,EAChB,cAAc,EACd,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,cAAc,EACtC,UAAU,CACX,CAAC;IACJ,CAAC;IAED,qDAAqD;IACrD,MAAM,WAAW,GAAG,IAAI,GAAG,EAAsB,CAAC;IAClD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,OAAO,GAAG,aAAa,CAAC,CAAC,CAAE,CAAC;QAClC,MAAM,GAAG,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;QAC7B,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,WAAW,CAAC,GAAG,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAC/B,CAAC;QACD,IAAI,GAAG,EAAE,CAAC;YACR,WAAW,CAAC,GAAG,CAAC,OAAO,CAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,uDAAuD;IACvD,MAAM,UAAU,GAAG,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/D,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACrC,QAAQ,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,GAAG,EAAE,EAAE,CACzB,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC;QAC9C,OAAO,eAAe,CAAC,IAAI,EAAE,UAAU,EAAE,OAAO,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC,CAAC,CACH,CACF,CAAC;IAEF,4CAA4C;IAC5C,cAAc,CAAC,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,cAAc,CAAC,CAAC;IAE/D,QAAQ,CAAC,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACpC,sBAAsB,CAAC,OAAO,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC;IAE9C,MAAM,YAAY,CAAC,OAAO,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,OAAO,CAAC,YAAY,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IACpD,KAAK,CAAC,UAAU,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC;IAC/C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,cAAc,CACrB,KAAoB,EACpB,OAAuB,EACvB,QAA4H,EAC5H,cAA2E;IAE3E,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,cAAc,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE1G,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YACnB,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBAC7C,KAAK,CAAC,YAAY,EAAE,CAAC;YACvB,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,KAAK,CAAC,iBAAiB,EAAE,CAAC;YAC1B,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;gBACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;gBAC7C,KAAK,CAAC,YAAY,EAAE,CAAC;YACvB,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,KAAK,CAAC,cAAc,EAAE,CAAC;YACvB,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,SAAS,EAAE,CAAC;YACrB,OAAO,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YAC7C,KAAK,CAAC,YAAY,EAAE,CAAC;YACrB,SAAS;QACX,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;YACtB,KAAK,CAAC,aAAa,EAAE,CAAC;QACxB,CAAC;aAAM,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;YACxB,KAAK,CAAC,QAAQ,EAAE,CAAC;QACnB,CAAC;QACD,IAAI,MAAM,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,GAAG,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;YACjD,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,CAAC,GAAG;gBACtC,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,UAAU,EAAE,MAAM,CAAC,UAAU;gBAC7B,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;gBACrB,KAAK,EAAE,IAAI,EAAE,KAAK;gBAClB,IAAI,EAAE,IAAI,EAAE,IAAI;aACjB,CAAC;YACF,KAAK,CAAC,WAAW,IAAI,MAAM,CAAC,UAAU,CAAC;YACvC,KAAK,CAAC,cAAc,EAAE,CAAC;QACzB,CAAC;IACH,CAAC;IAED,KAAK,MAAM,EAAE,cAAc,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,cAAc,EAAE,CAAC;QAC7D,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;QAC7C,IAAI,KAAK,IAAI,CAAC,KAAK,KAAK,SAAS,IAAI,IAAI,KAAK,SAAS,CAAC,EAAE,CAAC;YACzD,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC;YACpB,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC;QACpB,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,qBAAqB,CACzC,GAAW,EACX,SAAiB,EACjB,MAAiB,EACjB,KAAkB;IAElB,MAAM,UAAU,GAAG,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC;IACjD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,CAAC;IACrC,MAAM,cAAc,GAAG,MAAM,kBAAkB,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,KAAK,EAAE,CAAC;IAEvC,IAAI,UAAU,CAAC,MAAM,KAAK,IAAI,EAAE,CAAC;QAC/B,OAAO;YACL,cAAc,EAAE,UAAU,CAAC,MAAM;YACjC,eAAe,EAAE,CAAC;YAClB,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,cAAc,CAAC,MAAM;YACnC,eAAe,EAAE,UAAU,GAAG,CAAC;SAChC,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC,CAAC;IAChF,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,YAAY,GAAG,CAAC,CAAC;IAErB,KAAK,MAAM,IAAI,IAAI,cAAc,EAAE,CAAC;QAClC,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QACrD,IAAI,IAAI,CAAC,OAAO,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpC,IAAI,QAAQ;gBAAE,YAAY,EAAE,CAAC;YAC7B,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,IAAI,QAAQ,CAAC,IAAI,KAAK,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5C,aAAa,EAAE,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,YAAY,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,KAAK,MAAM,WAAW,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACtD,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,CAAC;YACnC,YAAY,EAAE,CAAC;QACjB,CAAC;IACH,CAAC;IAED,OAAO;QACL,cAAc,EAAE,UAAU,CAAC,MAAM;QACjC,eAAe,EAAE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,MAAM;QACnD,aAAa;QACb,YAAY;QACZ,aAAa,EAAE,QAAQ,CAAC,aAAa;QACrC,eAAe,EAAE,KAAK;KACvB,CAAC;AACJ,CAAC"}

package/dist/indexer/worker.d.ts

@@ -1,5 +1,5 @@
 import type { WorkspaceFile } from "../content/reader.js";
-import type { DescriptionProvider, EmbeddingProvider, KeywordIndex, VectorStore } from "../core/interfaces.js";
+import type { Chunk, DescriptionProvider, EmbeddingProvider, KeywordIndex, VectorStore } from "../core/interfaces.js";
 export interface WorkerResult {
     normalizedPath: string;
     hash: string;
@@ -13,6 +13,15 @@ export interface WorkerResult {
     isRemoved: boolean;
     hadChunks: boolean;
 }
+export interface PreparedFile {
+    normalizedPath: string;
+    hash: string;
+    fileLabel: string;
+    isModified: boolean;
+    earlyResult?: WorkerResult;
+    chunks?: Chunk[];
+    textToEmbed?: string[];
+}
 interface Logger {
     info(message: string): void;
     warn(message: string): void;
@@ -23,6 +32,15 @@ interface ManifestFile {
     chunkCount: number;
     indexedAt?: number;
 }
+export declare function prepareFile(file: WorkspaceFile, cwd: string, previous: ManifestFile | undefined, config: {
+    embedding: {
+        documentPrefix?: string;
+    };
+    chunking?: {
+        nodeTypes?: Record<string, string[]>;
+    };
+}, keywordIndex: KeywordIndex | undefined, descriptionProvider: DescriptionProvider | undefined, logger: Logger): Promise<PreparedFile>;
+export declare function storeFileChunks(prep: PreparedFile, embeddings: number[][], store: VectorStore, logger?: Logger): Promise<WorkerResult>;
 export declare function processFile(file: WorkspaceFile, cwd: string, previous: ManifestFile | undefined, config: {
     embedding: {
         documentPrefix?: string;

package/dist/indexer/worker.js

@@ -4,39 +4,39 @@ import { uuid } from "../chunker/uuid.js";
 import { isImageFile } from "../content/image.js";
 import { buildFileMetadataHeader } from "./metadata.js";
 import { generateDescriptions, buildFallbackDescription } from "./description-stage.js";
-export async function processFile(file, cwd, previous, config, store, keywordIndex, embedder, descriptionProvider, logger) {
+export async function prepareFile(file, cwd, previous, config, keywordIndex, descriptionProvider, logger) {
     const fileLabel = path.relative(cwd, file.filePath);
-    logger.debug(`  processFile: ${fileLabel}`);
     if (file.isEmpty) {
-        if (previous) {
-            await store.deleteByFilePath(file.normalizedPath);
-            keywordIndex?.removeByFilePath(file.normalizedPath);
-            logger.info(`  ${fileLabel} (empty, removed from index)`);
-            return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: true, isTooSmall: false, isRemoved: true, hadChunks: false };
-        }
-        logger.info(`  ${fileLabel} (empty, skipped)`);
-        return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: true, isTooSmall: false, isRemoved: false, hadChunks: false };
+        return {
+            normalizedPath: file.normalizedPath, hash: file.hash, fileLabel,
+            isModified: false,
+            earlyResult: {
+                normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel,
+                isNew: false, isModified: false, isUnchanged: false, isEmpty: true, isTooSmall: false, isRemoved: !!previous, hadChunks: false,
+            },
+        };
     }
     if (file.isTooSmall) {
-        if (previous) {
-            await store.deleteByFilePath(file.normalizedPath);
-            keywordIndex?.removeByFilePath(file.normalizedPath);
-            logger.info(`  ${fileLabel} (too small, removed from index)`);
-            return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: true, isRemoved: true, hadChunks: false };
-        }
-        logger.info(`  ${fileLabel} (too small, skipped)`);
-        return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: true, isRemoved: false, hadChunks: false };
+        return {
+            normalizedPath: file.normalizedPath, hash: file.hash, fileLabel,
+            isModified: false,
+            earlyResult: {
+                normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel,
+                isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: true, isRemoved: !!previous, hadChunks: false,
+            },
+        };
     }
     if (previous && previous.hash === file.hash) {
-        logger.info(`  ${fileLabel} (unchanged)`);
-        return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: false };
-    }
-    let isModified = false;
-    if (previous) {
-        await store.deleteByFilePath(file.normalizedPath);
-        keywordIndex?.removeByFilePath(file.normalizedPath);
-        isModified = true;
+        return {
+            normalizedPath: file.normalizedPath, hash: file.hash, fileLabel,
+            isModified: false,
+            earlyResult: {
+                normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel,
+                isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: false,
+            },
+        };
     }
+    const isModified = !!previous;
     let chunks;
     if (isImageFile(file.filePath) && file.content.trim().length > 0) {
         const imgExt = path.extname(file.filePath).toLowerCase();
@@ -46,11 +46,8 @@ export async function processFile(file, cwd, previous, config, store, keywordInd
                 id: uuid(),
                 content: metaHeader + " " + file.content,
                 metadata: {
-                    filePath: file.filePath,
-                    startLine: 1,
-                    endLine: 1,
-                    language: "image",
-                    contentType: "image",
+                    filePath: file.filePath, startLine: 1, endLine: 1,
+                    language: "image", contentType: "image",
                 },
             }];
     }
@@ -61,92 +58,140 @@ export async function processFile(file, cwd, previous, config, store, keywordInd
         });
     }
     if (chunks === null || chunks.length === 0) {
-        if (chunks === null) {
-            if (previous) {
-                return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: previous.chunkCount, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: true };
-            }
-            return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
+        if (chunks === null && previous) {
+            return {
+                normalizedPath: file.normalizedPath, hash: file.hash, fileLabel,
+                isModified: false,
+                earlyResult: {
+                    normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: previous.chunkCount, fileLabel,
+                    isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: true,
+                },
+            };
         }
-        logger.info(`  ${fileLabel} (0 chunks, removed from index)`);
-        return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
+        return {
+            normalizedPath: file.normalizedPath, hash: file.hash, fileLabel,
+            isModified: false,
+            earlyResult: {
+                normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel,
+                isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false,
+            },
+        };
     }
     keywordIndex?.addChunks(chunks);
-    try {
-        const docPrefix = config.embedding.documentPrefix ?? "";
-        const relPath = path.relative(cwd, file.filePath).replace(/\\/g, "/");
-        const isImage = isImageFile(file.filePath);
-        const metaHeader = isImage
-            ? ""
-            : buildFileMetadataHeader(file.filePath, cwd, file.content);
-        const textToEmbed = [];
-        if (descriptionProvider) {
-            const { descriptionMap } = await generateDescriptions(chunks, descriptionProvider, logger);
-            for (const chunk of chunks) {
-                if (isImage) {
-                    textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
-                }
-                else {
-                    const batchDesc = descriptionMap.get(chunk.id);
-                    if (batchDesc && batchDesc.trim().length > 0) {
-                        textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + batchDesc + "\n\n" + chunk.content);
-                    }
-                    else if (chunk.description) {
-                        textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.description + "\n\n" + chunk.content);
-                    }
-                    else {
-                        textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.content);
-                    }
-                }
+    const docPrefix = config.embedding.documentPrefix ?? "";
+    const relPath = path.relative(cwd, file.filePath).replace(/\\/g, "/");
+    const isImage = isImageFile(file.filePath);
+    const metaHeader = isImage ? "" : buildFileMetadataHeader(file.filePath, cwd, file.content);
+    const textToEmbed = [];
+    if (descriptionProvider) {
+        const { descriptionMap } = await generateDescriptions(chunks, descriptionProvider, logger);
+        for (const chunk of chunks) {
+            if (isImage) {
+                textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
             }
-        }
-        else {
-            for (const chunk of chunks) {
-                if (isImage) {
-                    chunk.description = chunk.content;
-                    textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
+            else {
+                const batchDesc = descriptionMap.get(chunk.id);
+                if (batchDesc && batchDesc.trim().length > 0) {
+                    textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + batchDesc + "\n\n" + chunk.content);
                 }
-                else {
-                    chunk.description = buildFallbackDescription(chunk);
+                else if (chunk.description) {
                     textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.description + "\n\n" + chunk.content);
                 }
+                else {
+                    textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.content);
+                }
             }
         }
-        const { embedBatch } = await import("../embedder/factory.js");
-        const embeddings = await embedBatch(embedder, textToEmbed, config.indexing.embedBatchSize, "document");
-        for (let i = 0; i < chunks.length; i++) {
-            const emb = embeddings[i];
-            if (Array.isArray(emb) && emb.length > 0 && typeof emb[0] === "number") {
-                chunks[i].embedding = emb;
+    }
+    else {
+        for (const chunk of chunks) {
+            if (isImage) {
+                chunk.description = chunk.content;
+                textToEmbed.push(docPrefix + relPath + "\n\n" + chunk.description);
             }
             else {
-                chunks[i].embedding = undefined;
+                chunk.description = buildFallbackDescription(chunk);
+                textToEmbed.push(docPrefix + relPath + "\n\n" + metaHeader + "\n\n" + chunk.description + "\n\n" + chunk.content);
             }
         }
-        const validChunks = chunks.filter((c) => c.embedding && c.embedding.length > 0);
-        if (validChunks.length > 0) {
-            await store.addChunks(validChunks);
+    }
+    return {
+        normalizedPath: file.normalizedPath,
+        hash: file.hash,
+        fileLabel,
+        isModified,
+        chunks,
+        textToEmbed,
+    };
+}
+export async function storeFileChunks(prep, embeddings, store, logger) {
+    if (prep.earlyResult)
+        return prep.earlyResult;
+    if (!prep.chunks || !prep.textToEmbed) {
+        return {
+            normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0, fileLabel: prep.fileLabel,
+            isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false,
+        };
+    }
+    for (let i = 0; i < prep.chunks.length; i++) {
+        const emb = embeddings[i];
+        if (Array.isArray(emb) && emb.length > 0 && typeof emb[0] === "number") {
+            prep.chunks[i].embedding = emb;
+        }
+        else {
+            prep.chunks[i].embedding = undefined;
         }
-        logger.info(`  ${fileLabel} (${chunks.length} chunks${isModified ? ", modified" : ", new"})`);
+    }
+    const validChunks = prep.chunks.filter((c) => c.embedding && c.embedding.length > 0);
+    if (validChunks.length > 0) {
+        await store.addChunks(validChunks);
+    }
+    logger?.info(`  ${prep.fileLabel} (${prep.chunks.length} chunks${prep.isModified ? ", modified" : ", new"})`);
+    return {
+        normalizedPath: prep.normalizedPath,
+        hash: prep.hash,
+        chunkCount: prep.chunks.length,
+        fileLabel: prep.fileLabel,
+        isNew: !prep.isModified,
+        isModified: prep.isModified,
+        isUnchanged: false,
+        isEmpty: false,
+        isTooSmall: false,
+        isRemoved: false,
+        hadChunks: prep.chunks.length > 0,
+    };
+}
+export async function processFile(file, cwd, previous, config, store, keywordIndex, embedder, descriptionProvider, logger) {
+    const prep = await prepareFile(file, cwd, previous, config, keywordIndex, descriptionProvider, logger);
+    if (prep.earlyResult) {
+        if ((prep.earlyResult.isEmpty || prep.earlyResult.isTooSmall) && prep.earlyResult.isRemoved) {
+            await store.deleteByFilePath(prep.normalizedPath);
+            keywordIndex?.removeByFilePath(prep.normalizedPath);
+        }
+        return prep.earlyResult;
+    }
+    if (prep.isModified) {
+        await store.deleteByFilePath(prep.normalizedPath);
+        keywordIndex?.removeByFilePath(prep.normalizedPath);
+    }
+    if (!prep.textToEmbed || prep.textToEmbed.length === 0) {
         return {
-            normalizedPath: file.normalizedPath,
-            hash: file.hash,
-            chunkCount: chunks.length,
-            fileLabel,
-            isNew: !isModified && !previous,
-            isModified,
-            isUnchanged: false,
-            isEmpty: false,
-            isTooSmall: false,
-            isRemoved: false,
-            hadChunks: chunks.length > 0,
+            normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0, fileLabel: prep.fileLabel,
+            isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: false,
         };
     }
+    try {
+        const { embedBatch } = await import("../embedder/factory.js");
+        const embeddings = await embedBatch(embedder, prep.textToEmbed, config.indexing.embedBatchSize, "document");
+        const result = await storeFileChunks(prep, embeddings, store);
+        return result;
+    }
     catch (err) {
-        logger.warn(`  ${fileLabel} (embed/store failed: ${err.message})`);
-        if (previous) {
-            return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: previous.chunkCount, fileLabel, isNew: false, isModified: false, isUnchanged: true, isEmpty: false, isTooSmall: false, isRemoved: false, hadChunks: true };
-        }
-        return { normalizedPath: file.normalizedPath, hash: file.hash, chunkCount: 0, fileLabel, isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false };
+        logger.warn(`  ${prep.fileLabel} (embed/store failed: ${err.message})`);
+        return {
+            normalizedPath: prep.normalizedPath, hash: prep.hash, chunkCount: 0, fileLabel: prep.fileLabel,
+            isNew: false, isModified: false, isUnchanged: false, isEmpty: false, isTooSmall: false, isRemoved: true, hadChunks: false,
+        };
     }
 }
 //# sourceMappingURL=worker.js.map