indexer-cli 0.2.1

package/dist/engine/indexer.js

@@ -0,0 +1,933 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.IndexerEngine = exports.createDefaultLanguagePlugins = exports.DEFAULT_LANGUAGE_PLUGIN_IDS = void 0;
+const node_crypto_1 = require("node:crypto");
+const promises_1 = require("node:fs/promises");
+const node_path_1 = require("node:path");
+const plugin_js_1 = require("../languages/plugin.js");
+const typescript_js_1 = require("../languages/typescript.js");
+const python_js_1 = require("../languages/python.js");
+const csharp_js_1 = require("../languages/csharp.js");
+const gdscript_js_1 = require("../languages/gdscript.js");
+const ruby_js_1 = require("../languages/ruby.js");
+const logger_js_1 = require("../core/logger.js");
+const config_js_1 = require("../core/config.js");
+const token_estimator_js_1 = require("../utils/token-estimator.js");
+const hash_js_1 = require("../utils/hash.js");
+const adaptive_js_1 = require("../chunking/adaptive.js");
+const architecture_js_1 = require("./architecture.js");
+const dependency_resolver_js_1 = require("./dependency-resolver.js");
+const scanner_js_1 = require("./scanner.js");
+const logger = new logger_js_1.SystemLogger("indexer-engine");
+exports.DEFAULT_LANGUAGE_PLUGIN_IDS = [
+    "typescript",
+    "python",
+    "csharp",
+    "gdscript",
+    "ruby",
+];
+const BUILTIN_LANGUAGE_PLUGIN_FACTORIES = {
+    typescript: () => new typescript_js_1.TypeScriptPlugin(),
+    python: () => new python_js_1.PythonPlugin(),
+    csharp: () => new csharp_js_1.CSharpPlugin(),
+    gdscript: () => new gdscript_js_1.GDScriptPlugin(),
+    ruby: () => new ruby_js_1.RubyPlugin(),
+};
+function normalizeImportKind(kind) {
+    if (kind === "require")
+        return "require";
+    if (kind === "dynamic_import")
+        return "dynamic_import";
+    return "import";
+}
+const createDefaultLanguagePlugins = (pluginIds) => {
+    const ids = pluginIds && pluginIds.length > 0 ? pluginIds : exports.DEFAULT_LANGUAGE_PLUGIN_IDS;
+    const normalizedIds = Array.from(new Set(ids
+        .map((value) => value.trim().toLowerCase())
+        .filter((value) => value.length > 0)));
+    return normalizedIds.map((id) => {
+        if (!(id in BUILTIN_LANGUAGE_PLUGIN_FACTORIES)) {
+            throw new Error(`Unsupported language plugin id: ${id}`);
+        }
+        return BUILTIN_LANGUAGE_PLUGIN_FACTORIES[id]();
+    });
+};
+exports.createDefaultLanguagePlugins = createDefaultLanguagePlugins;
+class IndexerEngine {
+    projectId;
+    repoRoot;
+    metadata;
+    vectors;
+    embedder;
+    git;
+    languagePluginRegistry = new plugin_js_1.LanguagePluginRegistry();
+    indexingOptions;
+    churnByFile = new Map();
+    tokenEstimator = new token_estimator_js_1.TokenEstimator();
+    chunker = new adaptive_js_1.AdaptiveChunker();
+    architectureGenerator;
+    constructor(options) {
+        this.projectId = options.projectId;
+        this.repoRoot = options.repoRoot;
+        this.metadata = options.metadata;
+        this.vectors = options.vectors;
+        this.embedder = options.embedder;
+        this.git = options.git;
+        const languagePlugins = options.languagePlugins ?? (0, exports.createDefaultLanguagePlugins)();
+        this.languagePluginRegistry.registerMany(languagePlugins);
+        const pluginCodeExtensions = Array.from(new Set(this.languagePluginRegistry
+            .list()
+            .flatMap((plugin) => plugin.fileExtensions.map((ext) => ext.toLowerCase()))));
+        this.indexingOptions = {
+            codeExtensions: options.indexingOptions?.codeExtensions &&
+                options.indexingOptions.codeExtensions.length > 0
+                ? options.indexingOptions.codeExtensions.map((ext) => ext.toLowerCase())
+                : pluginCodeExtensions,
+            skipImportChunksInVectors: options.indexingOptions?.skipImportChunksInVectors ?? false,
+        };
+        this.architectureGenerator = new architecture_js_1.ArchitectureGenerator(this.metadata, this.languagePluginRegistry.list());
+    }
+    get metadataWithProgress() {
+        return this.metadata;
+    }
+    getConfiguredOllamaNumCtx() {
+        if (config_js_1.config.get("embeddingProvider") !== "ollama") {
+            return null;
+        }
+        const numCtx = config_js_1.config.get("ollamaNumCtx");
+        return Number.isFinite(numCtx) && numCtx > 0 ? Math.floor(numCtx) : null;
+    }
+    isOllamaContextLengthError(error) {
+        const message = error instanceof Error ? error.message : String(error ?? "");
+        const normalized = message.toLowerCase();
+        return (normalized.includes("input length exceeds the context length") ||
+            (normalized.includes("context length") &&
+                normalized.includes("status=400")));
+    }
+    trimToTokenBudget(content, tokenBudget) {
+        if (tokenBudget <= 0 || content.length === 0) {
+            return content;
+        }
+        let trimmed = content;
+        let estimatedTokens = this.tokenEstimator.estimate(trimmed);
+        if (estimatedTokens <= tokenBudget) {
+            return trimmed;
+        }
+        const initialRatio = Math.min(1, tokenBudget / Math.max(estimatedTokens, 1));
+        const initialLength = Math.max(32, Math.floor(trimmed.length * initialRatio * 0.95));
+        trimmed = trimmed.slice(0, initialLength);
+        estimatedTokens = this.tokenEstimator.estimate(trimmed);
+        while (estimatedTokens > tokenBudget && trimmed.length > 32) {
+            const nextLength = Math.max(32, Math.floor(trimmed.length * 0.85));
+            if (nextLength >= trimmed.length) {
+                break;
+            }
+            trimmed = trimmed.slice(0, nextLength);
+            estimatedTokens = this.tokenEstimator.estimate(trimmed);
+        }
+        return trimmed;
+    }
+    async embedWithContextGuard(contents, operation) {
+        try {
+            return await this.embedder.embed(contents);
+        }
+        catch (error) {
+            const ollamaNumCtx = this.getConfiguredOllamaNumCtx();
+            if (!ollamaNumCtx || !this.isOllamaContextLengthError(error)) {
+                throw error;
+            }
+            const retryWithBudget = async (tokenBudget, passName) => {
+                const sizeDetails = contents.map((content, index) => {
+                    const originalTokens = this.tokenEstimator.estimate(content);
+                    const trimmedContent = this.trimToTokenBudget(content, tokenBudget);
+                    const trimmedTokens = this.tokenEstimator.estimate(trimmedContent);
+                    return {
+                        index,
+                        content: trimmedContent,
+                        originalChars: content.length,
+                        originalBytes: Buffer.byteLength(content, "utf8"),
+                        originalTokens,
+                        trimmedChars: trimmedContent.length,
+                        trimmedBytes: Buffer.byteLength(trimmedContent, "utf8"),
+                        trimmedTokens,
+                    };
+                });
+                logger.warn(`Embedding context overflow during ${operation}`, {
+                    operation,
+                    passName,
+                    embeddingModel: config_js_1.config.get("embeddingModel"),
+                    numCtx: ollamaNumCtx,
+                    tokenBudget,
+                    itemCount: sizeDetails.length,
+                    trimmedCount: sizeDetails.filter((item) => item.trimmedChars < item.originalChars).length,
+                    sizes: sizeDetails.map(({ content: _content, ...detail }) => detail),
+                });
+                return this.embedder.embed(sizeDetails.map((item) => item.content));
+            };
+            try {
+                return await retryWithBudget(Math.max(16, Math.floor(ollamaNumCtx * 0.9)), "first");
+            }
+            catch (retryError) {
+                if (!this.isOllamaContextLengthError(retryError)) {
+                    throw retryError;
+                }
+                return retryWithBudget(Math.max(16, Math.floor(ollamaNumCtx * 0.75)), "second");
+            }
+        }
+    }
+    shouldIndexChunkInVectors(chunk) {
+        return (!this.indexingOptions.skipImportChunksInVectors ||
+            chunk.chunkType !== "imports");
+    }
+    normalizeChunkType(value, chunkContent, primarySymbol) {
+        if (typeof value === "string") {
+            const normalized = value.trim().toLowerCase();
+            if (normalized === "imports")
+                return "imports";
+            if (normalized === "types")
+                return "types";
+            if (normalized === "impl")
+                return "impl";
+            if (normalized === "preamble")
+                return "preamble";
+            if (normalized === "declaration")
+                return "declaration";
+            if (normalized === "module_section")
+                return "module_section";
+            if (normalized === "full_file")
+                return "full_file";
+        }
+        if (primarySymbol) {
+            return "impl";
+        }
+        const trimmed = chunkContent.trim();
+        if (!trimmed)
+            return "full_file";
+        const lines = trimmed
+            .split(/\r?\n/)
+            .map((line) => line.trim())
+            .filter(Boolean);
+        if (lines.length > 0 &&
+            lines.filter((line) => /^(import\b|from\s+\S+\s+import\b|using\s+\S+\s*;|export\b.+\bfrom\b|const\s+.+\s*=\s*require\()/i.test(line)).length /
+                lines.length >=
+                0.8) {
+            return "imports";
+        }
+        if (/(^|\s)(interface|type|enum|protocol|trait|typedef|delegate)\b/i.test(trimmed)) {
+            return "types";
+        }
+        return "impl";
+    }
+    inferPrimarySymbol(chunk, symbolRecords) {
+        const explicitPrimary = typeof chunk.metadata?.primarySymbol === "string"
+            ? String(chunk.metadata.primarySymbol).trim()
+            : undefined;
+        if (explicitPrimary) {
+            return explicitPrimary;
+        }
+        const inChunk = symbolRecords.filter((symbol) => symbol.range.start.line >= chunk.range.startLine &&
+            symbol.range.start.line <= chunk.range.endLine);
+        if (inChunk.length === 0) {
+            return undefined;
+        }
+        const priority = new Map([
+            ["function", 5],
+            ["method", 5],
+            ["class", 4],
+            ["interface", 3],
+            ["type", 2],
+            ["variable", 1],
+        ]);
+        return inChunk.sort((a, b) => {
+            const aPriority = priority.get(a.kind) ?? 0;
+            const bPriority = priority.get(b.kind) ?? 0;
+            if (aPriority !== bPriority) {
+                return bPriority - aPriority;
+            }
+            return a.range.start.line - b.range.start.line;
+        })[0]?.name;
+    }
+    splitHeuristicChunks(content, languageId) {
+        const lines = content.split(/\r?\n/);
+        if (lines.length === 0) {
+            return [];
+        }
+        const isPython = languageId === "python";
+        const isCSharp = languageId === "csharp";
+        const isGDScript = languageId === "gdscript";
+        const isRuby = languageId === "ruby";
+        const isJSImport = languageId === "typescript" || languageId === "javascript";
+        if (!isPython && !isCSharp && !isGDScript && !isRuby && !isJSImport) {
+            return [];
+        }
+        const chunks = [];
+        const importPattern = isPython
+            ? /^(import\s+\S+|from\s+\S+\s+import\s+)/
+            : isCSharp
+                ? /^using\s+[^;]+;/
+                : isGDScript
+                    ? /^(extends\s+\S+|class_name\s+\S+|const\s+\S+\s*=\s*preload\(|var\s+\S+\s*=\s*preload\()/
+                    : isRuby
+                        ? /^(require|require_relative|include|extend)\b/
+                        : /^(import\s+|export\s+.+\s+from\s+|const\s+.+\s*=\s*require\()/;
+        let importEnd = 0;
+        for (let index = 0; index < lines.length; index += 1) {
+            const trimmed = lines[index].trim();
+            if (trimmed === "" ||
+                trimmed.startsWith("//") ||
+                trimmed.startsWith("#") ||
+                trimmed.startsWith("/*") ||
+                importPattern.test(trimmed)) {
+                if (importPattern.test(trimmed)) {
+                    importEnd = index + 1;
+                }
+                continue;
+            }
+            break;
+        }
+        if (importEnd > 0) {
+            const importContent = lines.slice(0, importEnd).join("\n").trim();
+            if (importContent) {
+                chunks.push({
+                    content: importContent,
+                    startLine: 1,
+                    endLine: importEnd,
+                    chunkType: "imports",
+                });
+            }
+        }
+        const definitions = [];
+        const pythonDefPattern = /^(?:async\s+def|def|class)\s+([A-Za-z_][A-Za-z0-9_]*)/;
+        const csharpTypePattern = /\b(class|interface|enum|struct|record)\s+([A-Za-z_][A-Za-z0-9_]*)/;
+        const csharpMethodPattern = /\b(?:public|private|protected|internal|static|virtual|override|sealed|partial|async|extern|unsafe|abstract|new|readonly|ref|out|in)+\s+[A-Za-z_][A-Za-z0-9_<>,\[\]?]*\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/;
+        const gdClassPattern = /^(?:class_name|class)\s+([A-Za-z_][A-Za-z0-9_]*)/;
+        const gdFuncPattern = /^(?:static\s+)?func\s+([A-Za-z_][A-Za-z0-9_]*)\s*\(/;
+        const rubyTypePattern = /^(?:class|module)\s+([A-Z][A-Za-z0-9_:]*)/;
+        const rubyMethodPattern = /^def\s+(?:self\.)?([A-Za-z_][A-Za-z0-9_]*[!?=]?)/;
+        const jsDefPattern = /^(?:export\s+)?(?:async\s+)?(?:function|class|const|let|var)\s+([A-Za-z_][A-Za-z0-9_]*)/;
+        for (let index = importEnd; index < lines.length; index += 1) {
+            const line = lines[index].trim();
+            if (!line)
+                continue;
+            if (isPython) {
+                const match = line.match(pythonDefPattern);
+                if (match) {
+                    definitions.push({
+                        line: index + 1,
+                        chunkType: /^class\b/.test(line) ? "types" : "impl",
+                        primarySymbol: match[1],
+                    });
+                }
+                continue;
+            }
+            if (isCSharp) {
+                const typeMatch = line.match(csharpTypePattern);
+                if (typeMatch) {
+                    definitions.push({
+                        line: index + 1,
+                        chunkType: typeMatch[1] === "interface" || typeMatch[1] === "enum"
+                            ? "types"
+                            : "impl",
+                        primarySymbol: typeMatch[2],
+                    });
+                    continue;
+                }
+                const methodMatch = line.match(csharpMethodPattern);
+                if (methodMatch) {
+                    definitions.push({
+                        line: index + 1,
+                        chunkType: "impl",
+                        primarySymbol: methodMatch[1],
+                    });
+                }
+                continue;
+            }
+            if (isGDScript) {
+                const classMatch = line.match(gdClassPattern);
+                if (classMatch) {
+                    definitions.push({
+                        line: index + 1,
+                        chunkType: "types",
+                        primarySymbol: classMatch[1],
+                    });
+                    continue;
+                }
+                const funcMatch = line.match(gdFuncPattern);
+                if (funcMatch) {
+                    definitions.push({
+                        line: index + 1,
+                        chunkType: "impl",
+                        primarySymbol: funcMatch[1],
+                    });
+                }
+                continue;
+            }
+            if (isRuby) {
+                const typeMatch = line.match(rubyTypePattern);
+                if (typeMatch) {
+                    definitions.push({
+                        line: index + 1,
+                        chunkType: "types",
+                        primarySymbol: typeMatch[1],
+                    });
+                    continue;
+                }
+                const methodMatch = line.match(rubyMethodPattern);
+                if (methodMatch) {
+                    definitions.push({
+                        line: index + 1,
+                        chunkType: "impl",
+                        primarySymbol: methodMatch[1],
+                    });
+                }
+                continue;
+            }
+            const jsMatch = line.match(jsDefPattern);
+            if (jsMatch) {
+                definitions.push({
+                    line: index + 1,
+                    chunkType: /^.*\bclass\b/.test(line) ? "types" : "impl",
+                    primarySymbol: jsMatch[1],
+                });
+            }
+        }
+        if (definitions.length === 0) {
+            const fallback = lines.slice(importEnd).join("\n").trim();
+            if (fallback) {
+                chunks.push({
+                    content: fallback,
+                    startLine: importEnd + 1,
+                    endLine: lines.length,
+                    chunkType: "impl",
+                });
+            }
+            return chunks;
+        }
+        for (let index = 0; index < definitions.length; index += 1) {
+            const current = definitions[index];
+            const next = definitions[index + 1];
+            const startLine = current.line;
+            const endLine = next ? Math.max(startLine, next.line - 1) : lines.length;
+            const chunkText = lines
+                .slice(startLine - 1, endLine)
+                .join("\n")
+                .trim();
+            if (!chunkText)
+                continue;
+            chunks.push({
+                content: chunkText,
+                startLine,
+                endLine,
+                chunkType: current.chunkType,
+                primarySymbol: current.primarySymbol,
+            });
+        }
+        return chunks;
+    }
+    deriveLanguageChunks(languagePlugin, parsed, languageId, filePath, content, symbolRecords) {
+        if (languagePlugin && parsed) {
+            const pluginChunks = languagePlugin.splitIntoChunks(parsed, {
+                targetTokens: 280,
+                maxTokens: 560,
+            });
+            if (pluginChunks.length > 0) {
+                return pluginChunks.map((chunk) => {
+                    const primarySymbol = this.inferPrimarySymbol(chunk, symbolRecords);
+                    return {
+                        content: chunk.content,
+                        startLine: chunk.range.startLine,
+                        endLine: chunk.range.endLine,
+                        chunkType: this.normalizeChunkType(chunk.metadata?.chunkType, chunk.content, primarySymbol),
+                        primarySymbol,
+                    };
+                });
+            }
+        }
+        const heuristicChunks = this.splitHeuristicChunks(content, languageId);
+        if (heuristicChunks.length > 0) {
+            return heuristicChunks;
+        }
+        return this.chunker
+            .chunk({ filePath, content, language: languageId })
+            .map((chunk) => ({
+            content: chunk.content,
+            startLine: chunk.startLine,
+            endLine: chunk.endLine,
+            chunkType: this.normalizeChunkType(chunk.type, chunk.content, chunk.primarySymbol),
+            primarySymbol: chunk.primarySymbol,
+        }));
+    }
+    async initialize() {
+        await Promise.all([
+            this.metadata.initialize(),
+            this.vectors.initialize(),
+            this.embedder.initialize(),
+        ]);
+    }
+    async close() {
+        await Promise.all([
+            this.metadata.close(),
+            this.vectors.close(),
+            this.embedder.close(),
+        ]);
+    }
+    async indexProject(options) {
+        const projectId = options.projectId ?? this.projectId;
+        const repoRoot = options.repoRoot ?? this.repoRoot;
+        const gitRef = options.gitRef ?? (await this.git.getHeadCommit(repoRoot)) ?? "unknown";
+        const errors = [];
+        await this.loadChurnByFile(repoRoot);
+        const latestSnapshot = await this.metadata.getLatestCompletedSnapshot(projectId);
+        const shouldDoFullReindex = options.isFullReindex ||
+            !latestSnapshot ||
+            !options.changedFiles ||
+            options.changedFiles.deleted.length +
+                options.changedFiles.modified.length +
+                options.changedFiles.added.length >
+                20000;
+        if (shouldDoFullReindex) {
+            return this.performFullReindex(projectId, gitRef, repoRoot, options.onProgress);
+        }
+        if (!latestSnapshot || !options.changedFiles) {
+            throw new Error("Latest snapshot not found for incremental indexing");
+        }
+        const snapshot = await this.createSnapshot(projectId, gitRef, "indexing");
+        const snapshotId = snapshot.id;
+        try {
+            await this.prepareIncrementalSnapshot({
+                projectId,
+                prevSnapshotId: latestSnapshot.id,
+                newSnapshotId: snapshotId,
+                diff: options.changedFiles,
+            });
+            const filesToIndex = [
+                ...options.changedFiles.modified,
+                ...options.changedFiles.added,
+            ].filter((filePath) => this.indexingOptions.codeExtensions.includes((0, node_path_1.extname)(filePath).toLowerCase()));
+            const totalFiles = (await this.metadata.listFiles(projectId, snapshotId))
+                .length;
+            const knownFiles = new Set((await this.metadata.listFiles(projectId, snapshotId)).map((file) => this.normalizePath(file.path)));
+            if (filesToIndex.length === 0) {
+                await this.metadata.updateSnapshotProgress(snapshotId, totalFiles, totalFiles);
+                await this.architectureGenerator.generate(projectId, snapshotId);
+                await this.metadata.updateSnapshotStatus(snapshotId, "completed");
+                await this.pruneHistoricalSnapshots(projectId, snapshotId);
+                return { snapshotId, filesIndexed: totalFiles, errors: [] };
+            }
+            await this.indexPreparedFiles({
+                projectId,
+                repoRoot,
+                gitRef,
+                snapshotId,
+                filesToIndex,
+                knownFiles,
+                totalFiles,
+                onProgress: options.onProgress,
+                errors,
+                operation: "incremental batch indexing",
+            });
+            await this.architectureGenerator.generate(projectId, snapshotId);
+            if (errors.length > 0) {
+                const message = `Incremental indexing completed with ${errors.length} preparation error${errors.length === 1 ? "" : "s"}`;
+                await this.metadata.updateSnapshotStatus(snapshotId, "failed", message);
+                throw new Error(message);
+            }
+            await this.metadata.updateSnapshotStatus(snapshotId, "completed");
+            await this.metadataWithProgress.updateSnapshotProgress(snapshotId, totalFiles, totalFiles);
+            await this.pruneHistoricalSnapshots(projectId, snapshotId);
+            return { snapshotId, filesIndexed: filesToIndex.length, errors };
+        }
+        catch (error) {
+            await this.metadata.updateSnapshotStatus(snapshotId, "failed", error instanceof Error ? error.message : String(error));
+            throw error;
+        }
+    }
+    async indexFile(options) {
+        const data = await this.prepareFileRecords(options);
+        const vectorChunks = data.chunkRecords.filter((chunk) => this.shouldIndexChunkInVectors(chunk));
+        await Promise.all([
+            this.metadata.upsertFile(options.projectId, data.fileRecord),
+            this.metadata.replaceChunks(options.projectId, options.snapshotId, options.filePath, data.chunkRecords),
+            this.metadata.replaceSymbols(options.projectId, options.snapshotId, options.filePath, data.symbolRecords),
+            this.metadata.replaceDependencies(options.projectId, options.snapshotId, options.filePath, data.dependencyRecords),
+            this.metadata.upsertFileMetrics(options.projectId, {
+                snapshotId: options.snapshotId,
+                filePath: options.filePath,
+                metrics: data.metrics,
+            }),
+            (async () => {
+                if (vectorChunks.length === 0) {
+                    return;
+                }
+                const embeddings = await this.embedWithContextGuard(vectorChunks.map((chunk) => data.chunksContent.get(chunk.chunkId) || ""), "single-file indexing");
+                await this.vectors.upsert(vectorChunks.map((chunk, index) => ({
+                    projectId: options.projectId,
+                    chunkId: chunk.chunkId,
+                    snapshotId: options.snapshotId,
+                    filePath: options.filePath,
+                    startLine: chunk.startLine,
+                    endLine: chunk.endLine,
+                    embedding: embeddings[index],
+                    contentHash: chunk.contentHash,
+                    chunkType: chunk.chunkType,
+                    primarySymbol: chunk.primarySymbol,
+                })));
+            })(),
+        ]);
+    }
+    async indexPreparedFiles(options) {
+        const batchSize = this.getBatchSize();
+        let processedCount = 0;
+        await this.metadataWithProgress.updateSnapshotProgress(options.snapshotId, 0, options.totalFiles);
+        await this.metadata.updateSnapshotProgress(options.snapshotId, 0, options.totalFiles);
+        for (let index = 0; index < options.filesToIndex.length; index += batchSize) {
+            const batch = options.filesToIndex.slice(index, index + batchSize);
+            const preparedData = await Promise.all(batch.map(async (filePath) => {
+                try {
+                    const content = await (0, promises_1.readFile)((0, node_path_1.join)(options.repoRoot, filePath), "utf8");
+                    return this.prepareFileRecords({
+                        snapshotId: options.snapshotId,
+                        projectId: options.projectId,
+                        filePath,
+                        content,
+                        gitRef: options.gitRef,
+                        knownFiles: options.knownFiles,
+                    });
+                }
+                catch (error) {
+                    const message = error instanceof Error ? error.message : String(error);
+                    logger.error(`Error preparing file ${filePath}`, { message });
+                    options.errors.push(`Failed to prepare ${filePath}: ${message}`);
+                    return null;
+                }
+            }));
+            const validData = preparedData.filter((value) => value !== null);
+            if (validData.length > 0) {
+                await this.metadata.transaction(async () => {
+                    for (const data of validData) {
+                        await this.metadata.upsertFile(options.projectId, data.fileRecord);
+                        await this.metadata.replaceChunks(options.projectId, options.snapshotId, data.filePath, data.chunkRecords);
+                        await this.metadata.replaceSymbols(options.projectId, options.snapshotId, data.filePath, data.symbolRecords);
+                        await this.metadata.replaceDependencies(options.projectId, options.snapshotId, data.filePath, data.dependencyRecords);
+                        await this.metadata.upsertFileMetrics(options.projectId, {
+                            snapshotId: options.snapshotId,
+                            filePath: data.filePath,
+                            metrics: data.metrics,
+                        });
+                    }
+                });
+                const vectorChunksWithContext = validData.flatMap((data) => data.chunkRecords
+                    .filter((chunk) => this.shouldIndexChunkInVectors(chunk))
+                    .map((chunk) => ({
+                    chunk,
+                    filePath: data.filePath,
+                    content: data.chunksContent.get(chunk.chunkId) || "",
+                })));
+                if (vectorChunksWithContext.length > 0) {
+                    const embeddings = await this.embedWithContextGuard(vectorChunksWithContext.map((item) => item.content), options.operation);
+                    await this.vectors.upsert(vectorChunksWithContext.map((item, index) => ({
+                        projectId: this.projectId,
+                        chunkId: item.chunk.chunkId,
+                        snapshotId: options.snapshotId,
+                        filePath: item.filePath,
+                        startLine: item.chunk.startLine,
+                        endLine: item.chunk.endLine,
+                        embedding: embeddings[index],
+                        contentHash: item.chunk.contentHash,
+                        chunkType: item.chunk.chunkType,
+                        primarySymbol: item.chunk.primarySymbol,
+                    })));
+                }
+            }
+            processedCount += batch.length;
+            options.onProgress?.(Math.min(processedCount, options.totalFiles), options.totalFiles);
+            if (processedCount % 100 === 0 || processedCount >= options.totalFiles) {
+                await this.metadataWithProgress.updateSnapshotProgress(options.snapshotId, processedCount, options.totalFiles);
+            }
+        }
+    }
+    getLanguagePlugin(filePath) {
+        return this.languagePluginRegistry.findByFilePath(filePath);
+    }
+    getLanguageIdFromPath(filePath) {
+        const ext = (0, node_path_1.extname)(filePath).toLowerCase();
+        switch (ext) {
+            case ".ts":
+            case ".tsx":
+            case ".mts":
+            case ".cts":
+            case ".js":
+            case ".jsx":
+            case ".mjs":
+            case ".cjs":
+                return "typescript";
+            case ".py":
+            case ".pyi":
+                return "python";
+            case ".cs":
+                return "csharp";
+            case ".gd":
+                return "gdscript";
+            case ".rb":
+                return "ruby";
+            default:
+                return "plaintext";
+        }
+    }
+    normalizePath(filePath) {
+        return filePath.replace(/\\/g, "/");
+    }
+    async loadChurnByFile(repoRoot) {
+        try {
+            const churn = await this.git.getChurnByFile(repoRoot, { sinceDays: 30 });
+            this.churnByFile = new Map(Object.entries(churn).map(([filePath, count]) => {
+                const normalized = this.normalizePath(filePath);
+                return [
+                    normalized.startsWith("./") ? normalized.slice(2) : normalized,
+                    count,
+                ];
+            }));
+        }
+        catch (error) {
+            logger.warn("Failed to load churn data", {
+                message: error instanceof Error ? error.message : String(error),
+            });
+            this.churnByFile = new Map();
+        }
+    }
+    async scanFiles(rootPath) {
+        return (0, scanner_js_1.scanProjectFiles)(rootPath, this.indexingOptions.codeExtensions);
+    }
+    async prepareFileRecords(options) {
+        const { snapshotId, filePath, content, knownFiles } = options;
+        const languagePlugin = this.getLanguagePlugin(filePath);
+        let languageId = this.getLanguageIdFromPath(filePath);
+        let symbolRecords = [];
+        let dependencyRecords = [];
+        let parsed;
+        if (languagePlugin) {
+            parsed = languagePlugin.parse({
+                path: filePath,
+                content,
+                languageHint: languagePlugin.id,
+                projectRoot: this.repoRoot,
+            });
+            languageId = parsed.languageId;
+            const symbols = languagePlugin.extractSymbols(parsed);
+            symbolRecords = symbols.map((symbol) => ({
+                snapshotId,
+                id: symbol.id,
+                filePath,
+                kind: symbol.kind,
+                name: symbol.name,
+                containerName: symbol.containerName,
+                exported: symbol.exported,
+                range: {
+                    start: {
+                        line: symbol.range.startLine,
+                        character: symbol.range.startCol,
+                    },
+                    end: { line: symbol.range.endLine, character: symbol.range.endCol },
+                },
+                signature: symbol.signature,
+                docComment: symbol.docComment,
+                metadata: symbol.metadata,
+            }));
+            const imports = languagePlugin.extractImports(parsed);
+            dependencyRecords = imports.map((dependency) => ({
+                ...(0, dependency_resolver_js_1.resolveDependency)(dependency.spec, filePath, knownFiles),
+                snapshotId,
+                id: dependency.id,
+                fromPath: filePath,
+                toSpecifier: dependency.spec,
+                kind: normalizeImportKind(dependency.kind),
+            }));
+        }
+        const chunks = this.deriveLanguageChunks(languagePlugin, parsed, languageId, filePath, content, symbolRecords);
+        const chunkRecords = [];
+        const chunksContent = new Map();
+        for (const chunk of chunks) {
+            const chunkId = (0, node_crypto_1.randomUUID)();
+            const chunkHash = (0, hash_js_1.computeHash)(chunk.content);
+            chunkRecords.push({
+                snapshotId,
+                chunkId,
+                filePath,
+                startLine: chunk.startLine,
+                endLine: chunk.endLine,
+                contentHash: chunkHash,
+                tokenEstimate: this.tokenEstimator.estimate(chunk.content),
+                chunkType: chunk.chunkType,
+                primarySymbol: chunk.primarySymbol,
+                hasOverlap: false,
+            });
+            chunksContent.set(chunkId, chunk.content);
+        }
+        const churn = this.churnByFile.get(this.normalizePath(filePath)) ?? 0;
+        return {
+            filePath,
+            content,
+            fileRecord: {
+                snapshotId,
+                path: filePath,
+                sha256: (0, hash_js_1.computeHash)(content),
+                mtimeMs: Date.now(),
+                size: content.length,
+                languageId,
+            },
+            chunkRecords,
+            chunksContent,
+            symbolRecords,
+            dependencyRecords,
+            metrics: {
+                complexity: 0,
+                maintainability: 100,
+                churn,
+                testCoverage: undefined,
+            },
+        };
+    }
+    async createSnapshot(projectId, gitRef, status) {
+        const snapshot = await this.metadata.createSnapshot(projectId, {
+            headCommit: gitRef,
+            indexedAt: Date.now(),
+        });
+        await this.metadata.updateSnapshotStatus(snapshot.id, status);
+        return { id: snapshot.id };
+    }
+    async prepareIncrementalSnapshot(options) {
+        const prevFiles = await this.metadata.listFiles(options.projectId, options.prevSnapshotId);
+        const modifiedSet = new Set(options.diff.modified);
+        const deletedSet = new Set(options.diff.deleted);
+        const unchangedFiles = prevFiles
+            .filter((file) => !modifiedSet.has(file.path) && !deletedSet.has(file.path))
+            .map((file) => file.path);
+        if (unchangedFiles.length === 0) {
+            return;
+        }
+        await this.metadata.copyUnchangedFileData(options.projectId, options.prevSnapshotId, options.newSnapshotId, unchangedFiles);
+        await this.vectors.copyVectors(options.projectId, options.prevSnapshotId, options.newSnapshotId, [
+            ...new Set([
+                ...options.diff.modified,
+                ...options.diff.deleted,
+                ...options.diff.added,
+            ]),
+        ]);
+    }
+    isTransientVectorDeleteError(error) {
+        const message = error instanceof Error ? error.message : String(error);
+        return ((message.includes("LanceError(IO)") &&
+            (message.includes("Not found:") ||
+                message.includes("Did not find any data files") ||
+                message.includes("/vectors.lance/data/"))) ||
+            message.includes("Commit conflict for version"));
+    }
+    async deleteProjectVectorsWithRetry(projectId) {
+        const delays = [0, 300, 1200];
+        for (let attempt = 0; attempt < delays.length; attempt += 1) {
+            if (delays[attempt] > 0) {
+                await new Promise((resolve) => setTimeout(resolve, delays[attempt]));
+            }
+            try {
+                await this.vectors.deleteByProject(projectId);
+                return;
+            }
+            catch (error) {
+                if (!this.isTransientVectorDeleteError(error) ||
+                    attempt === delays.length - 1) {
+                    throw error;
+                }
+            }
+        }
+    }
+    async deleteSnapshotVectorsWithRetry(projectId, snapshotId) {
+        const delays = [0, 300, 1200];
+        for (let attempt = 0; attempt < delays.length; attempt += 1) {
+            if (delays[attempt] > 0) {
+                await new Promise((resolve) => setTimeout(resolve, delays[attempt]));
+            }
+            try {
+                await this.vectors.deleteBySnapshot(projectId, snapshotId);
+                return;
+            }
+            catch (error) {
+                if (!this.isTransientVectorDeleteError(error) ||
+                    attempt === delays.length - 1) {
+                    throw error;
+                }
+            }
+        }
+    }
+    async pruneHistoricalSnapshots(projectId, keepSnapshotId) {
+        const staleSnapshotIds = await this.listStaleSnapshotIds(projectId, keepSnapshotId);
+        if (staleSnapshotIds.length === 0) {
+            return;
+        }
+        for (const snapshotId of staleSnapshotIds) {
+            await this.deleteSnapshotVectorsWithRetry(projectId, snapshotId);
+        }
+        await this.metadata.clearProjectMetadata(projectId, keepSnapshotId, {
+            preserveActiveIndexing: true,
+        });
+    }
+    async listStaleSnapshotIds(projectId, keepSnapshotId) {
+        const staleSnapshotIds = [];
+        const pageSize = 100;
+        for (let offset = 0;; offset += pageSize) {
+            const snapshots = await this.metadata.listSnapshots(projectId, {
+                limit: pageSize,
+                offset,
+            });
+            for (const snapshot of snapshots) {
+                if (snapshot.id !== keepSnapshotId) {
+                    staleSnapshotIds.push(snapshot.id);
+                }
+            }
+            if (snapshots.length < pageSize) {
+                return staleSnapshotIds;
+            }
+        }
+    }
+    async performFullReindex(projectId, gitRef, repoRoot, onProgress) {
+        const snapshot = await this.createSnapshot(projectId, gitRef, "indexing");
+        const snapshotId = snapshot.id;
+        const errors = [];
+        try {
+            await this.metadata.clearProjectMetadata(projectId, snapshotId);
+            await this.deleteProjectVectorsWithRetry(projectId);
+            const filesToIndex = await this.scanFiles(repoRoot);
+            if (filesToIndex.length === 0) {
+                await this.metadata.updateSnapshotProgress(snapshotId, 0, 0);
+                await this.architectureGenerator.generate(projectId, snapshotId);
+                await this.metadata.updateSnapshotStatus(snapshotId, "completed");
+                return { snapshotId, filesIndexed: 0, errors: [] };
+            }
+            await this.indexPreparedFiles({
+                projectId,
+                repoRoot,
+                gitRef,
+                snapshotId,
+                filesToIndex,
+                knownFiles: new Set(filesToIndex.map((filePath) => this.normalizePath(filePath))),
+                totalFiles: filesToIndex.length,
+                onProgress,
+                errors,
+                operation: "full reindex batch indexing",
+            });
+            await this.architectureGenerator.generate(projectId, snapshotId);
+            await this.metadata.updateSnapshotStatus(snapshotId, "completed");
+            await this.metadataWithProgress.updateSnapshotProgress(snapshotId, filesToIndex.length, filesToIndex.length);
+            return { snapshotId, filesIndexed: filesToIndex.length, errors };
+        }
+        catch (error) {
+            await this.metadata.updateSnapshotStatus(snapshotId, "failed", error instanceof Error ? error.message : String(error));
+            throw error;
+        }
+    }
+    getBatchSize() {
+        const value = config_js_1.config.get("indexBatchSize");
+        return Number.isFinite(value) && value > 0 ? value : 50;
+    }
+}
+exports.IndexerEngine = IndexerEngine;
+//# sourceMappingURL=indexer.js.map