kiri-mcp-server 0.9.6 → 0.10.0

package/dist/src/indexer/cli.js

@@ -1,17 +1,54 @@
 import { createHash } from "node:crypto";
+import { existsSync } from "node:fs";
 import { readFile, stat } from "node:fs/promises";
 import { join, resolve, extname } from "node:path";
 import { pathToFileURL } from "node:url";
 import { DuckDBClient } from "../shared/duckdb.js";
 import { generateEmbedding } from "../shared/embedding.js";
+import { acquireLock, releaseLock, LockfileError, getLockOwner } from "../shared/utils/lockfile.js";
+import { normalizeDbPath, ensureDbParentDir, getRepoPathCandidates } from "../shared/utils/path.js";
 import { analyzeSource, buildFallbackSnippet } from "./codeintel.js";
-import { getDefaultBranch, getHeadCommit, gitLsFiles } from "./git.js";
+import { getDefaultBranch, getHeadCommit, gitLsFiles, gitDiffNameOnly } from "./git.js";
 import { detectLanguage } from "./language.js";
-import { ensureBaseSchema } from "./schema.js";
+import { getIndexerQueue } from "./queue.js";
+import { ensureBaseSchema, ensureRepoMetaColumns, rebuildFTSIfNeeded } from "./schema.js";
 import { IndexWatcher } from "./watch.js";
 const MAX_SAMPLE_BYTES = 32_768;
 const MAX_FILE_BYTES = 32 * 1024 * 1024; // 32MB limit to prevent memory exhaustion
 const SCAN_BATCH_SIZE = 100; // Process files in batches to limit memory usage
+/**
+ * Maximum number of SQL placeholders per INSERT statement.
+ *
+ * DuckDB's internal limit is 65535 placeholders, but we use a conservative value of 30000 for:
+ * 1. Safety margin: Prevents stack overflow when building large SQL strings in JavaScript
+ * 2. Performance: Smaller batches reduce memory pressure and provide better error granularity
+ * 3. Compatibility: Works safely across different DuckDB versions and system configurations
+ *
+ * This value has been validated with real-world testing:
+ * - Successfully handles 10000+ files in batch-processing.spec.ts
+ * - Prevents "Maximum call stack size exceeded" errors (Issue #39)
+ * - Balances transaction throughput vs. individual batch size
+ *
+ * Example batch sizes with this limit:
+ * - 4-column table (blob): 7500 records per batch
+ * - 5-column table (dependency): 6000 records per batch
+ * - 9-column table (symbol): 3333 records per batch
+ */
+const MAX_SQL_PLACEHOLDERS = 30000;
+/**
+ * Calculate safe batch size for SQL INSERT operations based on columns per record.
+ * Ensures total placeholders per statement stays under MAX_SQL_PLACEHOLDERS.
+ *
+ * @param columnsPerRecord - Number of columns in the INSERT statement (must be positive)
+ * @returns Safe batch size that won't exceed placeholder limit
+ * @throws {Error} If columnsPerRecord is not a positive integer
+ */
+function calculateBatchSize(columnsPerRecord) {
+    if (columnsPerRecord <= 0 || !Number.isInteger(columnsPerRecord)) {
+        throw new Error(`columnsPerRecord must be a positive integer, got: ${columnsPerRecord}`);
+    }
+    return Math.floor(MAX_SQL_PLACEHOLDERS / columnsPerRecord);
+}
 function countLines(content) {
     if (content.length === 0) {
         return 0;
@@ -35,137 +72,237 @@ function isBinaryBuffer(buffer) {
  * @param defaultBranch - Default branch name (e.g., "main", "master"), or null if unknown
  * @returns The repository ID (auto-generated on first insert, reused thereafter)
  */
-async function ensureRepo(db, repoRoot, defaultBranch) {
-    // Atomically insert or update using ON CONFLICT to leverage auto-increment
-    // This eliminates the TOCTOU race condition present in manual ID generation
-    await db.run(`INSERT INTO repo (root, default_branch, indexed_at)
-     VALUES (?, ?, CURRENT_TIMESTAMP)
-     ON CONFLICT(root) DO UPDATE SET
-       default_branch = COALESCE(excluded.default_branch, repo.default_branch)`, [repoRoot, defaultBranch]);
-    // Fetch the ID of the existing or newly created repo
-    const rows = await db.all("SELECT id FROM repo WHERE root = ?", [repoRoot]);
+async function mergeLegacyRepoRows(db, canonicalRepoId, legacyRepoIds) {
+    if (legacyRepoIds.length === 0) {
+        return;
+    }
+    const referencingTables = await db.all(`SELECT DISTINCT c.table_name
+       FROM duckdb_columns() AS c
+       JOIN duckdb_tables() AS t
+         ON c.database_name = t.database_name
+        AND c.schema_name = t.schema_name
+        AND c.table_name = t.table_name
+      WHERE c.column_name = 'repo_id'
+        AND c.table_name <> 'repo'
+        AND t.table_type = 'BASE TABLE'`);
+    const safeTables = referencingTables
+        .map((row) => row.table_name)
+        .filter((name) => /^[A-Za-z0-9_]+$/.test(name));
+    await db.transaction(async () => {
+        for (const legacyRepoId of legacyRepoIds) {
+            for (const tableName of safeTables) {
+                await db.run(`UPDATE ${tableName} SET repo_id = ? WHERE repo_id = ?`, [
+                    canonicalRepoId,
+                    legacyRepoId,
+                ]);
+            }
+            await db.run("DELETE FROM repo WHERE id = ?", [legacyRepoId]);
+        }
+    });
+}
+async function ensureRepo(db, repoRoot, defaultBranch, candidateRoots) {
+    const searchRoots = Array.from(new Set([repoRoot, ...(candidateRoots ?? [])]));
+    const placeholders = searchRoots.map(() => "?").join(", ");
+    let rows = await db.all(`SELECT id, root FROM repo WHERE root IN (${placeholders})`, searchRoots);
+    if (rows.length === 0) {
+        await db.run(`INSERT INTO repo (root, default_branch, indexed_at)
+       VALUES (?, ?, CURRENT_TIMESTAMP)
+       ON CONFLICT(root) DO UPDATE SET
+         default_branch = COALESCE(excluded.default_branch, repo.default_branch)`, [repoRoot, defaultBranch]);
+        rows = await db.all(`SELECT id, root FROM repo WHERE root IN (${placeholders})`, searchRoots);
+    }
     if (rows.length === 0) {
         throw new Error("Failed to create or find repository record. Check database constraints and schema.");
     }
-    const row = rows[0];
-    if (!row) {
+    let canonicalRow = rows.find((row) => row.root === repoRoot) ?? rows[0];
+    if (!canonicalRow) {
         throw new Error("Failed to retrieve repository record. Database returned empty result.");
     }
-    return row.id;
-}
-async function persistBlobs(db, blobs) {
-    if (blobs.size === 0)
-        return;
-    // Use bulk insert for better performance
-    const blobArray = Array.from(blobs.values());
-    const placeholders = blobArray.map(() => "(?, ?, ?, ?)").join(", ");
-    const sql = `INSERT OR REPLACE INTO blob (hash, size_bytes, line_count, content) VALUES ${placeholders}`;
-    const params = [];
-    for (const blob of blobArray) {
-        params.push(blob.hash, blob.sizeBytes, blob.lineCount, blob.content);
+    if (canonicalRow.root !== repoRoot) {
+        await db.run("UPDATE repo SET root = ? WHERE id = ?", [repoRoot, canonicalRow.id]);
+        canonicalRow = { ...canonicalRow, root: repoRoot };
     }
-    await db.run(sql, params);
+    const legacyIds = rows.filter((row) => row.id !== canonicalRow.id).map((row) => row.id);
+    await mergeLegacyRepoRows(db, canonicalRow.id, legacyIds);
+    return canonicalRow.id;
 }
-async function persistTrees(db, repoId, commitHash, records) {
+/**
+ * Generic helper function to persist records in batches to prevent stack overflow.
+ * Splits large datasets into smaller batches and executes INSERT statements sequentially.
+ *
+ * IMPORTANT: This function must be called within an active database transaction.
+ * See runIndexer() for transaction management context.
+ *
+ * @param db - Database client (must be within an active transaction)
+ * @param records - Array of records to persist
+ * @param batchSize - Maximum number of records per INSERT statement
+ * @param buildInsert - Function that builds SQL and params for a batch
+ */
+async function persistInBatches(db, records, batchSize, buildInsert) {
     if (records.length === 0)
         return;
-    // Use bulk insert for better performance
-    const placeholders = records.map(() => "(?, ?, ?, ?, ?, ?, ?, ?)").join(", ");
-    const sql = `INSERT OR REPLACE INTO tree (repo_id, commit_hash, path, blob_hash, ext, lang, is_binary, mtime) VALUES ${placeholders}`;
-    const params = [];
-    for (const record of records) {
-        params.push(repoId, commitHash, record.path, record.blobHash, record.ext, record.lang, record.isBinary, record.mtimeIso);
+    for (let i = 0; i < records.length; i += batchSize) {
+        const batch = records.slice(i, i + batchSize);
+        const { sql, params } = buildInsert(batch);
+        try {
+            await db.run(sql, params);
+        }
+        catch (error) {
+            // バッチインデックスとサイズを含むエラーメッセージ（0-indexedの正確な範囲）
+            const batchInfo = `Batch ${Math.floor(i / batchSize) + 1}/${Math.ceil(records.length / batchSize)} (records ${i}-${i + batch.length - 1})`;
+            throw new Error(`Failed to persist batch: ${batchInfo}. Original error: ${error instanceof Error ? error.message : String(error)}`);
+        }
     }
-    await db.run(sql, params);
 }
+/**
+ * Persist blob records to database in batches to prevent stack overflow.
+ *
+ * IMPORTANT: This function must be called within an active database transaction.
+ * See runIndexer() for transaction management context.
+ *
+ * @param db - Database client (must be within an active transaction)
+ * @param blobs - Map of blob records to persist
+ */
+async function persistBlobs(db, blobs) {
+    const blobArray = Array.from(blobs.values());
+    const BATCH_SIZE = calculateBatchSize(4); // blob table has 4 columns
+    await persistInBatches(db, blobArray, BATCH_SIZE, (batch) => ({
+        sql: `INSERT OR REPLACE INTO blob (hash, size_bytes, line_count, content) VALUES ${batch.map(() => "(?, ?, ?, ?)").join(", ")}`,
+        params: batch.flatMap((blob) => [blob.hash, blob.sizeBytes, blob.lineCount, blob.content]),
+    }));
+}
+/**
+ * Persist tree records to database in batches to prevent stack overflow.
+ *
+ * IMPORTANT: This function must be called within an active database transaction.
+ * See runIndexer() for transaction management context.
+ *
+ * @param db - Database client (must be within an active transaction)
+ * @param repoId - Repository ID
+ * @param commitHash - Git commit hash
+ * @param records - File records to persist
+ */
+async function persistTrees(db, repoId, commitHash, records) {
+    const BATCH_SIZE = calculateBatchSize(8); // tree table has 8 columns
+    await persistInBatches(db, records, BATCH_SIZE, (batch) => ({
+        sql: `INSERT OR REPLACE INTO tree (repo_id, commit_hash, path, blob_hash, ext, lang, is_binary, mtime) VALUES ${batch.map(() => "(?, ?, ?, ?, ?, ?, ?, ?)").join(", ")}`,
+        params: batch.flatMap((record) => [
+            repoId,
+            commitHash,
+            record.path,
+            record.blobHash,
+            record.ext,
+            record.lang,
+            record.isBinary,
+            record.mtimeIso,
+        ]),
+    }));
+}
+/**
+ * Persist file records to database in batches to prevent stack overflow.
+ *
+ * IMPORTANT: This function must be called within an active database transaction.
+ * See runIndexer() for transaction management context.
+ *
+ * @param db - Database client (must be within an active transaction)
+ * @param repoId - Repository ID
+ * @param records - File records to persist
+ */
 async function persistFiles(db, repoId, records) {
-    if (records.length === 0)
-        return;
-    // Use bulk insert for better performance
-    const placeholders = records.map(() => "(?, ?, ?, ?, ?, ?, ?)").join(", ");
-    const sql = `INSERT OR REPLACE INTO file (repo_id, path, blob_hash, ext, lang, is_binary, mtime) VALUES ${placeholders}`;
-    const params = [];
-    for (const record of records) {
-        params.push(repoId, record.path, record.blobHash, record.ext, record.lang, record.isBinary, record.mtimeIso);
-    }
-    await db.run(sql, params);
+    const BATCH_SIZE = calculateBatchSize(7); // file table has 7 columns
+    await persistInBatches(db, records, BATCH_SIZE, (batch) => ({
+        sql: `INSERT OR REPLACE INTO file (repo_id, path, blob_hash, ext, lang, is_binary, mtime) VALUES ${batch.map(() => "(?, ?, ?, ?, ?, ?, ?)").join(", ")}`,
+        params: batch.flatMap((record) => [
+            repoId,
+            record.path,
+            record.blobHash,
+            record.ext,
+            record.lang,
+            record.isBinary,
+            record.mtimeIso,
+        ]),
+    }));
 }
+/**
+ * Persist symbol records to database in batches to prevent stack overflow.
+ *
+ * IMPORTANT: This function must be called within an active database transaction.
+ * See runIndexer() for transaction management context.
+ *
+ * @param db - Database client (must be within an active transaction)
+ * @param repoId - Repository ID
+ * @param records - Symbol records to persist
+ */
 async function persistSymbols(db, repoId, records) {
-    if (records.length === 0)
-        return;
-    // バッチサイズを1000に制限してスタックオーバーフローを防ぐ
-    const BATCH_SIZE = 1000;
-    for (let i = 0; i < records.length; i += BATCH_SIZE) {
-        const batch = records.slice(i, i + BATCH_SIZE);
-        const placeholders = batch.map(() => "(?, ?, ?, ?, ?, ?, ?, ?, ?)").join(", ");
-        const sql = `
-      INSERT OR REPLACE INTO symbol (
-        repo_id, path, symbol_id, name, kind, range_start_line, range_end_line, signature, doc
-      ) VALUES ${placeholders}
-    `;
-        const params = [];
-        for (const record of batch) {
-            params.push(repoId, record.path, record.symbolId, record.name, record.kind, record.rangeStartLine, record.rangeEndLine, record.signature, record.doc);
-        }
-        await db.run(sql, params);
-    }
+    const BATCH_SIZE = calculateBatchSize(9); // symbol table has 9 columns
+    await persistInBatches(db, records, BATCH_SIZE, (batch) => ({
+        sql: `INSERT OR REPLACE INTO symbol (repo_id, path, symbol_id, name, kind, range_start_line, range_end_line, signature, doc) VALUES ${batch.map(() => "(?, ?, ?, ?, ?, ?, ?, ?, ?)").join(", ")}`,
+        params: batch.flatMap((r) => [
+            repoId,
+            r.path,
+            r.symbolId,
+            r.name,
+            r.kind,
+            r.rangeStartLine,
+            r.rangeEndLine,
+            r.signature,
+            r.doc,
+        ]),
+    }));
 }
+/**
+ * Persist snippet records to database in batches to prevent stack overflow.
+ *
+ * IMPORTANT: This function must be called within an active database transaction.
+ * See runIndexer() for transaction management context.
+ *
+ * @param db - Database client (must be within an active transaction)
+ * @param repoId - Repository ID
+ * @param records - Snippet records to persist
+ */
 async function persistSnippets(db, repoId, records) {
-    if (records.length === 0)
-        return;
-    // バッチサイズを1000に制限してスタックオーバーフローを防ぐ
-    const BATCH_SIZE = 1000;
-    for (let i = 0; i < records.length; i += BATCH_SIZE) {
-        const batch = records.slice(i, i + BATCH_SIZE);
-        const placeholders = batch.map(() => "(?, ?, ?, ?, ?, ?)").join(", ");
-        const sql = `
-      INSERT OR REPLACE INTO snippet (
-        repo_id, path, snippet_id, start_line, end_line, symbol_id
-      ) VALUES ${placeholders}
-    `;
-        const params = [];
-        for (const record of batch) {
-            params.push(repoId, record.path, record.snippetId, record.startLine, record.endLine, record.symbolId);
-        }
-        await db.run(sql, params);
-    }
+    const BATCH_SIZE = calculateBatchSize(6); // snippet table has 6 columns
+    await persistInBatches(db, records, BATCH_SIZE, (batch) => ({
+        sql: `INSERT OR REPLACE INTO snippet (repo_id, path, snippet_id, start_line, end_line, symbol_id) VALUES ${batch.map(() => "(?, ?, ?, ?, ?, ?)").join(", ")}`,
+        params: batch.flatMap((r) => [repoId, r.path, r.snippetId, r.startLine, r.endLine, r.symbolId]),
+    }));
 }
+/**
+ * Persist file dependency records to database in batches to prevent stack overflow.
+ *
+ * MUST be called within a transaction.
+ * Batch size is dynamically calculated based on MAX_SQL_PLACEHOLDERS.
+ */
 async function persistDependencies(db, repoId, records) {
-    if (records.length === 0)
-        return;
-    // バッチサイズを1000に制限してスタックオーバーフローを防ぐ
-    const BATCH_SIZE = 1000;
-    for (let i = 0; i < records.length; i += BATCH_SIZE) {
-        const batch = records.slice(i, i + BATCH_SIZE);
-        const placeholders = batch.map(() => "(?, ?, ?, ?, ?)").join(", ");
-        const sql = `
-      INSERT OR REPLACE INTO dependency (
-        repo_id, src_path, dst_kind, dst, rel
-      ) VALUES ${placeholders}
-    `;
-        const params = [];
-        for (const record of batch) {
-            params.push(repoId, record.srcPath, record.dstKind, record.dst, record.rel);
-        }
-        await db.run(sql, params);
-    }
+    const BATCH_SIZE = calculateBatchSize(5); // dependency table has 5 columns
+    await persistInBatches(db, records, BATCH_SIZE, (batch) => ({
+        sql: `INSERT OR REPLACE INTO dependency (repo_id, src_path, dst_kind, dst, rel) VALUES ${batch.map(() => "(?, ?, ?, ?, ?)").join(", ")}`,
+        params: batch.flatMap((r) => [repoId, r.srcPath, r.dstKind, r.dst, r.rel]),
+    }));
 }
+/**
+ * Persist file embedding records to database in batches to prevent stack overflow.
+ *
+ * IMPORTANT: This function must be called within an active database transaction.
+ * See runIndexer() for transaction management context.
+ *
+ * @param db - Database client (must be within an active transaction)
+ * @param repoId - Repository ID
+ * @param records - Embedding records to persist
+ */
 async function persistEmbeddings(db, repoId, records) {
-    if (records.length === 0)
-        return;
-    const placeholders = records.map(() => "(?, ?, ?, ?, CURRENT_TIMESTAMP)").join(", ");
-    const sql = `
-    INSERT OR REPLACE INTO file_embedding (
-      repo_id, path, dims, vector_json, updated_at
-    ) VALUES ${placeholders}
-  `;
-    const params = [];
-    for (const record of records) {
-        params.push(repoId, record.path, record.dims, JSON.stringify(record.vector));
-    }
-    await db.run(sql, params);
+    const BATCH_SIZE = calculateBatchSize(4); // file_embedding table has 4 parameterized columns
+    await persistInBatches(db, records, BATCH_SIZE, (batch) => ({
+        sql: `INSERT OR REPLACE INTO file_embedding (repo_id, path, dims, vector_json, updated_at) VALUES ${batch.map(() => "(?, ?, ?, ?, CURRENT_TIMESTAMP)").join(", ")}`,
+        params: batch.flatMap((record) => [
+            repoId,
+            record.path,
+            record.dims,
+            JSON.stringify(record.vector),
+        ]),
+    }));
 }
-function buildCodeIntel(files, blobs) {
+async function buildCodeIntel(files, blobs, workspaceRoot) {
     const fileSet = new Set(files.map((file) => file.path));
     const symbols = [];
     const snippets = [];
@@ -178,7 +315,7 @@ function buildCodeIntel(files, blobs) {
         if (!blob || blob.content === null) {
             continue;
         }
-        const analysis = analyzeSource(file.path, file.lang, blob.content, fileSet);
+        const analysis = await analyzeSource(file.path, file.lang, blob.content, fileSet, workspaceRoot);
         for (const symbol of analysis.symbols) {
             symbols.push({
                 path: file.path,
@@ -234,9 +371,10 @@ async function scanFilesInBatches(repoRoot, paths) {
     const allBlobs = new Map();
     const allFiles = [];
     const allEmbeddings = [];
+    const allMissingPaths = [];
     for (let i = 0; i < paths.length; i += SCAN_BATCH_SIZE) {
         const batch = paths.slice(i, i + SCAN_BATCH_SIZE);
-        const { blobs, files, embeddings } = await scanFiles(repoRoot, batch);
+        const { blobs, files, embeddings, missingPaths } = await scanFiles(repoRoot, batch);
         // マージ: blobはhashでユニークなので重複排除
         for (const [hash, blob] of blobs) {
             if (!allBlobs.has(hash)) {
@@ -245,15 +383,22 @@ async function scanFilesInBatches(repoRoot, paths) {
         }
         allFiles.push(...files);
         allEmbeddings.push(...embeddings);
+        allMissingPaths.push(...missingPaths);
         // バッチデータを明示的にクリアしてGCを促す
         blobs.clear();
     }
-    return { blobs: allBlobs, files: allFiles, embeddings: allEmbeddings };
+    return {
+        blobs: allBlobs,
+        files: allFiles,
+        embeddings: allEmbeddings,
+        missingPaths: allMissingPaths,
+    };
 }
 async function scanFiles(repoRoot, paths) {
     const blobs = new Map();
     const files = [];
     const embeddings = [];
+    const missingPaths = [];
     for (const relativePath of paths) {
         const absolutePath = join(repoRoot, relativePath);
         try {
@@ -302,11 +447,17 @@ async function scanFiles(repoRoot, paths) {
             }
         }
         catch (error) {
+            // Fix #4: Track deleted files (ENOENT) for database cleanup
+            if (error.code === "ENOENT") {
+                missingPaths.push(relativePath);
+                continue;
+            }
+            // Other errors (permissions, etc.) - log and skip
             console.warn(`Cannot read ${relativePath} due to filesystem error. Fix file permissions or remove the file.`);
             console.warn(error);
         }
     }
-    return { blobs, files, embeddings };
+    return { blobs, files, embeddings, missingPaths };
 }
 /**
  * 既存のファイルハッシュをDBから取得する（インクリメンタルインデックス用）
@@ -383,161 +534,242 @@ async function deleteFileRecords(db, repoId, headCommit, path) {
     await db.run("DELETE FROM file WHERE repo_id = ? AND path = ?", [repoId, path]);
 }
 export async function runIndexer(options) {
-    const repoRoot = resolve(options.repoRoot);
-    const databasePath = resolve(options.databasePath);
-    const db = await DuckDBClient.connect({ databasePath, ensureDirectory: true });
-    try {
-        await ensureBaseSchema(db);
-        const [headCommit, defaultBranch] = await Promise.all([
-            getHeadCommit(repoRoot),
-            getDefaultBranch(repoRoot),
-        ]);
-        const repoId = await ensureRepo(db, repoRoot, defaultBranch);
-        // Incremental mode: only reindex files in changedPaths
-        if (options.changedPaths && options.changedPaths.length > 0) {
-            // First, reconcile deleted files (handle renames/deletions)
-            const deletedPaths = await reconcileDeletedFiles(db, repoId, repoRoot);
-            if (deletedPaths.length > 0) {
-                console.info(`Removed ${deletedPaths.length} deleted file(s) from index.`);
+    const repoPathCandidates = getRepoPathCandidates(options.repoRoot);
+    const repoRoot = repoPathCandidates[0];
+    if (!repoRoot) {
+        throw new Error(`Unable to resolve repository root for ${options.repoRoot}`);
+    }
+    let databasePath;
+    // Fix #2: Ensure parent directory exists BEFORE normalization
+    // This guarantees consistent path normalization on first and subsequent runs
+    await ensureDbParentDir(options.databasePath);
+    // Critical: Use normalizeDbPath to ensure consistent path across runs
+    // This prevents lock file and queue key bypass when DB is accessed via symlink
+    databasePath = normalizeDbPath(options.databasePath);
+    // DuckDB single-writer制約対応: 同じdatabasePathへの並列書き込みを防ぐため、
+    // databasePathごとのキューで直列化する
+    return getIndexerQueue(databasePath).add(async () => {
+        // Fix #1 & #2: Add file lock for multi-process safety (unless caller already holds lock)
+        const lockfilePath = `${databasePath}.lock`;
+        let lockAcquired = false;
+        if (!options.skipLocking) {
+            try {
+                acquireLock(lockfilePath);
+                lockAcquired = true;
             }
-            const existingHashes = await getExistingFileHashes(db, repoId);
-            const { blobs, files, embeddings } = await scanFilesInBatches(repoRoot, options.changedPaths);
-            // Filter out files that haven't actually changed (same hash)
-            const changedFiles = [];
-            const changedBlobs = new Map();
-            for (const file of files) {
-                const existingHash = existingHashes.get(file.path);
-                if (existingHash !== file.blobHash) {
-                    changedFiles.push(file);
-                    const blob = blobs.get(file.blobHash);
-                    if (blob) {
-                        changedBlobs.set(blob.hash, blob);
-                    }
+            catch (error) {
+                if (error instanceof LockfileError) {
+                    const ownerPid = error.ownerPid ?? getLockOwner(lockfilePath);
+                    const ownerInfo = ownerPid ? ` (PID: ${ownerPid})` : "";
+                    throw new Error(`Another indexing process${ownerInfo} holds the lock for ${databasePath}. Please wait for it to complete.`);
                 }
+                throw error;
             }
-            if (changedFiles.length === 0) {
-                console.info(`No actual changes detected in ${options.changedPaths.length} file(s). Skipping reindex.`);
-                // Still update timestamp to indicate we checked
-                if (defaultBranch) {
-                    await db.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP, default_branch = ? WHERE id = ?", [defaultBranch, repoId]);
+        }
+        let db = null;
+        try {
+            const dbClient = await DuckDBClient.connect({ databasePath, ensureDirectory: true });
+            db = dbClient;
+            await ensureBaseSchema(dbClient);
+            // Phase 3: Ensure FTS metadata columns exist for existing DBs (migration)
+            await ensureRepoMetaColumns(dbClient);
+            const [headCommit, defaultBranch] = await Promise.all([
+                getHeadCommit(repoRoot),
+                getDefaultBranch(repoRoot),
+            ]);
+            const repoId = await ensureRepo(dbClient, repoRoot, defaultBranch, repoPathCandidates);
+            // Incremental mode: only reindex files in changedPaths (empty array means no-op)
+            if (options.changedPaths) {
+                // First, reconcile deleted files (handle renames/deletions)
+                const deletedPaths = await reconcileDeletedFiles(dbClient, repoId, repoRoot);
+                if (deletedPaths.length > 0) {
+                    console.info(`Removed ${deletedPaths.length} deleted file(s) from index.`);
                 }
-                else {
-                    await db.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP WHERE id = ?", [repoId]);
+                const existingHashes = await getExistingFileHashes(dbClient, repoId);
+                const { blobs, files, embeddings, missingPaths } = await scanFilesInBatches(repoRoot, options.changedPaths);
+                // Filter out files that haven't actually changed (same hash)
+                const changedFiles = [];
+                const changedBlobs = new Map();
+                for (const file of files) {
+                    const existingHash = existingHashes.get(file.path);
+                    if (existingHash !== file.blobHash) {
+                        changedFiles.push(file);
+                        const blob = blobs.get(file.blobHash);
+                        if (blob) {
+                            changedBlobs.set(blob.hash, blob);
+                        }
+                    }
                 }
-                return;
-            }
-            // Process all changed files in a single transaction for atomicity
-            const fileSet = new Set(files.map((f) => f.path));
-            let processedCount = 0;
-            await db.transaction(async () => {
-                for (const file of changedFiles) {
-                    const blob = changedBlobs.get(file.blobHash);
-                    if (!blob)
-                        continue;
-                    // Build code intelligence for this file
-                    const fileSymbols = [];
-                    const fileSnippets = [];
-                    const fileDependencies = [];
-                    if (!file.isBinary && blob.content) {
-                        const analysis = analyzeSource(file.path, file.lang, blob.content, fileSet);
-                        for (const symbol of analysis.symbols) {
-                            fileSymbols.push({
-                                path: file.path,
-                                symbolId: symbol.symbolId,
-                                name: symbol.name,
-                                kind: symbol.kind,
-                                rangeStartLine: symbol.rangeStartLine,
-                                rangeEndLine: symbol.rangeEndLine,
-                                signature: symbol.signature,
-                                doc: symbol.doc,
-                            });
+                if (changedFiles.length === 0 && missingPaths.length === 0) {
+                    console.info(`No actual changes detected in ${options.changedPaths.length} file(s). Skipping reindex.`);
+                    // Fix #3 & #4: If files were deleted (git or watch mode), still need to dirty FTS and rebuild
+                    if (deletedPaths.length > 0) {
+                        console.info(`${deletedPaths.length} file(s) deleted (git) - marking FTS dirty`);
+                        if (defaultBranch) {
+                            await dbClient.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP, default_branch = ?, fts_dirty = true, fts_generation = fts_generation + 1 WHERE id = ?", [defaultBranch, repoId]);
                         }
-                        for (const snippet of analysis.snippets) {
+                        else {
+                            await dbClient.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP, fts_dirty = true, fts_generation = fts_generation + 1 WHERE id = ?", [repoId]);
+                        }
+                        await rebuildFTSIfNeeded(dbClient, repoId);
+                    }
+                    else {
+                        // No deletions either - just update timestamp
+                        if (defaultBranch) {
+                            await dbClient.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP, default_branch = ? WHERE id = ?", [defaultBranch, repoId]);
+                        }
+                        else {
+                            await dbClient.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP WHERE id = ?", [
+                                repoId,
+                            ]);
+                        }
+                    }
+                    return;
+                }
+                // Process all changed files in a single transaction for atomicity
+                const fileSet = new Set(files.map((f) => f.path));
+                const embeddingMap = new Map();
+                for (const embedding of embeddings) {
+                    embeddingMap.set(embedding.path, embedding);
+                }
+                let processedCount = 0;
+                await dbClient.transaction(async () => {
+                    // Fix #5: Handle deleted files from watch mode (uncommitted deletions) INSIDE transaction
+                    // This ensures deletion + FTS dirty flag update are atomic
+                    if (missingPaths.length > 0) {
+                        // Loop through each missing file and delete with headCommit
+                        for (const path of missingPaths) {
+                            await deleteFileRecords(dbClient, repoId, headCommit, path);
+                        }
+                        console.info(`Removed ${missingPaths.length} missing file(s) from index (watch mode deletion).`);
+                    }
+                    // Process changed files
+                    for (const file of changedFiles) {
+                        const blob = changedBlobs.get(file.blobHash);
+                        if (!blob)
+                            continue;
+                        // Build code intelligence for this file
+                        const fileSymbols = [];
+                        const fileSnippets = [];
+                        const fileDependencies = [];
+                        if (!file.isBinary && blob.content) {
+                            const analysis = await analyzeSource(file.path, file.lang, blob.content, fileSet, repoRoot);
+                            for (const symbol of analysis.symbols) {
+                                fileSymbols.push({
+                                    path: file.path,
+                                    symbolId: symbol.symbolId,
+                                    name: symbol.name,
+                                    kind: symbol.kind,
+                                    rangeStartLine: symbol.rangeStartLine,
+                                    rangeEndLine: symbol.rangeEndLine,
+                                    signature: symbol.signature,
+                                    doc: symbol.doc,
+                                });
+                            }
+                            for (const snippet of analysis.snippets) {
+                                fileSnippets.push({
+                                    path: file.path,
+                                    snippetId: snippet.startLine,
+                                    startLine: snippet.startLine,
+                                    endLine: snippet.endLine,
+                                    symbolId: snippet.symbolId,
+                                });
+                            }
+                            for (const dep of analysis.dependencies) {
+                                fileDependencies.push({
+                                    srcPath: file.path,
+                                    dstKind: dep.dstKind,
+                                    dst: dep.dst,
+                                    rel: dep.rel,
+                                });
+                            }
+                        }
+                        else {
+                            // Binary or no content: add fallback snippet
+                            const fallback = buildFallbackSnippet(blob.lineCount ?? 1);
                             fileSnippets.push({
                                 path: file.path,
-                                snippetId: snippet.startLine,
-                                startLine: snippet.startLine,
-                                endLine: snippet.endLine,
-                                symbolId: snippet.symbolId,
+                                snippetId: fallback.startLine,
+                                startLine: fallback.startLine,
+                                endLine: fallback.endLine,
+                                symbolId: fallback.symbolId,
                             });
                         }
-                        for (const dep of analysis.dependencies) {
-                            fileDependencies.push({
-                                srcPath: file.path,
-                                dstKind: dep.dstKind,
-                                dst: dep.dst,
-                                rel: dep.rel,
-                            });
+                        const fileEmbedding = embeddingMap.get(file.path) ?? null;
+                        // Delete old records for this file (within main transaction)
+                        await deleteFileRecords(dbClient, repoId, headCommit, file.path);
+                        // Insert new records (within main transaction)
+                        await persistBlobs(dbClient, new Map([[blob.hash, blob]]));
+                        await persistTrees(dbClient, repoId, headCommit, [file]);
+                        await persistFiles(dbClient, repoId, [file]);
+                        await persistSymbols(dbClient, repoId, fileSymbols);
+                        await persistSnippets(dbClient, repoId, fileSnippets);
+                        await persistDependencies(dbClient, repoId, fileDependencies);
+                        if (fileEmbedding) {
+                            await persistEmbeddings(dbClient, repoId, [fileEmbedding]);
                         }
+                        processedCount++;
                     }
-                    else {
-                        // Binary or no content: add fallback snippet
-                        const fallback = buildFallbackSnippet(blob.lineCount ?? 1);
-                        fileSnippets.push({
-                            path: file.path,
-                            snippetId: fallback.startLine,
-                            startLine: fallback.startLine,
-                            endLine: fallback.endLine,
-                            symbolId: fallback.symbolId,
-                        });
+                    // Update timestamp and mark FTS dirty inside transaction for atomicity
+                    // Fix: Increment fts_generation to prevent lost updates during concurrent rebuilds
+                    if (defaultBranch) {
+                        await dbClient.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP, default_branch = ?, fts_dirty = true, fts_generation = fts_generation + 1 WHERE id = ?", [defaultBranch, repoId]);
                     }
-                    const fileEmbedding = embeddings.find((e) => e.path === file.path) ?? null;
-                    // Delete old records for this file (within main transaction)
-                    await deleteFileRecords(db, repoId, headCommit, file.path);
-                    // Insert new records (within main transaction)
-                    await persistBlobs(db, new Map([[blob.hash, blob]]));
-                    await persistTrees(db, repoId, headCommit, [file]);
-                    await persistFiles(db, repoId, [file]);
-                    await persistSymbols(db, repoId, fileSymbols);
-                    await persistSnippets(db, repoId, fileSnippets);
-                    await persistDependencies(db, repoId, fileDependencies);
-                    if (fileEmbedding) {
-                        await persistEmbeddings(db, repoId, [fileEmbedding]);
+                    else {
+                        await dbClient.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP, fts_dirty = true, fts_generation = fts_generation + 1 WHERE id = ?", [repoId]);
                     }
-                    processedCount++;
-                }
-                // Update timestamp inside main transaction
+                });
+                console.info(`Incrementally indexed ${processedCount} changed file(s) for repo ${repoRoot} at ${databasePath} (commit ${headCommit.slice(0, 12)})`);
+                // Phase 2+3: Rebuild FTS index after incremental updates (dirty=true triggers rebuild)
+                await rebuildFTSIfNeeded(dbClient, repoId);
+                return;
+            }
+            // Full mode: reindex entire repository
+            const paths = await gitLsFiles(repoRoot);
+            const { blobs, files, embeddings, missingPaths } = await scanFilesInBatches(repoRoot, paths);
+            // In full mode, missingPaths should be rare (git ls-files returns existing files)
+            // But log them if they occur (race condition: file deleted between ls-files and scan)
+            if (missingPaths.length > 0) {
+                console.warn(`${missingPaths.length} file(s) disappeared during full reindex (race condition)`);
+            }
+            const codeIntel = await buildCodeIntel(files, blobs, repoRoot);
+            await dbClient.transaction(async () => {
+                await dbClient.run("DELETE FROM tree WHERE repo_id = ?", [repoId]);
+                await dbClient.run("DELETE FROM file WHERE repo_id = ?", [repoId]);
+                await dbClient.run("DELETE FROM symbol WHERE repo_id = ?", [repoId]);
+                await dbClient.run("DELETE FROM snippet WHERE repo_id = ?", [repoId]);
+                await dbClient.run("DELETE FROM dependency WHERE repo_id = ?", [repoId]);
+                await dbClient.run("DELETE FROM file_embedding WHERE repo_id = ?", [repoId]);
+                await persistBlobs(dbClient, blobs);
+                await persistTrees(dbClient, repoId, headCommit, files);
+                await persistFiles(dbClient, repoId, files);
+                await persistSymbols(dbClient, repoId, codeIntel.symbols);
+                await persistSnippets(dbClient, repoId, codeIntel.snippets);
+                await persistDependencies(dbClient, repoId, codeIntel.dependencies);
+                await persistEmbeddings(dbClient, repoId, embeddings);
+                // Update timestamp and mark FTS dirty inside transaction to ensure atomicity
+                // Fix: Increment fts_generation to prevent lost updates during concurrent rebuilds
                 if (defaultBranch) {
-                    await db.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP, default_branch = ? WHERE id = ?", [defaultBranch, repoId]);
+                    await dbClient.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP, default_branch = ?, fts_dirty = true, fts_generation = fts_generation + 1 WHERE id = ?", [defaultBranch, repoId]);
                 }
                 else {
-                    await db.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP WHERE id = ?", [repoId]);
+                    await dbClient.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP, fts_dirty = true, fts_generation = fts_generation + 1 WHERE id = ?", [repoId]);
                 }
             });
-            console.info(`Incrementally indexed ${processedCount} changed file(s) for repo ${repoRoot} at ${databasePath} (commit ${headCommit.slice(0, 12)})`);
-            return;
+            console.info(`Indexed ${files.length} files for repo ${repoRoot} at ${databasePath} (commit ${headCommit.slice(0, 12)})`);
+            // Phase 2+3: Force rebuild FTS index after full reindex
+            await rebuildFTSIfNeeded(dbClient, repoId, true);
         }
-        // Full mode: reindex entire repository
-        const paths = await gitLsFiles(repoRoot);
-        const { blobs, files, embeddings } = await scanFilesInBatches(repoRoot, paths);
-        const codeIntel = buildCodeIntel(files, blobs);
-        await db.transaction(async () => {
-            await db.run("DELETE FROM tree WHERE repo_id = ?", [repoId]);
-            await db.run("DELETE FROM file WHERE repo_id = ?", [repoId]);
-            await db.run("DELETE FROM symbol WHERE repo_id = ?", [repoId]);
-            await db.run("DELETE FROM snippet WHERE repo_id = ?", [repoId]);
-            await db.run("DELETE FROM dependency WHERE repo_id = ?", [repoId]);
-            await db.run("DELETE FROM file_embedding WHERE repo_id = ?", [repoId]);
-            await persistBlobs(db, blobs);
-            await persistTrees(db, repoId, headCommit, files);
-            await persistFiles(db, repoId, files);
-            await persistSymbols(db, repoId, codeIntel.symbols);
-            await persistSnippets(db, repoId, codeIntel.snippets);
-            await persistDependencies(db, repoId, codeIntel.dependencies);
-            await persistEmbeddings(db, repoId, embeddings);
-            // Update timestamp inside transaction to ensure atomicity
-            if (defaultBranch) {
-                await db.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP, default_branch = ? WHERE id = ?", [defaultBranch, repoId]);
+        finally {
+            // Fix #2: Ensure lock is released even if DB connection fails
+            if (db) {
+                await db.close();
             }
-            else {
-                await db.run("UPDATE repo SET indexed_at = CURRENT_TIMESTAMP WHERE id = ?", [repoId]);
+            if (lockAcquired) {
+                releaseLock(lockfilePath);
             }
-        });
-        console.info(`Indexed ${files.length} files for repo ${repoRoot} at ${databasePath} (commit ${headCommit.slice(0, 12)})`);
-    }
-    finally {
-        await db.close();
-    }
+        }
+    });
 }
 function parseArg(flag) {
     const index = process.argv.indexOf(flag);
@@ -554,12 +786,26 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
     const watch = process.argv.includes("--watch");
     const debounceMs = parseInt(parseArg("--debounce") ?? "500", 10);
     const options = { repoRoot, databasePath, full: full || !since };
-    if (since) {
-        options.since = since;
-    }
-    // Run initial indexing
-    runIndexer(options)
-        .then(async () => {
+    const main = async () => {
+        if (since) {
+            options.since = since;
+            if (!options.full) {
+                const diffPaths = await gitDiffNameOnly(repoRoot, since);
+                options.changedPaths = diffPaths;
+                if (diffPaths.length === 0) {
+                    console.info(`No tracked changes since ${since}. Skipping incremental scan.`);
+                }
+            }
+        }
+        const dbMissing = !existsSync(databasePath);
+        const shouldIndex = options.full || !options.changedPaths || options.changedPaths.length > 0 || dbMissing;
+        if (shouldIndex) {
+            await runIndexer(options);
+        }
+        else {
+            // No diff results and not running full indexing: keep metadata fresh without DB writes
+            console.info("No files to reindex. Database remains unchanged.");
+        }
         if (watch) {
             // Start watch mode after initial indexing completes
             const abortController = new AbortController();
@@ -578,8 +824,8 @@ if (import.meta.url === pathToFileURL(process.argv[1] ?? "").href) {
             process.on("SIGTERM", shutdownHandler);
             await watcher.start();
         }
-    })
-        .catch((error) => {
+    };
+    main().catch((error) => {
         console.error("Failed to index repository. Retry after resolving the logged error.");
         console.error(error);
         process.exitCode = 1;