@optave/codegraph 3.11.0 → 3.11.2

package/dist/domain/graph/builder/pipeline.js

@@ -7,20 +7,17 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import { performance } from 'node:perf_hooks';
-import { acquireAdvisoryLock, closeDb, closeDbPair, getBuildMeta, initSchema, MIGRATIONS, openDb, purgeFilesData, releaseAdvisoryLock, setBuildMeta, } from '../../../db/index.js';
+import { closeDb, closeDbPair, getBuildMeta, initSchema, MIGRATIONS, openDb, } from '../../../db/index.js';
 import { detectWorkspaces, loadConfig } from '../../../infrastructure/config.js';
 import { debug, info, warn } from '../../../infrastructure/logger.js';
 import { loadNative } from '../../../infrastructure/native.js';
-import { semverCompare } from '../../../infrastructure/update-check.js';
-import { normalizePath } from '../../../shared/constants.js';
 import { toErrorMessage } from '../../../shared/errors.js';
 import { CODEGRAPH_VERSION } from '../../../shared/version.js';
-import { classifyNativeDrops, formatDropExtensionSummary, getActiveEngine, getInstalledWasmExtensions, NATIVE_SUPPORTED_EXTENSIONS, parseFilesWasmForBackfill, } from '../../parser.js';
+import { getActiveEngine } from '../../parser.js';
 import { writeJournalHeader } from '../journal.js';
 import { setWorkspaces } from '../resolve.js';
 import { PipelineContext } from './context.js';
-import { batchInsertNodes, collectFiles as collectFilesUtil, fileHash, fileStat, loadPathAliases, readFileSafe, } from './helpers.js';
-import { NativeDbProxy } from './native-db-proxy.js';
+import { loadPathAliases } from './helpers.js';
 import { buildEdges } from './stages/build-edges.js';
 import { buildStructure } from './stages/build-structure.js';
 // Pipeline stages
@@ -28,9 +25,14 @@ import { collectFiles } from './stages/collect-files.js';
 import { detectChanges, detectNoChanges } from './stages/detect-changes.js';
 import { finalize } from './stages/finalize.js';
 import { insertNodes } from './stages/insert-nodes.js';
+import { closeNativeDb, refreshJsDb, reopenNativeDb, suspendNativeDb, } from './stages/native-db-lifecycle.js';
+import { tryNativeOrchestrator } from './stages/native-orchestrator.js';
 import { parseFiles } from './stages/parse-files.js';
 import { resolveImports } from './stages/resolve-imports.js';
 import { runAnalyses } from './stages/run-analyses.js';
+// Re-export computeWasmOnlyStaleFiles for backward compatibility with tests
+// that import from this module path (#1073 unit tests).
+export { computeWasmOnlyStaleFiles, } from './stages/native-orchestrator.js';
 // ── Setup helpers ───────────────────────────────────────────────────────
 function initializeEngine(ctx) {
     ctx.engineOpts = {
@@ -180,766 +182,8 @@ function formatTimingResult(ctx) {
         },
     };
 }
-// ── NativeDb lifecycle helpers ──────────────────────────────────────────
-/** Checkpoint WAL through rusqlite and close the native connection. */
-function closeNativeDb(ctx, label) {
-    if (!ctx.nativeDb)
-        return;
-    try {
-        ctx.nativeDb.exec('PRAGMA wal_checkpoint(TRUNCATE)');
-    }
-    catch (e) {
-        debug(`${label} WAL checkpoint failed: ${toErrorMessage(e)}`);
-    }
-    try {
-        ctx.nativeDb.close();
-    }
-    catch (e) {
-        debug(`${label} nativeDb close failed: ${toErrorMessage(e)}`);
-    }
-    ctx.nativeDb = undefined;
-}
-/** Try to reopen the native connection for a given pipeline phase. */
-function reopenNativeDb(ctx, label) {
-    if ((ctx.opts.engine ?? 'auto') === 'wasm')
-        return;
-    const native = loadNative();
-    if (!native?.NativeDatabase)
-        return;
-    try {
-        ctx.nativeDb = native.NativeDatabase.openReadWrite(ctx.dbPath);
-    }
-    catch (e) {
-        debug(`reopen nativeDb for ${label} failed: ${toErrorMessage(e)}`);
-        ctx.nativeDb = undefined;
-    }
-}
-/** Close nativeDb and clear stale references in engineOpts. */
-function suspendNativeDb(ctx, label) {
-    closeNativeDb(ctx, label);
-    if (ctx.engineOpts?.nativeDb) {
-        ctx.engineOpts.nativeDb = undefined;
-    }
-}
-/**
- * After native writes, reopen the JS db connection to get a fresh page cache.
- * Rusqlite WAL truncation invalidates better-sqlite3's internal WAL index,
- * causing SQLITE_CORRUPT on the next read (#715, #736).
- */
-function refreshJsDb(ctx) {
-    try {
-        ctx.db.close();
-    }
-    catch (e) {
-        debug(`refreshJsDb close failed: ${toErrorMessage(e)}`);
-    }
-    ctx.db = openDb(ctx.dbPath);
-}
-// ── Native orchestrator helpers ───────────────────────────────────────
-/** Determine whether the native orchestrator should be skipped. Returns a reason string, or null if it should run. */
-function shouldSkipNativeOrchestrator(ctx) {
-    if (ctx.forceFullRebuild)
-        return 'forceFullRebuild';
-    // v3.9.0 addon had buggy incremental purge (wrong SQL on analysis tables,
-    // scoped removal over-detection). Fixed in v3.9.1 by PR #865. Gate on
-    // < 3.9.1 so v3.9.1+ uses the fast Rust orchestrator path.
-    const orchestratorBuggy = !!ctx.engineVersion && semverCompare(ctx.engineVersion, '3.9.1') < 0;
-    if (orchestratorBuggy)
-        return `buggy addon ${ctx.engineVersion}`;
-    if (ctx.engineName !== 'native')
-        return `engine=${ctx.engineName}`;
-    return null;
-}
-/** Checkpoint WAL through rusqlite, close nativeDb, and reopen better-sqlite3.
- *  Returns false if the DB reopen fails (caller should return partial result). */
-function handoffWalAfterNativeBuild(ctx) {
-    closeNativeDb(ctx, 'post-native-build');
-    try {
-        ctx.db.close();
-    }
-    catch (e) {
-        debug(`handoffWal JS db close failed: ${toErrorMessage(e)}`);
-    }
-    try {
-        ctx.db = openDb(ctx.dbPath);
-        return true;
-    }
-    catch (reopenErr) {
-        warn(`Failed to reopen DB after native build: ${reopenErr.message}`);
-        return false;
-    }
-}
-/**
- * Reconstruct fileSymbols from the DB after a native orchestrator build.
- * When `scopeFiles` is provided, only loads those files (for analysis-only).
- * When omitted, loads all files (needed for structure rebuilds).
- */
-function reconstructFileSymbolsFromDb(ctx, scopeFiles) {
-    let query = 'SELECT file, name, kind, line, end_line as endLine FROM nodes WHERE file IS NOT NULL';
-    const params = [];
-    if (scopeFiles && scopeFiles.length > 0) {
-        const placeholders = scopeFiles.map(() => '?').join(',');
-        query += ` AND file IN (${placeholders})`;
-        params.push(...scopeFiles);
-    }
-    query += ' ORDER BY file, line';
-    const rows = ctx.db.prepare(query).all(...params);
-    const fileSymbols = new Map();
-    for (const row of rows) {
-        let entry = fileSymbols.get(row.file);
-        if (!entry) {
-            entry = {
-                definitions: [],
-                calls: [],
-                imports: [],
-                classes: [],
-                exports: [],
-                typeMap: new Map(),
-            };
-            fileSymbols.set(row.file, entry);
-        }
-        entry.definitions.push({
-            name: row.name,
-            kind: row.kind,
-            line: row.line,
-            endLine: row.endLine ?? undefined,
-        });
-    }
-    // Populate import/export counts from DB edges so buildStructure
-    // computes correct import_count/export_count in node_metrics.
-    // The extractor arrays aren't persisted to the DB, so we derive
-    // counts from edge data instead (#804).
-    const importCountRows = ctx.db
-        .prepare(`SELECT n.file, COUNT(*) AS cnt
-       FROM edges e JOIN nodes n ON e.source_id = n.id
-       WHERE e.kind IN ('imports', 'imports-type', 'dynamic-imports')
-         AND n.file IS NOT NULL
-       GROUP BY n.file`)
-        .all();
-    for (const row of importCountRows) {
-        const entry = fileSymbols.get(row.file);
-        if (entry)
-            entry.imports = new Array(row.cnt);
-    }
-    const exportCountRows = ctx.db
-        .prepare(`SELECT n_tgt.file, COUNT(DISTINCT n_tgt.id) AS cnt
-       FROM edges e
-       JOIN nodes n_tgt ON e.target_id = n_tgt.id
-       JOIN nodes n_src ON e.source_id = n_src.id
-       WHERE e.kind IN ('imports', 'imports-type', 'reexports')
-         AND n_tgt.file IS NOT NULL
-         AND n_src.file != n_tgt.file
-       GROUP BY n_tgt.file`)
-        .all();
-    for (const row of exportCountRows) {
-        const entry = fileSymbols.get(row.file);
-        if (entry)
-            entry.exports = new Array(row.cnt);
-    }
-    return fileSymbols;
-}
-/**
- * Run JS buildStructure() after native orchestrator to fill directory nodes + contains edges.
- * For full builds, passes changedFiles=null (full rebuild).
- * For incremental builds, passes the changed file list to scope the update.
- */
-async function runPostNativeStructure(ctx, allFileSymbols, isFullBuild, changedFiles) {
-    const structureStart = performance.now();
-    try {
-        const directories = new Set();
-        for (const relPath of allFileSymbols.keys()) {
-            const parts = relPath.split('/');
-            for (let i = 1; i < parts.length; i++) {
-                directories.add(parts.slice(0, i).join('/'));
-            }
-        }
-        const lineCountMap = new Map();
-        const cachedLineCounts = ctx.db
-            .prepare(`SELECT n.name AS file, m.line_count
-         FROM node_metrics m JOIN nodes n ON m.node_id = n.id
-         WHERE n.kind = 'file'`)
-            .all();
-        for (const row of cachedLineCounts) {
-            lineCountMap.set(row.file, row.line_count);
-        }
-        // Full builds need null (rebuild everything). Incremental builds pass the
-        // changed file list so buildStructure only updates those files' metrics
-        // and contains edges — matching the JS pipeline's medium-incremental path.
-        const changedFilePaths = isFullBuild || !changedFiles?.length ? null : changedFiles;
-        const { buildStructure: buildStructureFn } = (await import('../../../features/structure.js'));
-        buildStructureFn(ctx.db, allFileSymbols, ctx.rootDir, lineCountMap, directories, changedFilePaths);
-        debug(`Structure phase completed after native orchestrator${changedFilePaths ? ` (${changedFilePaths.length} files)` : ' (full)'}`);
-    }
-    catch (err) {
-        warn(`Structure phase failed after native build: ${toErrorMessage(err)}`);
-    }
-    return performance.now() - structureStart;
-}
-/**
- * JS fallback for AST/complexity/CFG/dataflow analysis after native orchestrator.
- * Used when the Rust addon doesn't include analysis persistence (older addon
- * version) or when analysis failed on the Rust side.
- */
-async function runPostNativeAnalysis(ctx, allFileSymbols, changedFiles) {
-    const timing = { astMs: 0, complexityMs: 0, cfgMs: 0, dataflowMs: 0 };
-    // Scope analysis fileSymbols to changed files only
-    let analysisFileSymbols;
-    if (changedFiles && changedFiles.length > 0) {
-        analysisFileSymbols = new Map();
-        for (const f of changedFiles) {
-            const entry = allFileSymbols.get(f);
-            if (entry)
-                analysisFileSymbols.set(f, entry);
-        }
-    }
-    else {
-        analysisFileSymbols = allFileSymbols;
-    }
-    // Reopen nativeDb for analysis features (suspend/resume WAL pattern).
-    const native = loadNative();
-    if (native?.NativeDatabase) {
-        try {
-            ctx.nativeDb = native.NativeDatabase.openReadWrite(ctx.dbPath);
-            if (ctx.engineOpts)
-                ctx.engineOpts.nativeDb = ctx.nativeDb;
-        }
-        catch {
-            ctx.nativeDb = undefined;
-            if (ctx.engineOpts)
-                ctx.engineOpts.nativeDb = undefined;
-        }
-    }
-    // Flush JS WAL pages once so Rust can see them, then no-op callbacks.
-    // Previously each feature called wal_checkpoint(TRUNCATE) individually
-    // (~68ms each × 3-4 features). One FULL checkpoint suffices.
-    if (ctx.nativeDb && ctx.engineOpts) {
-        ctx.db.pragma('wal_checkpoint(FULL)');
-        ctx.engineOpts.suspendJsDb = () => { };
-        ctx.engineOpts.resumeJsDb = () => { };
-    }
-    try {
-        const { runAnalyses: runAnalysesFn } = (await import('../../../ast-analysis/engine.js'));
-        const result = await runAnalysesFn(ctx.db, analysisFileSymbols, ctx.rootDir, ctx.opts, ctx.engineOpts);
-        timing.astMs = result.astMs ?? 0;
-        timing.complexityMs = result.complexityMs ?? 0;
-        timing.cfgMs = result.cfgMs ?? 0;
-        timing.dataflowMs = result.dataflowMs ?? 0;
-    }
-    catch (err) {
-        warn(`Analysis phases failed after native build: ${toErrorMessage(err)}`);
-    }
-    // Close nativeDb after analyses — TRUNCATE checkpoint flushes all Rust
-    // WAL writes so JS and external readers can see them. Runs once after
-    // all analysis features complete (not per-feature).
-    if (ctx.nativeDb) {
-        try {
-            ctx.nativeDb.exec('PRAGMA wal_checkpoint(TRUNCATE)');
-        }
-        catch {
-            /* ignore checkpoint errors */
-        }
-        try {
-            ctx.nativeDb.close();
-        }
-        catch {
-            /* ignore close errors */
-        }
-        ctx.nativeDb = undefined;
-        if (ctx.engineOpts) {
-            ctx.engineOpts.nativeDb = undefined;
-            ctx.engineOpts.suspendJsDb = undefined;
-            ctx.engineOpts.resumeJsDb = undefined;
-        }
-    }
-    return timing;
-}
-/** Format timing result from native orchestrator phases + JS post-processing. */
-function formatNativeTimingResult(p, structurePatchMs, analysisTiming) {
-    return {
-        phases: {
-            setupMs: +(p.setupMs ?? 0).toFixed(1),
-            collectMs: +(p.collectMs ?? 0).toFixed(1),
-            detectMs: +(p.detectMs ?? 0).toFixed(1),
-            parseMs: +(p.parseMs ?? 0).toFixed(1),
-            insertMs: +(p.insertMs ?? 0).toFixed(1),
-            resolveMs: +(p.resolveMs ?? 0).toFixed(1),
-            edgesMs: +(p.edgesMs ?? 0).toFixed(1),
-            structureMs: +((p.structureMs ?? 0) + structurePatchMs).toFixed(1),
-            rolesMs: +(p.rolesMs ?? 0).toFixed(1),
-            astMs: +(analysisTiming.astMs ?? 0).toFixed(1),
-            complexityMs: +(analysisTiming.complexityMs ?? 0).toFixed(1),
-            cfgMs: +(analysisTiming.cfgMs ?? 0).toFixed(1),
-            dataflowMs: +(analysisTiming.dataflowMs ?? 0).toFixed(1),
-            finalizeMs: +(p.finalizeMs ?? 0).toFixed(1),
-        },
-    };
-}
-/** Try the native build orchestrator. Returns a BuildResult on success, undefined to fall through to JS pipeline. */
-async function tryNativeOrchestrator(ctx) {
-    const skipReason = shouldSkipNativeOrchestrator(ctx);
-    if (skipReason) {
-        debug(`Skipping native orchestrator: ${skipReason}`);
-        return undefined;
-    }
-    // Open NativeDatabase on demand — deferred from setupPipeline to skip the
-    // ~60ms cost on no-op/early-exit builds. Close the better-sqlite3 connection
-    // first to avoid dual-connection WAL corruption.
-    if (!ctx.nativeDb && ctx.nativeAvailable) {
-        const native = loadNative();
-        if (native?.NativeDatabase) {
-            try {
-                // Close better-sqlite3 before opening rusqlite to avoid WAL conflicts.
-                // Uses raw close() instead of closeDb() intentionally — the advisory lock
-                // is kept and transferred to the NativeDbProxy below, not released here.
-                ctx.db.close();
-                acquireAdvisoryLock(ctx.dbPath);
-                ctx.nativeDb = native.NativeDatabase.openReadWrite(ctx.dbPath);
-                ctx.nativeDb.initSchema();
-                // Replace ctx.db with a NativeDbProxy so post-native JS fallback
-                // (structure, analysis) can use it without reopening better-sqlite3.
-                const proxy = new NativeDbProxy(ctx.nativeDb);
-                proxy.__lockPath = `${ctx.dbPath}.lock`;
-                ctx.db = proxy;
-                ctx.nativeFirstProxy = true;
-            }
-            catch (err) {
-                warn(`NativeDatabase setup failed, falling back to JS: ${toErrorMessage(err)}`);
-                try {
-                    ctx.nativeDb?.close();
-                }
-                catch (e) {
-                    debug(`tryNativeOrchestrator: close failed during fallback: ${toErrorMessage(e)}`);
-                }
-                ctx.nativeDb = undefined;
-                ctx.nativeFirstProxy = false; // defensive: reset in case future refactors move the assignment above throwing lines
-                releaseAdvisoryLock(`${ctx.dbPath}.lock`);
-                // Reopen better-sqlite3 for JS pipeline fallback
-                ctx.db = openDb(ctx.dbPath);
-            }
-        }
-    }
-    if (!ctx.nativeDb?.buildGraph)
-        return undefined;
-    const resultJson = ctx.nativeDb.buildGraph(ctx.rootDir, JSON.stringify(ctx.config), JSON.stringify(ctx.aliases), JSON.stringify(ctx.opts));
-    const result = JSON.parse(resultJson);
-    if (result.earlyExit) {
-        info('No changes detected');
-        // Even on no-op rebuilds, dropped-language files added since the last
-        // full build are still missing from `nodes`/`file_hashes` (#1083), and
-        // WASM-only files deleted from disk leave stale rows behind (#1073).
-        // The orchestrator's file_collector skipped them, so its earlyExit
-        // doesn't imply DB consistency. Run the gap repair before returning.
-        const gap = detectDroppedLanguageGap(ctx);
-        if (gap.missingAbs.length > 0 || gap.staleRel.length > 0) {
-            await backfillNativeDroppedFiles(ctx, gap);
-        }
-        closeDbPair({ db: ctx.db, nativeDb: ctx.nativeDb });
-        return 'early-exit';
-    }
-    // Log incremental status to match JS pipeline output
-    const changed = result.changedCount ?? 0;
-    const removed = result.removedCount ?? 0;
-    if (!result.isFullBuild && (changed > 0 || removed > 0)) {
-        info(`Incremental: ${changed} changed, ${removed} removed`);
-    }
-    const p = result.phases;
-    // Sync build_meta so JS-side version/engine checks work on next build.
-    // Use the binary's CARGO_PKG_VERSION (ctx.nativeBinaryVersion), not the
-    // platform package.json version (ctx.engineVersion). The Rust side's
-    // check_version_mismatch compares against CARGO_PKG_VERSION; writing
-    // the package.json value would create a permanent mismatch whenever
-    // the binary and platform package.json diverge — e.g., CI hot-swap
-    // via ci-install-native.mjs (#1066) — forcing every subsequent build
-    // to be a full rebuild.
-    //
-    // When the native addon doesn't expose engineVersion() (older addon),
-    // fall back to CODEGRAPH_VERSION — same fallback used by both
-    // checkEngineSchemaMismatch (read path) and persistBuildMetadata
-    // (the JS-pipeline write path in finalize.ts). Using ctx.engineVersion
-    // here would re-introduce the asymmetry this PR fixes for that case.
-    const nativeVersionForMeta = ctx.nativeBinaryVersion || CODEGRAPH_VERSION;
-    setBuildMeta(ctx.db, {
-        engine: ctx.engineName,
-        engine_version: nativeVersionForMeta,
-        codegraph_version: nativeVersionForMeta,
-        schema_version: String(ctx.schemaVersion),
-        built_at: new Date().toISOString(),
-    });
-    info(`Native build orchestrator completed: ${result.nodeCount ?? 0} nodes, ${result.edgeCount ?? 0} edges, ${result.fileCount ?? 0} files`);
-    // ── Post-native structure + analysis ──────────────────────────────
-    let analysisTiming = {
-        astMs: +(p.astMs ?? 0),
-        complexityMs: +(p.complexityMs ?? 0),
-        cfgMs: +(p.cfgMs ?? 0),
-        dataflowMs: +(p.dataflowMs ?? 0),
-    };
-    let structurePatchMs = 0;
-    // Skip JS structure when the Rust pipeline's small-incremental fast path
-    // already handled it. For full builds and large incrementals where Rust
-    // skipped structure, we must run the JS fallback.
-    const needsStructure = !result.structureHandled;
-    // When the Rust addon doesn't include analysis persistence (older addon
-    // version or analysis failed), fall back to JS-side analysis.
-    const needsAnalysisFallback = !result.analysisComplete &&
-        (ctx.opts.ast !== false ||
-            ctx.opts.complexity !== false ||
-            ctx.opts.cfg !== false ||
-            ctx.opts.dataflow !== false);
-    if (needsStructure || needsAnalysisFallback) {
-        // When analysis fallback is needed, handoff to better-sqlite3 — the
-        // analysis engine uses the suspend/resume WAL pattern that requires a
-        // real better-sqlite3 connection, not the NativeDbProxy.
-        if (needsAnalysisFallback && ctx.nativeFirstProxy) {
-            closeNativeDb(ctx, 'pre-analysis-fallback');
-            ctx.db = openDb(ctx.dbPath);
-            ctx.nativeFirstProxy = false;
-        }
-        else if (!ctx.nativeFirstProxy && !handoffWalAfterNativeBuild(ctx)) {
-            // DB reopen failed — return partial result
-            return formatNativeTimingResult(p, 0, analysisTiming);
-        }
-        const fileSymbols = reconstructFileSymbolsFromDb(ctx);
-        if (needsStructure) {
-            structurePatchMs = await runPostNativeStructure(ctx, fileSymbols, !!result.isFullBuild, result.changedFiles);
-        }
-        if (needsAnalysisFallback) {
-            analysisTiming = await runPostNativeAnalysis(ctx, fileSymbols, result.changedFiles);
-        }
-    }
-    // Engine parity: the native orchestrator silently drops files whose
-    // Rust extractor/grammar is missing or fails (e.g. HCL, Scala, Swift on
-    // stale native binaries). WASM handles those — backfill via WASM so both
-    // engines process the same file set (#967).
-    //
-    // Detect the gap once (fs walk + 2 DB queries, ~20–30ms) and use it for
-    // both gating and the backfill itself. On dirty incrementals/full builds
-    // the orchestrator signals trigger backfill, so the walk happens once
-    // (instead of redundantly inside backfill). On quiet incrementals we
-    // still pay the walk so we can detect brand-new files in dropped-language
-    // extensions — a gap that the orchestrator's `detect_removed_files`
-    // filter (#1070) leaves open (#1083, #1091). The pre-check is cheap
-    // because the expensive part (WASM re-parse of the missing set) is
-    // gated below.
-    const removedCount = result.removedCount ?? 0;
-    const changedCount = result.changedCount ?? 0;
-    const gap = detectDroppedLanguageGap(ctx);
-    if (result.isFullBuild ||
-        removedCount > 0 ||
-        changedCount > 0 ||
-        gap.missingAbs.length > 0 ||
-        gap.staleRel.length > 0) {
-        await backfillNativeDroppedFiles(ctx, gap);
-    }
-    closeDbPair({ db: ctx.db, nativeDb: ctx.nativeDb });
-    return formatNativeTimingResult(p, structurePatchMs, analysisTiming);
-}
-/**
- * Compute the WASM-only files present in the DB but missing from disk (#1073).
- *
- * Returns relative paths that:
- *   - appear in `existingNodes` or `existingHashes` (in DB),
- *   - are absent from `expected` (not on disk),
- *   - have an extension installed for WASM, AND
- *   - have an extension NOT covered by `nativeSupported` — Rust's
- *     `purge_changed_files` handles deletion for natively-supported extensions
- *     via its own `detect_removed_files`, so the caller must not double-purge.
- *
- * Extensions are lowercased before lookup to match the registry and Rust's
- * `LanguageKind::from_extension` (which normalises case for the languages
- * where both cases are conventional, e.g. R's `.r` / `.R`).
- *
- * DB paths are forced to forward slashes before comparison with `expected`
- * (which is always normalised). The on-disk invariant is that DB rows are
- * written with forward slashes, but a stale row written by older code on
- * Windows could carry back-slashes — normalising here makes the comparison
- * platform-safe and prevents false-positive purges of live rows. We replace
- * `\\` explicitly (rather than calling `normalizePath`, which only touches
- * `path.sep`) so the defence works when running on POSIX against a DB that
- * was migrated from Windows.
- *
- * Exported for unit testing.
- */
-export function computeWasmOnlyStaleFiles(input) {
-    const { existingNodes, existingHashes, expected, installedExts, nativeSupported } = input;
-    const stale = [];
-    const seen = new Set();
-    const consider = (rawRel) => {
-        const rel = rawRel.replace(/\\/g, '/');
-        if (expected.has(rel) || seen.has(rel))
-            return;
-        const ext = path.extname(rel).toLowerCase();
-        if (nativeSupported.has(ext))
-            return;
-        if (!installedExts.has(ext))
-            return;
-        seen.add(rel);
-        // Push the ORIGINAL raw path (not the normalised form) so the eventual
-        // `DELETE FROM nodes WHERE file = ?` predicate in `purgeFilesData`
-        // matches the actual stored row. The dedup `seen` set keeps the
-        // normalised form so a file written once with `\` and once with `/`
-        // is still treated as one entry — but the value the SQL sees has to
-        // be byte-identical to what's on disk in the DB.
-        stale.push(rawRel);
-    };
-    for (const rel of existingNodes)
-        consider(rel);
-    for (const rel of existingHashes)
-        consider(rel);
-    return stale;
-}
-/**
- * Group relative paths by their lowercased extension. Shape matches the bucket
- * type that `formatDropExtensionSummary` consumes, so callers can render a
- * log-friendly per-extension summary without going through `classifyNativeDrops`
- * when the reason is already known (e.g. the stale-purge path where every path
- * is guaranteed `unsupported-by-native`).
- */
-function groupByExtension(relPaths) {
-    const buckets = new Map();
-    for (const rel of relPaths) {
-        const ext = path.extname(rel).toLowerCase();
-        let list = buckets.get(ext);
-        if (!list) {
-            list = [];
-            buckets.set(ext, list);
-        }
-        list.push(rel);
-    }
-    return buckets;
-}
-/**
- * Detect files the native orchestrator silently dropped.
- *
- * Walks the filesystem and compares against `nodes` + `file_hashes`. A file
- * is "missing" if it's absent from EITHER table — both must be present for
- * the fast-skip pre-flight (#1054) to work, and the two can diverge (e.g.
- * legacy DBs where `nodes` was populated but `file_hashes` was not).
- *
- * Restricted to files with an installed WASM grammar; extensions in
- * `LANGUAGE_REGISTRY` without a shipped grammar (e.g. groovy on minimal
- * installs) can't be parsed by either engine, so they're not a native
- * regression — excluding them keeps the warn count in
- * `backfillNativeDroppedFiles` meaningful.
- *
- * Also detects WASM-only files deleted from disk (#1073). Rust's
- * `detect_removed_files` filter (#1070) skips files outside its supported
- * extensions, so deletions of WASM-only languages don't reach the native
- * purge path; the rest of the backfill only inserts rows, so without this
- * step stale `nodes`/`file_hashes` rows would linger across incremental
- * rebuilds until the next full rebuild.
- *
- * Cheap (no DB handoff, no parsing): used both to gate the backfill call
- * and as its working set. NativeDbProxy supports `.prepare().all()`, so
- * this works whether `ctx.db` is a proxy or a real better-sqlite3
- * connection — letting us skip the close-native / reopen-better-sqlite3
- * cost when there's nothing to backfill.
- */
-function detectDroppedLanguageGap(ctx) {
-    const collected = collectFilesUtil(ctx.rootDir, [], ctx.config, new Set());
-    const expected = new Set(collected.files.map((f) => normalizePath(path.relative(ctx.rootDir, f))));
-    const existingNodeRows = ctx.db
-        .prepare("SELECT DISTINCT file FROM nodes WHERE kind = 'file'")
-        .all();
-    const existingNodes = new Set(existingNodeRows.map((r) => r.file));
-    let existingHashes = new Set();
-    try {
-        const existingHashRows = ctx.db
-            .prepare('SELECT DISTINCT file FROM file_hashes')
-            .all();
-        existingHashes = new Set(existingHashRows.map((r) => r.file));
-    }
-    catch (e) {
-        // file_hashes table may not exist on legacy DBs; treat as fully missing
-        // so the backfill writes rows on the upsert path below.
-        debug(`detectDroppedLanguageGap: file_hashes read failed (table may not exist): ${toErrorMessage(e)}`);
-    }
-    const installedExts = getInstalledWasmExtensions();
-    const missingRel = [];
-    const missingAbs = [];
-    for (const rel of expected) {
-        if (existingNodes.has(rel) && existingHashes.has(rel))
-            continue;
-        const ext = path.extname(rel).toLowerCase();
-        if (!installedExts.has(ext))
-            continue;
-        missingRel.push(rel);
-        missingAbs.push(path.join(ctx.rootDir, rel));
-    }
-    const staleRel = computeWasmOnlyStaleFiles({
-        existingNodes,
-        existingHashes,
-        expected,
-        installedExts,
-        nativeSupported: NATIVE_SUPPORTED_EXTENSIONS,
-    });
-    return { missingRel, missingAbs, staleRel };
-}
-/**
- * Backfill files that the native orchestrator silently dropped during parse.
- * Falls back to WASM + inserts file/symbol nodes so engine counts match (#967).
- *
- * Also purges stale rows for WASM-only files deleted from disk (#1073), which
- * Rust's `detect_removed_files` filter (#1070) skips.
- *
- * Accepts a pre-computed `gap` from `detectDroppedLanguageGap` so the caller
- * can use the same scan for both gating and the actual backfill — avoiding
- * a redundant fs walk when the orchestrator's signals already triggered.
- */
-async function backfillNativeDroppedFiles(ctx, gap) {
-    const { missingRel, missingAbs, staleRel } = gap;
-    if (missingAbs.length === 0 && staleRel.length === 0)
-        return;
-    // Now that we know there's work to do, hand off to better-sqlite3 (needed
-    // for the INSERT path below).
-    if (ctx.nativeFirstProxy) {
-        closeNativeDb(ctx, 'pre-parity-backfill');
-        ctx.db = openDb(ctx.dbPath);
-        ctx.nativeFirstProxy = false;
-    }
-    const dbConn = ctx.db;
-    // Purge WASM-only files that were deleted from disk (#1073). Rust's
-    // detect_removed_files skips them and the insert path below never visits
-    // them, so without this their rows would persist across rebuilds until the
-    // next full rebuild reset the DB.
-    if (staleRel.length > 0) {
-        // `computeWasmOnlyStaleFiles` guarantees every path here has an extension
-        // outside NATIVE_SUPPORTED_EXTENSIONS, so `classifyNativeDrops` would
-        // always bucket 100% into `unsupported-by-native`. Build the extension
-        // summary directly to avoid a redundant classification pass.
-        const staleByExt = groupByExtension(staleRel);
-        info(`Detected ${staleRel.length} deleted WASM-only file(s) the native orchestrator skipped; purging stale rows: ${formatDropExtensionSummary(staleByExt)}`);
-        purgeFilesData(dbConn, staleRel);
-    }
-    if (missingAbs.length === 0)
-        return;
-    // Classify drops so users see per-extension reasons instead of just a count
-    // (#1011). `unsupported-by-native` is a legitimate parser limit (no Rust
-    // extractor); `native-extractor-failure` indicates a real native bug since
-    // the language IS supported by the addon yet the file was dropped anyway.
-    const { byReason, totals } = classifyNativeDrops(missingRel);
-    if (totals['unsupported-by-native'] > 0) {
-        info(`Native orchestrator skipped ${totals['unsupported-by-native']} file(s) in languages without a Rust extractor; backfilling via WASM: ${formatDropExtensionSummary(byReason['unsupported-by-native'])}`);
-    }
-    if (totals['native-extractor-failure'] > 0) {
-        warn(`Native orchestrator dropped ${totals['native-extractor-failure']} file(s) in natively-supported languages — likely a Rust extractor bug. Backfilling via WASM: ${formatDropExtensionSummary(byReason['native-extractor-failure'])}`);
-    }
-    const wasmResults = await parseFilesWasmForBackfill(missingAbs, ctx.rootDir);
-    const rows = [];
-    const exportKeys = [];
-    for (const [relPath, symbols] of wasmResults) {
-        // File row — mirrors insertDefinitionsAndExports: qualified_name is null.
-        rows.push([relPath, 'file', relPath, 0, null, null, null, null, null]);
-        for (const def of symbols.definitions ?? []) {
-            // Populate qualified_name/scope the same way the JS fallback does so
-            // downstream queries (cross-file references, "go to definition") find
-            // these symbols.
-            const dotIdx = def.name.lastIndexOf('.');
-            const scope = dotIdx !== -1 ? def.name.slice(0, dotIdx) : null;
-            rows.push([
-                def.name,
-                def.kind,
-                relPath,
-                def.line,
-                def.endLine ?? null,
-                null,
-                def.name,
-                scope,
-                def.visibility ?? null,
-            ]);
-        }
-        // Exports: insert the row (INSERT OR IGNORE — a matching definition row
-        // is a no-op) and queue a key for the second-pass exported=1 update, so
-        // queries filtering on exported=1 find backfilled symbols (#970).
-        for (const exp of symbols.exports ?? []) {
-            rows.push([exp.name, exp.kind, relPath, exp.line, null, null, exp.name, null, null]);
-            exportKeys.push([exp.name, exp.kind, relPath, exp.line]);
-        }
-    }
-    const db = dbConn;
-    batchInsertNodes(db, rows);
-    // Mark exported symbols in batches — mirrors insertDefinitionsAndExports.
-    if (exportKeys.length > 0) {
-        const EXPORT_CHUNK = 500;
-        const exportStmtCache = new Map();
-        for (let i = 0; i < exportKeys.length; i += EXPORT_CHUNK) {
-            const end = Math.min(i + EXPORT_CHUNK, exportKeys.length);
-            const chunkSize = end - i;
-            let updateStmt = exportStmtCache.get(chunkSize);
-            if (!updateStmt) {
-                const conditions = Array.from({ length: chunkSize }, () => '(name = ? AND kind = ? AND file = ? AND line = ?)').join(' OR ');
-                updateStmt = db.prepare(`UPDATE nodes SET exported = 1 WHERE ${conditions}`);
-                exportStmtCache.set(chunkSize, updateStmt);
-            }
-            const vals = [];
-            for (let j = i; j < end; j++) {
-                const k = exportKeys[j];
-                vals.push(k[0], k[1], k[2], k[3]);
-            }
-            updateStmt.run(...vals);
-        }
-    }
-    // Persist file_hashes rows for every backfilled file. The Rust orchestrator
-    // only hashes files it parsed itself, so without this step files in
-    // optional-language extensions (e.g. .clj when no Rust extractor exists)
-    // would be missing from `file_hashes` — permanently breaking the JS-side
-    // fast-skip pre-flight (#1054), which rejects on `collected file missing
-    // from file_hashes` and forces every no-op rebuild back through the full
-    // ~2s native pipeline (#1068).
-    //
-    // Iterates `missingRel` (every collected file the Rust orchestrator
-    // dropped), not `wasmResults`, so files that produced zero symbols still
-    // get a row.
-    try {
-        const upsertHash = db.prepare('INSERT OR REPLACE INTO file_hashes (file, hash, mtime, size) VALUES (?, ?, ?, ?)');
-        const writeHashes = db.transaction(() => {
-            for (let i = 0; i < missingRel.length; i++) {
-                const relPath = missingRel[i];
-                const absPath = missingAbs[i];
-                if (!relPath || !absPath)
-                    continue;
-                let code;
-                try {
-                    code = readFileSafe(absPath);
-                }
-                catch (e) {
-                    debug(`backfillNativeDroppedFiles: read failed for ${relPath}: ${toErrorMessage(e)}`);
-                    continue;
-                }
-                if (code === null)
-                    continue;
-                const stat = fileStat(absPath);
-                const mtime = stat ? stat.mtime : 0;
-                const size = stat ? stat.size : 0;
-                upsertHash.run(relPath, fileHash(code), mtime, size);
-            }
-        });
-        writeHashes();
-    }
-    catch (e) {
-        debug(`backfillNativeDroppedFiles: file_hashes write failed (table may not exist): ${toErrorMessage(e)}`);
-    }
-    // Free WASM parse trees from the inline backfill path (#1058).
-    // `parseFilesWasmInline` sets `symbols._tree` (a live web-tree-sitter Tree
-    // backed by WASM linear memory) on every result, but these symbols are
-    // consumed locally for DB row construction and never added to
-    // `ctx.allSymbols`, so the finalize-stage `releaseWasmTrees` sweep never
-    // sees them. Without this, trees leak WASM memory until process exit —
-    // bounded per run but cumulative across in-process integration tests.
-    // Mirrors the cleanup discipline established for #931.
-    for (const [, symbols] of wasmResults) {
-        const tree = symbols._tree;
-        if (tree && typeof tree.delete === 'function') {
-            try {
-                tree.delete();
-            }
-            catch {
-                /* ignore cleanup errors */
-            }
-        }
-        symbols._tree = undefined;
-        symbols._langId = undefined;
-    }
-}
+// Native db lifecycle and orchestrator helpers live in dedicated stage
+// modules — see `./stages/native-db-lifecycle.ts` and `./stages/native-orchestrator.ts`.
 // ── Pipeline stages execution ───────────────────────────────────────────
 async function runPipelineStages(ctx) {
     // ── WASM / fallback dual-connection mode ─────────────────────────────