arceus-s 1.6.4

package/dist/core/ingestion/pipeline-phases/parse-impl.js

@@ -0,0 +1,567 @@
+/**
+ * Parse implementation — chunked parse + resolve loop.
+ *
+ * This is the core parsing engine of the ingestion pipeline. It reads
+ * source files in byte-budget chunks (~20MB each), parses via worker
+ * pool (or sequential fallback), resolves imports/calls/heritage per
+ * chunk, and synthesizes wildcard import bindings.
+ *
+ * Consumed by the parse phase (`parse.ts`) — the phase file handles
+ * dependency wiring while the heavy implementation lives here.
+ *
+ * @module
+ */
+import { BindingAccumulator, enrichExportedTypeMap, } from '../binding-accumulator.js';
+import { processParsing, mergeChunkResults } from '../parsing-processor.js';
+import { fileContentHash, computeChunkHash } from '../../../storage/parse-cache.js';
+import { processImports, processImportsFromExtracted, buildImportResolutionContext, } from '../import-processor.js';
+import { EMPTY_INDEX } from '../import-resolvers/utils.js';
+import { processCalls, processCallsFromExtracted, processAssignmentsFromExtracted, processRoutesFromExtracted, seedCrossFileReceiverTypes, buildExportedTypeMapFromGraph, } from '../call-processor.js';
+import { buildHeritageMap } from '../model/heritage-map.js';
+import { processHeritage, processHeritageFromExtracted, extractExtractedHeritageFromFiles, getHeritageStrategyForLanguage, } from '../heritage-processor.js';
+import { createResolutionContext } from '../model/resolution-context.js';
+import { createASTCache } from '../ast-cache.js';
+import { getLanguageFromFilename } from '../../../_shared/index.js';
+import { readFileContents } from '../filesystem-walker.js';
+import { isLanguageAvailable } from '../../tree-sitter/parser-loader.js';
+import { createWorkerPool } from '../workers/worker-pool.js';
+import { extractFetchCallsFromFiles } from '../call-processor.js';
+import fs from 'node:fs';
+import path from 'node:path';
+import { fileURLToPath, pathToFileURL } from 'node:url';
+import { isDev } from '../utils/env.js';
+import { synthesizeWildcardImportBindings, needsSynthesis } from './wildcard-synthesis.js';
+import { extractORMQueriesInline } from './orm-extraction.js';
+import { logger } from '../../logger.js';
+// ── Constants ──────────────────────────────────────────────────────────────
+/** Max bytes of source content to load per parse chunk.
+ *
+ * Memory bound for the worker pool dispatch + a granularity knob for
+ * the parse cache. A single file change invalidates only its enclosing
+ * chunk, so smaller budgets → finer-grained invalidation.
+ *
+ * Override via ARC_CHUNK_BYTE_BUDGET (bytes) — the default of 2MB
+ * gives a useful invalidation floor (~1/N chunks on a multi-MB repo)
+ * while keeping worker dispatch overhead under 5% on cold runs.
+ */
+const CHUNK_BYTE_BUDGET = (() => {
+    const env = Number(process.env.ARC_CHUNK_BYTE_BUDGET);
+    if (Number.isFinite(env) && env > 0)
+        return env;
+    return 2 * 1024 * 1024;
+})();
+/**
+ * Chunked parse + resolve loop.
+ *
+ * Reads source in byte-budget chunks (~20MB each). For each chunk:
+ * 1. Parse via worker pool (or sequential fallback)
+ * 2. Resolve imports from extracted data
+ * 3. Synthesize wildcard import bindings (Go/Ruby/C++/Swift/Python)
+ * 4. Resolve heritage + routes per chunk; defer worker CALLS until all chunks
+ *    have contributed heritage so interface-dispatch implementor map is complete
+ * 5. Collect TypeEnv bindings for cross-file propagation
+ */
+export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, totalFiles, repoPath, pipelineStart, onProgress, options) {
+    const ctx = createResolutionContext();
+    const symbolTable = ctx.model.symbols;
+    const parseableScanned = scannedFiles.filter((f) => {
+        const lang = getLanguageFromFilename(f.path);
+        return lang && isLanguageAvailable(lang);
+    });
+    // Warn about files skipped due to unavailable parsers
+    const skippedByLang = new Map();
+    for (const f of scannedFiles) {
+        const lang = getLanguageFromFilename(f.path);
+        if (lang && !isLanguageAvailable(lang)) {
+            skippedByLang.set(lang, (skippedByLang.get(lang) || 0) + 1);
+        }
+    }
+    for (const [lang, count] of skippedByLang) {
+        logger.warn(`Skipping ${count} ${lang} file(s) — ${lang} parser not available (native binding may not have built). Try: npm rebuild tree-sitter-${lang}`);
+    }
+    // Sort parseableScanned alphabetically for stable chunk membership
+    // across runs (Finding 4). Without this, filesystem-scan order can
+    // shift between runs (notably on macOS APFS where directory entry
+    // order can change after modifications) — different files in the
+    // same chunk → different chunk hash → cache miss even when no file
+    // content changed. The cache also becomes platform-specific: a
+    // Linux-built cache misses on macOS for the same repo.
+    parseableScanned.sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0));
+    const totalParseable = parseableScanned.length;
+    if (totalParseable === 0) {
+        onProgress({
+            phase: 'parsing',
+            percent: 82,
+            message: 'No parseable files found — skipping parsing phase',
+            stats: { filesProcessed: 0, totalFiles: 0, nodesCreated: graph.nodeCount },
+        });
+    }
+    // Build byte-budget chunks
+    const chunks = [];
+    let currentChunk = [];
+    let currentBytes = 0;
+    for (const file of parseableScanned) {
+        if (currentChunk.length > 0 && currentBytes + file.size > CHUNK_BYTE_BUDGET) {
+            chunks.push(currentChunk);
+            currentChunk = [];
+            currentBytes = 0;
+        }
+        currentChunk.push(file.path);
+        currentBytes += file.size;
+    }
+    if (currentChunk.length > 0)
+        chunks.push(currentChunk);
+    const numChunks = chunks.length;
+    if (isDev) {
+        const totalMB = parseableScanned.reduce((s, f) => s + f.size, 0) / (1024 * 1024);
+        logger.info(`📂 Scan: ${totalFiles} paths, ${totalParseable} parseable (${totalMB.toFixed(0)}MB), ${numChunks} chunks @ ${CHUNK_BYTE_BUDGET / (1024 * 1024)}MB budget`);
+    }
+    onProgress({
+        phase: 'parsing',
+        percent: 20,
+        message: `Parsing ${totalParseable} files in ${numChunks} chunk${numChunks !== 1 ? 's' : ''}...`,
+        stats: { filesProcessed: 0, totalFiles: totalParseable, nodesCreated: graph.nodeCount },
+    });
+    // Don't spawn workers for tiny repos — overhead exceeds benefit.
+    // Test suites may lower the thresholds via `options.workerThresholdsForTest`
+    // to exercise the worker-pool path with small fixtures; see PipelineOptions.
+    const MIN_FILES_FOR_WORKERS = options?.workerThresholdsForTest?.minFiles ?? 15;
+    const MIN_BYTES_FOR_WORKERS = options?.workerThresholdsForTest?.minBytes ?? 512 * 1024;
+    const totalBytes = parseableScanned.reduce((s, f) => s + f.size, 0);
+    // Create worker pool once, reuse across chunks
+    let workerPool;
+    if (!options?.skipWorkers &&
+        (totalParseable >= MIN_FILES_FOR_WORKERS || totalBytes >= MIN_BYTES_FOR_WORKERS)) {
+        try {
+            let workerUrl = new URL('../workers/parse-worker.js', import.meta.url);
+            // When running under vitest, import.meta.url points to src/ where no .js exists.
+            // Fall back to the compiled dist/ worker so the pool can spawn real worker threads.
+            const thisDir = fileURLToPath(new URL('.', import.meta.url));
+            if (!fs.existsSync(fileURLToPath(workerUrl))) {
+                const distWorker = path.resolve(thisDir, '..', '..', '..', '..', 'dist', 'core', 'ingestion', 'workers', 'parse-worker.js');
+                if (fs.existsSync(distWorker)) {
+                    workerUrl = pathToFileURL(distWorker);
+                }
+            }
+            workerPool = createWorkerPool(workerUrl);
+        }
+        catch (err) {
+            logger.warn({ err: err.message }, 'Worker pool creation failed, using sequential fallback:');
+        }
+    }
+    let filesParsedSoFar = 0;
+    // Two caches with different lifetimes:
+    //   - `astCache` (chunk-local, cleared between chunks) — call /
+    //     heritage / import processors read it during parse to avoid
+    //     re-parsing within the same chunk.
+    //   - `scopeTreeCache` (total-parseable-sized, never cleared by
+    //     parse-impl) — exposed via ParseOutput so scope-resolution can
+    //     skip a second tree-sitter parse. Worker-mode parses don't
+    //     populate either; consumers fall back to a fresh parse.
+    // See plan docs/plans/2026-04-20-002-perf-parse-heritage-mro-plan.md (Unit 4).
+    const maxChunkFiles = chunks.reduce((max, c) => Math.max(max, c.length), 0);
+    let astCache = createASTCache(maxChunkFiles);
+    const scopeTreeCache = createASTCache(Math.max(parseableScanned.length, 1));
+    // Build import resolution context once — suffix index, file lists, resolve cache.
+    const importCtx = buildImportResolutionContext(allPaths);
+    const allPathObjects = allPaths.map((p) => ({ path: p }));
+    const sequentialChunkPaths = [];
+    const chunkNeedsSynthesis = chunks.map((paths) => paths.some((p) => {
+        const lang = getLanguageFromFilename(p);
+        return lang != null && needsSynthesis(lang);
+    }));
+    const exportedTypeMap = new Map();
+    const bindingAccumulator = new BindingAccumulator();
+    // Tracks whether per-chunk or fallback wildcard-binding synthesis already
+    // ran, so the unconditional final call below can be skipped when redundant.
+    // synthesizeWildcardImportBindings is graph-global; once any chunk runs it
+    // after parsing wildcard files, later non-wildcard chunks add no work for
+    // it, and later wildcard chunks re-run it themselves.
+    let hasSynthesized = false;
+    const allFetchCalls = [];
+    const allExtractedRoutes = [];
+    const allDecoratorRoutes = [];
+    const allToolDefs = [];
+    const allORMQueries = [];
+    const deferredWorkerCalls = [];
+    const deferredWorkerHeritage = [];
+    const deferredConstructorBindings = [];
+    const deferredAssignments = [];
+    // Aggregated per-file ParsedFile artifacts produced by workers' calls
+    // to `extractParsedFile`. Threaded through to the scope-resolution
+    // phase so it can SKIP its own re-extraction on cache hits — this is
+    // the second-half of the parse-cache speedup since scope-resolution's
+    // re-parse otherwise dominates the warm-cache wall-clock time.
+    const allParsedFiles = [];
+    // Incremental parse cache (Option B): chunk-level content-addressed.
+    // When the chunk's (filePath, content-hash) signature matches a prior
+    // run's, replay the cached ParseWorkerResult[] instead of dispatching
+    // to workers. See arc/src/storage/parse-cache.ts.
+    const parseCache = options?.parseCache;
+    let chunkCacheHits = 0;
+    let chunkCacheMisses = 0;
+    try {
+        for (let chunkIdx = 0; chunkIdx < numChunks; chunkIdx++) {
+            const chunkPaths = chunks[chunkIdx];
+            const chunkContents = await readFileContents(repoPath, chunkPaths);
+            const chunkFiles = chunkPaths
+                .filter((p) => chunkContents.has(p))
+                .map((p) => ({ path: p, content: chunkContents.get(p) }));
+            // Compute the chunk's content-hash signature (if cache available).
+            let chunkHash = null;
+            if (parseCache) {
+                const entries = chunkFiles.map((f) => ({
+                    filePath: f.path,
+                    contentHash: fileContentHash(f.content),
+                }));
+                chunkHash = computeChunkHash(entries);
+            }
+            let chunkWorkerData;
+            const cachedRaw = chunkHash ? parseCache.entries.get(chunkHash) : undefined;
+            // Track every chunk hash we touched so the orchestrator can
+            // prune stale entries (chunks whose composition no longer
+            // corresponds to a live chunk in the current scan) before saving.
+            if (parseCache && chunkHash)
+                parseCache.usedKeys.add(chunkHash);
+            if (cachedRaw && cachedRaw.length > 0) {
+                // Cache hit: replay the cached worker output through the same
+                // merge logic the live worker path uses.
+                chunkCacheHits++;
+                chunkWorkerData = mergeChunkResults(graph, symbolTable, cachedRaw);
+                if (isDev) {
+                    logger.info(`📦 parse-cache HIT: chunk ${chunkIdx + 1}/${numChunks} (${chunkFiles.length} files, ${chunkHash.slice(0, 8)})`);
+                }
+                // Progress update so UI advances even on a cache hit.
+                const cachedFiles = chunkFiles.length;
+                onProgress({
+                    phase: 'parsing',
+                    percent: Math.round(20 + ((filesParsedSoFar + cachedFiles) / totalParseable) * 62),
+                    message: `Parsing chunk ${chunkIdx + 1}/${numChunks} (cache)...`,
+                    stats: {
+                        filesProcessed: filesParsedSoFar + cachedFiles,
+                        totalFiles: totalParseable,
+                        nodesCreated: graph.nodeCount,
+                    },
+                });
+            }
+            else {
+                // Cache miss: dispatch to workers, capture the raw results, store
+                // them under the chunk hash for the next run.
+                chunkCacheMisses++;
+                const rawResults = [];
+                chunkWorkerData = await processParsing(graph, chunkFiles, symbolTable, astCache, scopeTreeCache, (current, _total, filePath) => {
+                    const globalCurrent = filesParsedSoFar + current;
+                    const parsingProgress = 20 + (globalCurrent / totalParseable) * 62;
+                    onProgress({
+                        phase: 'parsing',
+                        percent: Math.round(parsingProgress),
+                        message: `Parsing chunk ${chunkIdx + 1}/${numChunks}...`,
+                        detail: filePath,
+                        stats: {
+                            filesProcessed: globalCurrent,
+                            totalFiles: totalParseable,
+                            nodesCreated: graph.nodeCount,
+                        },
+                    });
+                }, workerPool, 
+                // Capture raw results only when we have a cache to write to —
+                // otherwise we'd retain extra arrays for nothing.
+                parseCache && chunkHash ? rawResults : undefined);
+                // Persist the raw results for this chunk hash. Sequential path
+                // doesn't populate rawResults (it writes directly to graph), so
+                // small repos without worker pool simply don't cache. That's fine.
+                if (parseCache && chunkHash && rawResults.length > 0) {
+                    parseCache.entries.set(chunkHash, rawResults);
+                    if (isDev) {
+                        logger.info(`📦 parse-cache MISS+store: chunk ${chunkIdx + 1}/${numChunks} (${chunkFiles.length} files, ${chunkHash.slice(0, 8)})`);
+                    }
+                }
+            }
+            const chunkBasePercent = 20 + (filesParsedSoFar / totalParseable) * 62;
+            if (chunkWorkerData) {
+                await processImportsFromExtracted(graph, allPathObjects, chunkWorkerData.imports, ctx, (current, total) => {
+                    onProgress({
+                        phase: 'parsing',
+                        percent: Math.round(chunkBasePercent),
+                        message: `Resolving imports (chunk ${chunkIdx + 1}/${numChunks})...`,
+                        detail: `${current}/${total} files`,
+                        stats: {
+                            filesProcessed: filesParsedSoFar,
+                            totalFiles: totalParseable,
+                            nodesCreated: graph.nodeCount,
+                        },
+                    });
+                }, repoPath, importCtx);
+                if (chunkNeedsSynthesis[chunkIdx]) {
+                    synthesizeWildcardImportBindings(graph, ctx);
+                    hasSynthesized = true;
+                }
+                if (exportedTypeMap.size > 0 && ctx.namedImportMap.size > 0) {
+                    const { enrichedCount } = seedCrossFileReceiverTypes(chunkWorkerData.calls, ctx.namedImportMap, exportedTypeMap);
+                    if (isDev && enrichedCount > 0) {
+                        logger.info(`🔗 E1: Seeded ${enrichedCount} cross-file receiver types (chunk ${chunkIdx + 1})`);
+                    }
+                }
+                for (const item of chunkWorkerData.calls)
+                    deferredWorkerCalls.push(item);
+                for (const item of chunkWorkerData.heritage)
+                    deferredWorkerHeritage.push(item);
+                for (const item of chunkWorkerData.constructorBindings)
+                    deferredConstructorBindings.push(item);
+                // Aggregate worker-produced ParsedFile artifacts so scope-
+                // resolution can use them as a re-extraction cache (skips its
+                // own tree-sitter re-parse on warm runs).
+                if (chunkWorkerData.parsedFiles?.length) {
+                    for (const item of chunkWorkerData.parsedFiles)
+                        allParsedFiles.push(item);
+                }
+                if (chunkWorkerData.assignments?.length) {
+                    for (const item of chunkWorkerData.assignments)
+                        deferredAssignments.push(item);
+                }
+                await Promise.all([
+                    processHeritageFromExtracted(graph, chunkWorkerData.heritage, ctx, (current, total) => {
+                        onProgress({
+                            phase: 'parsing',
+                            percent: Math.round(chunkBasePercent),
+                            message: `Resolving heritage (chunk ${chunkIdx + 1}/${numChunks})...`,
+                            detail: `${current}/${total} records`,
+                            stats: {
+                                filesProcessed: filesParsedSoFar,
+                                totalFiles: totalParseable,
+                                nodesCreated: graph.nodeCount,
+                            },
+                        });
+                    }),
+                    processRoutesFromExtracted(graph, chunkWorkerData.routes ?? [], ctx, (current, total) => {
+                        onProgress({
+                            phase: 'parsing',
+                            percent: Math.round(chunkBasePercent),
+                            message: `Resolving routes (chunk ${chunkIdx + 1}/${numChunks})...`,
+                            detail: `${current}/${total} routes`,
+                            stats: {
+                                filesProcessed: filesParsedSoFar,
+                                totalFiles: totalParseable,
+                                nodesCreated: graph.nodeCount,
+                            },
+                        });
+                    }),
+                ]);
+                if (chunkWorkerData.fileScopeBindings?.length) {
+                    for (const { filePath, bindings } of chunkWorkerData.fileScopeBindings) {
+                        if (typeof filePath !== 'string' || filePath.length === 0)
+                            continue;
+                        if (!Array.isArray(bindings))
+                            continue;
+                        const entries = [];
+                        for (const tuple of bindings) {
+                            if (!Array.isArray(tuple) || tuple.length !== 2)
+                                continue;
+                            const [varName, typeName] = tuple;
+                            if (typeof varName !== 'string' || typeof typeName !== 'string')
+                                continue;
+                            entries.push({ scope: '', varName, typeName });
+                        }
+                        if (entries.length > 0) {
+                            bindingAccumulator.appendFile(filePath, entries);
+                        }
+                    }
+                }
+                if (chunkWorkerData.fetchCalls?.length) {
+                    for (const item of chunkWorkerData.fetchCalls)
+                        allFetchCalls.push(item);
+                }
+                if (chunkWorkerData.routes?.length) {
+                    for (const item of chunkWorkerData.routes)
+                        allExtractedRoutes.push(item);
+                }
+                if (chunkWorkerData.decoratorRoutes?.length) {
+                    for (const item of chunkWorkerData.decoratorRoutes)
+                        allDecoratorRoutes.push(item);
+                }
+                if (chunkWorkerData.toolDefs?.length) {
+                    for (const item of chunkWorkerData.toolDefs)
+                        allToolDefs.push(item);
+                }
+                if (chunkWorkerData.ormQueries?.length) {
+                    for (const item of chunkWorkerData.ormQueries)
+                        allORMQueries.push(item);
+                }
+            }
+            else {
+                await processImports(graph, chunkFiles, astCache, ctx, undefined, repoPath, allPaths);
+                sequentialChunkPaths.push(chunkPaths);
+            }
+            filesParsedSoFar += chunkFiles.length;
+            astCache.clear();
+        }
+        if (isDev && parseCache && (chunkCacheHits > 0 || chunkCacheMisses > 0)) {
+            logger.info(`📦 parse-cache summary: ${chunkCacheHits} chunk hit(s), ${chunkCacheMisses} miss(es) across ${numChunks} chunk(s)`);
+        }
+        const fullWorkerHeritageMap = deferredWorkerHeritage.length > 0
+            ? buildHeritageMap(deferredWorkerHeritage, ctx, getHeritageStrategyForLanguage)
+            : undefined;
+        if (deferredWorkerCalls.length > 0) {
+            await processCallsFromExtracted(graph, deferredWorkerCalls, ctx, (current, total) => {
+                onProgress({
+                    phase: 'parsing',
+                    percent: 82,
+                    message: 'Resolving calls (all chunks)...',
+                    detail: `${current}/${total} files`,
+                    stats: {
+                        filesProcessed: filesParsedSoFar,
+                        totalFiles: totalParseable,
+                        nodesCreated: graph.nodeCount,
+                    },
+                });
+            }, deferredConstructorBindings.length > 0 ? deferredConstructorBindings : undefined, fullWorkerHeritageMap, bindingAccumulator);
+        }
+        if (deferredAssignments.length > 0) {
+            processAssignmentsFromExtracted(graph, deferredAssignments, ctx, deferredConstructorBindings.length > 0 ? deferredConstructorBindings : undefined, bindingAccumulator);
+        }
+    }
+    finally {
+        await workerPool?.terminate();
+    }
+    // Sequential fallback chunks.
+    //
+    // U6: wrap the fallback loop and the finalize/enrich steps in a try/finally
+    // so cleanup still runs on a mid-fallback throw. The `finally` guarantees:
+    //   1. `astCache.clear()` releases any tree-sitter trees held by the most
+    //      recently allocated per-chunk cache, mirroring the per-chunk
+    //      `astCache.clear()` calls on the happy path.
+    //   2. `bindingAccumulator.finalize()` runs before `crossFile` disposes the
+    //      accumulator downstream — callers that inspect partial TypeEnv state
+    //      (or consume it via `enrichExportedTypeMap` on a partial recovery)
+    //      still see a finalized accumulator.
+    //   3. `enrichExportedTypeMap` runs so any bindings already accumulated
+    //      are propagated into `exportedTypeMap` even if the fallback aborted.
+    //
+    // Disposal of the accumulator remains with `crossFile` (owned by U2). We do
+    // NOT call `bindingAccumulator.dispose()` here.
+    try {
+        if (sequentialChunkPaths.length > 0) {
+            synthesizeWildcardImportBindings(graph, ctx);
+            hasSynthesized = true;
+        }
+        const allSequentialHeritage = [];
+        const cachedSequentialChunkFiles = [];
+        for (const chunkPaths of sequentialChunkPaths) {
+            const chunkContents = await readFileContents(repoPath, chunkPaths);
+            const chunkFiles = chunkPaths
+                .filter((p) => chunkContents.has(p))
+                .map((p) => ({ path: p, content: chunkContents.get(p) }));
+            cachedSequentialChunkFiles.push(chunkFiles);
+            astCache = createASTCache(chunkFiles.length);
+            const sequentialHeritage = await extractExtractedHeritageFromFiles(chunkFiles, astCache);
+            for (const h of sequentialHeritage)
+                allSequentialHeritage.push(h);
+            astCache.clear();
+        }
+        const sequentialHeritageMap = allSequentialHeritage.length > 0
+            ? buildHeritageMap(allSequentialHeritage, ctx, getHeritageStrategyForLanguage)
+            : undefined;
+        for (let chunkIdx = 0; chunkIdx < sequentialChunkPaths.length; chunkIdx++) {
+            const chunkFiles = cachedSequentialChunkFiles[chunkIdx];
+            astCache = createASTCache(chunkFiles.length);
+            const rubyHeritage = await processCalls(graph, chunkFiles, astCache, ctx, undefined, exportedTypeMap, undefined, undefined, undefined, sequentialHeritageMap, bindingAccumulator);
+            await processHeritage(graph, chunkFiles, astCache, ctx);
+            if (rubyHeritage.length > 0) {
+                await processHeritageFromExtracted(graph, rubyHeritage, ctx);
+            }
+            const chunkFetchCalls = await extractFetchCallsFromFiles(chunkFiles, astCache);
+            if (chunkFetchCalls.length > 0) {
+                for (const item of chunkFetchCalls)
+                    allFetchCalls.push(item);
+            }
+            for (const f of chunkFiles) {
+                extractORMQueriesInline(f.path, f.content, allORMQueries);
+            }
+            astCache.clear();
+            cachedSequentialChunkFiles[chunkIdx] = [];
+        }
+        // Log resolution cache stats
+        if (isDev) {
+            const rcStats = ctx.getStats();
+            const total = rcStats.cacheHits + rcStats.cacheMisses;
+            const hitRate = total > 0 ? ((rcStats.cacheHits / total) * 100).toFixed(1) : '0';
+            logger.info(`🔍 Resolution cache: ${rcStats.cacheHits} hits, ${rcStats.cacheMisses} misses (${hitRate}% hit rate)`);
+        }
+    }
+    finally {
+        // Clearing an already-empty cache is a no-op, so this is idempotent-safe
+        // on the happy path where every per-chunk block already cleared astCache.
+        astCache.clear();
+        // Run finalize + enrichment inside try/catch so a cleanup failure never
+        // masks the original fallback error. finalize must precede crossFile's
+        // dispose (U2) and enrichExportedTypeMap depends on finalized bindings.
+        try {
+            bindingAccumulator.finalize();
+            const enriched = enrichExportedTypeMap(bindingAccumulator, graph, exportedTypeMap);
+            if (isDev && enriched > 0) {
+                logger.info(`🔗 Worker TypeEnv enrichment: ${enriched} fixpoint-inferred exports added to ExportedTypeMap`);
+            }
+        }
+        catch (enrichErr) {
+            if (isDev) {
+                logger.warn({ err: enrichErr.message }, 'Post-fallback finalize/enrich failed during cleanup:');
+            }
+        }
+    }
+    if (!hasSynthesized) {
+        const synthesized = synthesizeWildcardImportBindings(graph, ctx);
+        if (isDev && synthesized > 0) {
+            logger.info(`🔗 Synthesized ${synthesized} additional wildcard import bindings (Go/Ruby/C++/Swift/Python)`);
+        }
+    }
+    // Worker-path enrichment: if exportedTypeMap is empty (e.g. the worker pool
+    // built TypeEnv inside workers without access to SymbolTable), reconstruct
+    // the map from graph nodes + SymbolTable here in the main thread before
+    // handing the (now read-only) map to downstream phases. Doing it here means
+    // crossFile receives a fully-populated map and never needs to mutate it for
+    // initial-graph enrichment.
+    if (exportedTypeMap.size === 0 && graph.nodeCount > 0) {
+        const graphExports = buildExportedTypeMapFromGraph(graph, ctx.model.symbols);
+        for (const [fp, exports] of graphExports)
+            exportedTypeMap.set(fp, exports);
+    }
+    allPathObjects.length = 0;
+    // Safe to reset importCtx caches here: `importCtx` (ImportResolutionContext)
+    // is a scratch workspace used only during import path resolution. The
+    // `resolutionContext` (`ctx`) returned below is a distinct object — it owns
+    // the fully-populated, post-parse `importMap` / `namedImportMap` /
+    // `packageMap` / `moduleAliasMap` / `model`, and never references
+    // `importCtx`. Cross-file re-resolution in cross-file-impl.ts consumes only
+    // `ctx` (via `processCalls`), so clearing the suffix index / resolveCache /
+    // normalizedFileList here cannot lose import matches downstream.
+    importCtx.resolveCache.clear();
+    importCtx.index = EMPTY_INDEX;
+    importCtx.normalizedFileList = [];
+    return {
+        exportedTypeMap,
+        allFetchCalls,
+        allExtractedRoutes,
+        allDecoratorRoutes,
+        allToolDefs,
+        allORMQueries,
+        bindingAccumulator,
+        resolutionContext: ctx,
+        // Whether a worker pool was actually live for this run. False means the
+        // sequential fallback handled every chunk (either due to `skipWorkers`,
+        // the file-count/byte thresholds, or a pool-creation failure).
+        usedWorkerPool: workerPool !== undefined,
+        // Surface the persistent scope cache so downstream phases
+        // (scope-resolution) can skip re-parsing files that the
+        // sequential path already parsed. Survives chunk boundaries; the
+        // chunk-local `astCache` above is intentionally NOT exposed
+        // because parse-impl clears it between chunks.
+        scopeTreeCache,
+        // Per-file ParsedFile artifacts produced by workers' calls to
+        // `extractParsedFile`. Empty when only the sequential path ran
+        // (sequential doesn't go through the worker, and extracts ParsedFile
+        // inline rather than emitting it). Consumed by scope-resolution as
+        // a re-extraction cache: when the file's ParsedFile is here,
+        // scope-resolution skips its own `extractParsedFile` call.
+        parsedFiles: allParsedFiles,
+    };
+}

package/dist/core/ingestion/pipeline-phases/parse.d.ts

@@ -0,0 +1,88 @@
+/**
+ * Phase: parse
+ *
+ * Chunked parse + resolve loop: reads source in byte-budget chunks,
+ * parses via worker pool (or sequential fallback), resolves imports,
+ * heritage, and calls, synthesizes wildcard bindings.
+ *
+ * This phase encapsulates the entire `runChunkedParseAndResolve` function
+ * from the original pipeline. The chunk loop is a memory optimization
+ * internal to this phase, not a phase boundary.
+ *
+ * @deps    structure, markdown, cobol
+ * @reads   scannedFiles, allPaths, totalFiles (from structure)
+ * @writes  graph (Symbol nodes, IMPORTS/CALLS/EXTENDS/IMPLEMENTS/ACCESSES edges)
+ * @output  exportedTypeMap, allFetchCalls, allExtractedRoutes, allDecoratorRoutes,
+ *          allToolDefs, allORMQueries, bindingAccumulator
+ */
+import type { PipelinePhase } from './types.js';
+import type { BindingAccumulator } from '../binding-accumulator.js';
+import type { ParsedFile } from '../../../_shared/index.js';
+import type { ExtractedFetchCall, ExtractedRoute, ExtractedDecoratorRoute, ExtractedToolDef, ExtractedORMQuery } from '../workers/parse-worker.js';
+import type { createResolutionContext } from '../model/resolution-context.js';
+import type { ASTCache } from '../ast-cache.js';
+export interface ParseOutput {
+    /**
+     * Read-only snapshot of exported type bindings keyed by file path.
+     *
+     * Fully populated by `parse` (sequential path via `enrichExportedTypeMap`
+     * and worker path via `buildExportedTypeMapFromGraph` in the main thread).
+     * Downstream phases — including `crossFile` — receive it as a true
+     * `ReadonlyMap`; `crossFile` builds its own mutable working copy locally
+     * for per-file re-resolution writes, so this snapshot is never mutated
+     * after parse returns.
+     */
+    readonly exportedTypeMap: ReadonlyMap<string, ReadonlyMap<string, string>>;
+    readonly allFetchCalls: readonly ExtractedFetchCall[];
+    readonly allExtractedRoutes: readonly ExtractedRoute[];
+    readonly allDecoratorRoutes: readonly ExtractedDecoratorRoute[];
+    readonly allToolDefs: readonly ExtractedToolDef[];
+    readonly allORMQueries: readonly ExtractedORMQuery[];
+    bindingAccumulator: BindingAccumulator;
+    /** Resolution context from the parse phase — carries importMap, namedImportMap, etc. */
+    resolutionContext: ReturnType<typeof createResolutionContext>;
+    /** Pass-through: all file paths for downstream phases. */
+    readonly allPaths: readonly string[];
+    /** Pass-through: shared `allPathSet` from structure (built once, not per-phase). */
+    readonly allPathSet: ReadonlySet<string>;
+    /** Pass-through: total file count for progress reporting. */
+    totalFiles: number;
+    /**
+     * True if the parse phase spawned a live worker pool for this run.
+     * False means every chunk ran through the sequential fallback (skipWorkers,
+     * thresholds not met, or pool-creation failure). Primarily a test affordance:
+     * see `PipelineOptions.workerThresholdsForTest`.
+     */
+    readonly usedWorkerPool: boolean;
+    /**
+     * Cross-phase tree-sitter Tree cache populated by the sequential
+     * parse path. Separate from the chunk-local `astCache` used *inside*
+     * the parse phase (which is cleared between chunks) — this one
+     * survives the whole phase and hands Trees to scope-resolution so
+     * it can skip a second parse.
+     *
+     * Empty entries for files that ran through the worker pool
+     * (workers can't return native tree-sitter Trees across the
+     * MessageChannel). Cache miss is safe — consumers fall back to a
+     * fresh parse. See plan
+     * docs/plans/2026-04-20-002-perf-parse-heritage-mro-plan.md (Unit 4).
+     *
+     * Disposed by `scopeResolutionPhase` (the sole consumer) via
+     * `scopeTreeCache.clear()` after its extract loop finishes.
+     */
+    readonly scopeTreeCache: ASTCache;
+    /**
+     * Per-file `ParsedFile` artifacts produced by workers' calls to
+     * `extractParsedFile`. Threaded through to `scopeResolutionPhase`
+     * as a re-extraction cache: when a file's ParsedFile is present here,
+     * scope-resolution can skip its own `extractParsedFile` (which would
+     * otherwise re-parse the file with tree-sitter on the main thread,
+     * costing ~58s on a 1000-file repo).
+     *
+     * Empty for files that went through the sequential parse fallback —
+     * sequential doesn't emit ParsedFile artifacts; scope-resolution
+     * falls back to a fresh extract for those.
+     */
+    readonly parsedFiles: readonly ParsedFile[];
+}
+export declare const parsePhase: PipelinePhase<ParseOutput>;