gitnexus 1.6.6-rc.98 → 1.6.6

package/dist/core/ingestion/scope-resolution/pipeline/phase.js

@@ -3,23 +3,20 @@
  *
  * Generic registry-primary resolution phase (RFC #909 Ring 3).
  *
- * For every language in `MIGRATED_LANGUAGES` (per-language flag set)
- * whose provider is registered in `SCOPE_RESOLVERS`:
+ * For every language whose provider is registered in `SCOPE_RESOLVERS`:
  *   1. Filter scanned files by language extension.
  *   2. Read file contents.
  *   3. Drive the scope-based pipeline end-to-end via the generic
  *      `runScopeResolution(input, provider)` orchestrator.
  *   4. Emit IMPORTS / CALLS / ACCESSES / INHERITS / USES edges.
  *
- * Pairs with the per-language gates in `import-processor.ts` and
- * `call-processor.ts` that skip files when their language is registry-
- * primary, so we don't double-emit edges from both code paths.
+ * This is the sole resolution path — RING4-1 (#942) deleted the legacy
+ * call-resolution DAG, so there is no longer a per-language flag gating
+ * registry-vs-legacy.
  *
- * Adding a language is two changes:
- *   - Implement `ScopeResolver` in `languages/<lang>/scope-resolver.ts`
- *     and register it in `scope-resolution/pipeline/registry.ts`.
- *   - Add the language to `MIGRATED_LANGUAGES` in
- *     `registry-primary-flag.ts`.
+ * Adding a language is one change: implement `ScopeResolver` in
+ * `languages/<lang>/scope-resolver.ts` and register it in
+ * `scope-resolution/pipeline/registry.ts`.
  *
  * @deps    parse  (needs Symbol nodes already in the graph so emit-references
  *                  can attach edges to existing Function/Method/Class nodes)
@@ -27,12 +24,14 @@
  * @writes  graph (IMPORTS, CALLS, ACCESSES, INHERITS, USES)
  */
 import { getPhaseOutput } from '../../pipeline-phases/types.js';
-import { isRegistryPrimary } from '../../registry-primary-flag.js';
 import { getLanguageFromFilename } from '../../../../_shared/index.js';
 import { readFileContents } from '../../filesystem-walker.js';
 import { runScopeResolution } from './run.js';
+import { buildGraphNodeLookup } from '../graph-bridge/node-lookup.js';
 import { SCOPE_RESOLVERS } from './registry.js';
 import { isDev, isSemanticModelValidatorEnabled } from '../../utils/env.js';
+import { logHeapProbe } from '../../utils/heap-probe.js';
+import { clearParsedFileStore, loadParsedFilesForPaths, forceGc, } from '../../../../storage/parsedfile-store.js';
 import { logger } from '../../../logger.js';
 const NOOP_OUTPUT = Object.freeze({
     ran: false,
@@ -45,31 +44,25 @@ const NOOP_OUTPUT = Object.freeze({
 export const scopeResolutionPhase = {
     name: 'scopeResolution',
     // Depends on `parse` because emit-references attaches edges to
-    // already-existing Symbol nodes (Function/Method/Class). The legacy
-    // `parse` phase still creates those nodes; we only replace the
-    // import + call resolution layer.
+    // already-existing Symbol nodes (Function/Method/Class) that the `parse`
+    // phase creates.
     //
-    // Also depends on `crossFile` — we don't read crossFile's output
-    // directly (we have our own cross-file resolution), but crossFile
-    // writes EXTENDS edges that `buildMro` consumes via
-    // `iterRelationshipsByType('EXTENDS')`. Declaring the dep pins the
-    // ordering explicitly: without it, Kahn's runner could schedule
-    // scopeResolution before crossFile (both unblock after parse), and
-    // the MRO walk would miss heritage edges crossFile later adds.
+    // The `crossFile` dep is retained for stable ordering but is no longer
+    // load-bearing: inheritance (EXTENDS/IMPLEMENTS) edges are now emitted by
+    // this phase's own `preEmitInheritanceEdges` before `buildMro` runs, and
+    // since RING4-1 (#942) `crossFile` only disposes the BindingAccumulator
+    // (the legacy cross-file re-resolution it used to run was deleted with the
+    // call-resolution DAG).
     deps: ['parse', 'crossFile', 'structure'],
     async execute(ctx, deps) {
+        logHeapProbe('scopeResolution-enter');
         const { scannedFiles } = getPhaseOutput(deps, 'structure');
-        // Reach into the parse phase's AST cache so per-file extract can
-        // skip a second tree-sitter parse. Cache miss is safe (re-parses).
-        // Worker-mode parses leave the cache empty for those files; they
-        // also fall back to a fresh parse — no correctness impact.
         const parseOutput = getPhaseOutput(deps, 'parse');
-        const { scopeTreeCache, resolutionContext, parsedFiles: workerParsedFiles } = parseOutput;
+        const { model, parsedFiles: workerParsedFiles } = parseOutput;
         // SemanticModel populated during `parse`: scope-resolution consumes
         // TypeRegistry / MethodRegistry / SymbolTable lookups instead of
         // rebuilding parallel indexes. See ARCHITECTURE.md § "Semantic-model
         // source of truth".
-        const model = resolutionContext.model;
         // Build a per-file lookup of ParsedFile artifacts the workers (or
         // sequential extracts) already produced. Threading this into
         // `runScopeResolution` lets the per-language extract loop short-
@@ -81,6 +74,15 @@ export const scopeResolutionPhase = {
         for (const pf of workerParsedFiles) {
             preExtractedByPath.set(pf.filePath, pf);
         }
+        // Disk-backed ParsedFile store (#1983): on huge repos the parse phase
+        // flushes worker-produced ParsedFiles to disk (instead of `workerParsedFiles`
+        // heap) and we stream them back here per language. Reusing them means
+        // scope-resolution does ZERO tree-sitter parsing on the main thread, which
+        // is what eliminates the unbounded native re-parse leak that OOM'd large
+        // analyses. `undefined` when no storage path (tests / direct calls) — those
+        // runs use the in-heap `workerParsedFiles` above or fall back to a fresh
+        // extract per file inside runScopeResolution.
+        const parsedFileStorePath = ctx.options?.parseCache?.storagePath;
         // Drop pre-extracted entries for standalone providers — these
         // languages are skipped by the canonical guard below (line 164)
         // and never consume preExtractedByPath, so holding onto their
@@ -103,12 +105,30 @@ export const scopeResolutionPhase = {
         const perLanguage = new Map();
         // Pre-count files and languages for progress reporting. This avoids
         // a frozen progress bar during long scope-resolution runs (#1741).
+        // Uses primary-language file counts only; languages that expand their
+        // context via collectScopeContextPaths may process more files than shown.
         let totalScopeFiles = 0;
         let totalScopeLangs = 0;
-        for (const [lang] of SCOPE_RESOLVERS) {
-            if (!isRegistryPrimary(lang))
+        const allScannedPaths = new Set(scannedFiles.map((f) => f.path));
+        // Partition scanned files by language ONCE (O(F)). The previous code
+        // re-filtered all scannedFiles per language for the precount AND again in the
+        // per-language loop below — O(languages × files), ~2.3M getLanguageFromFilename
+        // calls on the Linux kernel. Bucketing once collapses that to O(F).
+        // Language-agnostic: keyed by the provider-supplied SupportedLanguages value.
+        const filesByLang = new Map();
+        for (const f of scannedFiles) {
+            const fileLang = getLanguageFromFilename(f.path);
+            if (fileLang === null)
                 continue;
-            const count = scannedFiles.filter((f) => getLanguageFromFilename(f.path) === lang).length;
+            let bucket = filesByLang.get(fileLang);
+            if (bucket === undefined) {
+                bucket = [];
+                filesByLang.set(fileLang, bucket);
+            }
+            bucket.push(f);
+        }
+        for (const [lang] of SCOPE_RESOLVERS) {
+            const count = filesByLang.get(lang)?.length ?? 0;
             if (count > 0) {
                 totalScopeLangs++;
                 totalScopeFiles += count;
@@ -125,9 +145,18 @@ export const scopeResolutionPhase = {
                 message: 'Resolving types',
             });
         }
+        // Build the graph-node lookup ONCE and share it across every language pass.
+        // It scans the whole graph (~2 GB on the kernel) and is language-agnostic,
+        // so the previous per-language rebuild burned that CPU+heap N times and, on
+        // a huge repo, a tiny language's full-graph copy overlapped the next
+        // language's — a real contributor to the scope-resolution memory peak.
+        // Bracket the whole-graph node-lookup build with probes — scanning every
+        // graph node is the silent multi-minute step before the first per-language
+        // marker on huge repos, so make it observable.
+        logHeapProbe('scope-setup-nodeLookup-start', `langs=${totalScopeLangs} files=${totalScopeFiles}`);
+        const sharedNodeLookup = totalScopeFiles > 0 ? buildGraphNodeLookup(ctx.graph) : undefined;
+        logHeapProbe('scope-setup-nodeLookup-end', `langs=${totalScopeLangs}`);
         for (const [lang, provider] of SCOPE_RESOLVERS) {
-            if (!isRegistryPrimary(lang))
-                continue;
             // Standalone providers (COBOL, JCL) don't emit graph edges yet
             // through the scope-resolution path. This is the canonical guard:
             // runScopeResolution is never called for standalone providers, which
@@ -136,17 +165,10 @@ export const scopeResolutionPhase = {
             // SCOPE_RESOLVERS.
             if (provider.languageProvider.parseStrategy === 'standalone')
                 continue;
-            const langFiles = scannedFiles.filter((f) => getLanguageFromFilename(f.path) === lang);
-            if (langFiles.length === 0)
+            const primaryLangFiles = filesByLang.get(lang) ?? [];
+            if (primaryLangFiles.length === 0)
                 continue;
-            const filePaths = langFiles.map((f) => f.path);
-            const contents = await readFileContents(ctx.repoPath, filePaths);
-            const files = [];
-            for (const fp of filePaths) {
-                const content = contents.get(fp);
-                if (content !== undefined)
-                    files.push({ path: fp, content });
-            }
+            const primaryFilePaths = primaryLangFiles.map((f) => f.path);
             // Load per-language import-resolution config (tsconfig paths,
             // composer.json autoload, go.mod, ...). One I/O round trip per
             // workspace pass — cached implicitly by the result handed to
@@ -154,7 +176,84 @@ export const scopeResolutionPhase = {
             const resolutionConfig = provider.loadResolutionConfig !== undefined
                 ? await provider.loadResolutionConfig(ctx.repoPath)
                 : undefined;
+            // Some languages (e.g. Vue) expand their file universe beyond the
+            // primary-language files via the `collectScopeContextPaths` hook.
+            // The hook receives raw source contents of the primary files so it
+            // can trace import closures without a second tree-sitter parse.
+            //
+            // To avoid reading primary files twice (once for the hook, once for
+            // the resolution pass), we read them upfront and merge with the
+            // extra context paths the hook may add.
+            // Stream this language's pre-built ParsedFiles in from the disk store
+            // FIRST (huge-repo path). Doing it before reading source lets us skip
+            // loading content for files the store already covers — for a provider
+            // with no content-consuming hook that source is pure dead weight once
+            // extraction is served from the store (~1.5 GB on the kernel's C pass).
+            // Merged into `preExtractedByPath`; the per-language release block below
+            // evicts these again before the next language, so only one language's
+            // ParsedFiles are resident at a time.
+            const loadStoreFor = async (paths) => {
+                if (!parsedFileStorePath)
+                    return;
+                const fromDisk = await loadParsedFilesForPaths(parsedFileStorePath, paths);
+                for (const [fp, pf] of fromDisk)
+                    preExtractedByPath.set(fp, pf);
+            };
+            // A provider that feeds source text into a post-extract hook
+            // (populateWorkspaceOwners / populateNamespaceSiblings /
+            // populateRangeBindings / emitPostResolutionEdges) needs content for ALL
+            // its files; one without those hooks only needs content for files the
+            // store does NOT cover (fresh-extract fallback). Keep this in sync with
+            // the getFileContents() call-sites in run.ts.
+            const providerNeedsAllContent = provider.populateWorkspaceOwners !== undefined ||
+                provider.populateNamespaceSiblings !== undefined ||
+                provider.populateRangeBindings !== undefined ||
+                provider.emitPostResolutionEdges !== undefined;
+            let scopeFilePaths;
+            let contents;
+            if (provider.collectScopeContextPaths !== undefined) {
+                // Context-expanding providers (e.g. Vue) need every primary file's
+                // source up front for the closure hook, so load it all.
+                const entryFileContents = await readFileContents(ctx.repoPath, primaryFilePaths);
+                scopeFilePaths = provider.collectScopeContextPaths({
+                    primaryFilePaths,
+                    preExtractedByPath,
+                    entryFileContents,
+                    allScannedPaths,
+                    resolutionConfig,
+                });
+                // Read only the extra context files (TS/JS etc.) not already loaded.
+                const extraPaths = [...scopeFilePaths].filter((p) => !entryFileContents.has(p));
+                const extraContents = await readFileContents(ctx.repoPath, extraPaths);
+                contents = new Map([...entryFileContents, ...extraContents]);
+                await loadStoreFor(scopeFilePaths);
+            }
+            else {
+                scopeFilePaths = new Set(primaryFilePaths);
+                await loadStoreFor(scopeFilePaths);
+                const pathsToRead = providerNeedsAllContent
+                    ? primaryFilePaths
+                    : primaryFilePaths.filter((p) => !preExtractedByPath.has(p));
+                contents = await readFileContents(ctx.repoPath, pathsToRead);
+            }
+            const filePaths = [...scopeFilePaths];
+            const files = [];
+            for (const fp of filePaths) {
+                const content = contents.get(fp);
+                if (content !== undefined) {
+                    files.push({ path: fp, content });
+                }
+                else if (preExtractedByPath.has(fp)) {
+                    // Store covers extraction for this file and we deliberately skipped
+                    // reading its source; the empty string is never consumed (the
+                    // extract loop uses the pre-extracted ParsedFile and this provider
+                    // has no content hook).
+                    files.push({ path: fp, content: '' });
+                }
+                // else: uncovered AND unreadable → skip (unchanged from prior behavior).
+            }
             const langFileCount = files.length;
+            logHeapProbe('scope-lang-start', `lang=${lang} files=${langFileCount} contentsLoaded=${contents.size}`);
             const langLabel = lang.charAt(0).toUpperCase() + lang.slice(1);
             currentLangIdx++;
             const langTag = totalScopeLangs > 1 ? `${langLabel} [${currentLangIdx}/${totalScopeLangs}]` : langLabel;
@@ -171,9 +270,10 @@ export const scopeResolutionPhase = {
                 graph: ctx.graph,
                 model,
                 files,
-                treeCache: scopeTreeCache,
                 resolutionConfig,
+                prebuiltNodeLookup: sharedNodeLookup,
                 preExtractedParsedFiles: preExtractedByPath,
+                scopeIndexStorePath: parsedFileStorePath,
                 recordResolutionOutcome: (outcome) => {
                     resolutionOutcomes.push(outcome);
                 },
@@ -220,11 +320,27 @@ export const scopeResolutionPhase = {
             // to reduce memory pressure. For large codebases (16K+ PHP files),
             // holding all source code simultaneously with scope trees causes OOM.
             // See: https://github.com/abhigyanpatwari/GitNexus/issues/1741
+            //
+            // Use `filePaths` (not `primaryFilePaths`) so that any context files
+            // added by `collectScopeContextPaths` (e.g. TS/JS files pulled in for
+            // Vue cross-file resolution) are also evicted and not held until GC.
             files.length = 0;
             contents.clear();
             for (const fp of filePaths) {
                 preExtractedByPath.delete(fp);
             }
+            // This language's ParsedFiles are now unreachable (runScopeResolution has
+            // returned and the Map entries are deleted). Force a GC HERE so a heavy
+            // language's ~17-20GB set (e.g. C/C++ on the Linux kernel) is reclaimed
+            // BEFORE the next language's store-load — instead of leaving V8 to collect
+            // it lazily under the next pass's allocation pressure (which, at a cap >=
+            // RAM, degrades into swap-thrash). Collects only dead objects: the live
+            // cross-file index of the next pass is untouched. The pre/post probe
+            // confirms whether old-space fragmentation defeats the reclaim.
+            logHeapProbe('lang-release-pre-gc', `lang=${lang}`);
+            forceGc();
+            logHeapProbe('lang-release-post-gc', `lang=${lang}`);
+            logHeapProbe('scope-lang-end', `lang=${lang} filesProcessed=${stats.filesProcessed}`);
             processedScopeFiles += langFileCount;
             anyRan = true;
             totalFiles += stats.filesProcessed;
@@ -247,13 +363,18 @@ export const scopeResolutionPhase = {
                 detail: 'complete',
             });
         }
-        // Dispose the cross-phase Tree cache — scope-resolution is the
-        // only consumer. Holding Trees past this point is pure memory
-        // pressure: downstream phases (mro, community, csv-generator)
-        // never read them, and tree-sitter Trees hold native-heap memory
-        // under WASM runtimes. ASTCache.clear() fires the LRU dispose
-        // handler which calls tree.delete?.() on each retained Tree.
-        scopeTreeCache.clear();
+        // Scope-resolution is the sole consumer of the disk-backed ParsedFile
+        // store; remove its shards now (can be many GB on a huge repo) so they
+        // don't linger in `.gitnexus`. Best-effort — never fail the phase on a
+        // cleanup error.
+        if (parsedFileStorePath) {
+            try {
+                await clearParsedFileStore(parsedFileStorePath);
+            }
+            catch {
+                /* best-effort cleanup */
+            }
+        }
         if (!anyRan)
             return NOOP_OUTPUT;
         return {

package/dist/core/ingestion/scope-resolution/pipeline/registry.d.ts

@@ -10,8 +10,7 @@
  */
 import { SupportedLanguages } from '../../../../_shared/index.js';
 import type { ScopeResolver } from '../contract/scope-resolver.js';
-/** Map of `SupportedLanguages` → `ScopeResolver`. The phase iterates
- *  this map intersected with `MIGRATED_LANGUAGES` (the per-language
- *  flag set) so adding a resolver here without flipping the flag is
- *  safe — the resolver sits idle until the language is migrated. */
+/** Map of `SupportedLanguages` → `ScopeResolver`. The scope-resolution phase
+ *  iterates this map directly — every registered resolver runs. This is the
+ *  single source of truth for which languages resolve via scope-resolution. */
 export declare const SCOPE_RESOLVERS: ReadonlyMap<SupportedLanguages, ScopeResolver>;

package/dist/core/ingestion/scope-resolution/pipeline/registry.js

@@ -22,10 +22,12 @@ import { javascriptScopeResolver } from '../../languages/javascript/scope-resolv
 import { kotlinScopeResolver } from '../../languages/kotlin/scope-resolver.js';
 import { rubyScopeResolver } from '../../languages/ruby/scope-resolver.js';
 import { cobolScopeResolver } from '../../languages/cobol/scope-resolver.js';
-/** Map of `SupportedLanguages` → `ScopeResolver`. The phase iterates
- *  this map intersected with `MIGRATED_LANGUAGES` (the per-language
- *  flag set) so adding a resolver here without flipping the flag is
- *  safe — the resolver sits idle until the language is migrated. */
+import { swiftScopeResolver } from '../../languages/swift/scope-resolver.js';
+import { dartScopeResolver } from '../../languages/dart/scope-resolver.js';
+import { vueScopeResolver } from '../../languages/vue/scope-resolver.js';
+/** Map of `SupportedLanguages` → `ScopeResolver`. The scope-resolution phase
+ *  iterates this map directly — every registered resolver runs. This is the
+ *  single source of truth for which languages resolve via scope-resolution. */
 export const SCOPE_RESOLVERS = new Map([
     [SupportedLanguages.Python, pythonScopeResolver],
     [SupportedLanguages.CSharp, csharpScopeResolver],
@@ -40,4 +42,7 @@ export const SCOPE_RESOLVERS = new Map([
     [SupportedLanguages.Kotlin, kotlinScopeResolver],
     [SupportedLanguages.Ruby, rubyScopeResolver],
     [SupportedLanguages.Cobol, cobolScopeResolver],
+    [SupportedLanguages.Swift, swiftScopeResolver],
+    [SupportedLanguages.Dart, dartScopeResolver],
+    [SupportedLanguages.Vue, vueScopeResolver],
 ]);

package/dist/core/ingestion/scope-resolution/pipeline/run.d.ts

@@ -26,6 +26,7 @@ import type { ParsedFile } from '../../../../_shared/index.js';
 import type { KnowledgeGraph } from '../../../graph/types.js';
 import type { MutableSemanticModel } from '../../model/semantic-model.js';
 import { type ResolveStats } from '../../resolve-references.js';
+import { buildGraphNodeLookup } from '../graph-bridge/node-lookup.js';
 import type { ScopeResolver } from '../contract/scope-resolver.js';
 import type { ResolutionOutcome, ResolutionOutcomeRecorder } from '../resolution-outcome.js';
 export type ScopeResolutionSubPhase = 'extracting' | 'analyzing types' | 'resolving references' | 'linking symbols';
@@ -47,15 +48,28 @@ interface RunScopeResolutionInput {
     }[];
     readonly onWarn?: (message: string) => void;
     /**
-     * Optional pre-parsed-Tree lookup keyed by file path. When the
-     * pipeline's parse phase ran sequentially, it populated an
-     * `ASTCache`; passing that here lets the per-file extract step
-     * skip a second `tree-sitter parser.parse(...)` call. Cache miss
-     * is safe — falls back to a fresh parse inside the provider.
+     * Optional pre-parsed-Tree lookup keyed by file path: a cache hit lets the
+     * per-file extract step skip a second `tree-sitter parser.parse(...)` call.
+     * Currently always empty — the only producer was the (removed) sequential
+     * parser, and workers can't return native Trees across the MessageChannel,
+     * so the parse phase no longer threads one. Kept as an extension point;
+     * cache miss is safe (the provider re-parses).
      */
     readonly treeCache?: {
         get(filePath: string): unknown;
     };
+    /**
+     * Optional graph-node lookup built ONCE by the caller and shared across
+     * every language pass. `buildGraphNodeLookup` scans the whole graph and is
+     * language-agnostic, so rebuilding it per language wastes both CPU and ~GBs
+     * of heap (on the kernel it is ~2 GB; a 5-file language would otherwise build
+     * its own full copy that then overlaps the next language's). When omitted
+     * (tests / isolated calls) the lookup is built locally as before. Providers
+     * that add graph nodes mid-pass (e.g. Ruby heritage Property nodes) still
+     * rebuild a fresh post-heritage lookup internally, so sharing the pre-loop
+     * base is safe.
+     */
+    readonly prebuiltNodeLookup?: ReturnType<typeof buildGraphNodeLookup>;
     /**
      * Opaque per-language import-resolution config (e.g. tsconfig path
      * aliases for TypeScript). Loaded once by the caller via
@@ -80,6 +94,15 @@ interface RunScopeResolutionInput {
      * Cache miss is safe — falls back to fresh extract.
      */
     readonly preExtractedParsedFiles?: ReadonlyMap<string, ParsedFile>;
+    /**
+     * Out-of-core scope index (disk-backed scope seal). When set AND `GITNEXUS_DISK_SCOPE_INDEX` is
+     * enabled, the per-language `scopeTree` is sealed to a disk-backed store at
+     * this path after resolve (before emit), and the heavy `Scope.bindings`
+     * payload is dropped from heap — lowering the per-language peak (kernel:
+     * ~20→~12 GB) so the analysis fits on smaller-RAM machines. Same storage path
+     * as the ParsedFile store; a sibling `scope-index-store/` dir.
+     */
+    readonly scopeIndexStorePath?: string;
     /**
      * Optional additive diagnostics sink. Resolver passes call this when they
      * intentionally suppress an edge; the graph remains unchanged.