@cleocode/nexus 2026.9.16 → 2026.9.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. package/dist/src/pipeline/community-processor.d.ts.map +1 -1
  2. package/dist/src/pipeline/community-processor.js +21 -0
  3. package/dist/src/pipeline/community-processor.js.map +1 -1
  4. package/dist/src/pipeline/extractors/typescript-extractor.d.ts.map +1 -1
  5. package/dist/src/pipeline/extractors/typescript-extractor.js +7 -1
  6. package/dist/src/pipeline/extractors/typescript-extractor.js.map +1 -1
  7. package/dist/src/pipeline/filesystem-walker.d.ts +38 -1
  8. package/dist/src/pipeline/filesystem-walker.d.ts.map +1 -1
  9. package/dist/src/pipeline/filesystem-walker.js +194 -46
  10. package/dist/src/pipeline/filesystem-walker.js.map +1 -1
  11. package/dist/src/pipeline/index.d.ts +52 -15
  12. package/dist/src/pipeline/index.d.ts.map +1 -1
  13. package/dist/src/pipeline/index.js +242 -103
  14. package/dist/src/pipeline/index.js.map +1 -1
  15. package/dist/src/pipeline/parse-cache.d.ts +72 -0
  16. package/dist/src/pipeline/parse-cache.d.ts.map +1 -0
  17. package/dist/src/pipeline/parse-cache.js +158 -0
  18. package/dist/src/pipeline/parse-cache.js.map +1 -0
  19. package/dist/src/pipeline/parse-loop.d.ts +31 -21
  20. package/dist/src/pipeline/parse-loop.d.ts.map +1 -1
  21. package/dist/src/pipeline/parse-loop.js +186 -285
  22. package/dist/src/pipeline/parse-loop.js.map +1 -1
  23. package/dist/src/pipeline/process-processor.d.ts.map +1 -1
  24. package/dist/src/pipeline/process-processor.js +4 -0
  25. package/dist/src/pipeline/process-processor.js.map +1 -1
  26. package/dist/src/pipeline/processors/access-processor.d.ts.map +1 -1
  27. package/dist/src/pipeline/processors/access-processor.js +14 -2
  28. package/dist/src/pipeline/processors/access-processor.js.map +1 -1
  29. package/dist/src/pipeline/workers/parse-worker.d.ts +10 -15
  30. package/dist/src/pipeline/workers/parse-worker.d.ts.map +1 -1
  31. package/dist/src/pipeline/workers/parse-worker.js +12 -12
  32. package/dist/src/pipeline/workers/parse-worker.js.map +1 -1
  33. package/dist/tsconfig.build.tsbuildinfo +1 -1
  34. package/dist/tsconfig.tsbuildinfo +1 -1
  35. package/package.json +2 -2
@@ -488,44 +488,37 @@ async function classifyFileCapabilities(file, repoPath, signal) {
488
488
  reason: 'No positive role evidence; potentially executable content remains unassessed',
489
489
  }));
490
490
  }
491
- // ---------------------------------------------------------------------------
492
- // Main parse loop
493
- // ---------------------------------------------------------------------------
494
- // ---------------------------------------------------------------------------
495
- // Parallel parse path helpers (Wave H — T540)
496
- // ---------------------------------------------------------------------------
491
+ /** Normalize an extractor result so every per-file array is present. */
492
+ function completeExtraction(result) {
493
+ return {
494
+ definitions: result.definitions,
495
+ imports: result.imports,
496
+ heritage: result.heritage,
497
+ calls: result.calls,
498
+ reExports: result.reExports ?? [],
499
+ accesses: result.accesses ?? [],
500
+ };
501
+ }
497
502
  /**
498
- * Run the parse loop using a worker pool for parallel parsing.
499
- *
500
- * Called by `runParseLoop` when the file count or total bytes exceeds the
501
- * worker pool thresholds. Falls back to returning `null` if the worker
502
- * script is not found so the caller can retry sequentially.
503
+ * Extract files through the worker pool (Wave H — T540).
503
504
  *
504
- * @returns ParseLoopResult on success, or null if workers could not be used.
505
+ * @returns Per-file outcomes, or `null` when the worker script is not built.
505
506
  */
506
- async function runParallelParseLoop(parseableFiles, graph, symbolTable, importCtx, repoPath, options) {
507
- const { tsconfigPaths = null, namedImportMap = new Map(), onProgress } = options;
507
+ async function extractInParallel(files, repoPath, options) {
508
508
  // Resolve the compiled worker script path (parallel to this file in dist/)
509
509
  let workerUrl;
510
510
  try {
511
- // In ESM, import.meta.url resolves relative to the current module file.
512
- // The worker script lives at the same directory level.
513
511
  workerUrl = new URL('./workers/parse-worker.js', import.meta.url);
514
- const workerPath = fileURLToPath(workerUrl);
515
- // Use dynamic require check — existsSync from 'node:fs'
516
512
  const { existsSync } = await import('node:fs');
517
- if (!existsSync(workerPath)) {
518
- return null; // Worker script not built yet — fall back to sequential
519
- }
513
+ if (!existsSync(fileURLToPath(workerUrl)))
514
+ return null; // not built — sequential fallback
520
515
  }
521
516
  catch {
522
517
  return null;
523
518
  }
524
- // Read all file contents first (sequential I/O, then dispatch in parallel)
525
- const total = parseableFiles.length;
519
+ const total = files.length;
526
520
  const workerInputs = [];
527
- for (let i = 0; i < parseableFiles.length; i++) {
528
- const file = parseableFiles[i];
521
+ for (const file of files) {
529
522
  try {
530
523
  const absPath = file.path.startsWith('/') ? file.path : `${repoPath}/${file.path}`;
531
524
  options.parserLimits?.signal?.throwIfAborted();
@@ -548,140 +541,152 @@ async function runParallelParseLoop(parseableFiles, graph, symbolTable, importCt
548
541
  }
549
542
  }
550
543
  const pool = createWorkerPool(workerUrl, undefined, options.parserLimits, options.parserExecution);
551
- let filesProcessedSoFar = 0;
552
544
  let workerResults;
553
545
  try {
554
546
  workerResults = await pool.dispatch(workerInputs, (filesProcessed) => {
555
- filesProcessedSoFar = filesProcessed;
556
- if (onProgress) {
557
- const lastFile = parseableFiles[Math.min(filesProcessed, total) - 1];
558
- onProgress(filesProcessed, total, lastFile?.path ?? '');
547
+ if (options.onProgress) {
548
+ const lastFile = files[Math.min(filesProcessed, total) - 1];
549
+ options.onProgress(filesProcessed, total, lastFile?.path ?? '');
559
550
  }
560
551
  });
561
552
  }
562
553
  finally {
563
554
  await pool.terminate().catch(() => undefined);
564
555
  }
565
- void filesProcessedSoFar; // used for progress, not needed after dispatch
566
- // Merge all worker results into graph, symbolTable, and accumulator arrays
567
- const allExtractedImports = [];
568
- const allParallelReExports = [];
569
- const allHeritage = [];
570
- const allCalls = [];
571
- // Workers invoke the same extractor, including access records.
572
- const allParallelAccesses = [];
573
- /** Files no worker could parse; the publish step decides what they mean. */
574
- const unparsedFiles = [];
556
+ if (!options.onProgress && total > 0) {
557
+ process.stderr.write(`[nexus] Parsing: ${total}/${total} files (100%) [parallel]\n`);
558
+ }
559
+ const pass = { extracted: new Map(), failures: new Map() };
575
560
  for (const workerResult of workerResults) {
576
- for (const report of workerResult.reports)
577
- options.onFileReport?.(report);
578
- // T12313: this threw on ANY unparsed file, which ended the run before the
579
- // pipeline could weigh how much of the repository was actually affected.
580
- // The reports are already forwarded above, so the failures travel onward
581
- // and the publish step decides — it tolerates a small number of recorded,
582
- // reported gaps and still refuses a generation that would misrepresent the
583
- // codebase. Deciding that here, per worker, could only ever see one chunk.
584
- const failed = workerResult.reports.filter((report) => report.status !== 'analyzed');
585
- if (failed.length > 0) {
586
- unparsedFiles.push(...failed);
587
- }
588
- allParallelAccesses.push(...workerResult.accesses);
589
- // Register symbols into SymbolTable and add graph nodes
590
- const fileGraphNodes = new Map();
591
- for (const sym of workerResult.symbols) {
592
- const graphNode = sym;
593
- registerInSymbolTable([graphNode], symbolTable);
594
- graph.addNode(graphNode);
595
- // Group by filePath for defines edge emission below
596
- if (graphNode.filePath) {
597
- const bucket = fileGraphNodes.get(graphNode.filePath) ?? [];
598
- bucket.push(graphNode);
599
- fileGraphNodes.set(graphNode.filePath, bucket);
600
- }
561
+ // T12313: failures travel onward as reports; the publish step weighs how
562
+ // much of the repository they affect instead of one chunk deciding alone.
563
+ for (const report of workerResult.reports) {
564
+ if (report.status !== 'analyzed')
565
+ pass.failures.set(report.path, report);
601
566
  }
602
- // Emit defines edges: file → each symbol it declares
603
- for (const [filePath, nodes] of fileGraphNodes) {
604
- emitDefinesEdges(filePath, nodes, graph);
605
- }
606
- allExtractedImports.push(...workerResult.imports);
607
- allParallelReExports.push(...workerResult.reExports);
608
- allHeritage.push(...workerResult.heritage);
609
- allCalls.push(...workerResult.calls);
610
- }
611
- if (onProgress && total > 0) {
612
- const lastFile = parseableFiles[total - 1];
613
- onProgress(total, total, lastFile?.path ?? '');
567
+ for (const file of workerResult.files)
568
+ pass.extracted.set(file.path, file);
614
569
  }
615
- else if (!onProgress && total > 0) {
616
- process.stderr.write(`[nexus] Parsing: ${total}/${total} files (100%) [parallel]\n`);
617
- }
618
- // Batch-resolve all extracted imports
619
- if (allExtractedImports.length > 0) {
620
- await processExtractedImports({
621
- imports: allExtractedImports,
622
- graph,
623
- importCtx,
624
- namedImportMap,
625
- tsconfigPaths,
626
- });
570
+ return pass;
571
+ }
572
+ /** Extract files one at a time in the calling thread. */
573
+ async function extractSequentially(files, repoPath, options) {
574
+ const pass = { extracted: new Map(), failures: new Map() };
575
+ const fail = (path, reason) => {
576
+ pass.failures.set(path, { path, status: 'failed', reason });
577
+ };
578
+ const parser = getParser();
579
+ if (!parser) {
580
+ for (const file of files)
581
+ fail(file.path, 'tree-sitter native module unavailable');
582
+ process.stderr.write('[nexus] WARNING: tree-sitter native module not available — parse loop skipped.\n');
583
+ return pass;
627
584
  }
628
- // Build barrel export map from worker-collected re-export records (T617)
629
- if (process.env['CLEO_BARREL_DEBUG']) {
630
- const coreInternalRecords = allParallelReExports.filter((re) => re.filePath.includes('core/src/internal'));
631
- process.stderr.write(`[parse-loop-debug] allParallelReExports.length = ${allParallelReExports.length}\n`);
632
- process.stderr.write(`[parse-loop-debug] core/src/internal records = ${coreInternalRecords.length}\n`);
633
- // Check if internal.ts was in the worker inputs
634
- const internalInput = workerInputs.find((f) => f.path.includes('core/src/internal'));
635
- process.stderr.write(`[parse-loop-debug] internal.ts in workerInputs = ${internalInput ? 'YES: ' + internalInput.path : 'NO'}\n`);
636
- // Check if internal.ts had symbols or imports
637
- for (const wr of workerResults) {
638
- const hasInternalSymbol = wr.symbols.some((s) => s.filePath?.includes('core/src/internal'));
639
- const hasInternalImport = wr.imports.some((i) => i.filePath.includes('core/src/internal'));
640
- const hasInternalReExport = wr.reExports.some((r) => r.filePath.includes('core/src/internal'));
641
- if (hasInternalSymbol || hasInternalImport || hasInternalReExport) {
642
- process.stderr.write(`[parse-loop-debug] Worker result: symbols=${hasInternalSymbol}, imports=${hasInternalImport}, reExports=${hasInternalReExport}\n`);
585
+ const total = files.length;
586
+ let filesProcessed = 0;
587
+ for (const file of files) {
588
+ options.parserLimits?.signal?.throwIfAborted();
589
+ filesProcessed++;
590
+ if (options.onProgress &&
591
+ (total <= 100 || filesProcessed % 10 === 0 || filesProcessed === total)) {
592
+ options.onProgress(filesProcessed, total, file.path);
593
+ }
594
+ else if (!options.onProgress && filesProcessed % 50 === 0) {
595
+ const pct = Math.round((filesProcessed / total) * 100);
596
+ process.stderr.write(`[nexus] Parsing: ${filesProcessed}/${total} files (${pct}%)...\n`);
597
+ }
598
+ const lang = detectLanguageFromPath(file.path);
599
+ if (!lang)
600
+ continue;
601
+ const grammarKey = grammarKeyForLanguage(lang);
602
+ if (!grammarKey)
603
+ continue;
604
+ const grammar = loadGrammar(grammarKey);
605
+ if (!grammar) {
606
+ fail(file.path, `No grammar for ${lang}`);
607
+ process.stderr.write(`[nexus] SKIP: no grammar for ${lang} (file: ${file.path})\n`);
608
+ continue;
609
+ }
610
+ let source;
611
+ try {
612
+ const absPath = file.path.startsWith('/') ? file.path : `${repoPath}/${file.path}`;
613
+ const bytes = await fs.readFile(absPath);
614
+ if (file.contentHash &&
615
+ createHash('sha256').update(bytes).digest('hex') !== file.contentHash) {
616
+ throw new Error('Source changed between scanning and parsing');
643
617
  }
618
+ source = bytes.toString('utf-8');
644
619
  }
620
+ catch (err) {
621
+ const msg = err instanceof Error ? err.message : String(err);
622
+ fail(file.path, `read: ${msg}`);
623
+ process.stderr.write(`[nexus] SKIP read error: ${file.path}: ${msg}\n`);
624
+ continue;
625
+ }
626
+ let rootNode;
627
+ try {
628
+ parser.setLanguage(grammar);
629
+ rootNode = parseOriginalSource(parser, source, options.parserLimits).rootNode;
630
+ }
631
+ catch (err) {
632
+ const msg = err instanceof Error ? err.message : String(err);
633
+ fail(file.path, `parse: ${msg}`);
634
+ options.parserLimits?.signal?.throwIfAborted();
635
+ process.stderr.write(`[nexus] SKIP parse error: ${file.path}: ${msg}\n`);
636
+ continue;
637
+ }
638
+ try {
639
+ const extracted = runExtractor(lang, rootNode, file.path, createHash('sha256').update(source).digest('hex'), options.publicationGeneration);
640
+ pass.extracted.set(file.path, {
641
+ path: file.path,
642
+ extraction: completeExtraction(extracted),
643
+ });
644
+ }
645
+ catch (err) {
646
+ const msg = err instanceof Error ? err.message : String(err);
647
+ fail(file.path, `extract: ${msg}`);
648
+ process.stderr.write(`[nexus] SKIP extract error: ${file.path}: ${msg}\n`);
649
+ continue;
650
+ }
651
+ // Yield to event loop periodically on large repos
652
+ if (filesProcessed % 100 === 0)
653
+ await Promise.resolve();
654
+ }
655
+ if (!options.onProgress && total > 0) {
656
+ process.stderr.write(`[nexus] Parsing: ${total}/${total} files (100%)\n`);
645
657
  }
646
- const parallelBarrelMap = buildBarrelExportMap(allParallelReExports, importCtx, tsconfigPaths);
647
- process.stderr.write(`[nexus] Barrel map: ${parallelBarrelMap.size} barrel files with re-export chains\n`);
648
- return { allHeritage, allCalls, allAccesses: allParallelAccesses, barrelMap: parallelBarrelMap };
658
+ return pass;
649
659
  }
650
660
  // ---------------------------------------------------------------------------
651
661
  // Main parse loop entry point
652
662
  // ---------------------------------------------------------------------------
653
663
  /**
654
- * Phase 3: Parse loop (parallel or sequential).
664
+ * Phase 3: Parse loop (parallel or sequential), then merge.
655
665
  *
656
- * For each TypeScript/JavaScript file in `files`:
657
- * 1. Read file content from disk
658
- * 2. Parse with tree-sitter (typescript or javascript grammar)
659
- * 3. Extract definitions → register in SymbolTable → add nodes to graph
660
- * 4. Extract imports → collect for batch resolution
661
- * 5. Extract heritage → accumulate for deferred edge emission
666
+ * 1. Classify every scanned file's role from its bytes.
667
+ * 2. Extract each parseable file that has no reusable cached extraction —
668
+ * through the worker pool when the file count or byte volume warrants it.
669
+ * 3. Merge fresh and reused extractions **in scan order**: register symbols,
670
+ * add nodes, emit DEFINES edges, and collect imports, re-exports, heritage,
671
+ * calls and accesses. The merge order is the same whichever path produced
672
+ * an extraction, so first-wins deduplication in the graph cannot depend on
673
+ * which files happened to be re-parsed.
674
+ * 4. Resolve all imports and build the barrel map over the merged input.
662
675
  *
663
- * After all files are processed:
664
- * - Resolves all collected imports via `processExtractedImports`
665
- * - Emits EXTENDS/IMPLEMENTS edges from accumulated heritage
676
+ * Files that fail to parse are reported, not merged; the publish step decides
677
+ * what a given share of failures means.
666
678
  *
667
- * Files that fail to parse (grammar unavailable, syntax error, read error)
668
- * are skipped with a warning to stderr — the loop continues.
669
- *
670
- * **Parallel mode** (Wave H): When `parseableFiles.length >= 15` or total
671
- * bytes >= 512 KB, the parallel path via the worker pool is attempted first.
672
- * If the worker script is not found (e.g. running from source without a
673
- * build), the sequential path is used as a fallback.
674
- *
675
- * @param files - All scanned files (non-TypeScript/JavaScript files are filtered)
679
+ * @param files - All scanned files (non-parseable files are classified and reported only)
676
680
  * @param graph - Knowledge graph to add nodes and relations to
677
681
  * @param symbolTable - Symbol table to register extracted symbols in
678
682
  * @param importCtx - Pre-built import resolution context (from Phase 3a)
679
683
  * @param repoPath - Absolute path to the repository root (used to resolve relative paths)
680
- * @param options - Optional tsconfig paths, named import map, and progress callback
681
- * @returns Accumulated heritage and call records for Phase 3c and Phase 3e
684
+ * @param options - Resolution inputs, reuse map, progress and report callbacks
685
+ * @returns Accumulated heritage, call and access records plus the barrel map
682
686
  */
683
687
  export async function runParseLoop(files, graph, symbolTable, importCtx, repoPath, options = {}) {
684
- const { tsconfigPaths = null, namedImportMap = new Map(), onProgress } = options;
688
+ const { tsconfigPaths = null, namedImportMap = new Map() } = options;
689
+ const reused = options.reusedExtractions ?? new Map();
685
690
  // Attach one capability contract to sequential and worker-returned reports.
686
691
  const callerReports = options.onFileReport;
687
692
  const classifications = new Map();
@@ -701,7 +706,6 @@ export async function runParseLoop(files, graph, symbolTable, importCtx, repoPat
701
706
  },
702
707
  });
703
708
  };
704
- options = { ...options, onFileReport: emitReport };
705
709
  const parseableFiles = [];
706
710
  for (const file of files) {
707
711
  options.parserLimits?.signal?.throwIfAborted();
@@ -734,173 +738,64 @@ export async function runParseLoop(files, graph, symbolTable, importCtx, repoPat
734
738
  else
735
739
  emitReport(report);
736
740
  }
737
- const total = parseableFiles.length;
738
- if (total === 0)
739
- return {
740
- allHeritage: [],
741
- allCalls: [],
742
- allAccesses: [],
743
- barrelMap: buildBarrelExportMap([], importCtx, tsconfigPaths),
744
- };
745
- // Wave H: Check thresholds for parallel dispatch
746
- const totalBytes = files.reduce((acc, f) => acc + f.size, 0);
747
- const useWorkers = Boolean(options.parserExecution) ||
748
- total >= WORKER_FILE_THRESHOLD ||
749
- totalBytes >= WORKER_BYTE_THRESHOLD;
750
- if (useWorkers && (options.parserExecution || !callerReports)) {
751
- process.stderr.write(`[nexus] Parallel parse: ${total} files, ${Math.round(totalBytes / 1024)}KB total — spawning worker pool\n`);
752
- try {
753
- const parallelResult = await runParallelParseLoop(parseableFiles, graph, symbolTable, importCtx, repoPath, { ...options, tsconfigPaths, namedImportMap, onProgress });
754
- if (parallelResult !== null) {
741
+ const toParse = parseableFiles.filter((file) => !reused.has(file.path));
742
+ let pass = { extracted: new Map(), failures: new Map() };
743
+ if (toParse.length > 0) {
744
+ const totalBytes = toParse.reduce((acc, f) => acc + f.size, 0);
745
+ const useWorkers = Boolean(options.parserExecution) ||
746
+ toParse.length >= WORKER_FILE_THRESHOLD ||
747
+ totalBytes >= WORKER_BYTE_THRESHOLD;
748
+ let parallel = null;
749
+ if (useWorkers && (options.parserExecution || !callerReports)) {
750
+ process.stderr.write(`[nexus] Parallel parse: ${toParse.length} files, ${Math.round(totalBytes / 1024)}KB total — spawning worker pool\n`);
751
+ try {
752
+ parallel = await extractInParallel(toParse, repoPath, options);
753
+ }
754
+ catch (err) {
755
+ const msg = err instanceof Error ? err.message : String(err);
756
+ throw new Error(`Parallel parser failed; generation not published: ${msg}`);
757
+ }
758
+ if (parallel === null) {
759
+ if (options.parserExecution)
760
+ throw new Error('Isolated parser executable unavailable');
761
+ process.stderr.write('[nexus] Worker script not found — using sequential parse.\n');
762
+ }
763
+ else {
755
764
  process.stderr.write('[nexus] Parallel parse complete.\n');
756
- return parallelResult;
757
765
  }
758
- if (options.parserExecution)
759
- throw new Error('Isolated parser executable unavailable');
760
- // Worker unavailable — fall through to sequential
761
- process.stderr.write('[nexus] Worker script not found — using sequential parse.\n');
762
- }
763
- catch (err) {
764
- const msg = err instanceof Error ? err.message : String(err);
765
- throw new Error(`Parallel parser failed; generation not published: ${msg}`);
766
766
  }
767
+ pass = parallel ?? (await extractSequentially(toParse, repoPath, options));
767
768
  }
769
+ // Merge in scan order — identical whether an extraction was fresh or reused.
768
770
  const allExtractedImports = [];
769
771
  const allReExports = [];
770
772
  const allHeritage = [];
771
773
  const allCalls = [];
772
774
  const allAccesses = [];
773
- const parser = getParser();
774
- if (!parser) {
775
- for (const file of parseableFiles) {
776
- options.onFileReport?.({
777
- path: file.path,
778
- status: 'failed',
779
- reason: 'tree-sitter native module unavailable',
780
- });
781
- }
782
- process.stderr.write('[nexus] WARNING: tree-sitter native module not available — parse loop skipped.\n');
783
- return {
784
- allHeritage: [],
785
- allCalls: [],
786
- allAccesses: [],
787
- barrelMap: buildBarrelExportMap([], importCtx, tsconfigPaths),
788
- };
789
- }
790
- let filesProcessed = 0;
791
775
  for (const file of parseableFiles) {
792
- options.parserLimits?.signal?.throwIfAborted();
793
- filesProcessed++;
794
- // Progress reporting (every file for small repos, every 10 for larger ones)
795
- if (onProgress && (total <= 100 || filesProcessed % 10 === 0 || filesProcessed === total)) {
796
- onProgress(filesProcessed, total, file.path);
797
- }
798
- else if (!onProgress && filesProcessed % 50 === 0) {
799
- // Fallback stderr progress for CLI usage without a callback
800
- const pct = Math.round((filesProcessed / total) * 100);
801
- process.stderr.write(`[nexus] Parsing: ${filesProcessed}/${total} files (${pct}%)...\n`);
802
- }
803
- // Determine grammar key
804
- const lang = detectLanguageFromPath(file.path);
805
- if (!lang)
806
- continue;
807
- const grammarKey = grammarKeyForLanguage(lang);
808
- if (!grammarKey)
809
- continue;
810
- // Load grammar
811
- const grammar = loadGrammar(grammarKey);
812
- if (!grammar) {
813
- options.onFileReport?.({
814
- path: file.path,
815
- status: 'failed',
816
- reason: `No grammar for ${lang}`,
817
- });
818
- process.stderr.write(`[nexus] SKIP: no grammar for ${lang} (file: ${file.path})\n`);
819
- continue;
820
- }
821
- // Read file content — relative path from filesystem walker, read from repoPath
822
- let source;
823
- try {
824
- const absPath = file.path.startsWith('/') ? file.path : `${repoPath}/${file.path}`;
825
- const bytes = await fs.readFile(absPath);
826
- if (file.contentHash &&
827
- createHash('sha256').update(bytes).digest('hex') !== file.contentHash) {
828
- throw new Error('Source changed between scanning and parsing');
829
- }
830
- source = bytes.toString('utf-8');
831
- }
832
- catch (err) {
833
- const msg = err instanceof Error ? err.message : String(err);
834
- options.onFileReport?.({ path: file.path, status: 'failed', reason: `read: ${msg}` });
835
- process.stderr.write(`[nexus] SKIP read error: ${file.path}: ${msg}\n`);
776
+ const fresh = pass.extracted.get(file.path);
777
+ const fileExtraction = fresh ?? reused.get(file.path);
778
+ if (!fileExtraction) {
779
+ const failure = pass.failures.get(file.path);
780
+ // Unknown language/grammar keys are silently skipped exactly as before.
781
+ if (failure)
782
+ emitReport(failure);
836
783
  continue;
837
784
  }
838
- // Parse with tree-sitter
839
- let rootNode;
840
- try {
841
- parser.setLanguage(grammar);
842
- const tree = parseOriginalSource(parser, source, options.parserLimits);
843
- rootNode = tree.rootNode;
844
- }
845
- catch (err) {
846
- const msg = err instanceof Error ? err.message : String(err);
847
- options.onFileReport?.({ path: file.path, status: 'failed', reason: `parse: ${msg}` });
848
- options.parserLimits?.signal?.throwIfAborted();
849
- process.stderr.write(`[nexus] SKIP parse error: ${file.path}: ${msg}\n`);
850
- continue;
851
- }
852
- // Extract definitions, imports, heritage, calls, and re-exports — dispatch by language
853
- let extracted;
854
- try {
855
- extracted = runExtractor(lang, rootNode, file.path, createHash('sha256').update(source).digest('hex'), options.publicationGeneration);
856
- }
857
- catch (err) {
858
- const msg = err instanceof Error ? err.message : String(err);
859
- options.onFileReport?.({ path: file.path, status: 'failed', reason: `extract: ${msg}` });
860
- process.stderr.write(`[nexus] SKIP extract error: ${file.path}: ${msg}\n`);
861
- continue;
862
- }
863
- options.onFileReport?.({ path: file.path, status: 'analyzed' });
864
- // Register in SymbolTable
785
+ // Capture the cache entry before any later phase mutates these nodes.
786
+ if (fresh)
787
+ options.onFileExtracted?.(fresh);
788
+ emitReport({ path: file.path, status: 'analyzed' });
789
+ const extracted = fileExtraction.extraction;
865
790
  registerInSymbolTable(extracted.definitions, symbolTable);
866
- // Add nodes to graph
867
- for (const node of extracted.definitions) {
791
+ for (const node of extracted.definitions)
868
792
  graph.addNode(node);
869
- }
870
- // Emit defines edges: file → each symbol it declares (T1836)
871
793
  emitDefinesEdges(file.path, extracted.definitions, graph);
872
- // Collect imports for batch resolution
873
- for (const imp of extracted.imports) {
874
- allExtractedImports.push(imp);
875
- }
876
- // Collect re-exports for barrel map construction (T617)
877
- if (extracted.reExports) {
878
- for (const re of extracted.reExports) {
879
- allReExports.push(re);
880
- }
881
- }
882
- // Collect heritage for deferred Phase 3c processing
883
- for (const h of extracted.heritage) {
884
- allHeritage.push(h);
885
- }
886
- // Collect calls for deferred Phase 3e resolution
887
- for (const c of extracted.calls) {
888
- allCalls.push(c);
889
- }
890
- // Collect access sites for deferred Phase 3f resolution (T1837)
891
- if (extracted.accesses) {
892
- for (const a of extracted.accesses) {
893
- allAccesses.push(a);
894
- }
895
- }
896
- // Yield to event loop periodically on large repos
897
- if (filesProcessed % 100 === 0) {
898
- await Promise.resolve();
899
- }
900
- }
901
- // Final stderr progress update
902
- if (!onProgress && total > 0) {
903
- process.stderr.write(`[nexus] Parsing: ${total}/${total} files (100%)\n`);
794
+ allExtractedImports.push(...extracted.imports);
795
+ allReExports.push(...extracted.reExports);
796
+ allHeritage.push(...extracted.heritage);
797
+ allCalls.push(...extracted.calls);
798
+ allAccesses.push(...extracted.accesses);
904
799
  }
905
800
  // Batch-resolve all extracted imports (populates namedImportMap for Phase 3e)
906
801
  if (allExtractedImports.length > 0) {
@@ -912,11 +807,17 @@ export async function runParseLoop(files, graph, symbolTable, importCtx, repoPat
912
807
  tsconfigPaths,
913
808
  });
914
809
  }
915
- // Build barrel export map from collected re-export records (T617)
810
+ // Build barrel export map from collected re-export records (T617).
916
811
  // Runs AFTER processExtractedImports so the resolve cache is warmed up.
917
812
  const barrelMap = buildBarrelExportMap(allReExports, importCtx, tsconfigPaths);
918
813
  process.stderr.write(`[nexus] Barrel map: ${barrelMap.size} barrel files with re-export chains\n`);
919
- // Return accumulated heritage, calls, accesses, and barrel map
920
- return { allHeritage, allCalls, allAccesses, barrelMap };
814
+ return {
815
+ allHeritage,
816
+ allCalls,
817
+ allAccesses,
818
+ barrelMap,
819
+ parsedFileCount: toParse.length,
820
+ reusedFileCount: parseableFiles.length - toParse.length,
821
+ };
921
822
  }
922
823
  //# sourceMappingURL=parse-loop.js.map