@cleocode/nexus 2026.9.15 → 2026.9.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/pipeline/community-processor.d.ts.map +1 -1
- package/dist/src/pipeline/community-processor.js +21 -0
- package/dist/src/pipeline/community-processor.js.map +1 -1
- package/dist/src/pipeline/extractors/typescript-extractor.d.ts.map +1 -1
- package/dist/src/pipeline/extractors/typescript-extractor.js +7 -1
- package/dist/src/pipeline/extractors/typescript-extractor.js.map +1 -1
- package/dist/src/pipeline/filesystem-walker.d.ts +25 -1
- package/dist/src/pipeline/filesystem-walker.d.ts.map +1 -1
- package/dist/src/pipeline/filesystem-walker.js +16 -4
- package/dist/src/pipeline/filesystem-walker.js.map +1 -1
- package/dist/src/pipeline/index.d.ts +52 -7
- package/dist/src/pipeline/index.d.ts.map +1 -1
- package/dist/src/pipeline/index.js +266 -96
- package/dist/src/pipeline/index.js.map +1 -1
- package/dist/src/pipeline/parse-cache.d.ts +72 -0
- package/dist/src/pipeline/parse-cache.d.ts.map +1 -0
- package/dist/src/pipeline/parse-cache.js +158 -0
- package/dist/src/pipeline/parse-cache.js.map +1 -0
- package/dist/src/pipeline/parse-loop.d.ts +31 -21
- package/dist/src/pipeline/parse-loop.d.ts.map +1 -1
- package/dist/src/pipeline/parse-loop.js +186 -276
- package/dist/src/pipeline/parse-loop.js.map +1 -1
- package/dist/src/pipeline/process-processor.d.ts.map +1 -1
- package/dist/src/pipeline/process-processor.js +4 -0
- package/dist/src/pipeline/process-processor.js.map +1 -1
- package/dist/src/pipeline/processors/access-processor.d.ts.map +1 -1
- package/dist/src/pipeline/processors/access-processor.js +14 -2
- package/dist/src/pipeline/processors/access-processor.js.map +1 -1
- package/dist/src/pipeline/workers/parse-worker.d.ts +10 -15
- package/dist/src/pipeline/workers/parse-worker.d.ts.map +1 -1
- package/dist/src/pipeline/workers/parse-worker.js +12 -12
- package/dist/src/pipeline/workers/parse-worker.js.map +1 -1
- package/dist/src/pipeline/workers/worker-pool.d.ts.map +1 -1
- package/dist/src/pipeline/workers/worker-pool.js +106 -4
- package/dist/src/pipeline/workers/worker-pool.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
|
@@ -488,44 +488,37 @@ async function classifyFileCapabilities(file, repoPath, signal) {
|
|
|
488
488
|
reason: 'No positive role evidence; potentially executable content remains unassessed',
|
|
489
489
|
}));
|
|
490
490
|
}
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
491
|
+
/** Normalize an extractor result so every per-file array is present. */
|
|
492
|
+
function completeExtraction(result) {
|
|
493
|
+
return {
|
|
494
|
+
definitions: result.definitions,
|
|
495
|
+
imports: result.imports,
|
|
496
|
+
heritage: result.heritage,
|
|
497
|
+
calls: result.calls,
|
|
498
|
+
reExports: result.reExports ?? [],
|
|
499
|
+
accesses: result.accesses ?? [],
|
|
500
|
+
};
|
|
501
|
+
}
|
|
497
502
|
/**
|
|
498
|
-
*
|
|
499
|
-
*
|
|
500
|
-
* Called by `runParseLoop` when the file count or total bytes exceeds the
|
|
501
|
-
* worker pool thresholds. Falls back to returning `null` if the worker
|
|
502
|
-
* script is not found so the caller can retry sequentially.
|
|
503
|
+
* Extract files through the worker pool (Wave H — T540).
|
|
503
504
|
*
|
|
504
|
-
* @returns
|
|
505
|
+
* @returns Per-file outcomes, or `null` when the worker script is not built.
|
|
505
506
|
*/
|
|
506
|
-
async function
|
|
507
|
-
const { tsconfigPaths = null, namedImportMap = new Map(), onProgress } = options;
|
|
507
|
+
async function extractInParallel(files, repoPath, options) {
|
|
508
508
|
// Resolve the compiled worker script path (parallel to this file in dist/)
|
|
509
509
|
let workerUrl;
|
|
510
510
|
try {
|
|
511
|
-
// In ESM, import.meta.url resolves relative to the current module file.
|
|
512
|
-
// The worker script lives at the same directory level.
|
|
513
511
|
workerUrl = new URL('./workers/parse-worker.js', import.meta.url);
|
|
514
|
-
const workerPath = fileURLToPath(workerUrl);
|
|
515
|
-
// Use dynamic require check — existsSync from 'node:fs'
|
|
516
512
|
const { existsSync } = await import('node:fs');
|
|
517
|
-
if (!existsSync(
|
|
518
|
-
return null; //
|
|
519
|
-
}
|
|
513
|
+
if (!existsSync(fileURLToPath(workerUrl)))
|
|
514
|
+
return null; // not built — sequential fallback
|
|
520
515
|
}
|
|
521
516
|
catch {
|
|
522
517
|
return null;
|
|
523
518
|
}
|
|
524
|
-
|
|
525
|
-
const total = parseableFiles.length;
|
|
519
|
+
const total = files.length;
|
|
526
520
|
const workerInputs = [];
|
|
527
|
-
for (
|
|
528
|
-
const file = parseableFiles[i];
|
|
521
|
+
for (const file of files) {
|
|
529
522
|
try {
|
|
530
523
|
const absPath = file.path.startsWith('/') ? file.path : `${repoPath}/${file.path}`;
|
|
531
524
|
options.parserLimits?.signal?.throwIfAborted();
|
|
@@ -548,131 +541,152 @@ async function runParallelParseLoop(parseableFiles, graph, symbolTable, importCt
|
|
|
548
541
|
}
|
|
549
542
|
}
|
|
550
543
|
const pool = createWorkerPool(workerUrl, undefined, options.parserLimits, options.parserExecution);
|
|
551
|
-
let filesProcessedSoFar = 0;
|
|
552
544
|
let workerResults;
|
|
553
545
|
try {
|
|
554
546
|
workerResults = await pool.dispatch(workerInputs, (filesProcessed) => {
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
onProgress(filesProcessed, total, lastFile?.path ?? '');
|
|
547
|
+
if (options.onProgress) {
|
|
548
|
+
const lastFile = files[Math.min(filesProcessed, total) - 1];
|
|
549
|
+
options.onProgress(filesProcessed, total, lastFile?.path ?? '');
|
|
559
550
|
}
|
|
560
551
|
});
|
|
561
552
|
}
|
|
562
553
|
finally {
|
|
563
554
|
await pool.terminate().catch(() => undefined);
|
|
564
555
|
}
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
const
|
|
569
|
-
const allHeritage = [];
|
|
570
|
-
const allCalls = [];
|
|
571
|
-
// Workers invoke the same extractor, including access records.
|
|
572
|
-
const allParallelAccesses = [];
|
|
556
|
+
if (!options.onProgress && total > 0) {
|
|
557
|
+
process.stderr.write(`[nexus] Parsing: ${total}/${total} files (100%) [parallel]\n`);
|
|
558
|
+
}
|
|
559
|
+
const pass = { extracted: new Map(), failures: new Map() };
|
|
573
560
|
for (const workerResult of workerResults) {
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
const
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
allParallelAccesses.push(...workerResult.accesses);
|
|
580
|
-
// Register symbols into SymbolTable and add graph nodes
|
|
581
|
-
const fileGraphNodes = new Map();
|
|
582
|
-
for (const sym of workerResult.symbols) {
|
|
583
|
-
const graphNode = sym;
|
|
584
|
-
registerInSymbolTable([graphNode], symbolTable);
|
|
585
|
-
graph.addNode(graphNode);
|
|
586
|
-
// Group by filePath for defines edge emission below
|
|
587
|
-
if (graphNode.filePath) {
|
|
588
|
-
const bucket = fileGraphNodes.get(graphNode.filePath) ?? [];
|
|
589
|
-
bucket.push(graphNode);
|
|
590
|
-
fileGraphNodes.set(graphNode.filePath, bucket);
|
|
591
|
-
}
|
|
592
|
-
}
|
|
593
|
-
// Emit defines edges: file → each symbol it declares
|
|
594
|
-
for (const [filePath, nodes] of fileGraphNodes) {
|
|
595
|
-
emitDefinesEdges(filePath, nodes, graph);
|
|
561
|
+
// T12313: failures travel onward as reports; the publish step weighs how
|
|
562
|
+
// much of the repository they affect instead of one chunk deciding alone.
|
|
563
|
+
for (const report of workerResult.reports) {
|
|
564
|
+
if (report.status !== 'analyzed')
|
|
565
|
+
pass.failures.set(report.path, report);
|
|
596
566
|
}
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
allHeritage.push(...workerResult.heritage);
|
|
600
|
-
allCalls.push(...workerResult.calls);
|
|
567
|
+
for (const file of workerResult.files)
|
|
568
|
+
pass.extracted.set(file.path, file);
|
|
601
569
|
}
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
tsconfigPaths,
|
|
617
|
-
});
|
|
570
|
+
return pass;
|
|
571
|
+
}
|
|
572
|
+
/** Extract files one at a time in the calling thread. */
|
|
573
|
+
async function extractSequentially(files, repoPath, options) {
|
|
574
|
+
const pass = { extracted: new Map(), failures: new Map() };
|
|
575
|
+
const fail = (path, reason) => {
|
|
576
|
+
pass.failures.set(path, { path, status: 'failed', reason });
|
|
577
|
+
};
|
|
578
|
+
const parser = getParser();
|
|
579
|
+
if (!parser) {
|
|
580
|
+
for (const file of files)
|
|
581
|
+
fail(file.path, 'tree-sitter native module unavailable');
|
|
582
|
+
process.stderr.write('[nexus] WARNING: tree-sitter native module not available — parse loop skipped.\n');
|
|
583
|
+
return pass;
|
|
618
584
|
}
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
const
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
585
|
+
const total = files.length;
|
|
586
|
+
let filesProcessed = 0;
|
|
587
|
+
for (const file of files) {
|
|
588
|
+
options.parserLimits?.signal?.throwIfAborted();
|
|
589
|
+
filesProcessed++;
|
|
590
|
+
if (options.onProgress &&
|
|
591
|
+
(total <= 100 || filesProcessed % 10 === 0 || filesProcessed === total)) {
|
|
592
|
+
options.onProgress(filesProcessed, total, file.path);
|
|
593
|
+
}
|
|
594
|
+
else if (!options.onProgress && filesProcessed % 50 === 0) {
|
|
595
|
+
const pct = Math.round((filesProcessed / total) * 100);
|
|
596
|
+
process.stderr.write(`[nexus] Parsing: ${filesProcessed}/${total} files (${pct}%)...\n`);
|
|
597
|
+
}
|
|
598
|
+
const lang = detectLanguageFromPath(file.path);
|
|
599
|
+
if (!lang)
|
|
600
|
+
continue;
|
|
601
|
+
const grammarKey = grammarKeyForLanguage(lang);
|
|
602
|
+
if (!grammarKey)
|
|
603
|
+
continue;
|
|
604
|
+
const grammar = loadGrammar(grammarKey);
|
|
605
|
+
if (!grammar) {
|
|
606
|
+
fail(file.path, `No grammar for ${lang}`);
|
|
607
|
+
process.stderr.write(`[nexus] SKIP: no grammar for ${lang} (file: ${file.path})\n`);
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
let source;
|
|
611
|
+
try {
|
|
612
|
+
const absPath = file.path.startsWith('/') ? file.path : `${repoPath}/${file.path}`;
|
|
613
|
+
const bytes = await fs.readFile(absPath);
|
|
614
|
+
if (file.contentHash &&
|
|
615
|
+
createHash('sha256').update(bytes).digest('hex') !== file.contentHash) {
|
|
616
|
+
throw new Error('Source changed between scanning and parsing');
|
|
634
617
|
}
|
|
618
|
+
source = bytes.toString('utf-8');
|
|
619
|
+
}
|
|
620
|
+
catch (err) {
|
|
621
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
622
|
+
fail(file.path, `read: ${msg}`);
|
|
623
|
+
process.stderr.write(`[nexus] SKIP read error: ${file.path}: ${msg}\n`);
|
|
624
|
+
continue;
|
|
635
625
|
}
|
|
626
|
+
let rootNode;
|
|
627
|
+
try {
|
|
628
|
+
parser.setLanguage(grammar);
|
|
629
|
+
rootNode = parseOriginalSource(parser, source, options.parserLimits).rootNode;
|
|
630
|
+
}
|
|
631
|
+
catch (err) {
|
|
632
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
633
|
+
fail(file.path, `parse: ${msg}`);
|
|
634
|
+
options.parserLimits?.signal?.throwIfAborted();
|
|
635
|
+
process.stderr.write(`[nexus] SKIP parse error: ${file.path}: ${msg}\n`);
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
try {
|
|
639
|
+
const extracted = runExtractor(lang, rootNode, file.path, createHash('sha256').update(source).digest('hex'), options.publicationGeneration);
|
|
640
|
+
pass.extracted.set(file.path, {
|
|
641
|
+
path: file.path,
|
|
642
|
+
extraction: completeExtraction(extracted),
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
catch (err) {
|
|
646
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
647
|
+
fail(file.path, `extract: ${msg}`);
|
|
648
|
+
process.stderr.write(`[nexus] SKIP extract error: ${file.path}: ${msg}\n`);
|
|
649
|
+
continue;
|
|
650
|
+
}
|
|
651
|
+
// Yield to event loop periodically on large repos
|
|
652
|
+
if (filesProcessed % 100 === 0)
|
|
653
|
+
await Promise.resolve();
|
|
636
654
|
}
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
655
|
+
if (!options.onProgress && total > 0) {
|
|
656
|
+
process.stderr.write(`[nexus] Parsing: ${total}/${total} files (100%)\n`);
|
|
657
|
+
}
|
|
658
|
+
return pass;
|
|
640
659
|
}
|
|
641
660
|
// ---------------------------------------------------------------------------
|
|
642
661
|
// Main parse loop entry point
|
|
643
662
|
// ---------------------------------------------------------------------------
|
|
644
663
|
/**
|
|
645
|
-
* Phase 3: Parse loop (parallel or sequential).
|
|
646
|
-
*
|
|
647
|
-
* For each TypeScript/JavaScript file in `files`:
|
|
648
|
-
* 1. Read file content from disk
|
|
649
|
-
* 2. Parse with tree-sitter (typescript or javascript grammar)
|
|
650
|
-
* 3. Extract definitions → register in SymbolTable → add nodes to graph
|
|
651
|
-
* 4. Extract imports → collect for batch resolution
|
|
652
|
-
* 5. Extract heritage → accumulate for deferred edge emission
|
|
664
|
+
* Phase 3: Parse loop (parallel or sequential), then merge.
|
|
653
665
|
*
|
|
654
|
-
*
|
|
655
|
-
*
|
|
656
|
-
*
|
|
666
|
+
* 1. Classify every scanned file's role from its bytes.
|
|
667
|
+
* 2. Extract each parseable file that has no reusable cached extraction —
|
|
668
|
+
* through the worker pool when the file count or byte volume warrants it.
|
|
669
|
+
* 3. Merge fresh and reused extractions **in scan order**: register symbols,
|
|
670
|
+
* add nodes, emit DEFINES edges, and collect imports, re-exports, heritage,
|
|
671
|
+
* calls and accesses. The merge order is the same whichever path produced
|
|
672
|
+
* an extraction, so first-wins deduplication in the graph cannot depend on
|
|
673
|
+
* which files happened to be re-parsed.
|
|
674
|
+
* 4. Resolve all imports and build the barrel map over the merged input.
|
|
657
675
|
*
|
|
658
|
-
* Files that fail to parse
|
|
659
|
-
*
|
|
676
|
+
* Files that fail to parse are reported, not merged; the publish step decides
|
|
677
|
+
* what a given share of failures means.
|
|
660
678
|
*
|
|
661
|
-
*
|
|
662
|
-
* bytes >= 512 KB, the parallel path via the worker pool is attempted first.
|
|
663
|
-
* If the worker script is not found (e.g. running from source without a
|
|
664
|
-
* build), the sequential path is used as a fallback.
|
|
665
|
-
*
|
|
666
|
-
* @param files - All scanned files (non-TypeScript/JavaScript files are filtered)
|
|
679
|
+
* @param files - All scanned files (non-parseable files are classified and reported only)
|
|
667
680
|
* @param graph - Knowledge graph to add nodes and relations to
|
|
668
681
|
* @param symbolTable - Symbol table to register extracted symbols in
|
|
669
682
|
* @param importCtx - Pre-built import resolution context (from Phase 3a)
|
|
670
683
|
* @param repoPath - Absolute path to the repository root (used to resolve relative paths)
|
|
671
|
-
* @param options -
|
|
672
|
-
* @returns Accumulated heritage and
|
|
684
|
+
* @param options - Resolution inputs, reuse map, progress and report callbacks
|
|
685
|
+
* @returns Accumulated heritage, call and access records plus the barrel map
|
|
673
686
|
*/
|
|
674
687
|
export async function runParseLoop(files, graph, symbolTable, importCtx, repoPath, options = {}) {
|
|
675
|
-
const { tsconfigPaths = null, namedImportMap = new Map()
|
|
688
|
+
const { tsconfigPaths = null, namedImportMap = new Map() } = options;
|
|
689
|
+
const reused = options.reusedExtractions ?? new Map();
|
|
676
690
|
// Attach one capability contract to sequential and worker-returned reports.
|
|
677
691
|
const callerReports = options.onFileReport;
|
|
678
692
|
const classifications = new Map();
|
|
@@ -692,7 +706,6 @@ export async function runParseLoop(files, graph, symbolTable, importCtx, repoPat
|
|
|
692
706
|
},
|
|
693
707
|
});
|
|
694
708
|
};
|
|
695
|
-
options = { ...options, onFileReport: emitReport };
|
|
696
709
|
const parseableFiles = [];
|
|
697
710
|
for (const file of files) {
|
|
698
711
|
options.parserLimits?.signal?.throwIfAborted();
|
|
@@ -725,173 +738,64 @@ export async function runParseLoop(files, graph, symbolTable, importCtx, repoPat
|
|
|
725
738
|
else
|
|
726
739
|
emitReport(report);
|
|
727
740
|
}
|
|
728
|
-
const
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
if (
|
|
741
|
+
const toParse = parseableFiles.filter((file) => !reused.has(file.path));
|
|
742
|
+
let pass = { extracted: new Map(), failures: new Map() };
|
|
743
|
+
if (toParse.length > 0) {
|
|
744
|
+
const totalBytes = toParse.reduce((acc, f) => acc + f.size, 0);
|
|
745
|
+
const useWorkers = Boolean(options.parserExecution) ||
|
|
746
|
+
toParse.length >= WORKER_FILE_THRESHOLD ||
|
|
747
|
+
totalBytes >= WORKER_BYTE_THRESHOLD;
|
|
748
|
+
let parallel = null;
|
|
749
|
+
if (useWorkers && (options.parserExecution || !callerReports)) {
|
|
750
|
+
process.stderr.write(`[nexus] Parallel parse: ${toParse.length} files, ${Math.round(totalBytes / 1024)}KB total — spawning worker pool\n`);
|
|
751
|
+
try {
|
|
752
|
+
parallel = await extractInParallel(toParse, repoPath, options);
|
|
753
|
+
}
|
|
754
|
+
catch (err) {
|
|
755
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
756
|
+
throw new Error(`Parallel parser failed; generation not published: ${msg}`);
|
|
757
|
+
}
|
|
758
|
+
if (parallel === null) {
|
|
759
|
+
if (options.parserExecution)
|
|
760
|
+
throw new Error('Isolated parser executable unavailable');
|
|
761
|
+
process.stderr.write('[nexus] Worker script not found — using sequential parse.\n');
|
|
762
|
+
}
|
|
763
|
+
else {
|
|
746
764
|
process.stderr.write('[nexus] Parallel parse complete.\n');
|
|
747
|
-
return parallelResult;
|
|
748
765
|
}
|
|
749
|
-
if (options.parserExecution)
|
|
750
|
-
throw new Error('Isolated parser executable unavailable');
|
|
751
|
-
// Worker unavailable — fall through to sequential
|
|
752
|
-
process.stderr.write('[nexus] Worker script not found — using sequential parse.\n');
|
|
753
|
-
}
|
|
754
|
-
catch (err) {
|
|
755
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
756
|
-
throw new Error(`Parallel parser failed; generation not published: ${msg}`);
|
|
757
766
|
}
|
|
767
|
+
pass = parallel ?? (await extractSequentially(toParse, repoPath, options));
|
|
758
768
|
}
|
|
769
|
+
// Merge in scan order — identical whether an extraction was fresh or reused.
|
|
759
770
|
const allExtractedImports = [];
|
|
760
771
|
const allReExports = [];
|
|
761
772
|
const allHeritage = [];
|
|
762
773
|
const allCalls = [];
|
|
763
774
|
const allAccesses = [];
|
|
764
|
-
const parser = getParser();
|
|
765
|
-
if (!parser) {
|
|
766
|
-
for (const file of parseableFiles) {
|
|
767
|
-
options.onFileReport?.({
|
|
768
|
-
path: file.path,
|
|
769
|
-
status: 'failed',
|
|
770
|
-
reason: 'tree-sitter native module unavailable',
|
|
771
|
-
});
|
|
772
|
-
}
|
|
773
|
-
process.stderr.write('[nexus] WARNING: tree-sitter native module not available — parse loop skipped.\n');
|
|
774
|
-
return {
|
|
775
|
-
allHeritage: [],
|
|
776
|
-
allCalls: [],
|
|
777
|
-
allAccesses: [],
|
|
778
|
-
barrelMap: buildBarrelExportMap([], importCtx, tsconfigPaths),
|
|
779
|
-
};
|
|
780
|
-
}
|
|
781
|
-
let filesProcessed = 0;
|
|
782
775
|
for (const file of parseableFiles) {
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
// Fallback stderr progress for CLI usage without a callback
|
|
791
|
-
const pct = Math.round((filesProcessed / total) * 100);
|
|
792
|
-
process.stderr.write(`[nexus] Parsing: ${filesProcessed}/${total} files (${pct}%)...\n`);
|
|
793
|
-
}
|
|
794
|
-
// Determine grammar key
|
|
795
|
-
const lang = detectLanguageFromPath(file.path);
|
|
796
|
-
if (!lang)
|
|
797
|
-
continue;
|
|
798
|
-
const grammarKey = grammarKeyForLanguage(lang);
|
|
799
|
-
if (!grammarKey)
|
|
800
|
-
continue;
|
|
801
|
-
// Load grammar
|
|
802
|
-
const grammar = loadGrammar(grammarKey);
|
|
803
|
-
if (!grammar) {
|
|
804
|
-
options.onFileReport?.({
|
|
805
|
-
path: file.path,
|
|
806
|
-
status: 'failed',
|
|
807
|
-
reason: `No grammar for ${lang}`,
|
|
808
|
-
});
|
|
809
|
-
process.stderr.write(`[nexus] SKIP: no grammar for ${lang} (file: ${file.path})\n`);
|
|
776
|
+
const fresh = pass.extracted.get(file.path);
|
|
777
|
+
const fileExtraction = fresh ?? reused.get(file.path);
|
|
778
|
+
if (!fileExtraction) {
|
|
779
|
+
const failure = pass.failures.get(file.path);
|
|
780
|
+
// Unknown language/grammar keys are silently skipped exactly as before.
|
|
781
|
+
if (failure)
|
|
782
|
+
emitReport(failure);
|
|
810
783
|
continue;
|
|
811
784
|
}
|
|
812
|
-
//
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
if (file.contentHash &&
|
|
818
|
-
createHash('sha256').update(bytes).digest('hex') !== file.contentHash) {
|
|
819
|
-
throw new Error('Source changed between scanning and parsing');
|
|
820
|
-
}
|
|
821
|
-
source = bytes.toString('utf-8');
|
|
822
|
-
}
|
|
823
|
-
catch (err) {
|
|
824
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
825
|
-
options.onFileReport?.({ path: file.path, status: 'failed', reason: `read: ${msg}` });
|
|
826
|
-
process.stderr.write(`[nexus] SKIP read error: ${file.path}: ${msg}\n`);
|
|
827
|
-
continue;
|
|
828
|
-
}
|
|
829
|
-
// Parse with tree-sitter
|
|
830
|
-
let rootNode;
|
|
831
|
-
try {
|
|
832
|
-
parser.setLanguage(grammar);
|
|
833
|
-
const tree = parseOriginalSource(parser, source, options.parserLimits);
|
|
834
|
-
rootNode = tree.rootNode;
|
|
835
|
-
}
|
|
836
|
-
catch (err) {
|
|
837
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
838
|
-
options.onFileReport?.({ path: file.path, status: 'failed', reason: `parse: ${msg}` });
|
|
839
|
-
options.parserLimits?.signal?.throwIfAborted();
|
|
840
|
-
process.stderr.write(`[nexus] SKIP parse error: ${file.path}: ${msg}\n`);
|
|
841
|
-
continue;
|
|
842
|
-
}
|
|
843
|
-
// Extract definitions, imports, heritage, calls, and re-exports — dispatch by language
|
|
844
|
-
let extracted;
|
|
845
|
-
try {
|
|
846
|
-
extracted = runExtractor(lang, rootNode, file.path, createHash('sha256').update(source).digest('hex'), options.publicationGeneration);
|
|
847
|
-
}
|
|
848
|
-
catch (err) {
|
|
849
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
850
|
-
options.onFileReport?.({ path: file.path, status: 'failed', reason: `extract: ${msg}` });
|
|
851
|
-
process.stderr.write(`[nexus] SKIP extract error: ${file.path}: ${msg}\n`);
|
|
852
|
-
continue;
|
|
853
|
-
}
|
|
854
|
-
options.onFileReport?.({ path: file.path, status: 'analyzed' });
|
|
855
|
-
// Register in SymbolTable
|
|
785
|
+
// Capture the cache entry before any later phase mutates these nodes.
|
|
786
|
+
if (fresh)
|
|
787
|
+
options.onFileExtracted?.(fresh);
|
|
788
|
+
emitReport({ path: file.path, status: 'analyzed' });
|
|
789
|
+
const extracted = fileExtraction.extraction;
|
|
856
790
|
registerInSymbolTable(extracted.definitions, symbolTable);
|
|
857
|
-
|
|
858
|
-
for (const node of extracted.definitions) {
|
|
791
|
+
for (const node of extracted.definitions)
|
|
859
792
|
graph.addNode(node);
|
|
860
|
-
}
|
|
861
|
-
// Emit defines edges: file → each symbol it declares (T1836)
|
|
862
793
|
emitDefinesEdges(file.path, extracted.definitions, graph);
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
if (extracted.reExports) {
|
|
869
|
-
for (const re of extracted.reExports) {
|
|
870
|
-
allReExports.push(re);
|
|
871
|
-
}
|
|
872
|
-
}
|
|
873
|
-
// Collect heritage for deferred Phase 3c processing
|
|
874
|
-
for (const h of extracted.heritage) {
|
|
875
|
-
allHeritage.push(h);
|
|
876
|
-
}
|
|
877
|
-
// Collect calls for deferred Phase 3e resolution
|
|
878
|
-
for (const c of extracted.calls) {
|
|
879
|
-
allCalls.push(c);
|
|
880
|
-
}
|
|
881
|
-
// Collect access sites for deferred Phase 3f resolution (T1837)
|
|
882
|
-
if (extracted.accesses) {
|
|
883
|
-
for (const a of extracted.accesses) {
|
|
884
|
-
allAccesses.push(a);
|
|
885
|
-
}
|
|
886
|
-
}
|
|
887
|
-
// Yield to event loop periodically on large repos
|
|
888
|
-
if (filesProcessed % 100 === 0) {
|
|
889
|
-
await Promise.resolve();
|
|
890
|
-
}
|
|
891
|
-
}
|
|
892
|
-
// Final stderr progress update
|
|
893
|
-
if (!onProgress && total > 0) {
|
|
894
|
-
process.stderr.write(`[nexus] Parsing: ${total}/${total} files (100%)\n`);
|
|
794
|
+
allExtractedImports.push(...extracted.imports);
|
|
795
|
+
allReExports.push(...extracted.reExports);
|
|
796
|
+
allHeritage.push(...extracted.heritage);
|
|
797
|
+
allCalls.push(...extracted.calls);
|
|
798
|
+
allAccesses.push(...extracted.accesses);
|
|
895
799
|
}
|
|
896
800
|
// Batch-resolve all extracted imports (populates namedImportMap for Phase 3e)
|
|
897
801
|
if (allExtractedImports.length > 0) {
|
|
@@ -903,11 +807,17 @@ export async function runParseLoop(files, graph, symbolTable, importCtx, repoPat
|
|
|
903
807
|
tsconfigPaths,
|
|
904
808
|
});
|
|
905
809
|
}
|
|
906
|
-
// Build barrel export map from collected re-export records (T617)
|
|
810
|
+
// Build barrel export map from collected re-export records (T617).
|
|
907
811
|
// Runs AFTER processExtractedImports so the resolve cache is warmed up.
|
|
908
812
|
const barrelMap = buildBarrelExportMap(allReExports, importCtx, tsconfigPaths);
|
|
909
813
|
process.stderr.write(`[nexus] Barrel map: ${barrelMap.size} barrel files with re-export chains\n`);
|
|
910
|
-
|
|
911
|
-
|
|
814
|
+
return {
|
|
815
|
+
allHeritage,
|
|
816
|
+
allCalls,
|
|
817
|
+
allAccesses,
|
|
818
|
+
barrelMap,
|
|
819
|
+
parsedFileCount: toParse.length,
|
|
820
|
+
reusedFileCount: parseableFiles.length - toParse.length,
|
|
821
|
+
};
|
|
912
822
|
}
|
|
913
823
|
//# sourceMappingURL=parse-loop.js.map
|