@cleocode/nexus 2026.9.16 → 2026.9.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/pipeline/community-processor.d.ts.map +1 -1
- package/dist/src/pipeline/community-processor.js +21 -0
- package/dist/src/pipeline/community-processor.js.map +1 -1
- package/dist/src/pipeline/extractors/typescript-extractor.d.ts.map +1 -1
- package/dist/src/pipeline/extractors/typescript-extractor.js +7 -1
- package/dist/src/pipeline/extractors/typescript-extractor.js.map +1 -1
- package/dist/src/pipeline/filesystem-walker.d.ts +38 -1
- package/dist/src/pipeline/filesystem-walker.d.ts.map +1 -1
- package/dist/src/pipeline/filesystem-walker.js +194 -46
- package/dist/src/pipeline/filesystem-walker.js.map +1 -1
- package/dist/src/pipeline/index.d.ts +52 -15
- package/dist/src/pipeline/index.d.ts.map +1 -1
- package/dist/src/pipeline/index.js +242 -103
- package/dist/src/pipeline/index.js.map +1 -1
- package/dist/src/pipeline/parse-cache.d.ts +72 -0
- package/dist/src/pipeline/parse-cache.d.ts.map +1 -0
- package/dist/src/pipeline/parse-cache.js +158 -0
- package/dist/src/pipeline/parse-cache.js.map +1 -0
- package/dist/src/pipeline/parse-loop.d.ts +31 -21
- package/dist/src/pipeline/parse-loop.d.ts.map +1 -1
- package/dist/src/pipeline/parse-loop.js +186 -285
- package/dist/src/pipeline/parse-loop.js.map +1 -1
- package/dist/src/pipeline/process-processor.d.ts.map +1 -1
- package/dist/src/pipeline/process-processor.js +4 -0
- package/dist/src/pipeline/process-processor.js.map +1 -1
- package/dist/src/pipeline/processors/access-processor.d.ts.map +1 -1
- package/dist/src/pipeline/processors/access-processor.js +14 -2
- package/dist/src/pipeline/processors/access-processor.js.map +1 -1
- package/dist/src/pipeline/workers/parse-worker.d.ts +10 -15
- package/dist/src/pipeline/workers/parse-worker.d.ts.map +1 -1
- package/dist/src/pipeline/workers/parse-worker.js +12 -12
- package/dist/src/pipeline/workers/parse-worker.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
|
@@ -488,44 +488,37 @@ async function classifyFileCapabilities(file, repoPath, signal) {
|
|
|
488
488
|
reason: 'No positive role evidence; potentially executable content remains unassessed',
|
|
489
489
|
}));
|
|
490
490
|
}
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
491
|
+
/** Normalize an extractor result so every per-file array is present. */
|
|
492
|
+
function completeExtraction(result) {
|
|
493
|
+
return {
|
|
494
|
+
definitions: result.definitions,
|
|
495
|
+
imports: result.imports,
|
|
496
|
+
heritage: result.heritage,
|
|
497
|
+
calls: result.calls,
|
|
498
|
+
reExports: result.reExports ?? [],
|
|
499
|
+
accesses: result.accesses ?? [],
|
|
500
|
+
};
|
|
501
|
+
}
|
|
497
502
|
/**
|
|
498
|
-
*
|
|
499
|
-
*
|
|
500
|
-
* Called by `runParseLoop` when the file count or total bytes exceeds the
|
|
501
|
-
* worker pool thresholds. Falls back to returning `null` if the worker
|
|
502
|
-
* script is not found so the caller can retry sequentially.
|
|
503
|
+
* Extract files through the worker pool (Wave H — T540).
|
|
503
504
|
*
|
|
504
|
-
* @returns
|
|
505
|
+
* @returns Per-file outcomes, or `null` when the worker script is not built.
|
|
505
506
|
*/
|
|
506
|
-
async function
|
|
507
|
-
const { tsconfigPaths = null, namedImportMap = new Map(), onProgress } = options;
|
|
507
|
+
async function extractInParallel(files, repoPath, options) {
|
|
508
508
|
// Resolve the compiled worker script path (parallel to this file in dist/)
|
|
509
509
|
let workerUrl;
|
|
510
510
|
try {
|
|
511
|
-
// In ESM, import.meta.url resolves relative to the current module file.
|
|
512
|
-
// The worker script lives at the same directory level.
|
|
513
511
|
workerUrl = new URL('./workers/parse-worker.js', import.meta.url);
|
|
514
|
-
const workerPath = fileURLToPath(workerUrl);
|
|
515
|
-
// Use dynamic require check — existsSync from 'node:fs'
|
|
516
512
|
const { existsSync } = await import('node:fs');
|
|
517
|
-
if (!existsSync(
|
|
518
|
-
return null; //
|
|
519
|
-
}
|
|
513
|
+
if (!existsSync(fileURLToPath(workerUrl)))
|
|
514
|
+
return null; // not built — sequential fallback
|
|
520
515
|
}
|
|
521
516
|
catch {
|
|
522
517
|
return null;
|
|
523
518
|
}
|
|
524
|
-
|
|
525
|
-
const total = parseableFiles.length;
|
|
519
|
+
const total = files.length;
|
|
526
520
|
const workerInputs = [];
|
|
527
|
-
for (
|
|
528
|
-
const file = parseableFiles[i];
|
|
521
|
+
for (const file of files) {
|
|
529
522
|
try {
|
|
530
523
|
const absPath = file.path.startsWith('/') ? file.path : `${repoPath}/${file.path}`;
|
|
531
524
|
options.parserLimits?.signal?.throwIfAborted();
|
|
@@ -548,140 +541,152 @@ async function runParallelParseLoop(parseableFiles, graph, symbolTable, importCt
|
|
|
548
541
|
}
|
|
549
542
|
}
|
|
550
543
|
const pool = createWorkerPool(workerUrl, undefined, options.parserLimits, options.parserExecution);
|
|
551
|
-
let filesProcessedSoFar = 0;
|
|
552
544
|
let workerResults;
|
|
553
545
|
try {
|
|
554
546
|
workerResults = await pool.dispatch(workerInputs, (filesProcessed) => {
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
onProgress(filesProcessed, total, lastFile?.path ?? '');
|
|
547
|
+
if (options.onProgress) {
|
|
548
|
+
const lastFile = files[Math.min(filesProcessed, total) - 1];
|
|
549
|
+
options.onProgress(filesProcessed, total, lastFile?.path ?? '');
|
|
559
550
|
}
|
|
560
551
|
});
|
|
561
552
|
}
|
|
562
553
|
finally {
|
|
563
554
|
await pool.terminate().catch(() => undefined);
|
|
564
555
|
}
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
const
|
|
569
|
-
const allHeritage = [];
|
|
570
|
-
const allCalls = [];
|
|
571
|
-
// Workers invoke the same extractor, including access records.
|
|
572
|
-
const allParallelAccesses = [];
|
|
573
|
-
/** Files no worker could parse; the publish step decides what they mean. */
|
|
574
|
-
const unparsedFiles = [];
|
|
556
|
+
if (!options.onProgress && total > 0) {
|
|
557
|
+
process.stderr.write(`[nexus] Parsing: ${total}/${total} files (100%) [parallel]\n`);
|
|
558
|
+
}
|
|
559
|
+
const pass = { extracted: new Map(), failures: new Map() };
|
|
575
560
|
for (const workerResult of workerResults) {
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
// and the publish step decides — it tolerates a small number of recorded,
|
|
582
|
-
// reported gaps and still refuses a generation that would misrepresent the
|
|
583
|
-
// codebase. Deciding that here, per worker, could only ever see one chunk.
|
|
584
|
-
const failed = workerResult.reports.filter((report) => report.status !== 'analyzed');
|
|
585
|
-
if (failed.length > 0) {
|
|
586
|
-
unparsedFiles.push(...failed);
|
|
587
|
-
}
|
|
588
|
-
allParallelAccesses.push(...workerResult.accesses);
|
|
589
|
-
// Register symbols into SymbolTable and add graph nodes
|
|
590
|
-
const fileGraphNodes = new Map();
|
|
591
|
-
for (const sym of workerResult.symbols) {
|
|
592
|
-
const graphNode = sym;
|
|
593
|
-
registerInSymbolTable([graphNode], symbolTable);
|
|
594
|
-
graph.addNode(graphNode);
|
|
595
|
-
// Group by filePath for defines edge emission below
|
|
596
|
-
if (graphNode.filePath) {
|
|
597
|
-
const bucket = fileGraphNodes.get(graphNode.filePath) ?? [];
|
|
598
|
-
bucket.push(graphNode);
|
|
599
|
-
fileGraphNodes.set(graphNode.filePath, bucket);
|
|
600
|
-
}
|
|
561
|
+
// T12313: failures travel onward as reports; the publish step weighs how
|
|
562
|
+
// much of the repository they affect instead of one chunk deciding alone.
|
|
563
|
+
for (const report of workerResult.reports) {
|
|
564
|
+
if (report.status !== 'analyzed')
|
|
565
|
+
pass.failures.set(report.path, report);
|
|
601
566
|
}
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
emitDefinesEdges(filePath, nodes, graph);
|
|
605
|
-
}
|
|
606
|
-
allExtractedImports.push(...workerResult.imports);
|
|
607
|
-
allParallelReExports.push(...workerResult.reExports);
|
|
608
|
-
allHeritage.push(...workerResult.heritage);
|
|
609
|
-
allCalls.push(...workerResult.calls);
|
|
610
|
-
}
|
|
611
|
-
if (onProgress && total > 0) {
|
|
612
|
-
const lastFile = parseableFiles[total - 1];
|
|
613
|
-
onProgress(total, total, lastFile?.path ?? '');
|
|
567
|
+
for (const file of workerResult.files)
|
|
568
|
+
pass.extracted.set(file.path, file);
|
|
614
569
|
}
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
570
|
+
return pass;
|
|
571
|
+
}
|
|
572
|
+
/** Extract files one at a time in the calling thread. */
|
|
573
|
+
async function extractSequentially(files, repoPath, options) {
|
|
574
|
+
const pass = { extracted: new Map(), failures: new Map() };
|
|
575
|
+
const fail = (path, reason) => {
|
|
576
|
+
pass.failures.set(path, { path, status: 'failed', reason });
|
|
577
|
+
};
|
|
578
|
+
const parser = getParser();
|
|
579
|
+
if (!parser) {
|
|
580
|
+
for (const file of files)
|
|
581
|
+
fail(file.path, 'tree-sitter native module unavailable');
|
|
582
|
+
process.stderr.write('[nexus] WARNING: tree-sitter native module not available — parse loop skipped.\n');
|
|
583
|
+
return pass;
|
|
627
584
|
}
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
const
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
585
|
+
const total = files.length;
|
|
586
|
+
let filesProcessed = 0;
|
|
587
|
+
for (const file of files) {
|
|
588
|
+
options.parserLimits?.signal?.throwIfAborted();
|
|
589
|
+
filesProcessed++;
|
|
590
|
+
if (options.onProgress &&
|
|
591
|
+
(total <= 100 || filesProcessed % 10 === 0 || filesProcessed === total)) {
|
|
592
|
+
options.onProgress(filesProcessed, total, file.path);
|
|
593
|
+
}
|
|
594
|
+
else if (!options.onProgress && filesProcessed % 50 === 0) {
|
|
595
|
+
const pct = Math.round((filesProcessed / total) * 100);
|
|
596
|
+
process.stderr.write(`[nexus] Parsing: ${filesProcessed}/${total} files (${pct}%)...\n`);
|
|
597
|
+
}
|
|
598
|
+
const lang = detectLanguageFromPath(file.path);
|
|
599
|
+
if (!lang)
|
|
600
|
+
continue;
|
|
601
|
+
const grammarKey = grammarKeyForLanguage(lang);
|
|
602
|
+
if (!grammarKey)
|
|
603
|
+
continue;
|
|
604
|
+
const grammar = loadGrammar(grammarKey);
|
|
605
|
+
if (!grammar) {
|
|
606
|
+
fail(file.path, `No grammar for ${lang}`);
|
|
607
|
+
process.stderr.write(`[nexus] SKIP: no grammar for ${lang} (file: ${file.path})\n`);
|
|
608
|
+
continue;
|
|
609
|
+
}
|
|
610
|
+
let source;
|
|
611
|
+
try {
|
|
612
|
+
const absPath = file.path.startsWith('/') ? file.path : `${repoPath}/${file.path}`;
|
|
613
|
+
const bytes = await fs.readFile(absPath);
|
|
614
|
+
if (file.contentHash &&
|
|
615
|
+
createHash('sha256').update(bytes).digest('hex') !== file.contentHash) {
|
|
616
|
+
throw new Error('Source changed between scanning and parsing');
|
|
643
617
|
}
|
|
618
|
+
source = bytes.toString('utf-8');
|
|
644
619
|
}
|
|
620
|
+
catch (err) {
|
|
621
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
622
|
+
fail(file.path, `read: ${msg}`);
|
|
623
|
+
process.stderr.write(`[nexus] SKIP read error: ${file.path}: ${msg}\n`);
|
|
624
|
+
continue;
|
|
625
|
+
}
|
|
626
|
+
let rootNode;
|
|
627
|
+
try {
|
|
628
|
+
parser.setLanguage(grammar);
|
|
629
|
+
rootNode = parseOriginalSource(parser, source, options.parserLimits).rootNode;
|
|
630
|
+
}
|
|
631
|
+
catch (err) {
|
|
632
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
633
|
+
fail(file.path, `parse: ${msg}`);
|
|
634
|
+
options.parserLimits?.signal?.throwIfAborted();
|
|
635
|
+
process.stderr.write(`[nexus] SKIP parse error: ${file.path}: ${msg}\n`);
|
|
636
|
+
continue;
|
|
637
|
+
}
|
|
638
|
+
try {
|
|
639
|
+
const extracted = runExtractor(lang, rootNode, file.path, createHash('sha256').update(source).digest('hex'), options.publicationGeneration);
|
|
640
|
+
pass.extracted.set(file.path, {
|
|
641
|
+
path: file.path,
|
|
642
|
+
extraction: completeExtraction(extracted),
|
|
643
|
+
});
|
|
644
|
+
}
|
|
645
|
+
catch (err) {
|
|
646
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
647
|
+
fail(file.path, `extract: ${msg}`);
|
|
648
|
+
process.stderr.write(`[nexus] SKIP extract error: ${file.path}: ${msg}\n`);
|
|
649
|
+
continue;
|
|
650
|
+
}
|
|
651
|
+
// Yield to event loop periodically on large repos
|
|
652
|
+
if (filesProcessed % 100 === 0)
|
|
653
|
+
await Promise.resolve();
|
|
654
|
+
}
|
|
655
|
+
if (!options.onProgress && total > 0) {
|
|
656
|
+
process.stderr.write(`[nexus] Parsing: ${total}/${total} files (100%)\n`);
|
|
645
657
|
}
|
|
646
|
-
|
|
647
|
-
process.stderr.write(`[nexus] Barrel map: ${parallelBarrelMap.size} barrel files with re-export chains\n`);
|
|
648
|
-
return { allHeritage, allCalls, allAccesses: allParallelAccesses, barrelMap: parallelBarrelMap };
|
|
658
|
+
return pass;
|
|
649
659
|
}
|
|
650
660
|
// ---------------------------------------------------------------------------
|
|
651
661
|
// Main parse loop entry point
|
|
652
662
|
// ---------------------------------------------------------------------------
|
|
653
663
|
/**
|
|
654
|
-
* Phase 3: Parse loop (parallel or sequential).
|
|
664
|
+
* Phase 3: Parse loop (parallel or sequential), then merge.
|
|
655
665
|
*
|
|
656
|
-
*
|
|
657
|
-
*
|
|
658
|
-
*
|
|
659
|
-
* 3.
|
|
660
|
-
*
|
|
661
|
-
*
|
|
666
|
+
* 1. Classify every scanned file's role from its bytes.
|
|
667
|
+
* 2. Extract each parseable file that has no reusable cached extraction —
|
|
668
|
+
* through the worker pool when the file count or byte volume warrants it.
|
|
669
|
+
* 3. Merge fresh and reused extractions **in scan order**: register symbols,
|
|
670
|
+
* add nodes, emit DEFINES edges, and collect imports, re-exports, heritage,
|
|
671
|
+
* calls and accesses. The merge order is the same whichever path produced
|
|
672
|
+
* an extraction, so first-wins deduplication in the graph cannot depend on
|
|
673
|
+
* which files happened to be re-parsed.
|
|
674
|
+
* 4. Resolve all imports and build the barrel map over the merged input.
|
|
662
675
|
*
|
|
663
|
-
*
|
|
664
|
-
*
|
|
665
|
-
* - Emits EXTENDS/IMPLEMENTS edges from accumulated heritage
|
|
676
|
+
* Files that fail to parse are reported, not merged; the publish step decides
|
|
677
|
+
* what a given share of failures means.
|
|
666
678
|
*
|
|
667
|
-
*
|
|
668
|
-
* are skipped with a warning to stderr — the loop continues.
|
|
669
|
-
*
|
|
670
|
-
* **Parallel mode** (Wave H): When `parseableFiles.length >= 15` or total
|
|
671
|
-
* bytes >= 512 KB, the parallel path via the worker pool is attempted first.
|
|
672
|
-
* If the worker script is not found (e.g. running from source without a
|
|
673
|
-
* build), the sequential path is used as a fallback.
|
|
674
|
-
*
|
|
675
|
-
* @param files - All scanned files (non-TypeScript/JavaScript files are filtered)
|
|
679
|
+
* @param files - All scanned files (non-parseable files are classified and reported only)
|
|
676
680
|
* @param graph - Knowledge graph to add nodes and relations to
|
|
677
681
|
* @param symbolTable - Symbol table to register extracted symbols in
|
|
678
682
|
* @param importCtx - Pre-built import resolution context (from Phase 3a)
|
|
679
683
|
* @param repoPath - Absolute path to the repository root (used to resolve relative paths)
|
|
680
|
-
* @param options -
|
|
681
|
-
* @returns Accumulated heritage and
|
|
684
|
+
* @param options - Resolution inputs, reuse map, progress and report callbacks
|
|
685
|
+
* @returns Accumulated heritage, call and access records plus the barrel map
|
|
682
686
|
*/
|
|
683
687
|
export async function runParseLoop(files, graph, symbolTable, importCtx, repoPath, options = {}) {
|
|
684
|
-
const { tsconfigPaths = null, namedImportMap = new Map()
|
|
688
|
+
const { tsconfigPaths = null, namedImportMap = new Map() } = options;
|
|
689
|
+
const reused = options.reusedExtractions ?? new Map();
|
|
685
690
|
// Attach one capability contract to sequential and worker-returned reports.
|
|
686
691
|
const callerReports = options.onFileReport;
|
|
687
692
|
const classifications = new Map();
|
|
@@ -701,7 +706,6 @@ export async function runParseLoop(files, graph, symbolTable, importCtx, repoPat
|
|
|
701
706
|
},
|
|
702
707
|
});
|
|
703
708
|
};
|
|
704
|
-
options = { ...options, onFileReport: emitReport };
|
|
705
709
|
const parseableFiles = [];
|
|
706
710
|
for (const file of files) {
|
|
707
711
|
options.parserLimits?.signal?.throwIfAborted();
|
|
@@ -734,173 +738,64 @@ export async function runParseLoop(files, graph, symbolTable, importCtx, repoPat
|
|
|
734
738
|
else
|
|
735
739
|
emitReport(report);
|
|
736
740
|
}
|
|
737
|
-
const
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
if (
|
|
741
|
+
const toParse = parseableFiles.filter((file) => !reused.has(file.path));
|
|
742
|
+
let pass = { extracted: new Map(), failures: new Map() };
|
|
743
|
+
if (toParse.length > 0) {
|
|
744
|
+
const totalBytes = toParse.reduce((acc, f) => acc + f.size, 0);
|
|
745
|
+
const useWorkers = Boolean(options.parserExecution) ||
|
|
746
|
+
toParse.length >= WORKER_FILE_THRESHOLD ||
|
|
747
|
+
totalBytes >= WORKER_BYTE_THRESHOLD;
|
|
748
|
+
let parallel = null;
|
|
749
|
+
if (useWorkers && (options.parserExecution || !callerReports)) {
|
|
750
|
+
process.stderr.write(`[nexus] Parallel parse: ${toParse.length} files, ${Math.round(totalBytes / 1024)}KB total — spawning worker pool\n`);
|
|
751
|
+
try {
|
|
752
|
+
parallel = await extractInParallel(toParse, repoPath, options);
|
|
753
|
+
}
|
|
754
|
+
catch (err) {
|
|
755
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
756
|
+
throw new Error(`Parallel parser failed; generation not published: ${msg}`);
|
|
757
|
+
}
|
|
758
|
+
if (parallel === null) {
|
|
759
|
+
if (options.parserExecution)
|
|
760
|
+
throw new Error('Isolated parser executable unavailable');
|
|
761
|
+
process.stderr.write('[nexus] Worker script not found — using sequential parse.\n');
|
|
762
|
+
}
|
|
763
|
+
else {
|
|
755
764
|
process.stderr.write('[nexus] Parallel parse complete.\n');
|
|
756
|
-
return parallelResult;
|
|
757
765
|
}
|
|
758
|
-
if (options.parserExecution)
|
|
759
|
-
throw new Error('Isolated parser executable unavailable');
|
|
760
|
-
// Worker unavailable — fall through to sequential
|
|
761
|
-
process.stderr.write('[nexus] Worker script not found — using sequential parse.\n');
|
|
762
|
-
}
|
|
763
|
-
catch (err) {
|
|
764
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
765
|
-
throw new Error(`Parallel parser failed; generation not published: ${msg}`);
|
|
766
766
|
}
|
|
767
|
+
pass = parallel ?? (await extractSequentially(toParse, repoPath, options));
|
|
767
768
|
}
|
|
769
|
+
// Merge in scan order — identical whether an extraction was fresh or reused.
|
|
768
770
|
const allExtractedImports = [];
|
|
769
771
|
const allReExports = [];
|
|
770
772
|
const allHeritage = [];
|
|
771
773
|
const allCalls = [];
|
|
772
774
|
const allAccesses = [];
|
|
773
|
-
const parser = getParser();
|
|
774
|
-
if (!parser) {
|
|
775
|
-
for (const file of parseableFiles) {
|
|
776
|
-
options.onFileReport?.({
|
|
777
|
-
path: file.path,
|
|
778
|
-
status: 'failed',
|
|
779
|
-
reason: 'tree-sitter native module unavailable',
|
|
780
|
-
});
|
|
781
|
-
}
|
|
782
|
-
process.stderr.write('[nexus] WARNING: tree-sitter native module not available — parse loop skipped.\n');
|
|
783
|
-
return {
|
|
784
|
-
allHeritage: [],
|
|
785
|
-
allCalls: [],
|
|
786
|
-
allAccesses: [],
|
|
787
|
-
barrelMap: buildBarrelExportMap([], importCtx, tsconfigPaths),
|
|
788
|
-
};
|
|
789
|
-
}
|
|
790
|
-
let filesProcessed = 0;
|
|
791
775
|
for (const file of parseableFiles) {
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
// Fallback stderr progress for CLI usage without a callback
|
|
800
|
-
const pct = Math.round((filesProcessed / total) * 100);
|
|
801
|
-
process.stderr.write(`[nexus] Parsing: ${filesProcessed}/${total} files (${pct}%)...\n`);
|
|
802
|
-
}
|
|
803
|
-
// Determine grammar key
|
|
804
|
-
const lang = detectLanguageFromPath(file.path);
|
|
805
|
-
if (!lang)
|
|
806
|
-
continue;
|
|
807
|
-
const grammarKey = grammarKeyForLanguage(lang);
|
|
808
|
-
if (!grammarKey)
|
|
809
|
-
continue;
|
|
810
|
-
// Load grammar
|
|
811
|
-
const grammar = loadGrammar(grammarKey);
|
|
812
|
-
if (!grammar) {
|
|
813
|
-
options.onFileReport?.({
|
|
814
|
-
path: file.path,
|
|
815
|
-
status: 'failed',
|
|
816
|
-
reason: `No grammar for ${lang}`,
|
|
817
|
-
});
|
|
818
|
-
process.stderr.write(`[nexus] SKIP: no grammar for ${lang} (file: ${file.path})\n`);
|
|
819
|
-
continue;
|
|
820
|
-
}
|
|
821
|
-
// Read file content — relative path from filesystem walker, read from repoPath
|
|
822
|
-
let source;
|
|
823
|
-
try {
|
|
824
|
-
const absPath = file.path.startsWith('/') ? file.path : `${repoPath}/${file.path}`;
|
|
825
|
-
const bytes = await fs.readFile(absPath);
|
|
826
|
-
if (file.contentHash &&
|
|
827
|
-
createHash('sha256').update(bytes).digest('hex') !== file.contentHash) {
|
|
828
|
-
throw new Error('Source changed between scanning and parsing');
|
|
829
|
-
}
|
|
830
|
-
source = bytes.toString('utf-8');
|
|
831
|
-
}
|
|
832
|
-
catch (err) {
|
|
833
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
834
|
-
options.onFileReport?.({ path: file.path, status: 'failed', reason: `read: ${msg}` });
|
|
835
|
-
process.stderr.write(`[nexus] SKIP read error: ${file.path}: ${msg}\n`);
|
|
776
|
+
const fresh = pass.extracted.get(file.path);
|
|
777
|
+
const fileExtraction = fresh ?? reused.get(file.path);
|
|
778
|
+
if (!fileExtraction) {
|
|
779
|
+
const failure = pass.failures.get(file.path);
|
|
780
|
+
// Unknown language/grammar keys are silently skipped exactly as before.
|
|
781
|
+
if (failure)
|
|
782
|
+
emitReport(failure);
|
|
836
783
|
continue;
|
|
837
784
|
}
|
|
838
|
-
//
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
rootNode = tree.rootNode;
|
|
844
|
-
}
|
|
845
|
-
catch (err) {
|
|
846
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
847
|
-
options.onFileReport?.({ path: file.path, status: 'failed', reason: `parse: ${msg}` });
|
|
848
|
-
options.parserLimits?.signal?.throwIfAborted();
|
|
849
|
-
process.stderr.write(`[nexus] SKIP parse error: ${file.path}: ${msg}\n`);
|
|
850
|
-
continue;
|
|
851
|
-
}
|
|
852
|
-
// Extract definitions, imports, heritage, calls, and re-exports — dispatch by language
|
|
853
|
-
let extracted;
|
|
854
|
-
try {
|
|
855
|
-
extracted = runExtractor(lang, rootNode, file.path, createHash('sha256').update(source).digest('hex'), options.publicationGeneration);
|
|
856
|
-
}
|
|
857
|
-
catch (err) {
|
|
858
|
-
const msg = err instanceof Error ? err.message : String(err);
|
|
859
|
-
options.onFileReport?.({ path: file.path, status: 'failed', reason: `extract: ${msg}` });
|
|
860
|
-
process.stderr.write(`[nexus] SKIP extract error: ${file.path}: ${msg}\n`);
|
|
861
|
-
continue;
|
|
862
|
-
}
|
|
863
|
-
options.onFileReport?.({ path: file.path, status: 'analyzed' });
|
|
864
|
-
// Register in SymbolTable
|
|
785
|
+
// Capture the cache entry before any later phase mutates these nodes.
|
|
786
|
+
if (fresh)
|
|
787
|
+
options.onFileExtracted?.(fresh);
|
|
788
|
+
emitReport({ path: file.path, status: 'analyzed' });
|
|
789
|
+
const extracted = fileExtraction.extraction;
|
|
865
790
|
registerInSymbolTable(extracted.definitions, symbolTable);
|
|
866
|
-
|
|
867
|
-
for (const node of extracted.definitions) {
|
|
791
|
+
for (const node of extracted.definitions)
|
|
868
792
|
graph.addNode(node);
|
|
869
|
-
}
|
|
870
|
-
// Emit defines edges: file → each symbol it declares (T1836)
|
|
871
793
|
emitDefinesEdges(file.path, extracted.definitions, graph);
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
if (extracted.reExports) {
|
|
878
|
-
for (const re of extracted.reExports) {
|
|
879
|
-
allReExports.push(re);
|
|
880
|
-
}
|
|
881
|
-
}
|
|
882
|
-
// Collect heritage for deferred Phase 3c processing
|
|
883
|
-
for (const h of extracted.heritage) {
|
|
884
|
-
allHeritage.push(h);
|
|
885
|
-
}
|
|
886
|
-
// Collect calls for deferred Phase 3e resolution
|
|
887
|
-
for (const c of extracted.calls) {
|
|
888
|
-
allCalls.push(c);
|
|
889
|
-
}
|
|
890
|
-
// Collect access sites for deferred Phase 3f resolution (T1837)
|
|
891
|
-
if (extracted.accesses) {
|
|
892
|
-
for (const a of extracted.accesses) {
|
|
893
|
-
allAccesses.push(a);
|
|
894
|
-
}
|
|
895
|
-
}
|
|
896
|
-
// Yield to event loop periodically on large repos
|
|
897
|
-
if (filesProcessed % 100 === 0) {
|
|
898
|
-
await Promise.resolve();
|
|
899
|
-
}
|
|
900
|
-
}
|
|
901
|
-
// Final stderr progress update
|
|
902
|
-
if (!onProgress && total > 0) {
|
|
903
|
-
process.stderr.write(`[nexus] Parsing: ${total}/${total} files (100%)\n`);
|
|
794
|
+
allExtractedImports.push(...extracted.imports);
|
|
795
|
+
allReExports.push(...extracted.reExports);
|
|
796
|
+
allHeritage.push(...extracted.heritage);
|
|
797
|
+
allCalls.push(...extracted.calls);
|
|
798
|
+
allAccesses.push(...extracted.accesses);
|
|
904
799
|
}
|
|
905
800
|
// Batch-resolve all extracted imports (populates namedImportMap for Phase 3e)
|
|
906
801
|
if (allExtractedImports.length > 0) {
|
|
@@ -912,11 +807,17 @@ export async function runParseLoop(files, graph, symbolTable, importCtx, repoPat
|
|
|
912
807
|
tsconfigPaths,
|
|
913
808
|
});
|
|
914
809
|
}
|
|
915
|
-
// Build barrel export map from collected re-export records (T617)
|
|
810
|
+
// Build barrel export map from collected re-export records (T617).
|
|
916
811
|
// Runs AFTER processExtractedImports so the resolve cache is warmed up.
|
|
917
812
|
const barrelMap = buildBarrelExportMap(allReExports, importCtx, tsconfigPaths);
|
|
918
813
|
process.stderr.write(`[nexus] Barrel map: ${barrelMap.size} barrel files with re-export chains\n`);
|
|
919
|
-
|
|
920
|
-
|
|
814
|
+
return {
|
|
815
|
+
allHeritage,
|
|
816
|
+
allCalls,
|
|
817
|
+
allAccesses,
|
|
818
|
+
barrelMap,
|
|
819
|
+
parsedFileCount: toParse.length,
|
|
820
|
+
reusedFileCount: parseableFiles.length - toParse.length,
|
|
821
|
+
};
|
|
921
822
|
}
|
|
922
823
|
//# sourceMappingURL=parse-loop.js.map
|