@cleocode/nexus 2026.9.15 → 2026.9.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/pipeline/community-processor.d.ts.map +1 -1
- package/dist/src/pipeline/community-processor.js +21 -0
- package/dist/src/pipeline/community-processor.js.map +1 -1
- package/dist/src/pipeline/extractors/typescript-extractor.d.ts.map +1 -1
- package/dist/src/pipeline/extractors/typescript-extractor.js +7 -1
- package/dist/src/pipeline/extractors/typescript-extractor.js.map +1 -1
- package/dist/src/pipeline/filesystem-walker.d.ts +25 -1
- package/dist/src/pipeline/filesystem-walker.d.ts.map +1 -1
- package/dist/src/pipeline/filesystem-walker.js +16 -4
- package/dist/src/pipeline/filesystem-walker.js.map +1 -1
- package/dist/src/pipeline/index.d.ts +52 -7
- package/dist/src/pipeline/index.d.ts.map +1 -1
- package/dist/src/pipeline/index.js +266 -96
- package/dist/src/pipeline/index.js.map +1 -1
- package/dist/src/pipeline/parse-cache.d.ts +72 -0
- package/dist/src/pipeline/parse-cache.d.ts.map +1 -0
- package/dist/src/pipeline/parse-cache.js +158 -0
- package/dist/src/pipeline/parse-cache.js.map +1 -0
- package/dist/src/pipeline/parse-loop.d.ts +31 -21
- package/dist/src/pipeline/parse-loop.d.ts.map +1 -1
- package/dist/src/pipeline/parse-loop.js +186 -276
- package/dist/src/pipeline/parse-loop.js.map +1 -1
- package/dist/src/pipeline/process-processor.d.ts.map +1 -1
- package/dist/src/pipeline/process-processor.js +4 -0
- package/dist/src/pipeline/process-processor.js.map +1 -1
- package/dist/src/pipeline/processors/access-processor.d.ts.map +1 -1
- package/dist/src/pipeline/processors/access-processor.js +14 -2
- package/dist/src/pipeline/processors/access-processor.js.map +1 -1
- package/dist/src/pipeline/workers/parse-worker.d.ts +10 -15
- package/dist/src/pipeline/workers/parse-worker.d.ts.map +1 -1
- package/dist/src/pipeline/workers/parse-worker.js +12 -12
- package/dist/src/pipeline/workers/parse-worker.js.map +1 -1
- package/dist/src/pipeline/workers/worker-pool.d.ts.map +1 -1
- package/dist/src/pipeline/workers/worker-pool.js +106 -4
- package/dist/src/pipeline/workers/worker-pool.js.map +1 -1
- package/dist/tsconfig.build.tsbuildinfo +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse cache — per-file extraction memo that makes incremental indexing exact.
|
|
3
|
+
*
|
|
4
|
+
* Cross-file resolution (imports, barrels, heritage, calls, accesses) needs the
|
|
5
|
+
* COMPLETE extraction of every file, and most of what it consumes — raw call
|
|
6
|
+
* sites, unresolved access sites, re-export records — is never stored in the
|
|
7
|
+
* published graph rows. Reconstructing those facts from `nexus_nodes` /
|
|
8
|
+
* `nexus_relations` would be an approximation, so the extraction itself is
|
|
9
|
+
* persisted instead.
|
|
10
|
+
*
|
|
11
|
+
* Extraction is a pure function of `(path, bytes, extractor build, publication
|
|
12
|
+
* generation)`. An entry is therefore reusable exactly when the file's content
|
|
13
|
+
* hash and the extractor fingerprint both match; the only generation-dependent
|
|
14
|
+
* token (the publication generation embedded in anonymous scope identities and
|
|
15
|
+
* reference evidence) is rewritten on reuse. Everything downstream of
|
|
16
|
+
* extraction — import resolution, barrel maps, heritage, call and access
|
|
17
|
+
* resolution, communities and flows — reruns over the merged extraction, so an
|
|
18
|
+
* incremental run computes the same function as a full rebuild over the same
|
|
19
|
+
* inputs rather than patching a previous answer.
|
|
20
|
+
*
|
|
21
|
+
* @task T12315
|
|
22
|
+
* @module pipeline/parse-cache
|
|
23
|
+
*/
|
|
24
|
+
import type { GraphParseCacheEntry } from '@cleocode/contracts';
|
|
25
|
+
import type { CommonExtractionResult } from './parse-loop.js';
|
|
26
|
+
/**
|
|
27
|
+
* One file's complete extraction output, keyed by the file it describes.
|
|
28
|
+
*
|
|
29
|
+
* Kept per file (never merged across files) so a file's facts can be replaced
|
|
30
|
+
* without disturbing any other file's.
|
|
31
|
+
*/
|
|
32
|
+
export interface FileExtraction {
|
|
33
|
+
/** Path relative to the repository root. */
|
|
34
|
+
path: string;
|
|
35
|
+
/** Declarations, imports, heritage, calls, re-exports and accesses of this file. */
|
|
36
|
+
extraction: Required<CommonExtractionResult>;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Fingerprint the extractor build: every module under the pipeline and code
|
|
40
|
+
* directories plus the installed grammar package versions.
|
|
41
|
+
*
|
|
42
|
+
* Deliberately over-inclusive — ANY change to pipeline code invalidates every
|
|
43
|
+
* cache entry. A missed invalidation would publish an incremental graph that
|
|
44
|
+
* silently differs from a full rebuild, which is worse than re-parsing.
|
|
45
|
+
*
|
|
46
|
+
* @returns A hex fingerprint, or `null` when the build cannot be fingerprinted
|
|
47
|
+
* (the caller must then refuse to reuse any entry).
|
|
48
|
+
*/
|
|
49
|
+
export declare function computeExtractorFingerprint(): string | null;
|
|
50
|
+
/**
|
|
51
|
+
* Encode a freshly parsed extraction for storage.
|
|
52
|
+
*
|
|
53
|
+
* Must be called before any downstream phase mutates the extracted nodes
|
|
54
|
+
* (community detection writes `communityId` onto them).
|
|
55
|
+
*
|
|
56
|
+
* @param file - The per-file extraction to encode.
|
|
57
|
+
* @param contentHash - SHA-256 of the parsed bytes.
|
|
58
|
+
* @param fingerprint - Extractor fingerprint from {@link computeExtractorFingerprint}.
|
|
59
|
+
* @param generation - Publication generation the extraction was produced under.
|
|
60
|
+
* @returns A cache entry ready for atomic publication.
|
|
61
|
+
*/
|
|
62
|
+
export declare function encodeParseCacheEntry(file: FileExtraction, contentHash: string, fingerprint: string, generation: string): GraphParseCacheEntry;
|
|
63
|
+
/**
|
|
64
|
+
* Decode a stored extraction and rebind it to the current publication generation.
|
|
65
|
+
*
|
|
66
|
+
* @param entry - Stored cache entry.
|
|
67
|
+
* @param generation - The publication generation of the run reusing it.
|
|
68
|
+
* @returns The extraction exactly as re-parsing the same bytes under `generation` would produce.
|
|
69
|
+
* @throws When the payload is not a decodable extraction; callers treat that file as changed.
|
|
70
|
+
*/
|
|
71
|
+
export declare function decodeParseCacheEntry(entry: GraphParseCacheEntry, generation: string): FileExtraction;
|
|
72
|
+
//# sourceMappingURL=parse-cache.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-cache.d.ts","sourceRoot":"","sources":["../../../src/pipeline/parse-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAQH,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAC;AAChE,OAAO,KAAK,EAAE,sBAAsB,EAAE,MAAM,iBAAiB,CAAC;AAE9D;;;;;GAKG;AACH,MAAM,WAAW,cAAc;IAC7B,4CAA4C;IAC5C,IAAI,EAAE,MAAM,CAAC;IACb,oFAAoF;IACpF,UAAU,EAAE,QAAQ,CAAC,sBAAsB,CAAC,CAAC;CAC9C;AAsCD;;;;;;;;;;GAUG;AACH,wBAAgB,2BAA2B,IAAI,MAAM,GAAG,IAAI,CAmC3D;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,qBAAqB,CACnC,IAAI,EAAE,cAAc,EACpB,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,MAAM,GACjB,oBAAoB,CAQtB;AAED;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CACnC,KAAK,EAAE,oBAAoB,EAC3B,UAAU,EAAE,MAAM,GACjB,cAAc,CAgBhB"}
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parse cache — per-file extraction memo that makes incremental indexing exact.
|
|
3
|
+
*
|
|
4
|
+
* Cross-file resolution (imports, barrels, heritage, calls, accesses) needs the
|
|
5
|
+
* COMPLETE extraction of every file, and most of what it consumes — raw call
|
|
6
|
+
* sites, unresolved access sites, re-export records — is never stored in the
|
|
7
|
+
* published graph rows. Reconstructing those facts from `nexus_nodes` /
|
|
8
|
+
* `nexus_relations` would be an approximation, so the extraction itself is
|
|
9
|
+
* persisted instead.
|
|
10
|
+
*
|
|
11
|
+
* Extraction is a pure function of `(path, bytes, extractor build, publication
|
|
12
|
+
* generation)`. An entry is therefore reusable exactly when the file's content
|
|
13
|
+
* hash and the extractor fingerprint both match; the only generation-dependent
|
|
14
|
+
* token (the publication generation embedded in anonymous scope identities and
|
|
15
|
+
* reference evidence) is rewritten on reuse. Everything downstream of
|
|
16
|
+
* extraction — import resolution, barrel maps, heritage, call and access
|
|
17
|
+
* resolution, communities and flows — reruns over the merged extraction, so an
|
|
18
|
+
* incremental run computes the same function as a full rebuild over the same
|
|
19
|
+
* inputs rather than patching a previous answer.
|
|
20
|
+
*
|
|
21
|
+
* @task T12315
|
|
22
|
+
* @module pipeline/parse-cache
|
|
23
|
+
*/
|
|
24
|
+
import { createHash } from 'node:crypto';
|
|
25
|
+
import { readdirSync, readFileSync } from 'node:fs';
|
|
26
|
+
import { createRequire } from 'node:module';
|
|
27
|
+
import { join, relative } from 'node:path';
|
|
28
|
+
import { fileURLToPath } from 'node:url';
|
|
29
|
+
import { gunzipSync, gzipSync } from 'node:zlib';
|
|
30
|
+
/**
|
|
31
|
+
* Serialization format version. Bump when the payload encoding (not the
|
|
32
|
+
* extractor) changes; extractor changes are caught by the fingerprint.
|
|
33
|
+
*/
|
|
34
|
+
const PAYLOAD_FORMAT = 'nexus-parse-cache/v1';
|
|
35
|
+
/** Grammar packages whose native behaviour determines extraction output. */
|
|
36
|
+
const GRAMMAR_PACKAGES = [
|
|
37
|
+
'tree-sitter',
|
|
38
|
+
'tree-sitter-typescript',
|
|
39
|
+
'tree-sitter-javascript',
|
|
40
|
+
'tree-sitter-python',
|
|
41
|
+
'tree-sitter-go',
|
|
42
|
+
'tree-sitter-rust',
|
|
43
|
+
];
|
|
44
|
+
let cachedFingerprint;
|
|
45
|
+
/** Collect every module file under a directory, in a stable order. */
|
|
46
|
+
function collectModuleFiles(directory, out) {
|
|
47
|
+
const entries = readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
|
|
48
|
+
for (const entry of entries) {
|
|
49
|
+
if (entry.name === '__tests__' || entry.name === 'node_modules')
|
|
50
|
+
continue;
|
|
51
|
+
const full = join(directory, entry.name);
|
|
52
|
+
if (entry.isDirectory())
|
|
53
|
+
collectModuleFiles(full, out);
|
|
54
|
+
else if (/\.(?:[cm]?js|ts)$/.test(entry.name) &&
|
|
55
|
+
!entry.name.endsWith('.d.ts') &&
|
|
56
|
+
!/\.test\.[cm]?[jt]s$/.test(entry.name))
|
|
57
|
+
out.push(full);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
/**
|
|
61
|
+
* Fingerprint the extractor build: every module under the pipeline and code
|
|
62
|
+
* directories plus the installed grammar package versions.
|
|
63
|
+
*
|
|
64
|
+
* Deliberately over-inclusive — ANY change to pipeline code invalidates every
|
|
65
|
+
* cache entry. A missed invalidation would publish an incremental graph that
|
|
66
|
+
* silently differs from a full rebuild, which is worse than re-parsing.
|
|
67
|
+
*
|
|
68
|
+
* @returns A hex fingerprint, or `null` when the build cannot be fingerprinted
|
|
69
|
+
* (the caller must then refuse to reuse any entry).
|
|
70
|
+
*/
|
|
71
|
+
export function computeExtractorFingerprint() {
|
|
72
|
+
if (cachedFingerprint !== undefined)
|
|
73
|
+
return cachedFingerprint;
|
|
74
|
+
try {
|
|
75
|
+
const hash = createHash('sha256').update(PAYLOAD_FORMAT);
|
|
76
|
+
const pipelineDir = fileURLToPath(new URL('.', import.meta.url));
|
|
77
|
+
const codeDir = fileURLToPath(new URL('../code/', import.meta.url));
|
|
78
|
+
const files = [];
|
|
79
|
+
collectModuleFiles(pipelineDir, files);
|
|
80
|
+
collectModuleFiles(codeDir, files);
|
|
81
|
+
if (files.length === 0)
|
|
82
|
+
throw new Error('no extractor modules found');
|
|
83
|
+
const packageSourceRoot = join(pipelineDir, '..');
|
|
84
|
+
for (const file of files) {
|
|
85
|
+
hash.update(relative(packageSourceRoot, file));
|
|
86
|
+
hash.update('\0');
|
|
87
|
+
hash.update(readFileSync(file));
|
|
88
|
+
}
|
|
89
|
+
const requireFromHere = createRequire(import.meta.url);
|
|
90
|
+
for (const pkg of GRAMMAR_PACKAGES) {
|
|
91
|
+
let version = 'absent';
|
|
92
|
+
try {
|
|
93
|
+
const manifest = JSON.parse(readFileSync(requireFromHere.resolve(`${pkg}/package.json`), 'utf8'));
|
|
94
|
+
version = manifest.version ?? 'unversioned';
|
|
95
|
+
}
|
|
96
|
+
catch {
|
|
97
|
+
// An uninstalled grammar is part of the build identity too.
|
|
98
|
+
}
|
|
99
|
+
hash.update(`${pkg}@${version}\0`);
|
|
100
|
+
}
|
|
101
|
+
hash.update(`node-abi:${process.versions.modules}`);
|
|
102
|
+
cachedFingerprint = hash.digest('hex');
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
cachedFingerprint = null;
|
|
106
|
+
}
|
|
107
|
+
return cachedFingerprint;
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Encode a freshly parsed extraction for storage.
|
|
111
|
+
*
|
|
112
|
+
* Must be called before any downstream phase mutates the extracted nodes
|
|
113
|
+
* (community detection writes `communityId` onto them).
|
|
114
|
+
*
|
|
115
|
+
* @param file - The per-file extraction to encode.
|
|
116
|
+
* @param contentHash - SHA-256 of the parsed bytes.
|
|
117
|
+
* @param fingerprint - Extractor fingerprint from {@link computeExtractorFingerprint}.
|
|
118
|
+
* @param generation - Publication generation the extraction was produced under.
|
|
119
|
+
* @returns A cache entry ready for atomic publication.
|
|
120
|
+
*/
|
|
121
|
+
export function encodeParseCacheEntry(file, contentHash, fingerprint, generation) {
|
|
122
|
+
return {
|
|
123
|
+
path: file.path,
|
|
124
|
+
contentHash,
|
|
125
|
+
fingerprint,
|
|
126
|
+
generation,
|
|
127
|
+
payload: gzipSync(JSON.stringify(file.extraction), { level: 1 }),
|
|
128
|
+
};
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Decode a stored extraction and rebind it to the current publication generation.
|
|
132
|
+
*
|
|
133
|
+
* @param entry - Stored cache entry.
|
|
134
|
+
* @param generation - The publication generation of the run reusing it.
|
|
135
|
+
* @returns The extraction exactly as re-parsing the same bytes under `generation` would produce.
|
|
136
|
+
* @throws When the payload is not a decodable extraction; callers treat that file as changed.
|
|
137
|
+
*/
|
|
138
|
+
export function decodeParseCacheEntry(entry, generation) {
|
|
139
|
+
if (!entry.generation)
|
|
140
|
+
throw new Error('Parse cache entry lacks its generation');
|
|
141
|
+
let text = gunzipSync(entry.payload).toString('utf8');
|
|
142
|
+
if (entry.generation !== generation)
|
|
143
|
+
text = text.split(entry.generation).join(generation);
|
|
144
|
+
const parsed = JSON.parse(text);
|
|
145
|
+
for (const key of [
|
|
146
|
+
'definitions',
|
|
147
|
+
'imports',
|
|
148
|
+
'heritage',
|
|
149
|
+
'calls',
|
|
150
|
+
'reExports',
|
|
151
|
+
'accesses',
|
|
152
|
+
]) {
|
|
153
|
+
if (!Array.isArray(parsed[key]))
|
|
154
|
+
throw new Error(`Parse cache entry lacks ${key}`);
|
|
155
|
+
}
|
|
156
|
+
return { path: entry.path, extraction: parsed };
|
|
157
|
+
}
|
|
158
|
+
//# sourceMappingURL=parse-cache.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parse-cache.js","sourceRoot":"","sources":["../../../src/pipeline/parse-cache.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;GAsBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAC5C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAC3C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,WAAW,CAAC;AAiBjD;;;GAGG;AACH,MAAM,cAAc,GAAG,sBAAsB,CAAC;AAE9C,4EAA4E;AAC5E,MAAM,gBAAgB,GAAG;IACvB,aAAa;IACb,wBAAwB;IACxB,wBAAwB;IACxB,oBAAoB;IACpB,gBAAgB;IAChB,kBAAkB;CACV,CAAC;AAEX,IAAI,iBAA4C,CAAC;AAEjD,sEAAsE;AACtE,SAAS,kBAAkB,CAAC,SAAiB,EAAE,GAAa;IAC1D,MAAM,OAAO,GAAG,WAAW,CAAC,SAAS,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAC5E,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/C,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc;YAAE,SAAS;QAC1E,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,IAAI,KAAK,CAAC,WAAW,EAAE;YAAE,kBAAkB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;aAClD,IACH,mBAAmB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YACpC,CAAC,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;YAC7B,CAAC,qBAAqB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;YAEvC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,2BAA2B;IACzC,IAAI,iBAAiB,KAAK,SAAS;QAAE,OAAO,iBAAiB,CAAC;IAC9D,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;QACzD,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACjE,MAAM,OAAO,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACpE,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,kBAAkB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC;QACvC,kBAAkB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QACnC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAC;QACtE,MAAM,iBAAiB,GAAG,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;QAClD,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC,CAAC;YAC/C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAClB,IAAI,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QAClC,CAAC;QACD,MAAM,eAAe,GAAG,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACvD,KAAK,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;YACnC,IAAI,OAAO,GAAG,QAAQ,CAAC;YACvB,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CACzB,YAAY,CAAC,eAAe,CAAC,OAAO,CAAC,GAAG,GAAG,eAAe,CAAC,EAAE,MAAM,CAAC,CAC7C,CAAC;gBAC1B,OAAO,GAAG,QAAQ,CAAC,OAAO,IAAI,aAAa,CAAC;YAC9C,CAAC;YAAC,MAAM,CAAC;gBACP,4DAA4D;YAC9D,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,OAAO,IAAI,CAAC,CAAC;QACrC,CAAC;QACD,IAAI,CAAC,MAAM,CAAC,YAAY,OAAO,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC;QACpD,iBAAiB,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACzC,CAAC;IAAC,MAAM,CAAC;QACP,iBAAiB,GAAG,IAAI,CAAC;IAC3B,CAAC;IACD,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,UAAU,qBAAqB,CACnC,IAAoB,EACpB,WAAmB,EACnB,WAAmB,EACnB,UAAkB;IAElB,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,WAAW;QACX,WAAW;QACX,UAAU;QACV,OAAO,EAAE,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,EAAE,KAAK,EAAE,CAAC,EAAE,CAAC;KACjE,CAAC;AACJ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CACnC,KAA2B,EAC3B,UAAkB;IAElB,IAAI,CAAC,KAAK,CAAC,UAAU;QAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAC;IACjF,IAAI,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACtD,IAAI,KAAK,CAAC,UAAU,KAAK,UAAU;QAAE,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC1F,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAqC,CAAC;IACpE,KAAK,MAAM,GAAG,IAAI;QAChB,aAAa;QACb,SAAS;QACT,UAAU;QACV,OAAO;QACP,WAAW;QACX,UAAU;KACF,EAAE,CAAC;QACX,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,2BAA2B,GAAG,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AAClD,CAAC"}
|
|
@@ -36,6 +36,7 @@ import { type ExtractedCall, type ExtractedHeritage, type ExtractedReExport } fr
|
|
|
36
36
|
import type { ScannedFile } from './filesystem-walker.js';
|
|
37
37
|
import type { BarrelExportMap, ExtractedImport, ImportResolutionContext, NamedImportMap, TsconfigPaths } from './import-processor.js';
|
|
38
38
|
import type { KnowledgeGraph } from './knowledge-graph.js';
|
|
39
|
+
import type { FileExtraction } from './parse-cache.js';
|
|
39
40
|
import { type ExtractedAccess } from './processors/access-processor.js';
|
|
40
41
|
import type { SymbolTable } from './symbol-table.js';
|
|
41
42
|
/**
|
|
@@ -97,6 +98,17 @@ export interface ParseLoopOptions {
|
|
|
97
98
|
* @param filePath - Current file path being processed
|
|
98
99
|
*/
|
|
99
100
|
onProgress?: (current: number, total: number, filePath: string) => void;
|
|
101
|
+
/**
|
|
102
|
+
* Extractions reused from the parse cache, keyed by path (T12315). A parseable
|
|
103
|
+
* file with an entry here is not re-parsed; its extraction is merged exactly
|
|
104
|
+
* where re-parsing it would have been, so resolution sees the same input.
|
|
105
|
+
*/
|
|
106
|
+
reusedExtractions?: ReadonlyMap<string, FileExtraction>;
|
|
107
|
+
/**
|
|
108
|
+
* Called once per freshly and successfully parsed file, before any later phase
|
|
109
|
+
* can mutate its nodes — the moment a parse-cache entry must be captured.
|
|
110
|
+
*/
|
|
111
|
+
onFileExtracted?: (file: FileExtraction) => void;
|
|
100
112
|
}
|
|
101
113
|
/**
|
|
102
114
|
* Result returned by the parse loop.
|
|
@@ -117,36 +129,34 @@ export interface ParseLoopResult {
|
|
|
117
129
|
* Used by the call resolution phase to trace imports through barrel index files.
|
|
118
130
|
*/
|
|
119
131
|
barrelMap: BarrelExportMap;
|
|
132
|
+
/** Parseable files handed to the parser this run. */
|
|
133
|
+
parsedFileCount: number;
|
|
134
|
+
/** Parseable files whose extraction came from {@link ParseLoopOptions.reusedExtractions}. */
|
|
135
|
+
reusedFileCount: number;
|
|
120
136
|
}
|
|
121
137
|
/**
|
|
122
|
-
* Phase 3: Parse loop (parallel or sequential).
|
|
123
|
-
*
|
|
124
|
-
* For each TypeScript/JavaScript file in `files`:
|
|
125
|
-
* 1. Read file content from disk
|
|
126
|
-
* 2. Parse with tree-sitter (typescript or javascript grammar)
|
|
127
|
-
* 3. Extract definitions → register in SymbolTable → add nodes to graph
|
|
128
|
-
* 4. Extract imports → collect for batch resolution
|
|
129
|
-
* 5. Extract heritage → accumulate for deferred edge emission
|
|
130
|
-
*
|
|
131
|
-
* After all files are processed:
|
|
132
|
-
* - Resolves all collected imports via `processExtractedImports`
|
|
133
|
-
* - Emits EXTENDS/IMPLEMENTS edges from accumulated heritage
|
|
138
|
+
* Phase 3: Parse loop (parallel or sequential), then merge.
|
|
134
139
|
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
140
|
+
* 1. Classify every scanned file's role from its bytes.
|
|
141
|
+
* 2. Extract each parseable file that has no reusable cached extraction —
|
|
142
|
+
* through the worker pool when the file count or byte volume warrants it.
|
|
143
|
+
* 3. Merge fresh and reused extractions **in scan order**: register symbols,
|
|
144
|
+
* add nodes, emit DEFINES edges, and collect imports, re-exports, heritage,
|
|
145
|
+
* calls and accesses. The merge order is the same whichever path produced
|
|
146
|
+
* an extraction, so first-wins deduplication in the graph cannot depend on
|
|
147
|
+
* which files happened to be re-parsed.
|
|
148
|
+
* 4. Resolve all imports and build the barrel map over the merged input.
|
|
137
149
|
*
|
|
138
|
-
*
|
|
139
|
-
*
|
|
140
|
-
* If the worker script is not found (e.g. running from source without a
|
|
141
|
-
* build), the sequential path is used as a fallback.
|
|
150
|
+
* Files that fail to parse are reported, not merged; the publish step decides
|
|
151
|
+
* what a given share of failures means.
|
|
142
152
|
*
|
|
143
|
-
* @param files - All scanned files (non-
|
|
153
|
+
* @param files - All scanned files (non-parseable files are classified and reported only)
|
|
144
154
|
* @param graph - Knowledge graph to add nodes and relations to
|
|
145
155
|
* @param symbolTable - Symbol table to register extracted symbols in
|
|
146
156
|
* @param importCtx - Pre-built import resolution context (from Phase 3a)
|
|
147
157
|
* @param repoPath - Absolute path to the repository root (used to resolve relative paths)
|
|
148
|
-
* @param options -
|
|
149
|
-
* @returns Accumulated heritage and
|
|
158
|
+
* @param options - Resolution inputs, reuse map, progress and report callbacks
|
|
159
|
+
* @returns Accumulated heritage, call and access records plus the barrel map
|
|
150
160
|
*/
|
|
151
161
|
export declare function runParseLoop(files: ScannedFile[], graph: KnowledgeGraph, symbolTable: SymbolTable, importCtx: ImportResolutionContext, repoPath: string, options?: ParseLoopOptions): Promise<ParseLoopResult>;
|
|
152
162
|
//# sourceMappingURL=parse-loop.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"parse-loop.d.ts","sourceRoot":"","sources":["../../../src/pipeline/parse-loop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAMH,OAAO,KAAK,EACV,oBAAoB,EACpB,SAAS,EAGT,qBAAqB,EACrB,mBAAmB,EACpB,MAAM,qBAAqB,CAAC;AAa7B,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EAEvB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EACV,eAAe,EACf,eAAe,EAEf,uBAAuB,EACvB,cAAc,EACd,aAAa,EACd,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAG3D,OAAO,EAAE,KAAK,eAAe,EAAmB,MAAM,kCAAkC,CAAC;AACzF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAkIrD;;;;;;;;;GASG;AACH,MAAM,WAAW,sBAAsB;IACrC,4DAA4D;IAC5D,WAAW,EAAE,SAAS,EAAE,CAAC;IACzB,0CAA0C;IAC1C,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,iCAAiC;IACjC,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,gEAAgE;IAChE,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAChC,yCAAyC;IACzC,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAC;CAC9B;AA8DD;;;;;;;;;;;;GAYG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,qBAAqB,EAC9B,qBAAqB,CAAC,EAAE,MAAM,GAC7B,sBAAsB,CAoBxB;AAmGD,6CAA6C;AAC7C,MAAM,WAAW,gBAAgB;IAC/B,mFAAmF;IACnF,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,sEAAsE;IACtE,YAAY,CAAC,EAAE,qBAAqB,CAAC;IACrC,sEAAsE;IACtE,eAAe,CAAC,EAAE,mBAAmB,CAAC;IACtC,qFAAqF;IACrF,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,IAAI,CAAC;IAEtD,4DAA4D;IAC5D,aAAa,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACrC,4EAA4E;IAC5E,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;
|
|
1
|
+
{"version":3,"file":"parse-loop.d.ts","sourceRoot":"","sources":["../../../src/pipeline/parse-loop.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAMH,OAAO,KAAK,EACV,oBAAoB,EACpB,SAAS,EAGT,qBAAqB,EACrB,mBAAmB,EACpB,MAAM,qBAAqB,CAAC;AAa7B,OAAO,EACL,KAAK,aAAa,EAClB,KAAK,iBAAiB,EACtB,KAAK,iBAAiB,EAEvB,MAAM,sCAAsC,CAAC;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC1D,OAAO,KAAK,EACV,eAAe,EACf,eAAe,EAEf,uBAAuB,EACvB,cAAc,EACd,aAAa,EACd,MAAM,uBAAuB,CAAC;AAE/B,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAG3D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACvD,OAAO,EAAE,KAAK,eAAe,EAAmB,MAAM,kCAAkC,CAAC;AACzF,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAkIrD;;;;;;;;;GASG;AACH,MAAM,WAAW,sBAAsB;IACrC,4DAA4D;IAC5D,WAAW,EAAE,SAAS,EAAE,CAAC;IACzB,0CAA0C;IAC1C,OAAO,EAAE,eAAe,EAAE,CAAC;IAC3B,iCAAiC;IACjC,QAAQ,EAAE,iBAAiB,EAAE,CAAC;IAC9B,gEAAgE;IAChE,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,0CAA0C;IAC1C,SAAS,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAChC,yCAAyC;IACzC,QAAQ,CAAC,EAAE,eAAe,EAAE,CAAC;CAC9B;AA8DD;;;;;;;;;;;;GAYG;AACH,wBAAgB,qBAAqB,CACnC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,qBAAqB,EAC9B,qBAAqB,CAAC,EAAE,MAAM,GAC7B,sBAAsB,CAoBxB;AAmGD,6CAA6C;AAC7C,MAAM,WAAW,gBAAgB;IAC/B,mFAAmF;IACnF,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,sEAAsE;IACtE,YAAY,CAAC,EAAE,qBAAqB,CAAC;IACrC,sEAAsE;IACtE,eAAe,CAAC,EAAE,mBAAmB,CAAC;IACtC,qFAAqF;IACrF,YAAY,CAAC,EAAE,CAAC,MAAM,EAAE,oBAAoB,KAAK,IAAI,CAAC;IAEtD,4DAA4D;IAC5D,aAAa,CAAC,EAAE,aAAa,GAAG,IAAI,CAAC;IACrC,4EAA4E;IAC5E,cAAc,CAAC,EAAE,cAAc,CAAC;IAChC;;;;;OAKG;IACH,UAAU,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;IACxE;;;;OAIG;IACH,iBAAiB,CAAC,EAAE,WAAW,CAAC,MAAM,EAAE,cAAc,CAAC,CAAC;IACxD;;;OAGG;IACH,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,cAAc,KAAK,IAAI,CAAC;CAClD;AAED;;;;;;GAMG;AACH,MAAM,WAAW,eAAe;IAC9B,6EAA6E;IAC7E,WAAW,EAAE,iBAAiB,EAAE,CAAC;IACjC,oFAAoF;IACpF,QAAQ,EAAE,aAAa,EAAE,CAAC;IAC1B,kFAAkF;IAClF,WAAW,EAAE,eAAe,EAAE,CAAC;IAC/B;;;OAGG;IACH,SAAS,EAAE,eAAe,CAAC;IAC3B,qDAAqD;IACrD,eAAe,EAAE,MAAM,CAAC;IACxB,6FAA6F;IAC7F,eAAe,EAAE,MAAM,CAAC;CACzB;AAifD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAsB,YAAY,CAChC,KAAK,EAAE,WAAW,EAAE,EACpB,KAAK,EAAE,cAAc,EACrB,WAAW,EAAE,WAAW,EACxB,SAAS,EAAE,uBAAuB,EAClC,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,gBAAqB,GAC7B,OAAO,CAAC,eAAe,CAAC,CA2I1B"}
|