gitnexus 1.6.11-rc.13 → 1.6.11-rc.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/incremental/derived-writeback.d.ts +36 -0
- package/dist/core/incremental/derived-writeback.js +68 -0
- package/dist/core/incremental/subgraph-extract.d.ts +6 -4
- package/dist/core/incremental/subgraph-extract.js +7 -5
- package/dist/core/ingestion/pipeline-phases/parse-impl.js +21 -18
- package/dist/core/ingestion/pipeline-phases/runner.d.ts +4 -1
- package/dist/core/ingestion/pipeline-phases/runner.js +34 -14
- package/dist/core/ingestion/pipeline.d.ts +6 -0
- package/dist/core/ingestion/pipeline.js +42 -11
- package/dist/core/ingestion/workers/parse-worker.js +4 -2
- package/dist/core/lbug/lbug-adapter.d.ts +59 -0
- package/dist/core/lbug/lbug-adapter.js +154 -1
- package/dist/core/run-analyze.js +120 -12
- package/dist/core/search/fts-indexes.d.ts +19 -1
- package/dist/core/search/fts-indexes.js +28 -1
- package/dist/storage/fs-atomic.d.ts +24 -0
- package/dist/storage/fs-atomic.js +86 -2
- package/dist/storage/parse-cache.js +24 -29
- package/dist/storage/parsedfile-store.d.ts +27 -62
- package/dist/storage/parsedfile-store.js +124 -295
- package/dist/storage/v8-sidecar.d.ts +48 -0
- package/dist/storage/v8-sidecar.js +347 -0
- package/dist/types/pipeline.d.ts +12 -0
- package/package.json +1 -1
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { KnowledgeGraph } from '../graph/types.js';
|
|
2
|
+
import type { FileHashDiff } from '../../storage/file-hash.js';
|
|
3
|
+
/** The FTS-backed members of `tables`. */
|
|
4
|
+
export declare const ftsTablesAmong: (tables: Iterable<string>) => Set<string>;
|
|
5
|
+
/**
|
|
6
|
+
* Whether a surgical incremental write may reuse the persisted derived layer.
|
|
7
|
+
*
|
|
8
|
+
* Deletions disqualify it: the persisted Community/Process rows and their
|
|
9
|
+
* MEMBER_OF / STEP_IN_PROCESS edges can reference nodes that no longer exist
|
|
10
|
+
* after this run, and nothing short of re-deriving can tell which.
|
|
11
|
+
*
|
|
12
|
+
* Added or content-changed files also disqualify it: they can introduce,
|
|
13
|
+
* rename, or retarget symbols and CALLS edges that Leiden and flow extraction
|
|
14
|
+
* consume. File-deletion-only was too weak a proof that the derived graph is
|
|
15
|
+
* still valid.
|
|
16
|
+
*/
|
|
17
|
+
export declare const shouldPreservePersistedDerivedGraph: (diff: Pick<FileHashDiff, "deleted" | "added" | "changed">) => boolean;
|
|
18
|
+
/**
|
|
19
|
+
* FTS-backed node tables that the fresh graph will WRITE rows into for
|
|
20
|
+
* `fileSet` — the inserting half of the DML.
|
|
21
|
+
*
|
|
22
|
+
* Callers must union this with a DB probe for the deleting half
|
|
23
|
+
* (`nodeTablesWithRowsForFiles`): a table whose last row in these files was
|
|
24
|
+
* just removed by the edit has nothing here, but still holds a stale row that
|
|
25
|
+
* the writeback must delete, and deleting it means taking its index down too.
|
|
26
|
+
*/
|
|
27
|
+
export declare const incrementalFtsTablesFromGraph: (graph: KnowledgeGraph, fileSet: ReadonlySet<string>) => Set<string>;
|
|
28
|
+
/**
|
|
29
|
+
* The node tables an incremental DETACH DELETE should target, given the FTS
|
|
30
|
+
* tables this run is rebuilding.
|
|
31
|
+
*
|
|
32
|
+
* Every non-FTS table (Folder, CodeElement, …) deletes as before. An FTS-backed
|
|
33
|
+
* table only deletes when its index is being rebuilt anyway, because deleting
|
|
34
|
+
* from it otherwise would mean DML against a live FTS index (#2589).
|
|
35
|
+
*/
|
|
36
|
+
export declare const nodeTablesForIncrementalDelete: (allNodeTables: readonly string[], rebuildingFtsTables: ReadonlySet<string>) => string[];
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Incremental derived-layer writeback helpers (#3016).
|
|
3
|
+
*
|
|
4
|
+
* The derived layers — Leiden communities, execution flows, and the FTS
|
|
5
|
+
* indexes — are graph-wide, so every analyze run rebuilt all three in full no
|
|
6
|
+
* matter how small the diff. A surgical incremental write can instead:
|
|
7
|
+
* - drop and rebuild only the FTS indexes whose tables hold rows in the
|
|
8
|
+
* write set (LadybugDB still cannot DML a table with a live FTS index —
|
|
9
|
+
* #2589 — so a table being written must still lose its index first);
|
|
10
|
+
* - leave the untouched tables' rows alone, so their indexes stay live;
|
|
11
|
+
* - reuse persisted Community/Process rows only when the file-hash diff is
|
|
12
|
+
* empty (no added, changed, or deleted files). Any content change can
|
|
13
|
+
* add, rename, or retarget symbols that Leiden and flow extraction
|
|
14
|
+
* consume — a no-deletion edit is not a validity proof.
|
|
15
|
+
*/
|
|
16
|
+
import { FTS_INDEXES } from '../search/fts-schema.js';
|
|
17
|
+
const FTS_TABLE_NAMES = new Set(FTS_INDEXES.map((i) => i.table));
|
|
18
|
+
/** The FTS-backed members of `tables`. */
|
|
19
|
+
export const ftsTablesAmong = (tables) => {
|
|
20
|
+
const out = new Set();
|
|
21
|
+
for (const table of tables) {
|
|
22
|
+
if (FTS_TABLE_NAMES.has(table))
|
|
23
|
+
out.add(table);
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Whether a surgical incremental write may reuse the persisted derived layer.
|
|
29
|
+
*
|
|
30
|
+
* Deletions disqualify it: the persisted Community/Process rows and their
|
|
31
|
+
* MEMBER_OF / STEP_IN_PROCESS edges can reference nodes that no longer exist
|
|
32
|
+
* after this run, and nothing short of re-deriving can tell which.
|
|
33
|
+
*
|
|
34
|
+
* Added or content-changed files also disqualify it: they can introduce,
|
|
35
|
+
* rename, or retarget symbols and CALLS edges that Leiden and flow extraction
|
|
36
|
+
* consume. File-deletion-only was too weak a proof that the derived graph is
|
|
37
|
+
* still valid.
|
|
38
|
+
*/
|
|
39
|
+
export const shouldPreservePersistedDerivedGraph = (diff) => diff.deleted.length === 0 && diff.added.length === 0 && diff.changed.length === 0;
|
|
40
|
+
/**
|
|
41
|
+
* FTS-backed node tables that the fresh graph will WRITE rows into for
|
|
42
|
+
* `fileSet` — the inserting half of the DML.
|
|
43
|
+
*
|
|
44
|
+
* Callers must union this with a DB probe for the deleting half
|
|
45
|
+
* (`nodeTablesWithRowsForFiles`): a table whose last row in these files was
|
|
46
|
+
* just removed by the edit has nothing here, but still holds a stale row that
|
|
47
|
+
* the writeback must delete, and deleting it means taking its index down too.
|
|
48
|
+
*/
|
|
49
|
+
export const incrementalFtsTablesFromGraph = (graph, fileSet) => {
|
|
50
|
+
const touched = new Set();
|
|
51
|
+
graph.forEachNode((n) => {
|
|
52
|
+
const filePath = n.properties?.filePath;
|
|
53
|
+
if (!filePath || !fileSet.has(filePath))
|
|
54
|
+
return;
|
|
55
|
+
if (FTS_TABLE_NAMES.has(n.label))
|
|
56
|
+
touched.add(n.label);
|
|
57
|
+
});
|
|
58
|
+
return touched;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* The node tables an incremental DETACH DELETE should target, given the FTS
|
|
62
|
+
* tables this run is rebuilding.
|
|
63
|
+
*
|
|
64
|
+
* Every non-FTS table (Folder, CodeElement, …) deletes as before. An FTS-backed
|
|
65
|
+
* table only deletes when its index is being rebuilt anyway, because deleting
|
|
66
|
+
* from it otherwise would mean DML against a live FTS index (#2589).
|
|
67
|
+
*/
|
|
68
|
+
export const nodeTablesForIncrementalDelete = (allNodeTables, rebuildingFtsTables) => allNodeTables.filter((tableName) => !FTS_TABLE_NAMES.has(tableName) || rebuildingFtsTables.has(tableName));
|
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
* replaced, produce a smaller KnowledgeGraph that contains:
|
|
7
7
|
*
|
|
8
8
|
* - Every node whose `properties.filePath` is in `toWriteSet`.
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* - Graph-wide Community/Process nodes unless `includeDerivedGraphWide`
|
|
10
|
+
* is false (#3016 incremental preserve). Spring metadata placeholders
|
|
11
|
+
* are always included.
|
|
12
12
|
* - Every relationship where AT LEAST ONE endpoint is in the writable
|
|
13
13
|
* set above. Relationships entirely between unchanged-file nodes
|
|
14
14
|
* are skipped — their rows are still in the DB and re-inserting
|
|
@@ -48,7 +48,9 @@
|
|
|
48
48
|
* IMPORTS from the pre-pipeline DB) covers that case instead.
|
|
49
49
|
*/
|
|
50
50
|
import type { KnowledgeGraph } from '../graph/types.js';
|
|
51
|
-
export declare const extractChangedSubgraph: (fullGraph: KnowledgeGraph, toWriteSet: ReadonlySet<string
|
|
51
|
+
export declare const extractChangedSubgraph: (fullGraph: KnowledgeGraph, toWriteSet: ReadonlySet<string>, options?: {
|
|
52
|
+
includeDerivedGraphWide?: boolean;
|
|
53
|
+
}) => KnowledgeGraph;
|
|
52
54
|
/**
|
|
53
55
|
* Public — derive the EFFECTIVE write-set: `toWriteSet` expanded by one
|
|
54
56
|
* hop along every file-owned edge in the new graph that crosses the
|
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
* replaced, produce a smaller KnowledgeGraph that contains:
|
|
7
7
|
*
|
|
8
8
|
* - Every node whose `properties.filePath` is in `toWriteSet`.
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* - Graph-wide Community/Process nodes unless `includeDerivedGraphWide`
|
|
10
|
+
* is false (#3016 incremental preserve). Spring metadata placeholders
|
|
11
|
+
* are always included.
|
|
12
12
|
* - Every relationship where AT LEAST ONE endpoint is in the writable
|
|
13
13
|
* set above. Relationships entirely between unchanged-file nodes
|
|
14
14
|
* are skipped — their rows are still in the DB and re-inserting
|
|
@@ -108,12 +108,14 @@ const indexNodeFilePaths = (fullGraph) => {
|
|
|
108
108
|
});
|
|
109
109
|
return idx;
|
|
110
110
|
};
|
|
111
|
-
export const extractChangedSubgraph = (fullGraph, toWriteSet) => {
|
|
111
|
+
export const extractChangedSubgraph = (fullGraph, toWriteSet, options) => {
|
|
112
112
|
const sub = createKnowledgeGraph();
|
|
113
113
|
const writableNodeIds = new Set();
|
|
114
|
+
const includeDerivedGraphWide = options?.includeDerivedGraphWide !== false;
|
|
114
115
|
fullGraph.forEachNode((n) => {
|
|
115
116
|
const filePath = n.properties?.filePath;
|
|
116
|
-
const
|
|
117
|
+
const derivedWide = includeDerivedGraphWide || (n.label !== 'Community' && n.label !== 'Process');
|
|
118
|
+
const include = (filePath && toWriteSet.has(filePath)) || (isGraphWideNode(n) && derivedWide);
|
|
117
119
|
if (include) {
|
|
118
120
|
sub.addNode(n);
|
|
119
121
|
writableNodeIds.add(n.id);
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
import { BindingAccumulator, enrichExportedTypeMap, } from '../binding-accumulator.js';
|
|
18
18
|
import { mergeChunkResults, dispatchChunkParse } from '../parsing-processor.js';
|
|
19
19
|
import { fileContentHash, computeChunkHash, loadParseCacheChunk, persistParseCacheChunk, PARSE_CACHE_VERSION, packParseCacheChunks, } from '../../../storage/parse-cache.js';
|
|
20
|
-
import { clearParsedFileStore, persistParsedFileChunk, loadParsedFilesForPaths, getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk,
|
|
20
|
+
import { clearParsedFileStore, persistParsedFileChunk, loadParsedFilesForPaths, getDurableParsedFileDir, loadDurableParsedFileIndex, prepareDurableParsedFileChunk, durableChunkHasShards, } from '../../../storage/parsedfile-store.js';
|
|
21
21
|
import { DEFAULT_PDG_MAX_FUNCTION_LINES } from '../cfg/collect.js';
|
|
22
22
|
import { processRoutesFromExtracted, resolveRouteHandlerSymbols, buildExportedTypeMapFromGraph, } from '../call-processor.js';
|
|
23
23
|
import { createSemanticModel } from '../model/index.js';
|
|
@@ -547,13 +547,13 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
547
547
|
// a sibling of the run-scoped store, NOT cleared per run. Workers write a
|
|
548
548
|
// shard per chunk hash; on a warm parse-cache hit we restore the chunk's
|
|
549
549
|
// shards into the run-scoped store so scope-resolution streams them without
|
|
550
|
-
// re-parsing. `
|
|
551
|
-
// PARSE_CACHE_VERSION (a mismatch ⇒ empty ⇒ every chunk
|
|
552
|
-
// repopulates the durable store
|
|
550
|
+
// re-parsing. `durableHitEntries` is the prior run's path-coverage index,
|
|
551
|
+
// version-gated by PARSE_CACHE_VERSION (a mismatch ⇒ empty ⇒ every chunk
|
|
552
|
+
// re-dispatches, which repopulates the durable store).
|
|
553
553
|
const durableParsedFileDir = parsedFileStorePath !== undefined ? getDurableParsedFileDir(parsedFileStorePath) : undefined;
|
|
554
|
-
const
|
|
554
|
+
const durableHitEntries = durableParsedFileDir !== undefined
|
|
555
555
|
? await loadDurableParsedFileIndex(durableParsedFileDir, PARSE_CACHE_VERSION)
|
|
556
|
-
: new
|
|
556
|
+
: new Map();
|
|
557
557
|
let chunkCacheHits = 0;
|
|
558
558
|
let chunkCacheMisses = 0;
|
|
559
559
|
let reparsedFileCount = 0;
|
|
@@ -607,7 +607,11 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
607
607
|
}
|
|
608
608
|
if (chunkWorkerData.parsedFiles?.length) {
|
|
609
609
|
if (parsedFileStorePath) {
|
|
610
|
-
await persistParsedFileChunk(parsedFileStorePath, `chunk-${chunkIdx}`, chunkWorkerData.parsedFiles);
|
|
610
|
+
const wrote = await persistParsedFileChunk(parsedFileStorePath, `chunk-${chunkIdx}`, chunkWorkerData.parsedFiles);
|
|
611
|
+
if (!wrote) {
|
|
612
|
+
for (const item of chunkWorkerData.parsedFiles)
|
|
613
|
+
allParsedFiles.push(item);
|
|
614
|
+
}
|
|
611
615
|
}
|
|
612
616
|
else {
|
|
613
617
|
for (const item of chunkWorkerData.parsedFiles)
|
|
@@ -794,7 +798,14 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
794
798
|
// store was introduced, or a pruned/version-stale shard — fall through to
|
|
795
799
|
// a worker re-dispatch to repopulate them. NEVER let scope-resolution
|
|
796
800
|
// re-extract on the main thread (the #1983 OOM the durable store closes).
|
|
797
|
-
const
|
|
801
|
+
const durableExpectedPaths = chunkHash === null ? undefined : durableHitEntries.get(chunkHash);
|
|
802
|
+
const durableHit = cachedRaw !== undefined &&
|
|
803
|
+
cachedRaw.length > 0 &&
|
|
804
|
+
chunkHash !== null &&
|
|
805
|
+
durableParsedFileDir !== undefined &&
|
|
806
|
+
parsedFileStorePath !== undefined &&
|
|
807
|
+
durableExpectedPaths !== undefined &&
|
|
808
|
+
(await durableChunkHasShards(parsedFileStorePath, chunkHash, durableExpectedPaths));
|
|
798
809
|
if (cachedRaw && cachedRaw.length > 0 && (durableHit || parsedFileStorePath === undefined)) {
|
|
799
810
|
// Cache hit: replay cached worker output. Finalize any parked worker
|
|
800
811
|
// chunk FIRST so deferred aggregation stays in chunk order, then merge
|
|
@@ -824,16 +835,8 @@ export async function runChunkedParseAndResolve(graph, scannedFiles, allPaths, t
|
|
|
824
835
|
nodesCreated: graph.nodeCount,
|
|
825
836
|
},
|
|
826
837
|
});
|
|
827
|
-
//
|
|
828
|
-
// store
|
|
829
|
-
// re-parse. A verbatim byte copy — byte-identical to a cold run.
|
|
830
|
-
if (durableHit && durableParsedFileDir && parsedFileStorePath && chunkHash) {
|
|
831
|
-
const restored = await restoreDurableParsedFileShard(durableParsedFileDir, parsedFileStorePath, chunkHash);
|
|
832
|
-
if (restored === 0) {
|
|
833
|
-
logger.warn(`parsedfile-cache: durable shards missing for cached chunk ` +
|
|
834
|
-
`${chunkHash.slice(0, 8)} — scope-resolution will re-extract these files`);
|
|
835
|
-
}
|
|
836
|
-
}
|
|
838
|
+
// The durable gate already snapshotted warm `.v8` shards into the
|
|
839
|
+
// run-scoped store for scope resolution.
|
|
837
840
|
await applyChunkResults(chunkWorkerData, chunkIdx, chunkFiles, chunkStartMs);
|
|
838
841
|
}
|
|
839
842
|
else {
|
|
@@ -17,6 +17,9 @@ import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js';
|
|
|
17
17
|
*
|
|
18
18
|
* @param phases All phases to execute (order doesn't matter — sorted internally)
|
|
19
19
|
* @param ctx Shared pipeline context
|
|
20
|
+
* @param seed Results of phases that already ran against this same context,
|
|
21
|
+
* available to `phases` as dependencies (#3016 deferred derived
|
|
22
|
+
* phases). Included in the returned map.
|
|
20
23
|
* @returns Map of phase name → PhaseResult (all completed phases)
|
|
21
24
|
*/
|
|
22
|
-
export declare function runPipeline(phases: readonly PipelinePhase[], ctx: PipelineContext): Promise<ReadonlyMap<string, PhaseResult<unknown>>>;
|
|
25
|
+
export declare function runPipeline(phases: readonly PipelinePhase[], ctx: PipelineContext, seed?: ReadonlyMap<string, PhaseResult<unknown>>): Promise<ReadonlyMap<string, PhaseResult<unknown>>>;
|
|
@@ -13,22 +13,30 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { isDev } from '../utils/env.js';
|
|
15
15
|
import { logger } from '../../logger.js';
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
* Returns phases in topological execution order.
|
|
19
|
-
*/
|
|
20
|
-
function topologicalSort(phases) {
|
|
21
|
-
const phaseMap = new Map();
|
|
16
|
+
function assertUniquePhaseNames(phases) {
|
|
17
|
+
const seen = new Set();
|
|
22
18
|
for (const phase of phases) {
|
|
23
|
-
if (
|
|
19
|
+
if (seen.has(phase.name)) {
|
|
24
20
|
throw new Error(`Duplicate phase name: '${phase.name}'`);
|
|
25
21
|
}
|
|
26
|
-
|
|
22
|
+
seen.add(phase.name);
|
|
27
23
|
}
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Validate that the phases form a valid dependency graph (no cycles, all deps present).
|
|
27
|
+
* Returns phases in topological execution order.
|
|
28
|
+
*
|
|
29
|
+
* `satisfied` names phases whose results are already available (a deferred
|
|
30
|
+
* follow-up run over the same context, #3016). Their edges are dropped rather
|
|
31
|
+
* than validated, because they are resolved by definition.
|
|
32
|
+
*/
|
|
33
|
+
function topologicalSort(phases, satisfied = new Set()) {
|
|
34
|
+
assertUniquePhaseNames(phases);
|
|
35
|
+
const phaseMap = new Map(phases.map((p) => [p.name, p]));
|
|
28
36
|
// Validate all deps exist
|
|
29
37
|
for (const phase of phases) {
|
|
30
38
|
for (const dep of phase.deps) {
|
|
31
|
-
if (!phaseMap.has(dep)) {
|
|
39
|
+
if (!phaseMap.has(dep) && !satisfied.has(dep)) {
|
|
32
40
|
throw new Error(`Phase '${phase.name}' depends on '${dep}', which is not registered`);
|
|
33
41
|
}
|
|
34
42
|
}
|
|
@@ -37,8 +45,9 @@ function topologicalSort(phases) {
|
|
|
37
45
|
const inDegree = new Map();
|
|
38
46
|
const reverseDeps = new Map();
|
|
39
47
|
for (const phase of phases) {
|
|
40
|
-
|
|
41
|
-
|
|
48
|
+
const pendingDeps = phase.deps.filter((dep) => !satisfied.has(dep));
|
|
49
|
+
inDegree.set(phase.name, pendingDeps.length);
|
|
50
|
+
for (const dep of pendingDeps) {
|
|
42
51
|
let rev = reverseDeps.get(dep);
|
|
43
52
|
if (!rev) {
|
|
44
53
|
rev = [];
|
|
@@ -125,12 +134,23 @@ function findCyclePath(remaining, phaseMap) {
|
|
|
125
134
|
*
|
|
126
135
|
* @param phases All phases to execute (order doesn't matter — sorted internally)
|
|
127
136
|
* @param ctx Shared pipeline context
|
|
137
|
+
* @param seed Results of phases that already ran against this same context,
|
|
138
|
+
* available to `phases` as dependencies (#3016 deferred derived
|
|
139
|
+
* phases). Included in the returned map.
|
|
128
140
|
* @returns Map of phase name → PhaseResult (all completed phases)
|
|
129
141
|
*/
|
|
130
|
-
export async function runPipeline(phases, ctx) {
|
|
142
|
+
export async function runPipeline(phases, ctx, seed) {
|
|
143
|
+
// A seeded phase has already run against this context; re-running it would
|
|
144
|
+
// apply its graph writes a second time. "Already ran" is the whole meaning of
|
|
145
|
+
// the seed, so honour it here rather than making every caller pre-filter.
|
|
146
|
+
const satisfied = new Set(seed?.keys() ?? []);
|
|
131
147
|
let sorted;
|
|
132
148
|
try {
|
|
133
|
-
|
|
149
|
+
// Duplicate names must be rejected on the caller-supplied list *before*
|
|
150
|
+
// seed-filtering. Filtering first would drop a seeded duplicate and let
|
|
151
|
+
// `topologicalSort` see a unique name (#3102).
|
|
152
|
+
assertUniquePhaseNames(phases);
|
|
153
|
+
sorted = topologicalSort(phases.filter((p) => !satisfied.has(p.name)), satisfied);
|
|
134
154
|
}
|
|
135
155
|
catch (err) {
|
|
136
156
|
// Emit a terminal 'error' progress event for graph-validation failures
|
|
@@ -152,7 +172,7 @@ export async function runPipeline(phases, ctx) {
|
|
|
152
172
|
}
|
|
153
173
|
throw err;
|
|
154
174
|
}
|
|
155
|
-
const results = new Map();
|
|
175
|
+
const results = new Map(seed);
|
|
156
176
|
for (const phase of sorted) {
|
|
157
177
|
const start = Date.now();
|
|
158
178
|
if (isDev) {
|
|
@@ -25,6 +25,12 @@ export interface PipelineOptions {
|
|
|
25
25
|
* to retain those nodes under `skipGraphPhases`.
|
|
26
26
|
*/
|
|
27
27
|
skipGraphPhases?: boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Skip only Leiden community detection and process/flow extraction (#3016).
|
|
30
|
+
* MRO/DI still run. Used on warm incremental analyze so persisted
|
|
31
|
+
* Community/Process rows can be kept instead of wipe+rewrite.
|
|
32
|
+
*/
|
|
33
|
+
skipDerivedGraphPhases?: boolean;
|
|
28
34
|
/** Per-advice Spring AOP candidate inspection cap. `0` disables this cap. */
|
|
29
35
|
springAopMaxCandidateInspectionsPerAdvice?: number;
|
|
30
36
|
/** Aggregate Spring AOP candidate inspection cap for one analysis. `0` disables this cap. */
|
|
@@ -61,8 +61,12 @@ export function buildPhaseList(options) {
|
|
|
61
61
|
.register(mroPhase, { enabledWhen: (o) => !o.skipGraphPhases })
|
|
62
62
|
.register(springAopInheritancePhase, { enabledWhen: (o) => !o.skipGraphPhases })
|
|
63
63
|
.register(diPhase, { enabledWhen: (o) => !o.skipGraphPhases })
|
|
64
|
-
.register(communitiesPhase, {
|
|
65
|
-
|
|
64
|
+
.register(communitiesPhase, {
|
|
65
|
+
enabledWhen: (o) => !o.skipGraphPhases && o.skipDerivedGraphPhases !== true,
|
|
66
|
+
})
|
|
67
|
+
.register(processesPhase, {
|
|
68
|
+
enabledWhen: (o) => !o.skipGraphPhases && o.skipDerivedGraphPhases !== true,
|
|
69
|
+
})
|
|
66
70
|
// Normalize a missing options object once here so phase predicates above
|
|
67
71
|
// take a required PipelineOptions and need no `?.` guard (#2080 review S1).
|
|
68
72
|
.build(options ?? {}));
|
|
@@ -91,17 +95,18 @@ export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
|
|
|
91
95
|
graphEmitSink = new GraphEmitSink(graph, options.graphEmitCsvDir);
|
|
92
96
|
}
|
|
93
97
|
const phases = buildPhaseList(options);
|
|
98
|
+
const ctx = {
|
|
99
|
+
repoPath,
|
|
100
|
+
graph: graphEmitSink ?? graph,
|
|
101
|
+
onProgress,
|
|
102
|
+
options,
|
|
103
|
+
pipelineStart,
|
|
104
|
+
graphEmit: graphEmitSink,
|
|
105
|
+
};
|
|
94
106
|
let graphEmitManifest;
|
|
95
107
|
let results;
|
|
96
108
|
try {
|
|
97
|
-
results = await runPipeline(phases,
|
|
98
|
-
repoPath,
|
|
99
|
-
graph: graphEmitSink ?? graph,
|
|
100
|
-
onProgress,
|
|
101
|
-
options,
|
|
102
|
-
pipelineStart,
|
|
103
|
-
graphEmit: graphEmitSink,
|
|
104
|
-
});
|
|
109
|
+
results = await runPipeline(phases, ctx);
|
|
105
110
|
graphEmitManifest = graphEmitSink?.finalize();
|
|
106
111
|
}
|
|
107
112
|
finally {
|
|
@@ -140,7 +145,7 @@ export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
|
|
|
140
145
|
nodesCreated: graph.nodeCount,
|
|
141
146
|
},
|
|
142
147
|
});
|
|
143
|
-
|
|
148
|
+
const result = {
|
|
144
149
|
// The RAW graph, deliberately — NOT `graphEmitSink`. Phases above received
|
|
145
150
|
// the sink so their reads are complete, but `loadGraphToLbug` feeds this to
|
|
146
151
|
// `streamAllCSVsToDisk`, and the sink's complete iterator would then emit
|
|
@@ -162,4 +167,30 @@ export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
|
|
|
162
167
|
pdgEmitManifest,
|
|
163
168
|
propertyInference,
|
|
164
169
|
};
|
|
170
|
+
// #3016: hand back a way to run the derived phases `skipDerivedGraphPhases`
|
|
171
|
+
// held back. Which phases those are is answered by re-asking the registry
|
|
172
|
+
// with only that flag cleared — the one form of the question that stays
|
|
173
|
+
// correct when a different predicate (`skipGraphPhases`) also disables them,
|
|
174
|
+
// since then they are absent for a reason a deferred run cannot fix and the
|
|
175
|
+
// filter yields nothing. The sink guard mirrors the `graph` note above: a
|
|
176
|
+
// streaming run is a full rebuild, which never sets the skip flag, so an
|
|
177
|
+
// active sink here means the two got combined by mistake — and deferred
|
|
178
|
+
// phases writing into a finalized sink would emit past its manifest.
|
|
179
|
+
const deferredDerivedPhases = options?.skipDerivedGraphPhases === true && graphEmitSink === undefined
|
|
180
|
+
? buildPhaseList({ ...options, skipDerivedGraphPhases: false }).filter((p) => (p.name === 'communities' || p.name === 'processes') && !results.has(p.name))
|
|
181
|
+
: [];
|
|
182
|
+
if (deferredDerivedPhases.length > 0) {
|
|
183
|
+
result.runDeferredDerivedPhases = async () => {
|
|
184
|
+
const derived = await runPipeline(deferredDerivedPhases, ctx, results);
|
|
185
|
+
// Presence-checked for the same reason as the block above: a phase the
|
|
186
|
+
// registry filtered out is absent, and `getPhaseOutput` throws on absent.
|
|
187
|
+
if (derived.has('communities')) {
|
|
188
|
+
result.communityResult = getPhaseOutput(derived, 'communities').communityResult;
|
|
189
|
+
}
|
|
190
|
+
if (derived.has('processes')) {
|
|
191
|
+
result.processResult = getPhaseOutput(derived, 'processes').processResult;
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
return result;
|
|
165
196
|
};
|
|
@@ -2469,8 +2469,10 @@ parentPort.on('message', (msg) => {
|
|
|
2469
2469
|
persistDurableParsedFileShardSync(DURABLE_PARSED_FILE_STORAGE_PATH, msg.chunkHash, threadId, seq, accumulated.parsedFiles);
|
|
2470
2470
|
}
|
|
2471
2471
|
if (PARSED_FILE_STORE_STORAGE_PATH) {
|
|
2472
|
-
persistParsedFileShardSync(PARSED_FILE_STORE_STORAGE_PATH, `w${threadId}-${seq}`, accumulated.parsedFiles);
|
|
2473
|
-
|
|
2472
|
+
const wrote = persistParsedFileShardSync(PARSED_FILE_STORE_STORAGE_PATH, `w${threadId}-${seq}`, accumulated.parsedFiles);
|
|
2473
|
+
if (wrote) {
|
|
2474
|
+
accumulated.parsedFiles = [];
|
|
2475
|
+
}
|
|
2474
2476
|
}
|
|
2475
2477
|
}
|
|
2476
2478
|
postResultCloneSafe(accumulated);
|
|
@@ -385,7 +385,66 @@ export declare const DELETE_FILES_CHUNK_SIZE = 200;
|
|
|
385
385
|
*/
|
|
386
386
|
export declare const deleteNodesForFiles: (filePaths: readonly string[], options?: {
|
|
387
387
|
onChunk?: (filesDone: number, filesTotal: number) => void;
|
|
388
|
+
/** When set, only these node tables are DETACH DELETEd (#3016). */
|
|
389
|
+
nodeTables?: readonly string[];
|
|
388
390
|
}) => Promise<void>;
|
|
391
|
+
/**
|
|
392
|
+
* Which of `candidateTables` currently hold at least one row for `filePaths`.
|
|
393
|
+
*
|
|
394
|
+
* The incremental writeback uses this to decide which FTS-backed tables it is
|
|
395
|
+
* about to DML (#3016). It has to be a question about the DB, not about the
|
|
396
|
+
* freshly built graph: an edit that DELETES the last Rust trait in a file
|
|
397
|
+
* leaves no Trait node in the new graph, but the old row is still in the index
|
|
398
|
+
* and still has to be deleted — and its FTS index still has to come down first.
|
|
399
|
+
*/
|
|
400
|
+
export declare const nodeTablesWithRowsForFiles: (filePaths: readonly string[], candidateTables: readonly string[]) => Promise<Set<string>>;
|
|
401
|
+
/**
|
|
402
|
+
* One MEMBER_OF / STEP_IN_PROCESS edge, carrying everything needed to recreate
|
|
403
|
+
* it byte-for-byte: both endpoint labels (so the re-MATCH is label-scoped
|
|
404
|
+
* rather than a scan of every node) and every column of the relationship
|
|
405
|
+
* table, `step` included — process traces order by it (`ORDER BY r.step`), so
|
|
406
|
+
* an edge restored without it silently scrambles the flow it belongs to.
|
|
407
|
+
*/
|
|
408
|
+
export interface DerivedRelSnapshot {
|
|
409
|
+
sourceId: string;
|
|
410
|
+
sourceLabel: string;
|
|
411
|
+
targetId: string;
|
|
412
|
+
targetLabel: string;
|
|
413
|
+
type: string;
|
|
414
|
+
confidence: number;
|
|
415
|
+
reason: string;
|
|
416
|
+
step: number;
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* Capture the MEMBER_OF / STEP_IN_PROCESS / ENTRY_POINT_OF edges owned by `filePaths`, before a
|
|
420
|
+
* surgical incremental write DETACH DELETEs their file-side endpoints (#3016).
|
|
421
|
+
*
|
|
422
|
+
* Only meaningful on the write plan that keeps the persisted Community/Process
|
|
423
|
+
* nodes: those nodes survive the delete, but the edges tying this run's changed
|
|
424
|
+
* files to them do not, and the pipeline did not re-derive them.
|
|
425
|
+
*
|
|
426
|
+
* Both endpoints are matched by an EXPLICIT label — `sourceTables` on one side,
|
|
427
|
+
* the edge type's fixed target table on the other — so the labels come from the
|
|
428
|
+
* query rather than the rows. `labels(n)[0]` over an unlabelled match returns
|
|
429
|
+
* an empty string on this engine, which silently produced a snapshot that
|
|
430
|
+
* restored nothing.
|
|
431
|
+
*
|
|
432
|
+
* Read failures propagate. This runs against a warm index whose derived tables
|
|
433
|
+
* the caller has already established exist, so a failure here is a real fault —
|
|
434
|
+
* and swallowing it would drop the edges silently, which looks identical to a
|
|
435
|
+
* repo that genuinely has no communities.
|
|
436
|
+
*/
|
|
437
|
+
export declare const snapshotDerivedRelsForFiles: (filePaths: readonly string[], sourceTables: readonly string[]) => Promise<DerivedRelSnapshot[]>;
|
|
438
|
+
/**
|
|
439
|
+
* Re-create the edges captured by `snapshotDerivedRelsForFiles`, after the
|
|
440
|
+
* incremental subgraph load has put their file-side endpoints back.
|
|
441
|
+
*
|
|
442
|
+
* Endpoints are matched by label + id, mirroring `fallbackRelationshipInserts`:
|
|
443
|
+
* an unlabelled `MATCH (a), (b)` is a cartesian product over the whole graph
|
|
444
|
+
* and does not finish on a real index. An endpoint the load did not restore
|
|
445
|
+
* simply matches nothing, so the edge is dropped rather than mis-attached.
|
|
446
|
+
*/
|
|
447
|
+
export declare const restoreDerivedRels: (rels: readonly DerivedRelSnapshot[]) => Promise<void>;
|
|
389
448
|
export declare const getEmbeddingTableName: () => string;
|
|
390
449
|
/**
|
|
391
450
|
* Return the distinct repo-relative paths of files that import
|