gitnexus 1.6.11-rc.13 → 1.6.11-rc.14
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/incremental/derived-writeback.d.ts +36 -0
- package/dist/core/incremental/derived-writeback.js +68 -0
- package/dist/core/incremental/subgraph-extract.d.ts +6 -4
- package/dist/core/incremental/subgraph-extract.js +7 -5
- package/dist/core/ingestion/pipeline-phases/runner.d.ts +4 -1
- package/dist/core/ingestion/pipeline-phases/runner.js +34 -14
- package/dist/core/ingestion/pipeline.d.ts +6 -0
- package/dist/core/ingestion/pipeline.js +42 -11
- package/dist/core/lbug/lbug-adapter.d.ts +59 -0
- package/dist/core/lbug/lbug-adapter.js +154 -1
- package/dist/core/run-analyze.js +120 -12
- package/dist/core/search/fts-indexes.d.ts +19 -1
- package/dist/core/search/fts-indexes.js +28 -1
- package/dist/types/pipeline.d.ts +12 -0
- package/package.json +1 -1
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { KnowledgeGraph } from '../graph/types.js';
|
|
2
|
+
import type { FileHashDiff } from '../../storage/file-hash.js';
|
|
3
|
+
/** The FTS-backed members of `tables`. */
|
|
4
|
+
export declare const ftsTablesAmong: (tables: Iterable<string>) => Set<string>;
|
|
5
|
+
/**
|
|
6
|
+
* Whether a surgical incremental write may reuse the persisted derived layer.
|
|
7
|
+
*
|
|
8
|
+
* Deletions disqualify it: the persisted Community/Process rows and their
|
|
9
|
+
* MEMBER_OF / STEP_IN_PROCESS edges can reference nodes that no longer exist
|
|
10
|
+
* after this run, and nothing short of re-deriving can tell which.
|
|
11
|
+
*
|
|
12
|
+
* Added or content-changed files also disqualify it: they can introduce,
|
|
13
|
+
* rename, or retarget symbols and CALLS edges that Leiden and flow extraction
|
|
14
|
+
* consume. File-deletion-only was too weak a proof that the derived graph is
|
|
15
|
+
* still valid.
|
|
16
|
+
*/
|
|
17
|
+
export declare const shouldPreservePersistedDerivedGraph: (diff: Pick<FileHashDiff, "deleted" | "added" | "changed">) => boolean;
|
|
18
|
+
/**
|
|
19
|
+
* FTS-backed node tables that the fresh graph will WRITE rows into for
|
|
20
|
+
* `fileSet` — the inserting half of the DML.
|
|
21
|
+
*
|
|
22
|
+
* Callers must union this with a DB probe for the deleting half
|
|
23
|
+
* (`nodeTablesWithRowsForFiles`): a table whose last row in these files was
|
|
24
|
+
* just removed by the edit has nothing here, but still holds a stale row that
|
|
25
|
+
* the writeback must delete, and deleting it means taking its index down too.
|
|
26
|
+
*/
|
|
27
|
+
export declare const incrementalFtsTablesFromGraph: (graph: KnowledgeGraph, fileSet: ReadonlySet<string>) => Set<string>;
|
|
28
|
+
/**
|
|
29
|
+
* The node tables an incremental DETACH DELETE should target, given the FTS
|
|
30
|
+
* tables this run is rebuilding.
|
|
31
|
+
*
|
|
32
|
+
* Every non-FTS table (Folder, CodeElement, …) deletes as before. An FTS-backed
|
|
33
|
+
* table only deletes when its index is being rebuilt anyway, because deleting
|
|
34
|
+
* from it otherwise would mean DML against a live FTS index (#2589).
|
|
35
|
+
*/
|
|
36
|
+
export declare const nodeTablesForIncrementalDelete: (allNodeTables: readonly string[], rebuildingFtsTables: ReadonlySet<string>) => string[];
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Incremental derived-layer writeback helpers (#3016).
|
|
3
|
+
*
|
|
4
|
+
* The derived layers — Leiden communities, execution flows, and the FTS
|
|
5
|
+
* indexes — are graph-wide, so every analyze run rebuilt all three in full no
|
|
6
|
+
* matter how small the diff. A surgical incremental write can instead:
|
|
7
|
+
* - drop and rebuild only the FTS indexes whose tables hold rows in the
|
|
8
|
+
* write set (LadybugDB still cannot DML a table with a live FTS index —
|
|
9
|
+
* #2589 — so a table being written must still lose its index first);
|
|
10
|
+
* - leave the untouched tables' rows alone, so their indexes stay live;
|
|
11
|
+
* - reuse persisted Community/Process rows only when the file-hash diff is
|
|
12
|
+
* empty (no added, changed, or deleted files). Any content change can
|
|
13
|
+
* add, rename, or retarget symbols that Leiden and flow extraction
|
|
14
|
+
* consume — a no-deletion edit is not a validity proof.
|
|
15
|
+
*/
|
|
16
|
+
import { FTS_INDEXES } from '../search/fts-schema.js';
|
|
17
|
+
const FTS_TABLE_NAMES = new Set(FTS_INDEXES.map((i) => i.table));
|
|
18
|
+
/** The FTS-backed members of `tables`. */
|
|
19
|
+
export const ftsTablesAmong = (tables) => {
|
|
20
|
+
const out = new Set();
|
|
21
|
+
for (const table of tables) {
|
|
22
|
+
if (FTS_TABLE_NAMES.has(table))
|
|
23
|
+
out.add(table);
|
|
24
|
+
}
|
|
25
|
+
return out;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Whether a surgical incremental write may reuse the persisted derived layer.
|
|
29
|
+
*
|
|
30
|
+
* Deletions disqualify it: the persisted Community/Process rows and their
|
|
31
|
+
* MEMBER_OF / STEP_IN_PROCESS edges can reference nodes that no longer exist
|
|
32
|
+
* after this run, and nothing short of re-deriving can tell which.
|
|
33
|
+
*
|
|
34
|
+
* Added or content-changed files also disqualify it: they can introduce,
|
|
35
|
+
* rename, or retarget symbols and CALLS edges that Leiden and flow extraction
|
|
36
|
+
* consume. File-deletion-only was too weak a proof that the derived graph is
|
|
37
|
+
* still valid.
|
|
38
|
+
*/
|
|
39
|
+
export const shouldPreservePersistedDerivedGraph = (diff) => diff.deleted.length === 0 && diff.added.length === 0 && diff.changed.length === 0;
|
|
40
|
+
/**
|
|
41
|
+
* FTS-backed node tables that the fresh graph will WRITE rows into for
|
|
42
|
+
* `fileSet` — the inserting half of the DML.
|
|
43
|
+
*
|
|
44
|
+
* Callers must union this with a DB probe for the deleting half
|
|
45
|
+
* (`nodeTablesWithRowsForFiles`): a table whose last row in these files was
|
|
46
|
+
* just removed by the edit has nothing here, but still holds a stale row that
|
|
47
|
+
* the writeback must delete, and deleting it means taking its index down too.
|
|
48
|
+
*/
|
|
49
|
+
export const incrementalFtsTablesFromGraph = (graph, fileSet) => {
|
|
50
|
+
const touched = new Set();
|
|
51
|
+
graph.forEachNode((n) => {
|
|
52
|
+
const filePath = n.properties?.filePath;
|
|
53
|
+
if (!filePath || !fileSet.has(filePath))
|
|
54
|
+
return;
|
|
55
|
+
if (FTS_TABLE_NAMES.has(n.label))
|
|
56
|
+
touched.add(n.label);
|
|
57
|
+
});
|
|
58
|
+
return touched;
|
|
59
|
+
};
|
|
60
|
+
/**
|
|
61
|
+
* The node tables an incremental DETACH DELETE should target, given the FTS
|
|
62
|
+
* tables this run is rebuilding.
|
|
63
|
+
*
|
|
64
|
+
* Every non-FTS table (Folder, CodeElement, …) deletes as before. An FTS-backed
|
|
65
|
+
* table only deletes when its index is being rebuilt anyway, because deleting
|
|
66
|
+
* from it otherwise would mean DML against a live FTS index (#2589).
|
|
67
|
+
*/
|
|
68
|
+
export const nodeTablesForIncrementalDelete = (allNodeTables, rebuildingFtsTables) => allNodeTables.filter((tableName) => !FTS_TABLE_NAMES.has(tableName) || rebuildingFtsTables.has(tableName));
|
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
* replaced, produce a smaller KnowledgeGraph that contains:
|
|
7
7
|
*
|
|
8
8
|
* - Every node whose `properties.filePath` is in `toWriteSet`.
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* - Graph-wide Community/Process nodes unless `includeDerivedGraphWide`
|
|
10
|
+
* is false (#3016 incremental preserve). Spring metadata placeholders
|
|
11
|
+
* are always included.
|
|
12
12
|
* - Every relationship where AT LEAST ONE endpoint is in the writable
|
|
13
13
|
* set above. Relationships entirely between unchanged-file nodes
|
|
14
14
|
* are skipped — their rows are still in the DB and re-inserting
|
|
@@ -48,7 +48,9 @@
|
|
|
48
48
|
* IMPORTS from the pre-pipeline DB) covers that case instead.
|
|
49
49
|
*/
|
|
50
50
|
import type { KnowledgeGraph } from '../graph/types.js';
|
|
51
|
-
export declare const extractChangedSubgraph: (fullGraph: KnowledgeGraph, toWriteSet: ReadonlySet<string
|
|
51
|
+
export declare const extractChangedSubgraph: (fullGraph: KnowledgeGraph, toWriteSet: ReadonlySet<string>, options?: {
|
|
52
|
+
includeDerivedGraphWide?: boolean;
|
|
53
|
+
}) => KnowledgeGraph;
|
|
52
54
|
/**
|
|
53
55
|
* Public — derive the EFFECTIVE write-set: `toWriteSet` expanded by one
|
|
54
56
|
* hop along every file-owned edge in the new graph that crosses the
|
|
@@ -6,9 +6,9 @@
|
|
|
6
6
|
* replaced, produce a smaller KnowledgeGraph that contains:
|
|
7
7
|
*
|
|
8
8
|
* - Every node whose `properties.filePath` is in `toWriteSet`.
|
|
9
|
-
* -
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* - Graph-wide Community/Process nodes unless `includeDerivedGraphWide`
|
|
10
|
+
* is false (#3016 incremental preserve). Spring metadata placeholders
|
|
11
|
+
* are always included.
|
|
12
12
|
* - Every relationship where AT LEAST ONE endpoint is in the writable
|
|
13
13
|
* set above. Relationships entirely between unchanged-file nodes
|
|
14
14
|
* are skipped — their rows are still in the DB and re-inserting
|
|
@@ -108,12 +108,14 @@ const indexNodeFilePaths = (fullGraph) => {
|
|
|
108
108
|
});
|
|
109
109
|
return idx;
|
|
110
110
|
};
|
|
111
|
-
export const extractChangedSubgraph = (fullGraph, toWriteSet) => {
|
|
111
|
+
export const extractChangedSubgraph = (fullGraph, toWriteSet, options) => {
|
|
112
112
|
const sub = createKnowledgeGraph();
|
|
113
113
|
const writableNodeIds = new Set();
|
|
114
|
+
const includeDerivedGraphWide = options?.includeDerivedGraphWide !== false;
|
|
114
115
|
fullGraph.forEachNode((n) => {
|
|
115
116
|
const filePath = n.properties?.filePath;
|
|
116
|
-
const
|
|
117
|
+
const derivedWide = includeDerivedGraphWide || (n.label !== 'Community' && n.label !== 'Process');
|
|
118
|
+
const include = (filePath && toWriteSet.has(filePath)) || (isGraphWideNode(n) && derivedWide);
|
|
117
119
|
if (include) {
|
|
118
120
|
sub.addNode(n);
|
|
119
121
|
writableNodeIds.add(n.id);
|
|
@@ -17,6 +17,9 @@ import type { PipelinePhase, PipelineContext, PhaseResult } from './types.js';
|
|
|
17
17
|
*
|
|
18
18
|
* @param phases All phases to execute (order doesn't matter — sorted internally)
|
|
19
19
|
* @param ctx Shared pipeline context
|
|
20
|
+
* @param seed Results of phases that already ran against this same context,
|
|
21
|
+
* available to `phases` as dependencies (#3016 deferred derived
|
|
22
|
+
* phases). Included in the returned map.
|
|
20
23
|
* @returns Map of phase name → PhaseResult (all completed phases)
|
|
21
24
|
*/
|
|
22
|
-
export declare function runPipeline(phases: readonly PipelinePhase[], ctx: PipelineContext): Promise<ReadonlyMap<string, PhaseResult<unknown>>>;
|
|
25
|
+
export declare function runPipeline(phases: readonly PipelinePhase[], ctx: PipelineContext, seed?: ReadonlyMap<string, PhaseResult<unknown>>): Promise<ReadonlyMap<string, PhaseResult<unknown>>>;
|
|
@@ -13,22 +13,30 @@
|
|
|
13
13
|
*/
|
|
14
14
|
import { isDev } from '../utils/env.js';
|
|
15
15
|
import { logger } from '../../logger.js';
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
* Returns phases in topological execution order.
|
|
19
|
-
*/
|
|
20
|
-
function topologicalSort(phases) {
|
|
21
|
-
const phaseMap = new Map();
|
|
16
|
+
function assertUniquePhaseNames(phases) {
|
|
17
|
+
const seen = new Set();
|
|
22
18
|
for (const phase of phases) {
|
|
23
|
-
if (
|
|
19
|
+
if (seen.has(phase.name)) {
|
|
24
20
|
throw new Error(`Duplicate phase name: '${phase.name}'`);
|
|
25
21
|
}
|
|
26
|
-
|
|
22
|
+
seen.add(phase.name);
|
|
27
23
|
}
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Validate that the phases form a valid dependency graph (no cycles, all deps present).
|
|
27
|
+
* Returns phases in topological execution order.
|
|
28
|
+
*
|
|
29
|
+
* `satisfied` names phases whose results are already available (a deferred
|
|
30
|
+
* follow-up run over the same context, #3016). Their edges are dropped rather
|
|
31
|
+
* than validated, because they are resolved by definition.
|
|
32
|
+
*/
|
|
33
|
+
function topologicalSort(phases, satisfied = new Set()) {
|
|
34
|
+
assertUniquePhaseNames(phases);
|
|
35
|
+
const phaseMap = new Map(phases.map((p) => [p.name, p]));
|
|
28
36
|
// Validate all deps exist
|
|
29
37
|
for (const phase of phases) {
|
|
30
38
|
for (const dep of phase.deps) {
|
|
31
|
-
if (!phaseMap.has(dep)) {
|
|
39
|
+
if (!phaseMap.has(dep) && !satisfied.has(dep)) {
|
|
32
40
|
throw new Error(`Phase '${phase.name}' depends on '${dep}', which is not registered`);
|
|
33
41
|
}
|
|
34
42
|
}
|
|
@@ -37,8 +45,9 @@ function topologicalSort(phases) {
|
|
|
37
45
|
const inDegree = new Map();
|
|
38
46
|
const reverseDeps = new Map();
|
|
39
47
|
for (const phase of phases) {
|
|
40
|
-
|
|
41
|
-
|
|
48
|
+
const pendingDeps = phase.deps.filter((dep) => !satisfied.has(dep));
|
|
49
|
+
inDegree.set(phase.name, pendingDeps.length);
|
|
50
|
+
for (const dep of pendingDeps) {
|
|
42
51
|
let rev = reverseDeps.get(dep);
|
|
43
52
|
if (!rev) {
|
|
44
53
|
rev = [];
|
|
@@ -125,12 +134,23 @@ function findCyclePath(remaining, phaseMap) {
|
|
|
125
134
|
*
|
|
126
135
|
* @param phases All phases to execute (order doesn't matter — sorted internally)
|
|
127
136
|
* @param ctx Shared pipeline context
|
|
137
|
+
* @param seed Results of phases that already ran against this same context,
|
|
138
|
+
* available to `phases` as dependencies (#3016 deferred derived
|
|
139
|
+
* phases). Included in the returned map.
|
|
128
140
|
* @returns Map of phase name → PhaseResult (all completed phases)
|
|
129
141
|
*/
|
|
130
|
-
export async function runPipeline(phases, ctx) {
|
|
142
|
+
export async function runPipeline(phases, ctx, seed) {
|
|
143
|
+
// A seeded phase has already run against this context; re-running it would
|
|
144
|
+
// apply its graph writes a second time. "Already ran" is the whole meaning of
|
|
145
|
+
// the seed, so honour it here rather than making every caller pre-filter.
|
|
146
|
+
const satisfied = new Set(seed?.keys() ?? []);
|
|
131
147
|
let sorted;
|
|
132
148
|
try {
|
|
133
|
-
|
|
149
|
+
// Duplicate names must be rejected on the caller-supplied list *before*
|
|
150
|
+
// seed-filtering. Filtering first would drop a seeded duplicate and let
|
|
151
|
+
// `topologicalSort` see a unique name (#3102).
|
|
152
|
+
assertUniquePhaseNames(phases);
|
|
153
|
+
sorted = topologicalSort(phases.filter((p) => !satisfied.has(p.name)), satisfied);
|
|
134
154
|
}
|
|
135
155
|
catch (err) {
|
|
136
156
|
// Emit a terminal 'error' progress event for graph-validation failures
|
|
@@ -152,7 +172,7 @@ export async function runPipeline(phases, ctx) {
|
|
|
152
172
|
}
|
|
153
173
|
throw err;
|
|
154
174
|
}
|
|
155
|
-
const results = new Map();
|
|
175
|
+
const results = new Map(seed);
|
|
156
176
|
for (const phase of sorted) {
|
|
157
177
|
const start = Date.now();
|
|
158
178
|
if (isDev) {
|
|
@@ -25,6 +25,12 @@ export interface PipelineOptions {
|
|
|
25
25
|
* to retain those nodes under `skipGraphPhases`.
|
|
26
26
|
*/
|
|
27
27
|
skipGraphPhases?: boolean;
|
|
28
|
+
/**
|
|
29
|
+
* Skip only Leiden community detection and process/flow extraction (#3016).
|
|
30
|
+
* MRO/DI still run. Used on warm incremental analyze so persisted
|
|
31
|
+
* Community/Process rows can be kept instead of wipe+rewrite.
|
|
32
|
+
*/
|
|
33
|
+
skipDerivedGraphPhases?: boolean;
|
|
28
34
|
/** Per-advice Spring AOP candidate inspection cap. `0` disables this cap. */
|
|
29
35
|
springAopMaxCandidateInspectionsPerAdvice?: number;
|
|
30
36
|
/** Aggregate Spring AOP candidate inspection cap for one analysis. `0` disables this cap. */
|
|
@@ -61,8 +61,12 @@ export function buildPhaseList(options) {
|
|
|
61
61
|
.register(mroPhase, { enabledWhen: (o) => !o.skipGraphPhases })
|
|
62
62
|
.register(springAopInheritancePhase, { enabledWhen: (o) => !o.skipGraphPhases })
|
|
63
63
|
.register(diPhase, { enabledWhen: (o) => !o.skipGraphPhases })
|
|
64
|
-
.register(communitiesPhase, {
|
|
65
|
-
|
|
64
|
+
.register(communitiesPhase, {
|
|
65
|
+
enabledWhen: (o) => !o.skipGraphPhases && o.skipDerivedGraphPhases !== true,
|
|
66
|
+
})
|
|
67
|
+
.register(processesPhase, {
|
|
68
|
+
enabledWhen: (o) => !o.skipGraphPhases && o.skipDerivedGraphPhases !== true,
|
|
69
|
+
})
|
|
66
70
|
// Normalize a missing options object once here so phase predicates above
|
|
67
71
|
// take a required PipelineOptions and need no `?.` guard (#2080 review S1).
|
|
68
72
|
.build(options ?? {}));
|
|
@@ -91,17 +95,18 @@ export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
|
|
|
91
95
|
graphEmitSink = new GraphEmitSink(graph, options.graphEmitCsvDir);
|
|
92
96
|
}
|
|
93
97
|
const phases = buildPhaseList(options);
|
|
98
|
+
const ctx = {
|
|
99
|
+
repoPath,
|
|
100
|
+
graph: graphEmitSink ?? graph,
|
|
101
|
+
onProgress,
|
|
102
|
+
options,
|
|
103
|
+
pipelineStart,
|
|
104
|
+
graphEmit: graphEmitSink,
|
|
105
|
+
};
|
|
94
106
|
let graphEmitManifest;
|
|
95
107
|
let results;
|
|
96
108
|
try {
|
|
97
|
-
results = await runPipeline(phases,
|
|
98
|
-
repoPath,
|
|
99
|
-
graph: graphEmitSink ?? graph,
|
|
100
|
-
onProgress,
|
|
101
|
-
options,
|
|
102
|
-
pipelineStart,
|
|
103
|
-
graphEmit: graphEmitSink,
|
|
104
|
-
});
|
|
109
|
+
results = await runPipeline(phases, ctx);
|
|
105
110
|
graphEmitManifest = graphEmitSink?.finalize();
|
|
106
111
|
}
|
|
107
112
|
finally {
|
|
@@ -140,7 +145,7 @@ export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
|
|
|
140
145
|
nodesCreated: graph.nodeCount,
|
|
141
146
|
},
|
|
142
147
|
});
|
|
143
|
-
|
|
148
|
+
const result = {
|
|
144
149
|
// The RAW graph, deliberately — NOT `graphEmitSink`. Phases above received
|
|
145
150
|
// the sink so their reads are complete, but `loadGraphToLbug` feeds this to
|
|
146
151
|
// `streamAllCSVsToDisk`, and the sink's complete iterator would then emit
|
|
@@ -162,4 +167,30 @@ export const runPipelineFromRepo = async (repoPath, onProgress, options) => {
|
|
|
162
167
|
pdgEmitManifest,
|
|
163
168
|
propertyInference,
|
|
164
169
|
};
|
|
170
|
+
// #3016: hand back a way to run the derived phases `skipDerivedGraphPhases`
|
|
171
|
+
// held back. Which phases those are is answered by re-asking the registry
|
|
172
|
+
// with only that flag cleared — the one form of the question that stays
|
|
173
|
+
// correct when a different predicate (`skipGraphPhases`) also disables them,
|
|
174
|
+
// since then they are absent for a reason a deferred run cannot fix and the
|
|
175
|
+
// filter yields nothing. The sink guard mirrors the `graph` note above: a
|
|
176
|
+
// streaming run is a full rebuild, which never sets the skip flag, so an
|
|
177
|
+
// active sink here means the two got combined by mistake — and deferred
|
|
178
|
+
// phases writing into a finalized sink would emit past its manifest.
|
|
179
|
+
const deferredDerivedPhases = options?.skipDerivedGraphPhases === true && graphEmitSink === undefined
|
|
180
|
+
? buildPhaseList({ ...options, skipDerivedGraphPhases: false }).filter((p) => (p.name === 'communities' || p.name === 'processes') && !results.has(p.name))
|
|
181
|
+
: [];
|
|
182
|
+
if (deferredDerivedPhases.length > 0) {
|
|
183
|
+
result.runDeferredDerivedPhases = async () => {
|
|
184
|
+
const derived = await runPipeline(deferredDerivedPhases, ctx, results);
|
|
185
|
+
// Presence-checked for the same reason as the block above: a phase the
|
|
186
|
+
// registry filtered out is absent, and `getPhaseOutput` throws on absent.
|
|
187
|
+
if (derived.has('communities')) {
|
|
188
|
+
result.communityResult = getPhaseOutput(derived, 'communities').communityResult;
|
|
189
|
+
}
|
|
190
|
+
if (derived.has('processes')) {
|
|
191
|
+
result.processResult = getPhaseOutput(derived, 'processes').processResult;
|
|
192
|
+
}
|
|
193
|
+
};
|
|
194
|
+
}
|
|
195
|
+
return result;
|
|
165
196
|
};
|
|
@@ -385,7 +385,66 @@ export declare const DELETE_FILES_CHUNK_SIZE = 200;
|
|
|
385
385
|
*/
|
|
386
386
|
export declare const deleteNodesForFiles: (filePaths: readonly string[], options?: {
|
|
387
387
|
onChunk?: (filesDone: number, filesTotal: number) => void;
|
|
388
|
+
/** When set, only these node tables are DETACH DELETEd (#3016). */
|
|
389
|
+
nodeTables?: readonly string[];
|
|
388
390
|
}) => Promise<void>;
|
|
391
|
+
/**
|
|
392
|
+
* Which of `candidateTables` currently hold at least one row for `filePaths`.
|
|
393
|
+
*
|
|
394
|
+
* The incremental writeback uses this to decide which FTS-backed tables it is
|
|
395
|
+
* about to DML (#3016). It has to be a question about the DB, not about the
|
|
396
|
+
* freshly built graph: an edit that DELETES the last Rust trait in a file
|
|
397
|
+
* leaves no Trait node in the new graph, but the old row is still in the index
|
|
398
|
+
* and still has to be deleted — and its FTS index still has to come down first.
|
|
399
|
+
*/
|
|
400
|
+
export declare const nodeTablesWithRowsForFiles: (filePaths: readonly string[], candidateTables: readonly string[]) => Promise<Set<string>>;
|
|
401
|
+
/**
|
|
402
|
+
* One MEMBER_OF / STEP_IN_PROCESS edge, carrying everything needed to recreate
|
|
403
|
+
* it byte-for-byte: both endpoint labels (so the re-MATCH is label-scoped
|
|
404
|
+
* rather than a scan of every node) and every column of the relationship
|
|
405
|
+
* table, `step` included — process traces order by it (`ORDER BY r.step`), so
|
|
406
|
+
* an edge restored without it silently scrambles the flow it belongs to.
|
|
407
|
+
*/
|
|
408
|
+
export interface DerivedRelSnapshot {
|
|
409
|
+
sourceId: string;
|
|
410
|
+
sourceLabel: string;
|
|
411
|
+
targetId: string;
|
|
412
|
+
targetLabel: string;
|
|
413
|
+
type: string;
|
|
414
|
+
confidence: number;
|
|
415
|
+
reason: string;
|
|
416
|
+
step: number;
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* Capture the MEMBER_OF / STEP_IN_PROCESS / ENTRY_POINT_OF edges owned by `filePaths`, before a
|
|
420
|
+
* surgical incremental write DETACH DELETEs their file-side endpoints (#3016).
|
|
421
|
+
*
|
|
422
|
+
* Only meaningful on the write plan that keeps the persisted Community/Process
|
|
423
|
+
* nodes: those nodes survive the delete, but the edges tying this run's changed
|
|
424
|
+
* files to them do not, and the pipeline did not re-derive them.
|
|
425
|
+
*
|
|
426
|
+
* Both endpoints are matched by an EXPLICIT label — `sourceTables` on one side,
|
|
427
|
+
* the edge type's fixed target table on the other — so the labels come from the
|
|
428
|
+
* query rather than the rows. `labels(n)[0]` over an unlabelled match returns
|
|
429
|
+
* an empty string on this engine, which silently produced a snapshot that
|
|
430
|
+
* restored nothing.
|
|
431
|
+
*
|
|
432
|
+
* Read failures propagate. This runs against a warm index whose derived tables
|
|
433
|
+
* the caller has already established exist, so a failure here is a real fault —
|
|
434
|
+
* and swallowing it would drop the edges silently, which looks identical to a
|
|
435
|
+
* repo that genuinely has no communities.
|
|
436
|
+
*/
|
|
437
|
+
export declare const snapshotDerivedRelsForFiles: (filePaths: readonly string[], sourceTables: readonly string[]) => Promise<DerivedRelSnapshot[]>;
|
|
438
|
+
/**
|
|
439
|
+
* Re-create the edges captured by `snapshotDerivedRelsForFiles`, after the
|
|
440
|
+
* incremental subgraph load has put their file-side endpoints back.
|
|
441
|
+
*
|
|
442
|
+
* Endpoints are matched by label + id, mirroring `fallbackRelationshipInserts`:
|
|
443
|
+
* an unlabelled `MATCH (a), (b)` is a cartesian product over the whole graph
|
|
444
|
+
* and does not finish on a real index. An endpoint the load did not restore
|
|
445
|
+
* simply matches nothing, so the edge is dropped rather than mis-attached.
|
|
446
|
+
*/
|
|
447
|
+
export declare const restoreDerivedRels: (rels: readonly DerivedRelSnapshot[]) => Promise<void>;
|
|
389
448
|
export declare const getEmbeddingTableName: () => string;
|
|
390
449
|
/**
|
|
391
450
|
* Return the distinct repo-relative paths of files that import
|
|
@@ -2303,7 +2303,8 @@ export const deleteNodesForFiles = async (filePaths, options = {}) => {
|
|
|
2303
2303
|
'skipping embedding-row deletes for this writeback.');
|
|
2304
2304
|
}
|
|
2305
2305
|
}
|
|
2306
|
-
|
|
2306
|
+
const tables = options.nodeTables ?? NODE_TABLES;
|
|
2307
|
+
for (const tableName of tables) {
|
|
2307
2308
|
// Community/Process are graph-wide (no filePath); the orchestrator
|
|
2308
2309
|
// drops them wholesale via deleteAllCommunitiesAndProcesses.
|
|
2309
2310
|
if (tableName === 'Community' || tableName === 'Process')
|
|
@@ -2314,6 +2315,158 @@ export const deleteNodesForFiles = async (filePaths, options = {}) => {
|
|
|
2314
2315
|
options.onChunk?.(Math.min((chunkIndex + 1) * DELETE_FILES_CHUNK_SIZE, filePaths.length), filePaths.length);
|
|
2315
2316
|
}
|
|
2316
2317
|
};
|
|
2318
|
+
/**
|
|
2319
|
+
* Which of `candidateTables` currently hold at least one row for `filePaths`.
|
|
2320
|
+
*
|
|
2321
|
+
* The incremental writeback uses this to decide which FTS-backed tables it is
|
|
2322
|
+
* about to DML (#3016). It has to be a question about the DB, not about the
|
|
2323
|
+
* freshly built graph: an edit that DELETES the last Rust trait in a file
|
|
2324
|
+
* leaves no Trait node in the new graph, but the old row is still in the index
|
|
2325
|
+
* and still has to be deleted — and its FTS index still has to come down first.
|
|
2326
|
+
*/
|
|
2327
|
+
export const nodeTablesWithRowsForFiles = async (filePaths, candidateTables) => {
|
|
2328
|
+
const c = conn;
|
|
2329
|
+
if (!c) {
|
|
2330
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
2331
|
+
}
|
|
2332
|
+
const found = new Set();
|
|
2333
|
+
return withConnLock(async () => {
|
|
2334
|
+
for (const batch of chunk(filePaths, DELETE_FILES_CHUNK_SIZE)) {
|
|
2335
|
+
const listLiteral = `[${batch.map((p) => formatCypherValue(p)).join(', ')}]`;
|
|
2336
|
+
for (const tableName of candidateTables) {
|
|
2337
|
+
// Graph-wide tables have no filePath column to filter on.
|
|
2338
|
+
if (tableName === 'Community' || tableName === 'Process')
|
|
2339
|
+
continue;
|
|
2340
|
+
if (found.has(tableName))
|
|
2341
|
+
continue;
|
|
2342
|
+
// determinism: probe — asks only whether the table has any row for
|
|
2343
|
+
// these files, so which row comes back cannot change the answer.
|
|
2344
|
+
const queryResult = await c.query(`MATCH (n:${escapeTableName(tableName)}) WHERE n.filePath IN ${listLiteral} ` +
|
|
2345
|
+
`RETURN n.id LIMIT 1`);
|
|
2346
|
+
try {
|
|
2347
|
+
const result = Array.isArray(queryResult) ? queryResult[0] : queryResult;
|
|
2348
|
+
if ((await result.getAll()).length > 0)
|
|
2349
|
+
found.add(tableName);
|
|
2350
|
+
}
|
|
2351
|
+
finally {
|
|
2352
|
+
await closeQueryResults(queryResult);
|
|
2353
|
+
}
|
|
2354
|
+
}
|
|
2355
|
+
}
|
|
2356
|
+
return found;
|
|
2357
|
+
});
|
|
2358
|
+
};
|
|
2359
|
+
/**
|
|
2360
|
+
* The graph-wide derived edges and the node table each one points at. Both are
|
|
2361
|
+
* produced by the derived phases (Leiden, flow extraction) rather than by
|
|
2362
|
+
* parsing, which is why an incremental run that skips those phases has to carry
|
|
2363
|
+
* them across the writeback itself.
|
|
2364
|
+
*/
|
|
2365
|
+
const DERIVED_REL_KINDS = [
|
|
2366
|
+
{ type: 'MEMBER_OF', targetLabel: 'Community' },
|
|
2367
|
+
{ type: 'STEP_IN_PROCESS', targetLabel: 'Process' },
|
|
2368
|
+
{ type: 'ENTRY_POINT_OF', targetLabel: 'Process' },
|
|
2369
|
+
];
|
|
2370
|
+
/**
|
|
2371
|
+
* Capture the MEMBER_OF / STEP_IN_PROCESS / ENTRY_POINT_OF edges owned by `filePaths`, before a
|
|
2372
|
+
* surgical incremental write DETACH DELETEs their file-side endpoints (#3016).
|
|
2373
|
+
*
|
|
2374
|
+
* Only meaningful on the write plan that keeps the persisted Community/Process
|
|
2375
|
+
* nodes: those nodes survive the delete, but the edges tying this run's changed
|
|
2376
|
+
* files to them do not, and the pipeline did not re-derive them.
|
|
2377
|
+
*
|
|
2378
|
+
* Both endpoints are matched by an EXPLICIT label — `sourceTables` on one side,
|
|
2379
|
+
* the edge type's fixed target table on the other — so the labels come from the
|
|
2380
|
+
* query rather than the rows. `labels(n)[0]` over an unlabelled match returns
|
|
2381
|
+
* an empty string on this engine, which silently produced a snapshot that
|
|
2382
|
+
* restored nothing.
|
|
2383
|
+
*
|
|
2384
|
+
* Read failures propagate. This runs against a warm index whose derived tables
|
|
2385
|
+
* the caller has already established exist, so a failure here is a real fault —
|
|
2386
|
+
* and swallowing it would drop the edges silently, which looks identical to a
|
|
2387
|
+
* repo that genuinely has no communities.
|
|
2388
|
+
*/
|
|
2389
|
+
export const snapshotDerivedRelsForFiles = async (filePaths, sourceTables) => {
|
|
2390
|
+
const c = conn;
|
|
2391
|
+
if (!c) {
|
|
2392
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
2393
|
+
}
|
|
2394
|
+
const out = [];
|
|
2395
|
+
return withConnLock(async () => {
|
|
2396
|
+
for (const batch of chunk(filePaths, DELETE_FILES_CHUNK_SIZE)) {
|
|
2397
|
+
const listLiteral = `[${batch.map((p) => formatCypherValue(p)).join(', ')}]`;
|
|
2398
|
+
for (const sourceLabel of sourceTables) {
|
|
2399
|
+
if (sourceLabel === 'Community' || sourceLabel === 'Process')
|
|
2400
|
+
continue;
|
|
2401
|
+
for (const { type, targetLabel } of DERIVED_REL_KINDS) {
|
|
2402
|
+
const queryResult = await c.query(`MATCH (n:${escapeTableName(sourceLabel)})-[r:${REL_TABLE_NAME}]->` +
|
|
2403
|
+
`(m:${escapeTableName(targetLabel)}) ` +
|
|
2404
|
+
`WHERE n.filePath IN ${listLiteral} AND r.type = ${formatCypherValue(type)} ` +
|
|
2405
|
+
`RETURN n.id AS sourceId, m.id AS targetId, ` +
|
|
2406
|
+
`r.confidence AS confidence, r.reason AS reason, r.step AS step`);
|
|
2407
|
+
try {
|
|
2408
|
+
const result = Array.isArray(queryResult) ? queryResult[0] : queryResult;
|
|
2409
|
+
for (const row of await result.getAll()) {
|
|
2410
|
+
const rec = row;
|
|
2411
|
+
if (typeof rec.sourceId !== 'string' || typeof rec.targetId !== 'string')
|
|
2412
|
+
continue;
|
|
2413
|
+
out.push({
|
|
2414
|
+
sourceId: rec.sourceId,
|
|
2415
|
+
sourceLabel,
|
|
2416
|
+
targetId: rec.targetId,
|
|
2417
|
+
targetLabel,
|
|
2418
|
+
type,
|
|
2419
|
+
confidence: typeof rec.confidence === 'number' ? rec.confidence : 1.0,
|
|
2420
|
+
reason: typeof rec.reason === 'string' ? rec.reason : '',
|
|
2421
|
+
step: typeof rec.step === 'number'
|
|
2422
|
+
? rec.step
|
|
2423
|
+
: typeof rec.step === 'bigint'
|
|
2424
|
+
? Number(rec.step)
|
|
2425
|
+
: 0,
|
|
2426
|
+
});
|
|
2427
|
+
}
|
|
2428
|
+
}
|
|
2429
|
+
finally {
|
|
2430
|
+
await closeQueryResults(queryResult);
|
|
2431
|
+
}
|
|
2432
|
+
}
|
|
2433
|
+
}
|
|
2434
|
+
}
|
|
2435
|
+
return out;
|
|
2436
|
+
});
|
|
2437
|
+
};
|
|
2438
|
+
/**
|
|
2439
|
+
* Re-create the edges captured by `snapshotDerivedRelsForFiles`, after the
|
|
2440
|
+
* incremental subgraph load has put their file-side endpoints back.
|
|
2441
|
+
*
|
|
2442
|
+
* Endpoints are matched by label + id, mirroring `fallbackRelationshipInserts`:
|
|
2443
|
+
* an unlabelled `MATCH (a), (b)` is a cartesian product over the whole graph
|
|
2444
|
+
* and does not finish on a real index. An endpoint the load did not restore
|
|
2445
|
+
* simply matches nothing, so the edge is dropped rather than mis-attached.
|
|
2446
|
+
*/
|
|
2447
|
+
export const restoreDerivedRels = async (rels) => {
|
|
2448
|
+
const c = conn;
|
|
2449
|
+
if (!c) {
|
|
2450
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
2451
|
+
}
|
|
2452
|
+
if (rels.length === 0)
|
|
2453
|
+
return;
|
|
2454
|
+
const escapeLabel = (label) => BACKTICK_TABLES.has(label) ? `\`${label}\`` : label;
|
|
2455
|
+
// No outer `withConnLock`: `queryAndDrain` takes the lock per statement, and
|
|
2456
|
+
// wrapping the loop as well trips the re-entry guard in conn-lock.ts. Same
|
|
2457
|
+
// shape as `fallbackRelationshipInserts`, the other per-edge CREATE loop.
|
|
2458
|
+
for (const rel of rels) {
|
|
2459
|
+
if (!NODE_TABLES.includes(rel.sourceLabel))
|
|
2460
|
+
continue;
|
|
2461
|
+
if (!NODE_TABLES.includes(rel.targetLabel))
|
|
2462
|
+
continue;
|
|
2463
|
+
await queryAndDrain(c, `MATCH (a:${escapeLabel(rel.sourceLabel)} {id: ${formatCypherValue(rel.sourceId)}}), ` +
|
|
2464
|
+
`(b:${escapeLabel(rel.targetLabel)} {id: ${formatCypherValue(rel.targetId)}}) ` +
|
|
2465
|
+
`CREATE (a)-[:${REL_TABLE_NAME} {type: ${formatCypherValue(rel.type)}, ` +
|
|
2466
|
+
`confidence: ${rel.confidence}, reason: ${formatCypherValue(rel.reason)}, ` +
|
|
2467
|
+
`step: ${rel.step}}]->(b)`);
|
|
2468
|
+
}
|
|
2469
|
+
};
|
|
2317
2470
|
export const getEmbeddingTableName = () => EMBEDDING_TABLE_NAME;
|
|
2318
2471
|
/**
|
|
2319
2472
|
* Return the distinct repo-relative paths of files that import
|
package/dist/core/run-analyze.js
CHANGED
|
@@ -21,11 +21,11 @@ import { logUnresolvedReceiverFiles, summarizeUnresolvedReceivers, } from './ing
|
|
|
21
21
|
import { summarizeUndecidedSatisfaction } from './ingestion/scope-resolution/undecided-satisfaction.js';
|
|
22
22
|
import { summarizeScopeExtractionFailures } from './ingestion/scope-resolution/scope-extraction-failures.js';
|
|
23
23
|
import { resetDegradedParseCounter } from './tree-sitter/safe-parse.js';
|
|
24
|
-
import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReusedStatement, closeLbug, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFiles, ensureEmbeddingRowDmlSafe, ensureFtsRowDmlSafe, readIndexCatalogSnapshot, INDEX_CATALOG_UNREADABLE, deleteAllCommunitiesAndProcesses, deleteAllInterprocTaintPaths, deleteAllCallSummaries, deleteAllInjects, deleteAllAdvisedBy, deleteSpringAopEvidenceNodes, deleteSpringAutoConfigurationDeclarations, deleteSpringAutoConfigurationSyntheticClasses, queryImportersBatch, loadFTSExtension, wipeLbugDbFiles, LbugWipeError, DELETE_FILES_CHUNK_SIZE, } from './lbug/lbug-adapter.js';
|
|
24
|
+
import { initLbug, loadGraphToLbug, getLbugStats, executeQuery, executeWithReusedStatement, closeLbug, closeLbugBeforeExit, loadCachedEmbeddings, deleteNodesForFiles, nodeTablesWithRowsForFiles, snapshotDerivedRelsForFiles, restoreDerivedRels, ensureEmbeddingRowDmlSafe, ensureFtsRowDmlSafe, readIndexCatalogSnapshot, INDEX_CATALOG_UNREADABLE, deleteAllCommunitiesAndProcesses, deleteAllInterprocTaintPaths, deleteAllCallSummaries, deleteAllInjects, deleteAllAdvisedBy, deleteSpringAopEvidenceNodes, deleteSpringAutoConfigurationDeclarations, deleteSpringAutoConfigurationSyntheticClasses, queryImportersBatch, loadFTSExtension, wipeLbugDbFiles, LbugWipeError, DELETE_FILES_CHUNK_SIZE, } from './lbug/lbug-adapter.js';
|
|
25
25
|
import { estimateBufferPool, setBufferPoolSizeHint, resolveNativeSafeStorageDir, } from './lbug/lbug-config.js';
|
|
26
26
|
import { escapeCypherString } from './lbug/cypher-escape.js';
|
|
27
27
|
import { chunk } from '../lib/utils.js';
|
|
28
|
-
import { buildSearchIndexesOrDegrade, ftsFailureIsFatal, createSearchFTSIndexes, summarizeFtsIndexBuildFailures, dropSearchFTSIndexes, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js';
|
|
28
|
+
import { buildSearchIndexesOrDegrade, ftsFailureIsFatal, createSearchFTSIndexes, summarizeFtsIndexBuildFailures, dropSearchFTSIndexes, missingSearchFTSIndexTables, initialiseSearchFTSStemmer, verifySearchFTSIndexes, } from './search/fts-indexes.js';
|
|
29
29
|
import { cjkSegmentationModeMismatch, getSearchFTSCjkSegmentation, initialiseSearchFTSCjkSegmentation, } from './search/cjk-segmentation.js';
|
|
30
30
|
import { getExtensionCapability, getExtensionCapabilities, getFtsCapability, resolveAnalyzeInstallPolicy, } from './lbug/extension-loader.js';
|
|
31
31
|
import { diagnoseExtensionLoad } from './lbug/extension-load-error.js';
|
|
@@ -43,6 +43,8 @@ import { computeFileHashes, diffFileHashes } from '../storage/file-hash.js';
|
|
|
43
43
|
import { extractChangedSubgraph, computeEffectiveWriteSet, } from './incremental/subgraph-extract.js';
|
|
44
44
|
import { shadowCandidatesFor } from './incremental/shadow-candidates.js';
|
|
45
45
|
import { shouldEscalateIncrementalWrite } from './incremental/escalation-gate.js';
|
|
46
|
+
import { ftsTablesAmong, incrementalFtsTablesFromGraph, nodeTablesForIncrementalDelete, shouldPreservePersistedDerivedGraph, } from './incremental/derived-writeback.js';
|
|
47
|
+
import { NODE_TABLES } from './lbug/schema.js';
|
|
46
48
|
import { loadParseCache, saveParseCache, pruneCache, PARSE_CACHE_VERSION, } from '../storage/parse-cache.js';
|
|
47
49
|
import { getDurableParsedFileDir, pruneAndSaveDurableParsedFileStore, } from '../storage/parsedfile-store.js';
|
|
48
50
|
import { getCurrentCommit, getCurrentBranch, getRemoteUrl, hasGitDir, getInferredRepoName, isWorkingTreeDirty, listWorkingTreeDirtyPaths, resolveRepoIdentityRoot, } from '../storage/git.js';
|
|
@@ -1306,6 +1308,23 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1306
1308
|
// pipeline has already run by then; that run emits non-streamed, precisely as
|
|
1307
1309
|
// `resolveStreamPdgEmit` — read fresh at the same point — behaves.)
|
|
1308
1310
|
const streamGraphEmitActive = resolveStreamGraphEmit(options);
|
|
1311
|
+
// #3016: hold back Leiden and flow extraction when the persisted metadata
|
|
1312
|
+
// says this run is a candidate for a surgical incremental write, whose
|
|
1313
|
+
// derived layer is reused rather than recomputed. Deliberately the same
|
|
1314
|
+
// conditions as the `isIncremental` decision below MINUS the two that only
|
|
1315
|
+
// the pipeline can answer (the analysis-feature re-check and a non-empty
|
|
1316
|
+
// file list), so this is a superset: every run that turns out incremental
|
|
1317
|
+
// had the phases skipped, and the runs that do not are caught by
|
|
1318
|
+
// `runDeferredDerivedPhases` once the write plan is known. Excluded on the
|
|
1319
|
+
// streaming path because that is a full rebuild by construction, and the
|
|
1320
|
+
// deferred phases must not write into a finalized emit sink.
|
|
1321
|
+
const skipDerivedGraphPhases = !streamGraphEmitActive &&
|
|
1322
|
+
!options.force &&
|
|
1323
|
+
!!existingMeta &&
|
|
1324
|
+
!!existingMeta.fileHashes &&
|
|
1325
|
+
Object.keys(existingMeta.fileHashes).length > 0 &&
|
|
1326
|
+
repoHasGit &&
|
|
1327
|
+
!schemaFingerprintMismatch(existingMeta.schemaFingerprint);
|
|
1309
1328
|
// ── Phase 1: Full Pipeline (0–60%) ────────────────────────────────
|
|
1310
1329
|
const pipelineResult = await runPipelineFromRepo(repoPath, (p) => {
|
|
1311
1330
|
const phaseLabel = PHASE_LABELS[p.phase] || p.phase;
|
|
@@ -1345,6 +1364,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1345
1364
|
? resolveNativeSafeStorageDir(storagePath, 'graph-csv')
|
|
1346
1365
|
: undefined,
|
|
1347
1366
|
fetchWrappers: options.fetchWrappers,
|
|
1367
|
+
skipDerivedGraphPhases,
|
|
1348
1368
|
});
|
|
1349
1369
|
// ── Phase 2: LadybugDB (60–85%) ──────────────────────────────────
|
|
1350
1370
|
progress('lbug', 60, 'Loading into LadybugDB...');
|
|
@@ -1394,6 +1414,26 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1394
1414
|
const hashDiff = isIncremental
|
|
1395
1415
|
? diffFileHashes(newFileHashes, existingMeta.fileHashes)
|
|
1396
1416
|
: undefined;
|
|
1417
|
+
// #3016: `skipDerivedGraphPhases` was decided BEFORE the pipeline, from the
|
|
1418
|
+
// persisted metadata alone, so it can only ever be a bet that this run stays
|
|
1419
|
+
// surgical. Settle the bet here, where `isIncremental` and the deletion set
|
|
1420
|
+
// are both known, and pay it off by running the held-back phases whenever the
|
|
1421
|
+
// write plan needs a freshly derived layer:
|
|
1422
|
+
// - not incremental → full rebuild writes the whole graph, and a graph
|
|
1423
|
+
// with no Community/Process nodes would publish an
|
|
1424
|
+
// index with no communities and no flows;
|
|
1425
|
+
// - added/changed/deleted files → the persisted derived layer can miss new
|
|
1426
|
+
// symbols, keep stale memberships, or reference
|
|
1427
|
+
// removed ids. Only an empty file-hash diff is a
|
|
1428
|
+
// proof that Leiden/flows still match.
|
|
1429
|
+
const preserveDerivedLayer = skipDerivedGraphPhases &&
|
|
1430
|
+
isIncremental &&
|
|
1431
|
+
!!hashDiff &&
|
|
1432
|
+
shouldPreservePersistedDerivedGraph(hashDiff);
|
|
1433
|
+
if (skipDerivedGraphPhases && !preserveDerivedLayer) {
|
|
1434
|
+
progress('communities', 58, 'Detecting code communities and flows...');
|
|
1435
|
+
await pipelineResult.runDeferredDerivedPhases?.();
|
|
1436
|
+
}
|
|
1397
1437
|
// #2 atomic index publish: on a full rebuild, build the fresh DB at a temp
|
|
1398
1438
|
// path and swap it over the live index in one rename at the very end, so a
|
|
1399
1439
|
// concurrent MCP reader opening mid-build only ever sees the previous
|
|
@@ -1592,6 +1632,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1592
1632
|
// collapse check compares the whole in-memory graph against the whole DB,
|
|
1593
1633
|
// which is only a like-for-like comparison on a full rebuild.
|
|
1594
1634
|
let wroteChangedSubgraphOnly = false;
|
|
1635
|
+
let incrementalFtsRebuildTables;
|
|
1595
1636
|
if (isIncremental && hashDiff) {
|
|
1596
1637
|
// ── Incremental DB writeback ───────────────────────────────────
|
|
1597
1638
|
// 0. Expand the writable set with transitive importers of
|
|
@@ -1854,6 +1895,14 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
1854
1895
|
const sizeForcedRebuild = shouldEscalateIncrementalWrite(filesToDelete.length, effectiveWriteSet.size, allFilePaths.length);
|
|
1855
1896
|
if (extensionForcedRebuild || sizeForcedRebuild) {
|
|
1856
1897
|
escalatedFullWrite = true;
|
|
1898
|
+
// #3016: escalation converts this run into a wipe + full bulk COPY of
|
|
1899
|
+
// the in-memory graph, so the derived layer the skip was betting on
|
|
1900
|
+
// preserving has to exist in that graph after all. Same reasoning as
|
|
1901
|
+
// the not-incremental branch above, just discovered later.
|
|
1902
|
+
if (preserveDerivedLayer) {
|
|
1903
|
+
progress('communities', 63, 'Detecting code communities and flows...');
|
|
1904
|
+
await pipelineResult.runDeferredDerivedPhases?.();
|
|
1905
|
+
}
|
|
1857
1906
|
// Every live cause is named, not just the first: a DB can carry BOTH a
|
|
1858
1907
|
// vector index and FTS indexes, and reporting one cause while the other
|
|
1859
1908
|
// is equally fatal is how #2841 stayed mis-diagnosed for so long. §5.D:
|
|
@@ -2054,7 +2103,53 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2054
2103
|
// the snapshot type exists to enforce.
|
|
2055
2104
|
if (buildPath === lbugPath)
|
|
2056
2105
|
liveIndexMutationStarted = true;
|
|
2057
|
-
|
|
2106
|
+
// FTS narrowing is independent of Leiden/flow reuse: even when this
|
|
2107
|
+
// run re-derives communities, Ladybug still cannot DML a live FTS
|
|
2108
|
+
// index (#2589), so only the tables this write set touches should
|
|
2109
|
+
// lose their index. The probe is a question about the DB rather than
|
|
2110
|
+
// the fresh graph — a symbol the edit DELETED is in no fresh graph
|
|
2111
|
+
// but is still a row that has to go.
|
|
2112
|
+
const tablesWithRows = await nodeTablesWithRowsForFiles(filesToDelete, NODE_TABLES);
|
|
2113
|
+
// Narrowing 1 — the FTS sweep, from "every configured index" to "the
|
|
2114
|
+
// indexes this run must touch". Three sources, and dropping any one of
|
|
2115
|
+
// them strands something:
|
|
2116
|
+
// - what the writeback DELETES (the probe above), because a symbol
|
|
2117
|
+
// the edit removed is in no fresh graph but is still a row;
|
|
2118
|
+
// - what it INSERTS (the fresh graph), because inserting under a live
|
|
2119
|
+
// FTS index is the same #2589 hazard as deleting under one;
|
|
2120
|
+
// - what is MISSING right now, because narrowing to the written
|
|
2121
|
+
// tables would otherwise leave keyword search degraded forever on
|
|
2122
|
+
// tables whose index a previous escalation dropped — the next full
|
|
2123
|
+
// rebuild would be the only thing that ever restored them.
|
|
2124
|
+
// An unreadable catalog proves nothing about that third set, so it
|
|
2125
|
+
// withdraws the narrowing entirely rather than guess.
|
|
2126
|
+
const missingFts = await missingSearchFTSIndexTables(indexCatalogRows);
|
|
2127
|
+
const touchedFts = missingFts
|
|
2128
|
+
? new Set([
|
|
2129
|
+
...ftsTablesAmong(tablesWithRows),
|
|
2130
|
+
...incrementalFtsTablesFromGraph(pipelineResult.graph, new Set(filesToDelete)),
|
|
2131
|
+
...missingFts,
|
|
2132
|
+
])
|
|
2133
|
+
: undefined;
|
|
2134
|
+
// Graph-wide Spring synthetic Class nodes are DETACH DELETEd on this
|
|
2135
|
+
// branch even when Class is not in the write set
|
|
2136
|
+
// (`deleteSpringAutoConfigurationSyntheticClasses`). Always include
|
|
2137
|
+
// Class so class_fts is not live across that DML (#2589), including
|
|
2138
|
+
// when the fresh graph no longer materializes the synthetics but the
|
|
2139
|
+
// DB still holds them.
|
|
2140
|
+
if (touchedFts) {
|
|
2141
|
+
touchedFts.add('Class');
|
|
2142
|
+
}
|
|
2143
|
+
incrementalFtsRebuildTables = touchedFts;
|
|
2144
|
+
// MEMBER_OF / STEP_IN_PROCESS / ENTRY_POINT_OF edges hang off the nodes
|
|
2145
|
+
// the DETACH DELETE below removes, so preserving the Community/Process
|
|
2146
|
+
// nodes preserves only half the layer unless these are reattached after
|
|
2147
|
+
// the subgraph write puts the member nodes back. Only the probed tables
|
|
2148
|
+
// can own such an edge, so they are the only ones worth scanning.
|
|
2149
|
+
const derivedSnapshot = preserveDerivedLayer
|
|
2150
|
+
? await snapshotDerivedRelsForFiles(filesToDelete, [...tablesWithRows])
|
|
2151
|
+
: [];
|
|
2152
|
+
await dropSearchFTSIndexes(indexCatalogRows, incrementalFtsRebuildTables);
|
|
2058
2153
|
// 1b. Remove the write set's existing rows — batched (#2409): one
|
|
2059
2154
|
// DETACH DELETE per table per 200-file chunk. The former per-file
|
|
2060
2155
|
// loop issued a count + delete per table per FILE — ~13k
|
|
@@ -2068,6 +2163,9 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2068
2163
|
progress('lbug', 62, `Removing rows for changed files (0/${filesToDelete.length})...`);
|
|
2069
2164
|
await deleteNodesForFiles(filesToDelete, {
|
|
2070
2165
|
onChunk: (done, total) => progress('lbug', 62, `Removing rows for changed files (${done}/${total})...`),
|
|
2166
|
+
nodeTables: incrementalFtsRebuildTables
|
|
2167
|
+
? nodeTablesForIncrementalDelete(NODE_TABLES, incrementalFtsRebuildTables)
|
|
2168
|
+
: undefined,
|
|
2071
2169
|
});
|
|
2072
2170
|
// Surgical path: Phase 3.5 restores exactly these files' embedding
|
|
2073
2171
|
// rows (FIX 3). Sound because deleteNodesForFiles propagates errors
|
|
@@ -2075,10 +2173,12 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2075
2173
|
// deterministically — and this process holds the exclusive DB lock,
|
|
2076
2174
|
// so no concurrent writer can disturb the derivation.
|
|
2077
2175
|
deletedFilePathsForRestore = new Set(filesToDelete);
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2081
|
-
|
|
2176
|
+
if (!preserveDerivedLayer) {
|
|
2177
|
+
// 2. Drop graph-wide nodes (Community, Process). They'll be re-inserted
|
|
2178
|
+
// from the fresh pipeline output below. Required for the
|
|
2179
|
+
// "Leiden runs on the FULL graph" correctness invariant.
|
|
2180
|
+
await deleteAllCommunitiesAndProcesses();
|
|
2181
|
+
}
|
|
2082
2182
|
// 2a. Drop INJECTS edges (DI collection injection, #2200) — their
|
|
2083
2183
|
// validity is a whole-program property (a third-file change to the
|
|
2084
2184
|
// interface or an implementer creates/invalidates edges between two
|
|
@@ -2125,7 +2225,9 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2125
2225
|
// only that. Unchanged-file rows in the DB stay untouched. Pass
|
|
2126
2226
|
// the SAME effectiveWriteSet so the subgraph and the deletes
|
|
2127
2227
|
// cover identical files (asymmetry would silently corrupt).
|
|
2128
|
-
const subgraph = extractChangedSubgraph(pipelineResult.graph, effectiveWriteSet
|
|
2228
|
+
const subgraph = extractChangedSubgraph(pipelineResult.graph, effectiveWriteSet, {
|
|
2229
|
+
includeDerivedGraphWide: !preserveDerivedLayer,
|
|
2230
|
+
});
|
|
2129
2231
|
wroteChangedSubgraphOnly = true;
|
|
2130
2232
|
await saveIncrementalDirtyState('load-graph', {
|
|
2131
2233
|
importerExpansion,
|
|
@@ -2138,6 +2240,9 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2138
2240
|
const pct = Math.min(84, 65 + Math.round((lbugMsgCount / (lbugMsgCount + 10)) * 19));
|
|
2139
2241
|
progress('lbug', pct, msg);
|
|
2140
2242
|
});
|
|
2243
|
+
if (preserveDerivedLayer && derivedSnapshot.length > 0) {
|
|
2244
|
+
await restoreDerivedRels(derivedSnapshot);
|
|
2245
|
+
}
|
|
2141
2246
|
}
|
|
2142
2247
|
// Boundary drain (#2409): checkpoint at the end of the incremental
|
|
2143
2248
|
// writeback so the WAL it accumulated never lingers into the FTS and
|
|
@@ -2189,6 +2294,7 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2189
2294
|
// pre-existing row (#2544/#2546) must not discard this run's otherwise-
|
|
2190
2295
|
// successful graph/embeddings work — only keyword search degrades.
|
|
2191
2296
|
const ftsResult = await buildSearchIndexesOrDegrade(executeQuery, {
|
|
2297
|
+
tables: incrementalFtsRebuildTables,
|
|
2192
2298
|
onIndexStart: options.verbose
|
|
2193
2299
|
? (table, indexName) => log(`FTS: creating ${table}.${indexName}`)
|
|
2194
2300
|
: undefined,
|
|
@@ -2833,8 +2939,9 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
2833
2939
|
files: pipelineResult.totalFileCount,
|
|
2834
2940
|
nodes: stats.nodes,
|
|
2835
2941
|
edges: stats.edges,
|
|
2836
|
-
communities: pipelineResult.communityResult?.stats.totalCommunities
|
|
2837
|
-
|
|
2942
|
+
communities: pipelineResult.communityResult?.stats.totalCommunities ??
|
|
2943
|
+
existingMeta?.stats?.communities,
|
|
2944
|
+
processes: pipelineResult.processResult?.stats.totalProcesses ?? existingMeta?.stats?.processes,
|
|
2838
2945
|
embeddings: persistedEmbeddingCount,
|
|
2839
2946
|
},
|
|
2840
2947
|
capabilities: {
|
|
@@ -3044,9 +3151,10 @@ async function runFullAnalysisInner(repoPath, options, callbacks, writeTarget, r
|
|
|
3044
3151
|
files: pipelineResult.totalFileCount,
|
|
3045
3152
|
nodes: stats.nodes,
|
|
3046
3153
|
edges: stats.edges,
|
|
3047
|
-
communities: pipelineResult.communityResult?.stats.totalCommunities
|
|
3154
|
+
communities: pipelineResult.communityResult?.stats.totalCommunities ??
|
|
3155
|
+
existingMeta?.stats?.communities,
|
|
3048
3156
|
clusters: aggregatedClusterCount,
|
|
3049
|
-
processes: pipelineResult.processResult?.stats.totalProcesses,
|
|
3157
|
+
processes: pipelineResult.processResult?.stats.totalProcesses ?? existingMeta?.stats?.processes,
|
|
3050
3158
|
}, undefined, {
|
|
3051
3159
|
skipAgentsMd: options.skipAgentsMd,
|
|
3052
3160
|
skipSkills: options.skipSkills,
|
|
@@ -59,6 +59,12 @@ export declare const SUPPORTED_FTS_STEMMERS: ReadonlySet<string>;
|
|
|
59
59
|
export interface CreateSearchFTSIndexesOptions {
|
|
60
60
|
onIndexStart?: (table: string, indexName: string) => void;
|
|
61
61
|
onIndexReady?: (table: string, indexName: string) => void;
|
|
62
|
+
/**
|
|
63
|
+
* When set, only these node-table names are dropped/rebuilt (#3016).
|
|
64
|
+
* Omit to rebuild every configured FTS index (full analyze / deleted-file
|
|
65
|
+
* incremental / `--repair-fts`).
|
|
66
|
+
*/
|
|
67
|
+
tables?: ReadonlySet<string>;
|
|
62
68
|
}
|
|
63
69
|
/**
|
|
64
70
|
* Resolve + validate `GITNEXUS_FTS_STEMMER` once, up front at analyze startup,
|
|
@@ -98,7 +104,19 @@ export declare function getSearchFTSStemmer(): string;
|
|
|
98
104
|
* contract, and the same one-shared-`SHOW_INDEXES`-read purpose, as the gates in
|
|
99
105
|
* `lbug-adapter.ts`. Omit it to have the sweep read the catalog itself.
|
|
100
106
|
*/
|
|
101
|
-
export declare function dropSearchFTSIndexes(indexRows?: IndexCatalogSnapshot): Promise<void>;
|
|
107
|
+
export declare function dropSearchFTSIndexes(indexRows?: IndexCatalogSnapshot, tables?: ReadonlySet<string>): Promise<void>;
|
|
108
|
+
/**
|
|
109
|
+
* The configured FTS tables whose index the catalog proves is ABSENT right now.
|
|
110
|
+
*
|
|
111
|
+
* `undefined` means the catalog could not be read, which proves nothing — the
|
|
112
|
+
* same fail-closed reading the sweep above applies. Callers narrowing a rebuild
|
|
113
|
+
* to a subset of tables (#3016) must union this in, or must not narrow at all
|
|
114
|
+
* when it is `undefined`: a run that rebuilds only the tables it wrote leaves
|
|
115
|
+
* keyword search permanently degraded on every table whose index went missing
|
|
116
|
+
* earlier (a prior escalation drops all of them, and only the next full rebuild
|
|
117
|
+
* would ever put them back).
|
|
118
|
+
*/
|
|
119
|
+
export declare function missingSearchFTSIndexTables(indexRows?: IndexCatalogSnapshot): Promise<Set<string> | undefined>;
|
|
102
120
|
/** One configured index that could not be (re)built, and why. */
|
|
103
121
|
export interface FtsIndexBuildFailure {
|
|
104
122
|
table: string;
|
|
@@ -172,7 +172,7 @@ export function getSearchFTSStemmer() {
|
|
|
172
172
|
* contract, and the same one-shared-`SHOW_INDEXES`-read purpose, as the gates in
|
|
173
173
|
* `lbug-adapter.ts`. Omit it to have the sweep read the catalog itself.
|
|
174
174
|
*/
|
|
175
|
-
export async function dropSearchFTSIndexes(indexRows) {
|
|
175
|
+
export async function dropSearchFTSIndexes(indexRows, tables) {
|
|
176
176
|
// One catalog read for the whole sweep, decided PER CONFIGURED INDEX on
|
|
177
177
|
// IDENTITY (#2841 cleanup review). `undefined` = the catalog could not be
|
|
178
178
|
// read, which proves nothing — attempt every drop rather than skip a real one,
|
|
@@ -193,6 +193,8 @@ export async function dropSearchFTSIndexes(indexRows) {
|
|
|
193
193
|
// whether the sweep ran or not.
|
|
194
194
|
const rows = await resolveGateRows(indexRows);
|
|
195
195
|
for (const { table, indexName } of FTS_INDEXES) {
|
|
196
|
+
if (tables && !tables.has(table))
|
|
197
|
+
continue;
|
|
196
198
|
// Skip only what the catalog POSITIVELY proves absent. Without this, a
|
|
197
199
|
// machine whose FTS extension cannot load, analyzing a DB that never carried
|
|
198
200
|
// an FTS index, pays one failed `CALL DROP_FTS_INDEX` per configured table on
|
|
@@ -209,6 +211,29 @@ export async function dropSearchFTSIndexes(indexRows) {
|
|
|
209
211
|
await dropFTSIndex(table, indexName);
|
|
210
212
|
}
|
|
211
213
|
}
|
|
214
|
+
/**
|
|
215
|
+
* The configured FTS tables whose index the catalog proves is ABSENT right now.
|
|
216
|
+
*
|
|
217
|
+
* `undefined` means the catalog could not be read, which proves nothing — the
|
|
218
|
+
* same fail-closed reading the sweep above applies. Callers narrowing a rebuild
|
|
219
|
+
* to a subset of tables (#3016) must union this in, or must not narrow at all
|
|
220
|
+
* when it is `undefined`: a run that rebuilds only the tables it wrote leaves
|
|
221
|
+
* keyword search permanently degraded on every table whose index went missing
|
|
222
|
+
* earlier (a prior escalation drops all of them, and only the next full rebuild
|
|
223
|
+
* would ever put them back).
|
|
224
|
+
*/
|
|
225
|
+
export async function missingSearchFTSIndexTables(indexRows) {
|
|
226
|
+
const rows = await resolveGateRows(indexRows);
|
|
227
|
+
if (rows === undefined)
|
|
228
|
+
return undefined;
|
|
229
|
+
const missing = new Set();
|
|
230
|
+
for (const { table, indexName } of FTS_INDEXES) {
|
|
231
|
+
const present = rows.some((row) => indexRowTable(row) === table && indexRowName(row) === indexName);
|
|
232
|
+
if (!present)
|
|
233
|
+
missing.add(table);
|
|
234
|
+
}
|
|
235
|
+
return missing;
|
|
236
|
+
}
|
|
212
237
|
/**
|
|
213
238
|
* Build every configured FTS index, and keep going when one of them fails
|
|
214
239
|
* (#2889).
|
|
@@ -232,6 +257,8 @@ export async function createSearchFTSIndexes(options) {
|
|
|
232
257
|
const stemmer = getSearchFTSStemmer();
|
|
233
258
|
const failures = [];
|
|
234
259
|
for (const { table, indexName, properties } of FTS_INDEXES) {
|
|
260
|
+
if (options?.tables && !options.tables.has(table))
|
|
261
|
+
continue;
|
|
235
262
|
options?.onIndexStart?.(table, indexName);
|
|
236
263
|
// Drop first so the live `properties` always win. `createFTSIndex` is
|
|
237
264
|
// idempotent-by-name (skips when the index already exists), so without the
|
package/dist/types/pipeline.d.ts
CHANGED
|
@@ -13,6 +13,18 @@ export interface PipelineResult {
|
|
|
13
13
|
totalFileCount: number;
|
|
14
14
|
communityResult?: CommunityDetectionResult;
|
|
15
15
|
processResult?: ProcessDetectionResult;
|
|
16
|
+
/**
|
|
17
|
+
* Runs the community/process phases that `skipDerivedGraphPhases` held back
|
|
18
|
+
* (#3016), against the same graph and phase outputs the pipeline already
|
|
19
|
+
* produced, and populates `communityResult`/`processResult` on this object.
|
|
20
|
+
*
|
|
21
|
+
* Present ONLY when those phases were skipped for that reason, so a caller
|
|
22
|
+
* that optimistically skipped them can still get a byte-identical derived
|
|
23
|
+
* layer on the paths that turn out to need one (full rebuild, escalated
|
|
24
|
+
* write, or an incremental run with deleted files). Absent means the phases
|
|
25
|
+
* either already ran or were disabled for an unrelated reason.
|
|
26
|
+
*/
|
|
27
|
+
runDeferredDerivedPhases?: () => Promise<void>;
|
|
16
28
|
/**
|
|
17
29
|
* Additive diagnostics for registry-primary resolution decisions that
|
|
18
30
|
* deliberately suppress edge emission. Empty means no diagnostic was
|
package/package.json
CHANGED