@optave/codegraph 3.9.1 → 3.9.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +95 -14
- package/dist/ast-analysis/engine.d.ts.map +1 -1
- package/dist/ast-analysis/engine.js +64 -0
- package/dist/ast-analysis/engine.js.map +1 -1
- package/dist/cli/commands/batch.d.ts.map +1 -1
- package/dist/cli/commands/batch.js +5 -17
- package/dist/cli/commands/batch.js.map +1 -1
- package/dist/cli/commands/structure.d.ts.map +1 -1
- package/dist/cli/commands/structure.js +18 -1
- package/dist/cli/commands/structure.js.map +1 -1
- package/dist/db/connection.d.ts +2 -0
- package/dist/db/connection.d.ts.map +1 -1
- package/dist/db/connection.js +2 -2
- package/dist/db/connection.js.map +1 -1
- package/dist/db/index.d.ts +1 -1
- package/dist/db/index.d.ts.map +1 -1
- package/dist/db/index.js +1 -1
- package/dist/db/index.js.map +1 -1
- package/dist/domain/analysis/context.d.ts.map +1 -1
- package/dist/domain/analysis/context.js +5 -15
- package/dist/domain/analysis/context.js.map +1 -1
- package/dist/domain/analysis/dependencies.d.ts +5 -5
- package/dist/domain/analysis/dependencies.d.ts.map +1 -1
- package/dist/domain/analysis/dependencies.js +6 -16
- package/dist/domain/analysis/dependencies.js.map +1 -1
- package/dist/domain/analysis/diff-impact.d.ts +12 -0
- package/dist/domain/analysis/diff-impact.d.ts.map +1 -1
- package/dist/domain/analysis/diff-impact.js +20 -1
- package/dist/domain/analysis/diff-impact.js.map +1 -1
- package/dist/domain/analysis/fn-impact.js +2 -2
- package/dist/domain/analysis/fn-impact.js.map +1 -1
- package/dist/domain/analysis/implementations.d.ts.map +1 -1
- package/dist/domain/analysis/implementations.js +3 -13
- package/dist/domain/analysis/implementations.js.map +1 -1
- package/dist/domain/graph/builder/context.d.ts +4 -0
- package/dist/domain/graph/builder/context.d.ts.map +1 -1
- package/dist/domain/graph/builder/context.js +4 -0
- package/dist/domain/graph/builder/context.js.map +1 -1
- package/dist/domain/graph/builder/native-db-proxy.d.ts +24 -0
- package/dist/domain/graph/builder/native-db-proxy.d.ts.map +1 -0
- package/dist/domain/graph/builder/native-db-proxy.js +91 -0
- package/dist/domain/graph/builder/native-db-proxy.js.map +1 -0
- package/dist/domain/graph/builder/pipeline.d.ts.map +1 -1
- package/dist/domain/graph/builder/pipeline.js +148 -79
- package/dist/domain/graph/builder/pipeline.js.map +1 -1
- package/dist/domain/graph/builder/stages/build-edges.d.ts.map +1 -1
- package/dist/domain/graph/builder/stages/build-edges.js +15 -2
- package/dist/domain/graph/builder/stages/build-edges.js.map +1 -1
- package/dist/domain/graph/builder/stages/build-structure.js +2 -2
- package/dist/domain/graph/builder/stages/build-structure.js.map +1 -1
- package/dist/domain/graph/builder/stages/detect-changes.d.ts.map +1 -1
- package/dist/domain/graph/builder/stages/detect-changes.js +6 -28
- package/dist/domain/graph/builder/stages/detect-changes.js.map +1 -1
- package/dist/domain/graph/builder/stages/finalize.js +1 -1
- package/dist/domain/graph/builder/stages/finalize.js.map +1 -1
- package/dist/domain/graph/builder/stages/insert-nodes.d.ts.map +1 -1
- package/dist/domain/graph/builder/stages/insert-nodes.js +16 -12
- package/dist/domain/graph/builder/stages/insert-nodes.js.map +1 -1
- package/dist/domain/graph/builder/stages/resolve-imports.d.ts.map +1 -1
- package/dist/domain/graph/builder/stages/resolve-imports.js +2 -3
- package/dist/domain/graph/builder/stages/resolve-imports.js.map +1 -1
- package/dist/domain/parser.d.ts.map +1 -1
- package/dist/domain/parser.js +11 -4
- package/dist/domain/parser.js.map +1 -1
- package/dist/domain/queries.d.ts +1 -1
- package/dist/domain/queries.d.ts.map +1 -1
- package/dist/domain/queries.js +1 -1
- package/dist/domain/queries.js.map +1 -1
- package/dist/features/ast.js +2 -2
- package/dist/features/ast.js.map +1 -1
- package/dist/features/audit.d.ts.map +1 -1
- package/dist/features/audit.js +3 -2
- package/dist/features/audit.js.map +1 -1
- package/dist/features/boundaries.d.ts.map +1 -1
- package/dist/features/boundaries.js +3 -5
- package/dist/features/boundaries.js.map +1 -1
- package/dist/features/branch-compare.d.ts.map +1 -1
- package/dist/features/branch-compare.js +2 -1
- package/dist/features/branch-compare.js.map +1 -1
- package/dist/features/cfg.d.ts +1 -1
- package/dist/features/cfg.d.ts.map +1 -1
- package/dist/features/cfg.js +52 -6
- package/dist/features/cfg.js.map +1 -1
- package/dist/features/complexity.d.ts.map +1 -1
- package/dist/features/complexity.js +7 -0
- package/dist/features/complexity.js.map +1 -1
- package/dist/features/flow.d.ts.map +1 -1
- package/dist/features/flow.js +2 -1
- package/dist/features/flow.js.map +1 -1
- package/dist/features/manifesto.d.ts.map +1 -1
- package/dist/features/manifesto.js +15 -1
- package/dist/features/manifesto.js.map +1 -1
- package/dist/infrastructure/config.d.ts +1 -0
- package/dist/infrastructure/config.d.ts.map +1 -1
- package/dist/infrastructure/config.js +1 -0
- package/dist/infrastructure/config.js.map +1 -1
- package/dist/infrastructure/update-check.d.ts +1 -1
- package/dist/infrastructure/update-check.js +3 -3
- package/dist/infrastructure/update-check.js.map +1 -1
- package/dist/presentation/batch.d.ts.map +1 -1
- package/dist/presentation/batch.js +1 -0
- package/dist/presentation/batch.js.map +1 -1
- package/dist/presentation/structure.d.ts +1 -1
- package/dist/presentation/structure.d.ts.map +1 -1
- package/dist/presentation/structure.js +1 -1
- package/dist/presentation/structure.js.map +1 -1
- package/dist/shared/normalize.d.ts +12 -0
- package/dist/shared/normalize.d.ts.map +1 -1
- package/dist/shared/normalize.js +4 -0
- package/dist/shared/normalize.js.map +1 -1
- package/dist/types.d.ts +2 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +7 -7
- package/src/ast-analysis/engine.ts +83 -0
- package/src/cli/commands/batch.ts +5 -26
- package/src/cli/commands/structure.ts +21 -1
- package/src/db/connection.ts +2 -2
- package/src/db/index.ts +2 -0
- package/src/domain/analysis/context.ts +5 -15
- package/src/domain/analysis/dependencies.ts +6 -16
- package/src/domain/analysis/diff-impact.ts +28 -1
- package/src/domain/analysis/fn-impact.ts +2 -2
- package/src/domain/analysis/implementations.ts +3 -13
- package/src/domain/graph/builder/context.ts +4 -0
- package/src/domain/graph/builder/native-db-proxy.ts +104 -0
- package/src/domain/graph/builder/pipeline.ts +171 -84
- package/src/domain/graph/builder/stages/build-edges.ts +15 -2
- package/src/domain/graph/builder/stages/build-structure.ts +2 -2
- package/src/domain/graph/builder/stages/detect-changes.ts +11 -33
- package/src/domain/graph/builder/stages/finalize.ts +1 -1
- package/src/domain/graph/builder/stages/insert-nodes.ts +17 -14
- package/src/domain/graph/builder/stages/resolve-imports.ts +2 -3
- package/src/domain/parser.ts +12 -4
- package/src/domain/queries.ts +1 -1
- package/src/features/ast.ts +2 -2
- package/src/features/audit.ts +3 -2
- package/src/features/boundaries.ts +3 -5
- package/src/features/branch-compare.ts +2 -3
- package/src/features/cfg.ts +51 -6
- package/src/features/complexity.ts +7 -0
- package/src/features/flow.ts +2 -1
- package/src/features/manifesto.ts +15 -1
- package/src/infrastructure/config.ts +1 -0
- package/src/infrastructure/update-check.ts +3 -3
- package/src/presentation/batch.ts +1 -0
- package/src/presentation/structure.ts +2 -2
- package/src/shared/normalize.ts +10 -0
- package/src/types.ts +2 -0
|
@@ -4,14 +4,17 @@
|
|
|
4
4
|
* This is the heart of the builder refactor (ROADMAP 3.9): the monolithic buildGraph()
|
|
5
5
|
* is decomposed into independently testable stages that communicate via PipelineContext.
|
|
6
6
|
*/
|
|
7
|
+
import fs from 'node:fs';
|
|
7
8
|
import path from 'node:path';
|
|
8
9
|
import { performance } from 'node:perf_hooks';
|
|
9
10
|
import {
|
|
11
|
+
acquireAdvisoryLock,
|
|
10
12
|
closeDbPair,
|
|
11
13
|
getBuildMeta,
|
|
12
14
|
initSchema,
|
|
13
15
|
MIGRATIONS,
|
|
14
16
|
openDb,
|
|
17
|
+
releaseAdvisoryLock,
|
|
15
18
|
setBuildMeta,
|
|
16
19
|
} from '../../../db/index.js';
|
|
17
20
|
import { detectWorkspaces, loadConfig } from '../../../infrastructure/config.js';
|
|
@@ -20,11 +23,18 @@ import { loadNative } from '../../../infrastructure/native.js';
|
|
|
20
23
|
import { semverCompare } from '../../../infrastructure/update-check.js';
|
|
21
24
|
import { toErrorMessage } from '../../../shared/errors.js';
|
|
22
25
|
import { CODEGRAPH_VERSION } from '../../../shared/version.js';
|
|
23
|
-
import type {
|
|
26
|
+
import type {
|
|
27
|
+
BetterSqlite3Database,
|
|
28
|
+
BuildGraphOpts,
|
|
29
|
+
BuildResult,
|
|
30
|
+
Definition,
|
|
31
|
+
ExtractorOutput,
|
|
32
|
+
} from '../../../types.js';
|
|
24
33
|
import { getActiveEngine } from '../../parser.js';
|
|
25
34
|
import { setWorkspaces } from '../resolve.js';
|
|
26
35
|
import { PipelineContext } from './context.js';
|
|
27
36
|
import { loadPathAliases } from './helpers.js';
|
|
37
|
+
import { NativeDbProxy } from './native-db-proxy.js';
|
|
28
38
|
import { buildEdges } from './stages/build-edges.js';
|
|
29
39
|
import { buildStructure } from './stages/build-structure.js';
|
|
30
40
|
// Pipeline stages
|
|
@@ -43,29 +53,11 @@ function initializeEngine(ctx: PipelineContext): void {
|
|
|
43
53
|
engine: ctx.opts.engine || 'auto',
|
|
44
54
|
dataflow: ctx.opts.dataflow !== false,
|
|
45
55
|
ast: ctx.opts.ast !== false,
|
|
46
|
-
nativeDb
|
|
47
|
-
//
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
// After native writes, resumeJsDb checkpoints through rusqlite so
|
|
52
|
-
// better-sqlite3 never reads WAL frames from a different SQLite library.
|
|
53
|
-
suspendJsDb: ctx.nativeDb
|
|
54
|
-
? () => {
|
|
55
|
-
ctx.db.pragma('wal_checkpoint(TRUNCATE)');
|
|
56
|
-
}
|
|
57
|
-
: undefined,
|
|
58
|
-
resumeJsDb: ctx.nativeDb
|
|
59
|
-
? () => {
|
|
60
|
-
try {
|
|
61
|
-
ctx.nativeDb?.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
62
|
-
} catch (e) {
|
|
63
|
-
debug(
|
|
64
|
-
`resumeJsDb: WAL checkpoint failed (nativeDb may already be closed): ${toErrorMessage(e)}`,
|
|
65
|
-
);
|
|
66
|
-
}
|
|
67
|
-
}
|
|
68
|
-
: undefined,
|
|
56
|
+
// nativeDb and WAL callbacks are set later when NativeDatabase is opened
|
|
57
|
+
// (deferred to skip overhead on no-op rebuilds).
|
|
58
|
+
nativeDb: undefined,
|
|
59
|
+
suspendJsDb: undefined,
|
|
60
|
+
resumeJsDb: undefined,
|
|
69
61
|
};
|
|
70
62
|
const { name: engineName, version: engineVersion } = getActiveEngine(ctx.engineOpts);
|
|
71
63
|
ctx.engineName = engineName as 'native' | 'wasm';
|
|
@@ -79,11 +71,10 @@ function checkEngineSchemaMismatch(ctx: PipelineContext): void {
|
|
|
79
71
|
ctx.forceFullRebuild = false;
|
|
80
72
|
if (!ctx.incremental) return;
|
|
81
73
|
|
|
82
|
-
//
|
|
83
|
-
//
|
|
84
|
-
|
|
85
|
-
const meta = (key: string): string | null =>
|
|
86
|
-
useNativeDb ? ctx.nativeDb!.getBuildMeta(key) : getBuildMeta(ctx.db, key);
|
|
74
|
+
// NativeDatabase is deferred until after change detection, so always use
|
|
75
|
+
// better-sqlite3 for metadata reads here. Reads are safe — WAL conflicts
|
|
76
|
+
// only arise from concurrent writes.
|
|
77
|
+
const meta = (key: string): string | null => getBuildMeta(ctx.db, key);
|
|
87
78
|
|
|
88
79
|
const prevEngine = meta('engine');
|
|
89
80
|
if (prevEngine && prevEngine !== ctx.engineName) {
|
|
@@ -129,34 +120,42 @@ function loadAliases(ctx: PipelineContext): void {
|
|
|
129
120
|
function setupPipeline(ctx: PipelineContext): void {
|
|
130
121
|
ctx.rootDir = path.resolve(ctx.rootDir);
|
|
131
122
|
ctx.dbPath = path.join(ctx.rootDir, '.codegraph', 'graph.db');
|
|
132
|
-
ctx.db = openDb(ctx.dbPath);
|
|
133
123
|
|
|
134
|
-
//
|
|
135
|
-
// better-sqlite3 (ctx.db) is still always opened — needed for queries and stages
|
|
136
|
-
// that haven't been migrated to rusqlite yet.
|
|
137
|
-
// Skip native DB entirely when user explicitly requested --engine wasm.
|
|
124
|
+
// Detect whether native engine is available.
|
|
138
125
|
const enginePref = ctx.opts.engine || 'auto';
|
|
139
126
|
const native = enginePref !== 'wasm' ? loadNative() : null;
|
|
140
|
-
|
|
127
|
+
ctx.nativeAvailable = !!native?.NativeDatabase;
|
|
128
|
+
|
|
129
|
+
// When native is available, use a NativeDbProxy backed by a single rusqlite
|
|
130
|
+
// connection. This eliminates the dual-connection WAL corruption problem.
|
|
131
|
+
// The Rust orchestrator handles the full pipeline; the proxy is used for any
|
|
132
|
+
// JS post-processing (e.g. structure fallback on large builds).
|
|
133
|
+
if (ctx.nativeAvailable && native?.NativeDatabase) {
|
|
141
134
|
try {
|
|
135
|
+
const dir = path.dirname(ctx.dbPath);
|
|
136
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
137
|
+
acquireAdvisoryLock(ctx.dbPath);
|
|
142
138
|
ctx.nativeDb = native.NativeDatabase.openReadWrite(ctx.dbPath);
|
|
143
139
|
ctx.nativeDb.initSchema();
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
ctx.
|
|
140
|
+
const proxy = new NativeDbProxy(ctx.nativeDb);
|
|
141
|
+
proxy.__lockPath = `${ctx.dbPath}.lock`;
|
|
142
|
+
ctx.db = proxy as unknown as typeof ctx.db;
|
|
143
|
+
ctx.nativeFirstProxy = true;
|
|
147
144
|
} catch (err) {
|
|
148
|
-
warn(`NativeDatabase setup failed, falling back to
|
|
145
|
+
warn(`NativeDatabase setup failed, falling back to better-sqlite3: ${toErrorMessage(err)}`);
|
|
149
146
|
try {
|
|
150
147
|
ctx.nativeDb?.close();
|
|
151
|
-
} catch
|
|
152
|
-
|
|
148
|
+
} catch {
|
|
149
|
+
/* ignore */
|
|
153
150
|
}
|
|
154
151
|
ctx.nativeDb = undefined;
|
|
152
|
+
ctx.nativeFirstProxy = false;
|
|
153
|
+
releaseAdvisoryLock(`${ctx.dbPath}.lock`);
|
|
154
|
+
ctx.db = openDb(ctx.dbPath);
|
|
155
|
+
initSchema(ctx.db);
|
|
155
156
|
}
|
|
156
|
-
// Always run JS initSchema so better-sqlite3 sees the schema —
|
|
157
|
-
// nativeDb is closed during pipeline stages and reopened for analyses.
|
|
158
|
-
initSchema(ctx.db);
|
|
159
157
|
} else {
|
|
158
|
+
ctx.db = openDb(ctx.dbPath);
|
|
160
159
|
initSchema(ctx.db);
|
|
161
160
|
}
|
|
162
161
|
|
|
@@ -267,15 +266,19 @@ interface NativeOrchestratorResult {
|
|
|
267
266
|
structureScope?: string[];
|
|
268
267
|
/** Whether the Rust pipeline handled the structure phase (small-incremental fast path). */
|
|
269
268
|
structureHandled?: boolean;
|
|
269
|
+
/** Whether the Rust pipeline wrote AST/complexity/CFG/dataflow to DB. */
|
|
270
|
+
analysisComplete?: boolean;
|
|
270
271
|
}
|
|
271
272
|
|
|
272
273
|
// ── Native orchestrator helpers ───────────────────────────────────────
|
|
273
274
|
|
|
274
275
|
/** Determine whether the native orchestrator should be skipped. Returns a reason string, or null if it should run. */
|
|
275
276
|
function shouldSkipNativeOrchestrator(ctx: PipelineContext): string | null {
|
|
276
|
-
if (process.env.CODEGRAPH_FORCE_JS_PIPELINE === '1') return 'CODEGRAPH_FORCE_JS_PIPELINE=1';
|
|
277
277
|
if (ctx.forceFullRebuild) return 'forceFullRebuild';
|
|
278
|
-
|
|
278
|
+
// v3.9.0 addon had buggy incremental purge (wrong SQL on analysis tables,
|
|
279
|
+
// scoped removal over-detection). Fixed in v3.9.1 by PR #865. Gate on
|
|
280
|
+
// < 3.9.1 so v3.9.1+ uses the fast Rust orchestrator path.
|
|
281
|
+
const orchestratorBuggy = !!ctx.engineVersion && semverCompare(ctx.engineVersion, '3.9.1') < 0;
|
|
279
282
|
if (orchestratorBuggy) return `buggy addon ${ctx.engineVersion}`;
|
|
280
283
|
if (ctx.engineName !== 'native') return `engine=${ctx.engineName}`;
|
|
281
284
|
return null;
|
|
@@ -452,7 +455,11 @@ async function runPostNativeStructure(
|
|
|
452
455
|
return performance.now() - structureStart;
|
|
453
456
|
}
|
|
454
457
|
|
|
455
|
-
/**
|
|
458
|
+
/**
|
|
459
|
+
* JS fallback for AST/complexity/CFG/dataflow analysis after native orchestrator.
|
|
460
|
+
* Used when the Rust addon doesn't include analysis persistence (older addon
|
|
461
|
+
* version) or when analysis failed on the Rust side.
|
|
462
|
+
*/
|
|
456
463
|
async function runPostNativeAnalysis(
|
|
457
464
|
ctx: PipelineContext,
|
|
458
465
|
allFileSymbols: Map<string, ExtractorOutput>,
|
|
@@ -484,14 +491,31 @@ async function runPostNativeAnalysis(
|
|
|
484
491
|
}
|
|
485
492
|
}
|
|
486
493
|
|
|
494
|
+
// Flush JS WAL pages once so Rust can see them, then no-op callbacks.
|
|
495
|
+
// Previously each feature called wal_checkpoint(TRUNCATE) individually
|
|
496
|
+
// (~68ms each × 3-4 features). One FULL checkpoint suffices.
|
|
497
|
+
if (ctx.nativeDb && ctx.engineOpts) {
|
|
498
|
+
ctx.db.pragma('wal_checkpoint(FULL)');
|
|
499
|
+
ctx.engineOpts.suspendJsDb = () => {};
|
|
500
|
+
ctx.engineOpts.resumeJsDb = () => {};
|
|
501
|
+
}
|
|
502
|
+
|
|
487
503
|
try {
|
|
488
|
-
const { runAnalyses: runAnalysesFn } = await import('../../../ast-analysis/engine.js')
|
|
504
|
+
const { runAnalyses: runAnalysesFn } = (await import('../../../ast-analysis/engine.js')) as {
|
|
505
|
+
runAnalyses: (
|
|
506
|
+
db: BetterSqlite3Database,
|
|
507
|
+
fileSymbols: Map<string, ExtractorOutput>,
|
|
508
|
+
rootDir: string,
|
|
509
|
+
opts: Record<string, unknown>,
|
|
510
|
+
engineOpts?: Record<string, unknown>,
|
|
511
|
+
) => Promise<{ astMs?: number; complexityMs?: number; cfgMs?: number; dataflowMs?: number }>;
|
|
512
|
+
};
|
|
489
513
|
const result = await runAnalysesFn(
|
|
490
514
|
ctx.db,
|
|
491
515
|
analysisFileSymbols,
|
|
492
516
|
ctx.rootDir,
|
|
493
|
-
ctx.opts,
|
|
494
|
-
ctx.engineOpts,
|
|
517
|
+
ctx.opts as Record<string, unknown>,
|
|
518
|
+
ctx.engineOpts as unknown as Record<string, unknown> | undefined,
|
|
495
519
|
);
|
|
496
520
|
timing.astMs = result.astMs ?? 0;
|
|
497
521
|
timing.complexityMs = result.complexityMs ?? 0;
|
|
@@ -501,7 +525,9 @@ async function runPostNativeAnalysis(
|
|
|
501
525
|
warn(`Analysis phases failed after native build: ${toErrorMessage(err)}`);
|
|
502
526
|
}
|
|
503
527
|
|
|
504
|
-
// Close nativeDb after analyses
|
|
528
|
+
// Close nativeDb after analyses — TRUNCATE checkpoint flushes all Rust
|
|
529
|
+
// WAL writes so JS and external readers can see them. Runs once after
|
|
530
|
+
// all analysis features complete (not per-feature).
|
|
505
531
|
if (ctx.nativeDb) {
|
|
506
532
|
try {
|
|
507
533
|
ctx.nativeDb.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
@@ -514,7 +540,11 @@ async function runPostNativeAnalysis(
|
|
|
514
540
|
/* ignore close errors */
|
|
515
541
|
}
|
|
516
542
|
ctx.nativeDb = undefined;
|
|
517
|
-
if (ctx.engineOpts)
|
|
543
|
+
if (ctx.engineOpts) {
|
|
544
|
+
ctx.engineOpts.nativeDb = undefined;
|
|
545
|
+
ctx.engineOpts.suspendJsDb = undefined;
|
|
546
|
+
ctx.engineOpts.resumeJsDb = undefined;
|
|
547
|
+
}
|
|
518
548
|
}
|
|
519
549
|
|
|
520
550
|
return timing;
|
|
@@ -553,6 +583,28 @@ async function tryNativeOrchestrator(
|
|
|
553
583
|
debug(`Skipping native orchestrator: ${skipReason}`);
|
|
554
584
|
return undefined;
|
|
555
585
|
}
|
|
586
|
+
|
|
587
|
+
// In native-first mode, nativeDb is already open from setupPipeline.
|
|
588
|
+
// Otherwise, open it on demand (deferred to skip overhead on no-op rebuilds).
|
|
589
|
+
if (!ctx.nativeDb && ctx.nativeAvailable) {
|
|
590
|
+
const native = loadNative();
|
|
591
|
+
if (native?.NativeDatabase) {
|
|
592
|
+
try {
|
|
593
|
+
ctx.nativeDb = native.NativeDatabase.openReadWrite(ctx.dbPath);
|
|
594
|
+
ctx.nativeDb.initSchema();
|
|
595
|
+
ctx.nativeDb.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
596
|
+
} catch (err) {
|
|
597
|
+
warn(`NativeDatabase setup failed, falling back to JS: ${toErrorMessage(err)}`);
|
|
598
|
+
try {
|
|
599
|
+
ctx.nativeDb?.close();
|
|
600
|
+
} catch (e) {
|
|
601
|
+
debug(`tryNativeOrchestrator: close failed during fallback: ${toErrorMessage(e)}`);
|
|
602
|
+
}
|
|
603
|
+
ctx.nativeDb = undefined;
|
|
604
|
+
}
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
|
|
556
608
|
if (!ctx.nativeDb?.buildGraph) return undefined;
|
|
557
609
|
|
|
558
610
|
const resultJson = ctx.nativeDb.buildGraph(
|
|
@@ -594,29 +646,40 @@ async function tryNativeOrchestrator(
|
|
|
594
646
|
);
|
|
595
647
|
|
|
596
648
|
// ── Post-native structure + analysis ──────────────────────────────
|
|
597
|
-
let analysisTiming = {
|
|
649
|
+
let analysisTiming = {
|
|
650
|
+
astMs: +(p.astMs ?? 0),
|
|
651
|
+
complexityMs: +(p.complexityMs ?? 0),
|
|
652
|
+
cfgMs: +(p.cfgMs ?? 0),
|
|
653
|
+
dataflowMs: +(p.dataflowMs ?? 0),
|
|
654
|
+
};
|
|
598
655
|
let structurePatchMs = 0;
|
|
599
|
-
const needsAnalysis =
|
|
600
|
-
ctx.opts.ast !== false ||
|
|
601
|
-
ctx.opts.complexity !== false ||
|
|
602
|
-
ctx.opts.cfg !== false ||
|
|
603
|
-
ctx.opts.dataflow !== false;
|
|
604
656
|
// Skip JS structure when the Rust pipeline's small-incremental fast path
|
|
605
657
|
// already handled it. For full builds and large incrementals where Rust
|
|
606
658
|
// skipped structure, we must run the JS fallback.
|
|
607
659
|
const needsStructure = !result.structureHandled;
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
660
|
+
// When the Rust addon doesn't include analysis persistence (older addon
|
|
661
|
+
// version or analysis failed), fall back to JS-side analysis.
|
|
662
|
+
const needsAnalysisFallback =
|
|
663
|
+
!result.analysisComplete &&
|
|
664
|
+
(ctx.opts.ast !== false ||
|
|
665
|
+
ctx.opts.complexity !== false ||
|
|
666
|
+
ctx.opts.cfg !== false ||
|
|
667
|
+
ctx.opts.dataflow !== false);
|
|
668
|
+
|
|
669
|
+
if (needsStructure || needsAnalysisFallback) {
|
|
670
|
+
// When analysis fallback is needed, handoff to better-sqlite3 — the
|
|
671
|
+
// analysis engine uses the suspend/resume WAL pattern that requires a
|
|
672
|
+
// real better-sqlite3 connection, not the NativeDbProxy.
|
|
673
|
+
if (needsAnalysisFallback && ctx.nativeFirstProxy) {
|
|
674
|
+
closeNativeDb(ctx, 'pre-analysis-fallback');
|
|
675
|
+
ctx.db = openDb(ctx.dbPath);
|
|
676
|
+
ctx.nativeFirstProxy = false;
|
|
677
|
+
} else if (!ctx.nativeFirstProxy && !handoffWalAfterNativeBuild(ctx)) {
|
|
611
678
|
// DB reopen failed — return partial result
|
|
612
679
|
return formatNativeTimingResult(p, 0, analysisTiming);
|
|
613
680
|
}
|
|
614
681
|
|
|
615
|
-
|
|
616
|
-
// analysis — no need to load the entire graph from DB. When structure
|
|
617
|
-
// was NOT handled, we need all files to build the complete directory tree.
|
|
618
|
-
const scopeFiles = needsStructure ? undefined : result.changedFiles;
|
|
619
|
-
const fileSymbols = reconstructFileSymbolsFromDb(ctx, scopeFiles);
|
|
682
|
+
const fileSymbols = reconstructFileSymbolsFromDb(ctx);
|
|
620
683
|
|
|
621
684
|
if (needsStructure) {
|
|
622
685
|
structurePatchMs = await runPostNativeStructure(
|
|
@@ -627,7 +690,7 @@ async function tryNativeOrchestrator(
|
|
|
627
690
|
);
|
|
628
691
|
}
|
|
629
692
|
|
|
630
|
-
if (
|
|
693
|
+
if (needsAnalysisFallback) {
|
|
631
694
|
analysisTiming = await runPostNativeAnalysis(ctx, fileSymbols, result.changedFiles);
|
|
632
695
|
}
|
|
633
696
|
}
|
|
@@ -639,14 +702,21 @@ async function tryNativeOrchestrator(
|
|
|
639
702
|
// ── Pipeline stages execution ───────────────────────────────────────────
|
|
640
703
|
|
|
641
704
|
async function runPipelineStages(ctx: PipelineContext): Promise<void> {
|
|
642
|
-
//
|
|
643
|
-
//
|
|
644
|
-
//
|
|
645
|
-
//
|
|
646
|
-
//
|
|
647
|
-
|
|
705
|
+
// ── WASM / fallback dual-connection mode ─────────────────────────────
|
|
706
|
+
// NativeDatabase is deferred — not opened during setup. collectFiles and
|
|
707
|
+
// detectChanges only need better-sqlite3. If no files changed, we exit
|
|
708
|
+
// early without ever opening the native connection, saving ~5ms.
|
|
709
|
+
// If nativeDb was opened by tryNativeOrchestrator (which fell through),
|
|
710
|
+
// suspend it now to avoid dual-connection WAL corruption during stages.
|
|
648
711
|
if (ctx.db && ctx.nativeDb) {
|
|
649
712
|
suspendNativeDb(ctx, 'pre-collect');
|
|
713
|
+
// When nativeFirstProxy is true, ctx.db is a NativeDbProxy wrapping the
|
|
714
|
+
// now-closed NativeDatabase. Replace it with a real better-sqlite3
|
|
715
|
+
// connection so the JS pipeline stages can operate normally.
|
|
716
|
+
if (ctx.nativeFirstProxy) {
|
|
717
|
+
ctx.db = openDb(ctx.dbPath);
|
|
718
|
+
ctx.nativeFirstProxy = false;
|
|
719
|
+
}
|
|
650
720
|
}
|
|
651
721
|
|
|
652
722
|
await collectFiles(ctx);
|
|
@@ -656,10 +726,13 @@ async function runPipelineStages(ctx: PipelineContext): Promise<void> {
|
|
|
656
726
|
|
|
657
727
|
await parseFiles(ctx);
|
|
658
728
|
|
|
659
|
-
//
|
|
660
|
-
//
|
|
661
|
-
//
|
|
662
|
-
|
|
729
|
+
// For small incremental builds (≤smallFilesThreshold files), skip the nativeDb open/close
|
|
730
|
+
// cycle for insertNodes — the WAL checkpoint + connection churn (~5-10ms)
|
|
731
|
+
// exceeds the napi bulk-insert savings on a handful of files. The JS
|
|
732
|
+
// fallback path inside insertNodes handles this case efficiently.
|
|
733
|
+
const smallIncremental =
|
|
734
|
+
!ctx.isFullBuild && ctx.allSymbols.size <= ctx.config.build.smallFilesThreshold;
|
|
735
|
+
if (ctx.nativeAvailable && ctx.engineName === 'native' && !smallIncremental) {
|
|
663
736
|
reopenNativeDb(ctx, 'insertNodes');
|
|
664
737
|
}
|
|
665
738
|
|
|
@@ -675,25 +748,39 @@ async function runPipelineStages(ctx: PipelineContext): Promise<void> {
|
|
|
675
748
|
await buildEdges(ctx);
|
|
676
749
|
await buildStructure(ctx);
|
|
677
750
|
|
|
678
|
-
// Reopen nativeDb for feature modules (ast, cfg, complexity, dataflow)
|
|
679
|
-
//
|
|
680
|
-
|
|
751
|
+
// Reopen nativeDb for feature modules (ast, cfg, complexity, dataflow).
|
|
752
|
+
// Skip for small incremental builds — same rationale as insertNodes above.
|
|
753
|
+
//
|
|
754
|
+
// Perf: do ONE upfront FULL checkpoint to flush JS WAL pages so Rust
|
|
755
|
+
// can see the latest rows, then make suspendJsDb/resumeJsDb no-ops.
|
|
756
|
+
// Previously each feature called wal_checkpoint(TRUNCATE) individually
|
|
757
|
+
// (~68ms each × 3-4 features = ~200-270ms overhead on incremental builds).
|
|
758
|
+
if (ctx.nativeAvailable && !smallIncremental) {
|
|
681
759
|
reopenNativeDb(ctx, 'analyses');
|
|
682
760
|
if (ctx.nativeDb && ctx.engineOpts) {
|
|
761
|
+
ctx.db.pragma('wal_checkpoint(FULL)');
|
|
683
762
|
ctx.engineOpts.nativeDb = ctx.nativeDb;
|
|
763
|
+
ctx.engineOpts.suspendJsDb = () => {};
|
|
764
|
+
ctx.engineOpts.resumeJsDb = () => {};
|
|
684
765
|
}
|
|
685
766
|
if (!ctx.nativeDb && ctx.engineOpts) {
|
|
686
767
|
ctx.engineOpts.nativeDb = undefined;
|
|
768
|
+
ctx.engineOpts.suspendJsDb = undefined;
|
|
769
|
+
ctx.engineOpts.resumeJsDb = undefined;
|
|
687
770
|
}
|
|
688
771
|
}
|
|
689
772
|
|
|
690
773
|
await runAnalyses(ctx);
|
|
691
774
|
|
|
692
|
-
//
|
|
693
|
-
//
|
|
694
|
-
//
|
|
695
|
-
// valid page cache in case finalize falls back to JS paths (#751).
|
|
775
|
+
// Flush Rust WAL writes (AST, complexity, CFG, dataflow) so the JS
|
|
776
|
+
// connection and any post-build readers can see them. One TRUNCATE
|
|
777
|
+
// here replaces the N per-feature resumeJsDb checkpoints (#checkpoint-opt).
|
|
696
778
|
if (ctx.nativeDb) {
|
|
779
|
+
try {
|
|
780
|
+
ctx.nativeDb.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
781
|
+
} catch (e) {
|
|
782
|
+
debug(`post-analyses WAL checkpoint failed: ${toErrorMessage(e)}`);
|
|
783
|
+
}
|
|
697
784
|
refreshJsDb(ctx);
|
|
698
785
|
}
|
|
699
786
|
|
|
@@ -342,7 +342,7 @@ function buildCallEdgesNative(
|
|
|
342
342
|
if (!fileNodeRow) continue;
|
|
343
343
|
|
|
344
344
|
const importedNames = buildImportedNamesForNative(ctx, relPath, symbols, rootDir);
|
|
345
|
-
const
|
|
345
|
+
const typeMapRaw: Array<{ name: string; typeName: string; confidence: number }> =
|
|
346
346
|
symbols.typeMap instanceof Map
|
|
347
347
|
? [...symbols.typeMap.entries()].map(([name, entry]) => ({
|
|
348
348
|
name,
|
|
@@ -352,6 +352,19 @@ function buildCallEdgesNative(
|
|
|
352
352
|
: Array.isArray(symbols.typeMap)
|
|
353
353
|
? (symbols.typeMap as Array<{ name: string; typeName: string; confidence: number }>)
|
|
354
354
|
: [];
|
|
355
|
+
// Deduplicate: keep highest-confidence entry per name (first-wins on tie),
|
|
356
|
+
// matching JS setTypeMapEntry semantics. The Map branch is already
|
|
357
|
+
// deduped by setTypeMapEntry — this loop is only needed for the Array
|
|
358
|
+
// branch (pre-rebuilt native addon) but runs unconditionally as
|
|
359
|
+
// belt-and-suspenders since it's a cheap O(n) pass.
|
|
360
|
+
const typeMapDedup = new Map<string, { name: string; typeName: string; confidence: number }>();
|
|
361
|
+
for (const entry of typeMapRaw) {
|
|
362
|
+
const existing = typeMapDedup.get(entry.name);
|
|
363
|
+
if (!existing || entry.confidence > existing.confidence) {
|
|
364
|
+
typeMapDedup.set(entry.name, entry);
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
const typeMap = [...typeMapDedup.values()];
|
|
355
368
|
nativeFiles.push({
|
|
356
369
|
file: relPath,
|
|
357
370
|
fileNodeId: fileNodeRow.id,
|
|
@@ -699,7 +712,7 @@ function loadNodes(ctx: PipelineContext): { rows: QueryNodeRow[]; scoped: boolea
|
|
|
699
712
|
const nodeKindFilter = `kind IN ('function','method','class','interface','struct','type','module','enum','trait','record','constant')`;
|
|
700
713
|
|
|
701
714
|
// Gate: only scope for small incremental on large codebases
|
|
702
|
-
if (!isFullBuild && fileSymbols.size <=
|
|
715
|
+
if (!isFullBuild && fileSymbols.size <= ctx.config.build.smallFilesThreshold) {
|
|
703
716
|
const existingFileCount = (
|
|
704
717
|
db.prepare("SELECT COUNT(*) as c FROM nodes WHERE kind = 'file'").get() as { c: number }
|
|
705
718
|
).c;
|
|
@@ -37,7 +37,7 @@ export async function buildStructure(ctx: PipelineContext): Promise<void> {
|
|
|
37
37
|
// For small incremental builds on large codebases, use a fast path that
|
|
38
38
|
// updates only the changed files' metrics via targeted SQL instead of
|
|
39
39
|
// loading ALL definitions from DB (~8ms) and recomputing ALL metrics (~15ms).
|
|
40
|
-
// Gate: ≤
|
|
40
|
+
// Gate: ≤smallFilesThreshold changed files AND significantly more existing files (>20) to
|
|
41
41
|
// avoid triggering on small test fixtures where directory metrics matter.
|
|
42
42
|
const useNativeReads = ctx.engineName === 'native' && !!ctx.nativeDb;
|
|
43
43
|
const existingFileCount = !isFullBuild
|
|
@@ -52,7 +52,7 @@ export async function buildStructure(ctx: PipelineContext): Promise<void> {
|
|
|
52
52
|
const useSmallIncrementalFastPath =
|
|
53
53
|
!isFullBuild &&
|
|
54
54
|
changedFileList != null &&
|
|
55
|
-
changedFileList.length <=
|
|
55
|
+
changedFileList.length <= ctx.config.build.smallFilesThreshold &&
|
|
56
56
|
existingFileCount > 20;
|
|
57
57
|
|
|
58
58
|
if (!isFullBuild && !useSmallIncrementalFastPath) {
|
|
@@ -58,26 +58,9 @@ function getChangedFiles(
|
|
|
58
58
|
db: BetterSqlite3Database,
|
|
59
59
|
allFiles: string[],
|
|
60
60
|
rootDir: string,
|
|
61
|
-
nativeDb?: NativeDatabase,
|
|
62
61
|
): ChangeResult {
|
|
63
|
-
//
|
|
64
|
-
|
|
65
|
-
const data = nativeDb.getFileHashData();
|
|
66
|
-
if (!data.exists) {
|
|
67
|
-
return {
|
|
68
|
-
changed: allFiles.map((f) => ({ file: f })),
|
|
69
|
-
removed: [],
|
|
70
|
-
isFullBuild: true,
|
|
71
|
-
};
|
|
72
|
-
}
|
|
73
|
-
const existing = new Map<string, FileHashRow>(data.rows.map((r) => [r.file, r]));
|
|
74
|
-
const removed = detectRemovedFiles(existing, allFiles, rootDir);
|
|
75
|
-
const journalResult = tryJournalTier(db, existing, rootDir, removed, data.maxMtime);
|
|
76
|
-
if (journalResult) return journalResult;
|
|
77
|
-
return mtimeAndHashTiers(existing, allFiles, rootDir, removed);
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
// WASM / fallback path
|
|
62
|
+
// NativeDatabase is not open during change detection (deferred to after
|
|
63
|
+
// early-exit check). All queries use better-sqlite3 here.
|
|
81
64
|
let hasTable = false;
|
|
82
65
|
try {
|
|
83
66
|
db.prepare('SELECT 1 FROM file_hashes LIMIT 1').get();
|
|
@@ -294,14 +277,14 @@ async function runPendingAnalysis(ctx: PipelineContext): Promise<boolean> {
|
|
|
294
277
|
rootDir,
|
|
295
278
|
analysisOpts,
|
|
296
279
|
);
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
280
|
+
const { runAnalyses } = await import('../../../../ast-analysis/engine.js');
|
|
281
|
+
await runAnalyses(
|
|
282
|
+
db,
|
|
283
|
+
analysisSymbols,
|
|
284
|
+
rootDir,
|
|
285
|
+
{ ast: false, complexity: false, cfg: needsCfg, dataflow: needsDataflow },
|
|
286
|
+
engineOpts,
|
|
287
|
+
);
|
|
305
288
|
return true;
|
|
306
289
|
}
|
|
307
290
|
|
|
@@ -487,12 +470,7 @@ export async function detectChanges(ctx: PipelineContext): Promise<void> {
|
|
|
487
470
|
}
|
|
488
471
|
const increResult =
|
|
489
472
|
incremental && !forceFullRebuild
|
|
490
|
-
? getChangedFiles(
|
|
491
|
-
db,
|
|
492
|
-
allFiles,
|
|
493
|
-
rootDir,
|
|
494
|
-
ctx.engineName === 'native' ? ctx.nativeDb : undefined,
|
|
495
|
-
)
|
|
473
|
+
? getChangedFiles(db, allFiles, rootDir)
|
|
496
474
|
: {
|
|
497
475
|
changed: allFiles.map((f): ChangedFile => ({ file: f })),
|
|
498
476
|
removed: [] as string[],
|
|
@@ -258,7 +258,7 @@ export async function finalize(ctx: PipelineContext): Promise<void> {
|
|
|
258
258
|
// immediately after build.
|
|
259
259
|
const pair = { db: ctx.db, nativeDb: ctx.nativeDb };
|
|
260
260
|
const isTempDir = path.resolve(rootDir).startsWith(path.resolve(tmpdir()));
|
|
261
|
-
if (!isFullBuild && allSymbols.size <=
|
|
261
|
+
if (!isFullBuild && allSymbols.size <= ctx.config.build.smallFilesThreshold && !isTempDir) {
|
|
262
262
|
closeDbPairDeferred(pair);
|
|
263
263
|
} else {
|
|
264
264
|
closeDbPair(pair);
|
|
@@ -159,23 +159,26 @@ function tryNativeInsert(ctx: PipelineContext): boolean {
|
|
|
159
159
|
}
|
|
160
160
|
const fileHashes = buildFileHashes(allSymbols, precomputedData, metadataUpdates, rootDir);
|
|
161
161
|
|
|
162
|
-
//
|
|
163
|
-
//
|
|
164
|
-
//
|
|
165
|
-
// written by the other (#696, #709, #715, #717).
|
|
162
|
+
// In native-first mode (single rusqlite connection), no WAL dance is needed.
|
|
163
|
+
// In dual-connection mode, checkpoint JS side before native write, then
|
|
164
|
+
// checkpoint native side after (#696, #709, #715, #717).
|
|
166
165
|
let result: boolean;
|
|
167
|
-
|
|
168
|
-
if (ctx.db) {
|
|
169
|
-
ctx.db.pragma('wal_checkpoint(TRUNCATE)');
|
|
170
|
-
}
|
|
166
|
+
if (ctx.nativeFirstProxy) {
|
|
171
167
|
result = ctx.nativeDb!.bulkInsertNodes(batches, fileHashes, removed);
|
|
172
|
-
}
|
|
168
|
+
} else {
|
|
173
169
|
try {
|
|
174
|
-
ctx.
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
170
|
+
if (ctx.db) {
|
|
171
|
+
ctx.db.pragma('wal_checkpoint(TRUNCATE)');
|
|
172
|
+
}
|
|
173
|
+
result = ctx.nativeDb!.bulkInsertNodes(batches, fileHashes, removed);
|
|
174
|
+
} finally {
|
|
175
|
+
try {
|
|
176
|
+
ctx.nativeDb?.exec('PRAGMA wal_checkpoint(TRUNCATE)');
|
|
177
|
+
} catch (e) {
|
|
178
|
+
debug(
|
|
179
|
+
`tryNativeInsert: WAL checkpoint failed (nativeDb may already be closed): ${toErrorMessage(e)}`,
|
|
180
|
+
);
|
|
181
|
+
}
|
|
179
182
|
}
|
|
180
183
|
}
|
|
181
184
|
return result;
|
|
@@ -34,15 +34,14 @@ function buildReexportMap(ctx: PipelineContext): void {
|
|
|
34
34
|
|
|
35
35
|
/**
|
|
36
36
|
* Find barrel files related to changed files for scoped re-parsing.
|
|
37
|
-
* For small incremental builds (<=
|
|
37
|
+
* For small incremental builds (<=smallFilesThreshold files), only barrels that re-export from
|
|
38
38
|
* or are imported by the changed files. For larger changes, all barrels.
|
|
39
39
|
*/
|
|
40
40
|
function findBarrelCandidates(ctx: PipelineContext): Array<{ file: string }> {
|
|
41
41
|
const { db, fileSymbols, rootDir, aliases } = ctx;
|
|
42
42
|
const changedRelPaths = new Set<string>(fileSymbols.keys());
|
|
43
43
|
|
|
44
|
-
|
|
45
|
-
if (changedRelPaths.size <= SMALL_CHANGE_THRESHOLD) {
|
|
44
|
+
if (changedRelPaths.size <= ctx.config.build.smallFilesThreshold) {
|
|
46
45
|
const allBarrelFiles = new Set(
|
|
47
46
|
(
|
|
48
47
|
db
|