@optave/codegraph 3.9.4 → 3.9.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -10
- package/dist/ast-analysis/engine.d.ts.map +1 -1
- package/dist/ast-analysis/engine.js +3 -2
- package/dist/ast-analysis/engine.js.map +1 -1
- package/dist/ast-analysis/rules/csharp.d.ts.map +1 -1
- package/dist/ast-analysis/rules/csharp.js +8 -1
- package/dist/ast-analysis/rules/csharp.js.map +1 -1
- package/dist/ast-analysis/rules/go.d.ts.map +1 -1
- package/dist/ast-analysis/rules/go.js +4 -1
- package/dist/ast-analysis/rules/go.js.map +1 -1
- package/dist/ast-analysis/rules/index.d.ts +6 -0
- package/dist/ast-analysis/rules/index.d.ts.map +1 -1
- package/dist/ast-analysis/rules/index.js +151 -4
- package/dist/ast-analysis/rules/index.js.map +1 -1
- package/dist/ast-analysis/rules/java.d.ts.map +1 -1
- package/dist/ast-analysis/rules/java.js +5 -1
- package/dist/ast-analysis/rules/java.js.map +1 -1
- package/dist/ast-analysis/rules/php.d.ts.map +1 -1
- package/dist/ast-analysis/rules/php.js +6 -1
- package/dist/ast-analysis/rules/php.js.map +1 -1
- package/dist/ast-analysis/rules/python.d.ts.map +1 -1
- package/dist/ast-analysis/rules/python.js +5 -1
- package/dist/ast-analysis/rules/python.js.map +1 -1
- package/dist/ast-analysis/rules/ruby.d.ts.map +1 -1
- package/dist/ast-analysis/rules/ruby.js +4 -1
- package/dist/ast-analysis/rules/ruby.js.map +1 -1
- package/dist/ast-analysis/rules/rust.d.ts.map +1 -1
- package/dist/ast-analysis/rules/rust.js +5 -1
- package/dist/ast-analysis/rules/rust.js.map +1 -1
- package/dist/ast-analysis/visitors/ast-store-visitor.d.ts +2 -1
- package/dist/ast-analysis/visitors/ast-store-visitor.d.ts.map +1 -1
- package/dist/ast-analysis/visitors/ast-store-visitor.js +129 -37
- package/dist/ast-analysis/visitors/ast-store-visitor.js.map +1 -1
- package/dist/cli/commands/watch.d.ts.map +1 -1
- package/dist/cli/commands/watch.js +2 -0
- package/dist/cli/commands/watch.js.map +1 -1
- package/dist/cli.js +24 -1
- package/dist/cli.js.map +1 -1
- package/dist/domain/graph/builder/context.d.ts +2 -0
- package/dist/domain/graph/builder/context.d.ts.map +1 -1
- package/dist/domain/graph/builder/context.js.map +1 -1
- package/dist/domain/graph/builder/helpers.d.ts +13 -2
- package/dist/domain/graph/builder/helpers.d.ts.map +1 -1
- package/dist/domain/graph/builder/helpers.js +30 -4
- package/dist/domain/graph/builder/helpers.js.map +1 -1
- package/dist/domain/graph/builder/pipeline.d.ts.map +1 -1
- package/dist/domain/graph/builder/pipeline.js +141 -3
- package/dist/domain/graph/builder/pipeline.js.map +1 -1
- package/dist/domain/graph/builder/stages/collect-files.d.ts.map +1 -1
- package/dist/domain/graph/builder/stages/collect-files.js +58 -26
- package/dist/domain/graph/builder/stages/collect-files.js.map +1 -1
- package/dist/domain/graph/builder/stages/detect-changes.d.ts.map +1 -1
- package/dist/domain/graph/builder/stages/detect-changes.js +54 -45
- package/dist/domain/graph/builder/stages/detect-changes.js.map +1 -1
- package/dist/domain/graph/builder/stages/finalize.d.ts.map +1 -1
- package/dist/domain/graph/builder/stages/finalize.js +17 -0
- package/dist/domain/graph/builder/stages/finalize.js.map +1 -1
- package/dist/domain/graph/journal.d.ts +15 -0
- package/dist/domain/graph/journal.d.ts.map +1 -1
- package/dist/domain/graph/journal.js +283 -28
- package/dist/domain/graph/journal.js.map +1 -1
- package/dist/domain/graph/watcher.d.ts +17 -0
- package/dist/domain/graph/watcher.d.ts.map +1 -1
- package/dist/domain/graph/watcher.js +23 -7
- package/dist/domain/graph/watcher.js.map +1 -1
- package/dist/domain/parser.d.ts +53 -4
- package/dist/domain/parser.d.ts.map +1 -1
- package/dist/domain/parser.js +278 -80
- package/dist/domain/parser.js.map +1 -1
- package/dist/domain/search/generator.d.ts.map +1 -1
- package/dist/domain/search/generator.js +28 -2
- package/dist/domain/search/generator.js.map +1 -1
- package/dist/domain/search/models.js +1 -1
- package/dist/domain/wasm-worker-entry.d.ts +24 -0
- package/dist/domain/wasm-worker-entry.d.ts.map +1 -0
- package/dist/domain/wasm-worker-entry.js +644 -0
- package/dist/domain/wasm-worker-entry.js.map +1 -0
- package/dist/domain/wasm-worker-pool.d.ts +59 -0
- package/dist/domain/wasm-worker-pool.d.ts.map +1 -0
- package/dist/domain/wasm-worker-pool.js +312 -0
- package/dist/domain/wasm-worker-pool.js.map +1 -0
- package/dist/domain/wasm-worker-protocol.d.ts +65 -0
- package/dist/domain/wasm-worker-protocol.d.ts.map +1 -0
- package/dist/domain/wasm-worker-protocol.js +13 -0
- package/dist/domain/wasm-worker-protocol.js.map +1 -0
- package/dist/extractors/javascript.js +146 -2
- package/dist/extractors/javascript.js.map +1 -1
- package/dist/features/ast.d.ts.map +1 -1
- package/dist/features/ast.js +11 -9
- package/dist/features/ast.js.map +1 -1
- package/dist/features/boundaries.d.ts +2 -2
- package/dist/features/boundaries.d.ts.map +1 -1
- package/dist/features/boundaries.js +2 -31
- package/dist/features/boundaries.js.map +1 -1
- package/dist/features/snapshot.d.ts.map +1 -1
- package/dist/features/snapshot.js +99 -13
- package/dist/features/snapshot.js.map +1 -1
- package/dist/graph/algorithms/louvain.d.ts.map +1 -1
- package/dist/graph/algorithms/louvain.js +2 -4
- package/dist/graph/algorithms/louvain.js.map +1 -1
- package/dist/infrastructure/config.d.ts.map +1 -1
- package/dist/infrastructure/config.js +12 -2
- package/dist/infrastructure/config.js.map +1 -1
- package/dist/shared/globs.d.ts +40 -0
- package/dist/shared/globs.d.ts.map +1 -0
- package/dist/shared/globs.js +126 -0
- package/dist/shared/globs.js.map +1 -0
- package/dist/types.d.ts +26 -1
- package/dist/types.d.ts.map +1 -1
- package/grammars/tree-sitter-c_sharp.wasm +0 -0
- package/grammars/tree-sitter-erlang.wasm +0 -0
- package/package.json +7 -7
- package/src/ast-analysis/engine.ts +11 -1
- package/src/ast-analysis/rules/csharp.ts +8 -1
- package/src/ast-analysis/rules/go.ts +4 -1
- package/src/ast-analysis/rules/index.ts +181 -4
- package/src/ast-analysis/rules/java.ts +5 -1
- package/src/ast-analysis/rules/php.ts +6 -1
- package/src/ast-analysis/rules/python.ts +5 -1
- package/src/ast-analysis/rules/ruby.ts +4 -1
- package/src/ast-analysis/rules/rust.ts +5 -1
- package/src/ast-analysis/visitors/ast-store-visitor.ts +129 -34
- package/src/cli/commands/watch.ts +2 -0
- package/src/cli.ts +31 -8
- package/src/domain/graph/builder/context.ts +2 -0
- package/src/domain/graph/builder/helpers.ts +53 -3
- package/src/domain/graph/builder/pipeline.ts +162 -3
- package/src/domain/graph/builder/stages/collect-files.ts +56 -26
- package/src/domain/graph/builder/stages/detect-changes.ts +57 -49
- package/src/domain/graph/builder/stages/finalize.ts +16 -0
- package/src/domain/graph/journal.ts +284 -27
- package/src/domain/graph/watcher.ts +29 -9
- package/src/domain/parser.ts +288 -73
- package/src/domain/search/generator.ts +34 -2
- package/src/domain/search/models.ts +1 -1
- package/src/domain/wasm-worker-entry.ts +798 -0
- package/src/domain/wasm-worker-pool.ts +330 -0
- package/src/domain/wasm-worker-protocol.ts +81 -0
- package/src/extractors/javascript.ts +149 -2
- package/src/features/ast.ts +22 -9
- package/src/features/boundaries.ts +2 -27
- package/src/features/snapshot.ts +93 -14
- package/src/graph/algorithms/louvain.ts +2 -4
- package/src/infrastructure/config.ts +12 -2
- package/src/shared/globs.ts +121 -0
- package/src/types.ts +26 -1
|
@@ -7,11 +7,13 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import fs from 'node:fs';
|
|
9
9
|
import path from 'node:path';
|
|
10
|
+
import { performance } from 'node:perf_hooks';
|
|
10
11
|
import { debug, info } from '../../../../infrastructure/logger.js';
|
|
11
12
|
import { normalizePath } from '../../../../shared/constants.js';
|
|
13
|
+
import { compileGlobs } from '../../../../shared/globs.js';
|
|
12
14
|
import { readJournal } from '../../journal.js';
|
|
13
15
|
import type { PipelineContext } from '../context.js';
|
|
14
|
-
import { collectFiles as collectFilesUtil } from '../helpers.js';
|
|
16
|
+
import { collectFiles as collectFilesUtil, passesIncludeExclude } from '../helpers.js';
|
|
15
17
|
|
|
16
18
|
/**
|
|
17
19
|
* Reconstruct allFiles from DB file_hashes + journal deltas.
|
|
@@ -20,7 +22,7 @@ import { collectFiles as collectFilesUtil } from '../helpers.js';
|
|
|
20
22
|
function tryFastCollect(
|
|
21
23
|
ctx: PipelineContext,
|
|
22
24
|
): { files: string[]; directories: Set<string> } | null {
|
|
23
|
-
const { db, rootDir } = ctx;
|
|
25
|
+
const { db, rootDir, config } = ctx;
|
|
24
26
|
const useNative = ctx.engineName === 'native' && !!ctx.nativeDb?.getCollectFilesData;
|
|
25
27
|
|
|
26
28
|
// 1. Check that file_hashes table exists and has entries
|
|
@@ -70,10 +72,20 @@ function tryFastCollect(
|
|
|
70
72
|
}
|
|
71
73
|
}
|
|
72
74
|
|
|
73
|
-
// 5. Convert to absolute paths and compute directories
|
|
75
|
+
// 5. Convert to absolute paths and compute directories, honoring
|
|
76
|
+
// config.include / config.exclude globs so incremental builds reflect
|
|
77
|
+
// config changes (paths from the DB were collected under older config).
|
|
78
|
+
const includeRegexes = compileGlobs(config?.include);
|
|
79
|
+
const excludeRegexes = compileGlobs(config?.exclude);
|
|
80
|
+
const hasGlobFilters = includeRegexes.length > 0 || excludeRegexes.length > 0;
|
|
81
|
+
|
|
74
82
|
const files: string[] = [];
|
|
75
83
|
const directories = new Set<string>();
|
|
76
84
|
for (const relPath of fileSet) {
|
|
85
|
+
if (hasGlobFilters) {
|
|
86
|
+
const normRel = normalizePath(relPath);
|
|
87
|
+
if (!passesIncludeExclude(normRel, includeRegexes, excludeRegexes)) continue;
|
|
88
|
+
}
|
|
77
89
|
const absPath = path.join(rootDir, relPath);
|
|
78
90
|
files.push(absPath);
|
|
79
91
|
directories.add(path.dirname(absPath));
|
|
@@ -89,42 +101,60 @@ export async function collectFiles(ctx: PipelineContext): Promise<void> {
|
|
|
89
101
|
const { rootDir, config, opts } = ctx;
|
|
90
102
|
|
|
91
103
|
if (opts.scope) {
|
|
92
|
-
// Scoped rebuild: rebuild only specified files
|
|
104
|
+
// Scoped rebuild: rebuild only specified files.
|
|
105
|
+
//
|
|
106
|
+
// Timer only wraps the filesystem-walk portion (existence checks + file
|
|
107
|
+
// list construction). Change-detection outputs (parseChanges, removed,
|
|
108
|
+
// isFullBuild) are attributed to detectMs for semantic consistency with
|
|
109
|
+
// the non-scoped path, even though this stage computes them.
|
|
110
|
+
const start = performance.now();
|
|
93
111
|
const scopedFiles = opts.scope.map((f: string) => normalizePath(f));
|
|
94
112
|
const existing: Array<{ file: string; relPath: string }> = [];
|
|
95
113
|
const missing: string[] = [];
|
|
96
|
-
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
114
|
+
try {
|
|
115
|
+
for (const rel of scopedFiles) {
|
|
116
|
+
const abs = path.join(rootDir, rel);
|
|
117
|
+
if (fs.existsSync(abs)) {
|
|
118
|
+
existing.push({ file: abs, relPath: rel });
|
|
119
|
+
} else {
|
|
120
|
+
missing.push(rel);
|
|
121
|
+
}
|
|
102
122
|
}
|
|
123
|
+
ctx.allFiles = existing.map((e) => e.file);
|
|
124
|
+
ctx.discoveredDirs = new Set(existing.map((e) => path.dirname(e.file)));
|
|
125
|
+
} finally {
|
|
126
|
+
ctx.timing.collectMs = performance.now() - start;
|
|
103
127
|
}
|
|
104
|
-
|
|
105
|
-
|
|
128
|
+
// Change-detection outputs — timed under detectMs for semantic parity.
|
|
129
|
+
const detectStart = performance.now();
|
|
106
130
|
ctx.parseChanges = existing;
|
|
107
131
|
ctx.metadataUpdates = [];
|
|
108
132
|
ctx.removed = missing;
|
|
109
133
|
ctx.isFullBuild = false;
|
|
134
|
+
ctx.timing.detectMs = (ctx.timing.detectMs ?? 0) + (performance.now() - detectStart);
|
|
110
135
|
info(`Scoped rebuild: ${existing.length} files to rebuild, ${missing.length} to purge`);
|
|
111
136
|
return;
|
|
112
137
|
}
|
|
113
138
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
if (
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
139
|
+
const start = performance.now();
|
|
140
|
+
try {
|
|
141
|
+
// Incremental fast path: reconstruct file list from DB + journal deltas
|
|
142
|
+
// instead of full recursive filesystem scan (~8ms savings on 473 files).
|
|
143
|
+
if (ctx.incremental && !ctx.forceFullRebuild) {
|
|
144
|
+
const fast = tryFastCollect(ctx);
|
|
145
|
+
if (fast) {
|
|
146
|
+
ctx.allFiles = fast.files;
|
|
147
|
+
ctx.discoveredDirs = fast.directories;
|
|
148
|
+
info(`Found ${ctx.allFiles.length} files (cached)`);
|
|
149
|
+
return;
|
|
150
|
+
}
|
|
123
151
|
}
|
|
124
|
-
}
|
|
125
152
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
153
|
+
const collected = collectFilesUtil(rootDir, [], config, new Set<string>());
|
|
154
|
+
ctx.allFiles = collected.files;
|
|
155
|
+
ctx.discoveredDirs = collected.directories;
|
|
156
|
+
info(`Found ${ctx.allFiles.length} files to parse`);
|
|
157
|
+
} finally {
|
|
158
|
+
ctx.timing.collectMs = performance.now() - start;
|
|
159
|
+
}
|
|
130
160
|
}
|
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import fs from 'node:fs';
|
|
9
9
|
import path from 'node:path';
|
|
10
|
+
import { performance } from 'node:perf_hooks';
|
|
10
11
|
import { closeDb } from '../../../../db/index.js';
|
|
11
12
|
import { debug, info } from '../../../../infrastructure/logger.js';
|
|
12
13
|
import { normalizePath } from '../../../../shared/constants.js';
|
|
@@ -512,59 +513,66 @@ function handleIncrementalBuild(ctx: PipelineContext): void {
|
|
|
512
513
|
}
|
|
513
514
|
|
|
514
515
|
export async function detectChanges(ctx: PipelineContext): Promise<void> {
|
|
515
|
-
const
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
.
|
|
532
|
-
.
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
if (
|
|
516
|
+
const start = performance.now();
|
|
517
|
+
try {
|
|
518
|
+
const { db, allFiles, rootDir, incremental, forceFullRebuild, opts } = ctx;
|
|
519
|
+
if ((opts as Record<string, unknown>).scope) {
|
|
520
|
+
handleScopedBuild(ctx);
|
|
521
|
+
return;
|
|
522
|
+
}
|
|
523
|
+
const increResult =
|
|
524
|
+
incremental && !forceFullRebuild
|
|
525
|
+
? getChangedFiles(db, allFiles, rootDir)
|
|
526
|
+
: {
|
|
527
|
+
changed: allFiles.map((f): ChangedFile => ({ file: f })),
|
|
528
|
+
removed: [] as string[],
|
|
529
|
+
isFullBuild: true,
|
|
530
|
+
};
|
|
531
|
+
ctx.removed = increResult.removed;
|
|
532
|
+
ctx.isFullBuild = increResult.isFullBuild;
|
|
533
|
+
ctx.parseChanges = increResult.changed
|
|
534
|
+
.filter((c) => !c.metadataOnly)
|
|
535
|
+
.map((c) => ({
|
|
536
|
+
file: c.file,
|
|
537
|
+
relPath: c.relPath,
|
|
538
|
+
content: c.content,
|
|
539
|
+
hash: c.hash,
|
|
540
|
+
stat: c.stat ? { mtime: Math.floor(c.stat.mtimeMs), size: c.stat.size } : undefined,
|
|
541
|
+
_reverseDepOnly: c._reverseDepOnly,
|
|
542
|
+
}));
|
|
543
|
+
ctx.metadataUpdates = increResult.changed
|
|
544
|
+
.filter(
|
|
545
|
+
(c): c is ChangedFile & { relPath: string; hash: string; stat: FileStat } =>
|
|
546
|
+
!!c.metadataOnly && !!c.relPath && !!c.hash && !!c.stat,
|
|
547
|
+
)
|
|
548
|
+
.map((c) => ({
|
|
549
|
+
relPath: c.relPath,
|
|
550
|
+
hash: c.hash,
|
|
551
|
+
stat: { mtime: Math.floor(c.stat.mtimeMs), size: c.stat.size },
|
|
552
|
+
}));
|
|
553
|
+
if (!ctx.isFullBuild && ctx.parseChanges.length === 0 && ctx.removed.length === 0) {
|
|
554
|
+
const ranAnalysis = await runPendingAnalysis(ctx);
|
|
555
|
+
if (ranAnalysis) {
|
|
556
|
+
closeDb(db);
|
|
557
|
+
writeJournalHeader(rootDir, Date.now());
|
|
558
|
+
ctx.earlyExit = true;
|
|
559
|
+
return;
|
|
560
|
+
}
|
|
561
|
+
healMetadata(ctx);
|
|
562
|
+
info('No changes detected. Graph is up to date.');
|
|
553
563
|
closeDb(db);
|
|
554
564
|
writeJournalHeader(rootDir, Date.now());
|
|
555
565
|
ctx.earlyExit = true;
|
|
556
566
|
return;
|
|
557
567
|
}
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
} else {
|
|
568
|
-
handleIncrementalBuild(ctx);
|
|
568
|
+
if (ctx.isFullBuild) {
|
|
569
|
+
handleFullBuild(ctx);
|
|
570
|
+
} else {
|
|
571
|
+
handleIncrementalBuild(ctx);
|
|
572
|
+
}
|
|
573
|
+
} finally {
|
|
574
|
+
// Additive to respect any partial detectMs contribution from collectFiles
|
|
575
|
+
// (scoped-rebuild path splits change-detection outputs across both stages).
|
|
576
|
+
ctx.timing.detectMs = (ctx.timing.detectMs ?? 0) + (performance.now() - start);
|
|
569
577
|
}
|
|
570
578
|
}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
*
|
|
4
4
|
* WASM cleanup, stats logging, drift detection, build metadata, registry, journal.
|
|
5
5
|
*/
|
|
6
|
+
import fs from 'node:fs';
|
|
6
7
|
import { tmpdir } from 'node:os';
|
|
7
8
|
import path from 'node:path';
|
|
8
9
|
import { performance } from 'node:perf_hooks';
|
|
@@ -88,6 +89,19 @@ function persistBuildMetadata(
|
|
|
88
89
|
// subsequent build to be a full rebuild.
|
|
89
90
|
const codeVersionToWrite =
|
|
90
91
|
ctx.engineName === 'native' && ctx.engineVersion ? ctx.engineVersion : CODEGRAPH_VERSION;
|
|
92
|
+
// Persist the repo root so downstream commands (e.g. `codegraph embed`)
|
|
93
|
+
// can resolve relative file paths regardless of the invoking cwd.
|
|
94
|
+
// Use realpathSync (symlink-resolving) to match the Rust engine's
|
|
95
|
+
// std::fs::canonicalize — otherwise the JS write here would overwrite the
|
|
96
|
+
// canonical path Rust wrote for native full builds and could re-introduce
|
|
97
|
+
// a non-canonical path when the project root is behind a symlink.
|
|
98
|
+
const resolvedRootDir = path.resolve(ctx.rootDir);
|
|
99
|
+
let rootDirToWrite = resolvedRootDir;
|
|
100
|
+
try {
|
|
101
|
+
rootDirToWrite = fs.realpathSync(resolvedRootDir);
|
|
102
|
+
} catch {
|
|
103
|
+
/* realpath can fail (e.g. path no longer exists); fall back to resolve() */
|
|
104
|
+
}
|
|
91
105
|
try {
|
|
92
106
|
if (useNativeDb) {
|
|
93
107
|
ctx.nativeDb!.setBuildMeta(
|
|
@@ -99,6 +113,7 @@ function persistBuildMetadata(
|
|
|
99
113
|
built_at: buildNow.toISOString(),
|
|
100
114
|
node_count: String(nodeCount),
|
|
101
115
|
edge_count: String(actualEdgeCount),
|
|
116
|
+
root_dir: rootDirToWrite,
|
|
102
117
|
}).map(([key, value]) => ({ key, value: String(value) })),
|
|
103
118
|
);
|
|
104
119
|
} else {
|
|
@@ -110,6 +125,7 @@ function persistBuildMetadata(
|
|
|
110
125
|
built_at: buildNow.toISOString(),
|
|
111
126
|
node_count: nodeCount,
|
|
112
127
|
edge_count: actualEdgeCount,
|
|
128
|
+
root_dir: rootDirToWrite,
|
|
113
129
|
});
|
|
114
130
|
}
|
|
115
131
|
} catch (err) {
|
|
@@ -1,9 +1,224 @@
|
|
|
1
|
+
import crypto from 'node:crypto';
|
|
1
2
|
import fs from 'node:fs';
|
|
2
3
|
import path from 'node:path';
|
|
3
4
|
import { debug, warn } from '../../infrastructure/logger.js';
|
|
4
5
|
|
|
5
6
|
export const JOURNAL_FILENAME = 'changes.journal';
|
|
6
7
|
const HEADER_PREFIX = '# codegraph-journal v1 ';
|
|
8
|
+
const LOCK_SUFFIX = '.lock';
|
|
9
|
+
const LOCK_TIMEOUT_MS = 5_000;
|
|
10
|
+
const LOCK_STALE_MS = 30_000;
|
|
11
|
+
const LOCK_RETRY_MS = 25;
|
|
12
|
+
|
|
13
|
+
// Busy-spin sleep avoids blocking the Node.js event loop (unlike Atomics.wait,
|
|
14
|
+
// which freezes all I/O and timer callbacks). The retry interval is short
|
|
15
|
+
// (25ms), so the CPU cost is negligible while keeping unrelated callbacks
|
|
16
|
+
// responsive in watcher processes.
|
|
17
|
+
function sleepSync(ms: number): void {
|
|
18
|
+
const end = process.hrtime.bigint() + BigInt(ms) * 1_000_000n;
|
|
19
|
+
while (process.hrtime.bigint() < end) {
|
|
20
|
+
/* spin */
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
function isPidAlive(pid: number): boolean {
|
|
25
|
+
if (!Number.isFinite(pid) || pid <= 0) return false;
|
|
26
|
+
try {
|
|
27
|
+
process.kill(pid, 0);
|
|
28
|
+
return true;
|
|
29
|
+
} catch (e) {
|
|
30
|
+
// EPERM means the process exists but we lack permission — still alive.
|
|
31
|
+
return (e as NodeJS.ErrnoException).code === 'EPERM';
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface AcquiredLock {
|
|
36
|
+
fd: number;
|
|
37
|
+
nonce: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Steal a stale lockfile atomically via write-tmp + rename.
|
|
42
|
+
*
|
|
43
|
+
* Using rename (which is atomic on POSIX and Windows) avoids the TOCTOU race
|
|
44
|
+
* inherent to the unlink + openSync('wx') pattern: if two stealers both
|
|
45
|
+
* observed the same stale holder, one's unlink could cross the other's fresh
|
|
46
|
+
* acquisition, admitting two writers into the critical section.
|
|
47
|
+
*
|
|
48
|
+
* After rename, we re-read the lockfile to confirm our nonce — if another
|
|
49
|
+
* stealer's rename landed after ours, they own the lock and we retry.
|
|
50
|
+
*/
|
|
51
|
+
function trySteal(lockPath: string): AcquiredLock | null {
|
|
52
|
+
const nonce = `${process.pid}-${crypto.randomBytes(8).toString('hex')}`;
|
|
53
|
+
const tmpPath = `${lockPath}.${nonce}.tmp`;
|
|
54
|
+
try {
|
|
55
|
+
fs.writeFileSync(tmpPath, `${process.pid}\n${nonce}\n`, { flag: 'w' });
|
|
56
|
+
} catch {
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
try {
|
|
61
|
+
// Atomic replace: overwrites the stale lockfile.
|
|
62
|
+
fs.renameSync(tmpPath, lockPath);
|
|
63
|
+
} catch {
|
|
64
|
+
try {
|
|
65
|
+
fs.unlinkSync(tmpPath);
|
|
66
|
+
} catch {
|
|
67
|
+
/* ignore */
|
|
68
|
+
}
|
|
69
|
+
return null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// Verify the nonce — another stealer's rename may have landed after ours.
|
|
73
|
+
let content: string;
|
|
74
|
+
try {
|
|
75
|
+
content = fs.readFileSync(lockPath, 'utf-8');
|
|
76
|
+
} catch {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
if (!content.includes(nonce)) {
|
|
80
|
+
// Lost the race to another stealer; do NOT unlink their live lockfile.
|
|
81
|
+
return null;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
let fd: number;
|
|
85
|
+
try {
|
|
86
|
+
// Re-open r+ so we have a persistent fd the caller can close on release.
|
|
87
|
+
fd = fs.openSync(lockPath, 'r+');
|
|
88
|
+
} catch {
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
return { fd, nonce };
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function acquireJournalLock(lockPath: string): AcquiredLock {
|
|
95
|
+
const start = Date.now();
|
|
96
|
+
for (;;) {
|
|
97
|
+
const nonce = `${process.pid}-${crypto.randomBytes(8).toString('hex')}`;
|
|
98
|
+
try {
|
|
99
|
+
const fd = fs.openSync(lockPath, 'wx');
|
|
100
|
+
try {
|
|
101
|
+
fs.writeSync(fd, `${process.pid}\n${nonce}\n`);
|
|
102
|
+
} catch {
|
|
103
|
+
// Stamp write failed (ENOSPC, I/O error). An empty lockfile would
|
|
104
|
+
// look stale to concurrent waiters (Number('') === 0, isPidAlive(0)
|
|
105
|
+
// returns false), so they'd steal our live lock. Release and retry.
|
|
106
|
+
try {
|
|
107
|
+
fs.closeSync(fd);
|
|
108
|
+
} catch {
|
|
109
|
+
/* ignore */
|
|
110
|
+
}
|
|
111
|
+
try {
|
|
112
|
+
fs.unlinkSync(lockPath);
|
|
113
|
+
} catch {
|
|
114
|
+
/* ignore */
|
|
115
|
+
}
|
|
116
|
+
if (Date.now() - start > LOCK_TIMEOUT_MS) {
|
|
117
|
+
throw new Error(
|
|
118
|
+
`Failed to acquire journal lock at ${lockPath} within ${LOCK_TIMEOUT_MS}ms`,
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
sleepSync(LOCK_RETRY_MS);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
return { fd, nonce };
|
|
125
|
+
} catch (e) {
|
|
126
|
+
if ((e as NodeJS.ErrnoException).code !== 'EEXIST') throw e;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
let holderAlive = true;
|
|
130
|
+
try {
|
|
131
|
+
const pidContent = fs.readFileSync(lockPath, 'utf-8').split('\n')[0]!.trim();
|
|
132
|
+
holderAlive = isPidAlive(Number(pidContent));
|
|
133
|
+
} catch {
|
|
134
|
+
/* unreadable — fall through to age check */
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
let shouldSteal = !holderAlive;
|
|
138
|
+
if (holderAlive) {
|
|
139
|
+
try {
|
|
140
|
+
const stat = fs.statSync(lockPath);
|
|
141
|
+
if (Date.now() - stat.mtimeMs > LOCK_STALE_MS) {
|
|
142
|
+
shouldSteal = true;
|
|
143
|
+
}
|
|
144
|
+
} catch {
|
|
145
|
+
/* stat failed — keep retrying */
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (shouldSteal) {
|
|
150
|
+
const stolen = trySteal(lockPath);
|
|
151
|
+
if (stolen) return stolen;
|
|
152
|
+
// Steal failed or lost the race — fall through to timeout check & retry.
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if (Date.now() - start > LOCK_TIMEOUT_MS) {
|
|
156
|
+
throw new Error(`Failed to acquire journal lock at ${lockPath} within ${LOCK_TIMEOUT_MS}ms`);
|
|
157
|
+
}
|
|
158
|
+
sleepSync(LOCK_RETRY_MS);
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
function releaseJournalLock(lockPath: string, lock: AcquiredLock): void {
|
|
163
|
+
try {
|
|
164
|
+
fs.closeSync(lock.fd);
|
|
165
|
+
} catch {
|
|
166
|
+
/* ignore */
|
|
167
|
+
}
|
|
168
|
+
// Only unlink if the lockfile still carries our nonce — if another stealer
|
|
169
|
+
// decided we were stale and replaced it, we must not unlink their live lock.
|
|
170
|
+
try {
|
|
171
|
+
const content = fs.readFileSync(lockPath, 'utf-8');
|
|
172
|
+
if (content.includes(lock.nonce)) {
|
|
173
|
+
fs.unlinkSync(lockPath);
|
|
174
|
+
}
|
|
175
|
+
} catch {
|
|
176
|
+
/* lockfile gone or unreadable — nothing to unlink */
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function sweepStaleTmpFiles(dir: string): void {
|
|
181
|
+
// Clean up orphaned .tmp files left behind when a process is killed after
|
|
182
|
+
// writeFileSync(tmpPath, ...) succeeds but before renameSync(tmpPath, lockPath)
|
|
183
|
+
// completes (trySteal path). Without this, tmp files accumulate silently in
|
|
184
|
+
// .codegraph/ across crash cycles. Only sweep ones older than LOCK_STALE_MS
|
|
185
|
+
// so we don't race an in-flight steal on another process.
|
|
186
|
+
let entries: fs.Dirent[];
|
|
187
|
+
try {
|
|
188
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
189
|
+
} catch {
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
const now = Date.now();
|
|
193
|
+
const prefix = `${JOURNAL_FILENAME}${LOCK_SUFFIX}.`;
|
|
194
|
+
for (const entry of entries) {
|
|
195
|
+
if (!entry.isFile() || !entry.name.startsWith(prefix) || !entry.name.endsWith('.tmp')) {
|
|
196
|
+
continue;
|
|
197
|
+
}
|
|
198
|
+
const tmpPath = path.join(dir, entry.name);
|
|
199
|
+
try {
|
|
200
|
+
const stat = fs.statSync(tmpPath);
|
|
201
|
+
if (now - stat.mtimeMs > LOCK_STALE_MS) {
|
|
202
|
+
fs.unlinkSync(tmpPath);
|
|
203
|
+
}
|
|
204
|
+
} catch {
|
|
205
|
+
/* stat/unlink raced another cleaner or was already removed — ignore */
|
|
206
|
+
}
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
function withJournalLock<T>(rootDir: string, fn: () => T): T {
|
|
211
|
+
const dir = path.join(rootDir, '.codegraph');
|
|
212
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
213
|
+
sweepStaleTmpFiles(dir);
|
|
214
|
+
const lockPath = path.join(dir, `${JOURNAL_FILENAME}${LOCK_SUFFIX}`);
|
|
215
|
+
const lock = acquireJournalLock(lockPath);
|
|
216
|
+
try {
|
|
217
|
+
return fn();
|
|
218
|
+
} finally {
|
|
219
|
+
releaseJournalLock(lockPath, lock);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
7
222
|
|
|
8
223
|
interface JournalResult {
|
|
9
224
|
valid: boolean;
|
|
@@ -63,43 +278,85 @@ export function appendJournalEntries(
|
|
|
63
278
|
rootDir: string,
|
|
64
279
|
entries: Array<{ file: string; deleted?: boolean }>,
|
|
65
280
|
): void {
|
|
66
|
-
|
|
67
|
-
|
|
281
|
+
withJournalLock(rootDir, () => {
|
|
282
|
+
const journalPath = path.join(rootDir, '.codegraph', JOURNAL_FILENAME);
|
|
68
283
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
284
|
+
if (!fs.existsSync(journalPath)) {
|
|
285
|
+
fs.writeFileSync(journalPath, `${HEADER_PREFIX}0\n`);
|
|
286
|
+
}
|
|
72
287
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
288
|
+
const lines = entries.map((e) => {
|
|
289
|
+
if (e.deleted) return `DELETED ${e.file}`;
|
|
290
|
+
return e.file;
|
|
291
|
+
});
|
|
76
292
|
|
|
77
|
-
|
|
78
|
-
if (e.deleted) return `DELETED ${e.file}`;
|
|
79
|
-
return e.file;
|
|
293
|
+
fs.appendFileSync(journalPath, `${lines.join('\n')}\n`);
|
|
80
294
|
});
|
|
81
|
-
|
|
82
|
-
fs.appendFileSync(journalPath, `${lines.join('\n')}\n`);
|
|
83
295
|
}
|
|
84
296
|
|
|
85
297
|
export function writeJournalHeader(rootDir: string, timestamp: number): void {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
298
|
+
withJournalLock(rootDir, () => {
|
|
299
|
+
const journalPath = path.join(rootDir, '.codegraph', JOURNAL_FILENAME);
|
|
300
|
+
const tmpPath = `${journalPath}.tmp`;
|
|
89
301
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
302
|
+
try {
|
|
303
|
+
fs.writeFileSync(tmpPath, `${HEADER_PREFIX}${timestamp}\n`);
|
|
304
|
+
fs.renameSync(tmpPath, journalPath);
|
|
305
|
+
} catch (err) {
|
|
306
|
+
warn(`Failed to write journal header: ${(err as Error).message}`);
|
|
307
|
+
try {
|
|
308
|
+
fs.unlinkSync(tmpPath);
|
|
309
|
+
} catch {
|
|
310
|
+
/* ignore */
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
});
|
|
314
|
+
}
|
|
93
315
|
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
316
|
+
/**
|
|
317
|
+
* Atomically append entries while advancing the header timestamp.
|
|
318
|
+
*
|
|
319
|
+
* Used by the watcher: without this, the header timestamp stays frozen at the
|
|
320
|
+
* last build's finalize time while entries accumulate, so the next build's
|
|
321
|
+
* Tier 0 check sees `journal.timestamp < MAX(file_hashes.mtime)`, rejects the
|
|
322
|
+
* journal, and falls through to the expensive mtime+size / hash scan.
|
|
323
|
+
*
|
|
324
|
+
* Writes a tmp file then renames — a crash mid-rename leaves the previous
|
|
325
|
+
* journal state intact.
|
|
326
|
+
*/
|
|
327
|
+
export function appendJournalEntriesAndStampHeader(
|
|
328
|
+
rootDir: string,
|
|
329
|
+
entries: Array<{ file: string; deleted?: boolean }>,
|
|
330
|
+
timestamp: number,
|
|
331
|
+
): void {
|
|
332
|
+
withJournalLock(rootDir, () => {
|
|
333
|
+
const journalPath = path.join(rootDir, '.codegraph', JOURNAL_FILENAME);
|
|
334
|
+
const tmpPath = `${journalPath}.tmp`;
|
|
335
|
+
|
|
336
|
+
let existingBody = '';
|
|
99
337
|
try {
|
|
100
|
-
fs.
|
|
338
|
+
const content = fs.readFileSync(journalPath, 'utf-8');
|
|
339
|
+
const newlineIdx = content.indexOf('\n');
|
|
340
|
+
if (newlineIdx >= 0) existingBody = content.slice(newlineIdx + 1);
|
|
101
341
|
} catch {
|
|
102
|
-
/*
|
|
342
|
+
/* no existing journal — fall through to write header + new entries */
|
|
103
343
|
}
|
|
104
|
-
|
|
344
|
+
if (existingBody && !existingBody.endsWith('\n')) existingBody = `${existingBody}\n`;
|
|
345
|
+
|
|
346
|
+
const newLines = entries.map((e) => (e.deleted ? `DELETED ${e.file}` : e.file));
|
|
347
|
+
const appended = newLines.length > 0 ? `${newLines.join('\n')}\n` : '';
|
|
348
|
+
const content = `${HEADER_PREFIX}${timestamp}\n${existingBody}${appended}`;
|
|
349
|
+
|
|
350
|
+
try {
|
|
351
|
+
fs.writeFileSync(tmpPath, content);
|
|
352
|
+
fs.renameSync(tmpPath, journalPath);
|
|
353
|
+
} catch (err) {
|
|
354
|
+
warn(`Failed to update journal: ${(err as Error).message}`);
|
|
355
|
+
try {
|
|
356
|
+
fs.unlinkSync(tmpPath);
|
|
357
|
+
} catch {
|
|
358
|
+
/* ignore */
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
});
|
|
105
362
|
}
|