@colbymchenry/codegraph-win32-arm64 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/codegraph.cmd +1 -1
- package/lib/dist/bin/codegraph.js +106 -19
- package/lib/dist/bin/codegraph.js.map +1 -1
- package/lib/dist/bin/command-supervision.d.ts +38 -1
- package/lib/dist/bin/command-supervision.d.ts.map +1 -1
- package/lib/dist/bin/command-supervision.js +8 -2
- package/lib/dist/bin/command-supervision.js.map +1 -1
- package/lib/dist/context/index.d.ts.map +1 -1
- package/lib/dist/context/index.js +41 -11
- package/lib/dist/context/index.js.map +1 -1
- package/lib/dist/db/index.d.ts +142 -0
- package/lib/dist/db/index.d.ts.map +1 -1
- package/lib/dist/db/index.js +267 -6
- package/lib/dist/db/index.js.map +1 -1
- package/lib/dist/db/migrations.d.ts +1 -1
- package/lib/dist/db/migrations.d.ts.map +1 -1
- package/lib/dist/db/migrations.js +31 -1
- package/lib/dist/db/migrations.js.map +1 -1
- package/lib/dist/db/queries.d.ts +113 -4
- package/lib/dist/db/queries.d.ts.map +1 -1
- package/lib/dist/db/queries.js +398 -24
- package/lib/dist/db/queries.js.map +1 -1
- package/lib/dist/db/schema.sql +13 -1
- package/lib/dist/db/sqlite-adapter.d.ts +3 -1
- package/lib/dist/db/sqlite-adapter.d.ts.map +1 -1
- package/lib/dist/db/sqlite-adapter.js +22 -4
- package/lib/dist/db/sqlite-adapter.js.map +1 -1
- package/lib/dist/db/wal-valve.d.ts +15 -2
- package/lib/dist/db/wal-valve.d.ts.map +1 -1
- package/lib/dist/db/wal-valve.js +88 -6
- package/lib/dist/db/wal-valve.js.map +1 -1
- package/lib/dist/extraction/grammars.d.ts.map +1 -1
- package/lib/dist/extraction/grammars.js +63 -0
- package/lib/dist/extraction/grammars.js.map +1 -1
- package/lib/dist/extraction/index.d.ts +32 -3
- package/lib/dist/extraction/index.d.ts.map +1 -1
- package/lib/dist/extraction/index.js +313 -56
- package/lib/dist/extraction/index.js.map +1 -1
- package/lib/dist/extraction/kernel/decode.d.ts +9 -0
- package/lib/dist/extraction/kernel/decode.d.ts.map +1 -0
- package/lib/dist/extraction/kernel/decode.js +162 -0
- package/lib/dist/extraction/kernel/decode.js.map +1 -0
- package/lib/dist/extraction/kernel/index.d.ts +64 -0
- package/lib/dist/extraction/kernel/index.d.ts.map +1 -0
- package/lib/dist/extraction/kernel/index.js +279 -0
- package/lib/dist/extraction/kernel/index.js.map +1 -0
- package/lib/dist/extraction/kernel/layout.d.ts +101 -0
- package/lib/dist/extraction/kernel/layout.d.ts.map +1 -0
- package/lib/dist/extraction/kernel/layout.js +104 -0
- package/lib/dist/extraction/kernel/layout.js.map +1 -0
- package/lib/dist/extraction/kernel/loader.d.ts +103 -0
- package/lib/dist/extraction/kernel/loader.d.ts.map +1 -0
- package/lib/dist/extraction/kernel/loader.js +146 -0
- package/lib/dist/extraction/kernel/loader.js.map +1 -0
- package/lib/dist/extraction/languages/c-cpp.d.ts +14 -0
- package/lib/dist/extraction/languages/c-cpp.d.ts.map +1 -1
- package/lib/dist/extraction/languages/c-cpp.js +849 -12
- package/lib/dist/extraction/languages/c-cpp.js.map +1 -1
- package/lib/dist/extraction/parse-pool.d.ts +8 -0
- package/lib/dist/extraction/parse-pool.d.ts.map +1 -1
- package/lib/dist/extraction/parse-pool.js +15 -0
- package/lib/dist/extraction/parse-pool.js.map +1 -1
- package/lib/dist/extraction/parse-worker.js +35 -1
- package/lib/dist/extraction/parse-worker.js.map +1 -1
- package/lib/dist/extraction/store-worker.d.ts +24 -0
- package/lib/dist/extraction/store-worker.d.ts.map +1 -0
- package/lib/dist/extraction/store-worker.js +139 -0
- package/lib/dist/extraction/store-worker.js.map +1 -0
- package/lib/dist/extraction/store-writer.d.ts +63 -0
- package/lib/dist/extraction/store-writer.d.ts.map +1 -0
- package/lib/dist/extraction/store-writer.js +174 -0
- package/lib/dist/extraction/store-writer.js.map +1 -0
- package/lib/dist/extraction/tree-sitter.d.ts +22 -4
- package/lib/dist/extraction/tree-sitter.d.ts.map +1 -1
- package/lib/dist/extraction/tree-sitter.js +151 -6
- package/lib/dist/extraction/tree-sitter.js.map +1 -1
- package/lib/dist/extraction/wasm/tree-sitter-c.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-cpp.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-dart.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-go.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-java.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-javascript.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-kotlin.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-php.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-python.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-ruby.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-rust.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-swift.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-tsx.wasm +0 -0
- package/lib/dist/extraction/wasm/tree-sitter-typescript.wasm +0 -0
- package/lib/dist/extraction/wasm-runtime-flags.d.ts +15 -0
- package/lib/dist/extraction/wasm-runtime-flags.d.ts.map +1 -1
- package/lib/dist/extraction/wasm-runtime-flags.js +27 -3
- package/lib/dist/extraction/wasm-runtime-flags.js.map +1 -1
- package/lib/dist/index.d.ts +19 -6
- package/lib/dist/index.d.ts.map +1 -1
- package/lib/dist/index.js +233 -17
- package/lib/dist/index.js.map +1 -1
- package/lib/dist/installer/beta-signup.d.ts +54 -0
- package/lib/dist/installer/beta-signup.d.ts.map +1 -0
- package/lib/dist/installer/beta-signup.js +178 -0
- package/lib/dist/installer/beta-signup.js.map +1 -0
- package/lib/dist/installer/index.d.ts +42 -0
- package/lib/dist/installer/index.d.ts.map +1 -1
- package/lib/dist/installer/index.js +105 -1
- package/lib/dist/installer/index.js.map +1 -1
- package/lib/dist/mcp/index.d.ts.map +1 -1
- package/lib/dist/mcp/index.js +8 -0
- package/lib/dist/mcp/index.js.map +1 -1
- package/lib/dist/mcp/liveness-watchdog.d.ts +18 -1
- package/lib/dist/mcp/liveness-watchdog.d.ts.map +1 -1
- package/lib/dist/mcp/liveness-watchdog.js +73 -13
- package/lib/dist/mcp/liveness-watchdog.js.map +1 -1
- package/lib/dist/mcp/proxy.js +1 -1
- package/lib/dist/mcp/proxy.js.map +1 -1
- package/lib/dist/mcp/session.d.ts +14 -0
- package/lib/dist/mcp/session.d.ts.map +1 -1
- package/lib/dist/mcp/session.js +22 -1
- package/lib/dist/mcp/session.js.map +1 -1
- package/lib/dist/mcp/tools.d.ts.map +1 -1
- package/lib/dist/mcp/tools.js +76 -1
- package/lib/dist/mcp/tools.js.map +1 -1
- package/lib/dist/resolution/c-fnptr-synthesizer.d.ts +1 -1
- package/lib/dist/resolution/c-fnptr-synthesizer.d.ts.map +1 -1
- package/lib/dist/resolution/c-fnptr-synthesizer.js +574 -132
- package/lib/dist/resolution/c-fnptr-synthesizer.js.map +1 -1
- package/lib/dist/resolution/callback-synthesizer.d.ts +54 -10
- package/lib/dist/resolution/callback-synthesizer.d.ts.map +1 -1
- package/lib/dist/resolution/callback-synthesizer.js +249 -157
- package/lib/dist/resolution/callback-synthesizer.js.map +1 -1
- package/lib/dist/resolution/cooperative-yield.d.ts +10 -2
- package/lib/dist/resolution/cooperative-yield.d.ts.map +1 -1
- package/lib/dist/resolution/cooperative-yield.js +6 -4
- package/lib/dist/resolution/cooperative-yield.js.map +1 -1
- package/lib/dist/resolution/frameworks/go.d.ts.map +1 -1
- package/lib/dist/resolution/frameworks/go.js +29 -6
- package/lib/dist/resolution/frameworks/go.js.map +1 -1
- package/lib/dist/resolution/import-resolver.d.ts +2 -3
- package/lib/dist/resolution/import-resolver.d.ts.map +1 -1
- package/lib/dist/resolution/import-resolver.js +166 -7
- package/lib/dist/resolution/import-resolver.js.map +1 -1
- package/lib/dist/resolution/index.d.ts +97 -2
- package/lib/dist/resolution/index.d.ts.map +1 -1
- package/lib/dist/resolution/index.js +783 -124
- package/lib/dist/resolution/index.js.map +1 -1
- package/lib/dist/resolution/memory-budget.d.ts +48 -0
- package/lib/dist/resolution/memory-budget.d.ts.map +1 -0
- package/lib/dist/resolution/memory-budget.js +162 -0
- package/lib/dist/resolution/memory-budget.js.map +1 -0
- package/lib/dist/resolution/name-matcher.d.ts +12 -1
- package/lib/dist/resolution/name-matcher.d.ts.map +1 -1
- package/lib/dist/resolution/name-matcher.js +517 -96
- package/lib/dist/resolution/name-matcher.js.map +1 -1
- package/lib/dist/resolution/resolver-pool.d.ts +106 -0
- package/lib/dist/resolution/resolver-pool.d.ts.map +1 -0
- package/lib/dist/resolution/resolver-pool.js +344 -0
- package/lib/dist/resolution/resolver-pool.js.map +1 -0
- package/lib/dist/resolution/resolver-worker.d.ts +17 -0
- package/lib/dist/resolution/resolver-worker.d.ts.map +1 -0
- package/lib/dist/resolution/resolver-worker.js +141 -0
- package/lib/dist/resolution/resolver-worker.js.map +1 -0
- package/lib/dist/resolution/types.d.ts +3 -0
- package/lib/dist/resolution/types.d.ts.map +1 -1
- package/lib/dist/sync/watcher.d.ts +25 -1
- package/lib/dist/sync/watcher.d.ts.map +1 -1
- package/lib/dist/sync/watcher.js +76 -3
- package/lib/dist/sync/watcher.js.map +1 -1
- package/lib/dist/types.d.ts +41 -2
- package/lib/dist/types.d.ts.map +1 -1
- package/lib/dist/types.js +26 -1
- package/lib/dist/types.js.map +1 -1
- package/lib/dist/ui/color.d.ts +21 -0
- package/lib/dist/ui/color.d.ts.map +1 -0
- package/lib/dist/ui/color.js +42 -0
- package/lib/dist/ui/color.js.map +1 -0
- package/lib/dist/ui/glyphs.d.ts +35 -10
- package/lib/dist/ui/glyphs.d.ts.map +1 -1
- package/lib/dist/ui/glyphs.js +59 -12
- package/lib/dist/ui/glyphs.js.map +1 -1
- package/lib/dist/ui/shimmer-progress.d.ts.map +1 -1
- package/lib/dist/ui/shimmer-progress.js +78 -4
- package/lib/dist/ui/shimmer-progress.js.map +1 -1
- package/lib/dist/ui/shimmer-worker.js +25 -17
- package/lib/dist/ui/shimmer-worker.js.map +1 -1
- package/lib/dist/ui/types.d.ts +0 -2
- package/lib/dist/ui/types.d.ts.map +1 -1
- package/lib/dist/upgrade/index.d.ts +39 -0
- package/lib/dist/upgrade/index.d.ts.map +1 -1
- package/lib/dist/upgrade/index.js +180 -19
- package/lib/dist/upgrade/index.js.map +1 -1
- package/lib/dist/upgrade/remove-binary.d.ts +87 -0
- package/lib/dist/upgrade/remove-binary.d.ts.map +1 -0
- package/lib/dist/upgrade/remove-binary.js +289 -0
- package/lib/dist/upgrade/remove-binary.js.map +1 -0
- package/lib/dist/upgrade/update-check.d.ts +92 -0
- package/lib/dist/upgrade/update-check.d.ts.map +1 -0
- package/lib/dist/upgrade/update-check.js +258 -0
- package/lib/dist/upgrade/update-check.js.map +1 -0
- package/lib/kernel/codegraph-kernel.node +0 -0
- package/lib/node_modules/.package-lock.json +1 -1
- package/lib/package.json +6 -1
- package/package.json +6 -2
|
@@ -57,6 +57,8 @@ const crypto = __importStar(require("crypto"));
|
|
|
57
57
|
const child_process_1 = require("child_process");
|
|
58
58
|
const tree_sitter_1 = require("./tree-sitter");
|
|
59
59
|
const parse_pool_1 = require("./parse-pool");
|
|
60
|
+
const store_writer_1 = require("./store-writer");
|
|
61
|
+
const kernel_1 = require("./kernel");
|
|
60
62
|
const grammars_1 = require("./grammars");
|
|
61
63
|
const project_config_1 = require("../project-config");
|
|
62
64
|
const directory_1 = require("../directory");
|
|
@@ -807,6 +809,11 @@ function findUnindexedIgnoredRepos(rootDir) {
|
|
|
807
809
|
if (includeIgnored?.ignores((0, utils_1.normalizePath)(dir)))
|
|
808
810
|
continue; // already opted in — nothing to nag about
|
|
809
811
|
for (const repo of findNestedGitRepos(path.join(rootDir, dir), dir)) {
|
|
812
|
+
// Per-repo opt-in check, mirroring findIgnoredEmbeddedRepos: a child
|
|
813
|
+
// pattern (`repos/a/`) doesn't match the parent dir above but DOES
|
|
814
|
+
// cover this repo — it's indexed, so don't nag about it (#1295).
|
|
815
|
+
if (includeIgnored?.ignores((0, utils_1.normalizePath)(repo)))
|
|
816
|
+
continue;
|
|
810
817
|
repos.push(repo);
|
|
811
818
|
if (repos.length >= UNINDEXED_IGNORED_REPO_HINT_CAP)
|
|
812
819
|
return repos;
|
|
@@ -837,9 +844,22 @@ function findIgnoredEmbeddedRepos(repoDir, includeIgnored, prefix) {
|
|
|
837
844
|
for (const dir of listIgnoredDirs(repoDir)) {
|
|
838
845
|
if (defaults.ignores(dir))
|
|
839
846
|
continue;
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
847
|
+
const nested = findNestedGitRepos(path.join(repoDir, dir), dir);
|
|
848
|
+
if (includeIgnored.ignores((0, utils_1.normalizePath)(prefix + dir))) {
|
|
849
|
+
// The whole ignored dir is opted in — every nested repo under it counts.
|
|
850
|
+
repos.push(...nested);
|
|
851
|
+
}
|
|
852
|
+
else {
|
|
853
|
+
// A single gitignore rule often covers the PARENT of the opted-in
|
|
854
|
+
// repos: `.gitignore: /repos/` lists `repos/` as ONE ignored entry,
|
|
855
|
+
// while `includeIgnored: ["repos/a/"]` (the CLI hint's own suggested
|
|
856
|
+
// spelling) names the child — which never matches the parent path, so
|
|
857
|
+
// the opt-in silently did nothing (#1295). Match each nested repo
|
|
858
|
+
// root individually so both spellings work. The walk is bounded
|
|
859
|
+
// (depth/entry caps in findNestedGitRepos) and only runs when
|
|
860
|
+
// includeIgnored is configured at all.
|
|
861
|
+
repos.push(...nested.filter((r) => includeIgnored.ignores((0, utils_1.normalizePath)(prefix + r))));
|
|
862
|
+
}
|
|
843
863
|
}
|
|
844
864
|
return repos;
|
|
845
865
|
}
|
|
@@ -1280,6 +1300,36 @@ function scanDirectoryWalk(rootDir, onProgress) {
|
|
|
1280
1300
|
}
|
|
1281
1301
|
return files;
|
|
1282
1302
|
}
|
|
1303
|
+
/**
|
|
1304
|
+
* Resurrect a resolution edge that is about to be dropped (its target symbol
|
|
1305
|
+
* was removed, renamed, or its whole file deleted) as the ORIGINAL unresolved
|
|
1306
|
+
* reference that created it, read from the refName/refKind stamp
|
|
1307
|
+
* `createEdges` writes into edge metadata. Inserted as status='pending', the
|
|
1308
|
+
* ref is consumed by the same sync's resolution sweep: it rebinds to an
|
|
1309
|
+
* alternative definition if one exists, or parks as status='failed' where the
|
|
1310
|
+
* #1240 retry finds it if the symbol later reappears.
|
|
1311
|
+
*
|
|
1312
|
+
* Returns null — drop silently, the pre-#1240 behavior — for edges without a
|
|
1313
|
+
* refName stamp (created before the stamp existed, or synthesized): rebuilding
|
|
1314
|
+
* a ref from the target's plain node name would strip the receiver/qualifier
|
|
1315
|
+
* context the original text carried (`h.greet` → `greet`) and could rebind
|
|
1316
|
+
* somewhere a full re-index never would. Silent beats wrong.
|
|
1317
|
+
*/
|
|
1318
|
+
function resurrectRefFromDroppedEdge(e) {
|
|
1319
|
+
const refName = e.metadata?.refName;
|
|
1320
|
+
if (typeof refName !== 'string' || refName.length === 0)
|
|
1321
|
+
return null;
|
|
1322
|
+
const refKind = typeof e.metadata?.refKind === 'string' ? e.metadata.refKind : e.kind;
|
|
1323
|
+
return {
|
|
1324
|
+
fromNodeId: e.source,
|
|
1325
|
+
referenceName: refName,
|
|
1326
|
+
referenceKind: refKind,
|
|
1327
|
+
line: e.line ?? 0,
|
|
1328
|
+
column: e.column ?? 0,
|
|
1329
|
+
filePath: e.sourceFilePath,
|
|
1330
|
+
language: e.sourceLanguage,
|
|
1331
|
+
};
|
|
1332
|
+
}
|
|
1283
1333
|
/**
|
|
1284
1334
|
* Extraction orchestrator
|
|
1285
1335
|
*/
|
|
@@ -1377,8 +1427,17 @@ class ExtractionOrchestrator {
|
|
|
1377
1427
|
// null in the normal case, or a promise to await (at this safe,
|
|
1378
1428
|
// between-transactions boundary) when the WAL has outrun the off-thread
|
|
1379
1429
|
// checkpointer past its hard cap. See db/wal-valve.ts.
|
|
1380
|
-
walBackpressure
|
|
1430
|
+
walBackpressure,
|
|
1431
|
+
// Fresh-DB store offload (perf): when set, per-file store bundles are
|
|
1432
|
+
// applied by a dedicated writer thread instead of the main thread. Only
|
|
1433
|
+
// passed for a COMPLETELY fresh database, where the main thread performs
|
|
1434
|
+
// no reads/writes during the parse loop, so one writer applying bundles
|
|
1435
|
+
// in file order preserves the #1015 determinism exactly.
|
|
1436
|
+
storeWriterOpts) {
|
|
1437
|
+
const tGrammar = Date.now();
|
|
1381
1438
|
await (0, grammars_1.initGrammars)();
|
|
1439
|
+
if (process.env.CODEGRAPH_SYNTH_TIMINGS)
|
|
1440
|
+
console.error(`[phase-timing] grammar-init: ${Date.now() - tGrammar}ms`);
|
|
1382
1441
|
const startTime = Date.now();
|
|
1383
1442
|
const errors = [];
|
|
1384
1443
|
let filesIndexed = 0;
|
|
@@ -1461,8 +1520,15 @@ class ExtractionOrchestrator {
|
|
|
1461
1520
|
let pool = null;
|
|
1462
1521
|
if (useWorker) {
|
|
1463
1522
|
// CODEGRAPH_PARSE_WORKERS: explicit worker count; 1 = the old single-worker
|
|
1464
|
-
// behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8)
|
|
1465
|
-
|
|
1523
|
+
// behaviour (the conservative rollback). Unset → clamp(cores-1, 1, 8),
|
|
1524
|
+
// with cores from availableParallelism — cpuset/affinity-honest, where
|
|
1525
|
+
// os.cpus() enumerates the host's CPUs and spawned 8 wasm workers (and
|
|
1526
|
+
// their grammar heaps) inside a 2-CPU container for zero extra
|
|
1527
|
+
// throughput (§7a.1). Floored so a 2-core box still gets 2 workers:
|
|
1528
|
+
// parse is worker-side CPU, and 1 worker measured 34% slower than the
|
|
1529
|
+
// old oversubscribed pool on the kernel-scale 2-cpuset envelope
|
|
1530
|
+
// (493s vs 369s) — main + store-worker don't fill the second core.
|
|
1531
|
+
const poolSize = (0, parse_pool_1.resolveParsePoolSize)(process.env.CODEGRAPH_PARSE_WORKERS, Math.max(3, os.availableParallelism()));
|
|
1466
1532
|
// Read each needed grammar's WASM ONCE here and hand the bytes to every
|
|
1467
1533
|
// worker, so spawns/respawns load grammars from memory instead of
|
|
1468
1534
|
// re-reading them from disk (#1231: on an HDD, respawn re-reads amplify
|
|
@@ -1478,11 +1544,32 @@ class ExtractionOrchestrator {
|
|
|
1478
1544
|
grammarBuffers,
|
|
1479
1545
|
});
|
|
1480
1546
|
log(`Parse worker pool: ${poolSize} worker(s)`);
|
|
1547
|
+
// Bulk index: every core will be needed — spawn the whole pool now so
|
|
1548
|
+
// worker boot overlaps the first read batches instead of trickling in
|
|
1549
|
+
// behind queue-pressure growth.
|
|
1550
|
+
pool.prewarm();
|
|
1481
1551
|
}
|
|
1482
1552
|
else {
|
|
1483
1553
|
// In-process fallback: load grammars locally and parse on the main thread.
|
|
1484
1554
|
await (0, grammars_1.loadGrammarsForLanguages)(neededLanguages);
|
|
1485
1555
|
}
|
|
1556
|
+
// Dedicated store writer thread (fresh DB only — see the parameter doc).
|
|
1557
|
+
// Same availability rule as the parse pool: needs the compiled worker
|
|
1558
|
+
// (absent when running from source in tests → main-thread fallback).
|
|
1559
|
+
const storeWorkerPath = path.join(__dirname, 'store-worker.js');
|
|
1560
|
+
let storeWriter = null;
|
|
1561
|
+
if (storeWriterOpts &&
|
|
1562
|
+
process.env.CODEGRAPH_NO_STORE_WORKER !== '1' &&
|
|
1563
|
+
fs.existsSync(storeWorkerPath)) {
|
|
1564
|
+
// Deliberately NOT awaiting ready(): worker_threads delivers messages in
|
|
1565
|
+
// order, so bundles posted while the worker is still booting queue
|
|
1566
|
+
// behind 'open'. A boot failure surfaces at the first drain() — same
|
|
1567
|
+
// propagation point as a store error.
|
|
1568
|
+
storeWriter = new store_writer_1.StoreWriter(storeWorkerPath, storeWriterOpts.dbPath, storeWriterOpts.fastInit);
|
|
1569
|
+
log('Store writer thread active');
|
|
1570
|
+
}
|
|
1571
|
+
/** Queue-depth bound for un-acked bundles (bundles hold whole node/edge arrays). */
|
|
1572
|
+
const STORE_WRITER_WINDOW = 64;
|
|
1486
1573
|
/**
|
|
1487
1574
|
* Parse one file: on the pool when available (the promise REJECTS on a worker
|
|
1488
1575
|
* crash/timeout — the caller records it and the retry pass re-attempts), or
|
|
@@ -1524,10 +1611,36 @@ class ExtractionOrchestrator {
|
|
|
1524
1611
|
const bp = walBackpressure?.();
|
|
1525
1612
|
if (bp)
|
|
1526
1613
|
await bp;
|
|
1527
|
-
//
|
|
1528
|
-
|
|
1614
|
+
// Kernel deferred-decode results carry table sizes in kernelCounts
|
|
1615
|
+
// (their object arrays are empty — decode happens at the store).
|
|
1616
|
+
const nodeCount = result.kernelCounts?.nodes ?? result.nodes.length;
|
|
1617
|
+
const edgeCount = result.kernelCounts?.edges ?? result.edges.length;
|
|
1618
|
+
// Store: on the writer thread when active (fresh DB — bundles applied
|
|
1619
|
+
// in the same file order this chain dispatches them), else on the main
|
|
1620
|
+
// thread (SQLite connections are per-thread).
|
|
1621
|
+
if (nodeCount > 0 || result.errors.length === 0) {
|
|
1529
1622
|
const language = (0, grammars_1.detectLanguage)(filePath, content, overrides);
|
|
1530
|
-
|
|
1623
|
+
if (storeWriter) {
|
|
1624
|
+
if (result.kernelBuffers) {
|
|
1625
|
+
// Buffers go to the writer as-is; the worker decodes + finalizes.
|
|
1626
|
+
// The main thread's only per-file work stays O(1) + the content hash.
|
|
1627
|
+
storeWriter.send({
|
|
1628
|
+
kernel: true,
|
|
1629
|
+
filePath,
|
|
1630
|
+
language,
|
|
1631
|
+
buffers: result.kernelBuffers,
|
|
1632
|
+
file: this.buildFileRecord(filePath, content, language, stats, nodeCount, result.errors),
|
|
1633
|
+
});
|
|
1634
|
+
}
|
|
1635
|
+
else {
|
|
1636
|
+
storeWriter.send(this.buildFreshStoreBundle(filePath, content, language, stats, result));
|
|
1637
|
+
}
|
|
1638
|
+
await storeWriter.waitBelow(STORE_WRITER_WINDOW);
|
|
1639
|
+
}
|
|
1640
|
+
else {
|
|
1641
|
+
const materialized = (0, kernel_1.materializeKernelResult)(result, filePath, language);
|
|
1642
|
+
await this.storeExtractionResult(filePath, content, language, stats, materialized, commitYield);
|
|
1643
|
+
}
|
|
1531
1644
|
}
|
|
1532
1645
|
if (result.errors.length > 0) {
|
|
1533
1646
|
for (const err of result.errors) {
|
|
@@ -1536,10 +1649,10 @@ class ExtractionOrchestrator {
|
|
|
1536
1649
|
}
|
|
1537
1650
|
errors.push(...result.errors);
|
|
1538
1651
|
}
|
|
1539
|
-
if (
|
|
1652
|
+
if (nodeCount > 0) {
|
|
1540
1653
|
filesIndexed++;
|
|
1541
|
-
totalNodes +=
|
|
1542
|
-
totalEdges +=
|
|
1654
|
+
totalNodes += nodeCount;
|
|
1655
|
+
totalEdges += edgeCount;
|
|
1543
1656
|
}
|
|
1544
1657
|
else if (result.errors.some((e) => e.severity === 'error')) {
|
|
1545
1658
|
filesErrored++;
|
|
@@ -1633,6 +1746,7 @@ class ExtractionOrchestrator {
|
|
|
1633
1746
|
await flushOrdered();
|
|
1634
1747
|
}
|
|
1635
1748
|
};
|
|
1749
|
+
const tParseLoop = Date.now();
|
|
1636
1750
|
for (let i = 0; i < files.length; i += FILE_IO_BATCH_SIZE) {
|
|
1637
1751
|
if (signal?.aborted) {
|
|
1638
1752
|
aborted = true;
|
|
@@ -1706,10 +1820,29 @@ class ExtractionOrchestrator {
|
|
|
1706
1820
|
if (!aborted) {
|
|
1707
1821
|
await Promise.all(inFlight);
|
|
1708
1822
|
await flushOrdered();
|
|
1709
|
-
if (flushError)
|
|
1823
|
+
if (flushError) {
|
|
1824
|
+
if (storeWriter)
|
|
1825
|
+
await storeWriter.close();
|
|
1710
1826
|
throw flushError;
|
|
1827
|
+
}
|
|
1828
|
+
// All bundles are posted; wait for the writer to apply them, then close
|
|
1829
|
+
// its connection BEFORE any main-thread DB work below (retry pass,
|
|
1830
|
+
// resolution) so exactly one connection writes at a time.
|
|
1831
|
+
if (storeWriter) {
|
|
1832
|
+
try {
|
|
1833
|
+
await storeWriter.drain();
|
|
1834
|
+
}
|
|
1835
|
+
finally {
|
|
1836
|
+
await storeWriter.close();
|
|
1837
|
+
storeWriter = null;
|
|
1838
|
+
}
|
|
1839
|
+
}
|
|
1711
1840
|
}
|
|
1841
|
+
if (process.env.CODEGRAPH_SYNTH_TIMINGS)
|
|
1842
|
+
console.error(`[phase-timing] parse-loop: ${Date.now() - tParseLoop}ms`);
|
|
1712
1843
|
if (signal?.aborted || aborted) {
|
|
1844
|
+
if (storeWriter)
|
|
1845
|
+
await storeWriter.close();
|
|
1713
1846
|
if (pool)
|
|
1714
1847
|
await pool.destroy();
|
|
1715
1848
|
return {
|
|
@@ -2037,15 +2170,52 @@ class ExtractionOrchestrator {
|
|
|
2037
2170
|
// This prevents FK violations when edges reference nodes that would
|
|
2038
2171
|
// be silently skipped by insertNode() (see issue #42).
|
|
2039
2172
|
const validNodes = result.nodes.filter((n) => n.id && n.kind && n.name && n.filePath && n.language);
|
|
2173
|
+
const insertedIds = new Set(validNodes.map((n) => n.id));
|
|
2174
|
+
const validEdges = result.edges.filter((e) => insertedIds.has(e.source) && insertedIds.has(e.target));
|
|
2175
|
+
const validRefs = result.unresolvedReferences
|
|
2176
|
+
.filter((ref) => insertedIds.has(ref.fromNodeId))
|
|
2177
|
+
.map((ref) => ({
|
|
2178
|
+
...ref,
|
|
2179
|
+
filePath: ref.filePath ?? filePath,
|
|
2180
|
+
language: ref.language ?? language,
|
|
2181
|
+
}));
|
|
2182
|
+
// Fast path for the common case (everything fits one chunk): the whole
|
|
2183
|
+
// file — nodes, edges, refs, file record — lands in ONE transaction with
|
|
2184
|
+
// no event-loop yields in between. Giant generated files keep the chunked
|
|
2185
|
+
// + yielding path below so the #850 watchdog heartbeat stays serviced.
|
|
2186
|
+
const fitsOneChunk = validNodes.length <= STORE_CHUNK &&
|
|
2187
|
+
validEdges.length <= STORE_CHUNK &&
|
|
2188
|
+
validRefs.length <= STORE_CHUNK;
|
|
2189
|
+
if (fitsOneChunk) {
|
|
2190
|
+
// Snapshot/re-resolution of cross-file incoming edges (below) still runs
|
|
2191
|
+
// for the sync path; on a fresh bulk index crossFileIncomingEdges is [].
|
|
2192
|
+
this.queries.storeFileBundle({
|
|
2193
|
+
nodes: validNodes,
|
|
2194
|
+
edges: validEdges,
|
|
2195
|
+
refs: validRefs,
|
|
2196
|
+
file: {
|
|
2197
|
+
path: filePath,
|
|
2198
|
+
contentHash,
|
|
2199
|
+
language,
|
|
2200
|
+
size: stats.size,
|
|
2201
|
+
modifiedAt: stats.mtimeMs,
|
|
2202
|
+
indexedAt: Date.now(),
|
|
2203
|
+
nodeCount: result.nodes.length,
|
|
2204
|
+
errors: result.errors.length > 0 ? result.errors : undefined,
|
|
2205
|
+
},
|
|
2206
|
+
});
|
|
2207
|
+
if (crossFileIncomingEdges.length > 0) {
|
|
2208
|
+
this.reattachCrossFileEdges(crossFileIncomingEdges, validNodes);
|
|
2209
|
+
}
|
|
2210
|
+
return;
|
|
2211
|
+
}
|
|
2040
2212
|
// Insert nodes (chunked — see STORE_CHUNK above)
|
|
2041
2213
|
for (let i = 0; i < validNodes.length; i += STORE_CHUNK) {
|
|
2042
2214
|
this.queries.insertNodes(validNodes.slice(i, i + STORE_CHUNK));
|
|
2043
2215
|
await onYield?.();
|
|
2044
2216
|
}
|
|
2045
2217
|
// Filter edges to only reference nodes that were actually inserted
|
|
2046
|
-
if (
|
|
2047
|
-
const insertedIds = new Set(validNodes.map((n) => n.id));
|
|
2048
|
-
const validEdges = result.edges.filter((e) => insertedIds.has(e.source) && insertedIds.has(e.target));
|
|
2218
|
+
if (validEdges.length > 0) {
|
|
2049
2219
|
for (let i = 0; i < validEdges.length; i += STORE_CHUNK) {
|
|
2050
2220
|
this.queries.insertEdges(validEdges.slice(i, i + STORE_CHUNK));
|
|
2051
2221
|
await onYield?.();
|
|
@@ -2056,40 +2226,26 @@ class ExtractionOrchestrator {
|
|
|
2056
2226
|
// (filePath, kind, name). Node ids include the source line, so any line
|
|
2057
2227
|
// shift in the callee file (e.g. a docstring-only edit above the symbol)
|
|
2058
2228
|
// changes every target id and a naive re-insert by old id would drop them
|
|
2059
|
-
// all. `insertEdges` still filters to endpoints that exist
|
|
2060
|
-
//
|
|
2061
|
-
//
|
|
2062
|
-
//
|
|
2229
|
+
// all. `insertEdges` still filters to endpoints that exist. This closes
|
|
2230
|
+
// the #899 edge-drop on `sync`.
|
|
2231
|
+
//
|
|
2232
|
+
// Edges whose callee (target) was renamed/removed during the re-index (no
|
|
2233
|
+
// match in `newNodesByKindName`) are not silently dropped anymore: each is
|
|
2234
|
+
// resurrected as its ORIGINAL unresolved ref (stamped on the edge as
|
|
2235
|
+
// metadata.refName/refKind at creation) so the same sync's resolution
|
|
2236
|
+
// sweep can rebind it to an alternative definition elsewhere, or park it
|
|
2237
|
+
// as status='failed' to be retried when the symbol reappears — the
|
|
2238
|
+
// removal-side counterpart of #1240. Edges without refName (built before
|
|
2239
|
+
// the stamp existed, or synthesized) still drop silently: reconstructing
|
|
2240
|
+
// a ref from the target's plain name would strip receiver/qualifier
|
|
2241
|
+
// context and risk a rebind a full re-index would never make.
|
|
2063
2242
|
if (crossFileIncomingEdges.length > 0) {
|
|
2064
|
-
|
|
2065
|
-
for (const n of validNodes) {
|
|
2066
|
-
newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
|
|
2067
|
-
}
|
|
2068
|
-
const reinserted = [];
|
|
2069
|
-
for (const e of crossFileIncomingEdges) {
|
|
2070
|
-
const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
|
|
2071
|
-
if (newTargetId) {
|
|
2072
|
-
reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
|
|
2073
|
-
}
|
|
2074
|
-
}
|
|
2075
|
-
if (reinserted.length > 0) {
|
|
2076
|
-
this.queries.insertEdges(reinserted);
|
|
2077
|
-
}
|
|
2243
|
+
this.reattachCrossFileEdges(crossFileIncomingEdges, validNodes);
|
|
2078
2244
|
}
|
|
2079
2245
|
// Insert unresolved references in batch with denormalized filePath/language
|
|
2080
|
-
|
|
2081
|
-
|
|
2082
|
-
|
|
2083
|
-
.filter((ref) => insertedIds.has(ref.fromNodeId))
|
|
2084
|
-
.map((ref) => ({
|
|
2085
|
-
...ref,
|
|
2086
|
-
filePath: ref.filePath ?? filePath,
|
|
2087
|
-
language: ref.language ?? language,
|
|
2088
|
-
}));
|
|
2089
|
-
for (let i = 0; i < refsWithContext.length; i += STORE_CHUNK) {
|
|
2090
|
-
this.queries.insertUnresolvedRefsBatch(refsWithContext.slice(i, i + STORE_CHUNK));
|
|
2091
|
-
await onYield?.();
|
|
2092
|
-
}
|
|
2246
|
+
for (let i = 0; i < validRefs.length; i += STORE_CHUNK) {
|
|
2247
|
+
this.queries.insertUnresolvedRefsBatch(validRefs.slice(i, i + STORE_CHUNK));
|
|
2248
|
+
await onYield?.();
|
|
2093
2249
|
}
|
|
2094
2250
|
// Insert file record
|
|
2095
2251
|
const fileRecord = {
|
|
@@ -2104,6 +2260,59 @@ class ExtractionOrchestrator {
|
|
|
2104
2260
|
};
|
|
2105
2261
|
this.queries.upsertFile(fileRecord);
|
|
2106
2262
|
}
|
|
2263
|
+
/**
|
|
2264
|
+
* Build one file's store bundle for the FRESH-DB path: no existing-file
|
|
2265
|
+
* check, no cross-file edge snapshot (both are re-index concerns — a fresh
|
|
2266
|
+
* database has neither). Filters mirror storeExtractionResult exactly.
|
|
2267
|
+
*/
|
|
2268
|
+
/** The FileRecord for a fresh-index store (nodeCount is the PRE-filter count). */
|
|
2269
|
+
buildFileRecord(filePath, content, language, stats, nodeCount, resultErrors) {
|
|
2270
|
+
return {
|
|
2271
|
+
path: filePath,
|
|
2272
|
+
contentHash: hashContent(content),
|
|
2273
|
+
language,
|
|
2274
|
+
size: stats.size,
|
|
2275
|
+
modifiedAt: stats.mtimeMs,
|
|
2276
|
+
indexedAt: Date.now(),
|
|
2277
|
+
nodeCount,
|
|
2278
|
+
errors: resultErrors.length > 0 ? resultErrors : undefined,
|
|
2279
|
+
};
|
|
2280
|
+
}
|
|
2281
|
+
buildFreshStoreBundle(filePath, content, language, stats, result) {
|
|
2282
|
+
return (0, store_writer_1.finalizeStoreBundle)(result, filePath, language, this.buildFileRecord(filePath, content, language, stats, result.nodes.length, result.errors));
|
|
2283
|
+
}
|
|
2284
|
+
/**
|
|
2285
|
+
* Re-attach cross-file incoming edges snapshotted before a re-index delete
|
|
2286
|
+
* (#899): re-resolve each edge's target to the re-indexed node's new id by
|
|
2287
|
+
* (kind, name); targets that vanished are resurrected as their original
|
|
2288
|
+
* unresolved ref (#1240's removal-side counterpart) when the edge carries
|
|
2289
|
+
* its refName stamp.
|
|
2290
|
+
*/
|
|
2291
|
+
reattachCrossFileEdges(crossFileIncomingEdges, validNodes) {
|
|
2292
|
+
const newNodesByKindName = new Map();
|
|
2293
|
+
for (const n of validNodes) {
|
|
2294
|
+
newNodesByKindName.set(`${n.kind}\0${n.name}`, n.id);
|
|
2295
|
+
}
|
|
2296
|
+
const reinserted = [];
|
|
2297
|
+
const resurrected = [];
|
|
2298
|
+
for (const e of crossFileIncomingEdges) {
|
|
2299
|
+
const newTargetId = newNodesByKindName.get(`${e.targetKind}\0${e.targetName}`);
|
|
2300
|
+
if (newTargetId) {
|
|
2301
|
+
reinserted.push({ source: e.source, target: newTargetId, kind: e.kind, metadata: e.metadata, line: e.line, column: e.column, provenance: e.provenance });
|
|
2302
|
+
}
|
|
2303
|
+
else {
|
|
2304
|
+
const ref = resurrectRefFromDroppedEdge(e);
|
|
2305
|
+
if (ref)
|
|
2306
|
+
resurrected.push(ref);
|
|
2307
|
+
}
|
|
2308
|
+
}
|
|
2309
|
+
if (reinserted.length > 0) {
|
|
2310
|
+
this.queries.insertEdges(reinserted);
|
|
2311
|
+
}
|
|
2312
|
+
if (resurrected.length > 0) {
|
|
2313
|
+
this.queries.insertUnresolvedRefsBatch(resurrected);
|
|
2314
|
+
}
|
|
2315
|
+
}
|
|
2107
2316
|
/**
|
|
2108
2317
|
* Sync the index with the current file state.
|
|
2109
2318
|
*
|
|
@@ -2112,7 +2321,17 @@ class ExtractionOrchestrator {
|
|
|
2112
2321
|
* changes. This works in non-git projects and catches committed changes from
|
|
2113
2322
|
* `git pull`/`checkout`/`merge`/`rebase` that `git status` cannot see.
|
|
2114
2323
|
*/
|
|
2115
|
-
async sync(onProgress
|
|
2324
|
+
async sync(onProgress,
|
|
2325
|
+
/**
|
|
2326
|
+
* Watcher fast path: the exact project-relative paths the OS reported as
|
|
2327
|
+
* changed. When provided, reconciliation runs over ONLY these paths —
|
|
2328
|
+
* per-path logic identical to the full walk (stat pre-filter, hash
|
|
2329
|
+
* confirm, the #1240 removal/resurrection flow) — skipping the O(repo)
|
|
2330
|
+
* scan and tracked-load. Callers must pass undefined whenever the change
|
|
2331
|
+
* set is not exactly known (directory removals, event overflow): the full
|
|
2332
|
+
* scan-diff remains the ground truth those cases need (#1285).
|
|
2333
|
+
*/
|
|
2334
|
+
scopedPaths) {
|
|
2116
2335
|
await (0, grammars_1.initGrammars)(); // Initialize WASM runtime (grammars loaded lazily below)
|
|
2117
2336
|
const startTime = Date.now();
|
|
2118
2337
|
let filesChecked = 0;
|
|
@@ -2137,15 +2356,37 @@ class ExtractionOrchestrator {
|
|
|
2137
2356
|
// changes from `git pull`/`checkout`/`merge`/`rebase` — which `git status`
|
|
2138
2357
|
// cannot see, because the working tree is clean afterward.
|
|
2139
2358
|
const tSyncScan = Date.now();
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
|
|
2359
|
+
let currentFiles;
|
|
2360
|
+
let trackedFiles;
|
|
2361
|
+
if (scopedPaths && scopedPaths.length > 0) {
|
|
2362
|
+
// Scoped reconcile: stat only the reported paths. filesChecked counts
|
|
2363
|
+
// the PATHS examined (not the files found) — it must stay non-zero even
|
|
2364
|
+
// when every scoped path was a deletion, because CodeGraph.watch()
|
|
2365
|
+
// reads `filesChecked === 0 && durationMs === 0` as the
|
|
2366
|
+
// lock-unavailable signature (#449).
|
|
2367
|
+
const unique = [...new Set(scopedPaths)];
|
|
2368
|
+
currentFiles = unique.filter((p) => fs.existsSync(path.join(this.rootDir, p)));
|
|
2369
|
+
trackedFiles = [];
|
|
2370
|
+
for (const p of unique) {
|
|
2371
|
+
const rec = this.queries.getFileByPath(p);
|
|
2372
|
+
if (rec)
|
|
2373
|
+
trackedFiles.push(rec);
|
|
2374
|
+
}
|
|
2375
|
+
filesChecked = unique.length;
|
|
2376
|
+
if (process.env.CODEGRAPH_SYNTH_TIMINGS)
|
|
2377
|
+
console.error(`[phase-timing] sync-scoped: ${Date.now() - tSyncScan}ms (${unique.length} paths, ${trackedFiles.length} tracked)`);
|
|
2378
|
+
}
|
|
2379
|
+
else {
|
|
2380
|
+
currentFiles = await scanDirectoryAsync(this.rootDir);
|
|
2381
|
+
if (process.env.CODEGRAPH_SYNTH_TIMINGS)
|
|
2382
|
+
console.error(`[phase-timing] sync-scan: ${Date.now() - tSyncScan}ms (${currentFiles.length} files)`);
|
|
2383
|
+
filesChecked = currentFiles.length;
|
|
2384
|
+
const tTracked = Date.now();
|
|
2385
|
+
trackedFiles = this.queries.getAllFiles();
|
|
2386
|
+
if (process.env.CODEGRAPH_SYNTH_TIMINGS)
|
|
2387
|
+
console.error(`[phase-timing] sync-tracked-load: ${Date.now() - tTracked}ms (${trackedFiles.length} tracked)`);
|
|
2388
|
+
}
|
|
2144
2389
|
const currentSet = new Set(currentFiles);
|
|
2145
|
-
const tTracked = Date.now();
|
|
2146
|
-
const trackedFiles = this.queries.getAllFiles();
|
|
2147
|
-
if (process.env.CODEGRAPH_SYNTH_TIMINGS)
|
|
2148
|
-
console.error(`[phase-timing] sync-tracked-load: ${Date.now() - tTracked}ms (${trackedFiles.length} tracked)`);
|
|
2149
2390
|
const trackedMap = new Map();
|
|
2150
2391
|
for (const f of trackedFiles) {
|
|
2151
2392
|
trackedMap.set(f.path, f);
|
|
@@ -2158,6 +2399,22 @@ class ExtractionOrchestrator {
|
|
|
2158
2399
|
let reconcileChecks = 0;
|
|
2159
2400
|
for (const tracked of trackedFiles) {
|
|
2160
2401
|
if (!currentSet.has(tracked.path) || !fs.existsSync(path.join(this.rootDir, tracked.path))) {
|
|
2402
|
+
// Before the cascade deletes them, resurrect incoming cross-file
|
|
2403
|
+
// resolution edges as their original refs (#1240 removal case): the
|
|
2404
|
+
// callers live in files this sync will NOT revisit, so this is their
|
|
2405
|
+
// only chance to rebind to an alternative definition — or to park as
|
|
2406
|
+
// failed until the symbol reappears somewhere. (A deleted file whose
|
|
2407
|
+
// CALLERS are also being deleted is fine: their nodes cascade later
|
|
2408
|
+
// in this loop and take the resurrected rows with them.)
|
|
2409
|
+
const incoming = this.queries.getCrossFileIncomingEdgesWithTarget(tracked.path);
|
|
2410
|
+
if (incoming.length > 0) {
|
|
2411
|
+
const resurrected = incoming
|
|
2412
|
+
.map((e) => resurrectRefFromDroppedEdge(e))
|
|
2413
|
+
.filter((r) => r !== null);
|
|
2414
|
+
if (resurrected.length > 0) {
|
|
2415
|
+
this.queries.insertUnresolvedRefsBatch(resurrected);
|
|
2416
|
+
}
|
|
2417
|
+
}
|
|
2161
2418
|
this.queries.deleteFile(tracked.path);
|
|
2162
2419
|
filesRemoved++;
|
|
2163
2420
|
}
|