gitnexus 1.6.8 → 1.6.9-rc.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/analyze.js +30 -7
- package/dist/core/ingestion/cfg/visitors/python-harvest.d.ts +9 -11
- package/dist/core/ingestion/cfg/visitors/python-harvest.js +85 -52
- package/dist/core/ingestion/scope-resolution/pipeline/run.js +5 -5
- package/dist/core/ingestion/taint/python-model.d.ts +10 -0
- package/dist/core/ingestion/taint/python-model.js +43 -0
- package/dist/core/ingestion/taint/typescript-model.d.ts +13 -5
- package/dist/core/ingestion/taint/typescript-model.js +20 -7
- package/dist/core/ingestion/workers/parse-worker.js +35 -27
- package/dist/core/lbug/conn-lock.d.ts +1 -0
- package/dist/core/lbug/conn-lock.js +61 -0
- package/dist/core/lbug/lbug-adapter.d.ts +21 -0
- package/dist/core/lbug/lbug-adapter.js +330 -196
- package/dist/core/lbug/shutdown-helpers.d.ts +18 -0
- package/dist/core/lbug/shutdown-helpers.js +36 -0
- package/dist/core/lbug/wal-checkpoint-driver.js +16 -0
- package/dist/core/lbug/wal-driver-state.d.ts +4 -0
- package/dist/core/lbug/wal-driver-state.js +23 -0
- package/dist/core/run-analyze.d.ts +9 -0
- package/dist/core/run-analyze.js +30 -5
- package/dist/server/analyze-job.js +6 -0
- package/dist/server/analyze-launch.js +7 -0
- package/dist/server/analyze-worker-core.d.ts +43 -0
- package/dist/server/analyze-worker-core.js +56 -0
- package/dist/server/analyze-worker.js +63 -32
- package/dist/storage/repo-manager.d.ts +14 -0
- package/dist/storage/repo-manager.js +24 -17
- package/package.json +1 -1
|
@@ -6,6 +6,8 @@ import { finished } from 'stream/promises';
|
|
|
6
6
|
import path from 'path';
|
|
7
7
|
import lbug from '@ladybugdb/core';
|
|
8
8
|
import { closeQueryResults } from './query-result-utils.js';
|
|
9
|
+
import { withConnLock } from './conn-lock.js';
|
|
10
|
+
import { isWalDriverActive } from './wal-driver-state.js';
|
|
9
11
|
import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, CREATE_VECTOR_INDEX_QUERY, STALE_HASH_SENTINEL, } from './schema.js';
|
|
10
12
|
import { streamAllCSVsToDisk } from './csv-generator.js';
|
|
11
13
|
import { getNodeLabel as deriveNodeLabel } from './rel-pair-routing.js';
|
|
@@ -123,6 +125,23 @@ export const splitRelCsvByLabelPair = async (csvPath, csvDir, validTables, getNo
|
|
|
123
125
|
};
|
|
124
126
|
let db = null;
|
|
125
127
|
let conn = null;
|
|
128
|
+
// Serialize every operation on the shared singleton `conn`. LadybugDB's
|
|
129
|
+
// Connection is single-writer and is NOT safe for concurrent query execution;
|
|
130
|
+
// the periodic WAL-checkpoint driver overlapping a long `--pdg` COPY on this
|
|
131
|
+
// connection corrupted native state (`double free or corruption`). Each
|
|
132
|
+
// singleton-`conn` helper below runs its full query + drain inside withConnLock.
|
|
133
|
+
// Invariant: a wrapped helper MUST NOT call another wrapped helper (re-entry
|
|
134
|
+
// self-deadlocks); all current holders are leaf-level. `streamQuery` is
|
|
135
|
+
// deliberately NOT wrapped — its per-row callback can re-enter the adapter and
|
|
136
|
+
// it only runs on the read path where the checkpoint driver is inactive.
|
|
137
|
+
// See conn-lock.ts for the full rationale.
|
|
138
|
+
//
|
|
139
|
+
// The gate that decides whether an op must take withConnLock: only operations on
|
|
140
|
+
// the shared singleton `conn` serialize. Per-file / temp connections (distinct
|
|
141
|
+
// native objects with no shared engine state) must NOT block on — or be blocked
|
|
142
|
+
// by — the singleton's lock. Reads the live `conn` binding at call time (it's
|
|
143
|
+
// reassigned only at open/close, never mid-load).
|
|
144
|
+
const isSharedSingletonConn = (c) => c === conn;
|
|
126
145
|
let currentDbPath = null;
|
|
127
146
|
let currentDbReadOnly = false;
|
|
128
147
|
let ftsLoaded = false;
|
|
@@ -379,8 +398,14 @@ const readQueryRows = async (queryResult) => {
|
|
|
379
398
|
return rows;
|
|
380
399
|
};
|
|
381
400
|
const queryAndDrain = async (targetConn, cypher) => {
|
|
382
|
-
const
|
|
383
|
-
|
|
401
|
+
const run = async () => {
|
|
402
|
+
const queryResult = await targetConn.query(cypher);
|
|
403
|
+
await drainQueryResult(queryResult);
|
|
404
|
+
};
|
|
405
|
+
// Serialize only when this runs on the shared singleton connection (the bulk
|
|
406
|
+
// node/relationship COPY captures `writeConn = conn`); per-file / temp
|
|
407
|
+
// connections skip the lock — see isSharedSingletonConn.
|
|
408
|
+
return isSharedSingletonConn(targetConn) ? withConnLock(run) : run();
|
|
384
409
|
};
|
|
385
410
|
const READ_ONLY_SHADOW_REPLAY_PROBE = 'MATCH (n) RETURN n LIMIT 1';
|
|
386
411
|
/**
|
|
@@ -940,7 +965,12 @@ pdgEmitManifest) => {
|
|
|
940
965
|
if (pairIdx % 5 === 0 || rows > 1000) {
|
|
941
966
|
log(`Loading edges: ${pairIdx}/${relsByPair.size} types (${fromLabel} -> ${toLabel})`);
|
|
942
967
|
}
|
|
943
|
-
|
|
968
|
+
// Use the captured `writeConn` (not the module-level `conn`) for the rel
|
|
969
|
+
// COPY, matching the node COPY above — one captured reference for the whole
|
|
970
|
+
// bulk load (#2264 review P3). Same object during analyze (`conn` is only
|
|
971
|
+
// reassigned at open/close under the session lock, never mid-load), so the
|
|
972
|
+
// queryAndDrain `targetConn === conn` lock gate still engages.
|
|
973
|
+
await copyCsvWithRetry(writeConn, copyQuery, (retryErr) => {
|
|
944
974
|
const retryMsg = retryErr instanceof Error ? retryErr.message : String(retryErr);
|
|
945
975
|
warnings.push(`${fromLabel}->${toLabel} (${rows} edges): ${retryMsg.slice(0, 80)}`);
|
|
946
976
|
failedPairEdges += rows;
|
|
@@ -1308,6 +1338,16 @@ export const executeQuery = async (cypher) => {
|
|
|
1308
1338
|
return await executePrepared(cypher, {});
|
|
1309
1339
|
};
|
|
1310
1340
|
export const streamQuery = async (cypher, onRow) => {
|
|
1341
|
+
if (isWalDriverActive()) {
|
|
1342
|
+
// streamQuery reads rows on the singleton connection WITHOUT withConnLock; if
|
|
1343
|
+
// the WAL-checkpoint driver is live, those reads could race a CHECKPOINT — the
|
|
1344
|
+
// #2264 corruption window. Today the serve/read path never runs the driver
|
|
1345
|
+
// (analyze runs in a forked worker), so this fails loud only if a future
|
|
1346
|
+
// in-process analyze overlaps a stream. Run analysis in a worker, or stop the
|
|
1347
|
+
// driver before streaming. See conn-lock.ts.
|
|
1348
|
+
throw new Error('streamQuery cannot run while the WAL-checkpoint driver is active (it would ' +
|
|
1349
|
+
'race a CHECKPOINT on the unlocked read connection — #2264).');
|
|
1350
|
+
}
|
|
1311
1351
|
if (!conn) {
|
|
1312
1352
|
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
1313
1353
|
}
|
|
@@ -1343,19 +1383,23 @@ export const streamQuery = async (cypher, onRow) => {
|
|
|
1343
1383
|
* Prevents Cypher injection by binding values as parameters.
|
|
1344
1384
|
*/
|
|
1345
1385
|
export const executePrepared = async (cypher, params) => {
|
|
1346
|
-
|
|
1386
|
+
const c = conn;
|
|
1387
|
+
if (!c) {
|
|
1347
1388
|
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
1348
1389
|
}
|
|
1349
|
-
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
|
|
1355
|
-
|
|
1390
|
+
return withConnLock(async () => {
|
|
1391
|
+
const stmt = await c.prepare(cypher);
|
|
1392
|
+
if (!stmt.isSuccess()) {
|
|
1393
|
+
const errMsg = await stmt.getErrorMessage();
|
|
1394
|
+
throw new Error(`Prepare failed: ${errMsg}`);
|
|
1395
|
+
}
|
|
1396
|
+
const queryResult = await c.execute(stmt, params);
|
|
1397
|
+
return await readQueryRows(queryResult);
|
|
1398
|
+
});
|
|
1356
1399
|
};
|
|
1357
1400
|
export const executeWithReusedStatement = async (cypher, paramsList) => {
|
|
1358
|
-
|
|
1401
|
+
const c = conn;
|
|
1402
|
+
if (!c) {
|
|
1359
1403
|
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
1360
1404
|
}
|
|
1361
1405
|
if (paramsList.length === 0)
|
|
@@ -1363,35 +1407,46 @@ export const executeWithReusedStatement = async (cypher, paramsList) => {
|
|
|
1363
1407
|
const SUB_BATCH_SIZE = 4;
|
|
1364
1408
|
for (let i = 0; i < paramsList.length; i += SUB_BATCH_SIZE) {
|
|
1365
1409
|
const subBatch = paramsList.slice(i, i + SUB_BATCH_SIZE);
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
|
|
1373
|
-
|
|
1410
|
+
// One critical section per sub-batch: the prepare + its executes run with
|
|
1411
|
+
// exclusive access to the connection (so the WAL checkpoint driver cannot
|
|
1412
|
+
// interleave a CHECKPOINT mid-batch), while the lock is released between
|
|
1413
|
+
// sub-batches to let the driver checkpoint during a long writeback.
|
|
1414
|
+
await withConnLock(async () => {
|
|
1415
|
+
const stmt = await c.prepare(cypher);
|
|
1416
|
+
if (!stmt.isSuccess()) {
|
|
1417
|
+
const errMsg = await stmt.getErrorMessage();
|
|
1418
|
+
throw new Error(`Prepare failed: ${errMsg}`);
|
|
1374
1419
|
}
|
|
1375
|
-
|
|
1376
|
-
|
|
1377
|
-
|
|
1378
|
-
|
|
1379
|
-
|
|
1380
|
-
|
|
1381
|
-
|
|
1420
|
+
try {
|
|
1421
|
+
for (const params of subBatch) {
|
|
1422
|
+
await drainQueryResult(await c.execute(stmt, params));
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
catch (e) {
|
|
1426
|
+
const msg = e instanceof Error ? e.message : String(e);
|
|
1427
|
+
const queryPreview = cypher.replace(/\s+/g, ' ').slice(0, 120);
|
|
1428
|
+
throw new Error(`Batch execution failed for rows ${i + 1}-${i + subBatch.length}: ${msg} (${queryPreview})`);
|
|
1429
|
+
}
|
|
1430
|
+
// Note: LadybugDB PreparedStatement doesn't require explicit close()
|
|
1431
|
+
});
|
|
1382
1432
|
}
|
|
1383
1433
|
};
|
|
1384
1434
|
export const getLbugStats = async () => {
|
|
1385
|
-
|
|
1435
|
+
const c = conn;
|
|
1436
|
+
if (!c)
|
|
1386
1437
|
return { nodes: 0, edges: 0 };
|
|
1438
|
+
// Called during analyze finalize while the WAL-checkpoint driver is still
|
|
1439
|
+
// running; each count read takes the connection lock so it cannot execute
|
|
1440
|
+
// concurrently with a driver CHECKPOINT. Per-query locking lets the driver
|
|
1441
|
+
// checkpoint between table counts rather than waiting for the whole sweep.
|
|
1387
1442
|
let totalNodes = 0;
|
|
1388
1443
|
for (const tableName of NODE_TABLES) {
|
|
1389
1444
|
try {
|
|
1390
|
-
|
|
1391
|
-
|
|
1392
|
-
|
|
1393
|
-
|
|
1394
|
-
}
|
|
1445
|
+
totalNodes += await withConnLock(async () => {
|
|
1446
|
+
const queryResult = await c.query(`MATCH (n:${escapeTableName(tableName)}) RETURN count(n) AS cnt`);
|
|
1447
|
+
const nodeRows = await readQueryRows(queryResult);
|
|
1448
|
+
return nodeRows.length > 0 ? Number(nodeRows[0]?.cnt ?? nodeRows[0]?.[0] ?? 0) : 0;
|
|
1449
|
+
});
|
|
1395
1450
|
}
|
|
1396
1451
|
catch {
|
|
1397
1452
|
// ignore
|
|
@@ -1399,11 +1454,11 @@ export const getLbugStats = async () => {
|
|
|
1399
1454
|
}
|
|
1400
1455
|
let totalEdges = 0;
|
|
1401
1456
|
try {
|
|
1402
|
-
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
}
|
|
1457
|
+
totalEdges = await withConnLock(async () => {
|
|
1458
|
+
const queryResult = await c.query(`MATCH ()-[r:${REL_TABLE_NAME}]->() RETURN count(r) AS cnt`);
|
|
1459
|
+
const edgeRows = await readQueryRows(queryResult);
|
|
1460
|
+
return edgeRows.length > 0 ? Number(edgeRows[0]?.cnt ?? edgeRows[0]?.[0] ?? 0) : 0;
|
|
1461
|
+
});
|
|
1407
1462
|
}
|
|
1408
1463
|
catch {
|
|
1409
1464
|
// ignore
|
|
@@ -1418,63 +1473,71 @@ export const getLbugStats = async () => {
|
|
|
1418
1473
|
* Detects old schema (no chunkIndex column) and returns empty cache to trigger rebuild.
|
|
1419
1474
|
*/
|
|
1420
1475
|
export const loadCachedEmbeddings = async () => {
|
|
1421
|
-
|
|
1476
|
+
const c = conn;
|
|
1477
|
+
if (!c) {
|
|
1422
1478
|
return { embeddingNodeIds: new Set(), embeddings: [] };
|
|
1423
1479
|
}
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
await readQueryRows(check);
|
|
1433
|
-
}
|
|
1434
|
-
catch {
|
|
1435
|
-
return { embeddingNodeIds: new Set(), embeddings: [] };
|
|
1436
|
-
}
|
|
1437
|
-
// Try to read contentHash alongside chunk columns
|
|
1438
|
-
let rows;
|
|
1439
|
-
let hasContentHash = true;
|
|
1480
|
+
// The whole read runs inside the connection lock (#2264 review P2). It's safe
|
|
1481
|
+
// today only by call-ordering (loadCachedEmbeddings runs before the WAL driver
|
|
1482
|
+
// starts), but the lock makes it robust to future reordering — a concurrent
|
|
1483
|
+
// CHECKPOINT on the singleton connection is the documented corruption trigger.
|
|
1484
|
+
// Leaf read: no nested withConnLock-wrapped helpers inside.
|
|
1485
|
+
return withConnLock(async () => {
|
|
1486
|
+
const embeddingNodeIds = new Set();
|
|
1487
|
+
const embeddings = [];
|
|
1440
1488
|
try {
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
hasContentHash = false;
|
|
1448
|
-
rows = await conn.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding`);
|
|
1489
|
+
// Schema migration detection: query with new columns to verify schema version.
|
|
1490
|
+
// Old schema only had (nodeId, embedding); new schema adds (id, chunkIndex, startLine, endLine, contentHash).
|
|
1491
|
+
// If the query fails (column missing), we return empty cache to force a full rebuild.
|
|
1492
|
+
try {
|
|
1493
|
+
const check = await c.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex LIMIT 1`);
|
|
1494
|
+
await readQueryRows(check);
|
|
1449
1495
|
}
|
|
1450
|
-
|
|
1451
|
-
|
|
1496
|
+
catch {
|
|
1497
|
+
return { embeddingNodeIds: new Set(), embeddings: [] };
|
|
1452
1498
|
}
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1499
|
+
// Try to read contentHash alongside chunk columns
|
|
1500
|
+
let rows;
|
|
1501
|
+
let hasContentHash = true;
|
|
1502
|
+
try {
|
|
1503
|
+
rows = await c.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding, e.contentHash AS contentHash`);
|
|
1504
|
+
}
|
|
1505
|
+
catch (err) {
|
|
1506
|
+
// Fallback for legacy DBs without contentHash column
|
|
1507
|
+
const msg = err?.message ?? '';
|
|
1508
|
+
if (isMissingColumnOrTableError(msg)) {
|
|
1509
|
+
hasContentHash = false;
|
|
1510
|
+
rows = await c.query(`MATCH (e:${EMBEDDING_TABLE_NAME}) RETURN e.nodeId AS nodeId, e.chunkIndex AS chunkIndex, e.startLine AS startLine, e.endLine AS endLine, e.embedding AS embedding`);
|
|
1511
|
+
}
|
|
1512
|
+
else {
|
|
1513
|
+
throw err;
|
|
1514
|
+
}
|
|
1515
|
+
}
|
|
1516
|
+
for (const row of await readQueryRows(rows)) {
|
|
1517
|
+
const nodeId = String(row.nodeId ?? row[0] ?? '');
|
|
1518
|
+
if (!nodeId)
|
|
1519
|
+
continue;
|
|
1520
|
+
embeddingNodeIds.add(nodeId);
|
|
1521
|
+
const embedding = row.embedding ?? row[4];
|
|
1522
|
+
if (embedding) {
|
|
1523
|
+
embeddings.push({
|
|
1524
|
+
nodeId,
|
|
1525
|
+
chunkIndex: Number(row.chunkIndex ?? row[1] ?? 0),
|
|
1526
|
+
startLine: Number(row.startLine ?? row[2] ?? 0),
|
|
1527
|
+
endLine: Number(row.endLine ?? row[3] ?? 0),
|
|
1528
|
+
embedding: Array.isArray(embedding)
|
|
1529
|
+
? embedding.map(Number)
|
|
1530
|
+
: Array.from(embedding).map(Number),
|
|
1531
|
+
contentHash: hasContentHash ? (row.contentHash ?? row[5] ?? undefined) : undefined,
|
|
1532
|
+
});
|
|
1533
|
+
}
|
|
1471
1534
|
}
|
|
1472
1535
|
}
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1536
|
+
catch {
|
|
1537
|
+
/* embedding table may not exist */
|
|
1538
|
+
}
|
|
1539
|
+
return { embeddingNodeIds, embeddings };
|
|
1540
|
+
});
|
|
1478
1541
|
};
|
|
1479
1542
|
/**
|
|
1480
1543
|
* Fetch existing embedding hashes from CodeEmbedding table for incremental embedding.
|
|
@@ -1552,11 +1615,14 @@ export const fetchExistingEmbeddingHashes = async (execQuery) => {
|
|
|
1552
1615
|
* @see safeClose — CHECKPOINT + connection/database close
|
|
1553
1616
|
*/
|
|
1554
1617
|
export const flushWAL = async () => {
|
|
1555
|
-
|
|
1618
|
+
const c = conn;
|
|
1619
|
+
if (!c)
|
|
1556
1620
|
return;
|
|
1557
1621
|
try {
|
|
1558
|
-
|
|
1559
|
-
|
|
1622
|
+
await withConnLock(async () => {
|
|
1623
|
+
const checkpointResult = await c.query('CHECKPOINT');
|
|
1624
|
+
await drainQueryResult(checkpointResult);
|
|
1625
|
+
});
|
|
1560
1626
|
}
|
|
1561
1627
|
catch (err) {
|
|
1562
1628
|
logger.debug(`GitNexus: LadybugDB CHECKPOINT skipped/failed during WAL flush: ${summarizeError(err)}`);
|
|
@@ -1577,10 +1643,16 @@ export const flushWAL = async () => {
|
|
|
1577
1643
|
* whether to retry.
|
|
1578
1644
|
*/
|
|
1579
1645
|
export const tryFlushWAL = async () => {
|
|
1580
|
-
|
|
1646
|
+
const c = conn;
|
|
1647
|
+
if (!c)
|
|
1581
1648
|
return false;
|
|
1582
|
-
|
|
1583
|
-
|
|
1649
|
+
// Runs on the periodic WAL-checkpoint driver. The lock makes this CHECKPOINT
|
|
1650
|
+
// wait for any in-flight COPY / writeback on the singleton connection instead
|
|
1651
|
+
// of executing concurrently with it (the `analyze --pdg` heap-corruption bug).
|
|
1652
|
+
await withConnLock(async () => {
|
|
1653
|
+
const checkpointResult = await c.query('CHECKPOINT');
|
|
1654
|
+
await drainQueryResult(checkpointResult);
|
|
1655
|
+
});
|
|
1584
1656
|
return true;
|
|
1585
1657
|
};
|
|
1586
1658
|
/**
|
|
@@ -1637,6 +1709,38 @@ export const safeClose = async () => {
|
|
|
1637
1709
|
await finalizeLbugSidecarsAfterClose(closingDbPath, { logger });
|
|
1638
1710
|
}
|
|
1639
1711
|
};
|
|
1712
|
+
/**
|
|
1713
|
+
* CHECKPOINT for durability, then DELIBERATELY skip the native connection/database
|
|
1714
|
+
* teardown. The name encodes the contract — there is no boolean flag to misuse:
|
|
1715
|
+
* call this ONLY from a path that guarantees a `process.exit` immediately after
|
|
1716
|
+
* (the CLI analyze success/SIGINT paths and the forked worker).
|
|
1717
|
+
*
|
|
1718
|
+
* LadybugDB's ClientContext/Connection destructor can double-free after large
|
|
1719
|
+
* --pdg writes (gdb: `double free or corruption` in ClientContext::~ClientContext
|
|
1720
|
+
* via NodeConnection::Close), aborting the process AFTER a fully-written,
|
|
1721
|
+
* checkpointed index. flushWAL already persisted the data; process exit reclaims
|
|
1722
|
+
* the native handles. We leave the handles referenced and module state intact so a
|
|
1723
|
+
* GC finalizer cannot run the same destructor before exit, and any post-analyze
|
|
1724
|
+
* read reuses the live connection. Mirrors the pool adapter's fire-and-forget
|
|
1725
|
+
* native teardown (pool-adapter.ts) and the ONNX native-cleanup philosophy.
|
|
1726
|
+
* Workaround for a LadybugDB engine bug (to be reported upstream).
|
|
1727
|
+
*
|
|
1728
|
+
* SAFETY: only valid when a process.exit is guaranteed to follow. Long-lived
|
|
1729
|
+
* callers (MCP server, tests) leave `skipNativeCloseOnExit` unset, so
|
|
1730
|
+
* runFullAnalysis closes for real via {@link closeLbug} — never this.
|
|
1731
|
+
*/
|
|
1732
|
+
export const closeLbugBeforeExit = async () => {
|
|
1733
|
+
await flushWAL();
|
|
1734
|
+
// NOTE (#2264): unlike safeClose, this deliberately does NOT run
|
|
1735
|
+
// finalizeLbugSidecarsAfterClose. That step inspects/quarantines orphan WAL +
|
|
1736
|
+
// sidecar files and is designed to run AFTER the native close has released the
|
|
1737
|
+
// WAL handle; running it here — with the connection still open — would risk a
|
|
1738
|
+
// Windows file-lock on the in-use WAL for no benefit. The CHECKPOINT above
|
|
1739
|
+
// already made the index durable, and the next run's preflightLbugSidecars
|
|
1740
|
+
// reconciles any residual WAL on open. The deferred sidecar housekeeping is the
|
|
1741
|
+
// accepted trade-off of skipping the native close to dodge the destructor
|
|
1742
|
+
// double-free.
|
|
1743
|
+
};
|
|
1640
1744
|
export const closeLbug = async () => {
|
|
1641
1745
|
await safeClose();
|
|
1642
1746
|
currentDbPath = null;
|
|
@@ -1676,10 +1780,17 @@ export const deleteNodesForFile = async (filePath, dbPath) => {
|
|
|
1676
1780
|
if (tableName === 'Community' || tableName === 'Process')
|
|
1677
1781
|
continue;
|
|
1678
1782
|
try {
|
|
1679
|
-
// First count how many we'll delete
|
|
1783
|
+
// First count how many we'll delete. On the singleton connection this
|
|
1784
|
+
// count runs inside withConnLock (incremental --pdg writeback executes
|
|
1785
|
+
// while the WAL driver is live); per-query/temp connections skip the
|
|
1786
|
+
// lock, matching queryAndDrain's `targetConn === conn` gate — the sibling
|
|
1787
|
+
// DETACH DELETE below already routes through it. (#2264)
|
|
1680
1788
|
const tn = escapeTableName(tableName);
|
|
1681
|
-
const
|
|
1682
|
-
const
|
|
1789
|
+
const countCypher = `MATCH (n:${tn}) WHERE n.filePath = '${escapedPath}' RETURN count(n) AS cnt`;
|
|
1790
|
+
const runCount = async () => readQueryRows(await targetConn.query(countCypher));
|
|
1791
|
+
const rows = isSharedSingletonConn(targetConn)
|
|
1792
|
+
? await withConnLock(runCount)
|
|
1793
|
+
: await runCount();
|
|
1683
1794
|
const count = Number(rows[0]?.cnt ?? rows[0]?.[0] ?? 0);
|
|
1684
1795
|
if (count > 0) {
|
|
1685
1796
|
// Delete nodes (and implicitly their relationships via DETACH)
|
|
@@ -1723,7 +1834,8 @@ export const getEmbeddingTableName = () => EMBEDDING_TABLE_NAME;
|
|
|
1723
1834
|
* exports.
|
|
1724
1835
|
*/
|
|
1725
1836
|
export const queryImporters = async (targetFilePath) => {
|
|
1726
|
-
|
|
1837
|
+
const c = conn;
|
|
1838
|
+
if (!c) {
|
|
1727
1839
|
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
1728
1840
|
}
|
|
1729
1841
|
const escaped = targetFilePath.replace(/'/g, "''");
|
|
@@ -1732,26 +1844,31 @@ export const queryImporters = async (targetFilePath) => {
|
|
|
1732
1844
|
WHERE r.type = 'IMPORTS' AND b.filePath = '${escaped}'
|
|
1733
1845
|
RETURN DISTINCT a.filePath AS importer
|
|
1734
1846
|
`;
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
1738
|
-
|
|
1739
|
-
|
|
1740
|
-
|
|
1741
|
-
|
|
1742
|
-
const
|
|
1743
|
-
|
|
1744
|
-
|
|
1847
|
+
// Runs inside the connection lock: queryImporters is called in the importer-BFS
|
|
1848
|
+
// loop during incremental --pdg writeback while the WAL driver is live, so an
|
|
1849
|
+
// unlocked conn.query here could race a concurrent CHECKPOINT on the singleton.
|
|
1850
|
+
return withConnLock(async () => {
|
|
1851
|
+
let queryResult;
|
|
1852
|
+
try {
|
|
1853
|
+
queryResult = await c.query(cypher);
|
|
1854
|
+
const result = Array.isArray(queryResult) ? queryResult[0] : queryResult;
|
|
1855
|
+
const rows = await result.getAll();
|
|
1856
|
+
const out = [];
|
|
1857
|
+
for (const row of rows) {
|
|
1858
|
+
const v = row.importer;
|
|
1859
|
+
if (typeof v === 'string' && v.length > 0)
|
|
1860
|
+
out.push(v);
|
|
1861
|
+
}
|
|
1862
|
+
return out;
|
|
1745
1863
|
}
|
|
1746
|
-
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
|
|
1753
|
-
|
|
1754
|
-
}
|
|
1864
|
+
catch {
|
|
1865
|
+
return [];
|
|
1866
|
+
}
|
|
1867
|
+
finally {
|
|
1868
|
+
if (queryResult)
|
|
1869
|
+
await closeQueryResults(queryResult);
|
|
1870
|
+
}
|
|
1871
|
+
});
|
|
1755
1872
|
};
|
|
1756
1873
|
/**
|
|
1757
1874
|
* Drop every Community and Process node (and their MEMBER_OF /
|
|
@@ -1761,31 +1878,38 @@ export const queryImporters = async (targetFilePath) => {
|
|
|
1761
1878
|
* "Leiden runs on the FULL graph" correctness invariant.
|
|
1762
1879
|
*/
|
|
1763
1880
|
export const deleteAllCommunitiesAndProcesses = async () => {
|
|
1764
|
-
|
|
1881
|
+
const c = conn;
|
|
1882
|
+
if (!c) {
|
|
1765
1883
|
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
1766
1884
|
}
|
|
1767
|
-
|
|
1768
|
-
|
|
1769
|
-
|
|
1770
|
-
|
|
1771
|
-
|
|
1772
|
-
|
|
1773
|
-
|
|
1774
|
-
|
|
1775
|
-
|
|
1776
|
-
await
|
|
1777
|
-
|
|
1885
|
+
// count + DETACH DELETE run inside the connection lock so they cannot execute
|
|
1886
|
+
// concurrently with the WAL-checkpoint driver's CHECKPOINT on the singleton
|
|
1887
|
+
// connection. This runs during incremental --pdg writeback while the driver is
|
|
1888
|
+
// live; mirrors the wrapped deleteAllInterprocTaintPaths / deleteAllCallSummaries.
|
|
1889
|
+
return withConnLock(async () => {
|
|
1890
|
+
let nodesDeleted = 0;
|
|
1891
|
+
for (const label of ['Community', 'Process']) {
|
|
1892
|
+
let countResult;
|
|
1893
|
+
try {
|
|
1894
|
+
countResult = await c.query(`MATCH (n:${label}) RETURN count(n) AS cnt`);
|
|
1895
|
+
const result = Array.isArray(countResult) ? countResult[0] : countResult;
|
|
1896
|
+
const rows = await result.getAll();
|
|
1897
|
+
const count = Number(rows[0]?.cnt ?? rows[0]?.[0] ?? 0);
|
|
1898
|
+
if (count > 0) {
|
|
1899
|
+
await closeQueryResults(await c.query(`MATCH (n:${label}) DETACH DELETE n`));
|
|
1900
|
+
nodesDeleted += count;
|
|
1901
|
+
}
|
|
1902
|
+
}
|
|
1903
|
+
catch {
|
|
1904
|
+
// Table may not exist yet on a freshly-initialized DB — fine.
|
|
1905
|
+
}
|
|
1906
|
+
finally {
|
|
1907
|
+
if (countResult)
|
|
1908
|
+
await closeQueryResults(countResult);
|
|
1778
1909
|
}
|
|
1779
1910
|
}
|
|
1780
|
-
|
|
1781
|
-
|
|
1782
|
-
}
|
|
1783
|
-
finally {
|
|
1784
|
-
if (countResult)
|
|
1785
|
-
await closeQueryResults(countResult);
|
|
1786
|
-
}
|
|
1787
|
-
}
|
|
1788
|
-
return { nodesDeleted };
|
|
1911
|
+
return { nodesDeleted };
|
|
1912
|
+
});
|
|
1789
1913
|
};
|
|
1790
1914
|
/**
|
|
1791
1915
|
* Drop every interprocedural `TAINT_PATH` relationship (#2084 M4 U6). Used at
|
|
@@ -1803,44 +1927,49 @@ export const deleteAllCommunitiesAndProcesses = async () => {
|
|
|
1803
1927
|
* plain DELETE on the typed CodeRelation rows — endpoints are untouched.
|
|
1804
1928
|
*/
|
|
1805
1929
|
export const deleteAllInterprocTaintPaths = async () => {
|
|
1806
|
-
|
|
1930
|
+
const c = conn;
|
|
1931
|
+
if (!c) {
|
|
1807
1932
|
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
1808
1933
|
}
|
|
1809
|
-
|
|
1810
|
-
|
|
1811
|
-
|
|
1812
|
-
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1934
|
+
// count + DELETE run as one critical section on the singleton connection so a
|
|
1935
|
+
// concurrent WAL-checkpoint cannot corrupt native state mid-delete (#pdg).
|
|
1936
|
+
return withConnLock(async () => {
|
|
1937
|
+
let edgesDeleted = 0;
|
|
1938
|
+
let countResult;
|
|
1939
|
+
try {
|
|
1940
|
+
countResult = await c.query(`MATCH ()-[r:CodeRelation]->() WHERE r.type = 'TAINT_PATH' RETURN count(r) AS cnt`);
|
|
1941
|
+
const result = Array.isArray(countResult) ? countResult[0] : countResult;
|
|
1942
|
+
const rows = await result.getAll();
|
|
1943
|
+
const count = Number(rows[0]?.cnt ?? rows[0]?.[0] ?? 0);
|
|
1944
|
+
if (count > 0) {
|
|
1945
|
+
await closeQueryResults(await c.query(`MATCH ()-[r:CodeRelation]->() WHERE r.type = 'TAINT_PATH' DELETE r`));
|
|
1946
|
+
edgesDeleted = count;
|
|
1947
|
+
}
|
|
1819
1948
|
}
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
1823
|
-
|
|
1824
|
-
|
|
1825
|
-
|
|
1826
|
-
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1949
|
+
catch (err) {
|
|
1950
|
+
// A missing table on a freshly-initialized DB is the benign, expected case
|
|
1951
|
+
// (the count query above is what throws) — stay silent. Any OTHER failure
|
|
1952
|
+
// (lock, disk, native error) would leave stale TAINT_PATH rows that the
|
|
1953
|
+
// subsequent re-extract then DUPLICATES (CodeRelation has no PK), so it
|
|
1954
|
+
// must ABORT the writeback (#2084 review P2-5): re-throw so the caller's
|
|
1955
|
+
// crash-recovery dirty flag forces a clean full rebuild on the next run,
|
|
1956
|
+
// rather than silently writing duplicate cross-function findings.
|
|
1957
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1958
|
+
if (/no table|not exist|not found|does not exist|Table .* does not exist/i.test(msg)) {
|
|
1959
|
+
if (countResult)
|
|
1960
|
+
await closeQueryResults(countResult);
|
|
1961
|
+
return { edgesDeleted };
|
|
1962
|
+
}
|
|
1831
1963
|
if (countResult)
|
|
1832
1964
|
await closeQueryResults(countResult);
|
|
1833
|
-
|
|
1965
|
+
throw new Error(`[taint-interproc] failed to clear existing TAINT_PATH edges before incremental ` +
|
|
1966
|
+
`re-write (${msg}) — aborting to avoid duplicate cross-function findings; ` +
|
|
1967
|
+
`the next run will full-rebuild`);
|
|
1834
1968
|
}
|
|
1835
1969
|
if (countResult)
|
|
1836
1970
|
await closeQueryResults(countResult);
|
|
1837
|
-
|
|
1838
|
-
|
|
1839
|
-
`the next run will full-rebuild`);
|
|
1840
|
-
}
|
|
1841
|
-
if (countResult)
|
|
1842
|
-
await closeQueryResults(countResult);
|
|
1843
|
-
return { edgesDeleted };
|
|
1971
|
+
return { edgesDeleted };
|
|
1972
|
+
});
|
|
1844
1973
|
};
|
|
1845
1974
|
/**
|
|
1846
1975
|
* Drop every `CALL_SUMMARY` relationship (PDG FU-C, U-C3). Used at the start of
|
|
@@ -1854,42 +1983,47 @@ export const deleteAllInterprocTaintPaths = async () => {
|
|
|
1854
1983
|
* an unchanged function's summary from being lost.
|
|
1855
1984
|
*/
|
|
1856
1985
|
export const deleteAllCallSummaries = async () => {
|
|
1857
|
-
|
|
1986
|
+
const c = conn;
|
|
1987
|
+
if (!c) {
|
|
1858
1988
|
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
1859
1989
|
}
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
|
|
1864
|
-
|
|
1865
|
-
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1990
|
+
// count + DELETE run as one critical section on the singleton connection so a
|
|
1991
|
+
// concurrent WAL-checkpoint cannot corrupt native state mid-delete (#pdg).
|
|
1992
|
+
return withConnLock(async () => {
|
|
1993
|
+
let edgesDeleted = 0;
|
|
1994
|
+
let countResult;
|
|
1995
|
+
try {
|
|
1996
|
+
countResult = await c.query(`MATCH ()-[r:CodeRelation]->() WHERE r.type = 'CALL_SUMMARY' RETURN count(r) AS cnt`);
|
|
1997
|
+
const result = Array.isArray(countResult) ? countResult[0] : countResult;
|
|
1998
|
+
const rows = await result.getAll();
|
|
1999
|
+
const count = Number(rows[0]?.cnt ?? rows[0]?.[0] ?? 0);
|
|
2000
|
+
if (count > 0) {
|
|
2001
|
+
await closeQueryResults(await c.query(`MATCH ()-[r:CodeRelation]->() WHERE r.type = 'CALL_SUMMARY' DELETE r`));
|
|
2002
|
+
edgesDeleted = count;
|
|
2003
|
+
}
|
|
1870
2004
|
}
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
2005
|
+
catch (err) {
|
|
2006
|
+
// A missing table on a freshly-initialized DB is the benign, expected case
|
|
2007
|
+
// (the count query is what throws) — stay silent. Any OTHER failure would
|
|
2008
|
+
// leave stale rows that the re-extract then DUPLICATES (CodeRelation has no
|
|
2009
|
+
// PK), so it must ABORT the writeback: re-throw so the caller's crash-
|
|
2010
|
+
// recovery dirty flag forces a clean full rebuild on the next run.
|
|
2011
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
2012
|
+
if (/no table|not exist|not found|does not exist|Table .* does not exist/i.test(msg)) {
|
|
2013
|
+
if (countResult)
|
|
2014
|
+
await closeQueryResults(countResult);
|
|
2015
|
+
return { edgesDeleted };
|
|
2016
|
+
}
|
|
1880
2017
|
if (countResult)
|
|
1881
2018
|
await closeQueryResults(countResult);
|
|
1882
|
-
|
|
2019
|
+
throw new Error(`[call-summary] failed to clear existing CALL_SUMMARY edges before incremental ` +
|
|
2020
|
+
`re-write (${msg}) — aborting to avoid duplicate summaries; ` +
|
|
2021
|
+
`the next run will full-rebuild`);
|
|
1883
2022
|
}
|
|
1884
2023
|
if (countResult)
|
|
1885
2024
|
await closeQueryResults(countResult);
|
|
1886
|
-
|
|
1887
|
-
|
|
1888
|
-
`the next run will full-rebuild`);
|
|
1889
|
-
}
|
|
1890
|
-
if (countResult)
|
|
1891
|
-
await closeQueryResults(countResult);
|
|
1892
|
-
return { edgesDeleted };
|
|
2025
|
+
return { edgesDeleted };
|
|
2026
|
+
});
|
|
1893
2027
|
};
|
|
1894
2028
|
// ============================================================================
|
|
1895
2029
|
// Full-Text Search (FTS) Functions
|