gitnexus 1.4.0 → 1.4.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -18
- package/dist/cli/analyze.js +37 -28
- package/dist/cli/augment.js +1 -1
- package/dist/cli/eval-server.d.ts +1 -1
- package/dist/cli/eval-server.js +1 -1
- package/dist/cli/index.js +1 -0
- package/dist/cli/mcp.js +1 -1
- package/dist/cli/setup.js +25 -13
- package/dist/cli/status.js +13 -4
- package/dist/cli/tool.d.ts +1 -1
- package/dist/cli/tool.js +2 -2
- package/dist/cli/wiki.js +2 -2
- package/dist/config/ignore-service.d.ts +25 -0
- package/dist/config/ignore-service.js +76 -0
- package/dist/config/supported-languages.d.ts +1 -0
- package/dist/config/supported-languages.js +1 -1
- package/dist/core/augmentation/engine.js +94 -67
- package/dist/core/embeddings/embedder.d.ts +1 -1
- package/dist/core/embeddings/embedder.js +1 -1
- package/dist/core/embeddings/embedding-pipeline.d.ts +3 -3
- package/dist/core/embeddings/embedding-pipeline.js +52 -25
- package/dist/core/embeddings/types.d.ts +1 -1
- package/dist/core/ingestion/call-processor.d.ts +6 -7
- package/dist/core/ingestion/call-processor.js +490 -127
- package/dist/core/ingestion/call-routing.d.ts +53 -0
- package/dist/core/ingestion/call-routing.js +108 -0
- package/dist/core/ingestion/entry-point-scoring.js +13 -2
- package/dist/core/ingestion/export-detection.js +1 -0
- package/dist/core/ingestion/filesystem-walker.js +4 -3
- package/dist/core/ingestion/framework-detection.js +9 -0
- package/dist/core/ingestion/heritage-processor.d.ts +3 -4
- package/dist/core/ingestion/heritage-processor.js +40 -50
- package/dist/core/ingestion/import-processor.d.ts +3 -5
- package/dist/core/ingestion/import-processor.js +41 -10
- package/dist/core/ingestion/parsing-processor.d.ts +2 -1
- package/dist/core/ingestion/parsing-processor.js +41 -4
- package/dist/core/ingestion/pipeline.d.ts +5 -1
- package/dist/core/ingestion/pipeline.js +174 -121
- package/dist/core/ingestion/resolution-context.d.ts +53 -0
- package/dist/core/ingestion/resolution-context.js +132 -0
- package/dist/core/ingestion/resolvers/index.d.ts +2 -0
- package/dist/core/ingestion/resolvers/index.js +2 -0
- package/dist/core/ingestion/resolvers/python.d.ts +19 -0
- package/dist/core/ingestion/resolvers/python.js +52 -0
- package/dist/core/ingestion/resolvers/ruby.d.ts +12 -0
- package/dist/core/ingestion/resolvers/ruby.js +15 -0
- package/dist/core/ingestion/resolvers/standard.js +0 -22
- package/dist/core/ingestion/resolvers/utils.js +2 -0
- package/dist/core/ingestion/symbol-table.d.ts +3 -0
- package/dist/core/ingestion/symbol-table.js +1 -0
- package/dist/core/ingestion/tree-sitter-queries.d.ts +3 -2
- package/dist/core/ingestion/tree-sitter-queries.js +53 -1
- package/dist/core/ingestion/type-env.d.ts +32 -10
- package/dist/core/ingestion/type-env.js +520 -47
- package/dist/core/ingestion/type-extractors/c-cpp.js +326 -1
- package/dist/core/ingestion/type-extractors/csharp.js +282 -2
- package/dist/core/ingestion/type-extractors/go.js +333 -2
- package/dist/core/ingestion/type-extractors/index.d.ts +3 -2
- package/dist/core/ingestion/type-extractors/index.js +3 -1
- package/dist/core/ingestion/type-extractors/jvm.js +537 -4
- package/dist/core/ingestion/type-extractors/php.js +387 -7
- package/dist/core/ingestion/type-extractors/python.js +356 -5
- package/dist/core/ingestion/type-extractors/ruby.d.ts +2 -0
- package/dist/core/ingestion/type-extractors/ruby.js +389 -0
- package/dist/core/ingestion/type-extractors/rust.js +399 -2
- package/dist/core/ingestion/type-extractors/shared.d.ts +116 -1
- package/dist/core/ingestion/type-extractors/shared.js +488 -14
- package/dist/core/ingestion/type-extractors/swift.js +95 -1
- package/dist/core/ingestion/type-extractors/types.d.ts +81 -0
- package/dist/core/ingestion/type-extractors/typescript.js +436 -2
- package/dist/core/ingestion/utils.d.ts +33 -2
- package/dist/core/ingestion/utils.js +399 -27
- package/dist/core/ingestion/workers/parse-worker.d.ts +18 -1
- package/dist/core/ingestion/workers/parse-worker.js +169 -19
- package/dist/core/{kuzu → lbug}/csv-generator.d.ts +1 -1
- package/dist/core/{kuzu → lbug}/csv-generator.js +1 -1
- package/dist/core/{kuzu/kuzu-adapter.d.ts → lbug/lbug-adapter.d.ts} +19 -19
- package/dist/core/{kuzu/kuzu-adapter.js → lbug/lbug-adapter.js} +70 -65
- package/dist/core/{kuzu → lbug}/schema.d.ts +1 -1
- package/dist/core/{kuzu → lbug}/schema.js +1 -1
- package/dist/core/search/bm25-index.d.ts +4 -4
- package/dist/core/search/bm25-index.js +10 -10
- package/dist/core/search/hybrid-search.d.ts +2 -2
- package/dist/core/search/hybrid-search.js +6 -6
- package/dist/core/tree-sitter/parser-loader.js +9 -2
- package/dist/core/wiki/generator.d.ts +2 -2
- package/dist/core/wiki/generator.js +4 -4
- package/dist/core/wiki/graph-queries.d.ts +4 -4
- package/dist/core/wiki/graph-queries.js +7 -7
- package/dist/mcp/core/{kuzu-adapter.d.ts → lbug-adapter.d.ts} +7 -7
- package/dist/mcp/core/{kuzu-adapter.js → lbug-adapter.js} +72 -43
- package/dist/mcp/local/local-backend.d.ts +6 -6
- package/dist/mcp/local/local-backend.js +25 -18
- package/dist/server/api.js +12 -12
- package/dist/server/mcp-http.d.ts +1 -1
- package/dist/server/mcp-http.js +1 -1
- package/dist/storage/repo-manager.d.ts +20 -2
- package/dist/storage/repo-manager.js +55 -1
- package/dist/types/pipeline.d.ts +1 -1
- package/package.json +5 -3
- package/dist/core/ingestion/symbol-resolver.d.ts +0 -32
- package/dist/core/ingestion/symbol-resolver.js +0 -83
|
@@ -2,14 +2,14 @@ import fs from 'fs/promises';
|
|
|
2
2
|
import { createReadStream } from 'fs';
|
|
3
3
|
import { createInterface } from 'readline';
|
|
4
4
|
import path from 'path';
|
|
5
|
-
import
|
|
5
|
+
import lbug from '@ladybugdb/core';
|
|
6
6
|
import { NODE_TABLES, REL_TABLE_NAME, SCHEMA_QUERIES, EMBEDDING_TABLE_NAME, } from './schema.js';
|
|
7
7
|
import { streamAllCSVsToDisk } from './csv-generator.js';
|
|
8
8
|
let db = null;
|
|
9
9
|
let conn = null;
|
|
10
10
|
let currentDbPath = null;
|
|
11
11
|
let ftsLoaded = false;
|
|
12
|
-
// Global session lock for operations that touch module-level
|
|
12
|
+
// Global session lock for operations that touch module-level lbug globals.
|
|
13
13
|
// This guarantees no DB switch can happen while an operation is running.
|
|
14
14
|
let sessionLock = Promise.resolve();
|
|
15
15
|
const runWithSessionLock = async (operation) => {
|
|
@@ -27,27 +27,27 @@ const runWithSessionLock = async (operation) => {
|
|
|
27
27
|
}
|
|
28
28
|
};
|
|
29
29
|
const normalizeCopyPath = (filePath) => filePath.replace(/\\/g, '/');
|
|
30
|
-
export const
|
|
31
|
-
return runWithSessionLock(() =>
|
|
30
|
+
export const initLbug = async (dbPath) => {
|
|
31
|
+
return runWithSessionLock(() => ensureLbugInitialized(dbPath));
|
|
32
32
|
};
|
|
33
33
|
/**
|
|
34
34
|
* Execute multiple queries against one repo DB atomically.
|
|
35
35
|
* While the callback runs, no other request can switch the active DB.
|
|
36
36
|
*/
|
|
37
|
-
export const
|
|
37
|
+
export const withLbugDb = async (dbPath, operation) => {
|
|
38
38
|
return runWithSessionLock(async () => {
|
|
39
|
-
await
|
|
39
|
+
await ensureLbugInitialized(dbPath);
|
|
40
40
|
return operation();
|
|
41
41
|
});
|
|
42
42
|
};
|
|
43
|
-
const
|
|
43
|
+
const ensureLbugInitialized = async (dbPath) => {
|
|
44
44
|
if (conn && currentDbPath === dbPath) {
|
|
45
45
|
return { db, conn };
|
|
46
46
|
}
|
|
47
|
-
await
|
|
47
|
+
await doInitLbug(dbPath);
|
|
48
48
|
return { db, conn };
|
|
49
49
|
};
|
|
50
|
-
const
|
|
50
|
+
const doInitLbug = async (dbPath) => {
|
|
51
51
|
// Different database requested — close the old one first
|
|
52
52
|
if (conn || db) {
|
|
53
53
|
try {
|
|
@@ -65,32 +65,36 @@ const doInitKuzu = async (dbPath) => {
|
|
|
65
65
|
currentDbPath = null;
|
|
66
66
|
ftsLoaded = false;
|
|
67
67
|
}
|
|
68
|
-
//
|
|
69
|
-
// If the path already exists, it must be a valid
|
|
68
|
+
// LadybugDB stores the database as a single file (not a directory).
|
|
69
|
+
// If the path already exists, it must be a valid LadybugDB database file.
|
|
70
70
|
// Remove stale empty directories or files from older versions.
|
|
71
71
|
try {
|
|
72
|
-
const stat = await fs.
|
|
73
|
-
if (stat.
|
|
74
|
-
//
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
72
|
+
const stat = await fs.lstat(dbPath);
|
|
73
|
+
if (stat.isSymbolicLink()) {
|
|
74
|
+
// Never follow symlinks — just remove the link itself
|
|
75
|
+
await fs.unlink(dbPath);
|
|
76
|
+
}
|
|
77
|
+
else if (stat.isDirectory()) {
|
|
78
|
+
// Verify path is within expected storage directory before deleting
|
|
79
|
+
const realPath = await fs.realpath(dbPath);
|
|
80
|
+
const parentDir = path.dirname(dbPath);
|
|
81
|
+
const realParent = await fs.realpath(parentDir);
|
|
82
|
+
if (!realPath.startsWith(realParent + path.sep) && realPath !== realParent) {
|
|
83
|
+
throw new Error(`Refusing to delete ${dbPath}: resolved path ${realPath} is outside storage directory`);
|
|
82
84
|
}
|
|
85
|
+
// Old-style directory database or empty leftover - remove it
|
|
86
|
+
await fs.rm(dbPath, { recursive: true, force: true });
|
|
83
87
|
}
|
|
84
|
-
// If it's a file, assume it's an existing
|
|
88
|
+
// If it's a file, assume it's an existing LadybugDB database - LadybugDB will open it
|
|
85
89
|
}
|
|
86
90
|
catch {
|
|
87
|
-
// Path doesn't exist, which is what
|
|
91
|
+
// Path doesn't exist, which is what LadybugDB wants for a new database
|
|
88
92
|
}
|
|
89
93
|
// Ensure parent directory exists
|
|
90
94
|
const parentDir = path.dirname(dbPath);
|
|
91
95
|
await fs.mkdir(parentDir, { recursive: true });
|
|
92
|
-
db = new
|
|
93
|
-
conn = new
|
|
96
|
+
db = new lbug.Database(dbPath);
|
|
97
|
+
conn = new lbug.Connection(db);
|
|
94
98
|
for (const schemaQuery of SCHEMA_QUERIES) {
|
|
95
99
|
try {
|
|
96
100
|
await conn.query(schemaQuery);
|
|
@@ -106,9 +110,9 @@ const doInitKuzu = async (dbPath) => {
|
|
|
106
110
|
currentDbPath = dbPath;
|
|
107
111
|
return { db, conn };
|
|
108
112
|
};
|
|
109
|
-
export const
|
|
113
|
+
export const loadGraphToLbug = async (graph, repoPath, storagePath, onProgress) => {
|
|
110
114
|
if (!conn) {
|
|
111
|
-
throw new Error('
|
|
115
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
112
116
|
}
|
|
113
117
|
const log = onProgress || (() => { });
|
|
114
118
|
const csvDir = path.join(storagePath, 'csv');
|
|
@@ -122,7 +126,7 @@ export const loadGraphToKuzu = async (graph, repoPath, storagePath, onProgress)
|
|
|
122
126
|
return 'Process';
|
|
123
127
|
return nodeId.split(':')[0];
|
|
124
128
|
};
|
|
125
|
-
// Bulk COPY all node CSVs (sequential —
|
|
129
|
+
// Bulk COPY all node CSVs (sequential — LadybugDB allows only one write txn at a time)
|
|
126
130
|
const nodeFiles = [...csvResult.nodeFiles.entries()];
|
|
127
131
|
const totalSteps = nodeFiles.length + 1; // +1 for relationships
|
|
128
132
|
let stepsDone = 0;
|
|
@@ -145,7 +149,7 @@ export const loadGraphToKuzu = async (graph, repoPath, storagePath, onProgress)
|
|
|
145
149
|
}
|
|
146
150
|
}
|
|
147
151
|
}
|
|
148
|
-
// Bulk COPY relationships — split by FROM→TO label pair (
|
|
152
|
+
// Bulk COPY relationships — split by FROM→TO label pair (LadybugDB requires it)
|
|
149
153
|
// Stream-read the relation CSV line by line to avoid exceeding V8 max string length
|
|
150
154
|
let relHeader = '';
|
|
151
155
|
const relsByPair = new Map();
|
|
@@ -254,10 +258,10 @@ export const loadGraphToKuzu = async (graph, repoPath, storagePath, onProgress)
|
|
|
254
258
|
catch { }
|
|
255
259
|
return { success: true, insertedRels, skippedRels, warnings };
|
|
256
260
|
};
|
|
257
|
-
//
|
|
261
|
+
// LadybugDB default ESCAPE is '\' (backslash), but our CSV uses RFC 4180 escaping ("" for literal quotes).
|
|
258
262
|
// Source code content is full of backslashes which confuse the auto-detection.
|
|
259
263
|
// We MUST explicitly set ESCAPE='"' to use RFC 4180 escaping, and disable auto_detect to prevent
|
|
260
|
-
//
|
|
264
|
+
// LadybugDB from overriding our settings based on sample rows.
|
|
261
265
|
const COPY_CSV_OPTS = `(HEADER=true, ESCAPE='"', DELIM=',', QUOTE='"', PARALLEL=false, auto_detect=false)`;
|
|
262
266
|
// Multi-language table names that were created with backticks in CODE_ELEMENT_BASE
|
|
263
267
|
// and must always be referenced with backticks in queries
|
|
@@ -289,10 +293,11 @@ const fallbackRelationshipInserts = async (validRelLines, validTables, getNodeLa
|
|
|
289
293
|
continue;
|
|
290
294
|
const confidence = parseFloat(confidenceStr) || 1.0;
|
|
291
295
|
const step = parseInt(stepStr) || 0;
|
|
296
|
+
const esc = (s) => s.replace(/'/g, "''").replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/\r/g, '\\r');
|
|
292
297
|
await conn.query(`
|
|
293
|
-
MATCH (a:${escapeLabel(fromLabel)} {id: '${fromId
|
|
294
|
-
(b:${escapeLabel(toLabel)} {id: '${toId
|
|
295
|
-
CREATE (a)-[:${REL_TABLE_NAME} {type: '${relType}', confidence: ${confidence}, reason: '${reason
|
|
298
|
+
MATCH (a:${escapeLabel(fromLabel)} {id: '${esc(fromId)}' }),
|
|
299
|
+
(b:${escapeLabel(toLabel)} {id: '${esc(toId)}' })
|
|
300
|
+
CREATE (a)-[:${REL_TABLE_NAME} {type: '${esc(relType)}', confidence: ${confidence}, reason: '${esc(reason)}', step: ${step}}]->(b)
|
|
296
301
|
`);
|
|
297
302
|
}
|
|
298
303
|
catch {
|
|
@@ -327,16 +332,16 @@ const getCopyQuery = (table, filePath) => {
|
|
|
327
332
|
return `COPY ${t}(id, name, filePath, startLine, endLine, content, description) FROM "${filePath}" ${COPY_CSV_OPTS}`;
|
|
328
333
|
};
|
|
329
334
|
/**
|
|
330
|
-
* Insert a single node to
|
|
335
|
+
* Insert a single node to LadybugDB
|
|
331
336
|
* @param label - Node type (File, Function, Class, etc.)
|
|
332
337
|
* @param properties - Node properties
|
|
333
|
-
* @param dbPath - Path to
|
|
338
|
+
* @param dbPath - Path to LadybugDB database (optional if already initialized)
|
|
334
339
|
*/
|
|
335
|
-
export const
|
|
340
|
+
export const insertNodeToLbug = async (label, properties, dbPath) => {
|
|
336
341
|
// Use provided dbPath or fall back to module-level db
|
|
337
342
|
const targetDbPath = dbPath || (db ? undefined : null);
|
|
338
343
|
if (!targetDbPath && !db) {
|
|
339
|
-
throw new Error('
|
|
344
|
+
throw new Error('LadybugDB not initialized. Provide dbPath or call initLbug first.');
|
|
340
345
|
}
|
|
341
346
|
try {
|
|
342
347
|
const escapeValue = (v) => {
|
|
@@ -345,7 +350,7 @@ export const insertNodeToKuzu = async (label, properties, dbPath) => {
|
|
|
345
350
|
if (typeof v === 'number')
|
|
346
351
|
return String(v);
|
|
347
352
|
// Escape backslashes first (for Windows paths), then single quotes
|
|
348
|
-
return `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "''")}'`;
|
|
353
|
+
return `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "''").replace(/\n/g, '\\n').replace(/\r/g, '\\r')}'`;
|
|
349
354
|
};
|
|
350
355
|
// Build INSERT query based on node type
|
|
351
356
|
const t = escapeTableName(label);
|
|
@@ -367,8 +372,8 @@ export const insertNodeToKuzu = async (label, properties, dbPath) => {
|
|
|
367
372
|
}
|
|
368
373
|
// Use per-query connection if dbPath provided (avoids lock conflicts)
|
|
369
374
|
if (targetDbPath) {
|
|
370
|
-
const tempDb = new
|
|
371
|
-
const tempConn = new
|
|
375
|
+
const tempDb = new lbug.Database(targetDbPath);
|
|
376
|
+
const tempConn = new lbug.Connection(tempDb);
|
|
372
377
|
try {
|
|
373
378
|
await tempConn.query(query);
|
|
374
379
|
return true;
|
|
@@ -398,12 +403,12 @@ export const insertNodeToKuzu = async (label, properties, dbPath) => {
|
|
|
398
403
|
}
|
|
399
404
|
};
|
|
400
405
|
/**
|
|
401
|
-
* Batch insert multiple nodes to
|
|
406
|
+
* Batch insert multiple nodes to LadybugDB using a single connection
|
|
402
407
|
* @param nodes - Array of {label, properties} to insert
|
|
403
|
-
* @param dbPath - Path to
|
|
408
|
+
* @param dbPath - Path to LadybugDB database
|
|
404
409
|
* @returns Object with success count and error count
|
|
405
410
|
*/
|
|
406
|
-
export const
|
|
411
|
+
export const batchInsertNodesToLbug = async (nodes, dbPath) => {
|
|
407
412
|
if (nodes.length === 0)
|
|
408
413
|
return { inserted: 0, failed: 0 };
|
|
409
414
|
const escapeValue = (v) => {
|
|
@@ -411,12 +416,12 @@ export const batchInsertNodesToKuzu = async (nodes, dbPath) => {
|
|
|
411
416
|
return 'NULL';
|
|
412
417
|
if (typeof v === 'number')
|
|
413
418
|
return String(v);
|
|
414
|
-
// Escape backslashes first (for Windows paths), then single quotes
|
|
415
|
-
return `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "''")}'`;
|
|
419
|
+
// Escape backslashes first (for Windows paths), then single quotes, then newlines
|
|
420
|
+
return `'${String(v).replace(/\\/g, '\\\\').replace(/'/g, "''").replace(/\n/g, '\\n').replace(/\r/g, '\\r')}'`;
|
|
416
421
|
};
|
|
417
422
|
// Open a single connection for all inserts
|
|
418
|
-
const tempDb = new
|
|
419
|
-
const tempConn = new
|
|
423
|
+
const tempDb = new lbug.Database(dbPath);
|
|
424
|
+
const tempConn = new lbug.Connection(tempDb);
|
|
420
425
|
let inserted = 0;
|
|
421
426
|
let failed = 0;
|
|
422
427
|
try {
|
|
@@ -462,10 +467,10 @@ export const batchInsertNodesToKuzu = async (nodes, dbPath) => {
|
|
|
462
467
|
};
|
|
463
468
|
export const executeQuery = async (cypher) => {
|
|
464
469
|
if (!conn) {
|
|
465
|
-
throw new Error('
|
|
470
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
466
471
|
}
|
|
467
472
|
const queryResult = await conn.query(cypher);
|
|
468
|
-
//
|
|
473
|
+
// LadybugDB uses getAll() instead of hasNext()/getNext()
|
|
469
474
|
// Query returns QueryResult for single queries, QueryResult[] for multi-statement
|
|
470
475
|
const result = Array.isArray(queryResult) ? queryResult[0] : queryResult;
|
|
471
476
|
const rows = await result.getAll();
|
|
@@ -473,7 +478,7 @@ export const executeQuery = async (cypher) => {
|
|
|
473
478
|
};
|
|
474
479
|
export const executeWithReusedStatement = async (cypher, paramsList) => {
|
|
475
480
|
if (!conn) {
|
|
476
|
-
throw new Error('
|
|
481
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
477
482
|
}
|
|
478
483
|
if (paramsList.length === 0)
|
|
479
484
|
return;
|
|
@@ -494,10 +499,10 @@ export const executeWithReusedStatement = async (cypher, paramsList) => {
|
|
|
494
499
|
// Log the error and continue with next batch
|
|
495
500
|
console.warn('Batch execution error:', e);
|
|
496
501
|
}
|
|
497
|
-
// Note:
|
|
502
|
+
// Note: LadybugDB PreparedStatement doesn't require explicit close()
|
|
498
503
|
}
|
|
499
504
|
};
|
|
500
|
-
export const
|
|
505
|
+
export const getLbugStats = async () => {
|
|
501
506
|
if (!conn)
|
|
502
507
|
return { nodes: 0, edges: 0 };
|
|
503
508
|
let totalNodes = 0;
|
|
@@ -529,7 +534,7 @@ export const getKuzuStats = async () => {
|
|
|
529
534
|
return { nodes: totalNodes, edges: totalEdges };
|
|
530
535
|
};
|
|
531
536
|
/**
|
|
532
|
-
* Load cached embeddings from
|
|
537
|
+
* Load cached embeddings from LadybugDB before a rebuild.
|
|
533
538
|
* Returns all embedding vectors so they can be re-inserted after the graph is reloaded,
|
|
534
539
|
* avoiding expensive re-embedding of unchanged nodes.
|
|
535
540
|
*/
|
|
@@ -559,7 +564,7 @@ export const loadCachedEmbeddings = async () => {
|
|
|
559
564
|
catch { /* embedding table may not exist */ }
|
|
560
565
|
return { embeddingNodeIds, embeddings };
|
|
561
566
|
};
|
|
562
|
-
export const
|
|
567
|
+
export const closeLbug = async () => {
|
|
563
568
|
if (conn) {
|
|
564
569
|
try {
|
|
565
570
|
await conn.close();
|
|
@@ -577,11 +582,11 @@ export const closeKuzu = async () => {
|
|
|
577
582
|
currentDbPath = null;
|
|
578
583
|
ftsLoaded = false;
|
|
579
584
|
};
|
|
580
|
-
export const
|
|
585
|
+
export const isLbugReady = () => conn !== null && db !== null;
|
|
581
586
|
/**
|
|
582
|
-
* Delete all nodes (and their relationships) for a specific file from
|
|
587
|
+
* Delete all nodes (and their relationships) for a specific file from LadybugDB
|
|
583
588
|
* @param filePath - The file path to delete nodes for
|
|
584
|
-
* @param dbPath - Optional path to
|
|
589
|
+
* @param dbPath - Optional path to LadybugDB for per-query connection
|
|
585
590
|
* @returns Object with counts of deleted nodes
|
|
586
591
|
*/
|
|
587
592
|
export const deleteNodesForFile = async (filePath, dbPath) => {
|
|
@@ -591,12 +596,12 @@ export const deleteNodesForFile = async (filePath, dbPath) => {
|
|
|
591
596
|
let tempConn = null;
|
|
592
597
|
let targetConn = conn;
|
|
593
598
|
if (usePerQuery) {
|
|
594
|
-
tempDb = new
|
|
595
|
-
tempConn = new
|
|
599
|
+
tempDb = new lbug.Database(dbPath);
|
|
600
|
+
tempConn = new lbug.Connection(tempDb);
|
|
596
601
|
targetConn = tempConn;
|
|
597
602
|
}
|
|
598
603
|
else if (!conn) {
|
|
599
|
-
throw new Error('
|
|
604
|
+
throw new Error('LadybugDB not initialized. Provide dbPath or call initLbug first.');
|
|
600
605
|
}
|
|
601
606
|
try {
|
|
602
607
|
let deletedNodes = 0;
|
|
@@ -661,7 +666,7 @@ export const loadFTSExtension = async () => {
|
|
|
661
666
|
if (ftsLoaded)
|
|
662
667
|
return;
|
|
663
668
|
if (!conn) {
|
|
664
|
-
throw new Error('
|
|
669
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
665
670
|
}
|
|
666
671
|
try {
|
|
667
672
|
await conn.query('INSTALL fts');
|
|
@@ -687,7 +692,7 @@ export const loadFTSExtension = async () => {
|
|
|
687
692
|
*/
|
|
688
693
|
export const createFTSIndex = async (tableName, indexName, properties, stemmer = 'porter') => {
|
|
689
694
|
if (!conn) {
|
|
690
|
-
throw new Error('
|
|
695
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
691
696
|
}
|
|
692
697
|
await loadFTSExtension();
|
|
693
698
|
const propList = properties.map(p => `'${p}'`).join(', ');
|
|
@@ -712,7 +717,7 @@ export const createFTSIndex = async (tableName, indexName, properties, stemmer =
|
|
|
712
717
|
*/
|
|
713
718
|
export const queryFTS = async (tableName, indexName, query, limit = 20, conjunctive = false) => {
|
|
714
719
|
if (!conn) {
|
|
715
|
-
throw new Error('
|
|
720
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
716
721
|
}
|
|
717
722
|
// Escape backslashes and single quotes to prevent Cypher injection
|
|
718
723
|
const escapedQuery = query.replace(/\\/g, '\\\\').replace(/'/g, "''");
|
|
@@ -751,7 +756,7 @@ export const queryFTS = async (tableName, indexName, query, limit = 20, conjunct
|
|
|
751
756
|
*/
|
|
752
757
|
export const dropFTSIndex = async (tableName, indexName) => {
|
|
753
758
|
if (!conn) {
|
|
754
|
-
throw new Error('
|
|
759
|
+
throw new Error('LadybugDB not initialized. Call initLbug first.');
|
|
755
760
|
}
|
|
756
761
|
try {
|
|
757
762
|
await conn.query(`CALL DROP_FTS_INDEX('${tableName}', '${indexName}')`);
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Full-Text Search via
|
|
2
|
+
* Full-Text Search via LadybugDB FTS
|
|
3
3
|
*
|
|
4
|
-
* Uses
|
|
4
|
+
* Uses LadybugDB's built-in full-text search indexes for keyword-based search.
|
|
5
5
|
* Always reads from the database (no cached state to drift).
|
|
6
6
|
*/
|
|
7
7
|
export interface BM25SearchResult {
|
|
@@ -10,7 +10,7 @@ export interface BM25SearchResult {
|
|
|
10
10
|
rank: number;
|
|
11
11
|
}
|
|
12
12
|
/**
|
|
13
|
-
* Search using
|
|
13
|
+
* Search using LadybugDB's built-in FTS (always fresh, reads from disk)
|
|
14
14
|
*
|
|
15
15
|
* Queries multiple node tables (File, Function, Class, Method) in parallel
|
|
16
16
|
* and merges results by filePath, summing scores for the same file.
|
|
@@ -20,4 +20,4 @@ export interface BM25SearchResult {
|
|
|
20
20
|
* @param repoId - If provided, queries will be routed via the MCP connection pool
|
|
21
21
|
* @returns Ranked search results from FTS indexes
|
|
22
22
|
*/
|
|
23
|
-
export declare const
|
|
23
|
+
export declare const searchFTSFromLbug: (query: string, limit?: number, repoId?: string) => Promise<BM25SearchResult[]>;
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Full-Text Search via
|
|
2
|
+
* Full-Text Search via LadybugDB FTS
|
|
3
3
|
*
|
|
4
|
-
* Uses
|
|
4
|
+
* Uses LadybugDB's built-in full-text search indexes for keyword-based search.
|
|
5
5
|
* Always reads from the database (no cached state to drift).
|
|
6
6
|
*/
|
|
7
|
-
import { queryFTS } from '../
|
|
7
|
+
import { queryFTS } from '../lbug/lbug-adapter.js';
|
|
8
8
|
/**
|
|
9
9
|
* Execute a single FTS query via a custom executor (for MCP connection pool).
|
|
10
|
-
* Returns the same shape as core queryFTS.
|
|
10
|
+
* Returns the same shape as core queryFTS (from LadybugDB adapter).
|
|
11
11
|
*/
|
|
12
12
|
async function queryFTSViaExecutor(executor, tableName, indexName, query, limit) {
|
|
13
13
|
// Escape single quotes and backslashes to prevent Cypher injection
|
|
@@ -34,7 +34,7 @@ async function queryFTSViaExecutor(executor, tableName, indexName, query, limit)
|
|
|
34
34
|
}
|
|
35
35
|
}
|
|
36
36
|
/**
|
|
37
|
-
* Search using
|
|
37
|
+
* Search using LadybugDB's built-in FTS (always fresh, reads from disk)
|
|
38
38
|
*
|
|
39
39
|
* Queries multiple node tables (File, Function, Class, Method) in parallel
|
|
40
40
|
* and merges results by filePath, summing scores for the same file.
|
|
@@ -44,13 +44,13 @@ async function queryFTSViaExecutor(executor, tableName, indexName, query, limit)
|
|
|
44
44
|
* @param repoId - If provided, queries will be routed via the MCP connection pool
|
|
45
45
|
* @returns Ranked search results from FTS indexes
|
|
46
46
|
*/
|
|
47
|
-
export const
|
|
47
|
+
export const searchFTSFromLbug = async (query, limit = 20, repoId) => {
|
|
48
48
|
let fileResults, functionResults, classResults, methodResults, interfaceResults;
|
|
49
49
|
if (repoId) {
|
|
50
50
|
// Use MCP connection pool via dynamic import
|
|
51
|
-
// IMPORTANT:
|
|
52
|
-
//
|
|
53
|
-
const { executeQuery } = await import('../../mcp/core/
|
|
51
|
+
// IMPORTANT: FTS queries run sequentially to avoid connection contention.
|
|
52
|
+
// The MCP pool supports multiple connections, but FTS is best run serially.
|
|
53
|
+
const { executeQuery } = await import('../../mcp/core/lbug-adapter.js');
|
|
54
54
|
const executor = (cypher) => executeQuery(repoId, cypher);
|
|
55
55
|
fileResults = await queryFTSViaExecutor(executor, 'File', 'file_fts', query, limit);
|
|
56
56
|
functionResults = await queryFTSViaExecutor(executor, 'Function', 'function_fts', query, limit);
|
|
@@ -59,7 +59,7 @@ export const searchFTSFromKuzu = async (query, limit = 20, repoId) => {
|
|
|
59
59
|
interfaceResults = await queryFTSViaExecutor(executor, 'Interface', 'interface_fts', query, limit);
|
|
60
60
|
}
|
|
61
61
|
else {
|
|
62
|
-
// Use core
|
|
62
|
+
// Use core lbug adapter (CLI / pipeline context) — also sequential for safety
|
|
63
63
|
fileResults = await queryFTS('File', 'file_fts', query, limit, false).catch(() => []);
|
|
64
64
|
functionResults = await queryFTS('Function', 'function_fts', query, limit, false).catch(() => []);
|
|
65
65
|
classResults = await queryFTS('Class', 'class_fts', query, limit, false).catch(() => []);
|
|
@@ -33,7 +33,7 @@ export interface HybridSearchResult {
|
|
|
33
33
|
export declare const mergeWithRRF: (bm25Results: BM25SearchResult[], semanticResults: SemanticSearchResult[], limit?: number) => HybridSearchResult[];
|
|
34
34
|
/**
|
|
35
35
|
* Check if hybrid search is available
|
|
36
|
-
*
|
|
36
|
+
* LadybugDB FTS is always available once the database is initialized.
|
|
37
37
|
* Semantic search is optional - hybrid works with just FTS if embeddings aren't ready.
|
|
38
38
|
*/
|
|
39
39
|
export declare const isHybridSearchReady: () => boolean;
|
|
@@ -43,7 +43,7 @@ export declare const isHybridSearchReady: () => boolean;
|
|
|
43
43
|
export declare const formatHybridResults: (results: HybridSearchResult[]) => string;
|
|
44
44
|
/**
|
|
45
45
|
* Execute BM25 + semantic search and merge with RRF.
|
|
46
|
-
* Uses
|
|
46
|
+
* Uses LadybugDB FTS for always-fresh BM25 results (no cached data).
|
|
47
47
|
* The semanticSearch function is injected to keep this module environment-agnostic.
|
|
48
48
|
*/
|
|
49
49
|
export declare const hybridSearch: (query: string, limit: number, executeQuery: (cypher: string) => Promise<any[]>, semanticSearch: (executeQuery: (cypher: string) => Promise<any[]>, query: string, k?: number) => Promise<SemanticSearchResult[]>) => Promise<HybridSearchResult[]>;
|
|
@@ -7,7 +7,7 @@
|
|
|
7
7
|
* This is the same approach used by Elasticsearch, Pinecone, and other
|
|
8
8
|
* production search systems.
|
|
9
9
|
*/
|
|
10
|
-
import {
|
|
10
|
+
import { searchFTSFromLbug } from './bm25-index.js';
|
|
11
11
|
/**
|
|
12
12
|
* RRF constant - standard value used in the literature
|
|
13
13
|
* Higher values give more weight to lower-ranked results
|
|
@@ -80,11 +80,11 @@ export const mergeWithRRF = (bm25Results, semanticResults, limit = 10) => {
|
|
|
80
80
|
};
|
|
81
81
|
/**
|
|
82
82
|
* Check if hybrid search is available
|
|
83
|
-
*
|
|
83
|
+
* LadybugDB FTS is always available once the database is initialized.
|
|
84
84
|
* Semantic search is optional - hybrid works with just FTS if embeddings aren't ready.
|
|
85
85
|
*/
|
|
86
86
|
export const isHybridSearchReady = () => {
|
|
87
|
-
return true; // FTS is always available via
|
|
87
|
+
return true; // FTS is always available via LadybugDB when DB is open
|
|
88
88
|
};
|
|
89
89
|
/**
|
|
90
90
|
* Format hybrid results for LLM consumption
|
|
@@ -107,12 +107,12 @@ export const formatHybridResults = (results) => {
|
|
|
107
107
|
};
|
|
108
108
|
/**
|
|
109
109
|
* Execute BM25 + semantic search and merge with RRF.
|
|
110
|
-
* Uses
|
|
110
|
+
* Uses LadybugDB FTS for always-fresh BM25 results (no cached data).
|
|
111
111
|
* The semanticSearch function is injected to keep this module environment-agnostic.
|
|
112
112
|
*/
|
|
113
113
|
export const hybridSearch = async (query, limit, executeQuery, semanticSearch) => {
|
|
114
|
-
// Use
|
|
115
|
-
const bm25Results = await
|
|
114
|
+
// Use LadybugDB FTS for always-fresh BM25 results
|
|
115
|
+
const bm25Results = await searchFTSFromLbug(query, limit);
|
|
116
116
|
const semanticResults = await semanticSearch(executeQuery, query, limit);
|
|
117
117
|
return mergeWithRRF(bm25Results, semanticResults, limit);
|
|
118
118
|
};
|
|
@@ -8,8 +8,8 @@ import CPP from 'tree-sitter-cpp';
|
|
|
8
8
|
import CSharp from 'tree-sitter-c-sharp';
|
|
9
9
|
import Go from 'tree-sitter-go';
|
|
10
10
|
import Rust from 'tree-sitter-rust';
|
|
11
|
-
import Kotlin from 'tree-sitter-kotlin';
|
|
12
11
|
import PHP from 'tree-sitter-php';
|
|
12
|
+
import Ruby from 'tree-sitter-ruby';
|
|
13
13
|
import { createRequire } from 'node:module';
|
|
14
14
|
import { SupportedLanguages } from '../../config/supported-languages.js';
|
|
15
15
|
// tree-sitter-swift is an optionalDependency — may not be installed
|
|
@@ -19,6 +19,12 @@ try {
|
|
|
19
19
|
Swift = _require('tree-sitter-swift');
|
|
20
20
|
}
|
|
21
21
|
catch { }
|
|
22
|
+
// tree-sitter-kotlin is an optionalDependency — may not be installed
|
|
23
|
+
let Kotlin = null;
|
|
24
|
+
try {
|
|
25
|
+
Kotlin = _require('tree-sitter-kotlin');
|
|
26
|
+
}
|
|
27
|
+
catch { }
|
|
22
28
|
let parser = null;
|
|
23
29
|
const languageMap = {
|
|
24
30
|
[SupportedLanguages.JavaScript]: JavaScript,
|
|
@@ -31,8 +37,9 @@ const languageMap = {
|
|
|
31
37
|
[SupportedLanguages.CSharp]: CSharp,
|
|
32
38
|
[SupportedLanguages.Go]: Go,
|
|
33
39
|
[SupportedLanguages.Rust]: Rust,
|
|
34
|
-
[SupportedLanguages.Kotlin]: Kotlin,
|
|
40
|
+
...(Kotlin ? { [SupportedLanguages.Kotlin]: Kotlin } : {}),
|
|
35
41
|
[SupportedLanguages.PHP]: PHP.php_only,
|
|
42
|
+
[SupportedLanguages.Ruby]: Ruby,
|
|
36
43
|
...(Swift ? { [SupportedLanguages.Swift]: Swift } : {}),
|
|
37
44
|
};
|
|
38
45
|
export const isLanguageAvailable = (language) => language in languageMap;
|
|
@@ -36,14 +36,14 @@ export declare class WikiGenerator {
|
|
|
36
36
|
private repoPath;
|
|
37
37
|
private storagePath;
|
|
38
38
|
private wikiDir;
|
|
39
|
-
private
|
|
39
|
+
private lbugPath;
|
|
40
40
|
private llmConfig;
|
|
41
41
|
private maxTokensPerModule;
|
|
42
42
|
private concurrency;
|
|
43
43
|
private options;
|
|
44
44
|
private onProgress;
|
|
45
45
|
private failedModules;
|
|
46
|
-
constructor(repoPath: string, storagePath: string,
|
|
46
|
+
constructor(repoPath: string, storagePath: string, lbugPath: string, llmConfig: LLMConfig, options?: WikiOptions, onProgress?: ProgressCallback);
|
|
47
47
|
private lastPercent;
|
|
48
48
|
/**
|
|
49
49
|
* Create streaming options that report LLM progress to the progress bar.
|
|
@@ -25,18 +25,18 @@ export class WikiGenerator {
|
|
|
25
25
|
repoPath;
|
|
26
26
|
storagePath;
|
|
27
27
|
wikiDir;
|
|
28
|
-
|
|
28
|
+
lbugPath;
|
|
29
29
|
llmConfig;
|
|
30
30
|
maxTokensPerModule;
|
|
31
31
|
concurrency;
|
|
32
32
|
options;
|
|
33
33
|
onProgress;
|
|
34
34
|
failedModules = [];
|
|
35
|
-
constructor(repoPath, storagePath,
|
|
35
|
+
constructor(repoPath, storagePath, lbugPath, llmConfig, options = {}, onProgress) {
|
|
36
36
|
this.repoPath = repoPath;
|
|
37
37
|
this.storagePath = storagePath;
|
|
38
38
|
this.wikiDir = path.join(storagePath, WIKI_DIR);
|
|
39
|
-
this.
|
|
39
|
+
this.lbugPath = lbugPath;
|
|
40
40
|
this.options = options;
|
|
41
41
|
this.llmConfig = llmConfig;
|
|
42
42
|
this.maxTokensPerModule = options.maxTokensPerModule ?? DEFAULT_MAX_TOKENS_PER_MODULE;
|
|
@@ -95,7 +95,7 @@ export class WikiGenerator {
|
|
|
95
95
|
}
|
|
96
96
|
// Init graph
|
|
97
97
|
this.onProgress('init', 2, 'Connecting to knowledge graph...');
|
|
98
|
-
await initWikiDb(this.
|
|
98
|
+
await initWikiDb(this.lbugPath);
|
|
99
99
|
let result;
|
|
100
100
|
try {
|
|
101
101
|
if (!forceMode && existingMeta && existingMeta.fromCommit) {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Graph Queries for Wiki Generation
|
|
3
3
|
*
|
|
4
4
|
* Encapsulated Cypher queries against the GitNexus knowledge graph.
|
|
5
|
-
* Uses the MCP-style pooled
|
|
5
|
+
* Uses the MCP-style pooled lbug-adapter for connection management.
|
|
6
6
|
*/
|
|
7
7
|
export interface FileWithExports {
|
|
8
8
|
filePath: string;
|
|
@@ -30,11 +30,11 @@ export interface ProcessInfo {
|
|
|
30
30
|
}>;
|
|
31
31
|
}
|
|
32
32
|
/**
|
|
33
|
-
* Initialize the
|
|
33
|
+
* Initialize the LadybugDB connection for wiki generation.
|
|
34
34
|
*/
|
|
35
|
-
export declare function initWikiDb(
|
|
35
|
+
export declare function initWikiDb(lbugPath: string): Promise<void>;
|
|
36
36
|
/**
|
|
37
|
-
* Close the
|
|
37
|
+
* Close the LadybugDB connection.
|
|
38
38
|
*/
|
|
39
39
|
export declare function closeWikiDb(): Promise<void>;
|
|
40
40
|
/**
|
|
@@ -2,21 +2,21 @@
|
|
|
2
2
|
* Graph Queries for Wiki Generation
|
|
3
3
|
*
|
|
4
4
|
* Encapsulated Cypher queries against the GitNexus knowledge graph.
|
|
5
|
-
* Uses the MCP-style pooled
|
|
5
|
+
* Uses the MCP-style pooled lbug-adapter for connection management.
|
|
6
6
|
*/
|
|
7
|
-
import {
|
|
7
|
+
import { initLbug, executeQuery, closeLbug } from '../../mcp/core/lbug-adapter.js';
|
|
8
8
|
const REPO_ID = '__wiki__';
|
|
9
9
|
/**
|
|
10
|
-
* Initialize the
|
|
10
|
+
* Initialize the LadybugDB connection for wiki generation.
|
|
11
11
|
*/
|
|
12
|
-
export async function initWikiDb(
|
|
13
|
-
await
|
|
12
|
+
export async function initWikiDb(lbugPath) {
|
|
13
|
+
await initLbug(REPO_ID, lbugPath);
|
|
14
14
|
}
|
|
15
15
|
/**
|
|
16
|
-
* Close the
|
|
16
|
+
* Close the LadybugDB connection.
|
|
17
17
|
*/
|
|
18
18
|
export async function closeWikiDb() {
|
|
19
|
-
await
|
|
19
|
+
await closeLbug(REPO_ID);
|
|
20
20
|
}
|
|
21
21
|
/**
|
|
22
22
|
* Get all source files with their exported symbol names and types.
|