devsmind-mcp 2.4.0 → 4.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +317 -234
- package/dist/cli/activity.d.ts +9 -0
- package/dist/cli/activity.js +69 -0
- package/dist/cli/activity.js.map +1 -0
- package/dist/cli/describe.d.ts +31 -0
- package/dist/cli/describe.js +201 -0
- package/dist/cli/describe.js.map +1 -0
- package/dist/cli/diff.d.ts +14 -0
- package/dist/cli/diff.js +161 -0
- package/dist/cli/diff.js.map +1 -0
- package/dist/cli/embed.d.ts +6 -0
- package/dist/cli/embed.js +109 -0
- package/dist/cli/embed.js.map +1 -0
- package/dist/cli/extract-agent.d.ts +66 -0
- package/dist/cli/extract-agent.js +192 -0
- package/dist/cli/extract-agent.js.map +1 -0
- package/dist/cli/feedback.d.ts +11 -0
- package/dist/cli/feedback.js +90 -0
- package/dist/cli/feedback.js.map +1 -0
- package/dist/cli/index.js +139 -9
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.d.ts +30 -0
- package/dist/cli/init.js +89 -18
- package/dist/cli/init.js.map +1 -1
- package/dist/cli/integrations/memory-topics.d.ts +55 -0
- package/dist/cli/integrations/memory-topics.js +318 -0
- package/dist/cli/integrations/memory-topics.js.map +1 -0
- package/dist/cli/integrations/memory.d.ts +17 -6
- package/dist/cli/integrations/memory.js +52 -139
- package/dist/cli/integrations/memory.js.map +1 -1
- package/dist/cli/integrations/prompt.d.ts +8 -0
- package/dist/cli/integrations/prompt.js +24 -3
- package/dist/cli/integrations/prompt.js.map +1 -1
- package/dist/cli/integrations/registry.d.ts +26 -9
- package/dist/cli/integrations/registry.js +61 -30
- package/dist/cli/integrations/registry.js.map +1 -1
- package/dist/cli/llm-client.d.ts +116 -0
- package/dist/cli/llm-client.js +322 -0
- package/dist/cli/llm-client.js.map +1 -0
- package/dist/cli/rule.d.ts +22 -2
- package/dist/cli/rule.js +110 -34
- package/dist/cli/rule.js.map +1 -1
- package/dist/cli/runner.d.ts +21 -0
- package/dist/cli/runner.js +94 -276
- package/dist/cli/runner.js.map +1 -1
- package/dist/cli/sync-progress.d.ts +9 -0
- package/dist/cli/sync-progress.js +41 -0
- package/dist/cli/sync-progress.js.map +1 -0
- package/dist/cli/sync.js +5 -2
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/view.js +3 -2
- package/dist/cli/view.js.map +1 -1
- package/dist/cli/workflow.js +26 -20
- package/dist/cli/workflow.js.map +1 -1
- package/dist/db/activity-graph.d.ts +55 -0
- package/dist/db/activity-graph.js +314 -0
- package/dist/db/activity-graph.js.map +1 -0
- package/dist/db/activity.d.ts +221 -0
- package/dist/db/activity.js +348 -0
- package/dist/db/activity.js.map +1 -0
- package/dist/db/analyze.js +18 -6
- package/dist/db/analyze.js.map +1 -1
- package/dist/db/database.d.ts +654 -102
- package/dist/db/database.js +1911 -558
- package/dist/db/database.js.map +1 -1
- package/dist/db/edges.d.ts +24 -0
- package/dist/db/edges.js +84 -0
- package/dist/db/edges.js.map +1 -1
- package/dist/db/embedder.d.ts +39 -0
- package/dist/db/embedder.js +0 -0
- package/dist/db/embedder.js.map +1 -0
- package/dist/db/feedback.d.ts +128 -0
- package/dist/db/feedback.js +182 -0
- package/dist/db/feedback.js.map +1 -0
- package/dist/db/file-diff.d.ts +32 -0
- package/dist/db/file-diff.js +110 -0
- package/dist/db/file-diff.js.map +1 -0
- package/dist/db/grep.d.ts +78 -0
- package/dist/db/grep.js +475 -0
- package/dist/db/grep.js.map +1 -0
- package/dist/db/index-build.d.ts +75 -0
- package/dist/db/index-build.js +177 -0
- package/dist/db/index-build.js.map +1 -0
- package/dist/db/message-revert.d.ts +63 -0
- package/dist/db/message-revert.js +258 -0
- package/dist/db/message-revert.js.map +1 -0
- package/dist/db/revert.d.ts +31 -0
- package/dist/db/revert.js +108 -0
- package/dist/db/revert.js.map +1 -0
- package/dist/db/schema.d.ts +49 -4
- package/dist/db/schema.js +125 -73
- package/dist/db/schema.js.map +1 -1
- package/dist/db/search-index.d.ts +65 -0
- package/dist/db/search-index.js +74 -0
- package/dist/db/search-index.js.map +1 -0
- package/dist/db/staging.d.ts +91 -5
- package/dist/db/staging.js +148 -22
- package/dist/db/staging.js.map +1 -1
- package/dist/mcp/server.d.ts +29 -7
- package/dist/mcp/server.js +2637 -878
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/vendor/3d-force-graph.min.js +5 -0
- package/dist/mcp/vendor/force-graph.min.js +5 -0
- package/dist/mcp/vendor/model/model_int8.onnx +0 -0
- package/dist/mcp/vendor/model/vocab.txt +30522 -0
- package/dist/mcp/vendor/three.min.js +7 -0
- package/dist/mcp/view.css +419 -0
- package/dist/mcp/view.html +161 -0
- package/dist/mcp/view.js +245 -0
- package/dist/mcp/view_chat.js +382 -0
- package/dist/mcp/view_graph.js +576 -0
- package/dist/mcp/visualizer.d.ts +25 -2
- package/dist/mcp/visualizer.js +31 -4
- package/dist/mcp/visualizer.js.map +1 -1
- package/dist/utils/ast.d.ts +89 -0
- package/dist/utils/ast.js +378 -10
- package/dist/utils/ast.js.map +1 -1
- package/dist/utils/diff.d.ts +44 -0
- package/dist/utils/diff.js +78 -0
- package/dist/utils/diff.js.map +1 -0
- package/dist/utils/edit.d.ts +6 -0
- package/dist/utils/edit.js +47 -14
- package/dist/utils/edit.js.map +1 -1
- package/dist/utils/scanner.d.ts +6 -4
- package/dist/utils/scanner.js +18 -8
- package/dist/utils/scanner.js.map +1 -1
- package/dist/utils/tokenize.d.ts +45 -0
- package/dist/utils/tokenize.js +129 -0
- package/dist/utils/tokenize.js.map +1 -0
- package/dist/utils/version.d.ts +14 -0
- package/dist/utils/version.js +61 -0
- package/dist/utils/version.js.map +1 -0
- package/package.json +18 -5
- package/dist/mcp/visualizer_2d.html +0 -635
- package/dist/mcp/visualizer_3d.html +0 -613
package/dist/cli/runner.js
CHANGED
|
@@ -36,6 +36,11 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.makeHttpRequest = makeHttpRequest;
|
|
40
|
+
exports.sleep = sleep;
|
|
41
|
+
exports.throttleRpm = throttleRpm;
|
|
42
|
+
exports.getAccessTokenFromServiceAccount = getAccessTokenFromServiceAccount;
|
|
43
|
+
exports.getVertexTokenCached = getVertexTokenCached;
|
|
39
44
|
exports.runBackgroundIndexing = runBackgroundIndexing;
|
|
40
45
|
exports.runBackgroundReindexing = runBackgroundReindexing;
|
|
41
46
|
const fs = __importStar(require("fs"));
|
|
@@ -48,9 +53,12 @@ const database_1 = require("../db/database");
|
|
|
48
53
|
const indexer_1 = require("../db/indexer");
|
|
49
54
|
const scanner_1 = require("../utils/scanner");
|
|
50
55
|
const config_1 = require("../utils/config");
|
|
51
|
-
const json_1 = require("../utils/json");
|
|
52
56
|
const ast_1 = require("../utils/ast");
|
|
53
57
|
const edges_1 = require("../db/edges");
|
|
58
|
+
const extract_agent_1 = require("./extract-agent");
|
|
59
|
+
const describe_1 = require("./describe");
|
|
60
|
+
/** Exported for reuse by `src/cli/llm-client.ts` (the `devsmind describe` backfill) — plain
|
|
61
|
+
* request/response, nothing indexing-specific about it. */
|
|
54
62
|
function makeHttpRequest(urlStr, method, headers, body) {
|
|
55
63
|
return new Promise((resolve, reject) => {
|
|
56
64
|
const isHttps = urlStr.startsWith('https');
|
|
@@ -93,8 +101,10 @@ function sleep(ms) {
|
|
|
93
101
|
}
|
|
94
102
|
// ── LLM request pacing ───────────────────────────────────────────────────
|
|
95
103
|
// Off by default — requests fire as fast as possible and 429s are handled by
|
|
96
|
-
// the retry/backoff in
|
|
97
|
-
// out requests and stay under a known quota instead of reacting after the fact.
|
|
104
|
+
// the retry/backoff in sendConversationTurnWithRetry (llm-client.ts). Pass --rpm to
|
|
105
|
+
// proactively space out requests and stay under a known quota instead of reacting after the fact.
|
|
106
|
+
// Module-global on purpose: `devsmind describe` shares this same pacing budget with indexing
|
|
107
|
+
// if both were ever run in the same process, which is the correct behavior (one shared quota).
|
|
98
108
|
let lastLlmCallAt = 0;
|
|
99
109
|
async function throttleRpm(rpm) {
|
|
100
110
|
if (!rpm || rpm <= 0)
|
|
@@ -270,22 +280,6 @@ class ProgressDisplay {
|
|
|
270
280
|
console.log(` ${checkmark} ${summary} \x1B[90m(total: ${fmtMs(elapsed)}, avg: ${avg > 0 ? fmtMs(avg) : '—'}/item)\x1B[0m`);
|
|
271
281
|
}
|
|
272
282
|
}
|
|
273
|
-
// Build standard taxonomy prompt text
|
|
274
|
-
const TAXONOMY_PROMPT = `
|
|
275
|
-
Choose node types from this taxonomy:
|
|
276
|
-
- UNIVERSAL: function | method | class | abstract_class | interface | type_alias | enum | constant | variable | module | namespace | decorator
|
|
277
|
-
- NESTJS: nest_module | nest_controller | nest_service | nest_provider | nest_guard | nest_interceptor | nest_pipe | nest_filter | nest_decorator | nest_middleware | nest_gateway | nest_resolver | nest_schema | nest_dto
|
|
278
|
-
- EXPRESS/FASTIFY: route_handler | middleware | router
|
|
279
|
-
- SPRING (Java): spring_controller | spring_service | spring_repository | spring_component | spring_bean | spring_config | spring_entity
|
|
280
|
-
- DJANGO/FASTAPI: django_view | django_model | django_serializer | django_form | django_signal | fastapi_router | fastapi_dependency
|
|
281
|
-
- GO: go_handler | go_middleware | go_struct | go_interface | go_func
|
|
282
|
-
- RUST: rust_struct | rust_impl | rust_trait | rust_enum | rust_fn | rust_macro
|
|
283
|
-
- REACT/NEXTJS: react_component | react_hook | react_context | react_hoc | react_page | next_page | next_layout | next_api_route | next_server_action
|
|
284
|
-
- ORM: prisma_model | typeorm_entity | mongoose_model | sqlalchemy_model
|
|
285
|
-
- REST/API/GRAPHQL: api_endpoint | rest_controller | graphql_resolver | graphql_query | graphql_mutation | graphql_schema
|
|
286
|
-
- CLI: cli_command | cli_option
|
|
287
|
-
- UTILITY: util_function | helper | validator | formatter
|
|
288
|
-
`;
|
|
289
283
|
// ── Vertex AI Authentication & Helper Functions ───────────────────────────
|
|
290
284
|
function base64UrlEncode(obj) {
|
|
291
285
|
return Buffer.from(JSON.stringify(obj))
|
|
@@ -374,237 +368,6 @@ async function getVertexTokenCached(saData) {
|
|
|
374
368
|
vertexTokenExpiry = Date.now() + 3600 * 1000;
|
|
375
369
|
return token;
|
|
376
370
|
}
|
|
377
|
-
async function extractWithVertex(model, token, projectId, location, filePath, code) {
|
|
378
|
-
const url = `https://${location}-aiplatform.googleapis.com/v1/projects/${projectId}/locations/${location}/publishers/google/models/${model}:generateContent`;
|
|
379
|
-
const systemPrompt = `You are a codebase indexing assistant. Your job is to analyze the source code file provided and extract all code structures (functions, methods, classes, controllers, services, interfaces, schema models, types) defined in the file.
|
|
380
|
-
Return ONLY a valid JSON object matching the schema:
|
|
381
|
-
{
|
|
382
|
-
"nodes": [
|
|
383
|
-
{
|
|
384
|
-
"node_id": "fully_qualified_identifier (e.g. Class.method or function)",
|
|
385
|
-
"name": "display_name",
|
|
386
|
-
"type": "type_from_taxonomy",
|
|
387
|
-
"signature": "param/return signature (optional)",
|
|
388
|
-
"code_snapshot": "the exact full source code block of this entity"
|
|
389
|
-
}
|
|
390
|
-
]
|
|
391
|
-
}
|
|
392
|
-
${TAXONOMY_PROMPT}
|
|
393
|
-
CRITICAL RULES:
|
|
394
|
-
1. ONLY extract code structures defined in the file. Do NOT extract imports or third-party libraries as nodes.
|
|
395
|
-
2. For each node, extract its exact code snippet as "code_snapshot".
|
|
396
|
-
3. DO NOT wrap JSON in markdown blocks (e.g. no \`\`\`json). Return raw JSON.
|
|
397
|
-
4. Be highly precise and return an empty JSON object if no code constructs are found.`;
|
|
398
|
-
const payload = {
|
|
399
|
-
contents: [
|
|
400
|
-
{
|
|
401
|
-
role: 'user',
|
|
402
|
-
parts: [
|
|
403
|
-
{
|
|
404
|
-
text: `File path: ${filePath}\n\nCode:\n${code}`
|
|
405
|
-
}
|
|
406
|
-
]
|
|
407
|
-
}
|
|
408
|
-
],
|
|
409
|
-
systemInstruction: {
|
|
410
|
-
parts: [
|
|
411
|
-
{
|
|
412
|
-
text: systemPrompt
|
|
413
|
-
}
|
|
414
|
-
]
|
|
415
|
-
},
|
|
416
|
-
generationConfig: {
|
|
417
|
-
responseMimeType: 'application/json'
|
|
418
|
-
}
|
|
419
|
-
};
|
|
420
|
-
const responseText = await makeHttpRequest(url, 'POST', {
|
|
421
|
-
'Content-Type': 'application/json',
|
|
422
|
-
'Authorization': `Bearer ${token}`
|
|
423
|
-
}, JSON.stringify(payload));
|
|
424
|
-
const parsed = (0, json_1.safeJsonParse)(responseText, {});
|
|
425
|
-
const text = parsed.candidates?.[0]?.content?.parts?.[0]?.text;
|
|
426
|
-
if (!text) {
|
|
427
|
-
return {};
|
|
428
|
-
}
|
|
429
|
-
return (0, json_1.safeJsonParse)(text, {});
|
|
430
|
-
}
|
|
431
|
-
async function extractWithGemini(model, key, filePath, code) {
|
|
432
|
-
const url = `https://generativelanguage.googleapis.com/v1beta/models/${model}:generateContent?key=${key}`;
|
|
433
|
-
const systemPrompt = `You are a codebase indexing assistant. Your job is to analyze the source code file provided and extract all code structures (functions, methods, classes, controllers, services, interfaces, schema models, types) defined in the file.
|
|
434
|
-
Return ONLY a valid JSON object matching the schema:
|
|
435
|
-
{
|
|
436
|
-
"nodes": [
|
|
437
|
-
{
|
|
438
|
-
"node_id": "fully_qualified_identifier (e.g. Class.method or function)",
|
|
439
|
-
"name": "display_name",
|
|
440
|
-
"type": "type_from_taxonomy",
|
|
441
|
-
"signature": "param/return signature (optional)",
|
|
442
|
-
"code_snapshot": "the exact full source code block of this entity"
|
|
443
|
-
}
|
|
444
|
-
]
|
|
445
|
-
}
|
|
446
|
-
${TAXONOMY_PROMPT}
|
|
447
|
-
CRITICAL RULES:
|
|
448
|
-
1. ONLY extract code structures defined in the file. Do NOT extract imports or third-party libraries as nodes.
|
|
449
|
-
2. For each node, extract its exact code snippet as "code_snapshot".
|
|
450
|
-
3. DO NOT wrap JSON in markdown blocks (e.g. no \`\`\`json). Return raw JSON.
|
|
451
|
-
4. Be highly precise and return an empty JSON object if no code constructs are found.`;
|
|
452
|
-
const payload = {
|
|
453
|
-
contents: [
|
|
454
|
-
{
|
|
455
|
-
parts: [
|
|
456
|
-
{
|
|
457
|
-
text: `File path: ${filePath}\n\nCode:\n${code}`
|
|
458
|
-
}
|
|
459
|
-
]
|
|
460
|
-
}
|
|
461
|
-
],
|
|
462
|
-
systemInstruction: {
|
|
463
|
-
parts: [
|
|
464
|
-
{
|
|
465
|
-
text: systemPrompt
|
|
466
|
-
}
|
|
467
|
-
]
|
|
468
|
-
},
|
|
469
|
-
generationConfig: {
|
|
470
|
-
responseMimeType: 'application/json'
|
|
471
|
-
}
|
|
472
|
-
};
|
|
473
|
-
const responseText = await makeHttpRequest(url, 'POST', { 'Content-Type': 'application/json' }, JSON.stringify(payload));
|
|
474
|
-
const parsed = (0, json_1.safeJsonParse)(responseText, {});
|
|
475
|
-
const text = parsed.candidates?.[0]?.content?.parts?.[0]?.text;
|
|
476
|
-
if (!text) {
|
|
477
|
-
return {};
|
|
478
|
-
}
|
|
479
|
-
return (0, json_1.safeJsonParse)(text, {});
|
|
480
|
-
}
|
|
481
|
-
async function extractWithOllama(url, model, filePath, code) {
|
|
482
|
-
const endpoint = `${url.replace(/\/$/, '')}/api/chat`;
|
|
483
|
-
const systemPrompt = `You are a codebase indexing assistant. Analyze this source code file and extract code structures (functions, classes, methods, endpoints).
|
|
484
|
-
Return ONLY a valid JSON object matching the schema:
|
|
485
|
-
{
|
|
486
|
-
"nodes": [
|
|
487
|
-
{
|
|
488
|
-
"node_id": "unique_string (e.g. Class.method or function)",
|
|
489
|
-
"name": "display_name",
|
|
490
|
-
"type": "type_from_taxonomy",
|
|
491
|
-
"signature": "param/return signature (optional)",
|
|
492
|
-
"code_snapshot": "the exact full source code block of this entity"
|
|
493
|
-
}
|
|
494
|
-
]
|
|
495
|
-
}
|
|
496
|
-
${TAXONOMY_PROMPT}
|
|
497
|
-
CRITICAL RULES:
|
|
498
|
-
1. ONLY extract constructs defined in this file. Do NOT extract third-party libraries or imports.
|
|
499
|
-
2. For each node, extract its exact code snippet as "code_snapshot".
|
|
500
|
-
3. Return a clean, valid JSON object.`;
|
|
501
|
-
const userPrompt = `File path: ${filePath}\n\nCode:\n${code}`;
|
|
502
|
-
const payload = {
|
|
503
|
-
model,
|
|
504
|
-
messages: [
|
|
505
|
-
{ role: 'system', content: systemPrompt },
|
|
506
|
-
{ role: 'user', content: userPrompt }
|
|
507
|
-
],
|
|
508
|
-
stream: false,
|
|
509
|
-
format: 'json'
|
|
510
|
-
};
|
|
511
|
-
const responseText = await makeHttpRequest(endpoint, 'POST', { 'Content-Type': 'application/json' }, JSON.stringify(payload));
|
|
512
|
-
const parsed = (0, json_1.safeJsonParse)(responseText, {});
|
|
513
|
-
const text = parsed.message?.content;
|
|
514
|
-
if (!text) {
|
|
515
|
-
return {};
|
|
516
|
-
}
|
|
517
|
-
return (0, json_1.safeJsonParse)(text, {});
|
|
518
|
-
}
|
|
519
|
-
async function extractNodesFromCode(provider, modelName, key, url, filePath, code, getVertexToken, vertexProjectId, vertexLocation, progress, chunkSize, chunkOverlap, rpm) {
|
|
520
|
-
// Chunking is opt-in: with no --chunk-size, the whole file always goes in one call.
|
|
521
|
-
const maxLines = chunkSize;
|
|
522
|
-
const overlap = chunkOverlap ?? 50;
|
|
523
|
-
const lines = code.split('\n');
|
|
524
|
-
const executeExtraction = async (codeChunk) => {
|
|
525
|
-
let retries = 5;
|
|
526
|
-
let backoffMs = 10000;
|
|
527
|
-
while (retries > 0) {
|
|
528
|
-
try {
|
|
529
|
-
await throttleRpm(rpm);
|
|
530
|
-
if (provider === 'gemini') {
|
|
531
|
-
return await extractWithGemini(modelName, key, filePath, codeChunk);
|
|
532
|
-
}
|
|
533
|
-
else if (provider === 'vertex') {
|
|
534
|
-
const token = await getVertexToken();
|
|
535
|
-
return await extractWithVertex(modelName, token, vertexProjectId, vertexLocation, filePath, codeChunk);
|
|
536
|
-
}
|
|
537
|
-
else {
|
|
538
|
-
return await extractWithOllama(url, modelName, filePath, codeChunk);
|
|
539
|
-
}
|
|
540
|
-
}
|
|
541
|
-
catch (err) {
|
|
542
|
-
retries--;
|
|
543
|
-
if (retries === 0) {
|
|
544
|
-
throw err;
|
|
545
|
-
}
|
|
546
|
-
const errMsg = err.message;
|
|
547
|
-
if (errMsg.includes('429')) {
|
|
548
|
-
progress.updateStatus(`Rate limited (429). Retrying in ${backoffMs / 1000}s...`);
|
|
549
|
-
await sleep(backoffMs);
|
|
550
|
-
backoffMs *= 2;
|
|
551
|
-
}
|
|
552
|
-
else {
|
|
553
|
-
progress.updateStatus(`API error. Retrying in 2s...`);
|
|
554
|
-
await sleep(2000);
|
|
555
|
-
}
|
|
556
|
-
}
|
|
557
|
-
}
|
|
558
|
-
return {};
|
|
559
|
-
};
|
|
560
|
-
if (!maxLines || lines.length <= maxLines) {
|
|
561
|
-
return await executeExtraction(code);
|
|
562
|
-
}
|
|
563
|
-
const relPath = path.relative(process.cwd(), filePath);
|
|
564
|
-
progress.log(`\x1B[90m[${relPath}] Large file (${lines.length} lines) - parsing in chunks to prevent LLM output truncation...\x1B[0m`);
|
|
565
|
-
const chunks = [];
|
|
566
|
-
let start = 0;
|
|
567
|
-
// Guard against overlap >= chunkSize, which would make the step <= 0 and loop forever.
|
|
568
|
-
const step = Math.max(1, maxLines - overlap);
|
|
569
|
-
while (start < lines.length) {
|
|
570
|
-
const end = Math.min(start + maxLines, lines.length);
|
|
571
|
-
chunks.push(lines.slice(start, end).join('\n'));
|
|
572
|
-
if (end === lines.length)
|
|
573
|
-
break;
|
|
574
|
-
start += step;
|
|
575
|
-
}
|
|
576
|
-
const seenNodeIds = new Map();
|
|
577
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
578
|
-
progress.updateStatus(`Sending chunk ${i + 1}/${chunks.length} to AI…`);
|
|
579
|
-
const chunkCode = chunks[i];
|
|
580
|
-
try {
|
|
581
|
-
const chunkResult = await executeExtraction(chunkCode);
|
|
582
|
-
if (chunkResult.nodes && Array.isArray(chunkResult.nodes)) {
|
|
583
|
-
for (const node of chunkResult.nodes) {
|
|
584
|
-
if (!node.node_id)
|
|
585
|
-
continue;
|
|
586
|
-
const existing = seenNodeIds.get(node.node_id);
|
|
587
|
-
if (!existing || (node.code_snapshot && (!existing.code_snapshot || node.code_snapshot.length > existing.code_snapshot.length))) {
|
|
588
|
-
seenNodeIds.set(node.node_id, node);
|
|
589
|
-
}
|
|
590
|
-
}
|
|
591
|
-
}
|
|
592
|
-
}
|
|
593
|
-
catch (err) {
|
|
594
|
-
progress.log(`\x1B[31mError extracting chunk ${i + 1}/${chunks.length}: ${err.message}\x1B[0m`);
|
|
595
|
-
throw err;
|
|
596
|
-
}
|
|
597
|
-
if (i < chunks.length - 1) {
|
|
598
|
-
if (provider === 'gemini' || provider === 'vertex') {
|
|
599
|
-
await sleep(1500);
|
|
600
|
-
}
|
|
601
|
-
else {
|
|
602
|
-
await sleep(100);
|
|
603
|
-
}
|
|
604
|
-
}
|
|
605
|
-
}
|
|
606
|
-
return { nodes: Array.from(seenNodeIds.values()) };
|
|
607
|
-
}
|
|
608
371
|
async function runBackgroundIndexing(opts) {
|
|
609
372
|
const resolvedDevmind = path.resolve(opts.devmindPath);
|
|
610
373
|
const chunkSize = opts.chunkSize;
|
|
@@ -613,6 +376,13 @@ async function runBackgroundIndexing(opts) {
|
|
|
613
376
|
const nodesOnly = !!opts.nodesOnly;
|
|
614
377
|
const edgesOnly = !!opts.edgesOnly;
|
|
615
378
|
const rpm = opts.rpm;
|
|
379
|
+
// Phase 3 (description backfill) is MANDATORY on a full run (neither --nodes-only nor
|
|
380
|
+
// --edges-only) — Phase 1/2 never write a description, so skipping it would leave a "finished"
|
|
381
|
+
// index that's not actually searchable by search_nodes' description-weighted BM25/vector layers.
|
|
382
|
+
// On --nodes-only it's an optional extra pass on top of the structure-only extraction, gated by
|
|
383
|
+
// --describe, since --nodes-only exists specifically for a fast partial run. --edges-only never
|
|
384
|
+
// resolves credentials or creates nodes, so it's never eligible (also rejected upfront in index.ts).
|
|
385
|
+
const shouldDescribe = edgesOnly ? false : (nodesOnly ? !!opts.describe : true);
|
|
616
386
|
// Repo scoping: restrict the whole operation to the named repos. Standalone-only.
|
|
617
387
|
const scopedRepos = opts.repos && opts.repos.length ? opts.repos : null;
|
|
618
388
|
const inScope = (nodeId) => !scopedRepos || scopedRepos.some(r => nodeId.startsWith(`{${r}}/`));
|
|
@@ -663,7 +433,13 @@ async function runBackgroundIndexing(opts) {
|
|
|
663
433
|
console.log(` Brain directory : ${resolvedDevmind}`);
|
|
664
434
|
console.log(` Provider : ${opts.provider}`);
|
|
665
435
|
console.log(` Connections : local AST resolution (always)`);
|
|
666
|
-
console.log(`
|
|
436
|
+
console.log(` Extraction : deterministic AST enumeration + agentic curation of ambiguous candidates only`);
|
|
437
|
+
console.log(` Describe : ${edgesOnly ? 'n/a (--edges-only never creates nodes)'
|
|
438
|
+
: nodesOnly ? (shouldDescribe ? 'enabled via --describe — Phase 3 will backfill descriptions after extraction' : 'disabled (pass --describe to also backfill descriptions after this --nodes-only run)')
|
|
439
|
+
: 'mandatory for a full run — Phase 3 will backfill descriptions after indexing'}`);
|
|
440
|
+
if (chunkSize) {
|
|
441
|
+
console.log(` ⚠ --chunk-size is ignored — extraction is per-candidate now (AST-enumerated), not whole-file-to-an-LLM, so chunking a huge file no longer applies.`);
|
|
442
|
+
}
|
|
667
443
|
console.log(` Rate limit : ${rpm ? `${rpm} req/min` : 'unthrottled'}`);
|
|
668
444
|
let modelName = opts.model || '';
|
|
669
445
|
let vertexSaData = null;
|
|
@@ -720,13 +496,18 @@ async function runBackgroundIndexing(opts) {
|
|
|
720
496
|
}
|
|
721
497
|
console.log(` Model : ${modelName}`);
|
|
722
498
|
}
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
499
|
+
// Built once here from whatever the provider-specific block above resolved — the ONE place
|
|
500
|
+
// per-file extraction needs credentials from, instead of threading five separate params
|
|
501
|
+
// through every call the way `extractNodesFromCode` used to.
|
|
502
|
+
const llmCreds = {
|
|
503
|
+
provider: opts.provider,
|
|
504
|
+
model: modelName,
|
|
505
|
+
apiKey: opts.key,
|
|
506
|
+
vertexSaData: vertexSaData || undefined,
|
|
507
|
+
vertexToken: vertexToken || undefined,
|
|
508
|
+
vertexProjectId,
|
|
509
|
+
vertexLocation,
|
|
510
|
+
url: opts.url
|
|
730
511
|
};
|
|
731
512
|
// 1. Open DB
|
|
732
513
|
const dbFile = path.join(resolvedDevmind, 'brain.db');
|
|
@@ -759,18 +540,23 @@ async function runBackgroundIndexing(opts) {
|
|
|
759
540
|
if (edgesOnly) {
|
|
760
541
|
// All nodes stay in the candidate pool (targets can live in any repo/file), but when
|
|
761
542
|
// scoped we only rebuild edges ORIGINATING from the named repos' nodes.
|
|
762
|
-
const
|
|
763
|
-
if (
|
|
543
|
+
const rawNodes = db.listNodes();
|
|
544
|
+
if (rawNodes.length === 0) {
|
|
764
545
|
console.error('❌ Error: --edges-only requires nodes to already exist. Run without --edges-only first (or with --nodes-only) to extract nodes.');
|
|
765
546
|
db.close();
|
|
766
547
|
process.exit(1);
|
|
767
548
|
}
|
|
768
|
-
const existingNodes = scopedRepos ?
|
|
549
|
+
const existingNodes = scopedRepos ? rawNodes.filter(n => inScope(n.id)) : rawNodes;
|
|
769
550
|
if (existingNodes.length === 0) {
|
|
770
551
|
console.error(`❌ Error: no nodes found for repo(s): ${scopedRepos?.join(', ')}. Extract nodes first.`);
|
|
771
552
|
db.close();
|
|
772
553
|
process.exit(1);
|
|
773
554
|
}
|
|
555
|
+
// Deterministic alias detection (RTK Query hook names, etc.) over this batch's files, BEFORE
|
|
556
|
+
// the candidate pool is fetched, so the resolver below sees any freshly-attached aliases in
|
|
557
|
+
// this SAME run rather than needing a second pass.
|
|
558
|
+
(0, edges_1.applyDeterministicAliases)(db, existingNodes.map(n => n.id));
|
|
559
|
+
const allNodes = db.listNodes();
|
|
774
560
|
let edgePad = (0, indexer_1.readScratchpad)(resolvedDevmind, padFile);
|
|
775
561
|
let resumeIndex = 0;
|
|
776
562
|
// Scoped runs never resume the shared scratchpad (its counts describe a different set).
|
|
@@ -898,10 +684,13 @@ async function runBackgroundIndexing(opts) {
|
|
|
898
684
|
continue;
|
|
899
685
|
}
|
|
900
686
|
const fileLines = code.split('\n').length;
|
|
901
|
-
progress.updateStatus(`
|
|
687
|
+
progress.updateStatus(`Enumerating candidates deterministically…`);
|
|
902
688
|
let result = {};
|
|
903
689
|
try {
|
|
904
|
-
result = await
|
|
690
|
+
result = await (0, extract_agent_1.extractFileWithCuration)(llmCreds, fileObj.absolutePath, {
|
|
691
|
+
rpm,
|
|
692
|
+
onLog: (line) => progress.log(line)
|
|
693
|
+
});
|
|
905
694
|
}
|
|
906
695
|
catch (err) {
|
|
907
696
|
progress.finishPhase(`Paused — API error. Run again to resume.`);
|
|
@@ -993,10 +782,15 @@ async function runBackgroundIndexing(opts) {
|
|
|
993
782
|
// PHASE 2: AI CONNECTION RESOLUTION / LINKING
|
|
994
783
|
// =========================================================================
|
|
995
784
|
if (pad.phase === 2 && !nodesOnly) {
|
|
996
|
-
const
|
|
997
|
-
const allNodeIds = allNodes.map(n => n.id);
|
|
785
|
+
const rawNodes = db.listNodes();
|
|
998
786
|
// Candidates are always all nodes; when scoped we only (re)build edges from the
|
|
999
787
|
// named repos' nodes and clear just those first so we don't wipe other repos' edges.
|
|
788
|
+
const activeNodesForAlias = scopedRepos ? rawNodes.filter(n => inScope(n.id)) : rawNodes;
|
|
789
|
+
// Deterministic alias detection (RTK Query hook names, etc.) BEFORE the candidate pool is
|
|
790
|
+
// fetched, so the resolver below sees any freshly-attached aliases in this SAME run.
|
|
791
|
+
(0, edges_1.applyDeterministicAliases)(db, activeNodesForAlias.map(n => n.id));
|
|
792
|
+
const allNodes = db.listNodes();
|
|
793
|
+
const allNodeIds = allNodes.map(n => n.id);
|
|
1000
794
|
const activeNodes = scopedRepos ? allNodes.filter(n => inScope(n.id)) : allNodes;
|
|
1001
795
|
const resumeIndex = pad.nodes_done || 0;
|
|
1002
796
|
if (scopedRepos && resumeIndex === 0) {
|
|
@@ -1037,6 +831,16 @@ async function runBackgroundIndexing(opts) {
|
|
|
1037
831
|
progress.finishPhase(`Phase 2 done — ${pad.connections_created} connection(s) linked across ${pad.nodes_total} node(s)`);
|
|
1038
832
|
(0, edges_1.finalizeMissingNodes)(resolvedDevmind, db, missingRefs);
|
|
1039
833
|
}
|
|
834
|
+
// =========================================================================
|
|
835
|
+
// PHASE 3: DESCRIPTION BACKFILL — mandatory on a full run, optional (--describe) on --nodes-only
|
|
836
|
+
// =========================================================================
|
|
837
|
+
let describeResult = null;
|
|
838
|
+
if (shouldDescribe) {
|
|
839
|
+
console.log(`\n${'═'.repeat(60)}`);
|
|
840
|
+
console.log(` Phase 3: Description Backfill`);
|
|
841
|
+
console.log(`${'═'.repeat(60)}\n`);
|
|
842
|
+
describeResult = await (0, describe_1.describePendingNodes)(db, llmCreds, { batchSize: opts.describeBatchSize, rpm });
|
|
843
|
+
}
|
|
1040
844
|
// Mark indexing session as fully complete
|
|
1041
845
|
pad.status = 'complete';
|
|
1042
846
|
pad.updated_at = new Date().toISOString();
|
|
@@ -1047,7 +851,10 @@ async function runBackgroundIndexing(opts) {
|
|
|
1047
851
|
console.log('\x1B[1m\x1B[32m ✔ Indexing complete!\x1B[0m');
|
|
1048
852
|
console.log(` ├─ Files indexed : \x1B[33m${pad.files_done}\x1B[0m`);
|
|
1049
853
|
console.log(` ├─ Nodes created : \x1B[33m${pad.nodes_created}\x1B[0m`);
|
|
1050
|
-
console.log(` └─ Connections : \x1B[33m${pad.connections_created}\x1B[0m`);
|
|
854
|
+
console.log(describeResult ? ` ├─ Connections : \x1B[33m${pad.connections_created}\x1B[0m` : ` └─ Connections : \x1B[33m${pad.connections_created}\x1B[0m`);
|
|
855
|
+
if (describeResult) {
|
|
856
|
+
console.log(` └─ Described : \x1B[33m${describeResult.described}\x1B[0m/${describeResult.pending}${describeResult.failed ? ` (${describeResult.failed} failed — re-run with --describe to retry)` : ''}`);
|
|
857
|
+
}
|
|
1051
858
|
console.log('');
|
|
1052
859
|
}
|
|
1053
860
|
async function runBackgroundReindexing(opts) {
|
|
@@ -1060,7 +867,10 @@ async function runBackgroundReindexing(opts) {
|
|
|
1060
867
|
console.log(` Brain directory : ${resolvedDevmind}`);
|
|
1061
868
|
console.log(` Provider : ${opts.provider}`);
|
|
1062
869
|
console.log(` Connections : local AST resolution (always)`);
|
|
1063
|
-
console.log(`
|
|
870
|
+
console.log(` Extraction : deterministic AST enumeration + agentic curation of ambiguous candidates only`);
|
|
871
|
+
if (chunkSize) {
|
|
872
|
+
console.log(` ⚠ --chunk-size is ignored — extraction is per-candidate now (AST-enumerated), not whole-file-to-an-LLM, so chunking a huge file no longer applies.`);
|
|
873
|
+
}
|
|
1064
874
|
console.log(` Rate limit : ${rpm ? `${rpm} req/min` : 'unthrottled'}`);
|
|
1065
875
|
let modelName = opts.model || '';
|
|
1066
876
|
let vertexSaData = null;
|
|
@@ -1112,13 +922,15 @@ async function runBackgroundReindexing(opts) {
|
|
|
1112
922
|
modelName = modelName || 'qwen2.5-coder';
|
|
1113
923
|
opts.url = opts.url || 'http://localhost:11434';
|
|
1114
924
|
}
|
|
1115
|
-
const
|
|
1116
|
-
|
|
1117
|
-
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
925
|
+
const llmCreds = {
|
|
926
|
+
provider: opts.provider,
|
|
927
|
+
model: modelName,
|
|
928
|
+
apiKey: opts.key,
|
|
929
|
+
vertexSaData: vertexSaData || undefined,
|
|
930
|
+
vertexToken: vertexToken || undefined,
|
|
931
|
+
vertexProjectId,
|
|
932
|
+
vertexLocation,
|
|
933
|
+
url: opts.url
|
|
1122
934
|
};
|
|
1123
935
|
console.log(` Model : ${modelName}`);
|
|
1124
936
|
// 1. Open DB
|
|
@@ -1219,10 +1031,13 @@ async function runBackgroundReindexing(opts) {
|
|
|
1219
1031
|
continue;
|
|
1220
1032
|
}
|
|
1221
1033
|
const fileLines = code.split('\n').length;
|
|
1222
|
-
progress.updateStatus(`
|
|
1034
|
+
progress.updateStatus(`Enumerating candidates deterministically…`);
|
|
1223
1035
|
let result = {};
|
|
1224
1036
|
try {
|
|
1225
|
-
result = await
|
|
1037
|
+
result = await (0, extract_agent_1.extractFileWithCuration)(llmCreds, fileObj.absolutePath, {
|
|
1038
|
+
rpm,
|
|
1039
|
+
onLog: (line) => progress.log(line)
|
|
1040
|
+
});
|
|
1226
1041
|
}
|
|
1227
1042
|
catch (err) {
|
|
1228
1043
|
if (fillGaps) {
|
|
@@ -1291,6 +1106,8 @@ async function runBackgroundReindexing(opts) {
|
|
|
1291
1106
|
// re-resolving the new nodes' own outbound edges isn't enough. This is local AST
|
|
1292
1107
|
// resolution (no LLM calls), so rebuilding it across the whole graph is cheap and
|
|
1293
1108
|
// safe to repeat.
|
|
1109
|
+
const rawNodesForAlias = db.listNodes();
|
|
1110
|
+
(0, edges_1.applyDeterministicAliases)(db, rawNodesForAlias.map(n => n.id));
|
|
1294
1111
|
const activeNodes = db.listNodes();
|
|
1295
1112
|
const allNodeIds = new Set(activeNodes.map(n => n.id));
|
|
1296
1113
|
console.log('\n🧹 Clearing existing connections for a full rebuild...');
|
|
@@ -1332,6 +1149,7 @@ async function runBackgroundReindexing(opts) {
|
|
|
1332
1149
|
}
|
|
1333
1150
|
// Phase 2: Resolving connections for modified nodes
|
|
1334
1151
|
if (newOrUpdatedNodeIds.length > 0) {
|
|
1152
|
+
(0, edges_1.applyDeterministicAliases)(db, newOrUpdatedNodeIds);
|
|
1335
1153
|
const activeNodes = db.listNodes();
|
|
1336
1154
|
const allNodeIds = activeNodes.map(n => n.id);
|
|
1337
1155
|
progress.startPhase(2, 'Incremental Connection Resolution', newOrUpdatedNodeIds.length, 0);
|