devsmind-mcp 2.4.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +326 -234
- package/dist/cli/activity.d.ts +9 -0
- package/dist/cli/activity.js +69 -0
- package/dist/cli/activity.js.map +1 -0
- package/dist/cli/describe.d.ts +31 -0
- package/dist/cli/describe.js +201 -0
- package/dist/cli/describe.js.map +1 -0
- package/dist/cli/diff.d.ts +14 -0
- package/dist/cli/diff.js +161 -0
- package/dist/cli/diff.js.map +1 -0
- package/dist/cli/embed.d.ts +6 -0
- package/dist/cli/embed.js +109 -0
- package/dist/cli/embed.js.map +1 -0
- package/dist/cli/extract-agent.d.ts +66 -0
- package/dist/cli/extract-agent.js +192 -0
- package/dist/cli/extract-agent.js.map +1 -0
- package/dist/cli/feedback.d.ts +11 -0
- package/dist/cli/feedback.js +90 -0
- package/dist/cli/feedback.js.map +1 -0
- package/dist/cli/index.js +128 -5
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.d.ts +30 -0
- package/dist/cli/init.js +89 -18
- package/dist/cli/init.js.map +1 -1
- package/dist/cli/integrations/memory-topics.d.ts +44 -0
- package/dist/cli/integrations/memory-topics.js +295 -0
- package/dist/cli/integrations/memory-topics.js.map +1 -0
- package/dist/cli/integrations/memory.d.ts +11 -5
- package/dist/cli/integrations/memory.js +132 -37
- package/dist/cli/integrations/memory.js.map +1 -1
- package/dist/cli/integrations/prompt.d.ts +8 -0
- package/dist/cli/integrations/prompt.js +24 -3
- package/dist/cli/integrations/prompt.js.map +1 -1
- package/dist/cli/integrations/registry.d.ts +19 -5
- package/dist/cli/integrations/registry.js +23 -13
- package/dist/cli/integrations/registry.js.map +1 -1
- package/dist/cli/llm-client.d.ts +116 -0
- package/dist/cli/llm-client.js +322 -0
- package/dist/cli/llm-client.js.map +1 -0
- package/dist/cli/rule.d.ts +22 -2
- package/dist/cli/rule.js +113 -32
- package/dist/cli/rule.js.map +1 -1
- package/dist/cli/runner.d.ts +21 -0
- package/dist/cli/runner.js +94 -276
- package/dist/cli/runner.js.map +1 -1
- package/dist/cli/sync-progress.d.ts +9 -0
- package/dist/cli/sync-progress.js +41 -0
- package/dist/cli/sync-progress.js.map +1 -0
- package/dist/cli/sync.js +5 -2
- package/dist/cli/sync.js.map +1 -1
- package/dist/cli/view.js +3 -2
- package/dist/cli/view.js.map +1 -1
- package/dist/cli/workflow.js +26 -20
- package/dist/cli/workflow.js.map +1 -1
- package/dist/db/activity.d.ts +200 -0
- package/dist/db/activity.js +347 -0
- package/dist/db/activity.js.map +1 -0
- package/dist/db/analyze.js +18 -6
- package/dist/db/analyze.js.map +1 -1
- package/dist/db/database.d.ts +583 -99
- package/dist/db/database.js +1825 -553
- package/dist/db/database.js.map +1 -1
- package/dist/db/edges.d.ts +24 -0
- package/dist/db/edges.js +84 -0
- package/dist/db/edges.js.map +1 -1
- package/dist/db/embedder.d.ts +39 -0
- package/dist/db/embedder.js +0 -0
- package/dist/db/embedder.js.map +1 -0
- package/dist/db/feedback.d.ts +128 -0
- package/dist/db/feedback.js +182 -0
- package/dist/db/feedback.js.map +1 -0
- package/dist/db/file-diff.d.ts +32 -0
- package/dist/db/file-diff.js +110 -0
- package/dist/db/file-diff.js.map +1 -0
- package/dist/db/grep.d.ts +78 -0
- package/dist/db/grep.js +475 -0
- package/dist/db/grep.js.map +1 -0
- package/dist/db/message-revert.d.ts +63 -0
- package/dist/db/message-revert.js +258 -0
- package/dist/db/message-revert.js.map +1 -0
- package/dist/db/revert.d.ts +31 -0
- package/dist/db/revert.js +108 -0
- package/dist/db/revert.js.map +1 -0
- package/dist/db/schema.d.ts +49 -4
- package/dist/db/schema.js +125 -73
- package/dist/db/schema.js.map +1 -1
- package/dist/db/search-index.d.ts +65 -0
- package/dist/db/search-index.js +74 -0
- package/dist/db/search-index.js.map +1 -0
- package/dist/db/staging.d.ts +90 -5
- package/dist/db/staging.js +148 -22
- package/dist/db/staging.js.map +1 -1
- package/dist/mcp/server.d.ts +29 -7
- package/dist/mcp/server.js +2584 -809
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/vendor/3d-force-graph.min.js +5 -0
- package/dist/mcp/vendor/force-graph.min.js +5 -0
- package/dist/mcp/vendor/model/model_int8.onnx +0 -0
- package/dist/mcp/vendor/model/vocab.txt +30522 -0
- package/dist/mcp/vendor/three.min.js +7 -0
- package/dist/mcp/view.css +419 -0
- package/dist/mcp/view.html +161 -0
- package/dist/mcp/view.js +245 -0
- package/dist/mcp/view_chat.js +382 -0
- package/dist/mcp/view_graph.js +576 -0
- package/dist/mcp/visualizer.d.ts +25 -2
- package/dist/mcp/visualizer.js +31 -4
- package/dist/mcp/visualizer.js.map +1 -1
- package/dist/utils/ast.d.ts +89 -0
- package/dist/utils/ast.js +378 -10
- package/dist/utils/ast.js.map +1 -1
- package/dist/utils/diff.d.ts +44 -0
- package/dist/utils/diff.js +78 -0
- package/dist/utils/diff.js.map +1 -0
- package/dist/utils/edit.d.ts +6 -0
- package/dist/utils/edit.js +47 -14
- package/dist/utils/edit.js.map +1 -1
- package/dist/utils/scanner.js +12 -4
- package/dist/utils/scanner.js.map +1 -1
- package/dist/utils/tokenize.d.ts +45 -0
- package/dist/utils/tokenize.js +129 -0
- package/dist/utils/tokenize.js.map +1 -0
- package/dist/utils/version.d.ts +14 -0
- package/dist/utils/version.js +61 -0
- package/dist/utils/version.js.map +1 -0
- package/package.json +18 -5
- package/dist/mcp/visualizer_2d.html +0 -635
- package/dist/mcp/visualizer_3d.html +0 -613
package/dist/db/database.js
CHANGED
|
@@ -36,30 +36,95 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
36
36
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
37
|
};
|
|
38
38
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
-
exports.DevMindDatabase = void 0;
|
|
39
|
+
exports.DevMindDatabase = exports.NO_STATIC_CALLERS_NOTE = exports.WORKFLOW_SIDECAR_FILE = exports.WORKFLOW_SCHEMA_VERSION = void 0;
|
|
40
|
+
exports.toCompactSearchResult = toCompactSearchResult;
|
|
40
41
|
exports.formatReasoning = formatReasoning;
|
|
41
42
|
exports.parseReasoningBlocks = parseReasoningBlocks;
|
|
42
43
|
const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
|
|
43
44
|
const crypto = __importStar(require("crypto"));
|
|
44
45
|
const fs = __importStar(require("fs"));
|
|
45
46
|
const path = __importStar(require("path"));
|
|
46
|
-
const zlib = __importStar(require("zlib"));
|
|
47
47
|
const schema_1 = require("./schema");
|
|
48
48
|
const config_1 = require("../utils/config");
|
|
49
49
|
const ast_1 = require("../utils/ast");
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
50
|
+
const tokenize_1 = require("../utils/tokenize");
|
|
51
|
+
const search_index_1 = require("./search-index");
|
|
52
|
+
const embedder_1 = require("./embedder");
|
|
53
|
+
const grep_1 = require("./grep");
|
|
54
|
+
/** A confident "0 callers" is a false negative on dynamic-dispatch/generated-binding code (RTK
|
|
55
|
+
* hooks, DI containers) — the AST resolver can't see those calls at all. Shared between
|
|
56
|
+
* `search_nodes`' drill-in hooks and `get_node_code`'s `used_by_note` so the two tools can never
|
|
57
|
+
* say this in two slightly different ways. */
|
|
58
|
+
/** How many budget-dropped node ids `getGraph` will name before it stops listing them — enough to
|
|
59
|
+
* act on, not so many that the omission list becomes its own oversized payload. */
|
|
60
|
+
const OMITTED_NODE_ID_CAP = 20;
|
|
61
|
+
/** Bumped when the shape written to `.devmind/workflows/<id>/workflow.json` changes. v2 replaced
|
|
62
|
+
* per-step `history_ids`/`pending_tasks` with `reasoning`/`node_ids`/`doc_paths`, and workflow
|
|
63
|
+
* `status`/`is_active` with `archived`. */
|
|
64
|
+
exports.WORKFLOW_SCHEMA_VERSION = 2;
|
|
65
|
+
/** The half of a workflow a v1 client has no field for, and therefore cannot overwrite when it
|
|
66
|
+
* rewrites `workflow.json` from its own columns. See `writeWorkflowToDisk`. */
|
|
67
|
+
exports.WORKFLOW_SIDECAR_FILE = 'v2.json';
|
|
68
|
+
exports.NO_STATIC_CALLERS_NOTE = 'no static callers found — may be used via dynamic dispatch or a generated binding; verify before assuming unused';
|
|
69
|
+
/** Tier 1 keeps a couple of sample lines; anything past this is bulk, not evidence. */
|
|
70
|
+
const COMPACT_SAMPLE_CAP = 2;
|
|
71
|
+
/** Tier 1 line truncation. 200 chars is enough to read a matching line in context; the full
|
|
72
|
+
* 400-char lines are the single biggest contributor to an oversized files bucket. */
|
|
73
|
+
const COMPACT_LINE_CAP = 200;
|
|
74
|
+
/**
|
|
75
|
+
* Shrink a search result to fit, in two tiers.
|
|
76
|
+
*
|
|
77
|
+
* Why two rather than an on/off switch: the sample lines and `code_matches` are genuinely the
|
|
78
|
+
* most useful part of a result — real agent feedback credits them with catching a live bug — so
|
|
79
|
+
* throwing all of them away at the first byte over a threshold overcorrects. Tier 1 drops what is
|
|
80
|
+
* bulk-without-signal (`match_counts`, `matched_terms`, `aliases`, `created_at`, `deprecated`, and
|
|
81
|
+
* the repeated `used_by_note` boilerplate) and thins the rest; only tier 2 gives up the evidence
|
|
82
|
+
* lines entirely and becomes a pure triage list.
|
|
83
|
+
*
|
|
84
|
+
* `confidence`/`relevance`/`found_by` and the `uses`/`used_by`/`history_count` drill-in hooks
|
|
85
|
+
* survive BOTH tiers on purpose. They are a handful of bytes each and they are precisely what a
|
|
86
|
+
* caller uses to decide which result to open next — dropping them would make a compact response
|
|
87
|
+
* smaller and useless at the same time.
|
|
88
|
+
*
|
|
89
|
+
* Pure: no DB access, no I/O. Kept here rather than in the MCP handler so it is unit-testable
|
|
90
|
+
* directly, and so it sits inside the coverage gate.
|
|
91
|
+
*/
|
|
92
|
+
function toCompactSearchResult(result, tier) {
|
|
93
|
+
const keepEvidence = tier === 1;
|
|
94
|
+
const trimLines = (lines) => lines.slice(0, COMPACT_SAMPLE_CAP).map(l => ({ ...l, line_content: l.line_content.slice(0, COMPACT_LINE_CAP) }));
|
|
95
|
+
return {
|
|
96
|
+
nodes: result.nodes.map(n => {
|
|
97
|
+
// Cast rather than narrow the RankedNode union: `code_matches` exists only on the fuzzy
|
|
98
|
+
// variant, and the identifier variant simply has no evidence lines to trim.
|
|
99
|
+
const src = n;
|
|
100
|
+
return {
|
|
101
|
+
id: src.id,
|
|
102
|
+
name: src.name,
|
|
103
|
+
type: src.type,
|
|
104
|
+
confidence: src.confidence,
|
|
105
|
+
relevance: src.relevance,
|
|
106
|
+
found_by: src.found_by,
|
|
107
|
+
file_path: src.file_path,
|
|
108
|
+
signature: src.signature,
|
|
109
|
+
description: src.description,
|
|
110
|
+
uses: src.uses,
|
|
111
|
+
used_by: src.used_by,
|
|
112
|
+
history_count: src.history_count,
|
|
113
|
+
code_matches: keepEvidence && src.code_matches ? trimLines(src.code_matches) : undefined
|
|
114
|
+
};
|
|
115
|
+
}),
|
|
116
|
+
files: result.files.map(f => ({
|
|
117
|
+
file_path: f.file_path,
|
|
118
|
+
total_matches: f.total_matches,
|
|
119
|
+
sample_lines: keepEvidence ? trimLines(f.sample_lines) : undefined
|
|
120
|
+
})),
|
|
121
|
+
files_total: result.files_total,
|
|
122
|
+
files_offset: result.files_offset,
|
|
123
|
+
nodes_total: result.nodes_total,
|
|
124
|
+
hint: result.hint,
|
|
125
|
+
truncated: result.truncated,
|
|
126
|
+
scope_note: result.scope_note
|
|
127
|
+
};
|
|
63
128
|
}
|
|
64
129
|
function formatReasoning(r) {
|
|
65
130
|
if (typeof r === 'string') {
|
|
@@ -77,6 +142,42 @@ function formatReasoning(r) {
|
|
|
77
142
|
];
|
|
78
143
|
return lines.join('\n');
|
|
79
144
|
}
|
|
145
|
+
/** Matches the `── Update @ … ──` separator `updateHistory` appends each same-session update under. */
|
|
146
|
+
const REASONING_SEPARATOR = /(\n*── Update @ [^\n]*──\n)/g;
|
|
147
|
+
/**
|
|
148
|
+
* Removes one block from an accumulated reasoning log, matched by its exact text and searched
|
|
149
|
+
* from the newest end.
|
|
150
|
+
*
|
|
151
|
+
* Not simply "drop the last block": an update that carries no code change (a `stage_change`, a
|
|
152
|
+
* bare `update_history`) appends reasoning without recording an edit, so blocks and edits are not
|
|
153
|
+
* one-to-one and positional removal would take the wrong one. Matching on content is exact when
|
|
154
|
+
* the block is there, and when it isn't the log is returned untouched — leaving a stale line is
|
|
155
|
+
* recoverable, mangling someone else's reasoning is not.
|
|
156
|
+
*
|
|
157
|
+
* Splitting on a capturing group keeps the separators in the result, so every block that stays
|
|
158
|
+
* keeps its original timestamp rather than being re-stamped on the way out.
|
|
159
|
+
*/
|
|
160
|
+
function dropReasoningBlock(raw, block) {
|
|
161
|
+
if (!raw || !block)
|
|
162
|
+
return raw;
|
|
163
|
+
const target = block.trim();
|
|
164
|
+
if (!target)
|
|
165
|
+
return raw;
|
|
166
|
+
// [block, sep, block, sep, block, …] — blocks at even indices, separators at odd.
|
|
167
|
+
const parts = raw.split(REASONING_SEPARATOR);
|
|
168
|
+
for (let i = parts.length - 1; i >= 0; i -= 2) {
|
|
169
|
+
if (parts[i].trim() !== target)
|
|
170
|
+
continue;
|
|
171
|
+
// Drop the block with the separator that introduced it. The first block has none, so it
|
|
172
|
+
// takes the separator that follows instead — whatever came after now leads the log.
|
|
173
|
+
if (i > 0)
|
|
174
|
+
parts.splice(i - 1, 2);
|
|
175
|
+
else
|
|
176
|
+
parts.splice(0, 2);
|
|
177
|
+
return parts.join('').replace(/^\n+/, '');
|
|
178
|
+
}
|
|
179
|
+
return raw;
|
|
180
|
+
}
|
|
80
181
|
/**
|
|
81
182
|
* Inverse of `formatReasoning`. A single history row accumulates every later update appended
|
|
82
183
|
* under a `── Update @ … ──` separator, so one stored blob can hold several changes — this
|
|
@@ -122,7 +223,15 @@ class DevMindDatabase {
|
|
|
122
223
|
db;
|
|
123
224
|
dbPath;
|
|
124
225
|
context = null;
|
|
125
|
-
|
|
226
|
+
/**
|
|
227
|
+
* `onSyncProgress`: optional, fires during the constructor's initial `syncFromDisk()` pass —
|
|
228
|
+
* the one silent stretch every CLI command pays on `new DevMindDatabase(...)` before it can
|
|
229
|
+
* print anything else. Large `.devmind` folders (mainly `history/`, which grows one file per
|
|
230
|
+
* edit — much faster than node count) can make that pass take minutes; without this, a caller
|
|
231
|
+
* has no way to tell "still working" from "hung". Omit it for silent construction (the MCP
|
|
232
|
+
* server's normal path, where per-open console spam would be noise, not signal).
|
|
233
|
+
*/
|
|
234
|
+
constructor(dbPath, opts) {
|
|
126
235
|
this.dbPath = dbPath;
|
|
127
236
|
// Open SQLite database
|
|
128
237
|
this.db = new better_sqlite3_1.default(dbPath);
|
|
@@ -138,7 +247,14 @@ class DevMindDatabase {
|
|
|
138
247
|
// Ignore context errors (e.g. running from scratch scripts)
|
|
139
248
|
}
|
|
140
249
|
// Auto-sync history and graph from disk JSONs
|
|
141
|
-
this.syncFromDisk();
|
|
250
|
+
this.syncFromDisk(opts?.onSyncProgress);
|
|
251
|
+
}
|
|
252
|
+
/** Throttles progress callbacks to ~100 updates across `total` items, regardless of scale —
|
|
253
|
+
* so a 500-file sync and a 500,000-file sync both report about as often, and the callback
|
|
254
|
+
* itself (a stdout write) never becomes the bottleneck it was being added to diagnose. */
|
|
255
|
+
static shouldReport(done, total) {
|
|
256
|
+
const every = Math.max(1, Math.floor(total / 100));
|
|
257
|
+
return done === total || done % every === 0;
|
|
142
258
|
}
|
|
143
259
|
initSchema() {
|
|
144
260
|
this.db.exec(schema_1.INIT_SCHEMA_SQL);
|
|
@@ -148,12 +264,45 @@ class DevMindDatabase {
|
|
|
148
264
|
catch {
|
|
149
265
|
// Column already exists, ignore
|
|
150
266
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
267
|
+
try {
|
|
268
|
+
this.db.exec('ALTER TABLE nodes ADD COLUMN description TEXT');
|
|
269
|
+
}
|
|
270
|
+
catch {
|
|
271
|
+
// Column already exists, ignore
|
|
272
|
+
}
|
|
273
|
+
try {
|
|
274
|
+
this.db.exec("ALTER TABLE nodes ADD COLUMN aliases TEXT DEFAULT '[]'");
|
|
275
|
+
}
|
|
276
|
+
catch {
|
|
277
|
+
// Column already exists, ignore
|
|
278
|
+
}
|
|
279
|
+
// Workflow v2 columns. Same additive, idempotent shape as the three above: on a fresh brain
|
|
280
|
+
// INIT_SCHEMA_SQL already created them so the ALTER throws and the catch absorbs it; on a brain
|
|
281
|
+
// created before v2 the ALTER is what actually adds them. Deliberately NOT paired with a
|
|
282
|
+
// DROP COLUMN for the fields they replace — `status`/`pending_tasks`/`history_ids` stay as
|
|
283
|
+
// vestigial columns, because dropping them buys nothing (they are nullable or defaulted, so
|
|
284
|
+
// nothing has to write them) and would break an older globally-installed CLI opening the same
|
|
285
|
+
// brain.db with "no such column".
|
|
286
|
+
for (const ddl of [
|
|
287
|
+
'ALTER TABLE workflow_steps ADD COLUMN reasoning TEXT',
|
|
288
|
+
'ALTER TABLE workflow_steps ADD COLUMN node_ids TEXT',
|
|
289
|
+
'ALTER TABLE workflow_steps ADD COLUMN doc_paths TEXT',
|
|
290
|
+
'ALTER TABLE workflows ADD COLUMN archived INTEGER NOT NULL DEFAULT 0'
|
|
291
|
+
]) {
|
|
292
|
+
try {
|
|
293
|
+
this.db.exec(ddl);
|
|
294
|
+
}
|
|
295
|
+
catch {
|
|
296
|
+
// Column already exists, ignore
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
this.backfillWorkflowStepNodeIds();
|
|
300
|
+
this.db.exec(`
|
|
301
|
+
CREATE TABLE IF NOT EXISTS system_meta (
|
|
302
|
+
key TEXT PRIMARY KEY,
|
|
303
|
+
value TEXT NOT NULL,
|
|
304
|
+
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
|
|
305
|
+
)
|
|
157
306
|
`);
|
|
158
307
|
}
|
|
159
308
|
getContext() {
|
|
@@ -170,10 +319,10 @@ class DevMindDatabase {
|
|
|
170
319
|
}
|
|
171
320
|
}
|
|
172
321
|
setSystemMeta(key, value) {
|
|
173
|
-
const stmt = this.db.prepare(`
|
|
174
|
-
INSERT INTO system_meta (key, value, updated_at)
|
|
175
|
-
VALUES (?, ?, CURRENT_TIMESTAMP)
|
|
176
|
-
ON CONFLICT(key) DO UPDATE SET value = ?, updated_at = CURRENT_TIMESTAMP
|
|
322
|
+
const stmt = this.db.prepare(`
|
|
323
|
+
INSERT INTO system_meta (key, value, updated_at)
|
|
324
|
+
VALUES (?, ?, CURRENT_TIMESTAMP)
|
|
325
|
+
ON CONFLICT(key) DO UPDATE SET value = ?, updated_at = CURRENT_TIMESTAMP
|
|
177
326
|
`);
|
|
178
327
|
stmt.run(key, value, value);
|
|
179
328
|
}
|
|
@@ -186,7 +335,7 @@ class DevMindDatabase {
|
|
|
186
335
|
*/
|
|
187
336
|
getNodesByFilePath(filePath) {
|
|
188
337
|
const stmt = this.db.prepare(`SELECT * FROM nodes WHERE deprecated = 0 AND REPLACE(LOWER(file_path), '\\', '/') = ?`);
|
|
189
|
-
return stmt.all((0, ast_1.normalizeFsPath)(filePath));
|
|
338
|
+
return DevMindDatabase.parseNodeRows(stmt.all((0, ast_1.normalizeFsPath)(filePath)));
|
|
190
339
|
}
|
|
191
340
|
close() {
|
|
192
341
|
this.db.close();
|
|
@@ -196,6 +345,10 @@ class DevMindDatabase {
|
|
|
196
345
|
const one = (sql) => {
|
|
197
346
|
try {
|
|
198
347
|
const row = this.db.prepare(sql).get();
|
|
348
|
+
/* istanbul ignore next -- every call site here is a `SELECT COUNT(*) AS c FROM ...`,
|
|
349
|
+
which always returns exactly one row; `row` can only be undefined if this helper is
|
|
350
|
+
ever repurposed for a query that can return zero rows. Kept as a real guard, not
|
|
351
|
+
because today's three call sites can hit it. */
|
|
199
352
|
return row ? row.c : 0;
|
|
200
353
|
}
|
|
201
354
|
catch {
|
|
@@ -226,9 +379,10 @@ class DevMindDatabase {
|
|
|
226
379
|
this.db.exec('DELETE FROM node_connections');
|
|
227
380
|
this.db.exec('DELETE FROM history');
|
|
228
381
|
this.db.exec('DELETE FROM nodes');
|
|
382
|
+
this.db.exec('DELETE FROM node_vectors');
|
|
229
383
|
this.db.exec('DELETE FROM system_meta');
|
|
230
384
|
const workspaceRoot = path.dirname(this.dbPath);
|
|
231
|
-
for (const dir of ['graph', 'history']) {
|
|
385
|
+
for (const dir of ['graph', 'history', 'vectors']) {
|
|
232
386
|
const p = path.join(workspaceRoot, dir);
|
|
233
387
|
if (fs.existsSync(p)) {
|
|
234
388
|
fs.rmSync(p, { recursive: true, force: true });
|
|
@@ -289,6 +443,11 @@ class DevMindDatabase {
|
|
|
289
443
|
upsertNode(node) {
|
|
290
444
|
const canonicalFp = (0, config_1.canonicalizePath)(node.file_path);
|
|
291
445
|
const existing = this.getNode(node.id);
|
|
446
|
+
// aliases JSON is only computed when the caller actually passed some — an ordinary
|
|
447
|
+
// edit/re-index of a node that ISN'T alias-bearing must not blank out aliases a prior
|
|
448
|
+
// detector pass (or Phase E's record_alias) already attached, same COALESCE idiom as
|
|
449
|
+
// description/signature below.
|
|
450
|
+
const aliasesJson = node.aliases ? JSON.stringify(Array.from(new Set(node.aliases))) : null;
|
|
292
451
|
if (existing) {
|
|
293
452
|
let finalPath = existing.file_path;
|
|
294
453
|
const paths = existing.file_path.split(',').map(p => p.trim()).filter(Boolean);
|
|
@@ -297,36 +456,143 @@ class DevMindDatabase {
|
|
|
297
456
|
paths.push(incoming);
|
|
298
457
|
finalPath = paths.join(', ');
|
|
299
458
|
}
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
459
|
+
// description follows the same COALESCE idiom as signature: an unspecified description
|
|
460
|
+
// on an ordinary edit must never blank out one already written — only an explicit new
|
|
461
|
+
// value (from add_description, or a description passed alongside this edit) overwrites it.
|
|
462
|
+
const stmt = this.db.prepare(`
|
|
463
|
+
UPDATE nodes
|
|
464
|
+
SET type = ?,
|
|
465
|
+
name = ?,
|
|
466
|
+
file_path = ?,
|
|
467
|
+
signature = COALESCE(?, signature),
|
|
468
|
+
description = COALESCE(?, description),
|
|
469
|
+
aliases = COALESCE(?, aliases),
|
|
470
|
+
deprecated = 0
|
|
471
|
+
WHERE id = ?
|
|
308
472
|
`);
|
|
309
|
-
stmt.run(node.type, node.name, finalPath, node.signature || null, node.id);
|
|
473
|
+
stmt.run(node.type, node.name, finalPath, node.signature || null, node.description || null, aliasesJson, node.id);
|
|
310
474
|
}
|
|
311
475
|
else {
|
|
312
|
-
const stmt = this.db.prepare(`
|
|
313
|
-
INSERT INTO nodes (id, type, name, file_path, signature)
|
|
314
|
-
VALUES (?, ?, ?, ?, ?)
|
|
476
|
+
const stmt = this.db.prepare(`
|
|
477
|
+
INSERT INTO nodes (id, type, name, file_path, signature, description, aliases)
|
|
478
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
315
479
|
`);
|
|
316
|
-
stmt.run(node.id, node.type, node.name, canonicalFp, node.signature || null);
|
|
480
|
+
stmt.run(node.id, node.type, node.name, canonicalFp, node.signature || null, node.description || null, aliasesJson ?? '[]');
|
|
317
481
|
}
|
|
318
482
|
this.writeGraphToDisk(canonicalFp);
|
|
319
483
|
}
|
|
484
|
+
/**
|
|
485
|
+
* Adds one alias to a node WITHOUT touching any it already has — the merge-safe counterpart to
|
|
486
|
+
* `upsertNode`'s replace-if-given aliases. This is what the batch graph-fix session's
|
|
487
|
+
* `record_alias` correction tool (Phase E) uses: it should never be able to accidentally drop an
|
|
488
|
+
* alias a deterministic detector pass already attached.
|
|
489
|
+
*/
|
|
490
|
+
addAlias(nodeId, alias) {
|
|
491
|
+
const node = this.getNode(nodeId);
|
|
492
|
+
if (!node)
|
|
493
|
+
return;
|
|
494
|
+
if (node.aliases.includes(alias))
|
|
495
|
+
return;
|
|
496
|
+
const next = [...node.aliases, alias];
|
|
497
|
+
this.db.prepare('UPDATE nodes SET aliases = ? WHERE id = ?').run(JSON.stringify(next), node.id);
|
|
498
|
+
this.writeGraphToDisk(node.file_path);
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* Stores a node's semantic vector and writes it to the committed `vectors/*.json` tree.
|
|
502
|
+
* `vector` must already be int8-quantized (`embedTextInt8`/`embedTextsInt8` in embedder.ts) and
|
|
503
|
+
* `descriptionHash` must be `hashDescription()` of the exact description it was computed from —
|
|
504
|
+
* this is the staleness key `getNodesNeedingEmbedding` checks against.
|
|
505
|
+
*/
|
|
506
|
+
upsertNodeVector(nodeId, vector, descriptionHash) {
|
|
507
|
+
const node = this.getNode(nodeId);
|
|
508
|
+
if (!node)
|
|
509
|
+
return;
|
|
510
|
+
const stmt = this.db.prepare(`
|
|
511
|
+
INSERT INTO node_vectors (node_id, model_id, dim, description_hash, vector)
|
|
512
|
+
VALUES (?, ?, ?, ?, ?)
|
|
513
|
+
ON CONFLICT(node_id) DO UPDATE SET
|
|
514
|
+
model_id = excluded.model_id,
|
|
515
|
+
dim = excluded.dim,
|
|
516
|
+
description_hash = excluded.description_hash,
|
|
517
|
+
vector = excluded.vector
|
|
518
|
+
`);
|
|
519
|
+
stmt.run(node.id, embedder_1.EMBEDDING_MODEL_ID, embedder_1.EMBEDDING_DIM, descriptionHash, Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength));
|
|
520
|
+
this.writeVectorsToDisk(node.file_path);
|
|
521
|
+
}
|
|
522
|
+
getNodeVector(nodeId) {
|
|
523
|
+
const stmt = this.db.prepare('SELECT model_id, dim, description_hash, vector FROM node_vectors WHERE node_id = ?');
|
|
524
|
+
const row = stmt.get(nodeId);
|
|
525
|
+
if (!row)
|
|
526
|
+
return null;
|
|
527
|
+
return {
|
|
528
|
+
modelId: row.model_id,
|
|
529
|
+
dim: row.dim,
|
|
530
|
+
descriptionHash: row.description_hash,
|
|
531
|
+
vector: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength)
|
|
532
|
+
};
|
|
533
|
+
}
|
|
534
|
+
/**
|
|
535
|
+
* Every non-deprecated, described node whose vector is missing, from a different model
|
|
536
|
+
* (`model_id` mismatch — e.g. a mismatched vector ignored during `syncFromDisk`), or stale
|
|
537
|
+
* (its description changed since the vector was computed). This is the work queue for both
|
|
538
|
+
* `devsmind embed` and the auto-embed hooks in `describe`/`add_description` — resumable and
|
|
539
|
+
* idempotent by construction, same shape as `describe.ts`'s own `WHERE description IS NULL`.
|
|
540
|
+
* `force: true` returns every described node regardless of vector state (model upgrades).
|
|
541
|
+
*/
|
|
542
|
+
getNodesNeedingEmbedding(force = false) {
|
|
543
|
+
const described = this.getAllNodes().filter(n => !n.deprecated && n.description);
|
|
544
|
+
if (force)
|
|
545
|
+
return described;
|
|
546
|
+
const vecRows = this.db.prepare('SELECT node_id, model_id, description_hash FROM node_vectors').all();
|
|
547
|
+
const vecMap = new Map(vecRows.map(r => [r.node_id, r]));
|
|
548
|
+
return described.filter(n => {
|
|
549
|
+
const v = vecMap.get(n.id);
|
|
550
|
+
if (!v)
|
|
551
|
+
return true;
|
|
552
|
+
if (v.model_id !== embedder_1.EMBEDDING_MODEL_ID)
|
|
553
|
+
return true;
|
|
554
|
+
if (v.description_hash !== (0, embedder_1.hashDescription)(n.description))
|
|
555
|
+
return true;
|
|
556
|
+
return false;
|
|
557
|
+
});
|
|
558
|
+
}
|
|
559
|
+
/**
|
|
560
|
+
* The `nodes.aliases` column is a JSON-array-in-TEXT blob — better-sqlite3 hands it back as a
|
|
561
|
+
* raw string, not a parsed array, on every `stmt.all()`/`stmt.get()`. A JSON string masquerading
|
|
562
|
+
* as `string[]` is a silent-wrong-answer hazard (both have `.length`, so a bug here would not
|
|
563
|
+
* throw, just quietly misbehave — e.g. counting characters instead of aliases). EVERY raw SQL
|
|
564
|
+
* read of the `nodes` table must route its rows through {@link parseNodeRow}/{@link
|
|
565
|
+
* parseNodeRows}, never cast `as DbNode`/`as DbNode[]` directly.
|
|
566
|
+
*/
|
|
567
|
+
static parseNodeAliases(raw) {
|
|
568
|
+
if (Array.isArray(raw))
|
|
569
|
+
return raw.filter(a => typeof a === 'string');
|
|
570
|
+
if (typeof raw !== 'string' || raw.length === 0)
|
|
571
|
+
return [];
|
|
572
|
+
try {
|
|
573
|
+
const parsed = JSON.parse(raw);
|
|
574
|
+
return Array.isArray(parsed) ? parsed.filter(a => typeof a === 'string') : [];
|
|
575
|
+
}
|
|
576
|
+
catch {
|
|
577
|
+
return [];
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
static parseNodeRow(row) {
|
|
581
|
+
return { ...row, aliases: DevMindDatabase.parseNodeAliases(row.aliases) };
|
|
582
|
+
}
|
|
583
|
+
static parseNodeRows(rows) {
|
|
584
|
+
return rows.map(DevMindDatabase.parseNodeRow);
|
|
585
|
+
}
|
|
320
586
|
getNode(id) {
|
|
321
587
|
const stmt = this.db.prepare('SELECT * FROM nodes WHERE id = ?');
|
|
322
588
|
const direct = stmt.get(id);
|
|
323
589
|
if (direct)
|
|
324
|
-
return direct;
|
|
590
|
+
return DevMindDatabase.parseNodeRow(direct);
|
|
325
591
|
if (!id.includes('#')) {
|
|
326
592
|
const suffixStmt = this.db.prepare("SELECT * FROM nodes WHERE id LIKE ? ESCAPE '\\' AND deprecated = 0");
|
|
327
593
|
const matches = suffixStmt.all(`%#${this.likeEscape(id)}`);
|
|
328
594
|
if (matches.length === 1) {
|
|
329
|
-
return matches[0];
|
|
595
|
+
return DevMindDatabase.parseNodeRow(matches[0]);
|
|
330
596
|
}
|
|
331
597
|
}
|
|
332
598
|
return null;
|
|
@@ -341,8 +607,13 @@ class DevMindDatabase {
|
|
|
341
607
|
this.deleteHistoryFilesForNode(resolvedId);
|
|
342
608
|
const stmt = this.db.prepare('DELETE FROM nodes WHERE id = ?');
|
|
343
609
|
stmt.run(resolvedId);
|
|
610
|
+
// node_vectors has no FK to nodes (see schema.ts) so this doesn't cascade — delete it
|
|
611
|
+
// explicitly rather than waiting for the next syncFromDisk() orphan sweep, or a vector
|
|
612
|
+
// search could keep surfacing a node that no longer exists until the next server restart.
|
|
613
|
+
this.db.prepare('DELETE FROM node_vectors WHERE node_id = ?').run(resolvedId);
|
|
344
614
|
if (node && node.file_path) {
|
|
345
615
|
this.writeGraphToDisk(node.file_path);
|
|
616
|
+
this.writeVectorsToDisk(node.file_path);
|
|
346
617
|
}
|
|
347
618
|
for (const p of inboundSourceFiles) {
|
|
348
619
|
this.writeGraphToDisk(p);
|
|
@@ -356,14 +627,19 @@ class DevMindDatabase {
|
|
|
356
627
|
const inboundSourceFiles = this.collectInboundSourceFiles(resolvedId);
|
|
357
628
|
const updateStmt = this.db.prepare('UPDATE nodes SET deprecated = 1 WHERE id = ?');
|
|
358
629
|
const deleteConnStmt = this.db.prepare('DELETE FROM node_connections WHERE source_node_id = ? OR target_node_id = ?');
|
|
630
|
+
// A deprecated node's vector is pure dead weight (writeVectorsToDisk already excludes
|
|
631
|
+
// deprecated nodes from the JSON) — drop it now rather than leaving an unused row behind.
|
|
632
|
+
const deleteVectorStmt = this.db.prepare('DELETE FROM node_vectors WHERE node_id = ?');
|
|
359
633
|
const tx = this.db.transaction(() => {
|
|
360
634
|
updateStmt.run(resolvedId);
|
|
361
635
|
deleteConnStmt.run(resolvedId, resolvedId);
|
|
636
|
+
deleteVectorStmt.run(resolvedId);
|
|
362
637
|
});
|
|
363
638
|
tx();
|
|
364
639
|
// Rewrite the node's own file (now carrying deprecated:1) and every caller file (so their
|
|
365
640
|
// stale inbound edges don't resurrect the connection on the next syncFromDisk()).
|
|
366
641
|
if (node && node.file_path) {
|
|
642
|
+
this.writeVectorsToDisk(node.file_path);
|
|
367
643
|
this.writeGraphToDisk(node.file_path);
|
|
368
644
|
}
|
|
369
645
|
for (const p of inboundSourceFiles) {
|
|
@@ -388,23 +664,29 @@ class DevMindDatabase {
|
|
|
388
664
|
this.db.pragma('foreign_keys = OFF');
|
|
389
665
|
try {
|
|
390
666
|
const runTx = this.db.transaction(() => {
|
|
391
|
-
const insertStmt = this.db.prepare(`
|
|
392
|
-
INSERT INTO nodes (id, type, name, file_path, signature, created_at, deprecated)
|
|
393
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
667
|
+
const insertStmt = this.db.prepare(`
|
|
668
|
+
INSERT INTO nodes (id, type, name, file_path, signature, description, aliases, created_at, deprecated)
|
|
669
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
394
670
|
`);
|
|
395
|
-
insertStmt.run(newId, node.type, name, filePath, node.signature, node.created_at, node.deprecated ? 1 : 0);
|
|
396
|
-
const updateSourceStmt = this.db.prepare(`
|
|
397
|
-
UPDATE node_connections SET source_node_id = ? WHERE source_node_id = ?
|
|
671
|
+
insertStmt.run(newId, node.type, name, filePath, node.signature, node.description, JSON.stringify(node.aliases), node.created_at, node.deprecated ? 1 : 0);
|
|
672
|
+
const updateSourceStmt = this.db.prepare(`
|
|
673
|
+
UPDATE node_connections SET source_node_id = ? WHERE source_node_id = ?
|
|
398
674
|
`);
|
|
399
675
|
updateSourceStmt.run(newId, resolvedOldId);
|
|
400
|
-
const updateTargetStmt = this.db.prepare(`
|
|
401
|
-
UPDATE node_connections SET target_node_id = ? WHERE target_node_id = ?
|
|
676
|
+
const updateTargetStmt = this.db.prepare(`
|
|
677
|
+
UPDATE node_connections SET target_node_id = ? WHERE target_node_id = ?
|
|
402
678
|
`);
|
|
403
679
|
updateTargetStmt.run(newId, resolvedOldId);
|
|
404
|
-
const updateHistoryStmt = this.db.prepare(`
|
|
405
|
-
UPDATE history SET node_id = ? WHERE node_id = ?
|
|
680
|
+
const updateHistoryStmt = this.db.prepare(`
|
|
681
|
+
UPDATE history SET node_id = ? WHERE node_id = ?
|
|
406
682
|
`);
|
|
407
683
|
updateHistoryStmt.run(newId, resolvedOldId);
|
|
684
|
+
// Carry any existing vector row to the new id — same reasoning as description above:
|
|
685
|
+
// a rename shouldn't force a re-embed. No-op if the node had no vector yet.
|
|
686
|
+
const updateVectorStmt = this.db.prepare(`
|
|
687
|
+
UPDATE node_vectors SET node_id = ? WHERE node_id = ?
|
|
688
|
+
`);
|
|
689
|
+
updateVectorStmt.run(newId, resolvedOldId);
|
|
408
690
|
const deleteOldStmt = this.db.prepare('DELETE FROM nodes WHERE id = ?');
|
|
409
691
|
deleteOldStmt.run(resolvedOldId);
|
|
410
692
|
});
|
|
@@ -413,9 +695,11 @@ class DevMindDatabase {
|
|
|
413
695
|
// Rewrite the OLD file's graph JSON too when the file itself moved, so the
|
|
414
696
|
// stale node entry doesn't linger under the old path's JSON on disk.
|
|
415
697
|
this.writeGraphToDisk(node.file_path);
|
|
698
|
+
this.writeVectorsToDisk(node.file_path);
|
|
416
699
|
}
|
|
417
700
|
if (filePath && filePath !== node.file_path) {
|
|
418
701
|
this.writeGraphToDisk(filePath);
|
|
702
|
+
this.writeVectorsToDisk(filePath);
|
|
419
703
|
}
|
|
420
704
|
// Edges pointing INTO the renamed node live in the SOURCE nodes' files' graph JSONs
|
|
421
705
|
// (which still reference oldId on disk). The DB was already repointed to newId above,
|
|
@@ -434,6 +718,69 @@ class DevMindDatabase {
|
|
|
434
718
|
this.db.pragma('foreign_keys = ON');
|
|
435
719
|
}
|
|
436
720
|
}
|
|
721
|
+
/**
|
|
722
|
+
* Merges `fromId` into `intoId` — the batch graph-fix session's `merge_nodes` correction, for
|
|
723
|
+
* when curation (or a human reviewing feedback) decides two node candidates were never really
|
|
724
|
+
* distinct entities. Unlike `renameNode` (which moves everything to a FRESH id), `intoId`
|
|
725
|
+
* already exists with its own rows here: `fromId`'s connections (both directions) and history
|
|
726
|
+
* are reassigned onto it, `fromId`'s aliases (plus its own name, so old references by that name
|
|
727
|
+
* still resolve) are folded into `intoId`'s alias set, and `fromId` is deprecated — not hard
|
|
728
|
+
* deleted, so its history stays reachable and the merge itself stays a reversible correction,
|
|
729
|
+
* not a destructive one.
|
|
730
|
+
*/
|
|
731
|
+
mergeNodes(fromId, intoId) {
|
|
732
|
+
const fromNode = this.getNode(fromId);
|
|
733
|
+
const intoNode = this.getNode(intoId);
|
|
734
|
+
if (!fromNode)
|
|
735
|
+
throw new Error(`mergeNodes: source node not found: ${fromId}`);
|
|
736
|
+
if (!intoNode)
|
|
737
|
+
throw new Error(`mergeNodes: target node not found: ${intoId}`);
|
|
738
|
+
const resolvedFrom = fromNode.id;
|
|
739
|
+
const resolvedInto = intoNode.id;
|
|
740
|
+
if (resolvedFrom === resolvedInto)
|
|
741
|
+
return; // already the same node — no-op, not an error
|
|
742
|
+
this.db.pragma('foreign_keys = OFF');
|
|
743
|
+
try {
|
|
744
|
+
const runTx = this.db.transaction(() => {
|
|
745
|
+
// Reassign fromId's connections onto intoId in both directions. INSERT OR IGNORE
|
|
746
|
+
// absorbs a duplicate-PK conflict when both nodes already shared a target/source; the
|
|
747
|
+
// `!== resolvedInto` guard drops what would otherwise become a self-referencing edge.
|
|
748
|
+
const outRows = this.db.prepare('SELECT target_node_id FROM node_connections WHERE source_node_id = ?').all(resolvedFrom);
|
|
749
|
+
for (const r of outRows) {
|
|
750
|
+
if (r.target_node_id === resolvedInto)
|
|
751
|
+
continue;
|
|
752
|
+
this.db.prepare('INSERT OR IGNORE INTO node_connections (source_node_id, target_node_id) VALUES (?, ?)').run(resolvedInto, r.target_node_id);
|
|
753
|
+
}
|
|
754
|
+
const inRows = this.db.prepare('SELECT source_node_id FROM node_connections WHERE target_node_id = ?').all(resolvedFrom);
|
|
755
|
+
for (const r of inRows) {
|
|
756
|
+
if (r.source_node_id === resolvedInto)
|
|
757
|
+
continue;
|
|
758
|
+
this.db.prepare('INSERT OR IGNORE INTO node_connections (source_node_id, target_node_id) VALUES (?, ?)').run(r.source_node_id, resolvedInto);
|
|
759
|
+
}
|
|
760
|
+
this.db.prepare('DELETE FROM node_connections WHERE source_node_id = ? OR target_node_id = ?').run(resolvedFrom, resolvedFrom);
|
|
761
|
+
this.db.prepare('UPDATE history SET node_id = ? WHERE node_id = ?').run(resolvedInto, resolvedFrom);
|
|
762
|
+
const mergedAliases = Array.from(new Set([...intoNode.aliases, ...fromNode.aliases, fromNode.name]));
|
|
763
|
+
this.db.prepare('UPDATE nodes SET aliases = ? WHERE id = ?').run(JSON.stringify(mergedAliases), resolvedInto);
|
|
764
|
+
this.db.prepare('UPDATE nodes SET deprecated = 1 WHERE id = ?').run(resolvedFrom);
|
|
765
|
+
});
|
|
766
|
+
runTx();
|
|
767
|
+
if (fromNode.file_path)
|
|
768
|
+
this.writeGraphToDisk(fromNode.file_path);
|
|
769
|
+
if (intoNode.file_path && intoNode.file_path !== fromNode.file_path)
|
|
770
|
+
this.writeGraphToDisk(intoNode.file_path);
|
|
771
|
+
// Edges that pointed INTO fromId now point at intoId in the DB, but the SOURCE files'
|
|
772
|
+
// on-disk JSON still says fromId until rewritten — same reasoning as renameNode's own use
|
|
773
|
+
// of this helper. Queried AFTER the transaction, so it reflects the already-repointed state.
|
|
774
|
+
this.rewriteInboundSourceFiles(resolvedInto);
|
|
775
|
+
const historyIds = this.db.prepare('SELECT id FROM history WHERE node_id = ?').all(resolvedInto);
|
|
776
|
+
for (const row of historyIds) {
|
|
777
|
+
this.patchHistoryDiskIdentity(row.id, resolvedInto, intoNode.name, intoNode.type, intoNode.file_path, intoNode.signature);
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
finally {
|
|
781
|
+
this.db.pragma('foreign_keys = ON');
|
|
782
|
+
}
|
|
783
|
+
}
|
|
437
784
|
/**
|
|
438
785
|
* Rewrites a history/[id].json file's identifying fields (node_id, node_metadata) in
|
|
439
786
|
* place, leaving code_snapshot/reasoning/timestamps untouched. Used after a rename so
|
|
@@ -468,10 +815,10 @@ class DevMindDatabase {
|
|
|
468
815
|
* callers that merely repoint them (rename) can rewrite after the fact.
|
|
469
816
|
*/
|
|
470
817
|
collectInboundSourceFiles(nodeId) {
|
|
471
|
-
const rows = this.db.prepare(`
|
|
472
|
-
SELECT DISTINCT n.file_path AS file_path
|
|
473
|
-
FROM node_connections c JOIN nodes n ON n.id = c.source_node_id
|
|
474
|
-
WHERE c.target_node_id = ?
|
|
818
|
+
const rows = this.db.prepare(`
|
|
819
|
+
SELECT DISTINCT n.file_path AS file_path
|
|
820
|
+
FROM node_connections c JOIN nodes n ON n.id = c.source_node_id
|
|
821
|
+
WHERE c.target_node_id = ?
|
|
475
822
|
`).all(nodeId);
|
|
476
823
|
const files = new Set();
|
|
477
824
|
for (const row of rows) {
|
|
@@ -531,9 +878,9 @@ class DevMindDatabase {
|
|
|
531
878
|
}
|
|
532
879
|
this.db.pragma('foreign_keys = OFF');
|
|
533
880
|
try {
|
|
534
|
-
const stmt = this.db.prepare(`
|
|
535
|
-
INSERT OR IGNORE INTO node_connections (source_node_id, target_node_id)
|
|
536
|
-
VALUES (?, ?)
|
|
881
|
+
const stmt = this.db.prepare(`
|
|
882
|
+
INSERT OR IGNORE INTO node_connections (source_node_id, target_node_id)
|
|
883
|
+
VALUES (?, ?)
|
|
537
884
|
`);
|
|
538
885
|
stmt.run(resolvedSrc, resolvedTgt);
|
|
539
886
|
if (srcNode.file_path) {
|
|
@@ -549,42 +896,119 @@ class DevMindDatabase {
|
|
|
549
896
|
const tgtNode = this.getNode(targetNodeId);
|
|
550
897
|
const resolvedSrc = srcNode ? srcNode.id : sourceNodeId;
|
|
551
898
|
const resolvedTgt = tgtNode ? tgtNode.id : targetNodeId;
|
|
552
|
-
const stmt = this.db.prepare(`
|
|
553
|
-
DELETE FROM node_connections
|
|
554
|
-
WHERE source_node_id = ? AND target_node_id = ?
|
|
899
|
+
const stmt = this.db.prepare(`
|
|
900
|
+
DELETE FROM node_connections
|
|
901
|
+
WHERE source_node_id = ? AND target_node_id = ?
|
|
555
902
|
`);
|
|
556
903
|
stmt.run(resolvedSrc, resolvedTgt);
|
|
557
904
|
if (srcNode && srcNode.file_path) {
|
|
558
905
|
this.writeGraphToDisk(srcNode.file_path);
|
|
559
906
|
}
|
|
560
907
|
}
|
|
561
|
-
|
|
908
|
+
/**
|
|
909
|
+
* `opts.limit`/`opts.offset` page a hub node's caller/callee list deterministically —
|
|
910
|
+
* `ORDER BY file_path, name` so a repeated call with the same offset returns the same slice,
|
|
911
|
+
* and so the local/nearby callers a reader actually wants tend to sort ahead of a scattered
|
|
912
|
+
* cross-repo tail (same file_path groups together). Omitting `opts` returns every row, exactly
|
|
913
|
+
* as before — every pre-existing call site keeps working unchanged.
|
|
914
|
+
*/
|
|
915
|
+
getConnections(nodeId, opts = {}) {
|
|
562
916
|
const node = this.getNode(nodeId);
|
|
563
917
|
const resolvedId = node ? node.id : nodeId;
|
|
564
|
-
const
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
918
|
+
const paging = opts.limit !== undefined ? ' ORDER BY n.file_path, n.name LIMIT ? OFFSET ?' : '';
|
|
919
|
+
const usesStmt = this.db.prepare(`
|
|
920
|
+
SELECT n.* FROM nodes n
|
|
921
|
+
JOIN node_connections c ON n.id = c.target_node_id
|
|
922
|
+
WHERE c.source_node_id = ?${paging}
|
|
568
923
|
`);
|
|
569
|
-
const usedByStmt = this.db.prepare(`
|
|
570
|
-
SELECT n.* FROM nodes n
|
|
571
|
-
JOIN node_connections c ON n.id = c.source_node_id
|
|
572
|
-
WHERE c.target_node_id =
|
|
924
|
+
const usedByStmt = this.db.prepare(`
|
|
925
|
+
SELECT n.* FROM nodes n
|
|
926
|
+
JOIN node_connections c ON n.id = c.source_node_id
|
|
927
|
+
WHERE c.target_node_id = ?${paging}
|
|
573
928
|
`);
|
|
929
|
+
const args = opts.limit !== undefined ? [resolvedId, opts.limit, opts.offset ?? 0] : [resolvedId];
|
|
574
930
|
return {
|
|
575
|
-
uses: usesStmt.all(
|
|
576
|
-
usedBy: usedByStmt.all(
|
|
931
|
+
uses: DevMindDatabase.parseNodeRows(usesStmt.all(...args)),
|
|
932
|
+
usedBy: DevMindDatabase.parseNodeRows(usedByStmt.all(...args))
|
|
577
933
|
};
|
|
578
934
|
}
|
|
935
|
+
/**
|
|
936
|
+
* Batched, COUNT-only connection degree for many nodes at once — the search-result drill-in
|
|
937
|
+
* hooks need this for ~20 nodes per call, and `getConnections` per-node would mean 20 pairs of
|
|
938
|
+
* full-row-fetching queries. One grouped COUNT each way instead (same shape as the degree
|
|
939
|
+
* subquery in {@link getGodEntities}). IDs not present in `node_connections` still get a
|
|
940
|
+
* `{uses:0, usedBy:0}` entry so callers never need an existence check.
|
|
941
|
+
*/
|
|
942
|
+
getConnectionCounts(ids) {
|
|
943
|
+
const result = new Map();
|
|
944
|
+
if (ids.length === 0)
|
|
945
|
+
return result;
|
|
946
|
+
for (const id of ids)
|
|
947
|
+
result.set(id, { uses: 0, usedBy: 0 });
|
|
948
|
+
const placeholders = ids.map(() => '?').join(',');
|
|
949
|
+
const usesStmt = this.db.prepare(`
|
|
950
|
+
SELECT source_node_id AS id, COUNT(*) AS n FROM node_connections
|
|
951
|
+
WHERE source_node_id IN (${placeholders}) GROUP BY source_node_id
|
|
952
|
+
`);
|
|
953
|
+
const usedByStmt = this.db.prepare(`
|
|
954
|
+
SELECT target_node_id AS id, COUNT(*) AS n FROM node_connections
|
|
955
|
+
WHERE target_node_id IN (${placeholders}) GROUP BY target_node_id
|
|
956
|
+
`);
|
|
957
|
+
for (const row of usesStmt.all(...ids)) {
|
|
958
|
+
const e = result.get(row.id);
|
|
959
|
+
if (e)
|
|
960
|
+
e.uses = row.n;
|
|
961
|
+
}
|
|
962
|
+
for (const row of usedByStmt.all(...ids)) {
|
|
963
|
+
const e = result.get(row.id);
|
|
964
|
+
if (e)
|
|
965
|
+
e.usedBy = row.n;
|
|
966
|
+
}
|
|
967
|
+
return result;
|
|
968
|
+
}
|
|
969
|
+
/** Batched history-entry count for many nodes at once — metadata only, no disk reads. */
|
|
970
|
+
getHistoryCounts(ids) {
|
|
971
|
+
const result = new Map();
|
|
972
|
+
if (ids.length === 0)
|
|
973
|
+
return result;
|
|
974
|
+
const placeholders = ids.map(() => '?').join(',');
|
|
975
|
+
const stmt = this.db.prepare(`
|
|
976
|
+
SELECT node_id AS id, COUNT(*) AS n FROM history
|
|
977
|
+
WHERE node_id IN (${placeholders}) GROUP BY node_id
|
|
978
|
+
`);
|
|
979
|
+
for (const row of stmt.all(...ids)) {
|
|
980
|
+
result.set(row.id, row.n);
|
|
981
|
+
}
|
|
982
|
+
return result;
|
|
983
|
+
}
|
|
984
|
+
/**
|
|
985
|
+
* Batched most-recent history timestamp for many nodes at once. Deliberately SQL-only (`MAX`
|
|
986
|
+
* over the indexed `updated_at` column) — unlike {@link getLatestHistory}, this never touches
|
|
987
|
+
* `populateHistoryFromDisk`, so it costs nothing beyond the query itself.
|
|
988
|
+
*/
|
|
989
|
+
getLastUpdatedMap(ids) {
|
|
990
|
+
const result = new Map();
|
|
991
|
+
if (ids.length === 0)
|
|
992
|
+
return result;
|
|
993
|
+
const placeholders = ids.map(() => '?').join(',');
|
|
994
|
+
const stmt = this.db.prepare(`
|
|
995
|
+
SELECT node_id AS id, MAX(updated_at) AS last FROM history
|
|
996
|
+
WHERE node_id IN (${placeholders}) GROUP BY node_id
|
|
997
|
+
`);
|
|
998
|
+
for (const row of stmt.all(...ids)) {
|
|
999
|
+
result.set(row.id, row.last);
|
|
1000
|
+
}
|
|
1001
|
+
return result;
|
|
1002
|
+
}
|
|
579
1003
|
// --- History Operations ---
|
|
580
1004
|
getLatestHistory(nodeId) {
|
|
581
1005
|
const node = this.getNode(nodeId);
|
|
582
1006
|
const resolvedId = node ? node.id : nodeId;
|
|
583
|
-
const stmt = this.db.prepare(`
|
|
584
|
-
SELECT id, node_id, session_id, created_at, updated_at FROM history
|
|
585
|
-
WHERE node_id = ?
|
|
586
|
-
ORDER BY updated_at DESC
|
|
587
|
-
LIMIT 1
|
|
1007
|
+
const stmt = this.db.prepare(`
|
|
1008
|
+
SELECT id, node_id, session_id, created_at, updated_at FROM history
|
|
1009
|
+
WHERE node_id = ?
|
|
1010
|
+
ORDER BY updated_at DESC
|
|
1011
|
+
LIMIT 1
|
|
588
1012
|
`);
|
|
589
1013
|
const row = stmt.get(resolvedId);
|
|
590
1014
|
if (!row)
|
|
@@ -594,11 +1018,11 @@ class DevMindDatabase {
|
|
|
594
1018
|
listHistory(nodeId) {
|
|
595
1019
|
const node = this.getNode(nodeId);
|
|
596
1020
|
const resolvedId = node ? node.id : nodeId;
|
|
597
|
-
const stmt = this.db.prepare(`
|
|
598
|
-
SELECT id, node_id, session_id, created_at, updated_at
|
|
599
|
-
FROM history
|
|
600
|
-
WHERE node_id = ?
|
|
601
|
-
ORDER BY updated_at DESC
|
|
1021
|
+
const stmt = this.db.prepare(`
|
|
1022
|
+
SELECT id, node_id, session_id, created_at, updated_at
|
|
1023
|
+
FROM history
|
|
1024
|
+
WHERE node_id = ?
|
|
1025
|
+
ORDER BY updated_at DESC
|
|
602
1026
|
`);
|
|
603
1027
|
return stmt.all(resolvedId);
|
|
604
1028
|
}
|
|
@@ -612,15 +1036,62 @@ class DevMindDatabase {
|
|
|
612
1036
|
getFullHistory(nodeId) {
|
|
613
1037
|
const node = this.getNode(nodeId);
|
|
614
1038
|
const resolvedId = node ? node.id : nodeId;
|
|
615
|
-
const stmt = this.db.prepare(`
|
|
616
|
-
SELECT id, node_id, session_id, created_at, updated_at
|
|
617
|
-
FROM history
|
|
618
|
-
WHERE node_id = ?
|
|
619
|
-
ORDER BY updated_at DESC
|
|
1039
|
+
const stmt = this.db.prepare(`
|
|
1040
|
+
SELECT id, node_id, session_id, created_at, updated_at
|
|
1041
|
+
FROM history
|
|
1042
|
+
WHERE node_id = ?
|
|
1043
|
+
ORDER BY updated_at DESC
|
|
620
1044
|
`);
|
|
621
1045
|
const rows = stmt.all(resolvedId);
|
|
622
1046
|
return rows.map(row => this.populateHistoryFromDisk(row));
|
|
623
1047
|
}
|
|
1048
|
+
/**
|
|
1049
|
+
* The last `limit` history entries' reasoning + timestamps only — no `code_snapshot`/`edits`.
|
|
1050
|
+
* Built for `get_node_code`'s default `history:"recent"` mode, which already returns the
|
|
1051
|
+
* CURRENT code: repeating past snapshots inline would just duplicate what's already in the
|
|
1052
|
+
* response. The full trail (snapshots + diffable edits) is `history:"full"`, served by
|
|
1053
|
+
* {@link getHistoryPage}. This answers "why does this look the way it does" cheaply enough to
|
|
1054
|
+
* attach to every get_node_code call by default, instead of leaving that as a round trip an AI
|
|
1055
|
+
* has to remember to make (or skip, and re-break a decision it never saw).
|
|
1056
|
+
*/
|
|
1057
|
+
getRecentHistorySummaries(nodeId, limit) {
|
|
1058
|
+
const node = this.getNode(nodeId);
|
|
1059
|
+
const resolvedId = node ? node.id : nodeId;
|
|
1060
|
+
const stmt = this.db.prepare(`
|
|
1061
|
+
SELECT id, node_id, session_id, created_at, updated_at
|
|
1062
|
+
FROM history
|
|
1063
|
+
WHERE node_id = ?
|
|
1064
|
+
ORDER BY updated_at DESC
|
|
1065
|
+
LIMIT ?
|
|
1066
|
+
`);
|
|
1067
|
+
const rows = stmt.all(resolvedId, limit);
|
|
1068
|
+
return rows.map(row => {
|
|
1069
|
+
const full = this.populateHistoryFromDisk(row);
|
|
1070
|
+
return { id: full.id, session_id: full.session_id, created_at: full.created_at, updated_at: full.updated_at, reasoning: full.reasoning };
|
|
1071
|
+
});
|
|
1072
|
+
}
|
|
1073
|
+
/**
|
|
1074
|
+
* `history:"full"`'s backing query — the same full-fidelity payload as {@link getFullHistory}
|
|
1075
|
+
* (code_snapshot + diffable edits per entry), but LIMIT/OFFSET applied IN SQL before any disk
|
|
1076
|
+
* read happens, not by slicing an already-fully-loaded array. `getFullHistory` reads every
|
|
1077
|
+
* revision's JSON off disk unconditionally; for a node with dozens of revisions that is dozens
|
|
1078
|
+
* of synchronous file reads to serve a request for the newest 5. `total` is the true count
|
|
1079
|
+
* before paging, same honesty contract as `nodes_total`/`files_total` elsewhere.
|
|
1080
|
+
*/
|
|
1081
|
+
getHistoryPage(nodeId, limit, offset) {
|
|
1082
|
+
const node = this.getNode(nodeId);
|
|
1083
|
+
const resolvedId = node ? node.id : nodeId;
|
|
1084
|
+
const totalRow = this.db.prepare('SELECT COUNT(*) AS c FROM history WHERE node_id = ?').get(resolvedId);
|
|
1085
|
+
const stmt = this.db.prepare(`
|
|
1086
|
+
SELECT id, node_id, session_id, created_at, updated_at
|
|
1087
|
+
FROM history
|
|
1088
|
+
WHERE node_id = ?
|
|
1089
|
+
ORDER BY updated_at DESC
|
|
1090
|
+
LIMIT ? OFFSET ?
|
|
1091
|
+
`);
|
|
1092
|
+
const rows = stmt.all(resolvedId, limit, offset);
|
|
1093
|
+
return { entries: rows.map(row => this.populateHistoryFromDisk(row)), total: totalRow.c };
|
|
1094
|
+
}
|
|
624
1095
|
/** Distinct source node ids of edges pointing INTO this node (its "used-by" callers). */
|
|
625
1096
|
getInboundSources(nodeId) {
|
|
626
1097
|
const rows = this.db
|
|
@@ -671,15 +1142,22 @@ class DevMindDatabase {
|
|
|
671
1142
|
if (node) {
|
|
672
1143
|
const live = this.extractLiveCode(node);
|
|
673
1144
|
if (live !== null) {
|
|
1145
|
+
const outdated = snapshot ? snapshot.code_snapshot !== live : undefined;
|
|
674
1146
|
return {
|
|
675
1147
|
exists: true,
|
|
676
1148
|
node_id: node.id,
|
|
677
1149
|
file_path: node.file_path,
|
|
678
1150
|
code: live,
|
|
679
1151
|
source: 'live',
|
|
680
|
-
// Snapshot exists but disagrees with disk → the
|
|
681
|
-
|
|
682
|
-
|
|
1152
|
+
// Snapshot exists but disagrees with disk → the RECORDED HISTORY has drifted, not the
|
|
1153
|
+
// `code` above — that was just read fresh from disk this call. Worth a message, not
|
|
1154
|
+
// just a bare flag: without it, this reads as "don't trust what you were just handed,"
|
|
1155
|
+
// which is backwards — it's the opposite thing (the graph's history) that's behind.
|
|
1156
|
+
snapshot_outdated: outdated,
|
|
1157
|
+
updated_at: snapshot?.updated_at,
|
|
1158
|
+
message: outdated
|
|
1159
|
+
? 'The code above is current — read live from disk this call. Only the recorded history snapshot is stale (it predates this edit); no need to re-read the file to double-check.'
|
|
1160
|
+
: undefined
|
|
683
1161
|
};
|
|
684
1162
|
}
|
|
685
1163
|
}
|
|
@@ -705,7 +1183,7 @@ class DevMindDatabase {
|
|
|
705
1183
|
getGraph(nodeId, maxDepth = 6, opts = {}) {
|
|
706
1184
|
const direction = opts.direction ?? 'both';
|
|
707
1185
|
const codeCharBudget = opts.codeCharBudget ?? 60_000;
|
|
708
|
-
const maxNodesLimit = 500;
|
|
1186
|
+
const maxNodesLimit = opts.maxNodes ?? 500;
|
|
709
1187
|
const visited = new Set();
|
|
710
1188
|
const nodes = [];
|
|
711
1189
|
const connections = [];
|
|
@@ -719,12 +1197,19 @@ class DevMindDatabase {
|
|
|
719
1197
|
// argument would find zero edges and return a lone root.
|
|
720
1198
|
const queue = [{ id: rootNode.id, depth: 0 }];
|
|
721
1199
|
visited.add(rootNode.id);
|
|
1200
|
+
rootNode.depth = 0;
|
|
722
1201
|
nodes.push(rootNode);
|
|
723
|
-
|
|
724
|
-
|
|
1202
|
+
// ORDER BY on both directions so the walk is REPRODUCIBLE: without it, sibling order within a
|
|
1203
|
+
// depth is whatever physical row order SQLite happens to return, which shifts after inserts,
|
|
1204
|
+
// deletes, or a VACUUM. That made two identical getGraph calls able to disagree about which
|
|
1205
|
+
// nodes got code when the budget ran out — the same query returning a different answer for no
|
|
1206
|
+
// visible reason. Both are index-ordered already (the PK autoindex outbound,
|
|
1207
|
+
// idx_node_connections_target inbound), so the planner elides the sort and this costs nothing.
|
|
1208
|
+
const usesStmt = this.db.prepare(`
|
|
1209
|
+
SELECT target_node_id FROM node_connections WHERE source_node_id = ? ORDER BY target_node_id
|
|
725
1210
|
`);
|
|
726
|
-
const usedByStmt = this.db.prepare(`
|
|
727
|
-
SELECT source_node_id FROM node_connections WHERE target_node_id = ?
|
|
1211
|
+
const usedByStmt = this.db.prepare(`
|
|
1212
|
+
SELECT source_node_id FROM node_connections WHERE target_node_id = ? ORDER BY source_node_id
|
|
728
1213
|
`);
|
|
729
1214
|
while (queue.length > 0 && nodes.length < maxNodesLimit) {
|
|
730
1215
|
const current = queue.shift();
|
|
@@ -745,10 +1230,16 @@ class DevMindDatabase {
|
|
|
745
1230
|
visited.add(targetId);
|
|
746
1231
|
const targetNode = this.getNode(targetId);
|
|
747
1232
|
if (targetNode) {
|
|
1233
|
+
targetNode.depth = current.depth + 1;
|
|
748
1234
|
nodes.push(targetNode);
|
|
749
1235
|
if (nodes.length >= maxNodesLimit)
|
|
750
1236
|
break;
|
|
751
1237
|
}
|
|
1238
|
+
// targetNode === null means `target_node_id` no longer resolves to a real node (the
|
|
1239
|
+
// node was deleted/renamed but the connections row survived). It still goes on the
|
|
1240
|
+
// queue/visited so a stale id is never re-processed, but — deliberately — no entry
|
|
1241
|
+
// is added to `nodes` for it. The edge pushed above therefore references an id that
|
|
1242
|
+
// will never appear in `nodes`; the dedup pass after the loop (see below) strips it.
|
|
752
1243
|
queue.push({ id: targetId, depth: current.depth + 1 });
|
|
753
1244
|
}
|
|
754
1245
|
}
|
|
@@ -769,6 +1260,7 @@ class DevMindDatabase {
|
|
|
769
1260
|
visited.add(sourceId);
|
|
770
1261
|
const sourceNode = this.getNode(sourceId);
|
|
771
1262
|
if (sourceNode) {
|
|
1263
|
+
sourceNode.depth = current.depth + 1;
|
|
772
1264
|
nodes.push(sourceNode);
|
|
773
1265
|
if (nodes.length >= maxNodesLimit)
|
|
774
1266
|
break;
|
|
@@ -778,24 +1270,50 @@ class DevMindDatabase {
|
|
|
778
1270
|
}
|
|
779
1271
|
}
|
|
780
1272
|
}
|
|
781
|
-
|
|
1273
|
+
// Two kinds of "more exists than was returned", reported honestly instead of silently:
|
|
1274
|
+
// 1. The node cap cut the walk short with the queue still non-empty.
|
|
1275
|
+
// 2. A connections row survives referencing a node this walk never added (deleted/renamed
|
|
1276
|
+
// node, or — same shape — a node dropped for being past the cap on a LATER queue entry
|
|
1277
|
+
// than the one whose edge pointed at it). Filtering here, once, after the walk, is
|
|
1278
|
+
// simpler and more certainly correct than trying to prevent every path that could create
|
|
1279
|
+
// one during the BFS itself.
|
|
1280
|
+
const nodesTruncated = nodes.length >= maxNodesLimit && queue.length > 0;
|
|
1281
|
+
const nodeIds = new Set(nodes.map(n => n.id));
|
|
1282
|
+
const cleanConnections = connections.filter(c => nodeIds.has(c.source_node_id) && nodeIds.has(c.target_node_id));
|
|
1283
|
+
const connectionsTruncated = cleanConnections.length < connections.length;
|
|
1284
|
+
const result = { nodes, connections: cleanConnections };
|
|
1285
|
+
if (nodesTruncated)
|
|
1286
|
+
result.nodes_truncated = true;
|
|
1287
|
+
if (connectionsTruncated)
|
|
1288
|
+
result.connections_truncated = true;
|
|
782
1289
|
if (opts.includeCode) {
|
|
783
1290
|
let spent = 0;
|
|
784
|
-
let
|
|
1291
|
+
let noCodeAvailable = 0;
|
|
1292
|
+
const omittedForBudget = [];
|
|
785
1293
|
// `nodes` is in BFS order (nearest the root first), so the budget is spent on the most
|
|
786
1294
|
// relevant code before anything is dropped.
|
|
787
1295
|
for (const [i, n] of nodes.entries()) {
|
|
788
1296
|
const live = this.extractLiveCode(n);
|
|
789
1297
|
const code = live ?? this.getLatestCode(n.id)?.code_snapshot ?? null;
|
|
790
1298
|
if (!code) {
|
|
791
|
-
|
|
1299
|
+
// Nothing to attach — a node whose symbol no longer resolves on disk and has no cached
|
|
1300
|
+
// snapshot. Counted SEPARATELY from a budget drop: these two used to share one
|
|
1301
|
+
// `nodes_without_code` counter, which meant a graph full of unresolvable nodes reported
|
|
1302
|
+
// `code_truncated: true` on a completely unspent budget. Raising the budget would then
|
|
1303
|
+
// do nothing, and the caller had no way to tell that from the response.
|
|
1304
|
+
noCodeAvailable++;
|
|
792
1305
|
continue;
|
|
793
1306
|
}
|
|
794
1307
|
// The root always gets its code — it is what was asked for, and dropping it would make
|
|
795
1308
|
// the response useless. Every other node must fit in the REMAINING budget, so a single
|
|
796
1309
|
// large node can't blow past the cap (it is skipped and counted, not truncated).
|
|
797
1310
|
if (i > 0 && spent + code.length > codeCharBudget) {
|
|
798
|
-
|
|
1311
|
+
// Recorded BY ID, not merely counted. An id is a valid argument to get_node_code, so a
|
|
1312
|
+
// caller who needs the rest can fetch exactly those nodes — where a bare count, or an
|
|
1313
|
+
// array index into a graph that gets re-derived on every call, tells them nothing they
|
|
1314
|
+
// can act on. Capped so a wide graph can't turn the omission list into its own payload.
|
|
1315
|
+
if (omittedForBudget.length < OMITTED_NODE_ID_CAP)
|
|
1316
|
+
omittedForBudget.push(n.id);
|
|
799
1317
|
continue;
|
|
800
1318
|
}
|
|
801
1319
|
n.code = code;
|
|
@@ -803,10 +1321,14 @@ class DevMindDatabase {
|
|
|
803
1321
|
spent += code.length;
|
|
804
1322
|
}
|
|
805
1323
|
result.code_chars = spent;
|
|
806
|
-
if (
|
|
1324
|
+
if (noCodeAvailable > 0 || omittedForBudget.length > 0) {
|
|
807
1325
|
result.code_truncated = true;
|
|
808
|
-
result.nodes_without_code =
|
|
1326
|
+
result.nodes_without_code = noCodeAvailable + omittedForBudget.length;
|
|
809
1327
|
}
|
|
1328
|
+
if (noCodeAvailable > 0)
|
|
1329
|
+
result.nodes_no_code_available = noCodeAvailable;
|
|
1330
|
+
if (omittedForBudget.length > 0)
|
|
1331
|
+
result.code_omitted_node_ids = omittedForBudget;
|
|
810
1332
|
}
|
|
811
1333
|
return result;
|
|
812
1334
|
}
|
|
@@ -826,7 +1348,9 @@ class DevMindDatabase {
|
|
|
826
1348
|
const resolvedId = node ? node.id : node_id;
|
|
827
1349
|
const formattedReasoning = formatReasoning(reasoning);
|
|
828
1350
|
const nowStr = new Date().toISOString();
|
|
829
|
-
const
|
|
1351
|
+
const newEdit = params.code_before === undefined
|
|
1352
|
+
? null
|
|
1353
|
+
: { at: nowStr, before: params.code_before ?? '', after: code_snapshot, reasoning: formattedReasoning };
|
|
830
1354
|
// 1-hour session boundary rule check
|
|
831
1355
|
const latest = this.getLatestHistory(resolvedId);
|
|
832
1356
|
if (latest) {
|
|
@@ -838,25 +1362,34 @@ class DevMindDatabase {
|
|
|
838
1362
|
// session). code_snapshot is always the latest state (git already owns version history for
|
|
839
1363
|
// code). reasoning is APPENDED, not overwritten — an earlier commit's "why" in this same
|
|
840
1364
|
// session is still real and still worth keeping; losing it silently is worse than a few
|
|
841
|
-
// extra lines in one file.
|
|
842
|
-
// this row valid: it never loses what it originally linked to, only gains more below it.
|
|
1365
|
+
// extra lines in one file.
|
|
843
1366
|
if (diffMs < 3600000) {
|
|
1367
|
+
/* istanbul ignore next -- `latest` came from `getLatestHistory()` -> `populateHistoryFromDisk()`,
|
|
1368
|
+
which always returns a STRING `reasoning` (either the disk JSON's string field, or the
|
|
1369
|
+
result of `formatReasoning(...)`, itself always a string) — so the `: ''` fallback here
|
|
1370
|
+
can never actually run. Kept as a real guard against a future change to
|
|
1371
|
+
`populateHistoryFromDisk`'s return shape, not because today's flow can reach it. */
|
|
844
1372
|
const previousReasoning = typeof latest.reasoning === 'string' ? latest.reasoning : '';
|
|
845
1373
|
const mergedReasoning = previousReasoning.trim().length > 0
|
|
846
1374
|
? `${previousReasoning}\n\n── Update @ ${nowStr} ──\n${formattedReasoning}`
|
|
847
1375
|
: formattedReasoning;
|
|
848
|
-
const updateStmt = this.db.prepare(`
|
|
849
|
-
UPDATE history
|
|
850
|
-
SET code_snapshot = '', reasoning = ?, updated_at = ?
|
|
851
|
-
WHERE id = ?
|
|
1376
|
+
const updateStmt = this.db.prepare(`
|
|
1377
|
+
UPDATE history
|
|
1378
|
+
SET code_snapshot = '', reasoning = ?, updated_at = ?
|
|
1379
|
+
WHERE id = ?
|
|
852
1380
|
`);
|
|
853
1381
|
updateStmt.run(mergedReasoning, nowStr, latest.id);
|
|
1382
|
+
// The edit trail APPENDS for the same reason reasoning does: this row now covers several
|
|
1383
|
+
// edits, and only a per-edit before/after lets a revert undo the last one rather than the
|
|
1384
|
+
// whole session's work — the window slides off updated_at, so one row can span hours.
|
|
1385
|
+
const mergedEdits = newEdit ? [...latest.edits, newEdit] : latest.edits;
|
|
854
1386
|
// Write/Update on disk
|
|
855
|
-
this.writeHistoryToDisk(latest.id, resolvedId, latest.session_id, latest.created_at, nowStr, code_snapshot, mergedReasoning);
|
|
1387
|
+
this.writeHistoryToDisk(latest.id, resolvedId, latest.session_id, latest.created_at, nowStr, code_snapshot, mergedReasoning, mergedEdits);
|
|
856
1388
|
return {
|
|
857
1389
|
...latest,
|
|
858
1390
|
code_snapshot,
|
|
859
1391
|
reasoning: mergedReasoning,
|
|
1392
|
+
edits: mergedEdits,
|
|
860
1393
|
updated_at: nowStr
|
|
861
1394
|
};
|
|
862
1395
|
}
|
|
@@ -864,13 +1397,14 @@ class DevMindDatabase {
|
|
|
864
1397
|
// Otherwise (or if no record exists), insert new history block
|
|
865
1398
|
const newId = crypto.randomUUID();
|
|
866
1399
|
const sessionId = params.session_id || crypto.randomUUID();
|
|
867
|
-
const insertStmt = this.db.prepare(`
|
|
868
|
-
INSERT INTO history (id, node_id, session_id, created_at, updated_at, code_snapshot, reasoning)
|
|
869
|
-
VALUES (?, ?, ?, ?, ?, '', ?)
|
|
1400
|
+
const insertStmt = this.db.prepare(`
|
|
1401
|
+
INSERT INTO history (id, node_id, session_id, created_at, updated_at, code_snapshot, reasoning)
|
|
1402
|
+
VALUES (?, ?, ?, ?, ?, '', ?)
|
|
870
1403
|
`);
|
|
871
1404
|
insertStmt.run(newId, resolvedId, sessionId, nowStr, nowStr, formattedReasoning);
|
|
1405
|
+
const newEdits = newEdit ? [newEdit] : [];
|
|
872
1406
|
// Write to disk
|
|
873
|
-
this.writeHistoryToDisk(newId, resolvedId, sessionId, nowStr, nowStr, code_snapshot, formattedReasoning);
|
|
1407
|
+
this.writeHistoryToDisk(newId, resolvedId, sessionId, nowStr, nowStr, code_snapshot, formattedReasoning, newEdits);
|
|
874
1408
|
return {
|
|
875
1409
|
id: newId,
|
|
876
1410
|
node_id: resolvedId,
|
|
@@ -878,195 +1412,688 @@ class DevMindDatabase {
|
|
|
878
1412
|
created_at: nowStr,
|
|
879
1413
|
updated_at: nowStr,
|
|
880
1414
|
code_snapshot,
|
|
881
|
-
reasoning: formattedReasoning
|
|
1415
|
+
reasoning: formattedReasoning,
|
|
1416
|
+
edits: newEdits
|
|
882
1417
|
};
|
|
883
1418
|
}
|
|
1419
|
+
/**
|
|
1420
|
+
* Removes the newest recorded edit from a history entry, leaving no trace of it.
|
|
1421
|
+
*
|
|
1422
|
+
* There used to be a citation guard here: a history row cited by a workflow step was emptied
|
|
1423
|
+
* rather than deleted, so the step was not left pointing at nothing. Workflow steps record
|
|
1424
|
+
* `node_ids` now, not history ids, so nothing cites a history row any more and the guard had
|
|
1425
|
+
* nothing left to check. Keeping it would have meant keeping the `history_ids` column alive
|
|
1426
|
+
* purely to protect a reference nothing makes.
|
|
1427
|
+
*
|
|
1428
|
+
* Callers are expected to have restored the file already; this only unwinds what was written
|
|
1429
|
+
* about it.
|
|
1430
|
+
*/
|
|
1431
|
+
eraseLastEdit(historyId) {
|
|
1432
|
+
const entry = this.getHistoryEntry(historyId);
|
|
1433
|
+
if (!entry)
|
|
1434
|
+
return { erased: false, entry_deleted: false, reason: 'history entry not found' };
|
|
1435
|
+
if (!entry.edits.length)
|
|
1436
|
+
return { erased: false, entry_deleted: false, reason: 'entry has no recorded edits' };
|
|
1437
|
+
const remaining = entry.edits.slice(0, -1);
|
|
1438
|
+
const dropped = entry.edits[entry.edits.length - 1];
|
|
1439
|
+
if (!remaining.length) {
|
|
1440
|
+
// Nothing left to keep the row for — no workflow step references a history id any more.
|
|
1441
|
+
this.db.prepare('DELETE FROM history WHERE id = ?').run(historyId);
|
|
1442
|
+
try {
|
|
1443
|
+
const f = path.join(path.dirname(this.dbPath), 'history', `${historyId}.json`);
|
|
1444
|
+
if (fs.existsSync(f))
|
|
1445
|
+
fs.unlinkSync(f);
|
|
1446
|
+
}
|
|
1447
|
+
catch { /* ignore */ }
|
|
1448
|
+
return { erased: true, entry_deleted: true };
|
|
1449
|
+
}
|
|
1450
|
+
const nowStr = new Date().toISOString();
|
|
1451
|
+
const newSnapshot = remaining[remaining.length - 1].after;
|
|
1452
|
+
const newReasoning = dropReasoningBlock(entry.reasoning, dropped.reasoning);
|
|
1453
|
+
this.db.prepare('UPDATE history SET reasoning = ?, updated_at = ? WHERE id = ?')
|
|
1454
|
+
.run(newReasoning, nowStr, historyId);
|
|
1455
|
+
this.writeHistoryToDisk(historyId, entry.node_id, entry.session_id, entry.created_at, nowStr, newSnapshot, newReasoning, remaining);
|
|
1456
|
+
return { erased: true, entry_deleted: false };
|
|
1457
|
+
}
|
|
884
1458
|
// --- Search Operations ---
|
|
885
1459
|
/**
|
|
886
|
-
*
|
|
887
|
-
*
|
|
888
|
-
*
|
|
889
|
-
*
|
|
890
|
-
*
|
|
1460
|
+
* The one search tool, covering both worlds in a single call:
|
|
1461
|
+
* - a primary `nodes` bucket — the indexed graph, found by exact identifier, then by three
|
|
1462
|
+
* fused rankers (BM25 over metadata, vector over descriptions, and code-body match), and
|
|
1463
|
+
* - a last-resort `files` bucket — a real filesystem grep of the configured repos, so files
|
|
1464
|
+
* the graph never models (CSS, JSON, config, markup, un-indexed code) are finally covered
|
|
1465
|
+
* in the same call instead of sending the caller off to an external grep.
|
|
1466
|
+
*
|
|
1467
|
+
* Inputs play to each layer's strength: the natural-language `query` drives the semantic vector
|
|
1468
|
+
* layer (and BM25); `opts.keywords` (literal, OR) drive grep and the code-body match, and also
|
|
1469
|
+
* feed BM25. If no keywords are given they're derived from the query's significant tokens, so a
|
|
1470
|
+
* natural-only call still gets code + file coverage.
|
|
1471
|
+
*
|
|
1472
|
+
* Speed is the point — the two slow layers (vector, grep) run concurrently, and the old ~9k
|
|
1473
|
+
* per-node snapshot read (the tool's former ~10-30s cost) is gone: code-body matching now rides
|
|
1474
|
+
* the single grep walk. See `grep.ts` and Phase 4 of the plan.
|
|
891
1475
|
*/
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
const
|
|
904
|
-
const
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
1476
|
+
/**
|
|
1477
|
+
* Attaches the drill-in hooks (`uses`/`used_by`/`history_count`/`last_updated`) to a batch of
|
|
1478
|
+
* search results in place, via ONE grouped query per hook instead of per-node fetches — see
|
|
1479
|
+
* {@link getConnectionCounts}/{@link getHistoryCounts}/{@link getLastUpdatedMap}. This is the
|
|
1480
|
+
* signal that turns a search result from a dead end into something worth drilling into with
|
|
1481
|
+
* `get_node_graph`/`get_node_history` — without it, nothing hints there's more to find.
|
|
1482
|
+
*/
|
|
1483
|
+
attachDrillInHooks(nodes) {
|
|
1484
|
+
if (nodes.length === 0)
|
|
1485
|
+
return nodes;
|
|
1486
|
+
const ids = nodes.map(n => n.id);
|
|
1487
|
+
const connCounts = this.getConnectionCounts(ids);
|
|
1488
|
+
const historyCounts = this.getHistoryCounts(ids);
|
|
1489
|
+
const lastUpdated = this.getLastUpdatedMap(ids);
|
|
1490
|
+
for (const n of nodes) {
|
|
1491
|
+
/* istanbul ignore next -- `connCounts` is built from `getConnectionCounts(ids)` on this
|
|
1492
|
+
SAME `ids` array a few lines up, and that helper pre-seeds a {uses:0,usedBy:0} entry for
|
|
1493
|
+
every id it's given before querying — so `connCounts.get(n.id)` can never miss here.
|
|
1494
|
+
Kept as a real guard against a future refactor decoupling the two, not because today's
|
|
1495
|
+
flow can reach the fallback. */
|
|
1496
|
+
const conn = connCounts.get(n.id) ?? { uses: 0, usedBy: 0 };
|
|
1497
|
+
n.uses = conn.uses;
|
|
1498
|
+
n.used_by = conn.usedBy;
|
|
1499
|
+
n.history_count = historyCounts.get(n.id) ?? 0;
|
|
1500
|
+
const lu = lastUpdated.get(n.id);
|
|
1501
|
+
if (lu)
|
|
1502
|
+
n.last_updated = lu;
|
|
1503
|
+
// Say so instead of asserting "unused", or this hook actively misleads instead of helping.
|
|
1504
|
+
if (conn.usedBy === 0) {
|
|
1505
|
+
n.used_by_note = exports.NO_STATIC_CALLERS_NOTE;
|
|
1506
|
+
}
|
|
915
1507
|
}
|
|
916
|
-
|
|
917
|
-
|
|
1508
|
+
return nodes;
|
|
1509
|
+
}
|
|
1510
|
+
async searchNodes(query, opts = {}) {
|
|
1511
|
+
const trimmedQuery = query?.trim();
|
|
1512
|
+
const hasQuery = !!trimmedQuery;
|
|
1513
|
+
const trimmedPattern = opts.pattern?.trim();
|
|
1514
|
+
const hasPattern = !!trimmedPattern;
|
|
1515
|
+
if (!hasQuery && !hasPattern) {
|
|
1516
|
+
throw new Error('searchNodes requires at least one of `query` or `pattern`.');
|
|
1517
|
+
}
|
|
1518
|
+
const caseInsensitive = opts.case_insensitive !== false;
|
|
1519
|
+
const scopePath = this.resolveSearchScopePath(opts.path);
|
|
1520
|
+
// Scoping `path` straight AT a lockfile or build artifact is honored, not overridden — the
|
|
1521
|
+
// same rule that has always applied to scoping at an ignored directory. What makes that
|
|
1522
|
+
// confusing is the SILENCE, not the exclusion: an empty result is indistinguishable from
|
|
1523
|
+
// "the pattern isn't in that file". Say so instead, so the agent stops rather than retrying
|
|
1524
|
+
// variations of a search that can never return anything.
|
|
1525
|
+
const scopeNote = scopePath && (0, grep_1.isDefaultIgnoredFile)(scopePath)
|
|
1526
|
+
? `path "${scopePath}" is a lockfile or generated artifact, excluded from search by default — no file content was scanned. Read the file directly if you genuinely need it.`
|
|
1527
|
+
: undefined;
|
|
1528
|
+
const filesOffset = opts.offset ?? 0;
|
|
1529
|
+
const filesLimit = opts.limit ?? 25;
|
|
1530
|
+
// `compact` is not just a projection flag — it SKIPS work. `annotateSampleLinesWithSymbol` is
|
|
1531
|
+
// the AST path (the same per-node span resolution that was 8.2s of a 9.6s query before its
|
|
1532
|
+
// file cap), and its only product is the `symbol` field ON the sample lines. A compact result
|
|
1533
|
+
// drops those lines entirely, so resolving them first would be pure waste. This is exactly
|
|
1534
|
+
// why compaction can't live wholly in the MCP handler: a projection applied after the fact
|
|
1535
|
+
// can trim the response, but it cannot un-spend the time that produced it.
|
|
1536
|
+
const compact = opts.compact === true;
|
|
1537
|
+
const grepOpts = { ignoredPaths: this.context?.config.ignored_paths, caseInsensitive, scopePath };
|
|
1538
|
+
// The pattern actually handed to the filesystem grep: the caller's own regex when given, else
|
|
1539
|
+
// a literal (escaped) OR of the query's own significant tokens — the same "derive from query"
|
|
1540
|
+
// fallback as before, just built from a real regex now instead of a keyword array. This is the
|
|
1541
|
+
// ONLY layer that ever runs — everything meaning-driven below is gated on `hasQuery`.
|
|
1542
|
+
const grepPattern = hasPattern
|
|
1543
|
+
? trimmedPattern
|
|
1544
|
+
: Array.from(new Set((0, tokenize_1.tokenizeText)(trimmedQuery))).map(grep_1.escapeRegExp).join('|');
|
|
1545
|
+
// BM25/vector/the identifier short-circuit are all driven by natural-language MEANING — a bare
|
|
1546
|
+
// regex has no meaning for them to tokenize or embed, so none of them run without a `query`.
|
|
1547
|
+
// Grep (and the code-match nodes it feeds via mapGrepHitsToNodes) is unconditional.
|
|
1548
|
+
const bm25Tokens = hasQuery ? Array.from(new Set((0, tokenize_1.tokenizeText)(trimmedQuery))) : [];
|
|
1549
|
+
if (hasQuery) {
|
|
1550
|
+
// Deliberately name/id ONLY — NOT description/reasoning. Those are free-text natural-language
|
|
1551
|
+
// fields (a description is a whole sentence); matching the query as a raw substring against
|
|
1552
|
+
// them turns "any short natural-language query that happens to appear inside some node's
|
|
1553
|
+
// description" into a false "exact identifier hit," which then skipped vector search AND the
|
|
1554
|
+
// grep-derived code layer entirely (see the comment below) for what was never actually an
|
|
1555
|
+
// identifier lookup. description/reasoning are already covered properly — with real ranking,
|
|
1556
|
+
// not a blind substring guess — by BM25 (`tokenSearchNodes`/`search-index.ts`) on the path
|
|
1557
|
+
// below, so nothing is lost by dropping them here; queries that used to wrongly short-circuit
|
|
1558
|
+
// now correctly fall through to BM25 + vector + code instead.
|
|
1559
|
+
//
|
|
1560
|
+
// Gated on `hasQuery` for a second, sharper reason too: this LIKE builds `%<query>%`, and an
|
|
1561
|
+
// ABSENT query would make that `%%` — matching every node in the database. A regex-only
|
|
1562
|
+
// search on a small repo (≤10 nodes) would then wrongly "short-circuit" and hand back the
|
|
1563
|
+
// ENTIRE graph as high-confidence identifier hits. Never reachable when `query` is empty.
|
|
1564
|
+
const stmt = this.db.prepare(`
|
|
1565
|
+
SELECT DISTINCT n.* FROM nodes n
|
|
1566
|
+
WHERE n.name LIKE ? ESCAPE '\\' OR n.id LIKE ? ESCAPE '\\'
|
|
1567
|
+
LIMIT 50
|
|
1568
|
+
`);
|
|
1569
|
+
const wildcard = `%${this.likeEscape(trimmedQuery)}%`;
|
|
1570
|
+
const identifierMatches = DevMindDatabase.parseNodeRows(stmt.all(wildcard, wildcard));
|
|
1571
|
+
// Exact-identifier short-circuit: a small, unambiguous set of literal name/id hits is trusted
|
|
1572
|
+
// outright and skips the rankers (semantic blur only hurts an exact symbol lookup). Grep
|
|
1573
|
+
// always runs here too (using `grepPattern`, the same derive-from-query-if-absent value the
|
|
1574
|
+
// main path below uses), so an agent that only ever sends `query` still gets a populated
|
|
1575
|
+
// `files` bucket instead of an empty one for no reason.
|
|
1576
|
+
if (identifierMatches.length > 0 && identifierMatches.length <= 10) {
|
|
1577
|
+
// An exact identifier hit is the most trustworthy result there is → high confidence, top
|
|
1578
|
+
// relevance, found by name.
|
|
1579
|
+
// Trust signals FIRST, before `description`. JSON.stringify preserves insertion order and
|
|
1580
|
+
// an agent reads a result top-down; with the plain `{...n, confidence}` spread these
|
|
1581
|
+
// landed after a full sentence of description, and real session feedback was that they got
|
|
1582
|
+
// skipped in favour of eyeballing node names. Ordering is the whole fix — the fields were
|
|
1583
|
+
// always there. Applied at BOTH construction sites (see the fused path below); doing only
|
|
1584
|
+
// one would leave the two paths disagreeing, and this identifier path is the more common
|
|
1585
|
+
// one for a symbol lookup.
|
|
1586
|
+
const nodes = identifierMatches.map(({ id, name, type, ...rest }) => ({
|
|
1587
|
+
id, name, type,
|
|
1588
|
+
confidence: 'high', relevance: 100, found_by: ['name'], matched_via: 'identifier',
|
|
1589
|
+
...rest,
|
|
1590
|
+
uses: 0, used_by: 0, history_count: 0
|
|
1591
|
+
}));
|
|
1592
|
+
this.attachDrillInHooks(nodes);
|
|
1593
|
+
const grep = await (0, grep_1.grepRepos)(this.repoRoots(), grepPattern, grepOpts);
|
|
1594
|
+
const filesResult = (0, grep_1.rankGrepHits)(grep.hits, { offset: filesOffset, maxFiles: filesLimit });
|
|
1595
|
+
if (!compact)
|
|
1596
|
+
this.annotateSampleLinesWithSymbol(filesResult.files);
|
|
1597
|
+
return {
|
|
1598
|
+
nodes,
|
|
1599
|
+
files: filesResult.files,
|
|
1600
|
+
files_total: filesResult.total,
|
|
1601
|
+
files_offset: filesOffset,
|
|
1602
|
+
nodes_total: nodes.length,
|
|
1603
|
+
truncated: grep.truncated || undefined,
|
|
1604
|
+
scope_note: scopeNote
|
|
1605
|
+
};
|
|
1606
|
+
}
|
|
1607
|
+
}
|
|
1608
|
+
// The two slow layers run CONCURRENTLY — vector inference and the filesystem grep walk. BM25
|
|
1609
|
+
// is synchronous indexed SQL, so it runs inline for free. Wall-clock ≈ max(vector, grep).
|
|
1610
|
+
//
|
|
1611
|
+
// Timed at every stage, opt-in via DEVSMIND_PERF_DEBUG — kept permanently (not stripped after
|
|
1612
|
+
// this pass) because it's how the mapGrepHitsToNodes bottleneck below was actually found: on a
|
|
1613
|
+
// real 8-repo query it was 8.2s of a 9.6s total, invisible from the outside since the tool
|
|
1614
|
+
// just looked uniformly slow. Zero cost when unset (`process.hrtime.bigint()` calls only;
|
|
1615
|
+
// the env check gates the one string-building/console.error).
|
|
1616
|
+
const perfDebug = !!process.env.DEVSMIND_PERF_DEBUG;
|
|
1617
|
+
const ms = (a, b) => (Number(b - a) / 1e6).toFixed(0);
|
|
1618
|
+
const perfStart = process.hrtime.bigint();
|
|
1619
|
+
const bm25Ranked = this.tokenSearchNodes(bm25Tokens);
|
|
1620
|
+
const perfAfterBm25 = process.hrtime.bigint();
|
|
1621
|
+
// Timed individually (not just the combined Promise.all) so a slow run can be attributed to
|
|
1622
|
+
// ONE of the two instead of leaving both under suspicion.
|
|
1623
|
+
let vectorMs = '?', grepMs = '?';
|
|
1624
|
+
const [vectorIds, grep] = await Promise.all([
|
|
1625
|
+
(hasQuery ? this.vectorSearchNodes(trimmedQuery) : Promise.resolve([])).then(r => { if (perfDebug)
|
|
1626
|
+
vectorMs = hasQuery ? ms(perfAfterBm25, process.hrtime.bigint()) : 'skipped(no query)'; return r; }),
|
|
1627
|
+
(0, grep_1.grepRepos)(this.repoRoots(), grepPattern, grepOpts).then(r => { if (perfDebug)
|
|
1628
|
+
grepMs = ms(perfAfterBm25, process.hrtime.bigint()); return r; })
|
|
1629
|
+
]);
|
|
1630
|
+
const perfAfterVectorGrep = process.hrtime.bigint();
|
|
1631
|
+
// The single grep walk feeds BOTH buckets: raw hits → the files bucket, and hits landing in
|
|
1632
|
+
// indexed source files → code-match nodes (this is the "code search that returns nodes").
|
|
1633
|
+
const filesResult = (0, grep_1.rankGrepHits)(grep.hits, { offset: filesOffset, maxFiles: filesLimit });
|
|
1634
|
+
const perfAfterRank = process.hrtime.bigint();
|
|
1635
|
+
const codeMatches = this.mapGrepHitsToNodes(grep.hits);
|
|
1636
|
+
const perfAfterCodeMatch = process.hrtime.bigint();
|
|
1637
|
+
const codeLinesById = new Map(codeMatches.map(c => [c.nodeId, c.lines]));
|
|
1638
|
+
// Bounded annotation (only the page actually returned — see the doc comment on the helper for
|
|
1639
|
+
// why this must NOT run over every raw hit). Timed and logged BEFORE the [perf] line below,
|
|
1640
|
+
// not after — it shares `locateNodeInFile`, the exact primitive mapGrepHitsToNodes' own doc
|
|
1641
|
+
// comment identifies as the historical bottleneck, so leaving it unmeasured would silently
|
|
1642
|
+
// exempt a real cost from the one line this file's instrumentation exists to catch it with.
|
|
1643
|
+
if (!compact)
|
|
1644
|
+
this.annotateSampleLinesWithSymbol(filesResult.files);
|
|
1645
|
+
const perfAfterAnnotate = process.hrtime.bigint();
|
|
1646
|
+
if (perfDebug) {
|
|
1647
|
+
console.error(`[perf] bm25=${ms(perfStart, perfAfterBm25)}ms vector=${vectorMs}ms grep=${grepMs}ms (combined=${ms(perfAfterBm25, perfAfterVectorGrep)}ms) rankGrepHits=${ms(perfAfterVectorGrep, perfAfterRank)}ms mapGrepHitsToNodes=${ms(perfAfterRank, perfAfterCodeMatch)}ms annotateSampleLinesWithSymbol=${ms(perfAfterCodeMatch, perfAfterAnnotate)}ms grepHits=${grep.hits.length} truncated=${grep.truncated}`);
|
|
1648
|
+
}
|
|
1649
|
+
if (bm25Ranked.length === 0 && vectorIds.length === 0 && codeMatches.length === 0) {
|
|
1650
|
+
// Neither the graph (metadata, meaning, code body) nor grep found anything meaningful.
|
|
1651
|
+
const base = {
|
|
1652
|
+
nodes: [],
|
|
1653
|
+
files: filesResult.files,
|
|
1654
|
+
files_total: filesResult.total,
|
|
1655
|
+
files_offset: filesOffset,
|
|
1656
|
+
nodes_total: 0,
|
|
1657
|
+
truncated: grep.truncated || undefined,
|
|
1658
|
+
scope_note: scopeNote
|
|
1659
|
+
};
|
|
1660
|
+
if (filesResult.files.length === 0) {
|
|
1661
|
+
base.hint = 'No meaningful match anywhere — not in any node\'s name, description, reasoning, or code body, and no file on disk contains this pattern. If you expected this to exist, retry with a broader pattern or different query terms; if it genuinely isn\'t in this codebase, that\'s a real answer — don\'t keep re-querying variations.';
|
|
1662
|
+
}
|
|
1663
|
+
return base;
|
|
1664
|
+
}
|
|
1665
|
+
// Fuse the THREE node rankings by rank (RRF) — BM25 (metadata), vector (meaning), code-match
|
|
1666
|
+
// (body). Each is strong at something the others miss; rank-fusion needs no score calibration.
|
|
1667
|
+
// But the RAW RRF float is a terrible thing to hand back (a #1-ranked hit tops out near 0.03,
|
|
1668
|
+
// which reads like "3% confident"), so it drives ORDER only — every node is then re-described
|
|
1669
|
+
// with human-meaningful signals: which layers found it, a high/medium/low confidence, and a
|
|
1670
|
+
// 0-100 relevance relative to the top hit.
|
|
1671
|
+
const bm25ById = new Map(bm25Ranked.map(n => [n.id, n]));
|
|
1672
|
+
const simById = new Map(vectorIds.map(v => [v.id, v.sim]));
|
|
1673
|
+
const fused = (0, search_index_1.reciprocalRankFusion)([bm25Ranked.map(n => n.id), vectorIds.map(v => v.id), codeMatches.map(c => c.nodeId)]);
|
|
1674
|
+
/* istanbul ignore next -- unreachable here: reaching this line already required at least one
|
|
1675
|
+
of bm25Ranked/vectorIds/codeMatches to be non-empty (the `if (... .length === 0 && ...)`
|
|
1676
|
+
guard above returns early otherwise), and reciprocalRankFusion's output is the union of its
|
|
1677
|
+
input rankings — so `fused` is always non-empty by the time this runs. Kept as a real guard
|
|
1678
|
+
against a future change decoupling that invariant, not because today's flow can reach it. */
|
|
1679
|
+
const topScore = fused.length ? fused[0].score : 1;
|
|
1680
|
+
const nodes = [];
|
|
1681
|
+
for (const { id, score } of fused.slice(0, 20)) {
|
|
1682
|
+
const bm25Hit = bm25ById.get(id);
|
|
1683
|
+
const sim = simById.get(id);
|
|
1684
|
+
const codeLines = codeLinesById.get(id);
|
|
1685
|
+
// Reconstruct a clean DbNode — never spread bm25Hit wholesale, or its internal fields leak.
|
|
1686
|
+
const src = bm25Hit ?? this.getNode(id);
|
|
1687
|
+
/* istanbul ignore next -- `fused`'s ids are drawn only from tokenSearchNodes/vectorSearchNodes/
|
|
1688
|
+
mapGrepHitsToNodes, all three of which query `nodes` directly (deprecated = 0) at call time
|
|
1689
|
+
a few lines above; nothing mutates the DB between those queries and this synchronous loop,
|
|
1690
|
+
so `this.getNode(id)` (the fallback when `bm25Hit` is absent) always finds a row here today.
|
|
1691
|
+
Kept as a real guard against a future async gap or query decoupling, not because today's
|
|
1692
|
+
flow can reach it. */
|
|
1693
|
+
if (!src)
|
|
1694
|
+
continue; // orphaned vector row surviving between sweeps — skip, don't crash
|
|
1695
|
+
/* istanbul ignore next -- `src` is always either a `tokenSearchNodes` row or a
|
|
1696
|
+
`getNode()` result, both of which route through `parseNodeRow`/`parseNodeAliases` and
|
|
1697
|
+
so always carry a real `string[]` aliases — the `?? []` fallback is unreachable in
|
|
1698
|
+
practice, kept only as a type-level safety net. */
|
|
1699
|
+
const srcAliases = src.aliases ?? [];
|
|
1700
|
+
const node = {
|
|
1701
|
+
id: src.id, type: src.type, name: src.name, file_path: src.file_path,
|
|
1702
|
+
signature: src.signature, description: src.description,
|
|
1703
|
+
aliases: srcAliases,
|
|
1704
|
+
deprecated: src.deprecated, created_at: src.created_at
|
|
1705
|
+
};
|
|
1706
|
+
const found_by = [];
|
|
1707
|
+
if (bm25Hit)
|
|
1708
|
+
found_by.push('keyword');
|
|
1709
|
+
if (sim !== undefined)
|
|
1710
|
+
found_by.push('meaning');
|
|
1711
|
+
if (codeLines)
|
|
1712
|
+
found_by.push('code');
|
|
1713
|
+
// Confidence from real evidence, not the fused float. Corroboration across ≥2 independent
|
|
1714
|
+
// layers is the strongest signal there is → high. A lone semantic match is graded by its
|
|
1715
|
+
// actual cosine (0.6+ is genuinely close; 0.35-0.45 is borderline). A lone keyword or code
|
|
1716
|
+
// hit already cleared its own floor, so it's real-but-uncorroborated → medium.
|
|
1717
|
+
let confidence;
|
|
1718
|
+
if (found_by.length >= 2) {
|
|
1719
|
+
confidence = 'high';
|
|
1720
|
+
}
|
|
1721
|
+
else if (found_by.length === 1 && found_by[0] === 'meaning' && sim !== undefined) {
|
|
1722
|
+
confidence = sim >= 0.6 ? 'high' : sim >= 0.45 ? 'medium' : 'low';
|
|
1723
|
+
}
|
|
1724
|
+
else {
|
|
1725
|
+
confidence = 'medium';
|
|
1726
|
+
}
|
|
1727
|
+
const matched_via = bm25Hit ? 'fuzzy' : (codeLines ? 'code' : 'semantic');
|
|
1728
|
+
const relevance = Math.max(1, Math.round((score / topScore) * 100));
|
|
1729
|
+
// Trust signals ahead of `description` — see the identifier short-circuit above for why.
|
|
1730
|
+
const { id: nodeId, name: nodeName, type: nodeType, ...nodeRest } = node;
|
|
1731
|
+
nodes.push({
|
|
1732
|
+
id: nodeId, name: nodeName, type: nodeType,
|
|
1733
|
+
confidence,
|
|
1734
|
+
relevance,
|
|
1735
|
+
found_by,
|
|
1736
|
+
matched_via,
|
|
1737
|
+
...nodeRest,
|
|
1738
|
+
matched_terms: bm25Hit ? bm25Hit.matched_terms : [],
|
|
1739
|
+
code_matches: codeLines,
|
|
1740
|
+
uses: 0, used_by: 0, history_count: 0
|
|
1741
|
+
});
|
|
1742
|
+
}
|
|
1743
|
+
this.attachDrillInHooks(nodes);
|
|
1744
|
+
return {
|
|
1745
|
+
nodes,
|
|
1746
|
+
files: filesResult.files,
|
|
1747
|
+
files_total: filesResult.total,
|
|
1748
|
+
files_offset: filesOffset,
|
|
1749
|
+
nodes_total: fused.length,
|
|
1750
|
+
truncated: grep.truncated || undefined,
|
|
1751
|
+
scope_note: scopeNote
|
|
1752
|
+
};
|
|
1753
|
+
}
|
|
1754
|
+
/**
|
|
1755
|
+
* The token-ranked (BM25) half of {@link searchNodes}. Looks up every query/keyword token in
|
|
1756
|
+
* the local `node_tokens` index (rebuilding it first if stale — see {@link ensureSearchIndexFresh}),
|
|
1757
|
+
* scores each candidate with {@link scoreCandidate} (IDF-weighted, saturating TF, per-field
|
|
1758
|
+
* weights favoring `description`), and applies the noise floor. Metadata only — identifier / id
|
|
1759
|
+
* / path / description / reasoning. Code-body matching is NO LONGER folded in here: it used to
|
|
1760
|
+
* read one history-JSON per node (~9k serial reads = the tool's ~10-30s cost), and is now
|
|
1761
|
+
* served far faster by the real filesystem grep in `searchNodes` (see `mapGrepHitsToNodes`).
|
|
1762
|
+
* Returns nodes tagged `matched_via:'fuzzy'`, ranked; `searchNodes` re-fuses them by RRF.
|
|
1763
|
+
*/
|
|
1764
|
+
tokenSearchNodes(tokens) {
|
|
1765
|
+
const uniq = Array.from(new Set(tokens));
|
|
1766
|
+
if (uniq.length === 0)
|
|
1767
|
+
return [];
|
|
1768
|
+
this.ensureSearchIndexFresh();
|
|
1769
|
+
const totalNodesRow = this.db.prepare('SELECT COUNT(*) as c FROM nodes WHERE deprecated = 0').get();
|
|
1770
|
+
const totalNodes = Math.max(totalNodesRow.c, 1);
|
|
1771
|
+
const placeholders = uniq.map(() => '?').join(',');
|
|
1772
|
+
// Document frequency per (token, field) computed ONCE via a grouped aggregate, not as a
|
|
1773
|
+
// per-ROW correlated subquery — that used to re-scan node_tokens once for EVERY matching row
|
|
1774
|
+
// (thousands, for a common word across an 8k-node/468k-token real corpus), which alone
|
|
1775
|
+
// measured at 9-10 SECONDS on a real production graph. One aggregate query + an in-memory
|
|
1776
|
+
// lookup does the identical computation in a few milliseconds.
|
|
1777
|
+
const docFreqRows = this.db.prepare(`
|
|
1778
|
+
SELECT token, field, COUNT(DISTINCT node_id) AS doc_freq
|
|
1779
|
+
FROM node_tokens
|
|
1780
|
+
WHERE token IN (${placeholders})
|
|
1781
|
+
GROUP BY token, field
|
|
1782
|
+
`).all(...uniq);
|
|
1783
|
+
const docFreqByKey = new Map();
|
|
1784
|
+
for (const r of docFreqRows)
|
|
1785
|
+
docFreqByKey.set(`${r.token}${r.field}`, r.doc_freq);
|
|
1786
|
+
const rows = this.db.prepare(`
|
|
1787
|
+
SELECT node_id, token, field, tf
|
|
1788
|
+
FROM node_tokens
|
|
1789
|
+
WHERE token IN (${placeholders})
|
|
1790
|
+
`).all(...uniq);
|
|
1791
|
+
const byNode = new Map();
|
|
1792
|
+
for (const row of rows) {
|
|
1793
|
+
const entry = byNode.get(row.node_id) || { matches: [], terms: new Set() };
|
|
1794
|
+
/* istanbul ignore next -- `rows` and `docFreqRows` are both filtered from `node_tokens` by
|
|
1795
|
+
the exact same `WHERE token IN (...)`, and `docFreqRows` is a `GROUP BY token, field` over
|
|
1796
|
+
that identical row set — so every (token, field) pair appearing in `rows` necessarily has
|
|
1797
|
+
a matching aggregate entry already. The `?? 1` fallback is unreachable in practice, kept
|
|
1798
|
+
only as a defensive default if the two queries are ever edited out of lockstep. */
|
|
1799
|
+
const docFreq = docFreqByKey.get(`${row.token}${row.field}`) ?? 1;
|
|
1800
|
+
entry.matches.push({ field: row.field, tf: row.tf, docFreq, totalNodes });
|
|
1801
|
+
entry.terms.add(row.token);
|
|
1802
|
+
byNode.set(row.node_id, entry);
|
|
1803
|
+
}
|
|
1804
|
+
if (byNode.size === 0)
|
|
918
1805
|
return [];
|
|
1806
|
+
// Reject the thinnest possible "match" instead of letting anything with one shared token
|
|
1807
|
+
// through: a query like "pending-orders/process-order" sharing only the generic word "order"
|
|
1808
|
+
// with an unrelated node is weak evidence, not a real hit. Two tunable gates:
|
|
1809
|
+
// (1) coverage — a multi-token query must match more than a single one of its distinct tokens
|
|
1810
|
+
// (a single-token query has nothing more to require, so this is a no-op for it);
|
|
1811
|
+
// (2) a minimum absolute score, since even one match CAN be strong (a rare word in the
|
|
1812
|
+
// description field) and shouldn't be discarded just for being one token.
|
|
1813
|
+
const minCoverage = Math.min(2, uniq.length);
|
|
1814
|
+
const MIN_BM25_SCORE = 0.75;
|
|
1815
|
+
const out = [];
|
|
1816
|
+
const nodeIds = Array.from(byNode.keys());
|
|
1817
|
+
const nodePlaceholders = nodeIds.map(() => '?').join(',');
|
|
1818
|
+
const nodeRows = DevMindDatabase.parseNodeRows(this.db.prepare(`SELECT * FROM nodes WHERE id IN (${nodePlaceholders}) AND deprecated = 0`).all(...nodeIds));
|
|
1819
|
+
for (const node of nodeRows) {
|
|
1820
|
+
const entry = byNode.get(node.id);
|
|
1821
|
+
if (entry.terms.size < minCoverage)
|
|
1822
|
+
continue;
|
|
1823
|
+
const score = (0, search_index_1.scoreCandidate)(entry.matches, search_index_1.DEFAULT_FIELD_WEIGHTS);
|
|
1824
|
+
if (score < MIN_BM25_SCORE)
|
|
1825
|
+
continue;
|
|
1826
|
+
out.push({ ...node, matched_via: 'fuzzy', matched_terms: Array.from(entry.terms), score, low_confidence: true });
|
|
1827
|
+
}
|
|
1828
|
+
return out.sort((a, b) => b.score - a.score).slice(0, 20);
|
|
1829
|
+
}
|
|
1830
|
+
/** Absolute filesystem roots for every configured repo — the search space for `grepRepos`. */
|
|
1831
|
+
repoRoots() {
|
|
1832
|
+
if (!this.context)
|
|
1833
|
+
return [];
|
|
1834
|
+
const roots = [];
|
|
1835
|
+
for (const repo of this.context.config.repos) {
|
|
1836
|
+
const root = (0, config_1.resolveRepoPath)(this.context, repo.name);
|
|
1837
|
+
if (root)
|
|
1838
|
+
roots.push(root);
|
|
919
1839
|
}
|
|
920
|
-
return
|
|
1840
|
+
return roots;
|
|
921
1841
|
}
|
|
922
1842
|
/**
|
|
923
|
-
*
|
|
924
|
-
*
|
|
925
|
-
*
|
|
1843
|
+
* Resolves and validates `search_nodes`' optional `path` scope — a single folder or file the
|
|
1844
|
+
* grep walk restricts to, instead of always walking every configured repo. A path outside every
|
|
1845
|
+
* repo root is REJECTED (thrown, not silently widened to "search everything") — a scope that
|
|
1846
|
+
* can't be honored should fail loudly, since silently ignoring it would search far more than
|
|
1847
|
+
* the caller asked for. `canonicalizePath` (already used throughout this file for path
|
|
1848
|
+
* comparisons) lowercases the Windows drive letter so `C:\...` and `c:\...` compare equal.
|
|
926
1849
|
*/
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
1850
|
+
resolveSearchScopePath(rawPath) {
|
|
1851
|
+
if (!rawPath || !rawPath.trim())
|
|
1852
|
+
return undefined;
|
|
1853
|
+
const resolved = (0, config_1.canonicalizePath)(rawPath.trim());
|
|
1854
|
+
const roots = this.repoRoots().map(config_1.canonicalizePath);
|
|
1855
|
+
const contained = roots.some(root => resolved === root || resolved.startsWith(root + path.sep));
|
|
1856
|
+
if (!contained) {
|
|
1857
|
+
throw new Error(`search_nodes: path "${rawPath}" is outside every configured repo (${roots.join(', ') || 'none configured'}).`);
|
|
932
1858
|
}
|
|
933
|
-
return
|
|
1859
|
+
return resolved;
|
|
934
1860
|
}
|
|
935
1861
|
/**
|
|
936
|
-
*
|
|
937
|
-
*
|
|
938
|
-
*
|
|
939
|
-
*
|
|
940
|
-
*
|
|
941
|
-
*
|
|
942
|
-
*
|
|
1862
|
+
* Maps raw grep hits back to graph nodes — this is "code search that returns nodes", rebuilt
|
|
1863
|
+
* on top of the single filesystem walk instead of the old ~9k per-node snapshot reads. For each
|
|
1864
|
+
* file that had hits AND contains indexed nodes: an AST-parseable file is resolved precisely
|
|
1865
|
+
* (each hit line → the one node whose line-range contains it, via {@link locateNodeInFile}), so
|
|
1866
|
+
* a keyword hitting one method in a 10-method file surfaces THAT node, not all ten. A non-AST
|
|
1867
|
+
* indexed file (a staged `.py`/`.go` node) falls back to coarse file→node. Returns node ids in
|
|
1868
|
+
* match-strength order (files with more hits first) plus the matching lines, for RRF + display.
|
|
1869
|
+
*
|
|
1870
|
+
* Only the top {@link CODE_MATCH_FILE_CAP} files by hit count are AST-resolved — measured on a
|
|
1871
|
+
* real 8-repo, 15k-grep-hit query, this step alone was 8.2s of a 9.6s total before the cap, one
|
|
1872
|
+
* `locateNodeInFile` AST walk per node per matched file with no bound on how many files that
|
|
1873
|
+
* could be. It doesn't cost result quality: the fused ranking downstream keeps only the top ~20
|
|
1874
|
+
* nodes anyway, and files with more hits are exactly the ones most likely to place there — files
|
|
1875
|
+
* past the cap would almost never have survived to the final result even fully resolved.
|
|
943
1876
|
*/
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
const
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
1877
|
+
mapGrepHitsToNodes(hits) {
|
|
1878
|
+
if (hits.length === 0)
|
|
1879
|
+
return [];
|
|
1880
|
+
// Group hit line numbers (+ content) by file, and rank files by hit count so the strongest
|
|
1881
|
+
// code matches come first in the returned order (which becomes their RRF rank).
|
|
1882
|
+
const byFile = new Map();
|
|
1883
|
+
for (const h of hits) {
|
|
1884
|
+
let arr = byFile.get(h.file_path);
|
|
1885
|
+
if (!arr) {
|
|
1886
|
+
arr = [];
|
|
1887
|
+
byFile.set(h.file_path, arr);
|
|
1888
|
+
}
|
|
1889
|
+
arr.push({ line_number: h.line_number, line_content: h.line_content });
|
|
1890
|
+
}
|
|
1891
|
+
const CODE_MATCH_FILE_CAP = 30;
|
|
1892
|
+
const filesByStrength = Array.from(byFile.entries())
|
|
1893
|
+
.sort((a, b) => b[1].length - a[1].length)
|
|
1894
|
+
.slice(0, CODE_MATCH_FILE_CAP);
|
|
1895
|
+
const perNode = new Map();
|
|
1896
|
+
for (const [filePath, lines] of filesByStrength) {
|
|
1897
|
+
const nodes = this.getNodesByFilePath(filePath);
|
|
1898
|
+
if (nodes.length === 0)
|
|
1899
|
+
continue; // an un-indexed file — lives in the files bucket only
|
|
1900
|
+
if ((0, ast_1.isAstParseable)(filePath)) {
|
|
1901
|
+
// Precise: resolve each node's line span once, assign every hit line to its container.
|
|
1902
|
+
const spans = this.computeSymbolSpans(filePath, nodes);
|
|
1903
|
+
for (const line of lines) {
|
|
1904
|
+
const containing = spans.find(s => line.line_number >= s.startLine && line.line_number <= s.endLine);
|
|
1905
|
+
if (!containing)
|
|
1906
|
+
continue; // hit was between symbols (an import, a top-level const) — file bucket has it
|
|
1907
|
+
let acc = perNode.get(containing.id);
|
|
1908
|
+
if (!acc) {
|
|
1909
|
+
acc = [];
|
|
1910
|
+
perNode.set(containing.id, acc);
|
|
975
1911
|
}
|
|
1912
|
+
if (acc.length < 5)
|
|
1913
|
+
acc.push(line);
|
|
976
1914
|
}
|
|
977
1915
|
}
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
}
|
|
984
|
-
if (nameLower.includes(token) || idLower.includes(token)) {
|
|
985
|
-
score += FIELD_WEIGHT.identifier;
|
|
986
|
-
tokenMatched = true;
|
|
987
|
-
}
|
|
988
|
-
if (reasoningLower.includes(token)) {
|
|
989
|
-
score += FIELD_WEIGHT.reasoning;
|
|
990
|
-
tokenMatched = true;
|
|
991
|
-
}
|
|
992
|
-
if (code && code.includes(token)) {
|
|
993
|
-
score += FIELD_WEIGHT.code;
|
|
994
|
-
tokenMatched = true;
|
|
1916
|
+
else {
|
|
1917
|
+
// Coarse fallback for indexed but non-AST files (staged .py/.go/etc): every node in the file.
|
|
1918
|
+
for (const n of nodes) {
|
|
1919
|
+
if (!perNode.has(n.id))
|
|
1920
|
+
perNode.set(n.id, lines.slice(0, 5));
|
|
995
1921
|
}
|
|
996
|
-
if (tokenMatched)
|
|
997
|
-
matchedTerms.add(token);
|
|
998
1922
|
}
|
|
999
|
-
|
|
1000
|
-
|
|
1923
|
+
}
|
|
1924
|
+
// perNode preserves insertion order = file-strength order, which is the ranking we want.
|
|
1925
|
+
return Array.from(perNode.entries()).map(([nodeId, lines]) => ({ nodeId, lines }));
|
|
1926
|
+
}
|
|
1927
|
+
/**
|
|
1928
|
+
* Resolves every indexed node's line span within one AST-parseable file — extracted out of
|
|
1929
|
+
* `mapGrepHitsToNodes` so `annotateSampleLinesWithSymbol` can reuse the identical span logic
|
|
1930
|
+
* instead of re-deriving it. Computed ONCE per file (not per line/hit): a file with several
|
|
1931
|
+
* sample lines would otherwise repeat the same `locateNodeInFile` AST walk once per line.
|
|
1932
|
+
* `nodes` is accepted rather than re-fetched so an existing `getNodesByFilePath` result (as
|
|
1933
|
+
* `mapGrepHitsToNodes` already has) isn't queried twice.
|
|
1934
|
+
*/
|
|
1935
|
+
computeSymbolSpans(filePath, nodes) {
|
|
1936
|
+
return nodes.map(n => {
|
|
1937
|
+
const parsed = (0, ast_1.parseNodeId)(n.id);
|
|
1938
|
+
const symbol = parsed ? parsed.symbolName : (n.id.split('#').pop() || n.name);
|
|
1939
|
+
const loc = symbol ? (0, ast_1.locateNodeInFile)(filePath, symbol) : null;
|
|
1940
|
+
return loc ? { id: n.id, name: n.name, startLine: loc.startLine, endLine: loc.endLine } : null;
|
|
1941
|
+
}).filter((s) => s !== null);
|
|
1942
|
+
}
|
|
1943
|
+
/**
|
|
1944
|
+
* Mutates each `RankedFile`'s `sample_lines` in place, tagging every line with the function/class
|
|
1945
|
+
* that contains it — the insight a plain filesystem grep can never give: not just "line 87
|
|
1946
|
+
* matched" but "line 87, inside `onLikeTap`". This is the annotation `search_nodes` offers that
|
|
1947
|
+
* makes it worth more than grep for a hit that lands inside real source.
|
|
1948
|
+
*
|
|
1949
|
+
* Deliberately bounded to the ALREADY-CAPPED page (`files`, ≤`maxFiles` entries × ≤5 sample
|
|
1950
|
+
* lines each — at most ~125 lookups) rather than every raw grep hit. `mapGrepHitsToNodes` is
|
|
1951
|
+
* measured at 8.2s of a 9.6s query on a real 8-repo search (see its doc comment) precisely from
|
|
1952
|
+
* unbounded per-hit AST resolution; annotating only what's actually returned keeps this at a
|
|
1953
|
+
* small, constant added cost regardless of how broad the pattern was or how many total hits it
|
|
1954
|
+
* produced. Silently no-ops for a file that isn't indexed or isn't AST-parseable — a `symbol` on
|
|
1955
|
+
* a sample line is a bonus, never a requirement.
|
|
1956
|
+
*/
|
|
1957
|
+
annotateSampleLinesWithSymbol(files) {
|
|
1958
|
+
for (const file of files) {
|
|
1959
|
+
if (!(0, ast_1.isAstParseable)(file.file_path))
|
|
1960
|
+
continue;
|
|
1961
|
+
const nodes = this.getNodesByFilePath(file.file_path);
|
|
1962
|
+
if (nodes.length === 0)
|
|
1963
|
+
continue;
|
|
1964
|
+
const spans = this.computeSymbolSpans(file.file_path, nodes);
|
|
1965
|
+
if (spans.length === 0)
|
|
1966
|
+
continue;
|
|
1967
|
+
for (const line of file.sample_lines) {
|
|
1968
|
+
const containing = spans.find(s => line.line_number >= s.startLine && line.line_number <= s.endLine);
|
|
1969
|
+
if (containing)
|
|
1970
|
+
line.symbol = containing.name;
|
|
1001
1971
|
}
|
|
1002
1972
|
}
|
|
1003
|
-
return scored.sort((a, b) => b.score - a.score).slice(0, 20);
|
|
1004
1973
|
}
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1039
|
-
|
|
1040
|
-
|
|
1041
|
-
const
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
`);
|
|
1060
|
-
const
|
|
1061
|
-
return
|
|
1974
|
+
/**
|
|
1975
|
+
* The semantic half of hybrid search: embeds the query with the same vendored ONNX model used
|
|
1976
|
+
* to embed every node's description (see embedder.ts), then linear-scans `node_vectors` for
|
|
1977
|
+
* cosine similarity. Linear is fine at this scale — ~9k nodes × 384 int8 dims is a few million
|
|
1978
|
+
* integer multiplications, well under a millisecond, far below the model's own inference time
|
|
1979
|
+
* for the query itself; revisit only past ~100k nodes. Returns each surviving node's raw cosine
|
|
1980
|
+
* alongside its id — the ORDER drives RRF, but the cosine value is kept so `searchNodes` can turn
|
|
1981
|
+
* it into a human-meaningful confidence (a 0.7 cosine is a strong match; a 0.36 is a weak one —
|
|
1982
|
+
* the fused RRF float can't express that). Returns [] — never throws — if the embedder is
|
|
1983
|
+
* unavailable, so `searchNodes` degrades to BM25+grep exactly as before.
|
|
1984
|
+
*/
|
|
1985
|
+
async vectorSearchNodes(query) {
|
|
1986
|
+
const queryVector = await (0, embedder_1.embedTextInt8)(query);
|
|
1987
|
+
if (!queryVector)
|
|
1988
|
+
return [];
|
|
1989
|
+
const rows = this.db.prepare(`
|
|
1990
|
+
SELECT nv.node_id AS node_id, nv.vector AS vector
|
|
1991
|
+
FROM node_vectors nv
|
|
1992
|
+
JOIN nodes n ON n.id = nv.node_id
|
|
1993
|
+
WHERE n.deprecated = 0 AND nv.model_id = ?
|
|
1994
|
+
`).all(embedder_1.EMBEDDING_MODEL_ID);
|
|
1995
|
+
if (rows.length === 0)
|
|
1996
|
+
return [];
|
|
1997
|
+
const scored = rows.map(r => ({
|
|
1998
|
+
id: r.node_id,
|
|
1999
|
+
sim: (0, embedder_1.cosineInt8)(queryVector, new Int8Array(r.vector.buffer, r.vector.byteOffset, r.vector.byteLength))
|
|
2000
|
+
}));
|
|
2001
|
+
scored.sort((a, b) => b.sim - a.sim);
|
|
2002
|
+
// A floor, not just a top-N cut: padding the ranking with genuinely unrelated nodes just to
|
|
2003
|
+
// fill a quota would inject noise into the RRF fusion. A low cosine here means "nothing is
|
|
2004
|
+
// semantically close" — that's a real signal that vector search found nothing, not a reason
|
|
2005
|
+
// to return its least-bad guesses. 0.35, not 0.2: MiniLM-class sentence embeddings commonly
|
|
2006
|
+
// put two UNRELATED short texts around 0.1-0.25 just from shared sentence structure/English
|
|
2007
|
+
// baseline, not real similarity — 0.2 was letting that noise floor through as if it meant
|
|
2008
|
+
// something. Tunable; revisit against real query/result pairs if this starts rejecting
|
|
2009
|
+
// matches that should have passed.
|
|
2010
|
+
const MIN_COSINE_SIMILARITY = 0.35;
|
|
2011
|
+
return scored.filter(s => s.sim > MIN_COSINE_SIMILARITY).slice(0, 50);
|
|
2012
|
+
}
|
|
2013
|
+
/**
|
|
2014
|
+
* Cheap, single-pass signal for "has the graph changed since node_tokens was last built":
|
|
2015
|
+
* how many non-deprecated nodes exist, the total length of every description (catches a
|
|
2016
|
+
* description being added OR edited — length almost never stays identical), and how many
|
|
2017
|
+
* history rows exist (catches new reasoning). Deliberately NOT based on an `updated_at`
|
|
2018
|
+
* column on `nodes` — there isn't one, and `syncFromDisk`'s destructive graph-rebuild pass
|
|
2019
|
+
* writes nodes via raw SQL, not through a single method that could easily be hooked, so a
|
|
2020
|
+
* write-path-by-write-path invalidation scheme would silently miss that path (and did, in an
|
|
2021
|
+
* earlier draft of this). A fingerprint checked lazily at query time can't be missed the same
|
|
2022
|
+
* way — it doesn't matter HOW the data changed, only THAT it did.
|
|
2023
|
+
*/
|
|
2024
|
+
searchIndexFingerprint() {
|
|
2025
|
+
const row = this.db.prepare(`
|
|
2026
|
+
SELECT COUNT(*) as node_count, COALESCE(SUM(LENGTH(description)), 0) as desc_len_sum
|
|
2027
|
+
FROM nodes WHERE deprecated = 0
|
|
2028
|
+
`).get();
|
|
2029
|
+
const historyRow = this.db.prepare('SELECT COUNT(*) as c FROM history').get();
|
|
2030
|
+
return `${row.node_count}:${row.desc_len_sum}:${historyRow.c}`;
|
|
2031
|
+
}
|
|
2032
|
+
/** Rebuilds `node_tokens` from scratch for every non-deprecated node's current
|
|
2033
|
+
* identifier/id/path/description and EVERY revision's reasoning (not just the latest — a
|
|
2034
|
+
* decision recorded three revisions ago must stay findable by `search_nodes`, since there is
|
|
2035
|
+
* no separate decisions-only search tool anymore; `GROUP_CONCAT` folds every history row's
|
|
2036
|
+
* reasoning into one field before tokenizing, so an older "Decision: …" is searchable exactly
|
|
2037
|
+
* like the newest one). Cheap at realistic node counts (hundreds to low-thousands) — a single
|
|
2038
|
+
* scan plus one batched transaction, not something that needs to be avoided; simplicity here
|
|
2039
|
+
* is worth more than incremental upkeep that a bypassed write path could silently defeat. */
|
|
2040
|
+
rebuildSearchIndex() {
|
|
2041
|
+
const nodes = this.db.prepare(`
|
|
2042
|
+
SELECT n.*, (
|
|
2043
|
+
SELECT GROUP_CONCAT(h.reasoning, ' ') FROM history h WHERE h.node_id = n.id
|
|
2044
|
+
) AS all_reasoning
|
|
2045
|
+
FROM nodes n
|
|
2046
|
+
WHERE n.deprecated = 0
|
|
2047
|
+
`).all();
|
|
2048
|
+
const del = this.db.prepare('DELETE FROM node_tokens');
|
|
2049
|
+
const ins = this.db.prepare('INSERT OR REPLACE INTO node_tokens (node_id, token, field, tf) VALUES (?, ?, ?, ?)');
|
|
2050
|
+
const tx = this.db.transaction(() => {
|
|
2051
|
+
del.run();
|
|
2052
|
+
for (const node of nodes) {
|
|
2053
|
+
const rows = [
|
|
2054
|
+
...(0, search_index_1.tokenizeNodeField)(node.name, 'identifier'),
|
|
2055
|
+
...(0, search_index_1.tokenizeNodeField)(node.id, 'identifier'),
|
|
2056
|
+
...(0, search_index_1.tokenizeNodeField)(node.file_path, 'path'),
|
|
2057
|
+
...(0, search_index_1.tokenizeNodeField)(node.description, 'description'),
|
|
2058
|
+
...(0, search_index_1.tokenizeNodeField)(node.all_reasoning, 'reasoning')
|
|
2059
|
+
];
|
|
2060
|
+
// Merge duplicate (field,token) pairs from the multiple sources above (e.g. name and id
|
|
2061
|
+
// both contributing the same word) into one summed tf, rather than letting the LATER
|
|
2062
|
+
// INSERT OR REPLACE silently discard the earlier one's count.
|
|
2063
|
+
const merged = new Map();
|
|
2064
|
+
for (const r of rows) {
|
|
2065
|
+
const key = `${r.field}:${r.token}`;
|
|
2066
|
+
const existing = merged.get(key);
|
|
2067
|
+
if (existing)
|
|
2068
|
+
existing.tf += r.tf;
|
|
2069
|
+
else
|
|
2070
|
+
merged.set(key, { ...r });
|
|
2071
|
+
}
|
|
2072
|
+
for (const r of merged.values())
|
|
2073
|
+
ins.run(node.id, r.token, r.field, r.tf);
|
|
2074
|
+
}
|
|
2075
|
+
});
|
|
2076
|
+
tx();
|
|
2077
|
+
}
|
|
2078
|
+
/** Compares the current fingerprint against what `node_tokens` was last built from, stored in
|
|
2079
|
+
* `system_meta`; rebuilds and updates the stored fingerprint only on a mismatch. A momentarily
|
|
2080
|
+
* stale index (between a real change and the next search call) only costs ranking quality on
|
|
2081
|
+
* that one call, never correctness — the next call rebuilds it. */
|
|
2082
|
+
ensureSearchIndexFresh() {
|
|
2083
|
+
const current = this.searchIndexFingerprint();
|
|
2084
|
+
const row = this.db.prepare(`SELECT value FROM system_meta WHERE key = 'search_index_fingerprint'`).get();
|
|
2085
|
+
if (row && row.value === current)
|
|
2086
|
+
return;
|
|
2087
|
+
this.rebuildSearchIndex();
|
|
2088
|
+
this.db.prepare(`INSERT OR REPLACE INTO system_meta (key, value, updated_at) VALUES ('search_index_fingerprint', ?, CURRENT_TIMESTAMP)`).run(current);
|
|
1062
2089
|
}
|
|
1063
2090
|
searchDecisions(query) {
|
|
1064
|
-
const stmt = this.db.prepare(`
|
|
1065
|
-
SELECT h.node_id, n.name as node_name, h.updated_at, h.reasoning
|
|
1066
|
-
FROM history h
|
|
1067
|
-
JOIN nodes n ON h.node_id = n.id
|
|
1068
|
-
WHERE h.reasoning LIKE ? ESCAPE '\\'
|
|
1069
|
-
ORDER BY h.updated_at DESC
|
|
2091
|
+
const stmt = this.db.prepare(`
|
|
2092
|
+
SELECT h.node_id, n.name as node_name, h.updated_at, h.reasoning
|
|
2093
|
+
FROM history h
|
|
2094
|
+
JOIN nodes n ON h.node_id = n.id
|
|
2095
|
+
WHERE h.reasoning LIKE ? ESCAPE '\\'
|
|
2096
|
+
ORDER BY h.updated_at DESC
|
|
1070
2097
|
`);
|
|
1071
2098
|
const wildcard = `%Decision: %${this.likeEscape(query)}%`;
|
|
1072
2099
|
return stmt.all(wildcard);
|
|
@@ -1074,17 +2101,17 @@ class DevMindDatabase {
|
|
|
1074
2101
|
searchCode(params) {
|
|
1075
2102
|
const { query, is_regex = false, case_insensitive = true } = params;
|
|
1076
2103
|
const historyDir = path.join(path.dirname(this.dbPath), 'history');
|
|
1077
|
-
const stmt = this.db.prepare(`
|
|
1078
|
-
SELECT h.id, n.id AS node_id, n.name AS node_name, n.file_path
|
|
1079
|
-
FROM nodes n
|
|
1080
|
-
JOIN history h ON h.node_id = n.id
|
|
1081
|
-
WHERE n.deprecated = 0
|
|
1082
|
-
AND h.id = (
|
|
1083
|
-
SELECT id FROM history
|
|
1084
|
-
WHERE node_id = n.id
|
|
1085
|
-
ORDER BY updated_at DESC
|
|
1086
|
-
LIMIT 1
|
|
1087
|
-
)
|
|
2104
|
+
const stmt = this.db.prepare(`
|
|
2105
|
+
SELECT h.id, n.id AS node_id, n.name AS node_name, n.file_path
|
|
2106
|
+
FROM nodes n
|
|
2107
|
+
JOIN history h ON h.node_id = n.id
|
|
2108
|
+
WHERE n.deprecated = 0
|
|
2109
|
+
AND h.id = (
|
|
2110
|
+
SELECT id FROM history
|
|
2111
|
+
WHERE node_id = n.id
|
|
2112
|
+
ORDER BY updated_at DESC
|
|
2113
|
+
LIMIT 1
|
|
2114
|
+
)
|
|
1088
2115
|
`);
|
|
1089
2116
|
const rows = stmt.all();
|
|
1090
2117
|
let matcher;
|
|
@@ -1139,20 +2166,25 @@ class DevMindDatabase {
|
|
|
1139
2166
|
return results.sort((a, b) => b.match_count - a.match_count);
|
|
1140
2167
|
}
|
|
1141
2168
|
getOrphanedNodes() {
|
|
1142
|
-
const stmt = this.db.prepare(`
|
|
1143
|
-
SELECT * FROM nodes
|
|
1144
|
-
WHERE deprecated = 0
|
|
1145
|
-
AND id NOT IN (SELECT DISTINCT source_node_id FROM node_connections)
|
|
1146
|
-
AND id NOT IN (SELECT DISTINCT target_node_id FROM node_connections)
|
|
2169
|
+
const stmt = this.db.prepare(`
|
|
2170
|
+
SELECT * FROM nodes
|
|
2171
|
+
WHERE deprecated = 0
|
|
2172
|
+
AND id NOT IN (SELECT DISTINCT source_node_id FROM node_connections)
|
|
2173
|
+
AND id NOT IN (SELECT DISTINCT target_node_id FROM node_connections)
|
|
1147
2174
|
`);
|
|
1148
|
-
return stmt.all();
|
|
2175
|
+
return DevMindDatabase.parseNodeRows(stmt.all());
|
|
1149
2176
|
}
|
|
1150
2177
|
getAllNodes() {
|
|
1151
2178
|
const stmt = this.db.prepare('SELECT * FROM nodes');
|
|
1152
|
-
return stmt.all();
|
|
2179
|
+
return DevMindDatabase.parseNodeRows(stmt.all());
|
|
1153
2180
|
}
|
|
1154
|
-
|
|
1155
|
-
|
|
2181
|
+
/**
|
|
2182
|
+
* The shared WHERE clause behind {@link listNodes} and {@link countNodes}. Extracted so the page
|
|
2183
|
+
* and its total can never drift apart — a `total` computed from even slightly different criteria
|
|
2184
|
+
* than the rows it describes is worse than no total at all, since it reads as authoritative.
|
|
2185
|
+
*/
|
|
2186
|
+
buildNodeFilterSql(filter) {
|
|
2187
|
+
let sql = ' WHERE 1=1';
|
|
1156
2188
|
const params = [];
|
|
1157
2189
|
if (filter?.type) {
|
|
1158
2190
|
sql += ' AND type = ?';
|
|
@@ -1172,8 +2204,34 @@ class DevMindDatabase {
|
|
|
1172
2204
|
if (!filter?.include_deprecated) {
|
|
1173
2205
|
sql += ' AND deprecated = 0';
|
|
1174
2206
|
}
|
|
1175
|
-
|
|
1176
|
-
|
|
2207
|
+
return { where: sql, params };
|
|
2208
|
+
}
|
|
2209
|
+
/**
|
|
2210
|
+
* Total nodes matching a filter, independent of any page. Exists so `list_nodes` can tell
|
|
2211
|
+
* "that's everything" from "there is more" — the same honesty contract `nodes_total` and
|
|
2212
|
+
* `files_total` already keep for search.
|
|
2213
|
+
*/
|
|
2214
|
+
countNodes(filter) {
|
|
2215
|
+
const { where, params } = this.buildNodeFilterSql(filter);
|
|
2216
|
+
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM nodes${where}`).get(...params);
|
|
2217
|
+
return row.c;
|
|
2218
|
+
}
|
|
2219
|
+
/**
|
|
2220
|
+
* `limit`/`offset` are OPTIONAL and, when omitted, this returns every matching row exactly as
|
|
2221
|
+
* before — the internal analysis callers (`analyze.ts`, `edges.ts`) legitimately need the whole
|
|
2222
|
+
* graph, so paging is opt-in rather than a default that would silently truncate them. Ordering
|
|
2223
|
+
* is applied only when paging, since an unordered LIMIT is a lottery: without it, "page 2" is
|
|
2224
|
+
* not guaranteed to exclude what "page 1" already returned.
|
|
2225
|
+
*/
|
|
2226
|
+
listNodes(filter) {
|
|
2227
|
+
const { where, params } = this.buildNodeFilterSql(filter);
|
|
2228
|
+
let sql = `SELECT * FROM nodes${where}`;
|
|
2229
|
+
const args = [...params];
|
|
2230
|
+
if (filter?.limit !== undefined) {
|
|
2231
|
+
sql += ' ORDER BY file_path, name LIMIT ? OFFSET ?';
|
|
2232
|
+
args.push(filter.limit, filter.offset ?? 0);
|
|
2233
|
+
}
|
|
2234
|
+
return DevMindDatabase.parseNodeRows(this.db.prepare(sql).all(...args));
|
|
1177
2235
|
}
|
|
1178
2236
|
getAllConnections() {
|
|
1179
2237
|
const stmt = this.db.prepare('SELECT * FROM node_connections');
|
|
@@ -1188,17 +2246,17 @@ class DevMindDatabase {
|
|
|
1188
2246
|
// All pure queries/graph traversal, no mutation, no LLM calls.
|
|
1189
2247
|
/** Nodes whose total (in + out) connection degree meets/exceeds `threshold` — architectural bottleneck candidates. */
|
|
1190
2248
|
getGodEntities(threshold = 15) {
|
|
1191
|
-
const stmt = this.db.prepare(`
|
|
1192
|
-
SELECT * FROM (
|
|
1193
|
-
SELECT n.id, n.name, n.file_path, (
|
|
1194
|
-
(SELECT COUNT(*) FROM node_connections c WHERE c.source_node_id = n.id) +
|
|
1195
|
-
(SELECT COUNT(*) FROM node_connections c WHERE c.target_node_id = n.id)
|
|
1196
|
-
) AS degree
|
|
1197
|
-
FROM nodes n
|
|
1198
|
-
WHERE n.deprecated = 0
|
|
1199
|
-
)
|
|
1200
|
-
WHERE degree >= ?
|
|
1201
|
-
ORDER BY degree DESC
|
|
2249
|
+
const stmt = this.db.prepare(`
|
|
2250
|
+
SELECT * FROM (
|
|
2251
|
+
SELECT n.id, n.name, n.file_path, (
|
|
2252
|
+
(SELECT COUNT(*) FROM node_connections c WHERE c.source_node_id = n.id) +
|
|
2253
|
+
(SELECT COUNT(*) FROM node_connections c WHERE c.target_node_id = n.id)
|
|
2254
|
+
) AS degree
|
|
2255
|
+
FROM nodes n
|
|
2256
|
+
WHERE n.deprecated = 0
|
|
2257
|
+
)
|
|
2258
|
+
WHERE degree >= ?
|
|
2259
|
+
ORDER BY degree DESC
|
|
1202
2260
|
`);
|
|
1203
2261
|
return stmt.all(threshold);
|
|
1204
2262
|
}
|
|
@@ -1216,6 +2274,12 @@ class DevMindDatabase {
|
|
|
1216
2274
|
const stack = [];
|
|
1217
2275
|
const onStack = new Set();
|
|
1218
2276
|
const dfs = (node) => {
|
|
2277
|
+
/* istanbul ignore if -- both of dfs's call sites (the outer `for` loop below, and the
|
|
2278
|
+
`for (const next of ...)` loop a few lines down) already check `cycles.length >=
|
|
2279
|
+
maxCycles` immediately before every single call to `dfs(...)`, so this repeats a guard
|
|
2280
|
+
that has always already passed by the time control reaches here. Kept as a real guard
|
|
2281
|
+
against a future call site that skips that pre-check, not because today's two call
|
|
2282
|
+
sites can reach it. */
|
|
1219
2283
|
if (cycles.length >= maxCycles)
|
|
1220
2284
|
return;
|
|
1221
2285
|
if (onStack.has(node)) {
|
|
@@ -1246,10 +2310,10 @@ class DevMindDatabase {
|
|
|
1246
2310
|
}
|
|
1247
2311
|
/** node_connections rows whose source or target no longer exists in `nodes` (broken by a non-transactional delete, or a sync race). */
|
|
1248
2312
|
getDanglingEdges() {
|
|
1249
|
-
const stmt = this.db.prepare(`
|
|
1250
|
-
SELECT * FROM node_connections
|
|
1251
|
-
WHERE source_node_id NOT IN (SELECT id FROM nodes)
|
|
1252
|
-
OR target_node_id NOT IN (SELECT id FROM nodes)
|
|
2313
|
+
const stmt = this.db.prepare(`
|
|
2314
|
+
SELECT * FROM node_connections
|
|
2315
|
+
WHERE source_node_id NOT IN (SELECT id FROM nodes)
|
|
2316
|
+
OR target_node_id NOT IN (SELECT id FROM nodes)
|
|
1253
2317
|
`);
|
|
1254
2318
|
return stmt.all();
|
|
1255
2319
|
}
|
|
@@ -1259,12 +2323,12 @@ class DevMindDatabase {
|
|
|
1259
2323
|
}
|
|
1260
2324
|
/** Node ids that differ only by case — a real collision risk on Windows's case-insensitive filesystem. */
|
|
1261
2325
|
getDuplicateNodeIds() {
|
|
1262
|
-
const stmt = this.db.prepare(`
|
|
1263
|
-
SELECT LOWER(id) AS lower_id, GROUP_CONCAT(id, '|') AS ids
|
|
1264
|
-
FROM nodes
|
|
1265
|
-
WHERE deprecated = 0
|
|
1266
|
-
GROUP BY lower_id
|
|
1267
|
-
HAVING COUNT(*) > 1
|
|
2326
|
+
const stmt = this.db.prepare(`
|
|
2327
|
+
SELECT LOWER(id) AS lower_id, GROUP_CONCAT(id, '|') AS ids
|
|
2328
|
+
FROM nodes
|
|
2329
|
+
WHERE deprecated = 0
|
|
2330
|
+
GROUP BY lower_id
|
|
2331
|
+
HAVING COUNT(*) > 1
|
|
1268
2332
|
`);
|
|
1269
2333
|
const rows = stmt.all();
|
|
1270
2334
|
return rows.map(r => ({ lowerId: r.lower_id, ids: r.ids.split('|') }));
|
|
@@ -1301,7 +2365,76 @@ class DevMindDatabase {
|
|
|
1301
2365
|
workflowsDir() {
|
|
1302
2366
|
return path.join(path.dirname(this.dbPath), 'workflows');
|
|
1303
2367
|
}
|
|
1304
|
-
/**
|
|
2368
|
+
/**
|
|
2369
|
+
* One-time backfill of `node_ids`/`reasoning` for steps written before v2, resolving each old
|
|
2370
|
+
* `history_ids` entry to the node it belongs to. Runs at open, right after the ALTERs, and is
|
|
2371
|
+
* self-limiting: it only touches rows where `node_ids IS NULL AND history_ids IS NOT NULL`, so
|
|
2372
|
+
* the second open finds nothing and the query costs one indexed scan.
|
|
2373
|
+
*
|
|
2374
|
+
* Best-effort by nature. Because of the 1-hour history merge, an old step's `history_ids` can
|
|
2375
|
+
* include rows an ADJACENT commit created, so a backfilled node list can be broader than what
|
|
2376
|
+
* that step actually touched. That is acceptable for pre-v2 rows — they were already imprecise,
|
|
2377
|
+
* which is exactly why the format changed — but it is why nothing presents backfilled data as
|
|
2378
|
+
* exact. A step whose history rows have since been pruned keeps its `summary` and gets an empty
|
|
2379
|
+
* list rather than being skipped, so it still appears on the timeline.
|
|
2380
|
+
*/
|
|
2381
|
+
backfillWorkflowStepNodeIds() {
|
|
2382
|
+
try {
|
|
2383
|
+
const stale = this.db
|
|
2384
|
+
.prepare(`SELECT id, history_ids FROM workflow_steps WHERE node_ids IS NULL AND history_ids IS NOT NULL`)
|
|
2385
|
+
.all();
|
|
2386
|
+
if (stale.length === 0)
|
|
2387
|
+
return;
|
|
2388
|
+
const lookup = this.db.prepare('SELECT node_id, reasoning FROM history WHERE id = ?');
|
|
2389
|
+
const update = this.db.prepare('UPDATE workflow_steps SET node_ids = ?, reasoning = COALESCE(reasoning, ?) WHERE id = ?');
|
|
2390
|
+
const run = this.db.transaction((rows) => {
|
|
2391
|
+
for (const row of rows) {
|
|
2392
|
+
let ids;
|
|
2393
|
+
try {
|
|
2394
|
+
ids = JSON.parse(row.history_ids);
|
|
2395
|
+
}
|
|
2396
|
+
catch {
|
|
2397
|
+
ids = [];
|
|
2398
|
+
}
|
|
2399
|
+
const nodeIds = [];
|
|
2400
|
+
let reasoning = null;
|
|
2401
|
+
if (Array.isArray(ids)) {
|
|
2402
|
+
for (const historyId of ids) {
|
|
2403
|
+
const hit = lookup.get(String(historyId));
|
|
2404
|
+
if (!hit)
|
|
2405
|
+
continue;
|
|
2406
|
+
if (!nodeIds.includes(hit.node_id))
|
|
2407
|
+
nodeIds.push(hit.node_id);
|
|
2408
|
+
if (reasoning === null && hit.reasoning)
|
|
2409
|
+
reasoning = hit.reasoning;
|
|
2410
|
+
}
|
|
2411
|
+
}
|
|
2412
|
+
update.run(JSON.stringify(nodeIds), reasoning, row.id);
|
|
2413
|
+
}
|
|
2414
|
+
});
|
|
2415
|
+
run(stale);
|
|
2416
|
+
}
|
|
2417
|
+
catch {
|
|
2418
|
+
// A brain mid-migration (or a workflow_steps table that predates history_ids entirely)
|
|
2419
|
+
// must not block opening the DB — the steps simply stay on their old shape.
|
|
2420
|
+
}
|
|
2421
|
+
}
|
|
2422
|
+
/**
|
|
2423
|
+
* Serializes the workflow + its steps + artifact index to disk so teammates can sync it via git.
|
|
2424
|
+
*
|
|
2425
|
+
* Written as TWO files, and the split is the whole point:
|
|
2426
|
+
*
|
|
2427
|
+
* - `workflow.json` keeps the shape a v1 client understands, so an older build reading it loses
|
|
2428
|
+
* nothing it ever had.
|
|
2429
|
+
* - `v2.json` holds everything v1 has no field for (`archived`, and per-step `reasoning` /
|
|
2430
|
+
* `node_ids` / `doc_paths`).
|
|
2431
|
+
*
|
|
2432
|
+
* A single file could not be made safe. `devsmind sync` calls `syncToDisk`, which re-serializes
|
|
2433
|
+
* every workflow.json from whatever columns the local build knows about — so a teammate who
|
|
2434
|
+
* pulls on an older version and syncs would rewrite every workflow and silently strip the new
|
|
2435
|
+
* fields, then commit that loss for everyone. An older build has no idea `v2.json` exists, so it
|
|
2436
|
+
* cannot rewrite it; the data survives the round trip and is merged back on the next read.
|
|
2437
|
+
*/
|
|
1305
2438
|
writeWorkflowToDisk(workflowId) {
|
|
1306
2439
|
try {
|
|
1307
2440
|
const workflow = this.db.prepare('SELECT * FROM workflows WHERE id = ?').get(workflowId);
|
|
@@ -1309,21 +2442,21 @@ class DevMindDatabase {
|
|
|
1309
2442
|
return;
|
|
1310
2443
|
const steps = this.db.prepare('SELECT * FROM workflow_steps WHERE workflow_id = ? ORDER BY step_index ASC').all(workflowId);
|
|
1311
2444
|
const artifacts = this.db.prepare('SELECT * FROM workflow_artifacts WHERE workflow_id = ? ORDER BY created_at ASC').all(workflowId);
|
|
1312
|
-
const activeId = this.getSystemMeta('active_workflow_id');
|
|
1313
2445
|
const data = {
|
|
2446
|
+
schema_version: exports.WORKFLOW_SCHEMA_VERSION,
|
|
1314
2447
|
id: workflow.id,
|
|
1315
2448
|
name: workflow.name,
|
|
1316
2449
|
description: workflow.description,
|
|
1317
|
-
|
|
2450
|
+
archived: workflow.archived ? 1 : 0,
|
|
1318
2451
|
created_at: workflow.created_at,
|
|
1319
2452
|
updated_at: workflow.updated_at,
|
|
1320
|
-
is_active: activeId === workflowId,
|
|
1321
2453
|
steps: steps.map(s => ({
|
|
1322
2454
|
id: s.id,
|
|
1323
2455
|
step_index: s.step_index,
|
|
1324
2456
|
summary: s.summary,
|
|
1325
|
-
|
|
1326
|
-
|
|
2457
|
+
reasoning: s.reasoning,
|
|
2458
|
+
node_ids: s.node_ids,
|
|
2459
|
+
doc_paths: s.doc_paths,
|
|
1327
2460
|
session_id: s.session_id,
|
|
1328
2461
|
created_at: s.created_at
|
|
1329
2462
|
})),
|
|
@@ -1339,86 +2472,123 @@ class DevMindDatabase {
|
|
|
1339
2472
|
const dir = path.join(this.workflowsDir(), workflowId);
|
|
1340
2473
|
fs.mkdirSync(dir, { recursive: true });
|
|
1341
2474
|
fs.writeFileSync(path.join(dir, 'workflow.json'), JSON.stringify(data, null, 2), 'utf-8');
|
|
2475
|
+
// The v1-invisible half. Keyed by step id rather than positional, so it still merges
|
|
2476
|
+
// correctly onto a workflow.json an older build reordered or rewrote.
|
|
2477
|
+
const sidecar = {
|
|
2478
|
+
schema_version: exports.WORKFLOW_SCHEMA_VERSION,
|
|
2479
|
+
archived: workflow.archived ? 1 : 0,
|
|
2480
|
+
steps: Object.fromEntries(steps
|
|
2481
|
+
.filter(s => s.reasoning || s.node_ids || s.doc_paths)
|
|
2482
|
+
.map(s => [s.id, { reasoning: s.reasoning, node_ids: s.node_ids, doc_paths: s.doc_paths }]))
|
|
2483
|
+
};
|
|
2484
|
+
fs.writeFileSync(path.join(dir, exports.WORKFLOW_SIDECAR_FILE), JSON.stringify(sidecar, null, 2), 'utf-8');
|
|
1342
2485
|
}
|
|
1343
2486
|
catch (err) {
|
|
1344
2487
|
console.warn('⚠️ DevsMind: Failed to write workflow JSON to disk:', err);
|
|
1345
2488
|
}
|
|
1346
2489
|
}
|
|
2490
|
+
/**
|
|
2491
|
+
* Creates a workflow. Note what it deliberately does NOT do any more: set a global "active"
|
|
2492
|
+
* pointer. Which workflow you are working on is a property of YOUR session, held locally — a
|
|
2493
|
+
* workflow is a shared record, and one shared pointer meant two sessions (or two teammates,
|
|
2494
|
+
* since the pointer synced through git) silently stole it from each other mid-work.
|
|
2495
|
+
*/
|
|
1347
2496
|
createWorkflow(name, description) {
|
|
1348
2497
|
const id = `wf_${crypto.randomUUID()}`;
|
|
1349
2498
|
const now = new Date().toISOString();
|
|
1350
|
-
this.db.prepare(`
|
|
1351
|
-
INSERT INTO workflows (id, name, description,
|
|
1352
|
-
VALUES (?, ?, ?,
|
|
2499
|
+
this.db.prepare(`
|
|
2500
|
+
INSERT INTO workflows (id, name, description, archived, created_at, updated_at)
|
|
2501
|
+
VALUES (?, ?, ?, 0, ?, ?)
|
|
1353
2502
|
`).run(id, name, description, now, now);
|
|
1354
|
-
this.setSystemMeta('active_workflow_id', id);
|
|
1355
2503
|
this.writeWorkflowToDisk(id);
|
|
1356
|
-
return { id, name, description,
|
|
2504
|
+
return { id, name, description, archived: 0, created_at: now, updated_at: now };
|
|
1357
2505
|
}
|
|
1358
2506
|
getWorkflow(id) {
|
|
1359
2507
|
const row = this.db.prepare('SELECT * FROM workflows WHERE id = ?').get(id);
|
|
1360
2508
|
return row || null;
|
|
1361
2509
|
}
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
2510
|
+
/**
|
|
2511
|
+
* Workflows newest-touched first — which is the ordering that replaces the old `status` field.
|
|
2512
|
+
* Live work floats up and abandoned threads sink on their own, so nothing has to be marked
|
|
2513
|
+
* "completed" by hand (nobody ever did, and a lifecycle field nobody maintains just lies).
|
|
2514
|
+
*
|
|
2515
|
+
* `query` matches name AND description, the search `searchWorkflows` never actually did: it
|
|
2516
|
+
* scanned step summaries and artifact names only, so looking a workflow up by its own name
|
|
2517
|
+
* returned nothing. Paging mirrors `listNodes` — `total` is the true count before the page.
|
|
2518
|
+
*/
|
|
2519
|
+
listWorkflows(opts) {
|
|
2520
|
+
const { where, params } = this.buildWorkflowFilterSql(opts);
|
|
2521
|
+
let sql = `SELECT * FROM workflows${where} ORDER BY updated_at DESC`;
|
|
2522
|
+
const args = [...params];
|
|
2523
|
+
if (opts?.limit !== undefined) {
|
|
2524
|
+
sql += ' LIMIT ? OFFSET ?';
|
|
2525
|
+
args.push(opts.limit, opts.offset ?? 0);
|
|
2526
|
+
}
|
|
2527
|
+
return this.db.prepare(sql).all(...args);
|
|
2528
|
+
}
|
|
2529
|
+
countWorkflows(opts) {
|
|
2530
|
+
const { where, params } = this.buildWorkflowFilterSql(opts);
|
|
2531
|
+
const row = this.db.prepare(`SELECT COUNT(*) AS c FROM workflows${where}`).get(...params);
|
|
2532
|
+
return row.c;
|
|
2533
|
+
}
|
|
2534
|
+
/** Shared WHERE builder, so a page and its `total` can never describe different criteria. */
|
|
2535
|
+
buildWorkflowFilterSql(opts) {
|
|
2536
|
+
let where = ' WHERE 1=1';
|
|
2537
|
+
const params = [];
|
|
2538
|
+
if (!opts?.includeArchived)
|
|
2539
|
+
where += ' AND archived = 0';
|
|
2540
|
+
if (opts?.query && opts.query.trim()) {
|
|
2541
|
+
where += " AND (LOWER(name) LIKE ? ESCAPE '\\' OR LOWER(description) LIKE ? ESCAPE '\\')";
|
|
2542
|
+
const like = `%${this.likeEscape(opts.query.trim().toLowerCase())}%`;
|
|
2543
|
+
params.push(like, like);
|
|
1369
2544
|
}
|
|
1370
|
-
return
|
|
1371
|
-
}
|
|
1372
|
-
/** Pauses the currently active workflow (if any) and clears the active pointer. */
|
|
1373
|
-
pauseWorkflow() {
|
|
1374
|
-
const active = this.getActiveWorkflow();
|
|
1375
|
-
if (!active)
|
|
1376
|
-
return null;
|
|
1377
|
-
const now = new Date().toISOString();
|
|
1378
|
-
this.db.prepare(`UPDATE workflows SET status = 'paused', updated_at = ? WHERE id = ?`).run(now, active.id);
|
|
1379
|
-
this.setSystemMeta('active_workflow_id', '');
|
|
1380
|
-
this.writeWorkflowToDisk(active.id);
|
|
1381
|
-
return { ...active, status: 'paused', updated_at: now };
|
|
1382
|
-
}
|
|
1383
|
-
/** Resumes `id`, auto-pausing whatever was previously active (only one workflow is active at a time). */
|
|
1384
|
-
resumeWorkflow(id) {
|
|
1385
|
-
const workflow = this.getWorkflow(id);
|
|
1386
|
-
if (!workflow)
|
|
1387
|
-
throw new Error(`Workflow not found: ${id}`);
|
|
1388
|
-
const currentActive = this.getActiveWorkflow();
|
|
1389
|
-
if (currentActive && currentActive.id !== id)
|
|
1390
|
-
this.pauseWorkflow();
|
|
1391
|
-
const now = new Date().toISOString();
|
|
1392
|
-
this.db.prepare(`UPDATE workflows SET status = 'active', updated_at = ? WHERE id = ?`).run(now, id);
|
|
1393
|
-
this.setSystemMeta('active_workflow_id', id);
|
|
1394
|
-
this.writeWorkflowToDisk(id);
|
|
1395
|
-
return { ...workflow, status: 'active', updated_at: now };
|
|
2545
|
+
return { where, params };
|
|
1396
2546
|
}
|
|
1397
|
-
|
|
2547
|
+
/**
|
|
2548
|
+
* Hides a workflow from the default listing. Deliberately NOT called "complete": a feature is
|
|
2549
|
+
* never finished, it just stops being worked on, and the old `completed` status promised a
|
|
2550
|
+
* lifecycle nobody maintained. Archiving claims only what it delivers, and is reversible.
|
|
2551
|
+
*/
|
|
2552
|
+
setWorkflowArchived(id, archived) {
|
|
1398
2553
|
const workflow = this.getWorkflow(id);
|
|
1399
2554
|
if (!workflow)
|
|
1400
2555
|
throw new Error(`Workflow not found: ${id}`);
|
|
1401
2556
|
const now = new Date().toISOString();
|
|
1402
|
-
this.db.prepare(
|
|
1403
|
-
if (this.getSystemMeta('active_workflow_id') === id)
|
|
1404
|
-
this.setSystemMeta('active_workflow_id', '');
|
|
2557
|
+
this.db.prepare('UPDATE workflows SET archived = ?, updated_at = ? WHERE id = ?').run(archived ? 1 : 0, now, id);
|
|
1405
2558
|
this.writeWorkflowToDisk(id);
|
|
1406
|
-
return { ...workflow,
|
|
2559
|
+
return { ...workflow, archived: archived ? 1 : 0, updated_at: now };
|
|
1407
2560
|
}
|
|
2561
|
+
/**
|
|
2562
|
+
* Appends one step. A step is either a COMMIT (summary + reasoning + the node ids it touched) or
|
|
2563
|
+
* a RESEARCH finding (summary + reasoning + the docs behind it, no nodes) — the second is the
|
|
2564
|
+
* only record of work that produced a decision but no code, which nothing else in DevsMind
|
|
2565
|
+
* captures: git has the diff and history has the per-node reasoning, but neither can tell you
|
|
2566
|
+
* what was evaluated and rejected.
|
|
2567
|
+
*
|
|
2568
|
+
* `reasoning` is stored, not joined from `history`, deliberately — see DbWorkflowStep.
|
|
2569
|
+
*/
|
|
1408
2570
|
addWorkflowStep(workflowId, opts) {
|
|
1409
2571
|
if (!this.getWorkflow(workflowId))
|
|
1410
2572
|
throw new Error(`Workflow not found: ${workflowId}`);
|
|
1411
2573
|
const id = crypto.randomUUID();
|
|
1412
2574
|
const now = new Date().toISOString();
|
|
1413
2575
|
const nextIndex = (this.db.prepare('SELECT MAX(step_index) AS m FROM workflow_steps WHERE workflow_id = ?').get(workflowId).m ?? 0) + 1;
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
2576
|
+
// Empty arrays store as NULL rather than "[]" so "this step touched nothing" and "this step
|
|
2577
|
+
// predates the column" read the same downstream — neither is a list worth rendering.
|
|
2578
|
+
const nodeIdsJson = opts.nodeIds && opts.nodeIds.length ? JSON.stringify(opts.nodeIds) : null;
|
|
2579
|
+
const docPathsJson = opts.docPaths && opts.docPaths.length ? JSON.stringify(opts.docPaths) : null;
|
|
2580
|
+
const reasoning = opts.reasoning || null;
|
|
2581
|
+
this.db.prepare(`
|
|
2582
|
+
INSERT INTO workflow_steps (id, workflow_id, step_index, summary, reasoning, node_ids, doc_paths, session_id, created_at)
|
|
2583
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
2584
|
+
`).run(id, workflowId, nextIndex, opts.summary, reasoning, nodeIdsJson, docPathsJson, opts.sessionId || null, now);
|
|
1419
2585
|
this.db.prepare(`UPDATE workflows SET updated_at = ? WHERE id = ?`).run(now, workflowId);
|
|
1420
2586
|
this.writeWorkflowToDisk(workflowId);
|
|
1421
|
-
return {
|
|
2587
|
+
return {
|
|
2588
|
+
id, workflow_id: workflowId, step_index: nextIndex, summary: opts.summary,
|
|
2589
|
+
reasoning, node_ids: nodeIdsJson, doc_paths: docPathsJson,
|
|
2590
|
+
session_id: opts.sessionId || null, created_at: now
|
|
2591
|
+
};
|
|
1422
2592
|
}
|
|
1423
2593
|
/** Writes `content` to `.devmind/workflows/<workflowId>/<artifactId>_<sourceName>` and records the DB row. */
|
|
1424
2594
|
addWorkflowArtifact(workflowId, opts) {
|
|
@@ -1431,32 +2601,38 @@ class DevMindDatabase {
|
|
|
1431
2601
|
fs.mkdirSync(dir, { recursive: true });
|
|
1432
2602
|
const filePath = path.join(dir, `${id}_${safeName}`);
|
|
1433
2603
|
fs.writeFileSync(filePath, opts.content, 'utf-8');
|
|
1434
|
-
this.db.prepare(`
|
|
1435
|
-
INSERT INTO workflow_artifacts (id, workflow_id, step_id, type, source_name, file_path, created_at)
|
|
1436
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
2604
|
+
this.db.prepare(`
|
|
2605
|
+
INSERT INTO workflow_artifacts (id, workflow_id, step_id, type, source_name, file_path, created_at)
|
|
2606
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1437
2607
|
`).run(id, workflowId, opts.stepId || null, opts.type, opts.sourceName, filePath, now);
|
|
1438
2608
|
this.db.prepare(`UPDATE workflows SET updated_at = ? WHERE id = ?`).run(now, workflowId);
|
|
1439
2609
|
this.writeWorkflowToDisk(workflowId);
|
|
1440
2610
|
return { id, workflow_id: workflowId, step_id: opts.stepId || null, type: opts.type, source_name: opts.sourceName, file_path: filePath, created_at: now };
|
|
1441
2611
|
}
|
|
2612
|
+
/**
|
|
2613
|
+
* The workflow's story: its steps in order, plus the docs attached to it.
|
|
2614
|
+
*
|
|
2615
|
+
* Paged, because this is now the ONLY read (it absorbed the old `workflow_get_steps`) and steps
|
|
2616
|
+
* carry their own reasoning, so an unbounded version of it would be the largest response the
|
|
2617
|
+
* server can produce. `steps_total` is exact regardless of the page — a short page must never
|
|
2618
|
+
* read as "that is the whole story".
|
|
2619
|
+
*
|
|
2620
|
+
* Artifact CONTENT is deliberately not returned. It used to be inlined whole, which on an
|
|
2621
|
+
* imported architecture doc is trivially tens of KB; the file path is enough, since the file is
|
|
2622
|
+
* on disk and the caller can read exactly the part it needs.
|
|
2623
|
+
*/
|
|
1442
2624
|
getWorkflowContext(id, opts) {
|
|
1443
2625
|
const workflow = this.getWorkflow(id);
|
|
1444
2626
|
if (!workflow)
|
|
1445
2627
|
throw new Error(`Workflow not found: ${id}`);
|
|
1446
|
-
const
|
|
1447
|
-
const
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
}
|
|
1455
|
-
catch {
|
|
1456
|
-
return a;
|
|
1457
|
-
}
|
|
1458
|
-
});
|
|
1459
|
-
return { workflow, steps, artifacts };
|
|
2628
|
+
const steps_total = this.db.prepare('SELECT COUNT(*) AS c FROM workflow_steps WHERE workflow_id = ?').get(id).c;
|
|
2629
|
+
const steps = this.getWorkflowSteps(id, opts);
|
|
2630
|
+
// `last_n` walks backwards from the end, so its offset is wherever that tail begins.
|
|
2631
|
+
const steps_offset = opts?.last_n && opts.last_n > 0
|
|
2632
|
+
? Math.max(0, steps_total - steps.length)
|
|
2633
|
+
: (opts?.offset ?? 0);
|
|
2634
|
+
const artifacts = this.db.prepare('SELECT * FROM workflow_artifacts WHERE workflow_id = ? ORDER BY created_at ASC').all(id);
|
|
2635
|
+
return { workflow, steps, steps_total, steps_offset, artifacts };
|
|
1460
2636
|
}
|
|
1461
2637
|
/**
|
|
1462
2638
|
* Returns steps for a workflow with optional pagination.
|
|
@@ -1476,99 +2652,13 @@ class DevMindDatabase {
|
|
|
1476
2652
|
}
|
|
1477
2653
|
return this.db.prepare('SELECT * FROM workflow_steps WHERE workflow_id = ? ORDER BY step_index ASC').all(workflowId);
|
|
1478
2654
|
}
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
const row = this.db.prepare('SELECT * FROM workflow_artifacts WHERE workflow_id = ? AND id = ?').get(workflowId, artifactId);
|
|
1487
|
-
if (!row)
|
|
1488
|
-
throw new Error(`Artifact not found: ${artifactId} in workflow ${workflowId}`);
|
|
1489
|
-
if (!fs.existsSync(row.file_path))
|
|
1490
|
-
throw new Error(`Artifact file missing on disk: ${row.file_path}`);
|
|
1491
|
-
const content = fs.readFileSync(row.file_path, 'utf-8');
|
|
1492
|
-
return { artifact: row, content };
|
|
1493
|
-
}
|
|
1494
|
-
/**
|
|
1495
|
-
* Full-text keyword search across all workflows' step summaries, pending_tasks,
|
|
1496
|
-
* and artifact source names. Optionally also searches artifact file content.
|
|
1497
|
-
* Returns a list of matches grouped by workflow.
|
|
1498
|
-
*/
|
|
1499
|
-
searchWorkflows(query, opts) {
|
|
1500
|
-
// Escaped so a query containing '%' or '_' matches those characters literally instead of
|
|
1501
|
-
// acting as SQL LIKE wildcards — otherwise `query: "%"` matches every row in the project.
|
|
1502
|
-
const lq = `%${this.likeEscape(query.toLowerCase())}%`;
|
|
1503
|
-
// Find matching steps
|
|
1504
|
-
const matchedStepRows = this.db.prepare(`
|
|
1505
|
-
SELECT ws.* FROM workflow_steps ws
|
|
1506
|
-
JOIN workflows w ON w.id = ws.workflow_id
|
|
1507
|
-
WHERE (LOWER(ws.summary) LIKE ? ESCAPE '\\' OR LOWER(IFNULL(ws.pending_tasks,'')) LIKE ? ESCAPE '\\')
|
|
1508
|
-
${opts?.status ? 'AND w.status = ?' : ''}
|
|
1509
|
-
ORDER BY ws.workflow_id, ws.step_index ASC
|
|
1510
|
-
`).all(...(opts?.status ? [lq, lq, opts.status] : [lq, lq]));
|
|
1511
|
-
// Find matching artifacts by source_name
|
|
1512
|
-
const matchedArtifactRows = this.db.prepare(`
|
|
1513
|
-
SELECT wa.* FROM workflow_artifacts wa
|
|
1514
|
-
JOIN workflows w ON w.id = wa.workflow_id
|
|
1515
|
-
WHERE LOWER(wa.source_name) LIKE ? ESCAPE '\\'
|
|
1516
|
-
${opts?.status ? 'AND w.status = ?' : ''}
|
|
1517
|
-
ORDER BY wa.workflow_id, wa.created_at ASC
|
|
1518
|
-
`).all(...(opts?.status ? [lq, opts.status] : [lq]));
|
|
1519
|
-
// If content search requested, also scan artifact files
|
|
1520
|
-
const contentMatchedArtifactIds = new Set();
|
|
1521
|
-
const artifactContentSnippets = new Map();
|
|
1522
|
-
if (opts?.include_artifact_content) {
|
|
1523
|
-
const allArtifacts = this.db.prepare(`SELECT wa.* FROM workflow_artifacts wa JOIN workflows w ON w.id = wa.workflow_id${opts.status ? ' WHERE w.status = ?' : ''}`).all(...(opts.status ? [opts.status] : []));
|
|
1524
|
-
const lqPlain = query.toLowerCase();
|
|
1525
|
-
for (const a of allArtifacts) {
|
|
1526
|
-
if (contentMatchedArtifactIds.has(a.id))
|
|
1527
|
-
continue;
|
|
1528
|
-
try {
|
|
1529
|
-
if (fs.existsSync(a.file_path)) {
|
|
1530
|
-
const text = fs.readFileSync(a.file_path, 'utf-8');
|
|
1531
|
-
const idx = text.toLowerCase().indexOf(lqPlain);
|
|
1532
|
-
if (idx !== -1) {
|
|
1533
|
-
contentMatchedArtifactIds.add(a.id);
|
|
1534
|
-
const start = Math.max(0, idx - 80);
|
|
1535
|
-
const end = Math.min(text.length, idx + query.length + 80);
|
|
1536
|
-
artifactContentSnippets.set(a.id, (start > 0 ? '…' : '') + text.slice(start, end) + (end < text.length ? '…' : ''));
|
|
1537
|
-
}
|
|
1538
|
-
}
|
|
1539
|
-
}
|
|
1540
|
-
catch { /* skip unreadable */ }
|
|
1541
|
-
}
|
|
1542
|
-
}
|
|
1543
|
-
// Collect all relevant workflow IDs
|
|
1544
|
-
const workflowIdSet = new Set([
|
|
1545
|
-
...matchedStepRows.map(s => s.workflow_id),
|
|
1546
|
-
...matchedArtifactRows.map(a => a.workflow_id),
|
|
1547
|
-
...Array.from(contentMatchedArtifactIds).map(id => {
|
|
1548
|
-
const r = this.db.prepare('SELECT workflow_id FROM workflow_artifacts WHERE id = ?').get(id);
|
|
1549
|
-
return r?.workflow_id || '';
|
|
1550
|
-
}).filter(Boolean)
|
|
1551
|
-
]);
|
|
1552
|
-
const results = [];
|
|
1553
|
-
for (const wid of workflowIdSet) {
|
|
1554
|
-
const workflow = this.getWorkflow(wid);
|
|
1555
|
-
if (!workflow)
|
|
1556
|
-
continue;
|
|
1557
|
-
const steps = matchedStepRows.filter(s => s.workflow_id === wid);
|
|
1558
|
-
const artByName = matchedArtifactRows.filter(a => a.workflow_id === wid);
|
|
1559
|
-
const artByContent = opts?.include_artifact_content
|
|
1560
|
-
? this.db.prepare('SELECT * FROM workflow_artifacts WHERE workflow_id = ?').all(wid).filter(a => contentMatchedArtifactIds.has(a.id) && !artByName.find(x => x.id === a.id))
|
|
1561
|
-
: [];
|
|
1562
|
-
const allArtifacts = [
|
|
1563
|
-
...artByName.map(a => ({ ...a, content_snippet: artifactContentSnippets.get(a.id) })),
|
|
1564
|
-
...artByContent.map(a => ({ ...a, content_snippet: artifactContentSnippets.get(a.id) }))
|
|
1565
|
-
];
|
|
1566
|
-
results.push({ workflow, matched_steps: steps, matched_artifacts: allArtifacts });
|
|
1567
|
-
}
|
|
1568
|
-
// Sort by most recently updated workflow first
|
|
1569
|
-
results.sort((a, b) => b.workflow.updated_at.localeCompare(a.workflow.updated_at));
|
|
1570
|
-
return results;
|
|
1571
|
-
}
|
|
2655
|
+
// NOTE: `readWorkflowArtifact` and `searchWorkflows` were removed here.
|
|
2656
|
+
// Artifacts are referenced by PATH now (workflow_add_step's doc_paths, plus the file paths
|
|
2657
|
+
// getWorkflowContext already returns), so nothing needs the DB to read a file back for it —
|
|
2658
|
+
// and inlining whole imported docs was the single largest thing a workflow response could emit.
|
|
2659
|
+
// searchWorkflows was replaced by listWorkflows({ query }): it scanned step summaries and
|
|
2660
|
+
// artifact names but NOT workflow name/description, so looking a workflow up by its own name
|
|
2661
|
+
// returned nothing — the one search anybody actually tries.
|
|
1572
2662
|
/**
|
|
1573
2663
|
* Imports an existing flow/architecture doc as a paused workflow (not active — importing
|
|
1574
2664
|
* a doc isn't the same as declaring active work). Idempotent on `name`: re-importing the
|
|
@@ -1591,14 +2681,14 @@ class DevMindDatabase {
|
|
|
1591
2681
|
return { workflow: { ...existing, description, updated_at: now }, created: false };
|
|
1592
2682
|
}
|
|
1593
2683
|
const id = `wf_${crypto.randomUUID()}`;
|
|
1594
|
-
this.db.prepare(`
|
|
1595
|
-
INSERT INTO workflows (id, name, description,
|
|
1596
|
-
VALUES (?, ?, ?,
|
|
2684
|
+
this.db.prepare(`
|
|
2685
|
+
INSERT INTO workflows (id, name, description, archived, created_at, updated_at)
|
|
2686
|
+
VALUES (?, ?, ?, 0, ?, ?)
|
|
1597
2687
|
`).run(id, name, description, now, now);
|
|
1598
2688
|
this.addWorkflowStep(id, { summary: `Imported existing flow documentation: ${sourceFileName}` });
|
|
1599
2689
|
this.addWorkflowArtifact(id, { type: 'imported_doc', sourceName: sourceFileName, content });
|
|
1600
2690
|
// writeWorkflowToDisk is already called inside addWorkflowArtifact/addWorkflowStep above
|
|
1601
|
-
return { workflow: { id, name, description,
|
|
2691
|
+
return { workflow: { id, name, description, archived: 0, created_at: now, updated_at: now }, created: true };
|
|
1602
2692
|
}
|
|
1603
2693
|
static SPURIOUS_NODE_NAMES = new Set([
|
|
1604
2694
|
'promise', 'map', 'set', 'json', 'console', 'error', 'object', 'function', 'array', 'string', 'number', 'boolean', 'regexp', 'date', 'math',
|
|
@@ -1610,9 +2700,9 @@ class DevMindDatabase {
|
|
|
1610
2700
|
* analyze`'s dry-run report (which just lists it). Never mutates the DB.
|
|
1611
2701
|
*/
|
|
1612
2702
|
findSpuriousAndMissingFileNodes(workspaceRoot) {
|
|
1613
|
-
const stmt = this.db.prepare(`
|
|
1614
|
-
SELECT id, name, file_path FROM nodes
|
|
1615
|
-
WHERE deprecated = 0
|
|
2703
|
+
const stmt = this.db.prepare(`
|
|
2704
|
+
SELECT id, name, file_path FROM nodes
|
|
2705
|
+
WHERE deprecated = 0
|
|
1616
2706
|
`);
|
|
1617
2707
|
const candidates = stmt.all();
|
|
1618
2708
|
const spurious = [];
|
|
@@ -1695,7 +2785,10 @@ class DevMindDatabase {
|
|
|
1695
2785
|
return {
|
|
1696
2786
|
...row,
|
|
1697
2787
|
code_snapshot: data.code_snapshot || '',
|
|
1698
|
-
reasoning: typeof data.reasoning === 'string' ? data.reasoning : formatReasoning(data.reasoning || '')
|
|
2788
|
+
reasoning: typeof data.reasoning === 'string' ? data.reasoning : formatReasoning(data.reasoning || ''),
|
|
2789
|
+
// Absent in every entry written before the edit trail existed — an empty trail is the
|
|
2790
|
+
// honest answer there: nothing to diff, nothing to revert.
|
|
2791
|
+
edits: Array.isArray(data.edits) ? data.edits : []
|
|
1699
2792
|
};
|
|
1700
2793
|
}
|
|
1701
2794
|
}
|
|
@@ -1705,10 +2798,11 @@ class DevMindDatabase {
|
|
|
1705
2798
|
return {
|
|
1706
2799
|
...row,
|
|
1707
2800
|
code_snapshot: '',
|
|
1708
|
-
reasoning: ''
|
|
2801
|
+
reasoning: '',
|
|
2802
|
+
edits: []
|
|
1709
2803
|
};
|
|
1710
2804
|
}
|
|
1711
|
-
writeHistoryToDisk(id, nodeId, sessionId, createdAt, updatedAt, codeSnapshot, reasoning) {
|
|
2805
|
+
writeHistoryToDisk(id, nodeId, sessionId, createdAt, updatedAt, codeSnapshot, reasoning, edits = []) {
|
|
1712
2806
|
try {
|
|
1713
2807
|
const historyDir = path.join(path.dirname(this.dbPath), 'history');
|
|
1714
2808
|
if (!fs.existsSync(historyDir)) {
|
|
@@ -1729,7 +2823,8 @@ class DevMindDatabase {
|
|
|
1729
2823
|
created_at: createdAt,
|
|
1730
2824
|
updated_at: updatedAt,
|
|
1731
2825
|
code_snapshot: codeSnapshot,
|
|
1732
|
-
reasoning
|
|
2826
|
+
reasoning,
|
|
2827
|
+
edits
|
|
1733
2828
|
};
|
|
1734
2829
|
const filePath = path.join(historyDir, `${id}.json`);
|
|
1735
2830
|
fs.writeFileSync(filePath, JSON.stringify(data, null, 2), 'utf-8');
|
|
@@ -1738,6 +2833,10 @@ class DevMindDatabase {
|
|
|
1738
2833
|
console.warn('⚠️ SQLite warning: Failed to write history JSON to disk:', err);
|
|
1739
2834
|
}
|
|
1740
2835
|
}
|
|
2836
|
+
/** The configured developer identity (`.devmind/.env`'s DEVELOPER_NAME), or null if unset. */
|
|
2837
|
+
getDeveloperName() {
|
|
2838
|
+
return this.context?.developer?.name || null;
|
|
2839
|
+
}
|
|
1741
2840
|
toRepoRelativePath(absolutePath) {
|
|
1742
2841
|
if (!absolutePath || !this.context)
|
|
1743
2842
|
return absolutePath;
|
|
@@ -1841,8 +2940,16 @@ class DevMindDatabase {
|
|
|
1841
2940
|
// Fallback: resolve relative to workspace root
|
|
1842
2941
|
return (0, config_1.canonicalizePath)(this.clampToRoot(workspaceRoot, path.resolve(workspaceRoot, repoRelativePath)));
|
|
1843
2942
|
}
|
|
1844
|
-
syncFromDisk() {
|
|
2943
|
+
syncFromDisk(onProgress) {
|
|
1845
2944
|
this.db.pragma('foreign_keys = OFF');
|
|
2945
|
+
// Captured BEFORE the walk starts (not after it finishes) so a file touched WHILE this sync
|
|
2946
|
+
// is running is still mtime >= this timestamp and gets correctly picked up on the NEXT sync,
|
|
2947
|
+
// rather than silently missed by a checkpoint that raced ahead of it.
|
|
2948
|
+
const syncStartedAtMs = Date.now();
|
|
2949
|
+
// 0 (never synced before, or system_meta was wiped by resetAll) disables every mtime skip
|
|
2950
|
+
// below unconditionally — every real file's mtimeMs is a large positive epoch value, so
|
|
2951
|
+
// `mtimeMs < 0` is never true and the first pass always processes everything, as before.
|
|
2952
|
+
const lastSyncedAtMs = Number(this.getSystemMeta('last_sync_checkpoint_ms')) || 0;
|
|
1846
2953
|
try {
|
|
1847
2954
|
const workspaceRoot = path.dirname(this.dbPath);
|
|
1848
2955
|
// 0. Auto-heal any legacy relative path records in SQLite.
|
|
@@ -1879,21 +2986,37 @@ class DevMindDatabase {
|
|
|
1879
2986
|
if (files.length > 0) {
|
|
1880
2987
|
const checkHistoryStmt = this.db.prepare('SELECT id FROM history WHERE id = ?');
|
|
1881
2988
|
const checkNodeStmt = this.db.prepare('SELECT id FROM nodes WHERE id = ?');
|
|
1882
|
-
const insertNodeStmt = this.db.prepare(`
|
|
1883
|
-
INSERT INTO nodes (id, type, name, file_path, signature, deprecated)
|
|
1884
|
-
VALUES (?, ?, ?, ?, ?, 0)
|
|
2989
|
+
const insertNodeStmt = this.db.prepare(`
|
|
2990
|
+
INSERT INTO nodes (id, type, name, file_path, signature, deprecated)
|
|
2991
|
+
VALUES (?, ?, ?, ?, ?, 0)
|
|
1885
2992
|
`);
|
|
1886
|
-
const insertHistoryStmt = this.db.prepare(`
|
|
1887
|
-
INSERT INTO history (id, node_id, session_id, created_at, updated_at, code_snapshot, reasoning)
|
|
1888
|
-
VALUES (?, ?, ?, ?, ?, '', ?)
|
|
2993
|
+
const insertHistoryStmt = this.db.prepare(`
|
|
2994
|
+
INSERT INTO history (id, node_id, session_id, created_at, updated_at, code_snapshot, reasoning)
|
|
2995
|
+
VALUES (?, ?, ?, ?, ?, '', ?)
|
|
1889
2996
|
`);
|
|
1890
2997
|
const syncHistoryTx = this.db.transaction(() => {
|
|
2998
|
+
let done = 0;
|
|
1891
2999
|
for (const file of files) {
|
|
3000
|
+
done++;
|
|
3001
|
+
if (onProgress && DevMindDatabase.shouldReport(done, files.length))
|
|
3002
|
+
onProgress('history', done, files.length);
|
|
1892
3003
|
try {
|
|
3004
|
+
// History files are immutable and always named `${id}.json` (every writer uses
|
|
3005
|
+
// this convention — see updateHistory/eraseLastEdit/etc.), so an already-synced
|
|
3006
|
+
// file can be identified from its NAME alone, before ever reading it. This is the
|
|
3007
|
+
// single biggest cost in syncFromDisk: history/ grows one file per edit forever,
|
|
3008
|
+
// so on a mature repo the overwhelming majority of files here are already synced
|
|
3009
|
+
// and this check-before-read turns them from a readFileSync+JSON.parse into a
|
|
3010
|
+
// single indexed SQLite lookup.
|
|
3011
|
+
if (checkHistoryStmt.get(file.slice(0, -'.json'.length)))
|
|
3012
|
+
continue;
|
|
1893
3013
|
const filePath = path.join(historyDir, file);
|
|
1894
3014
|
const data = JSON.parse(fs.readFileSync(filePath, 'utf-8'));
|
|
1895
3015
|
if (!data.id || !data.node_id)
|
|
1896
3016
|
continue;
|
|
3017
|
+
// Kept as a safety net in case a hand-written/legacy file's internal `id` ever
|
|
3018
|
+
// differs from its filename — the fast path above is an optimization, not a
|
|
3019
|
+
// replacement for this correctness check.
|
|
1897
3020
|
if (checkHistoryStmt.get(data.id))
|
|
1898
3021
|
continue;
|
|
1899
3022
|
if (!checkNodeStmt.get(data.node_id) && data.node_metadata) {
|
|
@@ -1933,18 +3056,31 @@ class DevMindDatabase {
|
|
|
1933
3056
|
if (jsonFiles.length > 0) {
|
|
1934
3057
|
const deleteNodesForFileStmt = this.db.prepare('DELETE FROM nodes WHERE file_path = ?');
|
|
1935
3058
|
const deleteConnsForNodesStmt = this.db.prepare('DELETE FROM node_connections WHERE source_node_id = ?');
|
|
1936
|
-
const insertNodeStmt = this.db.prepare(`
|
|
1937
|
-
INSERT OR REPLACE INTO nodes (id, type, name, file_path, signature, deprecated)
|
|
1938
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
3059
|
+
const insertNodeStmt = this.db.prepare(`
|
|
3060
|
+
INSERT OR REPLACE INTO nodes (id, type, name, file_path, signature, description, aliases, deprecated)
|
|
3061
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1939
3062
|
`);
|
|
1940
|
-
const insertConnStmt = this.db.prepare(`
|
|
1941
|
-
INSERT OR IGNORE INTO node_connections (source_node_id, target_node_id)
|
|
1942
|
-
VALUES (?, ?)
|
|
3063
|
+
const insertConnStmt = this.db.prepare(`
|
|
3064
|
+
INSERT OR IGNORE INTO node_connections (source_node_id, target_node_id)
|
|
3065
|
+
VALUES (?, ?)
|
|
1943
3066
|
`);
|
|
1944
3067
|
// Transaction for fast batch syncing
|
|
1945
3068
|
const syncGraphTx = this.db.transaction(() => {
|
|
3069
|
+
let done = 0;
|
|
1946
3070
|
for (const file of jsonFiles) {
|
|
3071
|
+
done++;
|
|
3072
|
+
if (onProgress && DevMindDatabase.shouldReport(done, jsonFiles.length))
|
|
3073
|
+
onProgress('graph', done, jsonFiles.length);
|
|
1947
3074
|
try {
|
|
3075
|
+
// Unlike history/, a graph JSON gets REWRITTEN in place on every edit to its
|
|
3076
|
+
// source file (same path, new content) — so identity alone can't tell us "already
|
|
3077
|
+
// synced." mtime can: if this file hasn't changed since the last successful sync,
|
|
3078
|
+
// the DB is already current for it (a `git pull`/checkout always bumps the mtime
|
|
3079
|
+
// of every file it actually changed, so this correctly still processes exactly
|
|
3080
|
+
// what came in on a pull). Skips the readFileSync+JSON.parse+delete+reinsert
|
|
3081
|
+
// entirely for the — typically overwhelming — majority of untouched files.
|
|
3082
|
+
if (fs.statSync(file).mtimeMs < lastSyncedAtMs)
|
|
3083
|
+
continue;
|
|
1948
3084
|
const data = JSON.parse(fs.readFileSync(file, 'utf-8'));
|
|
1949
3085
|
if (!data.file_path)
|
|
1950
3086
|
continue;
|
|
@@ -1960,7 +3096,8 @@ class DevMindDatabase {
|
|
|
1960
3096
|
const nodes = data.nodes || [];
|
|
1961
3097
|
for (const n of nodes) {
|
|
1962
3098
|
deleteConnsForNodesStmt.run(n.id);
|
|
1963
|
-
|
|
3099
|
+
const aliasesJson = Array.isArray(n.aliases) ? JSON.stringify(n.aliases) : '[]';
|
|
3100
|
+
insertNodeStmt.run(n.id, n.type, n.name, fileAbsPath, n.signature || null, n.description || null, aliasesJson, n.deprecated ? 1 : 0);
|
|
1964
3101
|
}
|
|
1965
3102
|
// Insert connections
|
|
1966
3103
|
const connections = data.connections || [];
|
|
@@ -1976,29 +3113,99 @@ class DevMindDatabase {
|
|
|
1976
3113
|
syncGraphTx();
|
|
1977
3114
|
}
|
|
1978
3115
|
}
|
|
3116
|
+
// 2.5. Sync Vector JSONs — MUST run after the graph pass (2), since that pass just
|
|
3117
|
+
// deleted and re-inserted the current node set; vectors are reconciled against nodes as
|
|
3118
|
+
// they now stand, not as they stood before this sync. Model-mismatched vectors (a
|
|
3119
|
+
// teammate on a different devsmind version) are ignored on import, never partially
|
|
3120
|
+
// trusted — those nodes simply fall back into the local `devsmind embed` queue. Then an
|
|
3121
|
+
// explicit orphan sweep, since node_vectors has no FK (this whole method runs with
|
|
3122
|
+
// foreign_keys=OFF, so a cascade would silently never fire during the graph pass anyway).
|
|
3123
|
+
const vectorsDir = path.join(workspaceRoot, 'vectors');
|
|
3124
|
+
if (fs.existsSync(vectorsDir)) {
|
|
3125
|
+
const walkSyncVec = (dir, fileList = []) => {
|
|
3126
|
+
const files = fs.readdirSync(dir);
|
|
3127
|
+
for (const file of files) {
|
|
3128
|
+
const filePath = path.join(dir, file);
|
|
3129
|
+
if (fs.statSync(filePath).isDirectory()) {
|
|
3130
|
+
walkSyncVec(filePath, fileList);
|
|
3131
|
+
}
|
|
3132
|
+
else if (file.endsWith('.json')) {
|
|
3133
|
+
fileList.push(filePath);
|
|
3134
|
+
}
|
|
3135
|
+
}
|
|
3136
|
+
return fileList;
|
|
3137
|
+
};
|
|
3138
|
+
const vectorJsonFiles = walkSyncVec(vectorsDir);
|
|
3139
|
+
if (vectorJsonFiles.length > 0) {
|
|
3140
|
+
const insertVectorStmt = this.db.prepare(`
|
|
3141
|
+
INSERT OR REPLACE INTO node_vectors (node_id, model_id, dim, description_hash, vector)
|
|
3142
|
+
VALUES (?, ?, ?, ?, ?)
|
|
3143
|
+
`);
|
|
3144
|
+
const syncVectorsTx = this.db.transaction(() => {
|
|
3145
|
+
let done = 0;
|
|
3146
|
+
for (const file of vectorJsonFiles) {
|
|
3147
|
+
done++;
|
|
3148
|
+
if (onProgress && DevMindDatabase.shouldReport(done, vectorJsonFiles.length))
|
|
3149
|
+
onProgress('vectors', done, vectorJsonFiles.length);
|
|
3150
|
+
try {
|
|
3151
|
+
// Same reasoning as the graph pass above: a vectors/*.json is rewritten in place
|
|
3152
|
+
// whenever its node's vector changes, so mtime — not identity — is what tells us
|
|
3153
|
+
// whether the DB is still current for this file.
|
|
3154
|
+
if (fs.statSync(file).mtimeMs < lastSyncedAtMs)
|
|
3155
|
+
continue;
|
|
3156
|
+
const data = JSON.parse(fs.readFileSync(file, 'utf-8'));
|
|
3157
|
+
if (!data.model_id || data.model_id !== embedder_1.EMBEDDING_MODEL_ID)
|
|
3158
|
+
continue;
|
|
3159
|
+
const vectors = data.vectors || {};
|
|
3160
|
+
for (const nodeId of Object.keys(vectors)) {
|
|
3161
|
+
const entry = vectors[nodeId];
|
|
3162
|
+
if (!entry || !entry.v || !entry.h)
|
|
3163
|
+
continue;
|
|
3164
|
+
const buf = Buffer.from(entry.v, 'base64');
|
|
3165
|
+
insertVectorStmt.run(nodeId, embedder_1.EMBEDDING_MODEL_ID, data.dim || embedder_1.EMBEDDING_DIM, entry.h, buf);
|
|
3166
|
+
}
|
|
3167
|
+
}
|
|
3168
|
+
catch (err) {
|
|
3169
|
+
// ignore malformed vectors JSON
|
|
3170
|
+
}
|
|
3171
|
+
}
|
|
3172
|
+
});
|
|
3173
|
+
syncVectorsTx();
|
|
3174
|
+
}
|
|
3175
|
+
}
|
|
3176
|
+
// Orphan sweep — always runs, even with no vectors/ dir, to catch nodes deleted or
|
|
3177
|
+
// renamed by the graph pass above that still had a (now-dangling) vector row.
|
|
3178
|
+
this.db.exec('DELETE FROM node_vectors WHERE node_id NOT IN (SELECT id FROM nodes)');
|
|
1979
3179
|
// 3. Sync Workflow JSONs
|
|
1980
3180
|
const workflowsDir = this.workflowsDir();
|
|
1981
3181
|
if (fs.existsSync(workflowsDir)) {
|
|
1982
|
-
const upsertWorkflow = this.db.prepare(`
|
|
1983
|
-
INSERT INTO workflows (id, name, description,
|
|
1984
|
-
VALUES (?, ?, ?, ?, ?, ?)
|
|
1985
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
1986
|
-
name = excluded.name,
|
|
1987
|
-
description = excluded.description,
|
|
1988
|
-
|
|
1989
|
-
updated_at = excluded.updated_at
|
|
3182
|
+
const upsertWorkflow = this.db.prepare(`
|
|
3183
|
+
INSERT INTO workflows (id, name, description, archived, created_at, updated_at)
|
|
3184
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
3185
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
3186
|
+
name = excluded.name,
|
|
3187
|
+
description = excluded.description,
|
|
3188
|
+
archived = excluded.archived,
|
|
3189
|
+
updated_at = excluded.updated_at
|
|
1990
3190
|
`);
|
|
1991
|
-
|
|
1992
|
-
|
|
1993
|
-
|
|
3191
|
+
// DO UPDATE, not INSERT OR IGNORE. A teammate who already has a step row would otherwise
|
|
3192
|
+
// never pick up `reasoning`/`node_ids`/`doc_paths` from a newer workflow.json — the row
|
|
3193
|
+
// exists, so the insert is ignored, and their brain stays permanently half-migrated with
|
|
3194
|
+
// no sign anything went wrong. Safe to overwrite because steps are append-only: nothing
|
|
3195
|
+
// edits one locally after it is written, so incoming disk state is always authoritative.
|
|
3196
|
+
const upsertStep = this.db.prepare(`
|
|
3197
|
+
INSERT INTO workflow_steps (id, workflow_id, step_index, summary, reasoning, node_ids, doc_paths, session_id, created_at)
|
|
3198
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
3199
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
3200
|
+
summary = excluded.summary,
|
|
3201
|
+
reasoning = COALESCE(excluded.reasoning, workflow_steps.reasoning),
|
|
3202
|
+
node_ids = COALESCE(excluded.node_ids, workflow_steps.node_ids),
|
|
3203
|
+
doc_paths = COALESCE(excluded.doc_paths, workflow_steps.doc_paths)
|
|
1994
3204
|
`);
|
|
1995
|
-
const upsertArtifact = this.db.prepare(`
|
|
1996
|
-
INSERT OR IGNORE INTO workflow_artifacts (id, workflow_id, step_id, type, source_name, file_path, created_at)
|
|
1997
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
3205
|
+
const upsertArtifact = this.db.prepare(`
|
|
3206
|
+
INSERT OR IGNORE INTO workflow_artifacts (id, workflow_id, step_id, type, source_name, file_path, created_at)
|
|
3207
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
1998
3208
|
`);
|
|
1999
|
-
// Track which workflow.json has is_active:true with the latest updated_at
|
|
2000
|
-
let bestActiveId = null;
|
|
2001
|
-
let bestActiveUpdatedAt = '';
|
|
2002
3209
|
const syncWorkflowsTx = this.db.transaction(() => {
|
|
2003
3210
|
const subdirs = fs.readdirSync(workflowsDir);
|
|
2004
3211
|
for (const subdir of subdirs) {
|
|
@@ -2009,32 +3216,48 @@ class DevMindDatabase {
|
|
|
2009
3216
|
const data = JSON.parse(fs.readFileSync(jsonPath, 'utf-8'));
|
|
2010
3217
|
if (!data.id || !data.name)
|
|
2011
3218
|
continue;
|
|
2012
|
-
|
|
3219
|
+
// The sidecar is authoritative for the fields v1 has no place for. It matters most
|
|
3220
|
+
// in exactly the case that looks fine: a teammate on an older build rewrote
|
|
3221
|
+
// workflow.json from their own columns, so it came back v1-shaped — but they could
|
|
3222
|
+
// not touch v2.json, so `archived` and every step's reasoning/node_ids/doc_paths are
|
|
3223
|
+
// still here to merge back on top.
|
|
3224
|
+
let sidecar = {};
|
|
3225
|
+
try {
|
|
3226
|
+
const sidecarPath = path.join(workflowsDir, subdir, exports.WORKFLOW_SIDECAR_FILE);
|
|
3227
|
+
if (fs.existsSync(sidecarPath))
|
|
3228
|
+
sidecar = JSON.parse(fs.readFileSync(sidecarPath, 'utf-8')) || {};
|
|
3229
|
+
}
|
|
3230
|
+
catch { /* a corrupt sidecar degrades to the v1 shape rather than losing the workflow */ }
|
|
3231
|
+
upsertWorkflow.run(data.id, data.name, data.description || '', (sidecar.archived ?? data.archived) ? 1 : 0, data.created_at || new Date().toISOString(), data.updated_at || new Date().toISOString());
|
|
2013
3232
|
for (const s of (data.steps || [])) {
|
|
2014
3233
|
if (!s.id)
|
|
2015
3234
|
continue;
|
|
2016
|
-
|
|
3235
|
+
// A genuinely v1 workflow (never written by this version) has no sidecar entry
|
|
3236
|
+
// either — those fields land as NULL and the step shows its summary alone, which
|
|
3237
|
+
// is all v1 ever stored. Nothing is lost that the old format ever held.
|
|
3238
|
+
const extra = sidecar.steps?.[s.id] || {};
|
|
3239
|
+
upsertStep.run(s.id, data.id, s.step_index, s.summary || '', extra.reasoning ?? s.reasoning ?? null, extra.node_ids ?? s.node_ids ?? null, extra.doc_paths ?? s.doc_paths ?? null, s.session_id || null, s.created_at || new Date().toISOString());
|
|
2017
3240
|
}
|
|
2018
3241
|
for (const a of (data.artifact_index || [])) {
|
|
2019
3242
|
if (!a.id)
|
|
2020
3243
|
continue;
|
|
2021
3244
|
upsertArtifact.run(a.id, data.id, a.step_id || null, a.type || 'unknown', a.source_name || '', a.file_path || '', a.created_at || new Date().toISOString());
|
|
2022
3245
|
}
|
|
2023
|
-
//
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
}
|
|
3246
|
+
// A v1 JSON's `is_active` is deliberately ignored. That flag is exactly how one
|
|
3247
|
+
// developer's "currently working on" state used to travel through git and take over
|
|
3248
|
+
// everyone else's — which workflow you are on is now local to your session and never
|
|
3249
|
+
// synced.
|
|
2028
3250
|
}
|
|
2029
3251
|
catch { /* skip malformed */ }
|
|
2030
3252
|
}
|
|
2031
3253
|
});
|
|
2032
3254
|
syncWorkflowsTx();
|
|
2033
|
-
// Restore active_workflow_id if not already set and a JSON claims active status
|
|
2034
|
-
if (bestActiveId && !this.getSystemMeta('active_workflow_id')) {
|
|
2035
|
-
this.setSystemMeta('active_workflow_id', bestActiveId);
|
|
2036
|
-
}
|
|
2037
3255
|
}
|
|
3256
|
+
// Only advance the checkpoint after every step above completed without throwing — an
|
|
3257
|
+
// exception anywhere earlier jumps straight to `catch` below, so this line is never
|
|
3258
|
+
// reached, and the NEXT sync correctly retries a full pass from the old (or absent)
|
|
3259
|
+
// checkpoint instead of wrongly believing a failed run succeeded.
|
|
3260
|
+
this.setSystemMeta('last_sync_checkpoint_ms', String(syncStartedAtMs));
|
|
2038
3261
|
}
|
|
2039
3262
|
catch (err) {
|
|
2040
3263
|
console.warn('⚠️ SQLite warning: Failed to sync from disk:', err);
|
|
@@ -2069,16 +3292,16 @@ class DevMindDatabase {
|
|
|
2069
3292
|
const absEsc = this.likeEscape(absPath);
|
|
2070
3293
|
const absLower = absPath.toLowerCase();
|
|
2071
3294
|
const absEscLower = absEsc.toLowerCase();
|
|
2072
|
-
const stmtNodes = this.db.prepare(`
|
|
2073
|
-
SELECT * FROM nodes
|
|
2074
|
-
WHERE (
|
|
2075
|
-
LOWER(file_path) = ? OR
|
|
2076
|
-
LOWER(file_path) LIKE ? ESCAPE '\\' OR
|
|
2077
|
-
LOWER(file_path) LIKE ? ESCAPE '\\' OR
|
|
2078
|
-
LOWER(file_path) LIKE ? ESCAPE '\\'
|
|
2079
|
-
)
|
|
3295
|
+
const stmtNodes = this.db.prepare(`
|
|
3296
|
+
SELECT * FROM nodes
|
|
3297
|
+
WHERE (
|
|
3298
|
+
LOWER(file_path) = ? OR
|
|
3299
|
+
LOWER(file_path) LIKE ? ESCAPE '\\' OR
|
|
3300
|
+
LOWER(file_path) LIKE ? ESCAPE '\\' OR
|
|
3301
|
+
LOWER(file_path) LIKE ? ESCAPE '\\'
|
|
3302
|
+
)
|
|
2080
3303
|
`);
|
|
2081
|
-
const nodes = stmtNodes.all(absLower, `${absEscLower}, %`, `%, ${absEscLower}`, `%, ${absEscLower}, %`);
|
|
3304
|
+
const nodes = DevMindDatabase.parseNodeRows(stmtNodes.all(absLower, `${absEscLower}, %`, `%, ${absEscLower}`, `%, ${absEscLower}, %`));
|
|
2082
3305
|
if (nodes.length === 0) {
|
|
2083
3306
|
// If no nodes left, delete the JSON file if it exists
|
|
2084
3307
|
if (fs.existsSync(graphJsonPath)) {
|
|
@@ -2090,9 +3313,9 @@ class DevMindDatabase {
|
|
|
2090
3313
|
const nodeIds = nodes.map(n => n.id);
|
|
2091
3314
|
const connections = [];
|
|
2092
3315
|
if (nodeIds.length > 0) {
|
|
2093
|
-
const stmtConn = this.db.prepare(`
|
|
2094
|
-
SELECT * FROM node_connections
|
|
2095
|
-
WHERE source_node_id = ?
|
|
3316
|
+
const stmtConn = this.db.prepare(`
|
|
3317
|
+
SELECT * FROM node_connections
|
|
3318
|
+
WHERE source_node_id = ?
|
|
2096
3319
|
`);
|
|
2097
3320
|
for (const id of nodeIds) {
|
|
2098
3321
|
const conns = stmtConn.all(id);
|
|
@@ -2107,6 +3330,8 @@ class DevMindDatabase {
|
|
|
2107
3330
|
name: n.name,
|
|
2108
3331
|
type: n.type,
|
|
2109
3332
|
signature: n.signature,
|
|
3333
|
+
description: n.description || undefined,
|
|
3334
|
+
aliases: n.aliases.length > 0 ? n.aliases : undefined,
|
|
2110
3335
|
deprecated: n.deprecated ? 1 : 0
|
|
2111
3336
|
})),
|
|
2112
3337
|
connections: connections.map(c => ({
|
|
@@ -2121,6 +3346,53 @@ class DevMindDatabase {
|
|
|
2121
3346
|
console.warn('⚠️ SQLite warning: Failed to write graph JSON to disk:', err);
|
|
2122
3347
|
}
|
|
2123
3348
|
}
|
|
3349
|
+
/**
|
|
3350
|
+
* Mirrors `writeGraphToDisk` exactly (same file-matching logic, same directory shape) but into
|
|
3351
|
+
* a separate `vectors/` tree rather than inside `graph/*.json` — deliberately, so opaque base64
|
|
3352
|
+
* blobs never pollute the human-readable, merge-friendly graph JSON. Deprecated nodes are
|
|
3353
|
+
* skipped here (unlike the graph, which keeps them): `searchNodes` never queries a deprecated
|
|
3354
|
+
* node's vector, so writing one is pure dead weight.
|
|
3355
|
+
*/
|
|
3356
|
+
writeVectorsToDisk(filePath) {
|
|
3357
|
+
try {
|
|
3358
|
+
if (!filePath)
|
|
3359
|
+
return;
|
|
3360
|
+
const workspaceRoot = (0, config_1.canonicalizePath)(path.dirname(this.dbPath));
|
|
3361
|
+
const absPath = (0, config_1.canonicalizePath)(filePath);
|
|
3362
|
+
const repoRelPath = this.toRepoRelativePath(absPath);
|
|
3363
|
+
const diskRelPath = repoRelPath.replace(/^\{([^}]+)\}/, '$1').replace(/\.[^/.]+$/, '.json');
|
|
3364
|
+
const vectorsJsonPath = path.join(workspaceRoot, 'vectors', diskRelPath);
|
|
3365
|
+
const absLower = absPath.toLowerCase();
|
|
3366
|
+
const absEscLower = this.likeEscape(absPath).toLowerCase();
|
|
3367
|
+
const stmt = this.db.prepare(`
|
|
3368
|
+
SELECT nv.node_id AS node_id, nv.description_hash AS description_hash, nv.vector AS vector
|
|
3369
|
+
FROM node_vectors nv
|
|
3370
|
+
JOIN nodes n ON n.id = nv.node_id
|
|
3371
|
+
WHERE n.deprecated = 0 AND (
|
|
3372
|
+
LOWER(n.file_path) = ? OR
|
|
3373
|
+
LOWER(n.file_path) LIKE ? ESCAPE '\\' OR
|
|
3374
|
+
LOWER(n.file_path) LIKE ? ESCAPE '\\' OR
|
|
3375
|
+
LOWER(n.file_path) LIKE ? ESCAPE '\\'
|
|
3376
|
+
) AND nv.model_id = ?
|
|
3377
|
+
`);
|
|
3378
|
+
const rows = stmt.all(absLower, `${absEscLower}, %`, `%, ${absEscLower}`, `%, ${absEscLower}, %`, embedder_1.EMBEDDING_MODEL_ID);
|
|
3379
|
+
if (rows.length === 0) {
|
|
3380
|
+
if (fs.existsSync(vectorsJsonPath))
|
|
3381
|
+
fs.unlinkSync(vectorsJsonPath);
|
|
3382
|
+
return;
|
|
3383
|
+
}
|
|
3384
|
+
const vectors = {};
|
|
3385
|
+
for (const r of rows) {
|
|
3386
|
+
vectors[r.node_id] = { h: r.description_hash, v: Buffer.from(r.vector).toString('base64') };
|
|
3387
|
+
}
|
|
3388
|
+
const data = { file_path: repoRelPath, model_id: embedder_1.EMBEDDING_MODEL_ID, dim: embedder_1.EMBEDDING_DIM, vectors };
|
|
3389
|
+
fs.mkdirSync(path.dirname(vectorsJsonPath), { recursive: true });
|
|
3390
|
+
fs.writeFileSync(vectorsJsonPath, JSON.stringify(data, null, 2), 'utf-8');
|
|
3391
|
+
}
|
|
3392
|
+
catch (err) {
|
|
3393
|
+
console.warn('⚠️ SQLite warning: Failed to write vectors JSON to disk:', err);
|
|
3394
|
+
}
|
|
3395
|
+
}
|
|
2124
3396
|
/** Force-syncs all database nodes and workflows to disk JSON files. */
|
|
2125
3397
|
syncToDisk() {
|
|
2126
3398
|
try {
|