minimem 0.0.3 → 0.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunk-ARZQHOJI.js +239 -0
- package/dist/chunk-ARZQHOJI.js.map +1 -0
- package/dist/chunk-GVWPZRF7.js +260 -0
- package/dist/chunk-GVWPZRF7.js.map +1 -0
- package/dist/cli/index.js +1411 -612
- package/dist/cli/index.js.map +1 -1
- package/dist/index.d.ts +375 -109
- package/dist/index.js +1330 -370
- package/dist/index.js.map +1 -1
- package/dist/internal.d.ts +65 -0
- package/dist/internal.js +33 -0
- package/dist/internal.js.map +1 -0
- package/dist/session.d.ts +96 -0
- package/dist/session.js +15 -0
- package/dist/session.js.map +1 -0
- package/package.json +10 -2
package/dist/index.js
CHANGED
|
@@ -1,187 +1,33 @@
|
|
|
1
|
+
import {
|
|
2
|
+
buildFileEntry,
|
|
3
|
+
chunkMarkdown,
|
|
4
|
+
cosineSimilarity,
|
|
5
|
+
ensureDir,
|
|
6
|
+
extractChunkMetadata,
|
|
7
|
+
hashText,
|
|
8
|
+
isMemoryPath,
|
|
9
|
+
listMemoryFiles,
|
|
10
|
+
logError,
|
|
11
|
+
parseEmbedding,
|
|
12
|
+
stripPrivateContent,
|
|
13
|
+
truncateUtf16Safe,
|
|
14
|
+
vectorToBlob
|
|
15
|
+
} from "./chunk-ARZQHOJI.js";
|
|
16
|
+
import {
|
|
17
|
+
addFrontmatter,
|
|
18
|
+
addSessionToContent,
|
|
19
|
+
extractSession,
|
|
20
|
+
parseFrontmatter,
|
|
21
|
+
serializeFrontmatter
|
|
22
|
+
} from "./chunk-GVWPZRF7.js";
|
|
23
|
+
|
|
1
24
|
// src/minimem.ts
|
|
2
25
|
import { randomUUID } from "crypto";
|
|
3
|
-
import
|
|
4
|
-
import
|
|
26
|
+
import fs from "fs/promises";
|
|
27
|
+
import path2 from "path";
|
|
5
28
|
import { DatabaseSync } from "sqlite";
|
|
6
29
|
import chokidar from "chokidar";
|
|
7
30
|
|
|
8
|
-
// src/internal.ts
|
|
9
|
-
import crypto from "crypto";
|
|
10
|
-
import fsSync from "fs";
|
|
11
|
-
import fs from "fs/promises";
|
|
12
|
-
import path from "path";
|
|
13
|
-
function ensureDir(dir) {
|
|
14
|
-
try {
|
|
15
|
-
fsSync.mkdirSync(dir, { recursive: true });
|
|
16
|
-
} catch {
|
|
17
|
-
}
|
|
18
|
-
return dir;
|
|
19
|
-
}
|
|
20
|
-
function normalizeRelPath(value) {
|
|
21
|
-
const trimmed = value.trim().replace(/^[./]+/, "");
|
|
22
|
-
return trimmed.replace(/\\/g, "/");
|
|
23
|
-
}
|
|
24
|
-
function isMemoryPath(relPath) {
|
|
25
|
-
const normalized = normalizeRelPath(relPath);
|
|
26
|
-
if (!normalized) return false;
|
|
27
|
-
if (normalized === "MEMORY.md" || normalized === "memory.md") return true;
|
|
28
|
-
return normalized.startsWith("memory/");
|
|
29
|
-
}
|
|
30
|
-
async function exists(filePath) {
|
|
31
|
-
try {
|
|
32
|
-
await fs.access(filePath);
|
|
33
|
-
return true;
|
|
34
|
-
} catch {
|
|
35
|
-
return false;
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
async function walkDir(dir, files) {
|
|
39
|
-
const entries = await fs.readdir(dir, { withFileTypes: true });
|
|
40
|
-
for (const entry of entries) {
|
|
41
|
-
const full = path.join(dir, entry.name);
|
|
42
|
-
if (entry.isDirectory()) {
|
|
43
|
-
await walkDir(full, files);
|
|
44
|
-
continue;
|
|
45
|
-
}
|
|
46
|
-
if (!entry.isFile()) continue;
|
|
47
|
-
if (!entry.name.endsWith(".md")) continue;
|
|
48
|
-
files.push(full);
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
async function listMemoryFiles(memoryDir) {
|
|
52
|
-
const result = [];
|
|
53
|
-
const memoryFile = path.join(memoryDir, "MEMORY.md");
|
|
54
|
-
const altMemoryFile = path.join(memoryDir, "memory.md");
|
|
55
|
-
if (await exists(memoryFile)) result.push(memoryFile);
|
|
56
|
-
if (await exists(altMemoryFile)) result.push(altMemoryFile);
|
|
57
|
-
const memorySubDir = path.join(memoryDir, "memory");
|
|
58
|
-
if (await exists(memorySubDir)) {
|
|
59
|
-
await walkDir(memorySubDir, result);
|
|
60
|
-
}
|
|
61
|
-
if (result.length <= 1) return result;
|
|
62
|
-
const seen = /* @__PURE__ */ new Set();
|
|
63
|
-
const deduped = [];
|
|
64
|
-
for (const entry of result) {
|
|
65
|
-
let key = entry;
|
|
66
|
-
try {
|
|
67
|
-
key = await fs.realpath(entry);
|
|
68
|
-
} catch {
|
|
69
|
-
}
|
|
70
|
-
if (seen.has(key)) continue;
|
|
71
|
-
seen.add(key);
|
|
72
|
-
deduped.push(entry);
|
|
73
|
-
}
|
|
74
|
-
return deduped;
|
|
75
|
-
}
|
|
76
|
-
function hashText(value) {
|
|
77
|
-
return crypto.createHash("sha256").update(value).digest("hex");
|
|
78
|
-
}
|
|
79
|
-
async function buildFileEntry(absPath, memoryDir) {
|
|
80
|
-
const stat = await fs.stat(absPath);
|
|
81
|
-
const content = await fs.readFile(absPath, "utf-8");
|
|
82
|
-
const hash = hashText(content);
|
|
83
|
-
return {
|
|
84
|
-
path: path.relative(memoryDir, absPath).replace(/\\/g, "/"),
|
|
85
|
-
absPath,
|
|
86
|
-
mtimeMs: stat.mtimeMs,
|
|
87
|
-
size: stat.size,
|
|
88
|
-
hash
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
function chunkMarkdown(content, chunking) {
|
|
92
|
-
const lines = content.split("\n");
|
|
93
|
-
if (lines.length === 0) return [];
|
|
94
|
-
const maxChars = Math.max(32, chunking.tokens * 4);
|
|
95
|
-
const overlapChars = Math.max(0, chunking.overlap * 4);
|
|
96
|
-
const chunks = [];
|
|
97
|
-
let current = [];
|
|
98
|
-
let currentChars = 0;
|
|
99
|
-
const flush = () => {
|
|
100
|
-
if (current.length === 0) return;
|
|
101
|
-
const firstEntry = current[0];
|
|
102
|
-
const lastEntry = current[current.length - 1];
|
|
103
|
-
if (!firstEntry || !lastEntry) return;
|
|
104
|
-
const text = current.map((entry) => entry.line).join("\n");
|
|
105
|
-
const startLine = firstEntry.lineNo;
|
|
106
|
-
const endLine = lastEntry.lineNo;
|
|
107
|
-
chunks.push({
|
|
108
|
-
startLine,
|
|
109
|
-
endLine,
|
|
110
|
-
text,
|
|
111
|
-
hash: hashText(text)
|
|
112
|
-
});
|
|
113
|
-
};
|
|
114
|
-
const carryOverlap = () => {
|
|
115
|
-
if (overlapChars <= 0 || current.length === 0) {
|
|
116
|
-
current = [];
|
|
117
|
-
currentChars = 0;
|
|
118
|
-
return;
|
|
119
|
-
}
|
|
120
|
-
let acc = 0;
|
|
121
|
-
const kept = [];
|
|
122
|
-
for (let i = current.length - 1; i >= 0; i -= 1) {
|
|
123
|
-
const entry = current[i];
|
|
124
|
-
if (!entry) continue;
|
|
125
|
-
acc += entry.line.length + 1;
|
|
126
|
-
kept.unshift(entry);
|
|
127
|
-
if (acc >= overlapChars) break;
|
|
128
|
-
}
|
|
129
|
-
current = kept;
|
|
130
|
-
currentChars = kept.reduce((sum, entry) => sum + entry.line.length + 1, 0);
|
|
131
|
-
};
|
|
132
|
-
for (let i = 0; i < lines.length; i += 1) {
|
|
133
|
-
const line = lines[i] ?? "";
|
|
134
|
-
const lineNo = i + 1;
|
|
135
|
-
const segments = [];
|
|
136
|
-
if (line.length === 0) {
|
|
137
|
-
segments.push("");
|
|
138
|
-
} else {
|
|
139
|
-
for (let start = 0; start < line.length; start += maxChars) {
|
|
140
|
-
segments.push(line.slice(start, start + maxChars));
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
for (const segment of segments) {
|
|
144
|
-
const lineSize = segment.length + 1;
|
|
145
|
-
if (currentChars + lineSize > maxChars && current.length > 0) {
|
|
146
|
-
flush();
|
|
147
|
-
carryOverlap();
|
|
148
|
-
}
|
|
149
|
-
current.push({ line: segment, lineNo });
|
|
150
|
-
currentChars += lineSize;
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
flush();
|
|
154
|
-
return chunks;
|
|
155
|
-
}
|
|
156
|
-
function parseEmbedding(raw) {
|
|
157
|
-
try {
|
|
158
|
-
const parsed = JSON.parse(raw);
|
|
159
|
-
return Array.isArray(parsed) ? parsed : [];
|
|
160
|
-
} catch {
|
|
161
|
-
return [];
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
function cosineSimilarity(a, b) {
|
|
165
|
-
if (a.length === 0 || b.length === 0) return 0;
|
|
166
|
-
const len = Math.min(a.length, b.length);
|
|
167
|
-
let dot = 0;
|
|
168
|
-
let normA = 0;
|
|
169
|
-
let normB = 0;
|
|
170
|
-
for (let i = 0; i < len; i += 1) {
|
|
171
|
-
const av = a[i] ?? 0;
|
|
172
|
-
const bv = b[i] ?? 0;
|
|
173
|
-
dot += av * bv;
|
|
174
|
-
normA += av * av;
|
|
175
|
-
normB += bv * bv;
|
|
176
|
-
}
|
|
177
|
-
if (normA === 0 || normB === 0) return 0;
|
|
178
|
-
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
179
|
-
}
|
|
180
|
-
function truncateUtf16Safe(text, maxChars) {
|
|
181
|
-
if (text.length <= maxChars) return text;
|
|
182
|
-
return text.slice(0, maxChars);
|
|
183
|
-
}
|
|
184
|
-
|
|
185
31
|
// src/search/hybrid.ts
|
|
186
32
|
function buildFtsQuery(raw) {
|
|
187
33
|
const tokens = raw.match(/[A-Za-z0-9_]+/g)?.map((t) => t.trim()).filter(Boolean) ?? [];
|
|
@@ -190,8 +36,11 @@ function buildFtsQuery(raw) {
|
|
|
190
36
|
return quoted.join(" AND ");
|
|
191
37
|
}
|
|
192
38
|
function bm25RankToScore(rank) {
|
|
193
|
-
|
|
194
|
-
|
|
39
|
+
if (!Number.isFinite(rank)) {
|
|
40
|
+
return 0;
|
|
41
|
+
}
|
|
42
|
+
const absRank = Math.abs(rank);
|
|
43
|
+
return 1 / (1 + absRank);
|
|
195
44
|
}
|
|
196
45
|
function mergeHybridResults(params) {
|
|
197
46
|
const byId = /* @__PURE__ */ new Map();
|
|
@@ -225,8 +74,17 @@ function mergeHybridResults(params) {
|
|
|
225
74
|
});
|
|
226
75
|
}
|
|
227
76
|
}
|
|
77
|
+
let vw = params.vectorWeight;
|
|
78
|
+
let tw = params.textWeight;
|
|
79
|
+
if (params.vector.length === 0 && params.keyword.length > 0) {
|
|
80
|
+
vw = 0;
|
|
81
|
+
tw = 1;
|
|
82
|
+
} else if (params.keyword.length === 0 && params.vector.length > 0) {
|
|
83
|
+
vw = 1;
|
|
84
|
+
tw = 0;
|
|
85
|
+
}
|
|
228
86
|
const merged = Array.from(byId.values()).map((entry) => {
|
|
229
|
-
const score =
|
|
87
|
+
const score = vw * entry.vectorScore + tw * entry.textScore;
|
|
230
88
|
return {
|
|
231
89
|
path: entry.path,
|
|
232
90
|
startLine: entry.startLine,
|
|
@@ -240,7 +98,33 @@ function mergeHybridResults(params) {
|
|
|
240
98
|
}
|
|
241
99
|
|
|
242
100
|
// src/search/search.ts
|
|
243
|
-
|
|
101
|
+
function buildKnowledgeFilterSql(opts) {
|
|
102
|
+
const clauses = [];
|
|
103
|
+
const params = [];
|
|
104
|
+
if (opts.knowledgeType) {
|
|
105
|
+
clauses.push(` AND c.knowledge_type = ?`);
|
|
106
|
+
params.push(opts.knowledgeType);
|
|
107
|
+
}
|
|
108
|
+
if (opts.minConfidence !== void 0) {
|
|
109
|
+
clauses.push(` AND c.confidence >= ?`);
|
|
110
|
+
params.push(opts.minConfidence);
|
|
111
|
+
}
|
|
112
|
+
if (opts.domain && opts.domain.length > 0) {
|
|
113
|
+
const domainPlaceholders = opts.domain.map(() => "?").join(", ");
|
|
114
|
+
clauses.push(
|
|
115
|
+
` AND EXISTS (SELECT 1 FROM json_each(c.domains) AS d WHERE d.value IN (${domainPlaceholders}))`
|
|
116
|
+
);
|
|
117
|
+
params.push(...opts.domain);
|
|
118
|
+
}
|
|
119
|
+
if (opts.entities && opts.entities.length > 0) {
|
|
120
|
+
const entityPlaceholders = opts.entities.map(() => "?").join(", ");
|
|
121
|
+
clauses.push(
|
|
122
|
+
` AND EXISTS (SELECT 1 FROM json_each(c.entities) AS e WHERE e.value IN (${entityPlaceholders}))`
|
|
123
|
+
);
|
|
124
|
+
params.push(...opts.entities);
|
|
125
|
+
}
|
|
126
|
+
return { sql: clauses.join(""), params };
|
|
127
|
+
}
|
|
244
128
|
async function searchVector(params) {
|
|
245
129
|
if (params.queryVec.length === 0 || params.limit <= 0) return [];
|
|
246
130
|
if (await params.ensureVectorReady(params.queryVec.length)) {
|
|
@@ -332,6 +216,7 @@ async function searchKeyword(params) {
|
|
|
332
216
|
}
|
|
333
217
|
|
|
334
218
|
// src/db/schema.ts
|
|
219
|
+
var SCHEMA_VERSION = 4;
|
|
335
220
|
function ensureMemoryIndexSchema(params) {
|
|
336
221
|
params.db.exec(`
|
|
337
222
|
CREATE TABLE IF NOT EXISTS meta (
|
|
@@ -339,6 +224,7 @@ function ensureMemoryIndexSchema(params) {
|
|
|
339
224
|
value TEXT NOT NULL
|
|
340
225
|
);
|
|
341
226
|
`);
|
|
227
|
+
const migrated = migrateIfNeeded(params.db, params.ftsTable);
|
|
342
228
|
params.db.exec(`
|
|
343
229
|
CREATE TABLE IF NOT EXISTS files (
|
|
344
230
|
path TEXT PRIMARY KEY,
|
|
@@ -401,9 +287,61 @@ function ensureMemoryIndexSchema(params) {
|
|
|
401
287
|
}
|
|
402
288
|
ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'");
|
|
403
289
|
ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'");
|
|
290
|
+
ensureColumn(params.db, "chunks", "type", "TEXT");
|
|
291
|
+
ensureColumn(params.db, "chunks", "knowledge_type", "TEXT");
|
|
292
|
+
ensureColumn(params.db, "chunks", "knowledge_id", "TEXT");
|
|
293
|
+
ensureColumn(params.db, "chunks", "domains", "TEXT");
|
|
294
|
+
ensureColumn(params.db, "chunks", "entities", "TEXT");
|
|
295
|
+
ensureColumn(params.db, "chunks", "confidence", "REAL");
|
|
404
296
|
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`);
|
|
405
297
|
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`);
|
|
406
|
-
|
|
298
|
+
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(type);`);
|
|
299
|
+
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_knowledge_type ON chunks(knowledge_type);`);
|
|
300
|
+
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_knowledge_id ON chunks(knowledge_id);`);
|
|
301
|
+
params.db.exec(`
|
|
302
|
+
CREATE TABLE IF NOT EXISTS knowledge_links (
|
|
303
|
+
from_id TEXT NOT NULL,
|
|
304
|
+
to_id TEXT NOT NULL,
|
|
305
|
+
relation TEXT NOT NULL,
|
|
306
|
+
layer TEXT,
|
|
307
|
+
weight REAL DEFAULT 0.5,
|
|
308
|
+
source_path TEXT,
|
|
309
|
+
created_at INTEGER,
|
|
310
|
+
PRIMARY KEY (from_id, to_id, relation)
|
|
311
|
+
);
|
|
312
|
+
`);
|
|
313
|
+
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_from ON knowledge_links(from_id);`);
|
|
314
|
+
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_to ON knowledge_links(to_id);`);
|
|
315
|
+
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_layer ON knowledge_links(layer);`);
|
|
316
|
+
params.db.prepare(
|
|
317
|
+
`INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)`
|
|
318
|
+
).run(String(SCHEMA_VERSION));
|
|
319
|
+
return { ftsAvailable, ...ftsError ? { ftsError } : {}, ...migrated ? { migrated } : {} };
|
|
320
|
+
}
|
|
321
|
+
function migrateIfNeeded(db, ftsTable) {
|
|
322
|
+
let storedVersion = 0;
|
|
323
|
+
try {
|
|
324
|
+
const row = db.prepare(
|
|
325
|
+
`SELECT value FROM meta WHERE key = 'schema_version'`
|
|
326
|
+
).get();
|
|
327
|
+
if (row) {
|
|
328
|
+
storedVersion = parseInt(row.value, 10) || 0;
|
|
329
|
+
}
|
|
330
|
+
} catch {
|
|
331
|
+
storedVersion = 0;
|
|
332
|
+
}
|
|
333
|
+
if (storedVersion >= SCHEMA_VERSION) return false;
|
|
334
|
+
if (storedVersion > 0 && storedVersion < SCHEMA_VERSION) {
|
|
335
|
+
db.exec(`DROP TABLE IF EXISTS files`);
|
|
336
|
+
db.exec(`DROP TABLE IF EXISTS chunks`);
|
|
337
|
+
db.exec(`DROP TABLE IF EXISTS knowledge_links`);
|
|
338
|
+
db.exec(`DROP TABLE IF EXISTS ${ftsTable}`);
|
|
339
|
+
try {
|
|
340
|
+
db.exec(`DROP TABLE IF EXISTS chunks_vec`);
|
|
341
|
+
} catch {
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
return storedVersion > 0;
|
|
407
345
|
}
|
|
408
346
|
function ensureColumn(db, table, column, definition) {
|
|
409
347
|
const rows = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
@@ -411,6 +349,122 @@ function ensureColumn(db, table, column, definition) {
|
|
|
411
349
|
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
|
412
350
|
}
|
|
413
351
|
|
|
352
|
+
// src/search/graph.ts
|
|
353
|
+
function getLinksFrom(db, fromId, opts) {
|
|
354
|
+
let sql = `SELECT from_id, to_id, relation, layer, weight, source_path FROM knowledge_links WHERE from_id = ?`;
|
|
355
|
+
const params = [fromId];
|
|
356
|
+
if (opts?.relation) {
|
|
357
|
+
sql += ` AND relation = ?`;
|
|
358
|
+
params.push(opts.relation);
|
|
359
|
+
}
|
|
360
|
+
if (opts?.layer) {
|
|
361
|
+
sql += ` AND layer = ?`;
|
|
362
|
+
params.push(opts.layer);
|
|
363
|
+
}
|
|
364
|
+
const rows = db.prepare(sql).all(...params);
|
|
365
|
+
return rows.map(toGraphLink);
|
|
366
|
+
}
|
|
367
|
+
function getLinksTo(db, toId, opts) {
|
|
368
|
+
let sql = `SELECT from_id, to_id, relation, layer, weight, source_path FROM knowledge_links WHERE to_id = ?`;
|
|
369
|
+
const params = [toId];
|
|
370
|
+
if (opts?.relation) {
|
|
371
|
+
sql += ` AND relation = ?`;
|
|
372
|
+
params.push(opts.relation);
|
|
373
|
+
}
|
|
374
|
+
if (opts?.layer) {
|
|
375
|
+
sql += ` AND layer = ?`;
|
|
376
|
+
params.push(opts.layer);
|
|
377
|
+
}
|
|
378
|
+
const rows = db.prepare(sql).all(...params);
|
|
379
|
+
return rows.map(toGraphLink);
|
|
380
|
+
}
|
|
381
|
+
function getNeighbors(db, startId, depth = 1, opts) {
|
|
382
|
+
const visited = /* @__PURE__ */ new Set([startId]);
|
|
383
|
+
const result = [];
|
|
384
|
+
let frontier = [startId];
|
|
385
|
+
for (let d = 1; d <= depth; d++) {
|
|
386
|
+
const nextFrontier = [];
|
|
387
|
+
for (const nodeId of frontier) {
|
|
388
|
+
const outgoing = getLinksFrom(db, nodeId, opts);
|
|
389
|
+
for (const link of outgoing) {
|
|
390
|
+
if (!visited.has(link.toId)) {
|
|
391
|
+
visited.add(link.toId);
|
|
392
|
+
nextFrontier.push(link.toId);
|
|
393
|
+
result.push({ id: link.toId, depth: d, link });
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
const incoming = getLinksTo(db, nodeId, opts);
|
|
397
|
+
for (const link of incoming) {
|
|
398
|
+
if (!visited.has(link.fromId)) {
|
|
399
|
+
visited.add(link.fromId);
|
|
400
|
+
nextFrontier.push(link.fromId);
|
|
401
|
+
result.push({ id: link.fromId, depth: d, link });
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
frontier = nextFrontier;
|
|
406
|
+
if (frontier.length === 0) break;
|
|
407
|
+
}
|
|
408
|
+
return result;
|
|
409
|
+
}
|
|
410
|
+
function getPathBetween(db, fromId, toId, maxDepth = 3) {
|
|
411
|
+
if (fromId === toId) return [];
|
|
412
|
+
const visited = /* @__PURE__ */ new Set([fromId]);
|
|
413
|
+
const parentLink = /* @__PURE__ */ new Map();
|
|
414
|
+
let frontier = [fromId];
|
|
415
|
+
for (let d = 0; d < maxDepth; d++) {
|
|
416
|
+
const nextFrontier = [];
|
|
417
|
+
for (const nodeId of frontier) {
|
|
418
|
+
const outgoing = getLinksFrom(db, nodeId);
|
|
419
|
+
for (const link of outgoing) {
|
|
420
|
+
if (!visited.has(link.toId)) {
|
|
421
|
+
visited.add(link.toId);
|
|
422
|
+
parentLink.set(link.toId, link);
|
|
423
|
+
if (link.toId === toId) {
|
|
424
|
+
return reconstructPath(parentLink, fromId, toId);
|
|
425
|
+
}
|
|
426
|
+
nextFrontier.push(link.toId);
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
const incoming = getLinksTo(db, nodeId);
|
|
430
|
+
for (const link of incoming) {
|
|
431
|
+
if (!visited.has(link.fromId)) {
|
|
432
|
+
visited.add(link.fromId);
|
|
433
|
+
parentLink.set(link.fromId, link);
|
|
434
|
+
if (link.fromId === toId) {
|
|
435
|
+
return reconstructPath(parentLink, fromId, toId);
|
|
436
|
+
}
|
|
437
|
+
nextFrontier.push(link.fromId);
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
frontier = nextFrontier;
|
|
442
|
+
if (frontier.length === 0) break;
|
|
443
|
+
}
|
|
444
|
+
return [];
|
|
445
|
+
}
|
|
446
|
+
function reconstructPath(parentLink, fromId, toId) {
|
|
447
|
+
const path4 = [];
|
|
448
|
+
let current = toId;
|
|
449
|
+
while (current !== fromId) {
|
|
450
|
+
const link = parentLink.get(current);
|
|
451
|
+
if (!link) break;
|
|
452
|
+
path4.unshift(link);
|
|
453
|
+
current = link.toId === current ? link.fromId : link.toId;
|
|
454
|
+
}
|
|
455
|
+
return path4;
|
|
456
|
+
}
|
|
457
|
+
function toGraphLink(row) {
|
|
458
|
+
return {
|
|
459
|
+
fromId: row.from_id,
|
|
460
|
+
toId: row.to_id,
|
|
461
|
+
relation: row.relation,
|
|
462
|
+
layer: row.layer,
|
|
463
|
+
weight: row.weight,
|
|
464
|
+
sourcePath: row.source_path
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
|
|
414
468
|
// src/db/sqlite-vec.ts
|
|
415
469
|
async function loadSqliteVecExtension(params) {
|
|
416
470
|
try {
|
|
@@ -431,8 +485,8 @@ async function loadSqliteVecExtension(params) {
|
|
|
431
485
|
}
|
|
432
486
|
|
|
433
487
|
// src/embeddings/embeddings.ts
|
|
434
|
-
import
|
|
435
|
-
import
|
|
488
|
+
import fsSync from "fs";
|
|
489
|
+
import path from "path";
|
|
436
490
|
import os from "os";
|
|
437
491
|
var DEFAULT_LOCAL_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
|
|
438
492
|
var DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
@@ -449,7 +503,7 @@ function createNoOpEmbeddingProvider() {
|
|
|
449
503
|
}
|
|
450
504
|
function resolveUserPath(filePath) {
|
|
451
505
|
if (filePath.startsWith("~/")) {
|
|
452
|
-
return
|
|
506
|
+
return path.join(os.homedir(), filePath.slice(2));
|
|
453
507
|
}
|
|
454
508
|
return filePath;
|
|
455
509
|
}
|
|
@@ -459,7 +513,7 @@ function canAutoSelectLocal(options) {
|
|
|
459
513
|
if (/^(hf:|https?:)/i.test(modelPath)) return false;
|
|
460
514
|
const resolved = resolveUserPath(modelPath);
|
|
461
515
|
try {
|
|
462
|
-
return
|
|
516
|
+
return fsSync.statSync(resolved).isFile();
|
|
463
517
|
} catch {
|
|
464
518
|
return false;
|
|
465
519
|
}
|
|
@@ -660,7 +714,6 @@ async function createEmbeddingProvider(options) {
|
|
|
660
714
|
return {
|
|
661
715
|
provider: createNoOpEmbeddingProvider(),
|
|
662
716
|
requestedProvider: "none"
|
|
663
|
-
// Type coercion for compatibility
|
|
664
717
|
};
|
|
665
718
|
}
|
|
666
719
|
const createProvider = async (id) => {
|
|
@@ -704,7 +757,6 @@ async function createEmbeddingProvider(options) {
|
|
|
704
757
|
provider: createNoOpEmbeddingProvider(),
|
|
705
758
|
requestedProvider,
|
|
706
759
|
fallbackFrom: "auto",
|
|
707
|
-
// Indicate this is a fallback
|
|
708
760
|
fallbackReason: "No embedding API available. Using BM25 full-text search only."
|
|
709
761
|
};
|
|
710
762
|
}
|
|
@@ -1347,7 +1399,6 @@ var EMBEDDING_RETRY_BASE_DELAY_MS = 500;
|
|
|
1347
1399
|
var EMBEDDING_RETRY_MAX_DELAY_MS = 8e3;
|
|
1348
1400
|
var EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 6e4;
|
|
1349
1401
|
var EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 6e4;
|
|
1350
|
-
var vectorToBlob2 = (embedding) => Buffer.from(new Float32Array(embedding).buffer);
|
|
1351
1402
|
var Minimem = class _Minimem {
|
|
1352
1403
|
memoryDir;
|
|
1353
1404
|
dbPath;
|
|
@@ -1373,10 +1424,11 @@ var Minimem = class _Minimem {
|
|
|
1373
1424
|
closed = false;
|
|
1374
1425
|
dirty = true;
|
|
1375
1426
|
syncing = null;
|
|
1427
|
+
syncLock = false;
|
|
1376
1428
|
embeddingOptions;
|
|
1377
1429
|
constructor(config) {
|
|
1378
|
-
this.memoryDir =
|
|
1379
|
-
this.dbPath = config.dbPath ??
|
|
1430
|
+
this.memoryDir = path2.resolve(config.memoryDir);
|
|
1431
|
+
this.dbPath = config.dbPath ?? path2.join(this.memoryDir, ".minimem", "index.db");
|
|
1380
1432
|
this.chunking = {
|
|
1381
1433
|
tokens: config.chunking?.tokens ?? 256,
|
|
1382
1434
|
overlap: config.chunking?.overlap ?? 32
|
|
@@ -1442,7 +1494,7 @@ var Minimem = class _Minimem {
|
|
|
1442
1494
|
}
|
|
1443
1495
|
}
|
|
1444
1496
|
openDatabase() {
|
|
1445
|
-
const dbDir =
|
|
1497
|
+
const dbDir = path2.dirname(this.dbPath);
|
|
1446
1498
|
ensureDir(dbDir);
|
|
1447
1499
|
return new DatabaseSync(this.dbPath);
|
|
1448
1500
|
}
|
|
@@ -1482,8 +1534,8 @@ var Minimem = class _Minimem {
|
|
|
1482
1534
|
}
|
|
1483
1535
|
ensureWatcher() {
|
|
1484
1536
|
if (this.watcher) return;
|
|
1485
|
-
const memorySubDir =
|
|
1486
|
-
const memoryFile =
|
|
1537
|
+
const memorySubDir = path2.join(this.memoryDir, "memory");
|
|
1538
|
+
const memoryFile = path2.join(this.memoryDir, "MEMORY.md");
|
|
1487
1539
|
this.watcher = chokidar.watch([memoryFile, memorySubDir], {
|
|
1488
1540
|
ignoreInitial: true,
|
|
1489
1541
|
persistent: true,
|
|
@@ -1516,13 +1568,13 @@ var Minimem = class _Minimem {
|
|
|
1516
1568
|
}
|
|
1517
1569
|
const storedMap = new Map(stored.map((f) => [f.path, f.mtime]));
|
|
1518
1570
|
for (const absPath of files) {
|
|
1519
|
-
const relPath =
|
|
1571
|
+
const relPath = path2.relative(this.memoryDir, absPath).replace(/\\/g, "/");
|
|
1520
1572
|
const storedMtime = storedMap.get(relPath);
|
|
1521
1573
|
if (storedMtime === void 0) {
|
|
1522
1574
|
this.debug?.(`Stale: new file ${relPath}`);
|
|
1523
1575
|
return true;
|
|
1524
1576
|
}
|
|
1525
|
-
const stat = await
|
|
1577
|
+
const stat = await fs.stat(absPath);
|
|
1526
1578
|
const currentMtime = Math.floor(stat.mtimeMs);
|
|
1527
1579
|
if (currentMtime !== storedMtime) {
|
|
1528
1580
|
this.debug?.(`Stale: mtime changed for ${relPath}`);
|
|
@@ -1572,8 +1624,14 @@ var Minimem = class _Minimem {
|
|
|
1572
1624
|
sourceFilterVec: sourceFilter,
|
|
1573
1625
|
sourceFilterChunks: sourceFilter
|
|
1574
1626
|
}).catch(() => []) : [];
|
|
1627
|
+
const typeFilterFn = opts?.type ? (id) => {
|
|
1628
|
+
const row = this.db.prepare(`SELECT type FROM chunks WHERE id = ?`).get(id);
|
|
1629
|
+
return row?.type === opts.type;
|
|
1630
|
+
} : void 0;
|
|
1575
1631
|
if (!this.hybrid.enabled) {
|
|
1576
|
-
|
|
1632
|
+
let results = vectorResults;
|
|
1633
|
+
if (typeFilterFn) results = results.filter((r) => typeFilterFn(r.id));
|
|
1634
|
+
return results.filter((entry) => entry.score >= minScore).slice(0, maxResults).map((r) => ({
|
|
1577
1635
|
path: r.path,
|
|
1578
1636
|
startLine: r.startLine,
|
|
1579
1637
|
endLine: r.endLine,
|
|
@@ -1581,8 +1639,14 @@ var Minimem = class _Minimem {
|
|
|
1581
1639
|
snippet: r.snippet
|
|
1582
1640
|
}));
|
|
1583
1641
|
}
|
|
1642
|
+
let filteredVector = vectorResults;
|
|
1643
|
+
let filteredKeyword = keywordResults;
|
|
1644
|
+
if (typeFilterFn) {
|
|
1645
|
+
filteredVector = vectorResults.filter((r) => typeFilterFn(r.id));
|
|
1646
|
+
filteredKeyword = keywordResults.filter((r) => typeFilterFn(r.id));
|
|
1647
|
+
}
|
|
1584
1648
|
const merged = mergeHybridResults({
|
|
1585
|
-
vector:
|
|
1649
|
+
vector: filteredVector.map((r) => ({
|
|
1586
1650
|
id: r.id,
|
|
1587
1651
|
path: r.path,
|
|
1588
1652
|
startLine: r.startLine,
|
|
@@ -1591,7 +1655,7 @@ var Minimem = class _Minimem {
|
|
|
1591
1655
|
snippet: r.snippet,
|
|
1592
1656
|
vectorScore: r.score
|
|
1593
1657
|
})),
|
|
1594
|
-
keyword:
|
|
1658
|
+
keyword: filteredKeyword.map((r) => ({
|
|
1595
1659
|
id: r.id,
|
|
1596
1660
|
path: r.path,
|
|
1597
1661
|
startLine: r.startLine,
|
|
@@ -1616,11 +1680,16 @@ var Minimem = class _Minimem {
|
|
|
1616
1680
|
await this.syncing;
|
|
1617
1681
|
return;
|
|
1618
1682
|
}
|
|
1683
|
+
if (this.syncLock) {
|
|
1684
|
+
return;
|
|
1685
|
+
}
|
|
1686
|
+
this.syncLock = true;
|
|
1619
1687
|
this.syncing = this.runSync(opts);
|
|
1620
1688
|
try {
|
|
1621
1689
|
await this.syncing;
|
|
1622
1690
|
} finally {
|
|
1623
1691
|
this.syncing = null;
|
|
1692
|
+
this.syncLock = false;
|
|
1624
1693
|
}
|
|
1625
1694
|
}
|
|
1626
1695
|
async runSync(opts) {
|
|
@@ -1647,13 +1716,16 @@ var Minimem = class _Minimem {
|
|
|
1647
1716
|
this.db.prepare(
|
|
1648
1717
|
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`
|
|
1649
1718
|
).run(stale.path, "memory");
|
|
1650
|
-
} catch {
|
|
1719
|
+
} catch (err) {
|
|
1720
|
+
logError("deleteStaleVectorEntries", err, this.debug);
|
|
1651
1721
|
}
|
|
1652
1722
|
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "memory");
|
|
1723
|
+
this.db.prepare(`DELETE FROM knowledge_links WHERE source_path = ?`).run(stale.path);
|
|
1653
1724
|
if (this.fts.enabled && this.fts.available) {
|
|
1654
1725
|
try {
|
|
1655
1726
|
this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`).run(stale.path, "memory", this.provider.model);
|
|
1656
|
-
} catch {
|
|
1727
|
+
} catch (err) {
|
|
1728
|
+
logError("deleteStaleFtsEntries", err, this.debug);
|
|
1657
1729
|
}
|
|
1658
1730
|
}
|
|
1659
1731
|
}
|
|
@@ -1670,8 +1742,15 @@ var Minimem = class _Minimem {
|
|
|
1670
1742
|
this.debug?.(`memory sync complete`, { files: files.length });
|
|
1671
1743
|
}
|
|
1672
1744
|
async indexFile(entry) {
|
|
1673
|
-
const content = await
|
|
1745
|
+
const content = await fs.readFile(entry.absPath, "utf-8");
|
|
1674
1746
|
const chunks = chunkMarkdown(content, this.chunking);
|
|
1747
|
+
const { frontmatter } = parseFrontmatter(content);
|
|
1748
|
+
const knowledgeType = frontmatter?.type ?? null;
|
|
1749
|
+
const knowledgeId = frontmatter?.id ?? null;
|
|
1750
|
+
const domains = frontmatter?.domain ?? null;
|
|
1751
|
+
const entities = frontmatter?.entities ?? null;
|
|
1752
|
+
const confidence = frontmatter?.confidence ?? null;
|
|
1753
|
+
const links = frontmatter?.links ?? null;
|
|
1675
1754
|
const embeddings = await this.embedChunks(chunks);
|
|
1676
1755
|
this.db.prepare(
|
|
1677
1756
|
`INSERT OR REPLACE INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)`
|
|
@@ -1680,23 +1759,27 @@ var Minimem = class _Minimem {
|
|
|
1680
1759
|
this.db.prepare(
|
|
1681
1760
|
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`
|
|
1682
1761
|
).run(entry.path, "memory");
|
|
1683
|
-
} catch {
|
|
1762
|
+
} catch (err) {
|
|
1763
|
+
logError("deleteOldVectorChunks", err, this.debug);
|
|
1684
1764
|
}
|
|
1685
1765
|
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(entry.path, "memory");
|
|
1686
1766
|
if (this.fts.enabled && this.fts.available) {
|
|
1687
1767
|
try {
|
|
1688
1768
|
this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`).run(entry.path, "memory", this.provider.model);
|
|
1689
|
-
} catch {
|
|
1769
|
+
} catch (err) {
|
|
1770
|
+
logError("deleteOldFtsChunks", err, this.debug);
|
|
1690
1771
|
}
|
|
1691
1772
|
}
|
|
1773
|
+
this.db.prepare(`DELETE FROM knowledge_links WHERE source_path = ?`).run(entry.path);
|
|
1692
1774
|
const now = Date.now();
|
|
1693
1775
|
for (let i = 0; i < chunks.length; i++) {
|
|
1694
1776
|
const chunk = chunks[i];
|
|
1695
1777
|
const embedding = embeddings[i] ?? [];
|
|
1696
1778
|
const chunkId = randomUUID();
|
|
1779
|
+
const meta = extractChunkMetadata(chunk.text);
|
|
1697
1780
|
this.db.prepare(
|
|
1698
|
-
`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
|
|
1699
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1781
|
+
`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at, type, knowledge_type, knowledge_id, domains, entities, confidence)
|
|
1782
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
1700
1783
|
).run(
|
|
1701
1784
|
chunkId,
|
|
1702
1785
|
entry.path,
|
|
@@ -1707,7 +1790,13 @@ var Minimem = class _Minimem {
|
|
|
1707
1790
|
this.provider.model,
|
|
1708
1791
|
chunk.text,
|
|
1709
1792
|
JSON.stringify(embedding),
|
|
1710
|
-
now
|
|
1793
|
+
now,
|
|
1794
|
+
meta.type ?? null,
|
|
1795
|
+
knowledgeType,
|
|
1796
|
+
knowledgeId,
|
|
1797
|
+
domains ? JSON.stringify(domains) : null,
|
|
1798
|
+
entities ? JSON.stringify(entities) : null,
|
|
1799
|
+
confidence
|
|
1711
1800
|
);
|
|
1712
1801
|
if (this.vector.available && embedding.length > 0) {
|
|
1713
1802
|
if (!this.vector.dims) {
|
|
@@ -1715,8 +1804,9 @@ var Minimem = class _Minimem {
|
|
|
1715
1804
|
this.ensureVectorTable(embedding.length);
|
|
1716
1805
|
}
|
|
1717
1806
|
try {
|
|
1718
|
-
this.db.prepare(`INSERT INTO ${VECTOR_TABLE} (id, embedding) VALUES (?, ?)`).run(chunkId,
|
|
1719
|
-
} catch {
|
|
1807
|
+
this.db.prepare(`INSERT INTO ${VECTOR_TABLE} (id, embedding) VALUES (?, ?)`).run(chunkId, vectorToBlob(embedding));
|
|
1808
|
+
} catch (err) {
|
|
1809
|
+
logError("insertVectorChunk", err, this.debug);
|
|
1720
1810
|
}
|
|
1721
1811
|
}
|
|
1722
1812
|
if (this.fts.enabled && this.fts.available) {
|
|
@@ -1733,10 +1823,28 @@ var Minimem = class _Minimem {
|
|
|
1733
1823
|
chunk.startLine,
|
|
1734
1824
|
chunk.endLine
|
|
1735
1825
|
);
|
|
1736
|
-
} catch {
|
|
1826
|
+
} catch (err) {
|
|
1827
|
+
logError("insertFtsChunk", err, this.debug);
|
|
1737
1828
|
}
|
|
1738
1829
|
}
|
|
1739
1830
|
}
|
|
1831
|
+
if (links && knowledgeId) {
|
|
1832
|
+
const upsertLink = this.db.prepare(
|
|
1833
|
+
`INSERT OR REPLACE INTO knowledge_links (from_id, to_id, relation, layer, weight, source_path, created_at)
|
|
1834
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
1835
|
+
);
|
|
1836
|
+
for (const link of links) {
|
|
1837
|
+
upsertLink.run(
|
|
1838
|
+
knowledgeId,
|
|
1839
|
+
link.target,
|
|
1840
|
+
link.relation,
|
|
1841
|
+
link.layer ?? null,
|
|
1842
|
+
0.5,
|
|
1843
|
+
entry.path,
|
|
1844
|
+
now
|
|
1845
|
+
);
|
|
1846
|
+
}
|
|
1847
|
+
}
|
|
1740
1848
|
}
|
|
1741
1849
|
async embedChunks(chunks) {
|
|
1742
1850
|
if (chunks.length === 0) return [];
|
|
@@ -1828,12 +1936,22 @@ var Minimem = class _Minimem {
|
|
|
1828
1936
|
}
|
|
1829
1937
|
async embedQueryWithTimeout(text) {
|
|
1830
1938
|
const timeout = this.provider.id === "local" ? EMBEDDING_QUERY_TIMEOUT_LOCAL_MS : EMBEDDING_QUERY_TIMEOUT_REMOTE_MS;
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
1939
|
+
const ac = new AbortController();
|
|
1940
|
+
const timer = setTimeout(() => ac.abort(), timeout);
|
|
1941
|
+
try {
|
|
1942
|
+
const result = await Promise.race([
|
|
1943
|
+
this.provider.embedQuery(text),
|
|
1944
|
+
new Promise((_, reject) => {
|
|
1945
|
+
ac.signal.addEventListener(
|
|
1946
|
+
"abort",
|
|
1947
|
+
() => reject(new Error("embedding query timeout"))
|
|
1948
|
+
);
|
|
1949
|
+
})
|
|
1950
|
+
]);
|
|
1951
|
+
return result;
|
|
1952
|
+
} finally {
|
|
1953
|
+
clearTimeout(timer);
|
|
1954
|
+
}
|
|
1837
1955
|
}
|
|
1838
1956
|
loadEmbeddingCache(hashes) {
|
|
1839
1957
|
const result = /* @__PURE__ */ new Map();
|
|
@@ -1926,9 +2044,9 @@ var Minimem = class _Minimem {
|
|
|
1926
2044
|
}
|
|
1927
2045
|
}
|
|
1928
2046
|
async readFile(relativePath) {
|
|
1929
|
-
const absPath =
|
|
2047
|
+
const absPath = path2.join(this.memoryDir, relativePath);
|
|
1930
2048
|
try {
|
|
1931
|
-
return await
|
|
2049
|
+
return await fs.readFile(absPath, "utf-8");
|
|
1932
2050
|
} catch {
|
|
1933
2051
|
return null;
|
|
1934
2052
|
}
|
|
@@ -1956,10 +2074,10 @@ var Minimem = class _Minimem {
|
|
|
1956
2074
|
*/
|
|
1957
2075
|
async writeFile(relativePath, content) {
|
|
1958
2076
|
this.validateMemoryPath(relativePath);
|
|
1959
|
-
const absPath =
|
|
1960
|
-
const dir =
|
|
1961
|
-
await
|
|
1962
|
-
await
|
|
2077
|
+
const absPath = path2.join(this.memoryDir, relativePath);
|
|
2078
|
+
const dir = path2.dirname(absPath);
|
|
2079
|
+
await fs.mkdir(dir, { recursive: true });
|
|
2080
|
+
await fs.writeFile(absPath, content, "utf-8");
|
|
1963
2081
|
this.dirty = true;
|
|
1964
2082
|
this.debug?.(`memory write: ${relativePath}`);
|
|
1965
2083
|
}
|
|
@@ -1968,18 +2086,18 @@ var Minimem = class _Minimem {
|
|
|
1968
2086
|
*/
|
|
1969
2087
|
async appendFile(relativePath, content) {
|
|
1970
2088
|
this.validateMemoryPath(relativePath);
|
|
1971
|
-
const absPath =
|
|
1972
|
-
const dir =
|
|
1973
|
-
await
|
|
2089
|
+
const absPath = path2.join(this.memoryDir, relativePath);
|
|
2090
|
+
const dir = path2.dirname(absPath);
|
|
2091
|
+
await fs.mkdir(dir, { recursive: true });
|
|
1974
2092
|
let toAppend = content;
|
|
1975
2093
|
try {
|
|
1976
|
-
const existing = await
|
|
2094
|
+
const existing = await fs.readFile(absPath, "utf-8");
|
|
1977
2095
|
if (existing.length > 0 && !existing.endsWith("\n")) {
|
|
1978
2096
|
toAppend = "\n" + content;
|
|
1979
2097
|
}
|
|
1980
2098
|
} catch {
|
|
1981
2099
|
}
|
|
1982
|
-
await
|
|
2100
|
+
await fs.appendFile(absPath, toAppend, "utf-8");
|
|
1983
2101
|
this.dirty = true;
|
|
1984
2102
|
this.debug?.(`memory append: ${relativePath}`);
|
|
1985
2103
|
}
|
|
@@ -1997,7 +2115,7 @@ var Minimem = class _Minimem {
|
|
|
1997
2115
|
*/
|
|
1998
2116
|
async listFiles() {
|
|
1999
2117
|
const files = await listMemoryFiles(this.memoryDir);
|
|
2000
|
-
return files.map((f) =>
|
|
2118
|
+
return files.map((f) => path2.relative(this.memoryDir, f).replace(/\\/g, "/"));
|
|
2001
2119
|
}
|
|
2002
2120
|
/**
|
|
2003
2121
|
* Validate that a path is within allowed memory locations
|
|
@@ -2035,6 +2153,70 @@ var Minimem = class _Minimem {
|
|
|
2035
2153
|
cacheCount: cacheRow.count
|
|
2036
2154
|
};
|
|
2037
2155
|
}
|
|
2156
|
+
/**
|
|
2157
|
+
* Search with knowledge metadata filters (domain, entities, confidence, type).
|
|
2158
|
+
* Runs a standard search then post-filters by knowledge columns.
|
|
2159
|
+
*/
|
|
2160
|
+
async knowledgeSearch(query, opts) {
|
|
2161
|
+
if (this.dirty || !this.watchConfig.enabled && await this.isStale()) {
|
|
2162
|
+
await this.sync({ reason: "knowledgeSearch" });
|
|
2163
|
+
}
|
|
2164
|
+
const cleaned = query.trim();
|
|
2165
|
+
if (!cleaned) return [];
|
|
2166
|
+
const minScore = opts?.minScore ?? this.queryConfig.minScore;
|
|
2167
|
+
const maxResults = opts?.maxResults ?? this.queryConfig.maxResults;
|
|
2168
|
+
const { sql: knowledgeWhere, params: knowledgeParams } = buildKnowledgeFilterSql({
|
|
2169
|
+
domain: opts?.domain,
|
|
2170
|
+
entities: opts?.entities,
|
|
2171
|
+
minConfidence: opts?.minConfidence,
|
|
2172
|
+
knowledgeType: opts?.knowledgeType
|
|
2173
|
+
});
|
|
2174
|
+
if (!knowledgeWhere) {
|
|
2175
|
+
return this.search(query, { maxResults, minScore });
|
|
2176
|
+
}
|
|
2177
|
+
const matchingRows = this.db.prepare(
|
|
2178
|
+
`SELECT id FROM chunks c WHERE c.model = ? AND c.source = 'memory'${knowledgeWhere}`
|
|
2179
|
+
).all(this.provider.model, ...knowledgeParams);
|
|
2180
|
+
const matchingIds = new Set(matchingRows.map((r) => r.id));
|
|
2181
|
+
if (matchingIds.size === 0) return [];
|
|
2182
|
+
const overFetch = Math.max(maxResults * 3, 30);
|
|
2183
|
+
const results = await this.search(query, {
|
|
2184
|
+
maxResults: overFetch,
|
|
2185
|
+
minScore
|
|
2186
|
+
});
|
|
2187
|
+
const filtered = [];
|
|
2188
|
+
for (const r of results) {
|
|
2189
|
+
const row = this.db.prepare(
|
|
2190
|
+
`SELECT id FROM chunks WHERE path = ? AND start_line = ? AND end_line = ? AND model = ?`
|
|
2191
|
+
).get(r.path, r.startLine, r.endLine, this.provider.model);
|
|
2192
|
+
if (row && matchingIds.has(row.id)) {
|
|
2193
|
+
filtered.push(r);
|
|
2194
|
+
if (filtered.length >= maxResults) break;
|
|
2195
|
+
}
|
|
2196
|
+
}
|
|
2197
|
+
return filtered;
|
|
2198
|
+
}
|
|
2199
|
+
/**
|
|
2200
|
+
* Get knowledge graph links from or to a node.
|
|
2201
|
+
*/
|
|
2202
|
+
getLinks(nodeId, direction = "from", opts) {
|
|
2203
|
+
if (direction === "from") {
|
|
2204
|
+
return getLinksFrom(this.db, nodeId, opts);
|
|
2205
|
+
}
|
|
2206
|
+
return getLinksTo(this.db, nodeId, opts);
|
|
2207
|
+
}
|
|
2208
|
+
/**
|
|
2209
|
+
* Get neighbor nodes via BFS traversal.
|
|
2210
|
+
*/
|
|
2211
|
+
getGraphNeighbors(nodeId, depth = 1, opts) {
|
|
2212
|
+
return getNeighbors(this.db, nodeId, depth, opts);
|
|
2213
|
+
}
|
|
2214
|
+
/**
|
|
2215
|
+
* Find shortest path between two knowledge nodes.
|
|
2216
|
+
*/
|
|
2217
|
+
getGraphPath(fromId, toId, maxDepth = 3) {
|
|
2218
|
+
return getPathBetween(this.db, fromId, toId, maxDepth);
|
|
2219
|
+
}
|
|
2038
2220
|
close() {
|
|
2039
2221
|
if (this.closed) return;
|
|
2040
2222
|
this.closed = true;
|
|
@@ -2048,7 +2230,8 @@ var Minimem = class _Minimem {
|
|
|
2048
2230
|
}
|
|
2049
2231
|
try {
|
|
2050
2232
|
this.db.close();
|
|
2051
|
-
} catch {
|
|
2233
|
+
} catch (err) {
|
|
2234
|
+
logError("dbClose", err, this.debug);
|
|
2052
2235
|
}
|
|
2053
2236
|
}
|
|
2054
2237
|
};
|
|
@@ -2076,12 +2259,152 @@ var MEMORY_SEARCH_TOOL = {
|
|
|
2076
2259
|
type: "array",
|
|
2077
2260
|
items: { type: "string" },
|
|
2078
2261
|
description: "Optional: filter to specific memory directories by name/path. If omitted, searches all configured directories."
|
|
2262
|
+
},
|
|
2263
|
+
detail: {
|
|
2264
|
+
type: "string",
|
|
2265
|
+
enum: ["compact", "full"],
|
|
2266
|
+
description: "Result detail level. 'compact' returns a lightweight index with short previews (~80 chars). 'full' returns complete snippets. Use 'compact' first, then memory_get_details for selected results. (default: 'compact')"
|
|
2267
|
+
},
|
|
2268
|
+
type: {
|
|
2269
|
+
type: "string",
|
|
2270
|
+
description: "Filter by observation type. Matches <!-- type: X --> comments in memory entries. Common types: decision, bugfix, feature, discovery, context, note."
|
|
2271
|
+
}
|
|
2272
|
+
},
|
|
2273
|
+
required: ["query"]
|
|
2274
|
+
}
|
|
2275
|
+
};
|
|
2276
|
+
var MEMORY_GET_DETAILS_TOOL = {
|
|
2277
|
+
name: "memory_get_details",
|
|
2278
|
+
description: "Fetch full text for specific memory chunks identified by path and line range. Use after memory_search with compact results to get details for selected items only. This two-step approach significantly reduces token usage.",
|
|
2279
|
+
inputSchema: {
|
|
2280
|
+
type: "object",
|
|
2281
|
+
properties: {
|
|
2282
|
+
results: {
|
|
2283
|
+
type: "array",
|
|
2284
|
+
items: { type: "object" },
|
|
2285
|
+
description: "Array of { path, startLine, endLine } objects from compact search results."
|
|
2286
|
+
},
|
|
2287
|
+
directories: {
|
|
2288
|
+
type: "array",
|
|
2289
|
+
items: { type: "string" },
|
|
2290
|
+
description: "Optional: filter to specific memory directories."
|
|
2291
|
+
}
|
|
2292
|
+
},
|
|
2293
|
+
required: ["results"]
|
|
2294
|
+
}
|
|
2295
|
+
};
|
|
2296
|
+
var KNOWLEDGE_SEARCH_TOOL = {
|
|
2297
|
+
name: "knowledge_search",
|
|
2298
|
+
description: "Search memory with knowledge metadata filters. Filter by domain, entities, confidence level, or knowledge type (observation, entity, domain-summary). Combines semantic search with structured knowledge filtering.",
|
|
2299
|
+
inputSchema: {
|
|
2300
|
+
type: "object",
|
|
2301
|
+
properties: {
|
|
2302
|
+
query: {
|
|
2303
|
+
type: "string",
|
|
2304
|
+
description: "Natural language search query"
|
|
2305
|
+
},
|
|
2306
|
+
domain: {
|
|
2307
|
+
type: "array",
|
|
2308
|
+
items: { type: "string" },
|
|
2309
|
+
description: "Filter to entries in these knowledge domains"
|
|
2310
|
+
},
|
|
2311
|
+
entities: {
|
|
2312
|
+
type: "array",
|
|
2313
|
+
items: { type: "string" },
|
|
2314
|
+
description: "Filter to entries referencing these entities"
|
|
2315
|
+
},
|
|
2316
|
+
minConfidence: {
|
|
2317
|
+
type: "number",
|
|
2318
|
+
description: "Minimum confidence threshold (0-1)"
|
|
2319
|
+
},
|
|
2320
|
+
knowledgeType: {
|
|
2321
|
+
type: "string",
|
|
2322
|
+
description: "Filter by knowledge type: observation, entity, domain-summary"
|
|
2323
|
+
},
|
|
2324
|
+
maxResults: {
|
|
2325
|
+
type: "number",
|
|
2326
|
+
description: "Maximum number of results (default: 10)"
|
|
2327
|
+
},
|
|
2328
|
+
minScore: {
|
|
2329
|
+
type: "number",
|
|
2330
|
+
description: "Minimum relevance score 0-1 (default: 0.3)"
|
|
2331
|
+
},
|
|
2332
|
+
directories: {
|
|
2333
|
+
type: "array",
|
|
2334
|
+
items: { type: "string" },
|
|
2335
|
+
description: "Optional: filter to specific memory directories"
|
|
2079
2336
|
}
|
|
2080
2337
|
},
|
|
2081
2338
|
required: ["query"]
|
|
2082
2339
|
}
|
|
2083
2340
|
};
|
|
2084
|
-
var
|
|
2341
|
+
var KNOWLEDGE_GRAPH_TOOL = {
|
|
2342
|
+
name: "knowledge_graph",
|
|
2343
|
+
description: "Traverse knowledge graph links from a note. Returns neighbor nodes connected by typed relationships (e.g., relates-to, supports, contradicts). Use depth parameter for multi-hop traversal.",
|
|
2344
|
+
inputSchema: {
|
|
2345
|
+
type: "object",
|
|
2346
|
+
properties: {
|
|
2347
|
+
nodeId: {
|
|
2348
|
+
type: "string",
|
|
2349
|
+
description: "The knowledge node ID to start traversal from"
|
|
2350
|
+
},
|
|
2351
|
+
depth: {
|
|
2352
|
+
type: "number",
|
|
2353
|
+
description: "Maximum traversal depth (default: 1, max: 3)",
|
|
2354
|
+
default: 1
|
|
2355
|
+
},
|
|
2356
|
+
relation: {
|
|
2357
|
+
type: "string",
|
|
2358
|
+
description: "Optional: filter to specific relation type"
|
|
2359
|
+
},
|
|
2360
|
+
layer: {
|
|
2361
|
+
type: "string",
|
|
2362
|
+
description: "Optional: filter to specific graph layer"
|
|
2363
|
+
},
|
|
2364
|
+
directories: {
|
|
2365
|
+
type: "array",
|
|
2366
|
+
items: { type: "string" },
|
|
2367
|
+
description: "Optional: filter to specific memory directories"
|
|
2368
|
+
}
|
|
2369
|
+
},
|
|
2370
|
+
required: ["nodeId"]
|
|
2371
|
+
}
|
|
2372
|
+
};
|
|
2373
|
+
var KNOWLEDGE_PATH_TOOL = {
|
|
2374
|
+
name: "knowledge_path",
|
|
2375
|
+
description: "Find the shortest path between two knowledge nodes in the graph. Uses BFS traversal up to a configurable max depth. Returns the sequence of links connecting the two nodes.",
|
|
2376
|
+
inputSchema: {
|
|
2377
|
+
type: "object",
|
|
2378
|
+
properties: {
|
|
2379
|
+
fromId: {
|
|
2380
|
+
type: "string",
|
|
2381
|
+
description: "Starting knowledge node ID"
|
|
2382
|
+
},
|
|
2383
|
+
toId: {
|
|
2384
|
+
type: "string",
|
|
2385
|
+
description: "Target knowledge node ID"
|
|
2386
|
+
},
|
|
2387
|
+
maxDepth: {
|
|
2388
|
+
type: "number",
|
|
2389
|
+
description: "Maximum path length (default: 3)",
|
|
2390
|
+
default: 3
|
|
2391
|
+
},
|
|
2392
|
+
directories: {
|
|
2393
|
+
type: "array",
|
|
2394
|
+
items: { type: "string" },
|
|
2395
|
+
description: "Optional: filter to specific memory directories"
|
|
2396
|
+
}
|
|
2397
|
+
},
|
|
2398
|
+
required: ["fromId", "toId"]
|
|
2399
|
+
}
|
|
2400
|
+
};
|
|
2401
|
+
var MEMORY_TOOLS = [
|
|
2402
|
+
MEMORY_SEARCH_TOOL,
|
|
2403
|
+
MEMORY_GET_DETAILS_TOOL,
|
|
2404
|
+
KNOWLEDGE_SEARCH_TOOL,
|
|
2405
|
+
KNOWLEDGE_GRAPH_TOOL,
|
|
2406
|
+
KNOWLEDGE_PATH_TOOL
|
|
2407
|
+
];
|
|
2085
2408
|
function getToolDefinitions() {
|
|
2086
2409
|
return MEMORY_TOOLS;
|
|
2087
2410
|
}
|
|
@@ -2110,6 +2433,14 @@ var MemoryToolExecutor = class {
|
|
|
2110
2433
|
switch (toolName) {
|
|
2111
2434
|
case "memory_search":
|
|
2112
2435
|
return await this.memorySearch(params);
|
|
2436
|
+
case "memory_get_details":
|
|
2437
|
+
return await this.memoryGetDetails(params);
|
|
2438
|
+
case "knowledge_search":
|
|
2439
|
+
return await this.knowledgeSearch(params);
|
|
2440
|
+
case "knowledge_graph":
|
|
2441
|
+
return await this.knowledgeGraph(params);
|
|
2442
|
+
case "knowledge_path":
|
|
2443
|
+
return await this.knowledgePath(params);
|
|
2113
2444
|
default:
|
|
2114
2445
|
return {
|
|
2115
2446
|
content: [{ type: "text", text: `Unknown tool: ${toolName}` }],
|
|
@@ -2124,37 +2455,43 @@ var MemoryToolExecutor = class {
|
|
|
2124
2455
|
};
|
|
2125
2456
|
}
|
|
2126
2457
|
}
|
|
2458
|
+
/**
|
|
2459
|
+
* Filter instances by directory names/paths
|
|
2460
|
+
*/
|
|
2461
|
+
filterInstances(directories) {
|
|
2462
|
+
if (!directories || directories.length === 0) return this.instances;
|
|
2463
|
+
const dirFilter = new Set(directories.map((d) => d.toLowerCase()));
|
|
2464
|
+
const filtered = this.instances.filter((i) => {
|
|
2465
|
+
const name = (i.name ?? i.memoryDir).toLowerCase();
|
|
2466
|
+
const dir = i.memoryDir.toLowerCase();
|
|
2467
|
+
return dirFilter.has(name) || dirFilter.has(dir) || [...dirFilter].some((f) => dir.includes(f) || name.includes(f));
|
|
2468
|
+
});
|
|
2469
|
+
return filtered.length > 0 ? filtered : null;
|
|
2470
|
+
}
|
|
2127
2471
|
async memorySearch(params) {
|
|
2128
2472
|
const maxResults = params.maxResults ?? 10;
|
|
2129
2473
|
const minScore = params.minScore;
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
|
|
2141
|
-
|
|
2142
|
-
|
|
2143
|
-
{
|
|
2144
|
-
type: "text",
|
|
2145
|
-
text: `No matching directories found. Available: ${available}`
|
|
2146
|
-
}
|
|
2147
|
-
],
|
|
2148
|
-
isError: true
|
|
2149
|
-
};
|
|
2150
|
-
}
|
|
2474
|
+
const detail = params.detail ?? "compact";
|
|
2475
|
+
const instancesToSearch = this.filterInstances(params.directories);
|
|
2476
|
+
if (!instancesToSearch) {
|
|
2477
|
+
const available = this.getDirectories().join(", ");
|
|
2478
|
+
return {
|
|
2479
|
+
content: [
|
|
2480
|
+
{
|
|
2481
|
+
type: "text",
|
|
2482
|
+
text: `No matching directories found. Available: ${available}`
|
|
2483
|
+
}
|
|
2484
|
+
],
|
|
2485
|
+
isError: true
|
|
2486
|
+
};
|
|
2151
2487
|
}
|
|
2152
2488
|
const allResults = [];
|
|
2153
2489
|
for (const instance of instancesToSearch) {
|
|
2154
2490
|
const perDirMax = Math.ceil(maxResults * 1.5);
|
|
2155
2491
|
const results = await instance.minimem.search(params.query, {
|
|
2156
2492
|
maxResults: perDirMax,
|
|
2157
|
-
minScore
|
|
2493
|
+
minScore,
|
|
2494
|
+
type: params.type
|
|
2158
2495
|
});
|
|
2159
2496
|
for (const result of results) {
|
|
2160
2497
|
allResults.push({
|
|
@@ -2171,21 +2508,203 @@ var MemoryToolExecutor = class {
|
|
|
2171
2508
|
};
|
|
2172
2509
|
}
|
|
2173
2510
|
const showSource = instancesToSearch.length > 1;
|
|
2174
|
-
|
|
2511
|
+
if (detail === "compact") {
|
|
2512
|
+
return this.formatCompactResults(topResults, showSource, instancesToSearch.length);
|
|
2513
|
+
}
|
|
2514
|
+
return this.formatFullResults(topResults, showSource, instancesToSearch.length);
|
|
2515
|
+
}
|
|
2516
|
+
formatCompactResults(results, showSource, dirCount) {
|
|
2517
|
+
const formatted = results.map((r, i) => {
|
|
2518
|
+
const location = `${r.path}:${r.startLine}-${r.endLine}`;
|
|
2519
|
+
const score = (r.score * 100).toFixed(0);
|
|
2520
|
+
const source = showSource ? ` [${r.memoryDir}]` : "";
|
|
2521
|
+
const preview = compactPreview(r.snippet);
|
|
2522
|
+
return `[${i}] ${location}${source} (${score}%) \u2014 ${preview}`;
|
|
2523
|
+
}).join("\n");
|
|
2524
|
+
const hint = "\n\nUse memory_get_details to fetch full text for selected results.";
|
|
2525
|
+
const dirSummary = dirCount > 1 ? `
|
|
2526
|
+
(Searched ${dirCount} directories)` : "";
|
|
2527
|
+
return {
|
|
2528
|
+
content: [{ type: "text", text: formatted + dirSummary + hint }]
|
|
2529
|
+
};
|
|
2530
|
+
}
|
|
2531
|
+
formatFullResults(results, showSource, dirCount) {
|
|
2532
|
+
const formatted = results.map((r, i) => {
|
|
2175
2533
|
const location = `${r.path}:${r.startLine}-${r.endLine}`;
|
|
2176
2534
|
const score = (r.score * 100).toFixed(1);
|
|
2177
2535
|
const source = showSource ? ` [${r.memoryDir}]` : "";
|
|
2178
2536
|
return `[${i + 1}] ${location}${source} (${score}% match)
|
|
2179
2537
|
${r.snippet}`;
|
|
2180
2538
|
}).join("\n\n");
|
|
2181
|
-
const dirSummary =
|
|
2539
|
+
const dirSummary = dirCount > 1 ? `
|
|
2182
2540
|
|
|
2183
|
-
(Searched ${
|
|
2541
|
+
(Searched ${dirCount} directories)` : "";
|
|
2184
2542
|
return {
|
|
2185
2543
|
content: [{ type: "text", text: formatted + dirSummary }]
|
|
2186
2544
|
};
|
|
2187
2545
|
}
|
|
2546
|
+
async memoryGetDetails(params) {
|
|
2547
|
+
if (!params.results || params.results.length === 0) {
|
|
2548
|
+
return {
|
|
2549
|
+
content: [{ type: "text", text: "No results specified." }],
|
|
2550
|
+
isError: true
|
|
2551
|
+
};
|
|
2552
|
+
}
|
|
2553
|
+
const instancesToSearch = this.filterInstances(params.directories);
|
|
2554
|
+
if (!instancesToSearch) {
|
|
2555
|
+
const available = this.getDirectories().join(", ");
|
|
2556
|
+
return {
|
|
2557
|
+
content: [
|
|
2558
|
+
{
|
|
2559
|
+
type: "text",
|
|
2560
|
+
text: `No matching directories found. Available: ${available}`
|
|
2561
|
+
}
|
|
2562
|
+
],
|
|
2563
|
+
isError: true
|
|
2564
|
+
};
|
|
2565
|
+
}
|
|
2566
|
+
const details = [];
|
|
2567
|
+
for (const ref of params.results) {
|
|
2568
|
+
let found = false;
|
|
2569
|
+
for (const instance of instancesToSearch) {
|
|
2570
|
+
const lineCount = ref.endLine - ref.startLine + 1;
|
|
2571
|
+
const result = await instance.minimem.readLines(ref.path, {
|
|
2572
|
+
from: ref.startLine,
|
|
2573
|
+
lines: lineCount
|
|
2574
|
+
});
|
|
2575
|
+
if (result) {
|
|
2576
|
+
const location = `${ref.path}:${result.startLine}-${result.endLine}`;
|
|
2577
|
+
details.push(`--- ${location} ---
|
|
2578
|
+
${result.content}`);
|
|
2579
|
+
found = true;
|
|
2580
|
+
break;
|
|
2581
|
+
}
|
|
2582
|
+
}
|
|
2583
|
+
if (!found) {
|
|
2584
|
+
details.push(`--- ${ref.path}:${ref.startLine}-${ref.endLine} ---
|
|
2585
|
+
(not found)`);
|
|
2586
|
+
}
|
|
2587
|
+
}
|
|
2588
|
+
return {
|
|
2589
|
+
content: [{ type: "text", text: details.join("\n\n") }]
|
|
2590
|
+
};
|
|
2591
|
+
}
|
|
2592
|
+
async knowledgeSearch(params) {
|
|
2593
|
+
const instancesToSearch = this.filterInstances(params.directories);
|
|
2594
|
+
if (!instancesToSearch) {
|
|
2595
|
+
const available = this.getDirectories().join(", ");
|
|
2596
|
+
return {
|
|
2597
|
+
content: [{ type: "text", text: `No matching directories found. Available: ${available}` }],
|
|
2598
|
+
isError: true
|
|
2599
|
+
};
|
|
2600
|
+
}
|
|
2601
|
+
const maxResults = params.maxResults ?? 10;
|
|
2602
|
+
const allResults = [];
|
|
2603
|
+
for (const instance of instancesToSearch) {
|
|
2604
|
+
const results = await instance.minimem.knowledgeSearch(params.query, {
|
|
2605
|
+
maxResults: Math.ceil(maxResults * 1.5),
|
|
2606
|
+
minScore: params.minScore,
|
|
2607
|
+
domain: params.domain,
|
|
2608
|
+
entities: params.entities,
|
|
2609
|
+
minConfidence: params.minConfidence,
|
|
2610
|
+
knowledgeType: params.knowledgeType
|
|
2611
|
+
});
|
|
2612
|
+
for (const result of results) {
|
|
2613
|
+
allResults.push({
|
|
2614
|
+
...result,
|
|
2615
|
+
memoryDir: instance.name ?? instance.memoryDir
|
|
2616
|
+
});
|
|
2617
|
+
}
|
|
2618
|
+
}
|
|
2619
|
+
allResults.sort((a, b) => b.score - a.score);
|
|
2620
|
+
const topResults = allResults.slice(0, maxResults);
|
|
2621
|
+
if (topResults.length === 0) {
|
|
2622
|
+
return { content: [{ type: "text", text: "No knowledge results found." }] };
|
|
2623
|
+
}
|
|
2624
|
+
const formatted = topResults.map((r, i) => {
|
|
2625
|
+
const location = `${r.path}:${r.startLine}-${r.endLine}`;
|
|
2626
|
+
const score = (r.score * 100).toFixed(0);
|
|
2627
|
+
const preview = compactPreview(r.snippet);
|
|
2628
|
+
return `[${i}] ${location} (${score}%) \u2014 ${preview}`;
|
|
2629
|
+
}).join("\n");
|
|
2630
|
+
return { content: [{ type: "text", text: formatted }] };
|
|
2631
|
+
}
|
|
2632
|
+
async knowledgeGraph(params) {
|
|
2633
|
+
const instancesToSearch = this.filterInstances(params.directories);
|
|
2634
|
+
if (!instancesToSearch) {
|
|
2635
|
+
const available = this.getDirectories().join(", ");
|
|
2636
|
+
return {
|
|
2637
|
+
content: [{ type: "text", text: `No matching directories found. Available: ${available}` }],
|
|
2638
|
+
isError: true
|
|
2639
|
+
};
|
|
2640
|
+
}
|
|
2641
|
+
const depth = Math.min(params.depth ?? 1, 3);
|
|
2642
|
+
const allNeighbors = [];
|
|
2643
|
+
for (const instance of instancesToSearch) {
|
|
2644
|
+
const neighbors = instance.minimem.getGraphNeighbors(params.nodeId, depth, {
|
|
2645
|
+
relation: params.relation,
|
|
2646
|
+
layer: params.layer
|
|
2647
|
+
});
|
|
2648
|
+
for (const n of neighbors) {
|
|
2649
|
+
allNeighbors.push({
|
|
2650
|
+
id: n.id,
|
|
2651
|
+
depth: n.depth,
|
|
2652
|
+
relation: n.link.relation,
|
|
2653
|
+
layer: n.link.layer,
|
|
2654
|
+
memoryDir: instance.name ?? instance.memoryDir
|
|
2655
|
+
});
|
|
2656
|
+
}
|
|
2657
|
+
}
|
|
2658
|
+
if (allNeighbors.length === 0) {
|
|
2659
|
+
return { content: [{ type: "text", text: `No neighbors found for node "${params.nodeId}".` }] };
|
|
2660
|
+
}
|
|
2661
|
+
const formatted = allNeighbors.map((n) => ` [depth=${n.depth}] ${n.id} \u2014(${n.relation})${n.layer ? ` [${n.layer}]` : ""}`).join("\n");
|
|
2662
|
+
return {
|
|
2663
|
+
content: [{ type: "text", text: `Neighbors of "${params.nodeId}":
|
|
2664
|
+
${formatted}` }]
|
|
2665
|
+
};
|
|
2666
|
+
}
|
|
2667
|
+
async knowledgePath(params) {
|
|
2668
|
+
const instancesToSearch = this.filterInstances(params.directories);
|
|
2669
|
+
if (!instancesToSearch) {
|
|
2670
|
+
const available = this.getDirectories().join(", ");
|
|
2671
|
+
return {
|
|
2672
|
+
content: [{ type: "text", text: `No matching directories found. Available: ${available}` }],
|
|
2673
|
+
isError: true
|
|
2674
|
+
};
|
|
2675
|
+
}
|
|
2676
|
+
const maxDepth = Math.min(params.maxDepth ?? 3, 5);
|
|
2677
|
+
for (const instance of instancesToSearch) {
|
|
2678
|
+
const path4 = instance.minimem.getGraphPath(params.fromId, params.toId, maxDepth);
|
|
2679
|
+
if (path4.length > 0) {
|
|
2680
|
+
const steps = path4.map((link) => ` ${link.fromId} \u2014(${link.relation})\u2192 ${link.toId}`).join("\n");
|
|
2681
|
+
return {
|
|
2682
|
+
content: [{
|
|
2683
|
+
type: "text",
|
|
2684
|
+
text: `Path from "${params.fromId}" to "${params.toId}" (${path4.length} steps):
|
|
2685
|
+
${steps}`
|
|
2686
|
+
}]
|
|
2687
|
+
};
|
|
2688
|
+
}
|
|
2689
|
+
}
|
|
2690
|
+
return {
|
|
2691
|
+
content: [{
|
|
2692
|
+
type: "text",
|
|
2693
|
+
text: `No path found from "${params.fromId}" to "${params.toId}" within depth ${maxDepth}.`
|
|
2694
|
+
}]
|
|
2695
|
+
};
|
|
2696
|
+
}
|
|
2188
2697
|
};
|
|
2698
|
+
function compactPreview(snippet) {
|
|
2699
|
+
const maxLen = 80;
|
|
2700
|
+
const lines = snippet.split("\n").filter((l) => l.trim());
|
|
2701
|
+
if (lines.length === 0) return "(empty)";
|
|
2702
|
+
const heading = lines.find((l) => l.startsWith("#"));
|
|
2703
|
+
const text = heading ?? lines[0];
|
|
2704
|
+
const cleaned = text.replace(/^#+\s*/, "").trim();
|
|
2705
|
+
if (cleaned.length <= maxLen) return `"${cleaned}"`;
|
|
2706
|
+
return `"${cleaned.slice(0, maxLen - 3)}..."`;
|
|
2707
|
+
}
|
|
2189
2708
|
function createToolExecutor(instances) {
|
|
2190
2709
|
return new MemoryToolExecutor(instances);
|
|
2191
2710
|
}
|
|
@@ -2347,133 +2866,570 @@ function generateMcpConfig(opts) {
|
|
|
2347
2866
|
};
|
|
2348
2867
|
}
|
|
2349
2868
|
|
|
2350
|
-
// src/
|
|
2351
|
-
import
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2869
|
+
// src/core/indexer.ts
|
|
2870
|
+
import { randomUUID as randomUUID2 } from "crypto";
|
|
2871
|
+
import fs2 from "fs/promises";
|
|
2872
|
+
import path3 from "path";
|
|
2873
|
+
var META_KEY2 = "memory_index_meta_v1";
|
|
2874
|
+
var EMBEDDING_CACHE_TABLE2 = "embedding_cache";
|
|
2875
|
+
var VECTOR_TABLE2 = "chunks_vec";
|
|
2876
|
+
var FTS_TABLE2 = "chunks_fts";
|
|
2877
|
+
var EMBEDDING_RETRY_MAX_ATTEMPTS2 = 3;
|
|
2878
|
+
var EMBEDDING_RETRY_BASE_DELAY_MS2 = 500;
|
|
2879
|
+
var EMBEDDING_RETRY_MAX_DELAY_MS2 = 8e3;
|
|
2880
|
+
var MemoryIndexer = class {
|
|
2881
|
+
config;
|
|
2882
|
+
db;
|
|
2883
|
+
provider;
|
|
2884
|
+
providerKey;
|
|
2885
|
+
openAi;
|
|
2886
|
+
gemini;
|
|
2887
|
+
// Vector/FTS state (shared with parent)
|
|
2888
|
+
vectorState;
|
|
2889
|
+
ftsAvailable;
|
|
2890
|
+
constructor(db, provider, config, options) {
|
|
2891
|
+
this.db = db;
|
|
2892
|
+
this.provider = provider;
|
|
2893
|
+
this.config = config;
|
|
2894
|
+
this.openAi = options?.openAi;
|
|
2895
|
+
this.gemini = options?.gemini;
|
|
2896
|
+
this.vectorState = options?.vectorState ?? { available: false };
|
|
2897
|
+
this.ftsAvailable = options?.ftsAvailable ?? false;
|
|
2898
|
+
this.providerKey = this.computeProviderKey();
|
|
2899
|
+
}
|
|
2900
|
+
/**
|
|
2901
|
+
* Update vector/FTS availability (called by parent when extensions load)
|
|
2902
|
+
*/
|
|
2903
|
+
setVectorState(state) {
|
|
2904
|
+
this.vectorState = state;
|
|
2905
|
+
}
|
|
2906
|
+
setFtsAvailable(available) {
|
|
2907
|
+
this.ftsAvailable = available;
|
|
2908
|
+
}
|
|
2909
|
+
getVectorDims() {
|
|
2910
|
+
return this.vectorState.dims;
|
|
2911
|
+
}
|
|
2912
|
+
/**
|
|
2913
|
+
* Compute a unique key for the current provider configuration
|
|
2914
|
+
*/
|
|
2915
|
+
computeProviderKey() {
|
|
2916
|
+
const parts = [this.provider.id, this.provider.model];
|
|
2917
|
+
if (this.openAi) {
|
|
2918
|
+
parts.push(this.openAi.baseUrl);
|
|
2919
|
+
}
|
|
2920
|
+
if (this.gemini) {
|
|
2921
|
+
parts.push(this.gemini.baseUrl);
|
|
2922
|
+
}
|
|
2923
|
+
return hashText(parts.join(":"));
|
|
2924
|
+
}
|
|
2925
|
+
/**
|
|
2926
|
+
* Read index metadata from database
|
|
2927
|
+
*/
|
|
2928
|
+
readMeta() {
|
|
2929
|
+
try {
|
|
2930
|
+
const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY2);
|
|
2931
|
+
if (!row?.value) return null;
|
|
2932
|
+
return JSON.parse(row.value);
|
|
2933
|
+
} catch {
|
|
2934
|
+
return null;
|
|
2935
|
+
}
|
|
2936
|
+
}
|
|
2937
|
+
/**
|
|
2938
|
+
* Write index metadata to database
|
|
2939
|
+
*/
|
|
2940
|
+
writeMeta(meta) {
|
|
2941
|
+
this.db.prepare(`INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)`).run(META_KEY2, JSON.stringify(meta));
|
|
2942
|
+
}
|
|
2943
|
+
/**
|
|
2944
|
+
* Check if the index is stale by comparing file mtimes
|
|
2945
|
+
*/
|
|
2946
|
+
async isStale() {
|
|
2947
|
+
try {
|
|
2948
|
+
const files = await listMemoryFiles(this.config.memoryDir);
|
|
2949
|
+
const stored = this.db.prepare(`SELECT path, mtime FROM files WHERE source = ?`).all("memory");
|
|
2950
|
+
if (files.length !== stored.length) {
|
|
2951
|
+
this.config.debug?.(`Stale: file count changed (${stored.length} -> ${files.length})`);
|
|
2952
|
+
return true;
|
|
2953
|
+
}
|
|
2954
|
+
const storedMap = new Map(stored.map((f) => [f.path, f.mtime]));
|
|
2955
|
+
for (const absPath of files) {
|
|
2956
|
+
const relPath = path3.relative(this.config.memoryDir, absPath).replace(/\\/g, "/");
|
|
2957
|
+
const storedMtime = storedMap.get(relPath);
|
|
2958
|
+
if (storedMtime === void 0) {
|
|
2959
|
+
this.config.debug?.(`Stale: new file ${relPath}`);
|
|
2960
|
+
return true;
|
|
2961
|
+
}
|
|
2962
|
+
const stat = await fs2.stat(absPath);
|
|
2963
|
+
const currentMtime = Math.floor(stat.mtimeMs);
|
|
2964
|
+
if (currentMtime !== storedMtime) {
|
|
2965
|
+
this.config.debug?.(`Stale: mtime changed for ${relPath}`);
|
|
2966
|
+
return true;
|
|
2388
2967
|
}
|
|
2389
2968
|
}
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2969
|
+
return false;
|
|
2970
|
+
} catch (err) {
|
|
2971
|
+
this.config.debug?.(`Stale check failed: ${String(err)}`);
|
|
2972
|
+
return true;
|
|
2973
|
+
}
|
|
2974
|
+
}
|
|
2975
|
+
/**
|
|
2976
|
+
* Check if a full reindex is needed based on configuration changes
|
|
2977
|
+
*/
|
|
2978
|
+
needsFullReindex(force) {
|
|
2979
|
+
const meta = this.readMeta();
|
|
2980
|
+
return force === true || !meta || meta.model !== this.provider.model || meta.provider !== this.provider.id || meta.providerKey !== this.providerKey || meta.chunkTokens !== this.config.chunking.tokens || meta.chunkOverlap !== this.config.chunking.overlap || this.vectorState.available && !meta?.vectorDims;
|
|
2981
|
+
}
|
|
2982
|
+
/**
|
|
2983
|
+
* Index all memory files, returns stats
|
|
2984
|
+
*/
|
|
2985
|
+
async indexAll(force) {
|
|
2986
|
+
const needsFullReindex = this.needsFullReindex(force);
|
|
2987
|
+
const files = await listMemoryFiles(this.config.memoryDir);
|
|
2988
|
+
const activePaths = /* @__PURE__ */ new Set();
|
|
2989
|
+
let filesProcessed = 0;
|
|
2990
|
+
let chunksCreated = 0;
|
|
2991
|
+
for (const absPath of files) {
|
|
2992
|
+
const entry = await buildFileEntry(absPath, this.config.memoryDir);
|
|
2993
|
+
activePaths.add(entry.path);
|
|
2994
|
+
const record = this.db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(entry.path, "memory");
|
|
2995
|
+
if (!needsFullReindex && record?.hash === entry.hash) {
|
|
2996
|
+
continue;
|
|
2395
2997
|
}
|
|
2998
|
+
const chunkCount = await this.indexFile(entry);
|
|
2999
|
+
filesProcessed++;
|
|
3000
|
+
chunksCreated += chunkCount;
|
|
2396
3001
|
}
|
|
3002
|
+
const staleRemoved = this.removeStaleEntries(activePaths);
|
|
3003
|
+
this.writeMeta({
|
|
3004
|
+
model: this.provider.model,
|
|
3005
|
+
provider: this.provider.id,
|
|
3006
|
+
providerKey: this.providerKey,
|
|
3007
|
+
chunkTokens: this.config.chunking.tokens,
|
|
3008
|
+
chunkOverlap: this.config.chunking.overlap,
|
|
3009
|
+
vectorDims: this.vectorState.dims
|
|
3010
|
+
});
|
|
3011
|
+
this.pruneEmbeddingCacheIfNeeded();
|
|
3012
|
+
return { filesProcessed, chunksCreated, staleRemoved };
|
|
2397
3013
|
}
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
const
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
function serializeFrontmatter(frontmatter) {
|
|
2415
|
-
const lines = ["---"];
|
|
2416
|
-
if (frontmatter.session) {
|
|
2417
|
-
lines.push("session:");
|
|
2418
|
-
const session = frontmatter.session;
|
|
2419
|
-
if (session.id) lines.push(` id: ${session.id}`);
|
|
2420
|
-
if (session.source) lines.push(` source: ${session.source}`);
|
|
2421
|
-
if (session.project) lines.push(` project: ${formatPath(session.project)}`);
|
|
2422
|
-
if (session.transcript) lines.push(` transcript: ${formatPath(session.transcript)}`);
|
|
2423
|
-
}
|
|
2424
|
-
if (frontmatter.created) {
|
|
2425
|
-
lines.push(`created: ${frontmatter.created}`);
|
|
2426
|
-
}
|
|
2427
|
-
if (frontmatter.updated) {
|
|
2428
|
-
lines.push(`updated: ${frontmatter.updated}`);
|
|
2429
|
-
}
|
|
2430
|
-
if (frontmatter.tags && frontmatter.tags.length > 0) {
|
|
2431
|
-
lines.push(`tags: [${frontmatter.tags.join(", ")}]`);
|
|
2432
|
-
}
|
|
2433
|
-
lines.push("---");
|
|
2434
|
-
return lines.join("\n") + "\n";
|
|
2435
|
-
}
|
|
2436
|
-
function addFrontmatter(content, frontmatter) {
|
|
2437
|
-
const { frontmatter: existing, body } = parseFrontmatter(content);
|
|
2438
|
-
const merged = {
|
|
2439
|
-
...existing,
|
|
2440
|
-
...frontmatter,
|
|
2441
|
-
session: {
|
|
2442
|
-
...existing?.session,
|
|
2443
|
-
...frontmatter.session
|
|
3014
|
+
/**
|
|
3015
|
+
* Index a single file
|
|
3016
|
+
*/
|
|
3017
|
+
async indexFile(entry) {
|
|
3018
|
+
const content = await fs2.readFile(entry.absPath, "utf-8");
|
|
3019
|
+
const chunks = chunkMarkdown(content, this.config.chunking);
|
|
3020
|
+
const embeddings = await this.embedChunks(chunks);
|
|
3021
|
+
this.db.prepare(
|
|
3022
|
+
`INSERT OR REPLACE INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)`
|
|
3023
|
+
).run(entry.path, "memory", entry.hash, Math.floor(entry.mtimeMs), entry.size);
|
|
3024
|
+
this.deleteChunksForFile(entry.path);
|
|
3025
|
+
const now = Date.now();
|
|
3026
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
3027
|
+
const chunk = chunks[i];
|
|
3028
|
+
const embedding = embeddings[i] ?? [];
|
|
3029
|
+
this.insertChunk(entry.path, chunk, embedding, now);
|
|
2444
3030
|
}
|
|
2445
|
-
|
|
2446
|
-
if (!merged.created) {
|
|
2447
|
-
merged.created = (/* @__PURE__ */ new Date()).toISOString();
|
|
3031
|
+
return chunks.length;
|
|
2448
3032
|
}
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2456
|
-
|
|
2457
|
-
|
|
2458
|
-
|
|
3033
|
+
/**
|
|
3034
|
+
* Delete all chunks for a file
|
|
3035
|
+
*/
|
|
3036
|
+
deleteChunksForFile(filePath) {
|
|
3037
|
+
try {
|
|
3038
|
+
this.db.prepare(
|
|
3039
|
+
`DELETE FROM ${VECTOR_TABLE2} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`
|
|
3040
|
+
).run(filePath, "memory");
|
|
3041
|
+
} catch {
|
|
3042
|
+
}
|
|
3043
|
+
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(filePath, "memory");
|
|
3044
|
+
if (this.config.ftsEnabled && this.ftsAvailable) {
|
|
3045
|
+
try {
|
|
3046
|
+
this.db.prepare(`DELETE FROM ${FTS_TABLE2} WHERE path = ? AND source = ? AND model = ?`).run(filePath, "memory", this.provider.model);
|
|
3047
|
+
} catch {
|
|
3048
|
+
}
|
|
3049
|
+
}
|
|
2459
3050
|
}
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
3051
|
+
/**
|
|
3052
|
+
* Insert a chunk into the database
|
|
3053
|
+
*/
|
|
3054
|
+
insertChunk(filePath, chunk, embedding, timestamp) {
|
|
3055
|
+
const chunkId = randomUUID2();
|
|
3056
|
+
this.db.prepare(
|
|
3057
|
+
`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at)
|
|
3058
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
3059
|
+
).run(
|
|
3060
|
+
chunkId,
|
|
3061
|
+
filePath,
|
|
3062
|
+
"memory",
|
|
3063
|
+
chunk.startLine,
|
|
3064
|
+
chunk.endLine,
|
|
3065
|
+
chunk.hash,
|
|
3066
|
+
this.provider.model,
|
|
3067
|
+
chunk.text,
|
|
3068
|
+
JSON.stringify(embedding),
|
|
3069
|
+
timestamp
|
|
3070
|
+
);
|
|
3071
|
+
if (this.vectorState.available && embedding.length > 0) {
|
|
3072
|
+
if (!this.vectorState.dims) {
|
|
3073
|
+
this.vectorState.dims = embedding.length;
|
|
3074
|
+
this.ensureVectorTable(embedding.length);
|
|
3075
|
+
}
|
|
3076
|
+
try {
|
|
3077
|
+
this.db.prepare(`INSERT INTO ${VECTOR_TABLE2} (id, embedding) VALUES (?, ?)`).run(chunkId, vectorToBlob(embedding));
|
|
3078
|
+
} catch {
|
|
3079
|
+
}
|
|
3080
|
+
}
|
|
3081
|
+
if (this.config.ftsEnabled && this.ftsAvailable) {
|
|
3082
|
+
try {
|
|
3083
|
+
this.db.prepare(
|
|
3084
|
+
`INSERT INTO ${FTS_TABLE2} (text, id, path, source, model, start_line, end_line)
|
|
3085
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
3086
|
+
).run(
|
|
3087
|
+
chunk.text,
|
|
3088
|
+
chunkId,
|
|
3089
|
+
filePath,
|
|
3090
|
+
"memory",
|
|
3091
|
+
this.provider.model,
|
|
3092
|
+
chunk.startLine,
|
|
3093
|
+
chunk.endLine
|
|
3094
|
+
);
|
|
3095
|
+
} catch {
|
|
3096
|
+
}
|
|
3097
|
+
}
|
|
3098
|
+
}
|
|
3099
|
+
/**
|
|
3100
|
+
* Remove stale file entries that no longer exist
|
|
3101
|
+
*/
|
|
3102
|
+
removeStaleEntries(activePaths) {
|
|
3103
|
+
const staleRows = this.db.prepare(`SELECT path FROM files WHERE source = ?`).all("memory");
|
|
3104
|
+
let removed = 0;
|
|
3105
|
+
for (const stale of staleRows) {
|
|
3106
|
+
if (activePaths.has(stale.path)) continue;
|
|
3107
|
+
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
|
|
3108
|
+
this.deleteChunksForFile(stale.path);
|
|
3109
|
+
removed++;
|
|
3110
|
+
}
|
|
3111
|
+
return removed;
|
|
3112
|
+
}
|
|
3113
|
+
/**
|
|
3114
|
+
* Create vector table with the given dimensions
|
|
3115
|
+
*/
|
|
3116
|
+
ensureVectorTable(dimensions) {
|
|
3117
|
+
if (!this.vectorState.available) return;
|
|
3118
|
+
try {
|
|
3119
|
+
this.db.exec(
|
|
3120
|
+
`CREATE VIRTUAL TABLE IF NOT EXISTS ${VECTOR_TABLE2} USING vec0(
|
|
3121
|
+
id TEXT PRIMARY KEY,
|
|
3122
|
+
embedding FLOAT[${dimensions}]
|
|
3123
|
+
)`
|
|
3124
|
+
);
|
|
3125
|
+
} catch (err) {
|
|
3126
|
+
this.config.debug?.(`vector table creation failed: ${String(err)}`);
|
|
3127
|
+
}
|
|
3128
|
+
}
|
|
3129
|
+
/**
|
|
3130
|
+
* Get embeddings for chunks, using cache when available
|
|
3131
|
+
*/
|
|
3132
|
+
async embedChunks(chunks) {
|
|
3133
|
+
if (chunks.length === 0) return [];
|
|
3134
|
+
const hashes = chunks.map((c) => c.hash);
|
|
3135
|
+
const cached = this.loadEmbeddingCache(hashes);
|
|
3136
|
+
const missing = [];
|
|
3137
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
3138
|
+
if (!cached.has(hashes[i])) {
|
|
3139
|
+
missing.push({ index: i, chunk: chunks[i] });
|
|
3140
|
+
}
|
|
3141
|
+
}
|
|
3142
|
+
if (missing.length > 0) {
|
|
3143
|
+
const texts = missing.map((m) => m.chunk.text);
|
|
3144
|
+
const newEmbeddings = await this.embedBatchWithRetry(texts);
|
|
3145
|
+
for (let i = 0; i < missing.length; i++) {
|
|
3146
|
+
const hash = missing[i].chunk.hash;
|
|
3147
|
+
const embedding = newEmbeddings[i] ?? [];
|
|
3148
|
+
cached.set(hash, embedding);
|
|
3149
|
+
this.upsertEmbeddingCache(hash, embedding);
|
|
3150
|
+
}
|
|
3151
|
+
}
|
|
3152
|
+
return hashes.map((h) => cached.get(h) ?? []);
|
|
3153
|
+
}
|
|
3154
|
+
/**
|
|
3155
|
+
* Embed texts with retry logic
|
|
3156
|
+
*/
|
|
3157
|
+
async embedBatchWithRetry(texts) {
|
|
3158
|
+
if (texts.length === 0) return [];
|
|
3159
|
+
if (this.config.batch.enabled) {
|
|
3160
|
+
try {
|
|
3161
|
+
return await this.embedWithBatchApi(texts);
|
|
3162
|
+
} catch (err) {
|
|
3163
|
+
this.config.debug?.(`batch embedding failed, falling back to direct: ${String(err)}`);
|
|
3164
|
+
}
|
|
3165
|
+
}
|
|
3166
|
+
let lastError = null;
|
|
3167
|
+
for (let attempt = 0; attempt < EMBEDDING_RETRY_MAX_ATTEMPTS2; attempt++) {
|
|
3168
|
+
try {
|
|
3169
|
+
return await this.provider.embedBatch(texts);
|
|
3170
|
+
} catch (err) {
|
|
3171
|
+
lastError = err instanceof Error ? err : new Error(String(err));
|
|
3172
|
+
if (attempt < EMBEDDING_RETRY_MAX_ATTEMPTS2 - 1) {
|
|
3173
|
+
const delay = Math.min(
|
|
3174
|
+
EMBEDDING_RETRY_MAX_DELAY_MS2,
|
|
3175
|
+
EMBEDDING_RETRY_BASE_DELAY_MS2 * Math.pow(2, attempt)
|
|
3176
|
+
);
|
|
3177
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
3178
|
+
}
|
|
3179
|
+
}
|
|
3180
|
+
}
|
|
3181
|
+
throw lastError;
|
|
3182
|
+
}
|
|
3183
|
+
/**
|
|
3184
|
+
* Use batch API for large embedding jobs
|
|
3185
|
+
*/
|
|
3186
|
+
async embedWithBatchApi(texts) {
|
|
3187
|
+
if (this.openAi) {
|
|
3188
|
+
const requests = texts.map((text, i) => ({
|
|
3189
|
+
custom_id: `chunk-${i}`,
|
|
3190
|
+
method: "POST",
|
|
3191
|
+
url: OPENAI_BATCH_ENDPOINT,
|
|
3192
|
+
body: { model: this.openAi.model, input: text }
|
|
3193
|
+
}));
|
|
3194
|
+
const results = await runOpenAiEmbeddingBatches({
|
|
3195
|
+
openAi: this.openAi,
|
|
3196
|
+
source: "minimem",
|
|
3197
|
+
requests,
|
|
3198
|
+
wait: this.config.batch.wait,
|
|
3199
|
+
pollIntervalMs: this.config.batch.pollIntervalMs,
|
|
3200
|
+
timeoutMs: this.config.batch.timeoutMs,
|
|
3201
|
+
concurrency: this.config.batch.concurrency,
|
|
3202
|
+
debug: this.config.debug
|
|
3203
|
+
});
|
|
3204
|
+
return texts.map((_, i) => results.get(`chunk-${i}`) ?? []);
|
|
3205
|
+
}
|
|
3206
|
+
if (this.gemini) {
|
|
3207
|
+
const requests = texts.map((text, i) => ({
|
|
3208
|
+
custom_id: `chunk-${i}`,
|
|
3209
|
+
content: { parts: [{ text }] },
|
|
3210
|
+
taskType: "RETRIEVAL_DOCUMENT"
|
|
3211
|
+
}));
|
|
3212
|
+
const results = await runGeminiEmbeddingBatches({
|
|
3213
|
+
gemini: this.gemini,
|
|
3214
|
+
source: "minimem",
|
|
3215
|
+
requests,
|
|
3216
|
+
wait: this.config.batch.wait,
|
|
3217
|
+
pollIntervalMs: this.config.batch.pollIntervalMs,
|
|
3218
|
+
timeoutMs: this.config.batch.timeoutMs,
|
|
3219
|
+
concurrency: this.config.batch.concurrency,
|
|
3220
|
+
debug: this.config.debug
|
|
3221
|
+
});
|
|
3222
|
+
return texts.map((_, i) => results.get(`chunk-${i}`) ?? []);
|
|
3223
|
+
}
|
|
3224
|
+
throw new Error("Batch API not available for local embeddings");
|
|
3225
|
+
}
|
|
3226
|
+
/**
|
|
3227
|
+
* Load embeddings from cache
|
|
3228
|
+
*/
|
|
3229
|
+
loadEmbeddingCache(hashes) {
|
|
3230
|
+
const result = /* @__PURE__ */ new Map();
|
|
3231
|
+
if (!this.config.cache.enabled || hashes.length === 0) return result;
|
|
3232
|
+
const placeholders = hashes.map(() => "?").join(",");
|
|
3233
|
+
const rows = this.db.prepare(
|
|
3234
|
+
`SELECT hash, embedding FROM ${EMBEDDING_CACHE_TABLE2}
|
|
3235
|
+
WHERE provider = ? AND model = ? AND provider_key = ? AND hash IN (${placeholders})`
|
|
3236
|
+
).all(this.provider.id, this.provider.model, this.providerKey, ...hashes);
|
|
3237
|
+
const now = Date.now();
|
|
3238
|
+
for (const row of rows) {
|
|
3239
|
+
result.set(row.hash, parseEmbedding(row.embedding));
|
|
3240
|
+
this.db.prepare(
|
|
3241
|
+
`UPDATE ${EMBEDDING_CACHE_TABLE2} SET updated_at = ?
|
|
3242
|
+
WHERE provider = ? AND model = ? AND provider_key = ? AND hash = ?`
|
|
3243
|
+
).run(now, this.provider.id, this.provider.model, this.providerKey, row.hash);
|
|
3244
|
+
}
|
|
3245
|
+
return result;
|
|
3246
|
+
}
|
|
3247
|
+
/**
|
|
3248
|
+
* Save embedding to cache
|
|
3249
|
+
*/
|
|
3250
|
+
upsertEmbeddingCache(hash, embedding) {
|
|
3251
|
+
if (!this.config.cache.enabled) return;
|
|
3252
|
+
const now = Date.now();
|
|
3253
|
+
this.db.prepare(
|
|
3254
|
+
`INSERT OR REPLACE INTO ${EMBEDDING_CACHE_TABLE2}
|
|
3255
|
+
(provider, model, provider_key, hash, embedding, dims, updated_at)
|
|
3256
|
+
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
3257
|
+
).run(
|
|
3258
|
+
this.provider.id,
|
|
3259
|
+
this.provider.model,
|
|
3260
|
+
this.providerKey,
|
|
3261
|
+
hash,
|
|
3262
|
+
JSON.stringify(embedding),
|
|
3263
|
+
embedding.length,
|
|
3264
|
+
now
|
|
3265
|
+
);
|
|
3266
|
+
}
|
|
3267
|
+
/**
|
|
3268
|
+
* Prune old cache entries if over limit
|
|
3269
|
+
*/
|
|
3270
|
+
pruneEmbeddingCacheIfNeeded() {
|
|
3271
|
+
if (!this.config.cache.enabled) return;
|
|
3272
|
+
const row = this.db.prepare(`SELECT COUNT(*) as count FROM ${EMBEDDING_CACHE_TABLE2}`).get();
|
|
3273
|
+
if (row.count <= this.config.cache.maxEntries) return;
|
|
3274
|
+
const excess = row.count - this.config.cache.maxEntries;
|
|
3275
|
+
this.db.prepare(
|
|
3276
|
+
`DELETE FROM ${EMBEDDING_CACHE_TABLE2}
|
|
3277
|
+
WHERE rowid IN (
|
|
3278
|
+
SELECT rowid FROM ${EMBEDDING_CACHE_TABLE2}
|
|
3279
|
+
ORDER BY updated_at ASC
|
|
3280
|
+
LIMIT ?
|
|
3281
|
+
)`
|
|
3282
|
+
).run(excess);
|
|
3283
|
+
}
|
|
3284
|
+
};
|
|
3285
|
+
|
|
3286
|
+
// src/core/searcher.ts
|
|
3287
|
+
var SNIPPET_MAX_CHARS2 = 700;
|
|
3288
|
+
var VECTOR_TABLE3 = "chunks_vec";
|
|
3289
|
+
var FTS_TABLE3 = "chunks_fts";
|
|
3290
|
+
var EMBEDDING_QUERY_TIMEOUT_REMOTE_MS2 = 6e4;
|
|
3291
|
+
var EMBEDDING_QUERY_TIMEOUT_LOCAL_MS2 = 5 * 6e4;
|
|
3292
|
+
var MemorySearcher = class {
|
|
3293
|
+
db;
|
|
3294
|
+
provider;
|
|
3295
|
+
config;
|
|
3296
|
+
// State from parent
|
|
3297
|
+
vectorState;
|
|
3298
|
+
ftsAvailable;
|
|
3299
|
+
// Callback to ensure vector is ready
|
|
3300
|
+
ensureVectorReadyFn;
|
|
3301
|
+
constructor(db, provider, config, options) {
|
|
3302
|
+
this.db = db;
|
|
3303
|
+
this.provider = provider;
|
|
3304
|
+
this.config = config;
|
|
3305
|
+
this.vectorState = options?.vectorState ?? { available: false };
|
|
3306
|
+
this.ftsAvailable = options?.ftsAvailable ?? false;
|
|
3307
|
+
this.ensureVectorReadyFn = options?.ensureVectorReady;
|
|
3308
|
+
}
|
|
3309
|
+
/**
|
|
3310
|
+
* Update vector/FTS availability (called by parent when extensions load)
|
|
3311
|
+
*/
|
|
3312
|
+
setVectorState(state) {
|
|
3313
|
+
this.vectorState = state;
|
|
3314
|
+
}
|
|
3315
|
+
setFtsAvailable(available) {
|
|
3316
|
+
this.ftsAvailable = available;
|
|
3317
|
+
}
|
|
3318
|
+
/**
|
|
3319
|
+
* Execute a search query
|
|
3320
|
+
*/
|
|
3321
|
+
async search(query, opts) {
|
|
3322
|
+
const cleaned = query.trim();
|
|
3323
|
+
if (!cleaned) return [];
|
|
3324
|
+
const minScore = opts?.minScore ?? this.config.query.minScore;
|
|
3325
|
+
const maxResults = opts?.maxResults ?? this.config.query.maxResults;
|
|
3326
|
+
const candidates = Math.min(
|
|
3327
|
+
200,
|
|
3328
|
+
Math.max(1, Math.floor(maxResults * this.config.hybrid.candidateMultiplier))
|
|
3329
|
+
);
|
|
3330
|
+
const sourceFilter = { sql: "", params: [] };
|
|
3331
|
+
const keywordResults = this.config.hybrid.enabled && this.ftsAvailable ? await searchKeyword({
|
|
3332
|
+
db: this.db,
|
|
3333
|
+
ftsTable: FTS_TABLE3,
|
|
3334
|
+
providerModel: this.provider.model,
|
|
3335
|
+
query: cleaned,
|
|
3336
|
+
limit: candidates,
|
|
3337
|
+
snippetMaxChars: SNIPPET_MAX_CHARS2,
|
|
3338
|
+
sourceFilter,
|
|
3339
|
+
buildFtsQuery,
|
|
3340
|
+
bm25RankToScore
|
|
3341
|
+
}).catch(() => []) : [];
|
|
3342
|
+
const queryVec = await this.embedQueryWithTimeout(cleaned);
|
|
3343
|
+
const hasVector = queryVec.some((v) => v !== 0);
|
|
3344
|
+
const vectorResults = hasVector ? await searchVector({
|
|
3345
|
+
db: this.db,
|
|
3346
|
+
vectorTable: VECTOR_TABLE3,
|
|
3347
|
+
providerModel: this.provider.model,
|
|
3348
|
+
queryVec,
|
|
3349
|
+
limit: candidates,
|
|
3350
|
+
snippetMaxChars: SNIPPET_MAX_CHARS2,
|
|
3351
|
+
ensureVectorReady: (dims) => this.ensureVectorReady(dims),
|
|
3352
|
+
sourceFilterVec: sourceFilter,
|
|
3353
|
+
sourceFilterChunks: sourceFilter
|
|
3354
|
+
}).catch(() => []) : [];
|
|
3355
|
+
if (!this.config.hybrid.enabled) {
|
|
3356
|
+
return vectorResults.filter((entry) => entry.score >= minScore).slice(0, maxResults).map((r) => ({
|
|
3357
|
+
path: r.path,
|
|
3358
|
+
startLine: r.startLine,
|
|
3359
|
+
endLine: r.endLine,
|
|
3360
|
+
score: r.score,
|
|
3361
|
+
snippet: r.snippet
|
|
3362
|
+
}));
|
|
3363
|
+
}
|
|
3364
|
+
const merged = mergeHybridResults({
|
|
3365
|
+
vector: vectorResults.map((r) => ({
|
|
3366
|
+
id: r.id,
|
|
3367
|
+
path: r.path,
|
|
3368
|
+
startLine: r.startLine,
|
|
3369
|
+
endLine: r.endLine,
|
|
3370
|
+
source: r.source,
|
|
3371
|
+
snippet: r.snippet,
|
|
3372
|
+
vectorScore: r.score
|
|
3373
|
+
})),
|
|
3374
|
+
keyword: keywordResults.map((r) => ({
|
|
3375
|
+
id: r.id,
|
|
3376
|
+
path: r.path,
|
|
3377
|
+
startLine: r.startLine,
|
|
3378
|
+
endLine: r.endLine,
|
|
3379
|
+
source: r.source,
|
|
3380
|
+
snippet: r.snippet,
|
|
3381
|
+
textScore: r.textScore
|
|
3382
|
+
})),
|
|
3383
|
+
vectorWeight: this.config.hybrid.vectorWeight,
|
|
3384
|
+
textWeight: this.config.hybrid.textWeight
|
|
3385
|
+
});
|
|
3386
|
+
return merged.filter((entry) => entry.score >= minScore).slice(0, maxResults).map((r) => ({
|
|
3387
|
+
path: r.path,
|
|
3388
|
+
startLine: r.startLine,
|
|
3389
|
+
endLine: r.endLine,
|
|
3390
|
+
score: r.score,
|
|
3391
|
+
snippet: r.snippet
|
|
3392
|
+
}));
|
|
3393
|
+
}
|
|
3394
|
+
/**
|
|
3395
|
+
* Embed a query string with timeout
|
|
3396
|
+
*/
|
|
3397
|
+
async embedQueryWithTimeout(text) {
|
|
3398
|
+
const timeout = this.provider.id === "local" ? EMBEDDING_QUERY_TIMEOUT_LOCAL_MS2 : EMBEDDING_QUERY_TIMEOUT_REMOTE_MS2;
|
|
3399
|
+
return Promise.race([
|
|
3400
|
+
this.provider.embedQuery(text),
|
|
3401
|
+
new Promise(
|
|
3402
|
+
(_, reject) => setTimeout(() => reject(new Error("embedding query timeout")), timeout)
|
|
3403
|
+
)
|
|
3404
|
+
]);
|
|
3405
|
+
}
|
|
3406
|
+
/**
|
|
3407
|
+
* Ensure vector extension is ready
|
|
3408
|
+
*/
|
|
3409
|
+
async ensureVectorReady(dims) {
|
|
3410
|
+
if (this.vectorState.available) return true;
|
|
3411
|
+
if (this.ensureVectorReadyFn) {
|
|
3412
|
+
return this.ensureVectorReadyFn(dims);
|
|
3413
|
+
}
|
|
3414
|
+
return false;
|
|
3415
|
+
}
|
|
3416
|
+
};
|
|
2466
3417
|
export {
|
|
3418
|
+
KNOWLEDGE_GRAPH_TOOL,
|
|
3419
|
+
KNOWLEDGE_PATH_TOOL,
|
|
3420
|
+
KNOWLEDGE_SEARCH_TOOL,
|
|
3421
|
+
MEMORY_GET_DETAILS_TOOL,
|
|
2467
3422
|
MEMORY_SEARCH_TOOL,
|
|
2468
3423
|
MEMORY_TOOLS,
|
|
2469
3424
|
McpServer,
|
|
3425
|
+
MemoryIndexer,
|
|
3426
|
+
MemorySearcher,
|
|
2470
3427
|
MemoryToolExecutor,
|
|
2471
3428
|
Minimem,
|
|
2472
3429
|
addFrontmatter,
|
|
2473
3430
|
addSessionToContent,
|
|
2474
|
-
bm25RankToScore,
|
|
2475
3431
|
buildFileEntry,
|
|
2476
|
-
|
|
3432
|
+
buildKnowledgeFilterSql,
|
|
2477
3433
|
chunkMarkdown,
|
|
2478
3434
|
cosineSimilarity,
|
|
2479
3435
|
createEmbeddingProvider,
|
|
@@ -2481,18 +3437,22 @@ export {
|
|
|
2481
3437
|
createMcpServer,
|
|
2482
3438
|
createOpenAiEmbeddingProvider,
|
|
2483
3439
|
createToolExecutor,
|
|
3440
|
+
extractChunkMetadata,
|
|
2484
3441
|
extractSession,
|
|
2485
3442
|
generateMcpConfig,
|
|
3443
|
+
getLinksFrom,
|
|
3444
|
+
getLinksTo,
|
|
3445
|
+
getNeighbors,
|
|
3446
|
+
getPathBetween,
|
|
2486
3447
|
getToolDefinitions,
|
|
2487
3448
|
hashText,
|
|
2488
3449
|
isMemoryPath,
|
|
2489
3450
|
listMemoryFiles,
|
|
2490
|
-
mergeHybridResults,
|
|
2491
|
-
normalizeRelPath,
|
|
2492
3451
|
parseFrontmatter,
|
|
2493
3452
|
runGeminiEmbeddingBatches,
|
|
2494
3453
|
runMcpServer,
|
|
2495
3454
|
runOpenAiEmbeddingBatches,
|
|
2496
|
-
serializeFrontmatter
|
|
3455
|
+
serializeFrontmatter,
|
|
3456
|
+
stripPrivateContent
|
|
2497
3457
|
};
|
|
2498
3458
|
//# sourceMappingURL=index.js.map
|