minimem 0.0.6 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -4
- package/dist/cli/chunk-BIYUNXYX.js +2689 -0
- package/dist/cli/chunk-BIYUNXYX.js.map +1 -0
- package/dist/cli/index.js +656 -2892
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/minimem-MQXSBGNG.js +8 -0
- package/dist/cli/minimem-MQXSBGNG.js.map +1 -0
- package/dist/index.cjs +391 -8
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +198 -1
- package/dist/index.d.ts +198 -1
- package/dist/index.js +378 -8
- package/dist/index.js.map +1 -1
- package/package.json +3 -2
package/dist/cli/index.js
CHANGED
|
@@ -1,21 +1,36 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
+
import {
|
|
3
|
+
Minimem,
|
|
4
|
+
addFrontmatter,
|
|
5
|
+
parseFrontmatter
|
|
6
|
+
} from "./chunk-BIYUNXYX.js";
|
|
2
7
|
|
|
3
8
|
// src/cli/index.ts
|
|
4
9
|
import { program } from "commander";
|
|
5
10
|
|
|
6
11
|
// src/cli/commands/init.ts
|
|
7
|
-
import
|
|
8
|
-
import
|
|
12
|
+
import fs5 from "fs/promises";
|
|
13
|
+
import path5 from "path";
|
|
9
14
|
|
|
10
15
|
// src/cli/config.ts
|
|
11
|
-
import
|
|
16
|
+
import fs2 from "fs/promises";
|
|
17
|
+
import fsSync from "fs";
|
|
12
18
|
import path2 from "path";
|
|
13
19
|
import os2 from "os";
|
|
14
20
|
import crypto from "crypto";
|
|
15
21
|
|
|
16
22
|
// src/cli/shared.ts
|
|
23
|
+
import * as fs from "fs";
|
|
17
24
|
import * as path from "path";
|
|
18
25
|
import * as os from "os";
|
|
26
|
+
function discoverMemoryDir(base) {
|
|
27
|
+
if (fs.existsSync(path.join(base, "MEMORY.md"))) return base;
|
|
28
|
+
const swarmDir = path.join(base, ".swarm", "minimem");
|
|
29
|
+
if (fs.existsSync(path.join(swarmDir, "config.json"))) return swarmDir;
|
|
30
|
+
const minimemDir = path.join(base, ".minimem");
|
|
31
|
+
if (fs.existsSync(path.join(minimemDir, "config.json"))) return minimemDir;
|
|
32
|
+
return base;
|
|
33
|
+
}
|
|
19
34
|
function resolveMemoryDir(options) {
|
|
20
35
|
const dir = Array.isArray(options.dir) ? options.dir[0] : options.dir;
|
|
21
36
|
if (dir) {
|
|
@@ -28,7 +43,7 @@ function resolveMemoryDir(options) {
|
|
|
28
43
|
if (options.global) {
|
|
29
44
|
return path.join(os.homedir(), ".minimem");
|
|
30
45
|
}
|
|
31
|
-
return process.cwd();
|
|
46
|
+
return discoverMemoryDir(process.cwd());
|
|
32
47
|
}
|
|
33
48
|
function resolveMemoryDirs(options) {
|
|
34
49
|
const dirs = [];
|
|
@@ -46,7 +61,7 @@ function resolveMemoryDirs(options) {
|
|
|
46
61
|
}
|
|
47
62
|
}
|
|
48
63
|
if (dirs.length === 0) {
|
|
49
|
-
dirs.push(process.cwd());
|
|
64
|
+
dirs.push(discoverMemoryDir(process.cwd()));
|
|
50
65
|
}
|
|
51
66
|
return [...new Set(dirs)];
|
|
52
67
|
}
|
|
@@ -90,8 +105,16 @@ function getGlobalDir() {
|
|
|
90
105
|
function getGlobalConfigPath() {
|
|
91
106
|
return path2.join(getGlobalDir(), CONFIG_DIR, CONFIG_FILENAME);
|
|
92
107
|
}
|
|
108
|
+
function resolveConfigSubdir(memoryDir) {
|
|
109
|
+
const envDir = process.env.MINIMEM_CONFIG_DIR;
|
|
110
|
+
if (envDir) return envDir;
|
|
111
|
+
if (fsSync.existsSync(path2.join(memoryDir, CONFIG_FILENAME))) return ".";
|
|
112
|
+
const swarmDir = path2.join(memoryDir, ".swarm", "minimem");
|
|
113
|
+
if (fsSync.existsSync(path2.join(swarmDir, CONFIG_FILENAME))) return path2.join(".swarm", "minimem");
|
|
114
|
+
return CONFIG_DIR;
|
|
115
|
+
}
|
|
93
116
|
function getConfigPath(memoryDir) {
|
|
94
|
-
return path2.join(memoryDir,
|
|
117
|
+
return path2.join(memoryDir, resolveConfigSubdir(memoryDir), CONFIG_FILENAME);
|
|
95
118
|
}
|
|
96
119
|
function getXdgConfigDir() {
|
|
97
120
|
return path2.join(os2.homedir(), XDG_CONFIG_DIR);
|
|
@@ -110,7 +133,7 @@ function expandPath(filePath) {
|
|
|
110
133
|
}
|
|
111
134
|
async function loadXdgConfig() {
|
|
112
135
|
try {
|
|
113
|
-
const content = await
|
|
136
|
+
const content = await fs2.readFile(getXdgConfigPath(), "utf-8");
|
|
114
137
|
return JSON.parse(content);
|
|
115
138
|
} catch {
|
|
116
139
|
return {};
|
|
@@ -119,8 +142,8 @@ async function loadXdgConfig() {
|
|
|
119
142
|
async function saveXdgConfig(config2) {
|
|
120
143
|
const configDir = getXdgConfigDir();
|
|
121
144
|
const configPath = getXdgConfigPath();
|
|
122
|
-
await
|
|
123
|
-
await
|
|
145
|
+
await fs2.mkdir(configDir, { recursive: true });
|
|
146
|
+
await fs2.writeFile(configPath, JSON.stringify(config2, null, 2), "utf-8");
|
|
124
147
|
}
|
|
125
148
|
async function getMachineId() {
|
|
126
149
|
const globalConfig = await loadXdgConfig();
|
|
@@ -135,7 +158,7 @@ async function getMachineId() {
|
|
|
135
158
|
}
|
|
136
159
|
async function loadConfigFile(configPath) {
|
|
137
160
|
try {
|
|
138
|
-
const content = await
|
|
161
|
+
const content = await fs2.readFile(configPath, "utf-8");
|
|
139
162
|
return JSON.parse(content);
|
|
140
163
|
} catch {
|
|
141
164
|
return {};
|
|
@@ -189,10 +212,10 @@ function deepMergeConfig(target, source) {
|
|
|
189
212
|
return result;
|
|
190
213
|
}
|
|
191
214
|
async function saveConfig(memoryDir, config2) {
|
|
192
|
-
const configDir = path2.join(memoryDir, CONFIG_DIR);
|
|
193
215
|
const configPath = getConfigPath(memoryDir);
|
|
194
|
-
|
|
195
|
-
await
|
|
216
|
+
const configDir = path2.dirname(configPath);
|
|
217
|
+
await fs2.mkdir(configDir, { recursive: true });
|
|
218
|
+
await fs2.writeFile(configPath, JSON.stringify(config2, null, 2), "utf-8");
|
|
196
219
|
}
|
|
197
220
|
function getDefaultConfig() {
|
|
198
221
|
return {
|
|
@@ -309,2745 +332,290 @@ function buildMinimemConfig(memoryDir, cliConfig, options) {
|
|
|
309
332
|
async function isInitialized(memoryDir) {
|
|
310
333
|
const configPath = getConfigPath(memoryDir);
|
|
311
334
|
try {
|
|
312
|
-
await
|
|
313
|
-
return true;
|
|
314
|
-
} catch {
|
|
315
|
-
return false;
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
function formatPath(filePath) {
|
|
319
|
-
const home = os2.homedir();
|
|
320
|
-
if (filePath.startsWith(home)) {
|
|
321
|
-
return "~" + filePath.slice(home.length);
|
|
322
|
-
}
|
|
323
|
-
return filePath;
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
// src/cli/commands/init.ts
|
|
327
|
-
var MEMORY_TEMPLATE = `# Memory
|
|
328
|
-
|
|
329
|
-
This is your memory file. Add notes, decisions, and context here.
|
|
330
|
-
|
|
331
|
-
## Quick Start
|
|
332
|
-
|
|
333
|
-
- Add daily logs in the \`memory/\` directory (e.g., \`memory/2024-01-15.md\`)
|
|
334
|
-
- Use \`minimem search <query>\` to find relevant memories
|
|
335
|
-
- Use \`minimem append <text>\` to quickly add to today's log
|
|
336
|
-
|
|
337
|
-
## Notes
|
|
338
|
-
|
|
339
|
-
`;
|
|
340
|
-
async function init(dir, options) {
|
|
341
|
-
const memoryDir = resolveMemoryDir({ dir, global: options.global });
|
|
342
|
-
const displayPath = formatPath(memoryDir);
|
|
343
|
-
if (!options.force && await isInitialized(memoryDir)) {
|
|
344
|
-
console.log(`Already initialized: ${displayPath}`);
|
|
345
|
-
console.log("Use --force to reinitialize");
|
|
346
|
-
return;
|
|
347
|
-
}
|
|
348
|
-
console.log(`Initializing minimem in ${displayPath}...`);
|
|
349
|
-
await fs2.mkdir(memoryDir, { recursive: true });
|
|
350
|
-
await fs2.mkdir(path3.join(memoryDir, "memory"), { recursive: true });
|
|
351
|
-
await fs2.mkdir(path3.join(memoryDir, ".minimem"), { recursive: true });
|
|
352
|
-
const memoryFilePath = path3.join(memoryDir, "MEMORY.md");
|
|
353
|
-
try {
|
|
354
|
-
await fs2.access(memoryFilePath);
|
|
355
|
-
console.log(" MEMORY.md already exists, skipping");
|
|
356
|
-
} catch {
|
|
357
|
-
await fs2.writeFile(memoryFilePath, MEMORY_TEMPLATE, "utf-8");
|
|
358
|
-
console.log(" Created MEMORY.md");
|
|
359
|
-
}
|
|
360
|
-
const config2 = getInitConfig();
|
|
361
|
-
await saveConfig(memoryDir, config2);
|
|
362
|
-
console.log(" Created .minimem/config.json");
|
|
363
|
-
const gitignorePath = path3.join(memoryDir, ".minimem", ".gitignore");
|
|
364
|
-
await fs2.writeFile(gitignorePath, "index.db\nindex.db-*\n", "utf-8");
|
|
365
|
-
console.log(" Created .minimem/.gitignore");
|
|
366
|
-
console.log();
|
|
367
|
-
console.log("Done! Your memory directory is ready.");
|
|
368
|
-
console.log();
|
|
369
|
-
console.log("Next steps:");
|
|
370
|
-
console.log(` 1. Set your embedding API key:`);
|
|
371
|
-
console.log(` export OPENAI_API_KEY=your-key`);
|
|
372
|
-
console.log(` # or: export GOOGLE_API_KEY=your-key`);
|
|
373
|
-
console.log();
|
|
374
|
-
console.log(` 2. Add some memories to MEMORY.md or memory/*.md`);
|
|
375
|
-
console.log();
|
|
376
|
-
console.log(` 3. Search your memories:`);
|
|
377
|
-
console.log(` minimem search "your query"${dir ? ` --dir ${dir}` : ""}`);
|
|
378
|
-
}
|
|
379
|
-
|
|
380
|
-
// src/minimem.ts
|
|
381
|
-
import { randomUUID } from "crypto";
|
|
382
|
-
import fs4 from "fs/promises";
|
|
383
|
-
import path6 from "path";
|
|
384
|
-
import { DatabaseSync } from "node:sqlite";
|
|
385
|
-
import chokidar from "chokidar";
|
|
386
|
-
|
|
387
|
-
// src/internal.ts
|
|
388
|
-
import crypto2 from "crypto";
|
|
389
|
-
import fsSync from "fs";
|
|
390
|
-
import fs3 from "fs/promises";
|
|
391
|
-
import path4 from "path";
|
|
392
|
-
function logError2(context, error, debug) {
|
|
393
|
-
if (!debug) return;
|
|
394
|
-
const message = error instanceof Error ? error.message : String(error);
|
|
395
|
-
debug(`[${context}] Error: ${message}`);
|
|
396
|
-
}
|
|
397
|
-
function ensureDir(dir, debug) {
|
|
398
|
-
try {
|
|
399
|
-
fsSync.mkdirSync(dir, { recursive: true });
|
|
400
|
-
} catch (error) {
|
|
401
|
-
const nodeError = error;
|
|
402
|
-
if (nodeError.code !== "EEXIST") {
|
|
403
|
-
logError2("ensureDir", error, debug);
|
|
404
|
-
}
|
|
405
|
-
}
|
|
406
|
-
return dir;
|
|
407
|
-
}
|
|
408
|
-
async function exists(filePath) {
|
|
409
|
-
try {
|
|
410
|
-
await fs3.access(filePath);
|
|
335
|
+
await fs2.access(configPath);
|
|
411
336
|
return true;
|
|
412
337
|
} catch {
|
|
413
338
|
return false;
|
|
414
339
|
}
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
const
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
if (entry.isDirectory()) {
|
|
421
|
-
await walkDir(full, files);
|
|
422
|
-
continue;
|
|
423
|
-
}
|
|
424
|
-
if (!entry.isFile()) continue;
|
|
425
|
-
if (!entry.name.endsWith(".md")) continue;
|
|
426
|
-
files.push(full);
|
|
427
|
-
}
|
|
428
|
-
}
|
|
429
|
-
async function listMemoryFiles(memoryDir) {
|
|
430
|
-
const result = [];
|
|
431
|
-
const memoryFile = path4.join(memoryDir, "MEMORY.md");
|
|
432
|
-
const altMemoryFile = path4.join(memoryDir, "memory.md");
|
|
433
|
-
const hasUpper = await exists(memoryFile);
|
|
434
|
-
const hasLower = await exists(altMemoryFile);
|
|
435
|
-
if (hasUpper && hasLower) {
|
|
436
|
-
let upperReal = memoryFile;
|
|
437
|
-
let lowerReal = altMemoryFile;
|
|
438
|
-
try {
|
|
439
|
-
upperReal = await fs3.realpath(memoryFile);
|
|
440
|
-
} catch {
|
|
441
|
-
}
|
|
442
|
-
try {
|
|
443
|
-
lowerReal = await fs3.realpath(altMemoryFile);
|
|
444
|
-
} catch {
|
|
445
|
-
}
|
|
446
|
-
if (upperReal !== lowerReal) {
|
|
447
|
-
throw new Error(
|
|
448
|
-
`Both MEMORY.md and memory.md exist in ${memoryDir}. Please remove one to avoid ambiguity.`
|
|
449
|
-
);
|
|
450
|
-
}
|
|
451
|
-
result.push(memoryFile);
|
|
452
|
-
} else if (hasUpper) {
|
|
453
|
-
result.push(memoryFile);
|
|
454
|
-
} else if (hasLower) {
|
|
455
|
-
result.push(altMemoryFile);
|
|
456
|
-
}
|
|
457
|
-
const memorySubDir = path4.join(memoryDir, "memory");
|
|
458
|
-
if (await exists(memorySubDir)) {
|
|
459
|
-
await walkDir(memorySubDir, result);
|
|
460
|
-
}
|
|
461
|
-
if (result.length <= 1) return result;
|
|
462
|
-
const seen = /* @__PURE__ */ new Set();
|
|
463
|
-
const deduped = [];
|
|
464
|
-
for (const entry of result) {
|
|
465
|
-
let key = entry;
|
|
466
|
-
try {
|
|
467
|
-
key = await fs3.realpath(entry);
|
|
468
|
-
} catch {
|
|
469
|
-
}
|
|
470
|
-
if (seen.has(key)) continue;
|
|
471
|
-
seen.add(key);
|
|
472
|
-
deduped.push(entry);
|
|
473
|
-
}
|
|
474
|
-
return deduped;
|
|
475
|
-
}
|
|
476
|
-
function hashText(value) {
|
|
477
|
-
return crypto2.createHash("sha256").update(value).digest("hex");
|
|
478
|
-
}
|
|
479
|
-
async function buildFileEntry(absPath, memoryDir) {
|
|
480
|
-
const stat = await fs3.stat(absPath);
|
|
481
|
-
const content = await fs3.readFile(absPath, "utf-8");
|
|
482
|
-
const hash = hashText(content);
|
|
483
|
-
return {
|
|
484
|
-
path: path4.relative(memoryDir, absPath).replace(/\\/g, "/"),
|
|
485
|
-
absPath,
|
|
486
|
-
mtimeMs: stat.mtimeMs,
|
|
487
|
-
size: stat.size,
|
|
488
|
-
hash
|
|
489
|
-
};
|
|
490
|
-
}
|
|
491
|
-
function stripPrivateContent(content) {
|
|
492
|
-
return content.replace(/<private>[\s\S]*?<\/private>/gi, (match) => {
|
|
493
|
-
const lineCount = match.split("\n").length;
|
|
494
|
-
return "\n".repeat(lineCount - 1);
|
|
495
|
-
});
|
|
496
|
-
}
|
|
497
|
-
function chunkMarkdown(content, chunking) {
|
|
498
|
-
const stripped = stripPrivateContent(content);
|
|
499
|
-
const lines = stripped.split("\n");
|
|
500
|
-
if (lines.length === 0) return [];
|
|
501
|
-
const maxChars = Math.max(32, chunking.tokens * 4);
|
|
502
|
-
const overlapChars = Math.max(0, chunking.overlap * 4);
|
|
503
|
-
const chunks = [];
|
|
504
|
-
let current = [];
|
|
505
|
-
let currentChars = 0;
|
|
506
|
-
const flush = () => {
|
|
507
|
-
if (current.length === 0) return;
|
|
508
|
-
const firstEntry = current[0];
|
|
509
|
-
const lastEntry = current[current.length - 1];
|
|
510
|
-
if (!firstEntry || !lastEntry) return;
|
|
511
|
-
const text = current.map((entry) => entry.line).join("\n");
|
|
512
|
-
const startLine = firstEntry.lineNo;
|
|
513
|
-
const endLine = lastEntry.lineNo;
|
|
514
|
-
chunks.push({
|
|
515
|
-
startLine,
|
|
516
|
-
endLine,
|
|
517
|
-
text,
|
|
518
|
-
hash: hashText(text)
|
|
519
|
-
});
|
|
520
|
-
};
|
|
521
|
-
const carryOverlap = () => {
|
|
522
|
-
if (overlapChars <= 0 || current.length === 0) {
|
|
523
|
-
current = [];
|
|
524
|
-
currentChars = 0;
|
|
525
|
-
return;
|
|
526
|
-
}
|
|
527
|
-
let acc = 0;
|
|
528
|
-
const kept = [];
|
|
529
|
-
for (let i = current.length - 1; i >= 0; i -= 1) {
|
|
530
|
-
const entry = current[i];
|
|
531
|
-
if (!entry) continue;
|
|
532
|
-
acc += entry.line.length + 1;
|
|
533
|
-
kept.unshift(entry);
|
|
534
|
-
if (acc >= overlapChars) break;
|
|
535
|
-
}
|
|
536
|
-
current = kept;
|
|
537
|
-
currentChars = kept.reduce((sum, entry) => sum + entry.line.length + 1, 0);
|
|
538
|
-
};
|
|
539
|
-
for (let i = 0; i < lines.length; i += 1) {
|
|
540
|
-
const line = lines[i] ?? "";
|
|
541
|
-
const lineNo = i + 1;
|
|
542
|
-
const segments = [];
|
|
543
|
-
if (line.length === 0) {
|
|
544
|
-
segments.push("");
|
|
545
|
-
} else {
|
|
546
|
-
for (let start = 0; start < line.length; start += maxChars) {
|
|
547
|
-
segments.push(line.slice(start, start + maxChars));
|
|
548
|
-
}
|
|
549
|
-
}
|
|
550
|
-
for (const segment of segments) {
|
|
551
|
-
const lineSize = segment.length + 1;
|
|
552
|
-
if (currentChars + lineSize > maxChars && current.length > 0) {
|
|
553
|
-
flush();
|
|
554
|
-
carryOverlap();
|
|
555
|
-
}
|
|
556
|
-
current.push({ line: segment, lineNo });
|
|
557
|
-
currentChars += lineSize;
|
|
558
|
-
}
|
|
559
|
-
}
|
|
560
|
-
flush();
|
|
561
|
-
return chunks;
|
|
562
|
-
}
|
|
563
|
-
function extractChunkMetadata(text) {
|
|
564
|
-
const typeMatch = text.match(/<!--\s*type:\s*([\w-]+)\s*-->/i);
|
|
565
|
-
return typeMatch ? { type: typeMatch[1].toLowerCase() } : {};
|
|
566
|
-
}
|
|
567
|
-
function parseEmbedding(raw) {
|
|
568
|
-
try {
|
|
569
|
-
const parsed = JSON.parse(raw);
|
|
570
|
-
return Array.isArray(parsed) ? parsed : [];
|
|
571
|
-
} catch {
|
|
572
|
-
return [];
|
|
573
|
-
}
|
|
574
|
-
}
|
|
575
|
-
function cosineSimilarity(a, b) {
|
|
576
|
-
if (a.length === 0 || b.length === 0) return 0;
|
|
577
|
-
const len = Math.min(a.length, b.length);
|
|
578
|
-
let dot = 0;
|
|
579
|
-
let normA = 0;
|
|
580
|
-
let normB = 0;
|
|
581
|
-
for (let i = 0; i < len; i += 1) {
|
|
582
|
-
const av = a[i] ?? 0;
|
|
583
|
-
const bv = b[i] ?? 0;
|
|
584
|
-
dot += av * bv;
|
|
585
|
-
normA += av * av;
|
|
586
|
-
normB += bv * bv;
|
|
587
|
-
}
|
|
588
|
-
if (normA === 0 || normB === 0) return 0;
|
|
589
|
-
return dot / (Math.sqrt(normA) * Math.sqrt(normB));
|
|
590
|
-
}
|
|
591
|
-
function truncateUtf16Safe(text, maxChars) {
|
|
592
|
-
if (text.length <= maxChars) return text;
|
|
593
|
-
return text.slice(0, maxChars);
|
|
594
|
-
}
|
|
595
|
-
function vectorToBlob(embedding) {
|
|
596
|
-
return Buffer.from(new Float32Array(embedding).buffer);
|
|
597
|
-
}
|
|
598
|
-
|
|
599
|
-
// src/search/hybrid.ts
|
|
600
|
-
function buildFtsQuery(raw) {
|
|
601
|
-
const tokens = raw.match(/[A-Za-z0-9_]+/g)?.map((t) => t.trim()).filter(Boolean) ?? [];
|
|
602
|
-
if (tokens.length === 0) return null;
|
|
603
|
-
const quoted = tokens.map((t) => `"${t.replaceAll('"', "")}"`);
|
|
604
|
-
return quoted.join(" AND ");
|
|
605
|
-
}
|
|
606
|
-
function bm25RankToScore(rank) {
|
|
607
|
-
if (!Number.isFinite(rank)) {
|
|
608
|
-
return 0;
|
|
609
|
-
}
|
|
610
|
-
const absRank = Math.abs(rank);
|
|
611
|
-
return 1 / (1 + absRank);
|
|
612
|
-
}
|
|
613
|
-
function mergeHybridResults(params) {
|
|
614
|
-
const byId = /* @__PURE__ */ new Map();
|
|
615
|
-
for (const r of params.vector) {
|
|
616
|
-
byId.set(r.id, {
|
|
617
|
-
id: r.id,
|
|
618
|
-
path: r.path,
|
|
619
|
-
startLine: r.startLine,
|
|
620
|
-
endLine: r.endLine,
|
|
621
|
-
source: r.source,
|
|
622
|
-
snippet: r.snippet,
|
|
623
|
-
vectorScore: r.vectorScore,
|
|
624
|
-
textScore: 0
|
|
625
|
-
});
|
|
626
|
-
}
|
|
627
|
-
for (const r of params.keyword) {
|
|
628
|
-
const existing = byId.get(r.id);
|
|
629
|
-
if (existing) {
|
|
630
|
-
existing.textScore = r.textScore;
|
|
631
|
-
if (r.snippet && r.snippet.length > 0) existing.snippet = r.snippet;
|
|
632
|
-
} else {
|
|
633
|
-
byId.set(r.id, {
|
|
634
|
-
id: r.id,
|
|
635
|
-
path: r.path,
|
|
636
|
-
startLine: r.startLine,
|
|
637
|
-
endLine: r.endLine,
|
|
638
|
-
source: r.source,
|
|
639
|
-
snippet: r.snippet,
|
|
640
|
-
vectorScore: 0,
|
|
641
|
-
textScore: r.textScore
|
|
642
|
-
});
|
|
643
|
-
}
|
|
644
|
-
}
|
|
645
|
-
let vw = params.vectorWeight;
|
|
646
|
-
let tw = params.textWeight;
|
|
647
|
-
if (params.vector.length === 0 && params.keyword.length > 0) {
|
|
648
|
-
vw = 0;
|
|
649
|
-
tw = 1;
|
|
650
|
-
} else if (params.keyword.length === 0 && params.vector.length > 0) {
|
|
651
|
-
vw = 1;
|
|
652
|
-
tw = 0;
|
|
653
|
-
}
|
|
654
|
-
const merged = Array.from(byId.values()).map((entry) => {
|
|
655
|
-
const score = vw * entry.vectorScore + tw * entry.textScore;
|
|
656
|
-
return {
|
|
657
|
-
path: entry.path,
|
|
658
|
-
startLine: entry.startLine,
|
|
659
|
-
endLine: entry.endLine,
|
|
660
|
-
score,
|
|
661
|
-
snippet: entry.snippet,
|
|
662
|
-
source: entry.source
|
|
663
|
-
};
|
|
664
|
-
});
|
|
665
|
-
return merged.sort((a, b) => b.score - a.score);
|
|
666
|
-
}
|
|
667
|
-
|
|
668
|
-
// src/search/search.ts
|
|
669
|
-
function buildKnowledgeFilterSql(opts) {
|
|
670
|
-
const clauses = [];
|
|
671
|
-
const params = [];
|
|
672
|
-
if (opts.knowledgeType) {
|
|
673
|
-
clauses.push(` AND c.knowledge_type = ?`);
|
|
674
|
-
params.push(opts.knowledgeType);
|
|
675
|
-
}
|
|
676
|
-
if (opts.minConfidence !== void 0) {
|
|
677
|
-
clauses.push(` AND c.confidence >= ?`);
|
|
678
|
-
params.push(opts.minConfidence);
|
|
679
|
-
}
|
|
680
|
-
if (opts.domain && opts.domain.length > 0) {
|
|
681
|
-
const domainPlaceholders = opts.domain.map(() => "?").join(", ");
|
|
682
|
-
clauses.push(
|
|
683
|
-
` AND EXISTS (SELECT 1 FROM json_each(c.domains) AS d WHERE d.value IN (${domainPlaceholders}))`
|
|
684
|
-
);
|
|
685
|
-
params.push(...opts.domain);
|
|
686
|
-
}
|
|
687
|
-
if (opts.entities && opts.entities.length > 0) {
|
|
688
|
-
const entityPlaceholders = opts.entities.map(() => "?").join(", ");
|
|
689
|
-
clauses.push(
|
|
690
|
-
` AND EXISTS (SELECT 1 FROM json_each(c.entities) AS e WHERE e.value IN (${entityPlaceholders}))`
|
|
691
|
-
);
|
|
692
|
-
params.push(...opts.entities);
|
|
693
|
-
}
|
|
694
|
-
return { sql: clauses.join(""), params };
|
|
695
|
-
}
|
|
696
|
-
async function searchVector(params) {
|
|
697
|
-
if (params.queryVec.length === 0 || params.limit <= 0) return [];
|
|
698
|
-
if (await params.ensureVectorReady(params.queryVec.length)) {
|
|
699
|
-
const rows = params.db.prepare(
|
|
700
|
-
`SELECT c.id, c.path, c.start_line, c.end_line, c.text,
|
|
701
|
-
c.source,
|
|
702
|
-
vec_distance_cosine(v.embedding, ?) AS dist
|
|
703
|
-
FROM ${params.vectorTable} v
|
|
704
|
-
JOIN chunks c ON c.id = v.id
|
|
705
|
-
WHERE c.model = ?${params.sourceFilterVec.sql}
|
|
706
|
-
ORDER BY dist ASC
|
|
707
|
-
LIMIT ?`
|
|
708
|
-
).all(
|
|
709
|
-
vectorToBlob(params.queryVec),
|
|
710
|
-
params.providerModel,
|
|
711
|
-
...params.sourceFilterVec.params,
|
|
712
|
-
params.limit
|
|
713
|
-
);
|
|
714
|
-
return rows.map((row) => ({
|
|
715
|
-
id: row.id,
|
|
716
|
-
path: row.path,
|
|
717
|
-
startLine: row.start_line,
|
|
718
|
-
endLine: row.end_line,
|
|
719
|
-
score: 1 - row.dist,
|
|
720
|
-
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
|
721
|
-
source: row.source
|
|
722
|
-
}));
|
|
723
|
-
}
|
|
724
|
-
const candidates = listChunks({
|
|
725
|
-
db: params.db,
|
|
726
|
-
providerModel: params.providerModel,
|
|
727
|
-
sourceFilter: params.sourceFilterChunks
|
|
728
|
-
});
|
|
729
|
-
const scored = candidates.map((chunk) => ({
|
|
730
|
-
chunk,
|
|
731
|
-
score: cosineSimilarity(params.queryVec, chunk.embedding)
|
|
732
|
-
})).filter((entry) => Number.isFinite(entry.score));
|
|
733
|
-
return scored.sort((a, b) => b.score - a.score).slice(0, params.limit).map((entry) => ({
|
|
734
|
-
id: entry.chunk.id,
|
|
735
|
-
path: entry.chunk.path,
|
|
736
|
-
startLine: entry.chunk.startLine,
|
|
737
|
-
endLine: entry.chunk.endLine,
|
|
738
|
-
score: entry.score,
|
|
739
|
-
snippet: truncateUtf16Safe(entry.chunk.text, params.snippetMaxChars),
|
|
740
|
-
source: entry.chunk.source
|
|
741
|
-
}));
|
|
742
|
-
}
|
|
743
|
-
function listChunks(params) {
|
|
744
|
-
const rows = params.db.prepare(
|
|
745
|
-
`SELECT id, path, start_line, end_line, text, embedding, source
|
|
746
|
-
FROM chunks
|
|
747
|
-
WHERE model = ?${params.sourceFilter.sql}`
|
|
748
|
-
).all(params.providerModel, ...params.sourceFilter.params);
|
|
749
|
-
return rows.map((row) => ({
|
|
750
|
-
id: row.id,
|
|
751
|
-
path: row.path,
|
|
752
|
-
startLine: row.start_line,
|
|
753
|
-
endLine: row.end_line,
|
|
754
|
-
text: row.text,
|
|
755
|
-
embedding: parseEmbedding(row.embedding),
|
|
756
|
-
source: row.source
|
|
757
|
-
}));
|
|
758
|
-
}
|
|
759
|
-
async function searchKeyword(params) {
|
|
760
|
-
if (params.limit <= 0) return [];
|
|
761
|
-
const ftsQuery = params.buildFtsQuery(params.query);
|
|
762
|
-
if (!ftsQuery) return [];
|
|
763
|
-
const rows = params.db.prepare(
|
|
764
|
-
`SELECT id, path, source, start_line, end_line, text,
|
|
765
|
-
bm25(${params.ftsTable}) AS rank
|
|
766
|
-
FROM ${params.ftsTable}
|
|
767
|
-
WHERE ${params.ftsTable} MATCH ? AND model = ?${params.sourceFilter.sql}
|
|
768
|
-
ORDER BY rank ASC
|
|
769
|
-
LIMIT ?`
|
|
770
|
-
).all(ftsQuery, params.providerModel, ...params.sourceFilter.params, params.limit);
|
|
771
|
-
return rows.map((row) => {
|
|
772
|
-
const textScore = params.bm25RankToScore(row.rank);
|
|
773
|
-
return {
|
|
774
|
-
id: row.id,
|
|
775
|
-
path: row.path,
|
|
776
|
-
startLine: row.start_line,
|
|
777
|
-
endLine: row.end_line,
|
|
778
|
-
score: textScore,
|
|
779
|
-
textScore,
|
|
780
|
-
snippet: truncateUtf16Safe(row.text, params.snippetMaxChars),
|
|
781
|
-
source: row.source
|
|
782
|
-
};
|
|
783
|
-
});
|
|
784
|
-
}
|
|
785
|
-
|
|
786
|
-
// src/db/schema.ts
|
|
787
|
-
var SCHEMA_VERSION = 4;
|
|
788
|
-
function ensureMemoryIndexSchema(params) {
|
|
789
|
-
params.db.exec(`
|
|
790
|
-
CREATE TABLE IF NOT EXISTS meta (
|
|
791
|
-
key TEXT PRIMARY KEY,
|
|
792
|
-
value TEXT NOT NULL
|
|
793
|
-
);
|
|
794
|
-
`);
|
|
795
|
-
const migrated = migrateIfNeeded(params.db, params.ftsTable);
|
|
796
|
-
params.db.exec(`
|
|
797
|
-
CREATE TABLE IF NOT EXISTS files (
|
|
798
|
-
path TEXT PRIMARY KEY,
|
|
799
|
-
source TEXT NOT NULL DEFAULT 'memory',
|
|
800
|
-
hash TEXT NOT NULL,
|
|
801
|
-
mtime INTEGER NOT NULL,
|
|
802
|
-
size INTEGER NOT NULL
|
|
803
|
-
);
|
|
804
|
-
`);
|
|
805
|
-
params.db.exec(`
|
|
806
|
-
CREATE TABLE IF NOT EXISTS chunks (
|
|
807
|
-
id TEXT PRIMARY KEY,
|
|
808
|
-
path TEXT NOT NULL,
|
|
809
|
-
source TEXT NOT NULL DEFAULT 'memory',
|
|
810
|
-
start_line INTEGER NOT NULL,
|
|
811
|
-
end_line INTEGER NOT NULL,
|
|
812
|
-
hash TEXT NOT NULL,
|
|
813
|
-
model TEXT NOT NULL,
|
|
814
|
-
text TEXT NOT NULL,
|
|
815
|
-
embedding TEXT NOT NULL,
|
|
816
|
-
updated_at INTEGER NOT NULL
|
|
817
|
-
);
|
|
818
|
-
`);
|
|
819
|
-
params.db.exec(`
|
|
820
|
-
CREATE TABLE IF NOT EXISTS ${params.embeddingCacheTable} (
|
|
821
|
-
provider TEXT NOT NULL,
|
|
822
|
-
model TEXT NOT NULL,
|
|
823
|
-
provider_key TEXT NOT NULL,
|
|
824
|
-
hash TEXT NOT NULL,
|
|
825
|
-
embedding TEXT NOT NULL,
|
|
826
|
-
dims INTEGER,
|
|
827
|
-
updated_at INTEGER NOT NULL,
|
|
828
|
-
PRIMARY KEY (provider, model, provider_key, hash)
|
|
829
|
-
);
|
|
830
|
-
`);
|
|
831
|
-
params.db.exec(
|
|
832
|
-
`CREATE INDEX IF NOT EXISTS idx_embedding_cache_updated_at ON ${params.embeddingCacheTable}(updated_at);`
|
|
833
|
-
);
|
|
834
|
-
let ftsAvailable = false;
|
|
835
|
-
let ftsError;
|
|
836
|
-
if (params.ftsEnabled) {
|
|
837
|
-
try {
|
|
838
|
-
params.db.exec(
|
|
839
|
-
`CREATE VIRTUAL TABLE IF NOT EXISTS ${params.ftsTable} USING fts5(
|
|
840
|
-
text,
|
|
841
|
-
id UNINDEXED,
|
|
842
|
-
path UNINDEXED,
|
|
843
|
-
source UNINDEXED,
|
|
844
|
-
model UNINDEXED,
|
|
845
|
-
start_line UNINDEXED,
|
|
846
|
-
end_line UNINDEXED
|
|
847
|
-
);`
|
|
848
|
-
);
|
|
849
|
-
ftsAvailable = true;
|
|
850
|
-
} catch (err) {
|
|
851
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
852
|
-
ftsAvailable = false;
|
|
853
|
-
ftsError = message;
|
|
854
|
-
}
|
|
855
|
-
}
|
|
856
|
-
ensureColumn(params.db, "files", "source", "TEXT NOT NULL DEFAULT 'memory'");
|
|
857
|
-
ensureColumn(params.db, "chunks", "source", "TEXT NOT NULL DEFAULT 'memory'");
|
|
858
|
-
ensureColumn(params.db, "chunks", "type", "TEXT");
|
|
859
|
-
ensureColumn(params.db, "chunks", "knowledge_type", "TEXT");
|
|
860
|
-
ensureColumn(params.db, "chunks", "knowledge_id", "TEXT");
|
|
861
|
-
ensureColumn(params.db, "chunks", "domains", "TEXT");
|
|
862
|
-
ensureColumn(params.db, "chunks", "entities", "TEXT");
|
|
863
|
-
ensureColumn(params.db, "chunks", "confidence", "REAL");
|
|
864
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_path ON chunks(path);`);
|
|
865
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_source ON chunks(source);`);
|
|
866
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_type ON chunks(type);`);
|
|
867
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_knowledge_type ON chunks(knowledge_type);`);
|
|
868
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_chunks_knowledge_id ON chunks(knowledge_id);`);
|
|
869
|
-
params.db.exec(`
|
|
870
|
-
CREATE TABLE IF NOT EXISTS knowledge_links (
|
|
871
|
-
from_id TEXT NOT NULL,
|
|
872
|
-
to_id TEXT NOT NULL,
|
|
873
|
-
relation TEXT NOT NULL,
|
|
874
|
-
layer TEXT,
|
|
875
|
-
weight REAL DEFAULT 0.5,
|
|
876
|
-
source_path TEXT,
|
|
877
|
-
created_at INTEGER,
|
|
878
|
-
PRIMARY KEY (from_id, to_id, relation)
|
|
879
|
-
);
|
|
880
|
-
`);
|
|
881
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_from ON knowledge_links(from_id);`);
|
|
882
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_to ON knowledge_links(to_id);`);
|
|
883
|
-
params.db.exec(`CREATE INDEX IF NOT EXISTS idx_kl_layer ON knowledge_links(layer);`);
|
|
884
|
-
params.db.prepare(
|
|
885
|
-
`INSERT OR REPLACE INTO meta (key, value) VALUES ('schema_version', ?)`
|
|
886
|
-
).run(String(SCHEMA_VERSION));
|
|
887
|
-
return { ftsAvailable, ...ftsError ? { ftsError } : {}, ...migrated ? { migrated } : {} };
|
|
888
|
-
}
|
|
889
|
-
function migrateIfNeeded(db, ftsTable) {
|
|
890
|
-
let storedVersion = 0;
|
|
891
|
-
try {
|
|
892
|
-
const row = db.prepare(
|
|
893
|
-
`SELECT value FROM meta WHERE key = 'schema_version'`
|
|
894
|
-
).get();
|
|
895
|
-
if (row) {
|
|
896
|
-
storedVersion = parseInt(row.value, 10) || 0;
|
|
897
|
-
}
|
|
898
|
-
} catch {
|
|
899
|
-
storedVersion = 0;
|
|
900
|
-
}
|
|
901
|
-
if (storedVersion >= SCHEMA_VERSION) return false;
|
|
902
|
-
if (storedVersion > 0 && storedVersion < SCHEMA_VERSION) {
|
|
903
|
-
db.exec(`DROP TABLE IF EXISTS files`);
|
|
904
|
-
db.exec(`DROP TABLE IF EXISTS chunks`);
|
|
905
|
-
db.exec(`DROP TABLE IF EXISTS knowledge_links`);
|
|
906
|
-
db.exec(`DROP TABLE IF EXISTS ${ftsTable}`);
|
|
907
|
-
try {
|
|
908
|
-
db.exec(`DROP TABLE IF EXISTS chunks_vec`);
|
|
909
|
-
} catch {
|
|
910
|
-
}
|
|
911
|
-
}
|
|
912
|
-
return storedVersion > 0;
|
|
913
|
-
}
|
|
914
|
-
function ensureColumn(db, table, column, definition) {
|
|
915
|
-
const rows = db.prepare(`PRAGMA table_info(${table})`).all();
|
|
916
|
-
if (rows.some((row) => row.name === column)) return;
|
|
917
|
-
db.exec(`ALTER TABLE ${table} ADD COLUMN ${column} ${definition}`);
|
|
918
|
-
}
|
|
919
|
-
|
|
920
|
-
// src/session.ts
|
|
921
|
-
import * as os3 from "os";
|
|
922
|
-
function parseFrontmatter(content) {
|
|
923
|
-
const frontmatterRegex = /^---\n([\s\S]*?)\n---\n/;
|
|
924
|
-
const match = content.match(frontmatterRegex);
|
|
925
|
-
if (!match) {
|
|
926
|
-
return { frontmatter: void 0, body: content };
|
|
927
|
-
}
|
|
928
|
-
const yamlContent = match[1];
|
|
929
|
-
const body = content.slice(match[0].length);
|
|
930
|
-
try {
|
|
931
|
-
const frontmatter = parseSimpleYaml(yamlContent);
|
|
932
|
-
return { frontmatter, body };
|
|
933
|
-
} catch {
|
|
934
|
-
return { frontmatter: void 0, body: content };
|
|
935
|
-
}
|
|
936
|
-
}
|
|
937
|
-
function parseSimpleYaml(yaml) {
|
|
938
|
-
const lines = yaml.split("\n");
|
|
939
|
-
return parseYamlBlock(lines, 0, 0, lines.length).value;
|
|
940
|
-
}
|
|
941
|
-
function parseYamlBlock(lines, indent, startIdx, endIdx) {
|
|
942
|
-
const result = {};
|
|
943
|
-
let i = startIdx;
|
|
944
|
-
while (i < endIdx) {
|
|
945
|
-
const line = lines[i];
|
|
946
|
-
if (!line || !line.trim()) {
|
|
947
|
-
i++;
|
|
948
|
-
continue;
|
|
949
|
-
}
|
|
950
|
-
const lineIndent = getIndent(line);
|
|
951
|
-
if (lineIndent < indent) break;
|
|
952
|
-
if (lineIndent > indent) {
|
|
953
|
-
i++;
|
|
954
|
-
continue;
|
|
955
|
-
}
|
|
956
|
-
const keyMatch = line.match(/^(\s*)([\w-]+):\s*(.*)?$/);
|
|
957
|
-
if (!keyMatch) {
|
|
958
|
-
i++;
|
|
959
|
-
continue;
|
|
960
|
-
}
|
|
961
|
-
const [, , key, rawValue] = keyMatch;
|
|
962
|
-
const value = rawValue?.trim() ?? "";
|
|
963
|
-
if (value === "" || value === void 0) {
|
|
964
|
-
const nextNonEmpty = findNextNonEmptyLine(lines, i + 1, endIdx);
|
|
965
|
-
if (nextNonEmpty < endIdx) {
|
|
966
|
-
const nextLine = lines[nextNonEmpty];
|
|
967
|
-
const nextIndent = getIndent(nextLine);
|
|
968
|
-
if (nextIndent > indent) {
|
|
969
|
-
if (nextLine.trimStart().startsWith("- ")) {
|
|
970
|
-
const listResult = parseYamlList(lines, nextIndent, i + 1, endIdx);
|
|
971
|
-
result[key] = listResult.value;
|
|
972
|
-
i = listResult.nextIdx;
|
|
973
|
-
} else {
|
|
974
|
-
const blockResult = parseYamlBlock(lines, nextIndent, i + 1, endIdx);
|
|
975
|
-
result[key] = blockResult.value;
|
|
976
|
-
i = blockResult.nextIdx;
|
|
977
|
-
}
|
|
978
|
-
continue;
|
|
979
|
-
}
|
|
980
|
-
}
|
|
981
|
-
result[key] = null;
|
|
982
|
-
i++;
|
|
983
|
-
} else {
|
|
984
|
-
result[key] = parseYamlValue(value);
|
|
985
|
-
i++;
|
|
986
|
-
}
|
|
987
|
-
}
|
|
988
|
-
return { value: result, nextIdx: i };
|
|
989
|
-
}
|
|
990
|
-
function parseYamlList(lines, indent, startIdx, endIdx) {
|
|
991
|
-
const result = [];
|
|
992
|
-
let i = startIdx;
|
|
993
|
-
while (i < endIdx) {
|
|
994
|
-
const line = lines[i];
|
|
995
|
-
if (!line || !line.trim()) {
|
|
996
|
-
i++;
|
|
997
|
-
continue;
|
|
998
|
-
}
|
|
999
|
-
const lineIndent = getIndent(line);
|
|
1000
|
-
if (lineIndent < indent) break;
|
|
1001
|
-
if (lineIndent > indent) {
|
|
1002
|
-
i++;
|
|
1003
|
-
continue;
|
|
1004
|
-
}
|
|
1005
|
-
const trimmed = line.trimStart();
|
|
1006
|
-
if (!trimmed.startsWith("- ")) break;
|
|
1007
|
-
const itemContent = trimmed.slice(2).trim();
|
|
1008
|
-
if (itemContent === "" || itemContent === void 0) {
|
|
1009
|
-
const nextNonEmpty = findNextNonEmptyLine(lines, i + 1, endIdx);
|
|
1010
|
-
if (nextNonEmpty < endIdx) {
|
|
1011
|
-
const nextIndent = getIndent(lines[nextNonEmpty]);
|
|
1012
|
-
if (nextIndent > indent) {
|
|
1013
|
-
const blockResult = parseYamlBlock(lines, nextIndent, i + 1, endIdx);
|
|
1014
|
-
result.push(blockResult.value);
|
|
1015
|
-
i = blockResult.nextIdx;
|
|
1016
|
-
continue;
|
|
1017
|
-
}
|
|
1018
|
-
}
|
|
1019
|
-
result.push(null);
|
|
1020
|
-
i++;
|
|
1021
|
-
} else {
|
|
1022
|
-
const kvMatch = itemContent.match(/^([\w-]+):\s*(.*)$/);
|
|
1023
|
-
if (kvMatch) {
|
|
1024
|
-
const obj = {};
|
|
1025
|
-
const [, firstKey, firstVal] = kvMatch;
|
|
1026
|
-
obj[firstKey] = parseYamlValue(firstVal?.trim() ?? "");
|
|
1027
|
-
const itemKeyIndent = indent + 2;
|
|
1028
|
-
let j = i + 1;
|
|
1029
|
-
while (j < endIdx) {
|
|
1030
|
-
const nextLine = lines[j];
|
|
1031
|
-
if (!nextLine || !nextLine.trim()) {
|
|
1032
|
-
j++;
|
|
1033
|
-
continue;
|
|
1034
|
-
}
|
|
1035
|
-
const nextLineIndent = getIndent(nextLine);
|
|
1036
|
-
if (nextLineIndent < itemKeyIndent) break;
|
|
1037
|
-
if (nextLineIndent === itemKeyIndent) {
|
|
1038
|
-
const nextKv = nextLine.match(/^\s*([\w-]+):\s*(.*)$/);
|
|
1039
|
-
if (nextKv) {
|
|
1040
|
-
const [, nk, nv] = nextKv;
|
|
1041
|
-
obj[nk] = parseYamlValue(nv?.trim() ?? "");
|
|
1042
|
-
j++;
|
|
1043
|
-
continue;
|
|
1044
|
-
}
|
|
1045
|
-
}
|
|
1046
|
-
break;
|
|
1047
|
-
}
|
|
1048
|
-
result.push(obj);
|
|
1049
|
-
i = j;
|
|
1050
|
-
} else {
|
|
1051
|
-
result.push(parseYamlValue(itemContent));
|
|
1052
|
-
i++;
|
|
1053
|
-
}
|
|
1054
|
-
}
|
|
1055
|
-
}
|
|
1056
|
-
return { value: result, nextIdx: i };
|
|
1057
|
-
}
|
|
1058
|
-
function getIndent(line) {
|
|
1059
|
-
const match = line.match(/^(\s*)/);
|
|
1060
|
-
return match ? match[1].length : 0;
|
|
1061
|
-
}
|
|
1062
|
-
function findNextNonEmptyLine(lines, from, end) {
|
|
1063
|
-
for (let i = from; i < end; i++) {
|
|
1064
|
-
if (lines[i]?.trim()) return i;
|
|
1065
|
-
}
|
|
1066
|
-
return end;
|
|
1067
|
-
}
|
|
1068
|
-
function parseYamlValue(value) {
|
|
1069
|
-
if (value === "") return null;
|
|
1070
|
-
if (value.startsWith('"') && value.endsWith('"') || value.startsWith("'") && value.endsWith("'")) {
|
|
1071
|
-
return value.slice(1, -1);
|
|
1072
|
-
}
|
|
1073
|
-
if (value === "null" || value === "~") return null;
|
|
1074
|
-
if (value === "true") return true;
|
|
1075
|
-
if (value === "false") return false;
|
|
1076
|
-
const num = Number(value);
|
|
1077
|
-
if (!isNaN(num) && value !== "") return num;
|
|
1078
|
-
if (value.startsWith("[") && value.endsWith("]")) {
|
|
1079
|
-
const inner = value.slice(1, -1);
|
|
1080
|
-
if (inner.trim() === "") return [];
|
|
1081
|
-
return inner.split(",").map((s) => parseYamlValue(s.trim()));
|
|
1082
|
-
}
|
|
1083
|
-
return value;
|
|
1084
|
-
}
|
|
1085
|
-
function serializeFrontmatter(frontmatter) {
|
|
1086
|
-
const lines = ["---"];
|
|
1087
|
-
if (frontmatter.id) {
|
|
1088
|
-
lines.push(`id: ${frontmatter.id}`);
|
|
1089
|
-
}
|
|
1090
|
-
if (frontmatter.type) {
|
|
1091
|
-
lines.push(`type: ${frontmatter.type}`);
|
|
1092
|
-
}
|
|
1093
|
-
if (frontmatter.session) {
|
|
1094
|
-
lines.push("session:");
|
|
1095
|
-
const session = frontmatter.session;
|
|
1096
|
-
if (session.id) lines.push(` id: ${session.id}`);
|
|
1097
|
-
if (session.source) lines.push(` source: ${session.source}`);
|
|
1098
|
-
if (session.project) lines.push(` project: ${formatPath2(session.project)}`);
|
|
1099
|
-
if (session.transcript) lines.push(` transcript: ${formatPath2(session.transcript)}`);
|
|
1100
|
-
}
|
|
1101
|
-
if (frontmatter.created) {
|
|
1102
|
-
lines.push(`created: ${frontmatter.created}`);
|
|
1103
|
-
}
|
|
1104
|
-
if (frontmatter.updated) {
|
|
1105
|
-
lines.push(`updated: ${frontmatter.updated}`);
|
|
1106
|
-
}
|
|
1107
|
-
if (frontmatter.tags && frontmatter.tags.length > 0) {
|
|
1108
|
-
lines.push(`tags: [${frontmatter.tags.join(", ")}]`);
|
|
1109
|
-
}
|
|
1110
|
-
if (frontmatter.domain && frontmatter.domain.length > 0) {
|
|
1111
|
-
lines.push(`domain: [${frontmatter.domain.join(", ")}]`);
|
|
1112
|
-
}
|
|
1113
|
-
if (frontmatter.entities && frontmatter.entities.length > 0) {
|
|
1114
|
-
lines.push(`entities: [${frontmatter.entities.join(", ")}]`);
|
|
1115
|
-
}
|
|
1116
|
-
if (frontmatter.confidence !== void 0) {
|
|
1117
|
-
lines.push(`confidence: ${frontmatter.confidence}`);
|
|
1118
|
-
}
|
|
1119
|
-
if (frontmatter.source) {
|
|
1120
|
-
lines.push("source:");
|
|
1121
|
-
if (frontmatter.source.origin) lines.push(` origin: ${frontmatter.source.origin}`);
|
|
1122
|
-
if (frontmatter.source.trajectories && frontmatter.source.trajectories.length > 0) {
|
|
1123
|
-
lines.push(` trajectories: [${frontmatter.source.trajectories.join(", ")}]`);
|
|
1124
|
-
}
|
|
1125
|
-
if (frontmatter.source.agentId) lines.push(` agentId: ${frontmatter.source.agentId}`);
|
|
1126
|
-
}
|
|
1127
|
-
if (frontmatter.links && frontmatter.links.length > 0) {
|
|
1128
|
-
lines.push("links:");
|
|
1129
|
-
for (const link of frontmatter.links) {
|
|
1130
|
-
lines.push(` - target: ${link.target}`);
|
|
1131
|
-
lines.push(` relation: ${link.relation}`);
|
|
1132
|
-
if (link.layer) lines.push(` layer: ${link.layer}`);
|
|
1133
|
-
}
|
|
1134
|
-
}
|
|
1135
|
-
if (frontmatter.supersedes !== void 0) {
|
|
1136
|
-
lines.push(`supersedes: ${frontmatter.supersedes === null ? "~" : frontmatter.supersedes}`);
|
|
1137
|
-
}
|
|
1138
|
-
lines.push("---");
|
|
1139
|
-
return lines.join("\n") + "\n";
|
|
1140
|
-
}
|
|
1141
|
-
function addFrontmatter(content, frontmatter) {
|
|
1142
|
-
const { frontmatter: existing, body } = parseFrontmatter(content);
|
|
1143
|
-
const merged = {
|
|
1144
|
-
...existing,
|
|
1145
|
-
...frontmatter,
|
|
1146
|
-
session: {
|
|
1147
|
-
...existing?.session,
|
|
1148
|
-
...frontmatter.session
|
|
1149
|
-
}
|
|
1150
|
-
};
|
|
1151
|
-
if (!merged.created) {
|
|
1152
|
-
merged.created = (/* @__PURE__ */ new Date()).toISOString();
|
|
1153
|
-
}
|
|
1154
|
-
merged.updated = (/* @__PURE__ */ new Date()).toISOString();
|
|
1155
|
-
return serializeFrontmatter(merged) + body;
|
|
1156
|
-
}
|
|
1157
|
-
function formatPath2(filePath) {
|
|
1158
|
-
const home = os3.homedir();
|
|
1159
|
-
if (filePath.startsWith(home)) {
|
|
1160
|
-
return "~" + filePath.slice(home.length);
|
|
1161
|
-
}
|
|
1162
|
-
return filePath;
|
|
1163
|
-
}
|
|
1164
|
-
|
|
1165
|
-
// src/search/graph.ts
|
|
1166
|
-
function getLinksFrom(db, fromId, opts) {
|
|
1167
|
-
let sql = `SELECT from_id, to_id, relation, layer, weight, source_path FROM knowledge_links WHERE from_id = ?`;
|
|
1168
|
-
const params = [fromId];
|
|
1169
|
-
if (opts?.relation) {
|
|
1170
|
-
sql += ` AND relation = ?`;
|
|
1171
|
-
params.push(opts.relation);
|
|
1172
|
-
}
|
|
1173
|
-
if (opts?.layer) {
|
|
1174
|
-
sql += ` AND layer = ?`;
|
|
1175
|
-
params.push(opts.layer);
|
|
1176
|
-
}
|
|
1177
|
-
const rows = db.prepare(sql).all(...params);
|
|
1178
|
-
return rows.map(toGraphLink);
|
|
1179
|
-
}
|
|
1180
|
-
function getLinksTo(db, toId, opts) {
|
|
1181
|
-
let sql = `SELECT from_id, to_id, relation, layer, weight, source_path FROM knowledge_links WHERE to_id = ?`;
|
|
1182
|
-
const params = [toId];
|
|
1183
|
-
if (opts?.relation) {
|
|
1184
|
-
sql += ` AND relation = ?`;
|
|
1185
|
-
params.push(opts.relation);
|
|
1186
|
-
}
|
|
1187
|
-
if (opts?.layer) {
|
|
1188
|
-
sql += ` AND layer = ?`;
|
|
1189
|
-
params.push(opts.layer);
|
|
1190
|
-
}
|
|
1191
|
-
const rows = db.prepare(sql).all(...params);
|
|
1192
|
-
return rows.map(toGraphLink);
|
|
1193
|
-
}
|
|
1194
|
-
function getNeighbors(db, startId, depth = 1, opts) {
|
|
1195
|
-
const visited = /* @__PURE__ */ new Set([startId]);
|
|
1196
|
-
const result = [];
|
|
1197
|
-
let frontier = [startId];
|
|
1198
|
-
for (let d = 1; d <= depth; d++) {
|
|
1199
|
-
const nextFrontier = [];
|
|
1200
|
-
for (const nodeId of frontier) {
|
|
1201
|
-
const outgoing = getLinksFrom(db, nodeId, opts);
|
|
1202
|
-
for (const link of outgoing) {
|
|
1203
|
-
if (!visited.has(link.toId)) {
|
|
1204
|
-
visited.add(link.toId);
|
|
1205
|
-
nextFrontier.push(link.toId);
|
|
1206
|
-
result.push({ id: link.toId, depth: d, link });
|
|
1207
|
-
}
|
|
1208
|
-
}
|
|
1209
|
-
const incoming = getLinksTo(db, nodeId, opts);
|
|
1210
|
-
for (const link of incoming) {
|
|
1211
|
-
if (!visited.has(link.fromId)) {
|
|
1212
|
-
visited.add(link.fromId);
|
|
1213
|
-
nextFrontier.push(link.fromId);
|
|
1214
|
-
result.push({ id: link.fromId, depth: d, link });
|
|
1215
|
-
}
|
|
1216
|
-
}
|
|
1217
|
-
}
|
|
1218
|
-
frontier = nextFrontier;
|
|
1219
|
-
if (frontier.length === 0) break;
|
|
1220
|
-
}
|
|
1221
|
-
return result;
|
|
1222
|
-
}
|
|
1223
|
-
function getPathBetween(db, fromId, toId, maxDepth = 3) {
|
|
1224
|
-
if (fromId === toId) return [];
|
|
1225
|
-
const visited = /* @__PURE__ */ new Set([fromId]);
|
|
1226
|
-
const parentLink = /* @__PURE__ */ new Map();
|
|
1227
|
-
let frontier = [fromId];
|
|
1228
|
-
for (let d = 0; d < maxDepth; d++) {
|
|
1229
|
-
const nextFrontier = [];
|
|
1230
|
-
for (const nodeId of frontier) {
|
|
1231
|
-
const outgoing = getLinksFrom(db, nodeId);
|
|
1232
|
-
for (const link of outgoing) {
|
|
1233
|
-
if (!visited.has(link.toId)) {
|
|
1234
|
-
visited.add(link.toId);
|
|
1235
|
-
parentLink.set(link.toId, link);
|
|
1236
|
-
if (link.toId === toId) {
|
|
1237
|
-
return reconstructPath(parentLink, fromId, toId);
|
|
1238
|
-
}
|
|
1239
|
-
nextFrontier.push(link.toId);
|
|
1240
|
-
}
|
|
1241
|
-
}
|
|
1242
|
-
const incoming = getLinksTo(db, nodeId);
|
|
1243
|
-
for (const link of incoming) {
|
|
1244
|
-
if (!visited.has(link.fromId)) {
|
|
1245
|
-
visited.add(link.fromId);
|
|
1246
|
-
parentLink.set(link.fromId, link);
|
|
1247
|
-
if (link.fromId === toId) {
|
|
1248
|
-
return reconstructPath(parentLink, fromId, toId);
|
|
1249
|
-
}
|
|
1250
|
-
nextFrontier.push(link.fromId);
|
|
1251
|
-
}
|
|
1252
|
-
}
|
|
1253
|
-
}
|
|
1254
|
-
frontier = nextFrontier;
|
|
1255
|
-
if (frontier.length === 0) break;
|
|
1256
|
-
}
|
|
1257
|
-
return [];
|
|
1258
|
-
}
|
|
1259
|
-
function reconstructPath(parentLink, fromId, toId) {
|
|
1260
|
-
const path20 = [];
|
|
1261
|
-
let current = toId;
|
|
1262
|
-
while (current !== fromId) {
|
|
1263
|
-
const link = parentLink.get(current);
|
|
1264
|
-
if (!link) break;
|
|
1265
|
-
path20.unshift(link);
|
|
1266
|
-
current = link.toId === current ? link.fromId : link.toId;
|
|
1267
|
-
}
|
|
1268
|
-
return path20;
|
|
1269
|
-
}
|
|
1270
|
-
function toGraphLink(row) {
|
|
1271
|
-
return {
|
|
1272
|
-
fromId: row.from_id,
|
|
1273
|
-
toId: row.to_id,
|
|
1274
|
-
relation: row.relation,
|
|
1275
|
-
layer: row.layer,
|
|
1276
|
-
weight: row.weight,
|
|
1277
|
-
sourcePath: row.source_path
|
|
1278
|
-
};
|
|
1279
|
-
}
|
|
1280
|
-
|
|
1281
|
-
// src/db/sqlite-vec.ts
|
|
1282
|
-
async function loadSqliteVecExtension(params) {
|
|
1283
|
-
try {
|
|
1284
|
-
const sqliteVec = await import("sqlite-vec");
|
|
1285
|
-
const resolvedPath = params.extensionPath?.trim() ? params.extensionPath.trim() : void 0;
|
|
1286
|
-
const extensionPath = resolvedPath ?? sqliteVec.getLoadablePath();
|
|
1287
|
-
params.db.enableLoadExtension(true);
|
|
1288
|
-
if (resolvedPath) {
|
|
1289
|
-
params.db.loadExtension(extensionPath);
|
|
1290
|
-
} else {
|
|
1291
|
-
sqliteVec.load(params.db);
|
|
1292
|
-
}
|
|
1293
|
-
return { ok: true, extensionPath };
|
|
1294
|
-
} catch (err) {
|
|
1295
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
1296
|
-
return { ok: false, error: message };
|
|
1297
|
-
}
|
|
1298
|
-
}
|
|
1299
|
-
|
|
1300
|
-
// src/embeddings/embeddings.ts
|
|
1301
|
-
import fsSync2 from "fs";
|
|
1302
|
-
import path5 from "path";
|
|
1303
|
-
import os4 from "os";
|
|
1304
|
-
var DEFAULT_LOCAL_MODEL = "hf:ggml-org/embeddinggemma-300M-GGUF/embeddinggemma-300M-Q8_0.gguf";
|
|
1305
|
-
var DEFAULT_OPENAI_EMBEDDING_MODEL = "text-embedding-3-small";
|
|
1306
|
-
var DEFAULT_OPENAI_BASE_URL = "https://api.openai.com/v1";
|
|
1307
|
-
var DEFAULT_GEMINI_EMBEDDING_MODEL = "gemini-embedding-001";
|
|
1308
|
-
var DEFAULT_GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v1beta";
|
|
1309
|
-
function createNoOpEmbeddingProvider() {
|
|
1310
|
-
return {
|
|
1311
|
-
id: "none",
|
|
1312
|
-
model: "bm25-only",
|
|
1313
|
-
embedQuery: async () => [],
|
|
1314
|
-
embedBatch: async (texts) => texts.map(() => [])
|
|
1315
|
-
};
|
|
1316
|
-
}
|
|
1317
|
-
function resolveUserPath(filePath) {
|
|
1318
|
-
if (filePath.startsWith("~/")) {
|
|
1319
|
-
return path5.join(os4.homedir(), filePath.slice(2));
|
|
1320
|
-
}
|
|
1321
|
-
return filePath;
|
|
1322
|
-
}
|
|
1323
|
-
function canAutoSelectLocal(options) {
|
|
1324
|
-
const modelPath = options.local?.modelPath?.trim();
|
|
1325
|
-
if (!modelPath) return false;
|
|
1326
|
-
if (/^(hf:|https?:)/i.test(modelPath)) return false;
|
|
1327
|
-
const resolved = resolveUserPath(modelPath);
|
|
1328
|
-
try {
|
|
1329
|
-
return fsSync2.statSync(resolved).isFile();
|
|
1330
|
-
} catch {
|
|
1331
|
-
return false;
|
|
1332
|
-
}
|
|
1333
|
-
}
|
|
1334
|
-
function isMissingApiKeyError(err) {
|
|
1335
|
-
const message = formatError(err);
|
|
1336
|
-
return message.includes("API key") || message.includes("apiKey");
|
|
1337
|
-
}
|
|
1338
|
-
async function importNodeLlamaCpp() {
|
|
1339
|
-
const llama = await import("node-llama-cpp");
|
|
1340
|
-
return llama;
|
|
1341
|
-
}
|
|
1342
|
-
async function createLocalEmbeddingProvider(options) {
|
|
1343
|
-
const modelPath = options.local?.modelPath?.trim() || DEFAULT_LOCAL_MODEL;
|
|
1344
|
-
const modelCacheDir = options.local?.modelCacheDir?.trim();
|
|
1345
|
-
const { getLlama, resolveModelFile, LlamaLogLevel } = await importNodeLlamaCpp();
|
|
1346
|
-
let llama = null;
|
|
1347
|
-
let embeddingModel = null;
|
|
1348
|
-
let embeddingContext = null;
|
|
1349
|
-
const ensureContext = async () => {
|
|
1350
|
-
if (!llama) {
|
|
1351
|
-
llama = await getLlama({ logLevel: LlamaLogLevel.error });
|
|
1352
|
-
}
|
|
1353
|
-
if (!embeddingModel) {
|
|
1354
|
-
const resolved = await resolveModelFile(modelPath, modelCacheDir || void 0);
|
|
1355
|
-
embeddingModel = await llama.loadModel({ modelPath: resolved });
|
|
1356
|
-
}
|
|
1357
|
-
if (!embeddingContext) {
|
|
1358
|
-
embeddingContext = await embeddingModel.createEmbeddingContext();
|
|
1359
|
-
}
|
|
1360
|
-
return embeddingContext;
|
|
1361
|
-
};
|
|
1362
|
-
return {
|
|
1363
|
-
id: "local",
|
|
1364
|
-
model: modelPath,
|
|
1365
|
-
embedQuery: async (text) => {
|
|
1366
|
-
const ctx = await ensureContext();
|
|
1367
|
-
const embedding = await ctx.getEmbeddingFor(text);
|
|
1368
|
-
return Array.from(embedding.vector);
|
|
1369
|
-
},
|
|
1370
|
-
embedBatch: async (texts) => {
|
|
1371
|
-
const ctx = await ensureContext();
|
|
1372
|
-
const embeddings = await Promise.all(
|
|
1373
|
-
texts.map(async (text) => {
|
|
1374
|
-
const embedding = await ctx.getEmbeddingFor(text);
|
|
1375
|
-
return Array.from(embedding.vector);
|
|
1376
|
-
})
|
|
1377
|
-
);
|
|
1378
|
-
return embeddings;
|
|
1379
|
-
}
|
|
1380
|
-
};
|
|
1381
|
-
}
|
|
1382
|
-
function normalizeOpenAiModel(model) {
|
|
1383
|
-
const trimmed = model.trim();
|
|
1384
|
-
if (!trimmed) return DEFAULT_OPENAI_EMBEDDING_MODEL;
|
|
1385
|
-
if (trimmed.startsWith("openai/")) return trimmed.slice("openai/".length);
|
|
1386
|
-
return trimmed;
|
|
1387
|
-
}
|
|
1388
|
-
function resolveOpenAiApiKey(options) {
|
|
1389
|
-
const apiKey = options.openai?.apiKey?.trim();
|
|
1390
|
-
if (apiKey) return apiKey;
|
|
1391
|
-
const envKey = process.env.OPENAI_API_KEY?.trim();
|
|
1392
|
-
if (envKey) return envKey;
|
|
1393
|
-
throw new Error("OpenAI API key not found. Set OPENAI_API_KEY env var or pass openai.apiKey option.");
|
|
1394
|
-
}
|
|
1395
|
-
async function createOpenAiEmbeddingProvider(options) {
|
|
1396
|
-
const apiKey = resolveOpenAiApiKey(options);
|
|
1397
|
-
const baseUrl = options.openai?.baseUrl?.trim() || DEFAULT_OPENAI_BASE_URL;
|
|
1398
|
-
const headerOverrides = options.openai?.headers ?? {};
|
|
1399
|
-
const headers = {
|
|
1400
|
-
"Content-Type": "application/json",
|
|
1401
|
-
Authorization: `Bearer ${apiKey}`,
|
|
1402
|
-
...headerOverrides
|
|
1403
|
-
};
|
|
1404
|
-
const model = normalizeOpenAiModel(options.model || "");
|
|
1405
|
-
const client = { baseUrl, headers, model };
|
|
1406
|
-
const url = `${baseUrl.replace(/\/$/, "")}/embeddings`;
|
|
1407
|
-
const embed = async (input) => {
|
|
1408
|
-
if (input.length === 0) return [];
|
|
1409
|
-
const res = await fetch(url, {
|
|
1410
|
-
method: "POST",
|
|
1411
|
-
headers: client.headers,
|
|
1412
|
-
body: JSON.stringify({ model: client.model, input })
|
|
1413
|
-
});
|
|
1414
|
-
if (!res.ok) {
|
|
1415
|
-
const text = await res.text();
|
|
1416
|
-
throw new Error(`openai embeddings failed: ${res.status} ${text}`);
|
|
1417
|
-
}
|
|
1418
|
-
const payload = await res.json();
|
|
1419
|
-
const data = payload.data ?? [];
|
|
1420
|
-
return data.map((entry) => entry.embedding ?? []);
|
|
1421
|
-
};
|
|
1422
|
-
return {
|
|
1423
|
-
provider: {
|
|
1424
|
-
id: "openai",
|
|
1425
|
-
model: client.model,
|
|
1426
|
-
embedQuery: async (text) => {
|
|
1427
|
-
const [vec] = await embed([text]);
|
|
1428
|
-
return vec ?? [];
|
|
1429
|
-
},
|
|
1430
|
-
embedBatch: embed
|
|
1431
|
-
},
|
|
1432
|
-
client
|
|
1433
|
-
};
|
|
1434
|
-
}
|
|
1435
|
-
function normalizeGeminiModel(model) {
|
|
1436
|
-
const trimmed = model.trim();
|
|
1437
|
-
if (!trimmed) return DEFAULT_GEMINI_EMBEDDING_MODEL;
|
|
1438
|
-
const withoutPrefix = trimmed.replace(/^models\//, "");
|
|
1439
|
-
if (withoutPrefix.startsWith("gemini/")) return withoutPrefix.slice("gemini/".length);
|
|
1440
|
-
if (withoutPrefix.startsWith("google/")) return withoutPrefix.slice("google/".length);
|
|
1441
|
-
return withoutPrefix;
|
|
1442
|
-
}
|
|
1443
|
-
function normalizeGeminiBaseUrl(raw) {
|
|
1444
|
-
const trimmed = raw.replace(/\/+$/, "");
|
|
1445
|
-
const openAiIndex = trimmed.indexOf("/openai");
|
|
1446
|
-
if (openAiIndex > -1) return trimmed.slice(0, openAiIndex);
|
|
1447
|
-
return trimmed;
|
|
1448
|
-
}
|
|
1449
|
-
function buildGeminiModelPath(model) {
|
|
1450
|
-
return model.startsWith("models/") ? model : `models/${model}`;
|
|
1451
|
-
}
|
|
1452
|
-
function resolveGeminiApiKey(options) {
|
|
1453
|
-
const apiKey = options.gemini?.apiKey?.trim();
|
|
1454
|
-
if (apiKey) return apiKey;
|
|
1455
|
-
const googleKey = process.env.GOOGLE_API_KEY?.trim();
|
|
1456
|
-
if (googleKey) return googleKey;
|
|
1457
|
-
const geminiKey = process.env.GEMINI_API_KEY?.trim();
|
|
1458
|
-
if (geminiKey) return geminiKey;
|
|
1459
|
-
throw new Error("Gemini API key not found. Set GOOGLE_API_KEY or GEMINI_API_KEY env var or pass gemini.apiKey option.");
|
|
1460
|
-
}
|
|
1461
|
-
async function createGeminiEmbeddingProvider(options) {
|
|
1462
|
-
const apiKey = resolveGeminiApiKey(options);
|
|
1463
|
-
const rawBaseUrl = options.gemini?.baseUrl?.trim() || DEFAULT_GEMINI_BASE_URL;
|
|
1464
|
-
const baseUrl = normalizeGeminiBaseUrl(rawBaseUrl);
|
|
1465
|
-
const headerOverrides = options.gemini?.headers ?? {};
|
|
1466
|
-
const headers = {
|
|
1467
|
-
"Content-Type": "application/json",
|
|
1468
|
-
"x-goog-api-key": apiKey,
|
|
1469
|
-
...headerOverrides
|
|
1470
|
-
};
|
|
1471
|
-
const model = normalizeGeminiModel(options.model || "");
|
|
1472
|
-
const modelPath = buildGeminiModelPath(model);
|
|
1473
|
-
const client = { baseUrl, headers, model, modelPath };
|
|
1474
|
-
const embedUrl = `${baseUrl}/${modelPath}:embedContent`;
|
|
1475
|
-
const batchUrl = `${baseUrl}/${modelPath}:batchEmbedContents`;
|
|
1476
|
-
const embedQuery = async (text) => {
|
|
1477
|
-
if (!text.trim()) return [];
|
|
1478
|
-
const res = await fetch(embedUrl, {
|
|
1479
|
-
method: "POST",
|
|
1480
|
-
headers: client.headers,
|
|
1481
|
-
body: JSON.stringify({
|
|
1482
|
-
content: { parts: [{ text }] },
|
|
1483
|
-
taskType: "RETRIEVAL_QUERY"
|
|
1484
|
-
})
|
|
1485
|
-
});
|
|
1486
|
-
if (!res.ok) {
|
|
1487
|
-
const payload2 = await res.text();
|
|
1488
|
-
throw new Error(`gemini embeddings failed: ${res.status} ${payload2}`);
|
|
1489
|
-
}
|
|
1490
|
-
const payload = await res.json();
|
|
1491
|
-
return payload.embedding?.values ?? [];
|
|
1492
|
-
};
|
|
1493
|
-
const embedBatch = async (texts) => {
|
|
1494
|
-
if (texts.length === 0) return [];
|
|
1495
|
-
const requests = texts.map((text) => ({
|
|
1496
|
-
model: modelPath,
|
|
1497
|
-
content: { parts: [{ text }] },
|
|
1498
|
-
taskType: "RETRIEVAL_DOCUMENT"
|
|
1499
|
-
}));
|
|
1500
|
-
const res = await fetch(batchUrl, {
|
|
1501
|
-
method: "POST",
|
|
1502
|
-
headers: client.headers,
|
|
1503
|
-
body: JSON.stringify({ requests })
|
|
1504
|
-
});
|
|
1505
|
-
if (!res.ok) {
|
|
1506
|
-
const payload2 = await res.text();
|
|
1507
|
-
throw new Error(`gemini embeddings failed: ${res.status} ${payload2}`);
|
|
1508
|
-
}
|
|
1509
|
-
const payload = await res.json();
|
|
1510
|
-
const embeddings = Array.isArray(payload.embeddings) ? payload.embeddings : [];
|
|
1511
|
-
return texts.map((_, index) => embeddings[index]?.values ?? []);
|
|
1512
|
-
};
|
|
1513
|
-
return {
|
|
1514
|
-
provider: {
|
|
1515
|
-
id: "gemini",
|
|
1516
|
-
model: client.model,
|
|
1517
|
-
embedQuery,
|
|
1518
|
-
embedBatch
|
|
1519
|
-
},
|
|
1520
|
-
client
|
|
1521
|
-
};
|
|
1522
|
-
}
|
|
1523
|
-
async function createEmbeddingProvider(options) {
|
|
1524
|
-
const requestedProvider = options.provider;
|
|
1525
|
-
const fallback = options.fallback ?? "none";
|
|
1526
|
-
if (requestedProvider === "none") {
|
|
1527
|
-
return {
|
|
1528
|
-
provider: createNoOpEmbeddingProvider(),
|
|
1529
|
-
requestedProvider: "none"
|
|
1530
|
-
};
|
|
1531
|
-
}
|
|
1532
|
-
const createProvider = async (id) => {
|
|
1533
|
-
if (id === "local") {
|
|
1534
|
-
const provider2 = await createLocalEmbeddingProvider(options);
|
|
1535
|
-
return { provider: provider2 };
|
|
1536
|
-
}
|
|
1537
|
-
if (id === "gemini") {
|
|
1538
|
-
const { provider: provider2, client: client2 } = await createGeminiEmbeddingProvider(options);
|
|
1539
|
-
return { provider: provider2, gemini: client2 };
|
|
1540
|
-
}
|
|
1541
|
-
const { provider, client } = await createOpenAiEmbeddingProvider(options);
|
|
1542
|
-
return { provider, openAi: client };
|
|
1543
|
-
};
|
|
1544
|
-
const formatPrimaryError = (err, provider) => provider === "local" ? formatLocalSetupError(err) : formatError(err);
|
|
1545
|
-
if (requestedProvider === "auto") {
|
|
1546
|
-
const missingKeyErrors = [];
|
|
1547
|
-
let localError = null;
|
|
1548
|
-
if (canAutoSelectLocal(options)) {
|
|
1549
|
-
try {
|
|
1550
|
-
const local = await createProvider("local");
|
|
1551
|
-
return { ...local, requestedProvider };
|
|
1552
|
-
} catch (err) {
|
|
1553
|
-
localError = formatLocalSetupError(err);
|
|
1554
|
-
}
|
|
1555
|
-
}
|
|
1556
|
-
for (const provider of ["openai", "gemini"]) {
|
|
1557
|
-
try {
|
|
1558
|
-
const result = await createProvider(provider);
|
|
1559
|
-
return { ...result, requestedProvider };
|
|
1560
|
-
} catch (err) {
|
|
1561
|
-
const message = formatPrimaryError(err, provider);
|
|
1562
|
-
if (isMissingApiKeyError(err)) {
|
|
1563
|
-
missingKeyErrors.push(message);
|
|
1564
|
-
continue;
|
|
1565
|
-
}
|
|
1566
|
-
throw new Error(message);
|
|
1567
|
-
}
|
|
1568
|
-
}
|
|
1569
|
-
return {
|
|
1570
|
-
provider: createNoOpEmbeddingProvider(),
|
|
1571
|
-
requestedProvider,
|
|
1572
|
-
fallbackFrom: "auto",
|
|
1573
|
-
fallbackReason: "No embedding API available. Using BM25 full-text search only."
|
|
1574
|
-
};
|
|
1575
|
-
}
|
|
1576
|
-
try {
|
|
1577
|
-
const primary = await createProvider(requestedProvider);
|
|
1578
|
-
return { ...primary, requestedProvider };
|
|
1579
|
-
} catch (primaryErr) {
|
|
1580
|
-
const reason = formatPrimaryError(primaryErr, requestedProvider);
|
|
1581
|
-
if (fallback && fallback !== "none" && fallback !== requestedProvider) {
|
|
1582
|
-
try {
|
|
1583
|
-
const fallbackResult = await createProvider(fallback);
|
|
1584
|
-
return {
|
|
1585
|
-
...fallbackResult,
|
|
1586
|
-
requestedProvider,
|
|
1587
|
-
fallbackFrom: requestedProvider,
|
|
1588
|
-
fallbackReason: reason
|
|
1589
|
-
};
|
|
1590
|
-
} catch (fallbackErr) {
|
|
1591
|
-
throw new Error(`${reason}
|
|
1592
|
-
|
|
1593
|
-
Fallback to ${fallback} failed: ${formatError(fallbackErr)}`);
|
|
1594
|
-
}
|
|
1595
|
-
}
|
|
1596
|
-
throw new Error(reason);
|
|
1597
|
-
}
|
|
1598
|
-
}
|
|
1599
|
-
function formatError(err) {
|
|
1600
|
-
if (err instanceof Error) return err.message;
|
|
1601
|
-
return String(err);
|
|
1602
|
-
}
|
|
1603
|
-
function isNodeLlamaCppMissing(err) {
|
|
1604
|
-
if (!(err instanceof Error)) return false;
|
|
1605
|
-
const code = err.code;
|
|
1606
|
-
if (code === "ERR_MODULE_NOT_FOUND") {
|
|
1607
|
-
return err.message.includes("node-llama-cpp");
|
|
1608
|
-
}
|
|
1609
|
-
return false;
|
|
1610
|
-
}
|
|
1611
|
-
function formatLocalSetupError(err) {
|
|
1612
|
-
const detail = formatError(err);
|
|
1613
|
-
const missing = isNodeLlamaCppMissing(err);
|
|
1614
|
-
return [
|
|
1615
|
-
"Local embeddings unavailable.",
|
|
1616
|
-
missing ? "Reason: optional dependency node-llama-cpp is missing (or failed to install)." : detail ? `Reason: ${detail}` : void 0,
|
|
1617
|
-
missing && detail ? `Detail: ${detail}` : null,
|
|
1618
|
-
"To enable local embeddings:",
|
|
1619
|
-
"1) Use Node 22 LTS (recommended for installs/updates)",
|
|
1620
|
-
missing ? "2) Install node-llama-cpp: npm install node-llama-cpp" : null,
|
|
1621
|
-
"3) If you use pnpm: pnpm approve-builds (select node-llama-cpp), then pnpm rebuild node-llama-cpp",
|
|
1622
|
-
'Or set provider = "openai" or "gemini" (remote).'
|
|
1623
|
-
].filter(Boolean).join("\n");
|
|
1624
|
-
}
|
|
1625
|
-
|
|
1626
|
-
// src/embeddings/batch-openai.ts
|
|
1627
|
-
var OPENAI_BATCH_ENDPOINT = "/v1/embeddings";
|
|
1628
|
-
var OPENAI_BATCH_COMPLETION_WINDOW = "24h";
|
|
1629
|
-
var OPENAI_BATCH_MAX_REQUESTS = 5e4;
|
|
1630
|
-
function getOpenAiBaseUrl(openAi) {
|
|
1631
|
-
return openAi.baseUrl?.replace(/\/$/, "") ?? "";
|
|
1632
|
-
}
|
|
1633
|
-
function getOpenAiHeaders(openAi, params) {
|
|
1634
|
-
const headers = openAi.headers ? { ...openAi.headers } : {};
|
|
1635
|
-
if (params.json) {
|
|
1636
|
-
if (!headers["Content-Type"] && !headers["content-type"]) {
|
|
1637
|
-
headers["Content-Type"] = "application/json";
|
|
1638
|
-
}
|
|
1639
|
-
} else {
|
|
1640
|
-
delete headers["Content-Type"];
|
|
1641
|
-
delete headers["content-type"];
|
|
1642
|
-
}
|
|
1643
|
-
return headers;
|
|
1644
|
-
}
|
|
1645
|
-
function splitOpenAiBatchRequests(requests) {
|
|
1646
|
-
if (requests.length <= OPENAI_BATCH_MAX_REQUESTS) return [requests];
|
|
1647
|
-
const groups = [];
|
|
1648
|
-
for (let i = 0; i < requests.length; i += OPENAI_BATCH_MAX_REQUESTS) {
|
|
1649
|
-
groups.push(requests.slice(i, i + OPENAI_BATCH_MAX_REQUESTS));
|
|
1650
|
-
}
|
|
1651
|
-
return groups;
|
|
1652
|
-
}
|
|
1653
|
-
async function retryAsync(fn, opts) {
|
|
1654
|
-
let lastError;
|
|
1655
|
-
for (let attempt = 0; attempt < opts.attempts; attempt++) {
|
|
1656
|
-
try {
|
|
1657
|
-
return await fn();
|
|
1658
|
-
} catch (err) {
|
|
1659
|
-
lastError = err;
|
|
1660
|
-
if (!opts.shouldRetry(err) || attempt === opts.attempts - 1) {
|
|
1661
|
-
throw err;
|
|
1662
|
-
}
|
|
1663
|
-
const delay = Math.min(
|
|
1664
|
-
opts.maxDelayMs,
|
|
1665
|
-
opts.minDelayMs * Math.pow(2, attempt) * (1 + Math.random() * opts.jitter)
|
|
1666
|
-
);
|
|
1667
|
-
await new Promise((resolve3) => setTimeout(resolve3, delay));
|
|
1668
|
-
}
|
|
1669
|
-
}
|
|
1670
|
-
throw lastError;
|
|
1671
|
-
}
|
|
1672
|
-
async function submitOpenAiBatch(params) {
|
|
1673
|
-
const baseUrl = getOpenAiBaseUrl(params.openAi);
|
|
1674
|
-
const jsonl = params.requests.map((request) => JSON.stringify(request)).join("\n");
|
|
1675
|
-
const form = new FormData();
|
|
1676
|
-
form.append("purpose", "batch");
|
|
1677
|
-
form.append(
|
|
1678
|
-
"file",
|
|
1679
|
-
new Blob([jsonl], { type: "application/jsonl" }),
|
|
1680
|
-
`memory-embeddings.${hashText(String(Date.now()))}.jsonl`
|
|
1681
|
-
);
|
|
1682
|
-
const fileRes = await fetch(`${baseUrl}/files`, {
|
|
1683
|
-
method: "POST",
|
|
1684
|
-
headers: getOpenAiHeaders(params.openAi, { json: false }),
|
|
1685
|
-
body: form
|
|
1686
|
-
});
|
|
1687
|
-
if (!fileRes.ok) {
|
|
1688
|
-
const text = await fileRes.text();
|
|
1689
|
-
throw new Error(`openai batch file upload failed: ${fileRes.status} ${text}`);
|
|
1690
|
-
}
|
|
1691
|
-
const filePayload = await fileRes.json();
|
|
1692
|
-
if (!filePayload.id) {
|
|
1693
|
-
throw new Error("openai batch file upload failed: missing file id");
|
|
1694
|
-
}
|
|
1695
|
-
const batchRes = await retryAsync(
|
|
1696
|
-
async () => {
|
|
1697
|
-
const res = await fetch(`${baseUrl}/batches`, {
|
|
1698
|
-
method: "POST",
|
|
1699
|
-
headers: getOpenAiHeaders(params.openAi, { json: true }),
|
|
1700
|
-
body: JSON.stringify({
|
|
1701
|
-
input_file_id: filePayload.id,
|
|
1702
|
-
endpoint: OPENAI_BATCH_ENDPOINT,
|
|
1703
|
-
completion_window: OPENAI_BATCH_COMPLETION_WINDOW,
|
|
1704
|
-
metadata: {
|
|
1705
|
-
source: params.source
|
|
1706
|
-
}
|
|
1707
|
-
})
|
|
1708
|
-
});
|
|
1709
|
-
if (!res.ok) {
|
|
1710
|
-
const text = await res.text();
|
|
1711
|
-
const err = new Error(`openai batch create failed: ${res.status} ${text}`);
|
|
1712
|
-
err.status = res.status;
|
|
1713
|
-
throw err;
|
|
1714
|
-
}
|
|
1715
|
-
return res;
|
|
1716
|
-
},
|
|
1717
|
-
{
|
|
1718
|
-
attempts: 3,
|
|
1719
|
-
minDelayMs: 300,
|
|
1720
|
-
maxDelayMs: 2e3,
|
|
1721
|
-
jitter: 0.2,
|
|
1722
|
-
shouldRetry: (err) => {
|
|
1723
|
-
const status2 = err.status;
|
|
1724
|
-
return status2 === 429 || typeof status2 === "number" && status2 >= 500;
|
|
1725
|
-
}
|
|
1726
|
-
}
|
|
1727
|
-
);
|
|
1728
|
-
return await batchRes.json();
|
|
1729
|
-
}
|
|
1730
|
-
async function fetchOpenAiBatchStatus(params) {
|
|
1731
|
-
const baseUrl = getOpenAiBaseUrl(params.openAi);
|
|
1732
|
-
const res = await fetch(`${baseUrl}/batches/${params.batchId}`, {
|
|
1733
|
-
headers: getOpenAiHeaders(params.openAi, { json: true })
|
|
1734
|
-
});
|
|
1735
|
-
if (!res.ok) {
|
|
1736
|
-
const text = await res.text();
|
|
1737
|
-
throw new Error(`openai batch status failed: ${res.status} ${text}`);
|
|
1738
|
-
}
|
|
1739
|
-
return await res.json();
|
|
1740
|
-
}
|
|
1741
|
-
async function fetchOpenAiFileContent(params) {
|
|
1742
|
-
const baseUrl = getOpenAiBaseUrl(params.openAi);
|
|
1743
|
-
const res = await fetch(`${baseUrl}/files/${params.fileId}/content`, {
|
|
1744
|
-
headers: getOpenAiHeaders(params.openAi, { json: true })
|
|
1745
|
-
});
|
|
1746
|
-
if (!res.ok) {
|
|
1747
|
-
const text = await res.text();
|
|
1748
|
-
throw new Error(`openai batch file content failed: ${res.status} ${text}`);
|
|
1749
|
-
}
|
|
1750
|
-
return await res.text();
|
|
1751
|
-
}
|
|
1752
|
-
function parseOpenAiBatchOutput(text) {
|
|
1753
|
-
if (!text.trim()) return [];
|
|
1754
|
-
return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
|
|
1755
|
-
}
|
|
1756
|
-
async function readOpenAiBatchError(params) {
|
|
1757
|
-
try {
|
|
1758
|
-
const content = await fetchOpenAiFileContent({
|
|
1759
|
-
openAi: params.openAi,
|
|
1760
|
-
fileId: params.errorFileId
|
|
1761
|
-
});
|
|
1762
|
-
const lines = parseOpenAiBatchOutput(content);
|
|
1763
|
-
const first = lines.find((line) => line.error?.message || line.response?.body?.error);
|
|
1764
|
-
const message = first?.error?.message ?? (typeof first?.response?.body?.error?.message === "string" ? first?.response?.body?.error?.message : void 0);
|
|
1765
|
-
return message;
|
|
1766
|
-
} catch (err) {
|
|
1767
|
-
const message = err instanceof Error ? err.message : String(err);
|
|
1768
|
-
return message ? `error file unavailable: ${message}` : void 0;
|
|
1769
|
-
}
|
|
1770
|
-
}
|
|
1771
|
-
async function waitForOpenAiBatch(params) {
|
|
1772
|
-
const start = Date.now();
|
|
1773
|
-
let current = params.initial;
|
|
1774
|
-
while (true) {
|
|
1775
|
-
const status2 = current ?? await fetchOpenAiBatchStatus({
|
|
1776
|
-
openAi: params.openAi,
|
|
1777
|
-
batchId: params.batchId
|
|
1778
|
-
});
|
|
1779
|
-
const state = status2.status ?? "unknown";
|
|
1780
|
-
if (state === "completed") {
|
|
1781
|
-
if (!status2.output_file_id) {
|
|
1782
|
-
throw new Error(`openai batch ${params.batchId} completed without output file`);
|
|
1783
|
-
}
|
|
1784
|
-
return {
|
|
1785
|
-
outputFileId: status2.output_file_id,
|
|
1786
|
-
errorFileId: status2.error_file_id ?? void 0
|
|
1787
|
-
};
|
|
1788
|
-
}
|
|
1789
|
-
if (["failed", "expired", "cancelled", "canceled"].includes(state)) {
|
|
1790
|
-
const detail = status2.error_file_id ? await readOpenAiBatchError({ openAi: params.openAi, errorFileId: status2.error_file_id }) : void 0;
|
|
1791
|
-
const suffix = detail ? `: ${detail}` : "";
|
|
1792
|
-
throw new Error(`openai batch ${params.batchId} ${state}${suffix}`);
|
|
1793
|
-
}
|
|
1794
|
-
if (!params.wait) {
|
|
1795
|
-
throw new Error(`openai batch ${params.batchId} still ${state}; wait disabled`);
|
|
1796
|
-
}
|
|
1797
|
-
if (Date.now() - start > params.timeoutMs) {
|
|
1798
|
-
throw new Error(`openai batch ${params.batchId} timed out after ${params.timeoutMs}ms`);
|
|
1799
|
-
}
|
|
1800
|
-
params.debug?.(`openai batch ${params.batchId} ${state}; waiting ${params.pollIntervalMs}ms`);
|
|
1801
|
-
await new Promise((resolve3) => setTimeout(resolve3, params.pollIntervalMs));
|
|
1802
|
-
current = void 0;
|
|
1803
|
-
}
|
|
1804
|
-
}
|
|
1805
|
-
async function runWithConcurrency(tasks, limit) {
|
|
1806
|
-
if (tasks.length === 0) return [];
|
|
1807
|
-
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
|
|
1808
|
-
const results = Array.from({ length: tasks.length });
|
|
1809
|
-
let next = 0;
|
|
1810
|
-
let firstError = null;
|
|
1811
|
-
const workers = Array.from({ length: resolvedLimit }, async () => {
|
|
1812
|
-
while (true) {
|
|
1813
|
-
if (firstError) return;
|
|
1814
|
-
const index = next;
|
|
1815
|
-
next += 1;
|
|
1816
|
-
if (index >= tasks.length) return;
|
|
1817
|
-
try {
|
|
1818
|
-
results[index] = await tasks[index]();
|
|
1819
|
-
} catch (err) {
|
|
1820
|
-
firstError = err;
|
|
1821
|
-
return;
|
|
1822
|
-
}
|
|
1823
|
-
}
|
|
1824
|
-
});
|
|
1825
|
-
await Promise.allSettled(workers);
|
|
1826
|
-
if (firstError) throw firstError;
|
|
1827
|
-
return results;
|
|
1828
|
-
}
|
|
1829
|
-
async function runOpenAiEmbeddingBatches(params) {
|
|
1830
|
-
if (params.requests.length === 0) return /* @__PURE__ */ new Map();
|
|
1831
|
-
const groups = splitOpenAiBatchRequests(params.requests);
|
|
1832
|
-
const byCustomId = /* @__PURE__ */ new Map();
|
|
1833
|
-
const tasks = groups.map((group, groupIndex) => async () => {
|
|
1834
|
-
const batchInfo = await submitOpenAiBatch({
|
|
1835
|
-
openAi: params.openAi,
|
|
1836
|
-
requests: group,
|
|
1837
|
-
source: params.source
|
|
1838
|
-
});
|
|
1839
|
-
if (!batchInfo.id) {
|
|
1840
|
-
throw new Error("openai batch create failed: missing batch id");
|
|
1841
|
-
}
|
|
1842
|
-
params.debug?.("memory embeddings: openai batch created", {
|
|
1843
|
-
batchId: batchInfo.id,
|
|
1844
|
-
status: batchInfo.status,
|
|
1845
|
-
group: groupIndex + 1,
|
|
1846
|
-
groups: groups.length,
|
|
1847
|
-
requests: group.length
|
|
1848
|
-
});
|
|
1849
|
-
if (!params.wait && batchInfo.status !== "completed") {
|
|
1850
|
-
throw new Error(
|
|
1851
|
-
`openai batch ${batchInfo.id} submitted; enable batch.wait to await completion`
|
|
1852
|
-
);
|
|
1853
|
-
}
|
|
1854
|
-
const completed = batchInfo.status === "completed" ? {
|
|
1855
|
-
outputFileId: batchInfo.output_file_id ?? "",
|
|
1856
|
-
errorFileId: batchInfo.error_file_id ?? void 0
|
|
1857
|
-
} : await waitForOpenAiBatch({
|
|
1858
|
-
openAi: params.openAi,
|
|
1859
|
-
batchId: batchInfo.id,
|
|
1860
|
-
wait: params.wait,
|
|
1861
|
-
pollIntervalMs: params.pollIntervalMs,
|
|
1862
|
-
timeoutMs: params.timeoutMs,
|
|
1863
|
-
debug: params.debug,
|
|
1864
|
-
initial: batchInfo
|
|
1865
|
-
});
|
|
1866
|
-
if (!completed.outputFileId) {
|
|
1867
|
-
throw new Error(`openai batch ${batchInfo.id} completed without output file`);
|
|
1868
|
-
}
|
|
1869
|
-
const content = await fetchOpenAiFileContent({
|
|
1870
|
-
openAi: params.openAi,
|
|
1871
|
-
fileId: completed.outputFileId
|
|
1872
|
-
});
|
|
1873
|
-
const outputLines = parseOpenAiBatchOutput(content);
|
|
1874
|
-
const errors = [];
|
|
1875
|
-
const remaining = new Set(group.map((request) => request.custom_id));
|
|
1876
|
-
for (const line of outputLines) {
|
|
1877
|
-
const customId = line.custom_id;
|
|
1878
|
-
if (!customId) continue;
|
|
1879
|
-
remaining.delete(customId);
|
|
1880
|
-
if (line.error?.message) {
|
|
1881
|
-
errors.push(`${customId}: ${line.error.message}`);
|
|
1882
|
-
continue;
|
|
1883
|
-
}
|
|
1884
|
-
const response = line.response;
|
|
1885
|
-
const statusCode = response?.status_code ?? 0;
|
|
1886
|
-
if (statusCode >= 400) {
|
|
1887
|
-
const message = response?.body?.error?.message ?? (typeof response?.body === "string" ? response.body : void 0) ?? "unknown error";
|
|
1888
|
-
errors.push(`${customId}: ${message}`);
|
|
1889
|
-
continue;
|
|
1890
|
-
}
|
|
1891
|
-
const data = response?.body?.data ?? [];
|
|
1892
|
-
const embedding = data[0]?.embedding ?? [];
|
|
1893
|
-
if (embedding.length === 0) {
|
|
1894
|
-
errors.push(`${customId}: empty embedding`);
|
|
1895
|
-
continue;
|
|
1896
|
-
}
|
|
1897
|
-
byCustomId.set(customId, embedding);
|
|
1898
|
-
}
|
|
1899
|
-
if (errors.length > 0) {
|
|
1900
|
-
throw new Error(`openai batch ${batchInfo.id} failed: ${errors.join("; ")}`);
|
|
1901
|
-
}
|
|
1902
|
-
if (remaining.size > 0) {
|
|
1903
|
-
throw new Error(`openai batch ${batchInfo.id} missing ${remaining.size} embedding responses`);
|
|
1904
|
-
}
|
|
1905
|
-
});
|
|
1906
|
-
params.debug?.("memory embeddings: openai batch submit", {
|
|
1907
|
-
requests: params.requests.length,
|
|
1908
|
-
groups: groups.length,
|
|
1909
|
-
wait: params.wait,
|
|
1910
|
-
concurrency: params.concurrency,
|
|
1911
|
-
pollIntervalMs: params.pollIntervalMs,
|
|
1912
|
-
timeoutMs: params.timeoutMs
|
|
1913
|
-
});
|
|
1914
|
-
await runWithConcurrency(tasks, params.concurrency);
|
|
1915
|
-
return byCustomId;
|
|
1916
|
-
}
|
|
1917
|
-
|
|
1918
|
-
// src/embeddings/batch-gemini.ts
|
|
1919
|
-
var GEMINI_BATCH_MAX_REQUESTS = 5e4;
|
|
1920
|
-
function getGeminiBaseUrl(gemini) {
|
|
1921
|
-
return gemini.baseUrl?.replace(/\/$/, "") ?? "";
|
|
1922
|
-
}
|
|
1923
|
-
function getGeminiHeaders(gemini, params) {
|
|
1924
|
-
const headers = gemini.headers ? { ...gemini.headers } : {};
|
|
1925
|
-
if (params.json) {
|
|
1926
|
-
if (!headers["Content-Type"] && !headers["content-type"]) {
|
|
1927
|
-
headers["Content-Type"] = "application/json";
|
|
1928
|
-
}
|
|
1929
|
-
} else {
|
|
1930
|
-
delete headers["Content-Type"];
|
|
1931
|
-
delete headers["content-type"];
|
|
1932
|
-
}
|
|
1933
|
-
return headers;
|
|
1934
|
-
}
|
|
1935
|
-
function getGeminiUploadUrl(baseUrl) {
|
|
1936
|
-
if (baseUrl.includes("/v1beta")) {
|
|
1937
|
-
return baseUrl.replace(/\/v1beta\/?$/, "/upload/v1beta");
|
|
1938
|
-
}
|
|
1939
|
-
return `${baseUrl.replace(/\/$/, "")}/upload`;
|
|
1940
|
-
}
|
|
1941
|
-
function splitGeminiBatchRequests(requests) {
|
|
1942
|
-
if (requests.length <= GEMINI_BATCH_MAX_REQUESTS) return [requests];
|
|
1943
|
-
const groups = [];
|
|
1944
|
-
for (let i = 0; i < requests.length; i += GEMINI_BATCH_MAX_REQUESTS) {
|
|
1945
|
-
groups.push(requests.slice(i, i + GEMINI_BATCH_MAX_REQUESTS));
|
|
1946
|
-
}
|
|
1947
|
-
return groups;
|
|
1948
|
-
}
|
|
1949
|
-
function buildGeminiUploadBody(params) {
|
|
1950
|
-
const boundary = `minimem-${hashText(params.displayName)}`;
|
|
1951
|
-
const jsonPart = JSON.stringify({
|
|
1952
|
-
file: {
|
|
1953
|
-
displayName: params.displayName,
|
|
1954
|
-
mimeType: "application/jsonl"
|
|
1955
|
-
}
|
|
1956
|
-
});
|
|
1957
|
-
const delimiter = `--${boundary}\r
|
|
1958
|
-
`;
|
|
1959
|
-
const closeDelimiter = `--${boundary}--\r
|
|
1960
|
-
`;
|
|
1961
|
-
const parts = [
|
|
1962
|
-
`${delimiter}Content-Type: application/json; charset=UTF-8\r
|
|
1963
|
-
\r
|
|
1964
|
-
${jsonPart}\r
|
|
1965
|
-
`,
|
|
1966
|
-
`${delimiter}Content-Type: application/jsonl; charset=UTF-8\r
|
|
1967
|
-
\r
|
|
1968
|
-
${params.jsonl}\r
|
|
1969
|
-
`,
|
|
1970
|
-
closeDelimiter
|
|
1971
|
-
];
|
|
1972
|
-
const body = new Blob([parts.join("")], { type: "multipart/related" });
|
|
1973
|
-
return {
|
|
1974
|
-
body,
|
|
1975
|
-
contentType: `multipart/related; boundary=${boundary}`
|
|
1976
|
-
};
|
|
1977
|
-
}
|
|
1978
|
-
async function submitGeminiBatch(params) {
|
|
1979
|
-
const baseUrl = getGeminiBaseUrl(params.gemini);
|
|
1980
|
-
const jsonl = params.requests.map(
|
|
1981
|
-
(request) => JSON.stringify({
|
|
1982
|
-
key: request.custom_id,
|
|
1983
|
-
request: {
|
|
1984
|
-
content: request.content,
|
|
1985
|
-
task_type: request.taskType
|
|
1986
|
-
}
|
|
1987
|
-
})
|
|
1988
|
-
).join("\n");
|
|
1989
|
-
const displayName = `memory-embeddings-${hashText(String(Date.now()))}`;
|
|
1990
|
-
const uploadPayload = buildGeminiUploadBody({ jsonl, displayName });
|
|
1991
|
-
const uploadUrl = `${getGeminiUploadUrl(baseUrl)}/files?uploadType=multipart`;
|
|
1992
|
-
const fileRes = await fetch(uploadUrl, {
|
|
1993
|
-
method: "POST",
|
|
1994
|
-
headers: {
|
|
1995
|
-
...getGeminiHeaders(params.gemini, { json: false }),
|
|
1996
|
-
"Content-Type": uploadPayload.contentType
|
|
1997
|
-
},
|
|
1998
|
-
body: uploadPayload.body
|
|
1999
|
-
});
|
|
2000
|
-
if (!fileRes.ok) {
|
|
2001
|
-
const text2 = await fileRes.text();
|
|
2002
|
-
throw new Error(`gemini batch file upload failed: ${fileRes.status} ${text2}`);
|
|
2003
|
-
}
|
|
2004
|
-
const filePayload = await fileRes.json();
|
|
2005
|
-
const fileId = filePayload.name ?? filePayload.file?.name;
|
|
2006
|
-
if (!fileId) {
|
|
2007
|
-
throw new Error("gemini batch file upload failed: missing file id");
|
|
2008
|
-
}
|
|
2009
|
-
const batchBody = {
|
|
2010
|
-
batch: {
|
|
2011
|
-
displayName: `memory-embeddings-${params.source}`,
|
|
2012
|
-
inputConfig: {
|
|
2013
|
-
file_name: fileId
|
|
2014
|
-
}
|
|
2015
|
-
}
|
|
2016
|
-
};
|
|
2017
|
-
const batchEndpoint = `${baseUrl}/${params.gemini.modelPath}:asyncBatchEmbedContent`;
|
|
2018
|
-
const batchRes = await fetch(batchEndpoint, {
|
|
2019
|
-
method: "POST",
|
|
2020
|
-
headers: getGeminiHeaders(params.gemini, { json: true }),
|
|
2021
|
-
body: JSON.stringify(batchBody)
|
|
2022
|
-
});
|
|
2023
|
-
if (batchRes.ok) {
|
|
2024
|
-
return await batchRes.json();
|
|
2025
|
-
}
|
|
2026
|
-
const text = await batchRes.text();
|
|
2027
|
-
if (batchRes.status === 404) {
|
|
2028
|
-
throw new Error(
|
|
2029
|
-
"gemini batch create failed: 404 (asyncBatchEmbedContent not available for this model/baseUrl). Disable batch.enabled or switch providers."
|
|
2030
|
-
);
|
|
2031
|
-
}
|
|
2032
|
-
throw new Error(`gemini batch create failed: ${batchRes.status} ${text}`);
|
|
2033
|
-
}
|
|
2034
|
-
async function fetchGeminiBatchStatus(params) {
|
|
2035
|
-
const baseUrl = getGeminiBaseUrl(params.gemini);
|
|
2036
|
-
const name = params.batchName.startsWith("batches/") ? params.batchName : `batches/${params.batchName}`;
|
|
2037
|
-
const statusUrl = `${baseUrl}/${name}`;
|
|
2038
|
-
const res = await fetch(statusUrl, {
|
|
2039
|
-
headers: getGeminiHeaders(params.gemini, { json: true })
|
|
2040
|
-
});
|
|
2041
|
-
if (!res.ok) {
|
|
2042
|
-
const text = await res.text();
|
|
2043
|
-
throw new Error(`gemini batch status failed: ${res.status} ${text}`);
|
|
2044
|
-
}
|
|
2045
|
-
return await res.json();
|
|
2046
|
-
}
|
|
2047
|
-
async function fetchGeminiFileContent(params) {
|
|
2048
|
-
const baseUrl = getGeminiBaseUrl(params.gemini);
|
|
2049
|
-
const file = params.fileId.startsWith("files/") ? params.fileId : `files/${params.fileId}`;
|
|
2050
|
-
const downloadUrl = `${baseUrl}/${file}:download`;
|
|
2051
|
-
const res = await fetch(downloadUrl, {
|
|
2052
|
-
headers: getGeminiHeaders(params.gemini, { json: true })
|
|
2053
|
-
});
|
|
2054
|
-
if (!res.ok) {
|
|
2055
|
-
const text = await res.text();
|
|
2056
|
-
throw new Error(`gemini batch file content failed: ${res.status} ${text}`);
|
|
2057
|
-
}
|
|
2058
|
-
return await res.text();
|
|
2059
|
-
}
|
|
2060
|
-
function parseGeminiBatchOutput(text) {
|
|
2061
|
-
if (!text.trim()) return [];
|
|
2062
|
-
return text.split("\n").map((line) => line.trim()).filter(Boolean).map((line) => JSON.parse(line));
|
|
2063
|
-
}
|
|
2064
|
-
async function waitForGeminiBatch(params) {
|
|
2065
|
-
const start = Date.now();
|
|
2066
|
-
let current = params.initial;
|
|
2067
|
-
while (true) {
|
|
2068
|
-
const status2 = current ?? await fetchGeminiBatchStatus({
|
|
2069
|
-
gemini: params.gemini,
|
|
2070
|
-
batchName: params.batchName
|
|
2071
|
-
});
|
|
2072
|
-
const state = status2.state ?? "UNKNOWN";
|
|
2073
|
-
if (["SUCCEEDED", "COMPLETED", "DONE"].includes(state)) {
|
|
2074
|
-
const outputFileId = status2.outputConfig?.file ?? status2.outputConfig?.fileId ?? status2.metadata?.output?.responsesFile;
|
|
2075
|
-
if (!outputFileId) {
|
|
2076
|
-
throw new Error(`gemini batch ${params.batchName} completed without output file`);
|
|
2077
|
-
}
|
|
2078
|
-
return { outputFileId };
|
|
2079
|
-
}
|
|
2080
|
-
if (["FAILED", "CANCELLED", "CANCELED", "EXPIRED"].includes(state)) {
|
|
2081
|
-
const message = status2.error?.message ?? "unknown error";
|
|
2082
|
-
throw new Error(`gemini batch ${params.batchName} ${state}: ${message}`);
|
|
2083
|
-
}
|
|
2084
|
-
if (!params.wait) {
|
|
2085
|
-
throw new Error(`gemini batch ${params.batchName} still ${state}; wait disabled`);
|
|
2086
|
-
}
|
|
2087
|
-
if (Date.now() - start > params.timeoutMs) {
|
|
2088
|
-
throw new Error(`gemini batch ${params.batchName} timed out after ${params.timeoutMs}ms`);
|
|
2089
|
-
}
|
|
2090
|
-
params.debug?.(`gemini batch ${params.batchName} ${state}; waiting ${params.pollIntervalMs}ms`);
|
|
2091
|
-
await new Promise((resolve3) => setTimeout(resolve3, params.pollIntervalMs));
|
|
2092
|
-
current = void 0;
|
|
2093
|
-
}
|
|
2094
|
-
}
|
|
2095
|
-
async function runWithConcurrency2(tasks, limit) {
|
|
2096
|
-
if (tasks.length === 0) return [];
|
|
2097
|
-
const resolvedLimit = Math.max(1, Math.min(limit, tasks.length));
|
|
2098
|
-
const results = Array.from({ length: tasks.length });
|
|
2099
|
-
let next = 0;
|
|
2100
|
-
let firstError = null;
|
|
2101
|
-
const workers = Array.from({ length: resolvedLimit }, async () => {
|
|
2102
|
-
while (true) {
|
|
2103
|
-
if (firstError) return;
|
|
2104
|
-
const index = next;
|
|
2105
|
-
next += 1;
|
|
2106
|
-
if (index >= tasks.length) return;
|
|
2107
|
-
try {
|
|
2108
|
-
results[index] = await tasks[index]();
|
|
2109
|
-
} catch (err) {
|
|
2110
|
-
firstError = err;
|
|
2111
|
-
return;
|
|
2112
|
-
}
|
|
2113
|
-
}
|
|
2114
|
-
});
|
|
2115
|
-
await Promise.allSettled(workers);
|
|
2116
|
-
if (firstError) throw firstError;
|
|
2117
|
-
return results;
|
|
2118
|
-
}
|
|
2119
|
-
async function runGeminiEmbeddingBatches(params) {
|
|
2120
|
-
if (params.requests.length === 0) return /* @__PURE__ */ new Map();
|
|
2121
|
-
const groups = splitGeminiBatchRequests(params.requests);
|
|
2122
|
-
const byCustomId = /* @__PURE__ */ new Map();
|
|
2123
|
-
const tasks = groups.map((group, groupIndex) => async () => {
|
|
2124
|
-
const batchInfo = await submitGeminiBatch({
|
|
2125
|
-
gemini: params.gemini,
|
|
2126
|
-
requests: group,
|
|
2127
|
-
source: params.source
|
|
2128
|
-
});
|
|
2129
|
-
const batchName = batchInfo.name ?? "";
|
|
2130
|
-
if (!batchName) {
|
|
2131
|
-
throw new Error("gemini batch create failed: missing batch name");
|
|
2132
|
-
}
|
|
2133
|
-
params.debug?.("memory embeddings: gemini batch created", {
|
|
2134
|
-
batchName,
|
|
2135
|
-
state: batchInfo.state,
|
|
2136
|
-
group: groupIndex + 1,
|
|
2137
|
-
groups: groups.length,
|
|
2138
|
-
requests: group.length
|
|
2139
|
-
});
|
|
2140
|
-
if (!params.wait && batchInfo.state && !["SUCCEEDED", "COMPLETED", "DONE"].includes(batchInfo.state)) {
|
|
2141
|
-
throw new Error(
|
|
2142
|
-
`gemini batch ${batchName} submitted; enable batch.wait to await completion`
|
|
2143
|
-
);
|
|
2144
|
-
}
|
|
2145
|
-
const completed = batchInfo.state && ["SUCCEEDED", "COMPLETED", "DONE"].includes(batchInfo.state) ? {
|
|
2146
|
-
outputFileId: batchInfo.outputConfig?.file ?? batchInfo.outputConfig?.fileId ?? batchInfo.metadata?.output?.responsesFile ?? ""
|
|
2147
|
-
} : await waitForGeminiBatch({
|
|
2148
|
-
gemini: params.gemini,
|
|
2149
|
-
batchName,
|
|
2150
|
-
wait: params.wait,
|
|
2151
|
-
pollIntervalMs: params.pollIntervalMs,
|
|
2152
|
-
timeoutMs: params.timeoutMs,
|
|
2153
|
-
debug: params.debug,
|
|
2154
|
-
initial: batchInfo
|
|
2155
|
-
});
|
|
2156
|
-
if (!completed.outputFileId) {
|
|
2157
|
-
throw new Error(`gemini batch ${batchName} completed without output file`);
|
|
2158
|
-
}
|
|
2159
|
-
const content = await fetchGeminiFileContent({
|
|
2160
|
-
gemini: params.gemini,
|
|
2161
|
-
fileId: completed.outputFileId
|
|
2162
|
-
});
|
|
2163
|
-
const outputLines = parseGeminiBatchOutput(content);
|
|
2164
|
-
const errors = [];
|
|
2165
|
-
const remaining = new Set(group.map((request) => request.custom_id));
|
|
2166
|
-
for (const line of outputLines) {
|
|
2167
|
-
const customId = line.key ?? line.custom_id ?? line.request_id;
|
|
2168
|
-
if (!customId) continue;
|
|
2169
|
-
remaining.delete(customId);
|
|
2170
|
-
if (line.error?.message) {
|
|
2171
|
-
errors.push(`${customId}: ${line.error.message}`);
|
|
2172
|
-
continue;
|
|
2173
|
-
}
|
|
2174
|
-
if (line.response?.error?.message) {
|
|
2175
|
-
errors.push(`${customId}: ${line.response.error.message}`);
|
|
2176
|
-
continue;
|
|
2177
|
-
}
|
|
2178
|
-
const embedding = line.embedding?.values ?? line.response?.embedding?.values ?? [];
|
|
2179
|
-
if (embedding.length === 0) {
|
|
2180
|
-
errors.push(`${customId}: empty embedding`);
|
|
2181
|
-
continue;
|
|
2182
|
-
}
|
|
2183
|
-
byCustomId.set(customId, embedding);
|
|
2184
|
-
}
|
|
2185
|
-
if (errors.length > 0) {
|
|
2186
|
-
throw new Error(`gemini batch ${batchName} failed: ${errors.join("; ")}`);
|
|
2187
|
-
}
|
|
2188
|
-
if (remaining.size > 0) {
|
|
2189
|
-
throw new Error(`gemini batch ${batchName} missing ${remaining.size} embedding responses`);
|
|
2190
|
-
}
|
|
2191
|
-
});
|
|
2192
|
-
params.debug?.("memory embeddings: gemini batch submit", {
|
|
2193
|
-
requests: params.requests.length,
|
|
2194
|
-
groups: groups.length,
|
|
2195
|
-
wait: params.wait,
|
|
2196
|
-
concurrency: params.concurrency,
|
|
2197
|
-
pollIntervalMs: params.pollIntervalMs,
|
|
2198
|
-
timeoutMs: params.timeoutMs
|
|
2199
|
-
});
|
|
2200
|
-
await runWithConcurrency2(tasks, params.concurrency);
|
|
2201
|
-
return byCustomId;
|
|
2202
|
-
}
|
|
2203
|
-
|
|
2204
|
-
// src/minimem.ts
|
|
2205
|
-
var META_KEY = "memory_index_meta_v1";
|
|
2206
|
-
var SNIPPET_MAX_CHARS = 700;
|
|
2207
|
-
var VECTOR_TABLE = "chunks_vec";
|
|
2208
|
-
var FTS_TABLE = "chunks_fts";
|
|
2209
|
-
var EMBEDDING_CACHE_TABLE = "embedding_cache";
|
|
2210
|
-
var EMBEDDING_RETRY_MAX_ATTEMPTS = 3;
|
|
2211
|
-
var EMBEDDING_RETRY_BASE_DELAY_MS = 500;
|
|
2212
|
-
var EMBEDDING_RETRY_MAX_DELAY_MS = 8e3;
|
|
2213
|
-
var EMBEDDING_QUERY_TIMEOUT_REMOTE_MS = 6e4;
|
|
2214
|
-
var EMBEDDING_QUERY_TIMEOUT_LOCAL_MS = 5 * 6e4;
|
|
2215
|
-
var Minimem = class _Minimem {
|
|
2216
|
-
memoryDir;
|
|
2217
|
-
dbPath;
|
|
2218
|
-
chunking;
|
|
2219
|
-
cache;
|
|
2220
|
-
hybrid;
|
|
2221
|
-
queryConfig;
|
|
2222
|
-
watchConfig;
|
|
2223
|
-
batchConfig;
|
|
2224
|
-
vectorExtensionPath;
|
|
2225
|
-
debug;
|
|
2226
|
-
provider;
|
|
2227
|
-
openAi;
|
|
2228
|
-
gemini;
|
|
2229
|
-
providerKey = "";
|
|
2230
|
-
providerFallbackReason;
|
|
2231
|
-
db;
|
|
2232
|
-
vector;
|
|
2233
|
-
fts;
|
|
2234
|
-
vectorReady = null;
|
|
2235
|
-
watcher = null;
|
|
2236
|
-
watchTimer = null;
|
|
2237
|
-
closed = false;
|
|
2238
|
-
dirty = true;
|
|
2239
|
-
syncing = null;
|
|
2240
|
-
syncLock = false;
|
|
2241
|
-
embeddingOptions;
|
|
2242
|
-
constructor(config2) {
|
|
2243
|
-
this.memoryDir = path6.resolve(config2.memoryDir);
|
|
2244
|
-
this.dbPath = config2.dbPath ?? path6.join(this.memoryDir, ".minimem", "index.db");
|
|
2245
|
-
this.chunking = {
|
|
2246
|
-
tokens: config2.chunking?.tokens ?? 256,
|
|
2247
|
-
overlap: config2.chunking?.overlap ?? 32
|
|
2248
|
-
};
|
|
2249
|
-
this.cache = {
|
|
2250
|
-
enabled: config2.cache?.enabled ?? true,
|
|
2251
|
-
maxEntries: config2.cache?.maxEntries ?? 1e4
|
|
2252
|
-
};
|
|
2253
|
-
this.hybrid = {
|
|
2254
|
-
enabled: config2.hybrid?.enabled ?? true,
|
|
2255
|
-
vectorWeight: config2.hybrid?.vectorWeight ?? 0.7,
|
|
2256
|
-
textWeight: config2.hybrid?.textWeight ?? 0.3,
|
|
2257
|
-
candidateMultiplier: config2.hybrid?.candidateMultiplier ?? 2
|
|
2258
|
-
};
|
|
2259
|
-
this.queryConfig = {
|
|
2260
|
-
maxResults: config2.query?.maxResults ?? 10,
|
|
2261
|
-
minScore: config2.query?.minScore ?? 0.3
|
|
2262
|
-
};
|
|
2263
|
-
this.watchConfig = {
|
|
2264
|
-
enabled: config2.watch?.enabled ?? true,
|
|
2265
|
-
debounceMs: config2.watch?.debounceMs ?? 1e3
|
|
2266
|
-
};
|
|
2267
|
-
this.batchConfig = {
|
|
2268
|
-
enabled: config2.batch?.enabled ?? false,
|
|
2269
|
-
wait: config2.batch?.wait ?? true,
|
|
2270
|
-
concurrency: config2.batch?.concurrency ?? 2,
|
|
2271
|
-
pollIntervalMs: config2.batch?.pollIntervalMs ?? 2e3,
|
|
2272
|
-
timeoutMs: config2.batch?.timeoutMs ?? 60 * 60 * 1e3
|
|
2273
|
-
};
|
|
2274
|
-
this.vectorExtensionPath = config2.vectorExtensionPath;
|
|
2275
|
-
this.debug = config2.debug;
|
|
2276
|
-
this.embeddingOptions = config2.embedding;
|
|
2277
|
-
this.vector = {
|
|
2278
|
-
enabled: true,
|
|
2279
|
-
available: null,
|
|
2280
|
-
extensionPath: this.vectorExtensionPath
|
|
2281
|
-
};
|
|
2282
|
-
this.fts = { enabled: this.hybrid.enabled, available: false };
|
|
2283
|
-
}
|
|
2284
|
-
static async create(config2) {
|
|
2285
|
-
const instance = new _Minimem(config2);
|
|
2286
|
-
await instance.initialize();
|
|
2287
|
-
return instance;
|
|
2288
|
-
}
|
|
2289
|
-
async initialize() {
|
|
2290
|
-
const providerResult = await createEmbeddingProvider(this.embeddingOptions);
|
|
2291
|
-
this.provider = providerResult.provider;
|
|
2292
|
-
this.openAi = providerResult.openAi;
|
|
2293
|
-
this.gemini = providerResult.gemini;
|
|
2294
|
-
this.providerKey = this.computeProviderKey();
|
|
2295
|
-
this.providerFallbackReason = providerResult.fallbackReason;
|
|
2296
|
-
if (this.provider.id === "none") {
|
|
2297
|
-
this.debug?.("Running in BM25-only mode (no embedding API available)");
|
|
2298
|
-
}
|
|
2299
|
-
this.db = this.openDatabase();
|
|
2300
|
-
this.ensureSchema();
|
|
2301
|
-
const meta = this.readMeta();
|
|
2302
|
-
if (meta?.vectorDims) {
|
|
2303
|
-
this.vector.dims = meta.vectorDims;
|
|
2304
|
-
}
|
|
2305
|
-
if (this.watchConfig.enabled) {
|
|
2306
|
-
this.ensureWatcher();
|
|
2307
|
-
}
|
|
2308
|
-
}
|
|
2309
|
-
openDatabase() {
|
|
2310
|
-
const dbDir = path6.dirname(this.dbPath);
|
|
2311
|
-
ensureDir(dbDir);
|
|
2312
|
-
return new DatabaseSync(this.dbPath);
|
|
2313
|
-
}
|
|
2314
|
-
ensureSchema() {
|
|
2315
|
-
const result = ensureMemoryIndexSchema({
|
|
2316
|
-
db: this.db,
|
|
2317
|
-
embeddingCacheTable: EMBEDDING_CACHE_TABLE,
|
|
2318
|
-
ftsTable: FTS_TABLE,
|
|
2319
|
-
ftsEnabled: this.fts.enabled
|
|
2320
|
-
});
|
|
2321
|
-
this.fts.available = result.ftsAvailable;
|
|
2322
|
-
if (result.ftsError) {
|
|
2323
|
-
this.fts.loadError = result.ftsError;
|
|
2324
|
-
}
|
|
2325
|
-
}
|
|
2326
|
-
computeProviderKey() {
|
|
2327
|
-
const parts = [this.provider.id, this.provider.model];
|
|
2328
|
-
if (this.openAi) {
|
|
2329
|
-
parts.push(this.openAi.baseUrl);
|
|
2330
|
-
}
|
|
2331
|
-
if (this.gemini) {
|
|
2332
|
-
parts.push(this.gemini.baseUrl);
|
|
2333
|
-
}
|
|
2334
|
-
return hashText(parts.join(":"));
|
|
2335
|
-
}
|
|
2336
|
-
readMeta() {
|
|
2337
|
-
try {
|
|
2338
|
-
const row = this.db.prepare(`SELECT value FROM meta WHERE key = ?`).get(META_KEY);
|
|
2339
|
-
if (!row?.value) return null;
|
|
2340
|
-
return JSON.parse(row.value);
|
|
2341
|
-
} catch {
|
|
2342
|
-
return null;
|
|
2343
|
-
}
|
|
2344
|
-
}
|
|
2345
|
-
writeMeta(meta) {
|
|
2346
|
-
this.db.prepare(`INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)`).run(META_KEY, JSON.stringify(meta));
|
|
2347
|
-
}
|
|
2348
|
-
ensureWatcher() {
|
|
2349
|
-
if (this.watcher) return;
|
|
2350
|
-
const memorySubDir = path6.join(this.memoryDir, "memory");
|
|
2351
|
-
const memoryFile = path6.join(this.memoryDir, "MEMORY.md");
|
|
2352
|
-
this.watcher = chokidar.watch([memoryFile, memorySubDir], {
|
|
2353
|
-
ignoreInitial: true,
|
|
2354
|
-
persistent: true,
|
|
2355
|
-
awaitWriteFinish: { stabilityThreshold: 200, pollInterval: 50 }
|
|
2356
|
-
});
|
|
2357
|
-
const scheduleSync = () => {
|
|
2358
|
-
this.dirty = true;
|
|
2359
|
-
if (this.watchTimer) clearTimeout(this.watchTimer);
|
|
2360
|
-
this.watchTimer = setTimeout(() => {
|
|
2361
|
-
void this.sync({ reason: "watch" }).catch((err) => {
|
|
2362
|
-
this.debug?.(`memory sync failed (watch): ${String(err)}`);
|
|
2363
|
-
});
|
|
2364
|
-
}, this.watchConfig.debounceMs);
|
|
2365
|
-
};
|
|
2366
|
-
this.watcher.on("add", scheduleSync);
|
|
2367
|
-
this.watcher.on("change", scheduleSync);
|
|
2368
|
-
this.watcher.on("unlink", scheduleSync);
|
|
2369
|
-
}
|
|
2370
|
-
/**
|
|
2371
|
-
* Check if the index is stale by comparing file mtimes against stored values.
|
|
2372
|
-
* This is a lightweight check (stat calls only, no file reads).
|
|
2373
|
-
*/
|
|
2374
|
-
async isStale() {
|
|
2375
|
-
try {
|
|
2376
|
-
const files = await listMemoryFiles(this.memoryDir);
|
|
2377
|
-
const stored = this.db.prepare(`SELECT path, mtime FROM files WHERE source = ?`).all("memory");
|
|
2378
|
-
if (files.length !== stored.length) {
|
|
2379
|
-
this.debug?.(`Stale: file count changed (${stored.length} -> ${files.length})`);
|
|
2380
|
-
return true;
|
|
2381
|
-
}
|
|
2382
|
-
const storedMap = new Map(stored.map((f) => [f.path, f.mtime]));
|
|
2383
|
-
for (const absPath of files) {
|
|
2384
|
-
const relPath = path6.relative(this.memoryDir, absPath).replace(/\\/g, "/");
|
|
2385
|
-
const storedMtime = storedMap.get(relPath);
|
|
2386
|
-
if (storedMtime === void 0) {
|
|
2387
|
-
this.debug?.(`Stale: new file ${relPath}`);
|
|
2388
|
-
return true;
|
|
2389
|
-
}
|
|
2390
|
-
const stat = await fs4.stat(absPath);
|
|
2391
|
-
const currentMtime = Math.floor(stat.mtimeMs);
|
|
2392
|
-
if (currentMtime !== storedMtime) {
|
|
2393
|
-
this.debug?.(`Stale: mtime changed for ${relPath}`);
|
|
2394
|
-
return true;
|
|
2395
|
-
}
|
|
2396
|
-
}
|
|
2397
|
-
return false;
|
|
2398
|
-
} catch (err) {
|
|
2399
|
-
this.debug?.(`Stale check failed: ${String(err)}`);
|
|
2400
|
-
return true;
|
|
2401
|
-
}
|
|
2402
|
-
}
|
|
2403
|
-
async search(query, opts) {
|
|
2404
|
-
if (this.dirty || !this.watchConfig.enabled && await this.isStale()) {
|
|
2405
|
-
await this.sync({ reason: "search" });
|
|
2406
|
-
}
|
|
2407
|
-
const cleaned = query.trim();
|
|
2408
|
-
if (!cleaned) return [];
|
|
2409
|
-
const minScore = opts?.minScore ?? this.queryConfig.minScore;
|
|
2410
|
-
const maxResults = opts?.maxResults ?? this.queryConfig.maxResults;
|
|
2411
|
-
const candidates = Math.min(
|
|
2412
|
-
200,
|
|
2413
|
-
Math.max(1, Math.floor(maxResults * this.hybrid.candidateMultiplier))
|
|
2414
|
-
);
|
|
2415
|
-
const sourceFilter = { sql: "", params: [] };
|
|
2416
|
-
const keywordResults = this.hybrid.enabled && this.fts.available ? await searchKeyword({
|
|
2417
|
-
db: this.db,
|
|
2418
|
-
ftsTable: FTS_TABLE,
|
|
2419
|
-
providerModel: this.provider.model,
|
|
2420
|
-
query: cleaned,
|
|
2421
|
-
limit: candidates,
|
|
2422
|
-
snippetMaxChars: SNIPPET_MAX_CHARS,
|
|
2423
|
-
sourceFilter,
|
|
2424
|
-
buildFtsQuery,
|
|
2425
|
-
bm25RankToScore
|
|
2426
|
-
}).catch(() => []) : [];
|
|
2427
|
-
const queryVec = await this.embedQueryWithTimeout(cleaned);
|
|
2428
|
-
const hasVector = queryVec.some((v) => v !== 0);
|
|
2429
|
-
const vectorResults = hasVector ? await searchVector({
|
|
2430
|
-
db: this.db,
|
|
2431
|
-
vectorTable: VECTOR_TABLE,
|
|
2432
|
-
providerModel: this.provider.model,
|
|
2433
|
-
queryVec,
|
|
2434
|
-
limit: candidates,
|
|
2435
|
-
snippetMaxChars: SNIPPET_MAX_CHARS,
|
|
2436
|
-
ensureVectorReady: (dims) => this.ensureVectorReady(dims),
|
|
2437
|
-
sourceFilterVec: sourceFilter,
|
|
2438
|
-
sourceFilterChunks: sourceFilter
|
|
2439
|
-
}).catch(() => []) : [];
|
|
2440
|
-
const typeFilterFn = opts?.type ? (id) => {
|
|
2441
|
-
const row = this.db.prepare(`SELECT type FROM chunks WHERE id = ?`).get(id);
|
|
2442
|
-
return row?.type === opts.type;
|
|
2443
|
-
} : void 0;
|
|
2444
|
-
if (!this.hybrid.enabled) {
|
|
2445
|
-
let results = vectorResults;
|
|
2446
|
-
if (typeFilterFn) results = results.filter((r) => typeFilterFn(r.id));
|
|
2447
|
-
return results.filter((entry) => entry.score >= minScore).slice(0, maxResults).map((r) => ({
|
|
2448
|
-
path: r.path,
|
|
2449
|
-
startLine: r.startLine,
|
|
2450
|
-
endLine: r.endLine,
|
|
2451
|
-
score: r.score,
|
|
2452
|
-
snippet: r.snippet
|
|
2453
|
-
}));
|
|
2454
|
-
}
|
|
2455
|
-
let filteredVector = vectorResults;
|
|
2456
|
-
let filteredKeyword = keywordResults;
|
|
2457
|
-
if (typeFilterFn) {
|
|
2458
|
-
filteredVector = vectorResults.filter((r) => typeFilterFn(r.id));
|
|
2459
|
-
filteredKeyword = keywordResults.filter((r) => typeFilterFn(r.id));
|
|
2460
|
-
}
|
|
2461
|
-
const merged = mergeHybridResults({
|
|
2462
|
-
vector: filteredVector.map((r) => ({
|
|
2463
|
-
id: r.id,
|
|
2464
|
-
path: r.path,
|
|
2465
|
-
startLine: r.startLine,
|
|
2466
|
-
endLine: r.endLine,
|
|
2467
|
-
source: r.source,
|
|
2468
|
-
snippet: r.snippet,
|
|
2469
|
-
vectorScore: r.score
|
|
2470
|
-
})),
|
|
2471
|
-
keyword: filteredKeyword.map((r) => ({
|
|
2472
|
-
id: r.id,
|
|
2473
|
-
path: r.path,
|
|
2474
|
-
startLine: r.startLine,
|
|
2475
|
-
endLine: r.endLine,
|
|
2476
|
-
source: r.source,
|
|
2477
|
-
snippet: r.snippet,
|
|
2478
|
-
textScore: r.textScore
|
|
2479
|
-
})),
|
|
2480
|
-
vectorWeight: this.hybrid.vectorWeight,
|
|
2481
|
-
textWeight: this.hybrid.textWeight
|
|
2482
|
-
});
|
|
2483
|
-
return merged.filter((entry) => entry.score >= minScore).slice(0, maxResults).map((r) => ({
|
|
2484
|
-
path: r.path,
|
|
2485
|
-
startLine: r.startLine,
|
|
2486
|
-
endLine: r.endLine,
|
|
2487
|
-
score: r.score,
|
|
2488
|
-
snippet: r.snippet
|
|
2489
|
-
}));
|
|
2490
|
-
}
|
|
2491
|
-
async sync(opts) {
|
|
2492
|
-
if (this.syncing) {
|
|
2493
|
-
await this.syncing;
|
|
2494
|
-
return;
|
|
2495
|
-
}
|
|
2496
|
-
if (this.syncLock) {
|
|
2497
|
-
return;
|
|
2498
|
-
}
|
|
2499
|
-
this.syncLock = true;
|
|
2500
|
-
this.syncing = this.runSync(opts);
|
|
2501
|
-
try {
|
|
2502
|
-
await this.syncing;
|
|
2503
|
-
} finally {
|
|
2504
|
-
this.syncing = null;
|
|
2505
|
-
this.syncLock = false;
|
|
2506
|
-
}
|
|
2507
|
-
}
|
|
2508
|
-
async runSync(opts) {
|
|
2509
|
-
this.debug?.(`memory sync starting`, { reason: opts?.reason });
|
|
2510
|
-
await this.ensureVectorReady();
|
|
2511
|
-
const meta = this.readMeta();
|
|
2512
|
-
const needsFullReindex = opts?.force || !meta || meta.model !== this.provider.model || meta.provider !== this.provider.id || meta.providerKey !== this.providerKey || meta.chunkTokens !== this.chunking.tokens || meta.chunkOverlap !== this.chunking.overlap || this.vector.available && !meta?.vectorDims;
|
|
2513
|
-
const files = await listMemoryFiles(this.memoryDir);
|
|
2514
|
-
const activePaths = /* @__PURE__ */ new Set();
|
|
2515
|
-
for (const absPath of files) {
|
|
2516
|
-
const entry = await buildFileEntry(absPath, this.memoryDir);
|
|
2517
|
-
activePaths.add(entry.path);
|
|
2518
|
-
const record = this.db.prepare(`SELECT hash FROM files WHERE path = ? AND source = ?`).get(entry.path, "memory");
|
|
2519
|
-
if (!needsFullReindex && record?.hash === entry.hash) {
|
|
2520
|
-
continue;
|
|
2521
|
-
}
|
|
2522
|
-
await this.indexFile(entry);
|
|
2523
|
-
}
|
|
2524
|
-
const staleRows = this.db.prepare(`SELECT path FROM files WHERE source = ?`).all("memory");
|
|
2525
|
-
for (const stale of staleRows) {
|
|
2526
|
-
if (activePaths.has(stale.path)) continue;
|
|
2527
|
-
this.db.prepare(`DELETE FROM files WHERE path = ? AND source = ?`).run(stale.path, "memory");
|
|
2528
|
-
try {
|
|
2529
|
-
this.db.prepare(
|
|
2530
|
-
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`
|
|
2531
|
-
).run(stale.path, "memory");
|
|
2532
|
-
} catch (err) {
|
|
2533
|
-
logError2("deleteStaleVectorEntries", err, this.debug);
|
|
2534
|
-
}
|
|
2535
|
-
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(stale.path, "memory");
|
|
2536
|
-
this.db.prepare(`DELETE FROM knowledge_links WHERE source_path = ?`).run(stale.path);
|
|
2537
|
-
if (this.fts.enabled && this.fts.available) {
|
|
2538
|
-
try {
|
|
2539
|
-
this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`).run(stale.path, "memory", this.provider.model);
|
|
2540
|
-
} catch (err) {
|
|
2541
|
-
logError2("deleteStaleFtsEntries", err, this.debug);
|
|
2542
|
-
}
|
|
2543
|
-
}
|
|
2544
|
-
}
|
|
2545
|
-
this.writeMeta({
|
|
2546
|
-
model: this.provider.model,
|
|
2547
|
-
provider: this.provider.id,
|
|
2548
|
-
providerKey: this.providerKey,
|
|
2549
|
-
chunkTokens: this.chunking.tokens,
|
|
2550
|
-
chunkOverlap: this.chunking.overlap,
|
|
2551
|
-
vectorDims: this.vector.dims
|
|
2552
|
-
});
|
|
2553
|
-
this.pruneEmbeddingCacheIfNeeded();
|
|
2554
|
-
this.dirty = false;
|
|
2555
|
-
this.debug?.(`memory sync complete`, { files: files.length });
|
|
2556
|
-
}
|
|
2557
|
-
async indexFile(entry) {
|
|
2558
|
-
const content = await fs4.readFile(entry.absPath, "utf-8");
|
|
2559
|
-
const chunks = chunkMarkdown(content, this.chunking);
|
|
2560
|
-
const { frontmatter } = parseFrontmatter(content);
|
|
2561
|
-
const knowledgeType = frontmatter?.type ?? null;
|
|
2562
|
-
const knowledgeId = frontmatter?.id ?? null;
|
|
2563
|
-
const domains = frontmatter?.domain ?? null;
|
|
2564
|
-
const entities = frontmatter?.entities ?? null;
|
|
2565
|
-
const confidence = frontmatter?.confidence ?? null;
|
|
2566
|
-
const links = frontmatter?.links ?? null;
|
|
2567
|
-
const embeddings = await this.embedChunks(chunks);
|
|
2568
|
-
this.db.prepare(
|
|
2569
|
-
`INSERT OR REPLACE INTO files (path, source, hash, mtime, size) VALUES (?, ?, ?, ?, ?)`
|
|
2570
|
-
).run(entry.path, "memory", entry.hash, Math.floor(entry.mtimeMs), entry.size);
|
|
2571
|
-
try {
|
|
2572
|
-
this.db.prepare(
|
|
2573
|
-
`DELETE FROM ${VECTOR_TABLE} WHERE id IN (SELECT id FROM chunks WHERE path = ? AND source = ?)`
|
|
2574
|
-
).run(entry.path, "memory");
|
|
2575
|
-
} catch (err) {
|
|
2576
|
-
logError2("deleteOldVectorChunks", err, this.debug);
|
|
2577
|
-
}
|
|
2578
|
-
this.db.prepare(`DELETE FROM chunks WHERE path = ? AND source = ?`).run(entry.path, "memory");
|
|
2579
|
-
if (this.fts.enabled && this.fts.available) {
|
|
2580
|
-
try {
|
|
2581
|
-
this.db.prepare(`DELETE FROM ${FTS_TABLE} WHERE path = ? AND source = ? AND model = ?`).run(entry.path, "memory", this.provider.model);
|
|
2582
|
-
} catch (err) {
|
|
2583
|
-
logError2("deleteOldFtsChunks", err, this.debug);
|
|
2584
|
-
}
|
|
2585
|
-
}
|
|
2586
|
-
this.db.prepare(`DELETE FROM knowledge_links WHERE source_path = ?`).run(entry.path);
|
|
2587
|
-
const now = Date.now();
|
|
2588
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
2589
|
-
const chunk = chunks[i];
|
|
2590
|
-
const embedding = embeddings[i] ?? [];
|
|
2591
|
-
const chunkId = randomUUID();
|
|
2592
|
-
const meta = extractChunkMetadata(chunk.text);
|
|
2593
|
-
this.db.prepare(
|
|
2594
|
-
`INSERT INTO chunks (id, path, source, start_line, end_line, hash, model, text, embedding, updated_at, type, knowledge_type, knowledge_id, domains, entities, confidence)
|
|
2595
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`
|
|
2596
|
-
).run(
|
|
2597
|
-
chunkId,
|
|
2598
|
-
entry.path,
|
|
2599
|
-
"memory",
|
|
2600
|
-
chunk.startLine,
|
|
2601
|
-
chunk.endLine,
|
|
2602
|
-
chunk.hash,
|
|
2603
|
-
this.provider.model,
|
|
2604
|
-
chunk.text,
|
|
2605
|
-
JSON.stringify(embedding),
|
|
2606
|
-
now,
|
|
2607
|
-
meta.type ?? null,
|
|
2608
|
-
knowledgeType,
|
|
2609
|
-
knowledgeId,
|
|
2610
|
-
domains ? JSON.stringify(domains) : null,
|
|
2611
|
-
entities ? JSON.stringify(entities) : null,
|
|
2612
|
-
confidence
|
|
2613
|
-
);
|
|
2614
|
-
if (this.vector.available && embedding.length > 0) {
|
|
2615
|
-
if (!this.vector.dims) {
|
|
2616
|
-
this.vector.dims = embedding.length;
|
|
2617
|
-
this.ensureVectorTable(embedding.length);
|
|
2618
|
-
}
|
|
2619
|
-
try {
|
|
2620
|
-
this.db.prepare(`INSERT INTO ${VECTOR_TABLE} (id, embedding) VALUES (?, ?)`).run(chunkId, vectorToBlob(embedding));
|
|
2621
|
-
} catch (err) {
|
|
2622
|
-
logError2("insertVectorChunk", err, this.debug);
|
|
2623
|
-
}
|
|
2624
|
-
}
|
|
2625
|
-
if (this.fts.enabled && this.fts.available) {
|
|
2626
|
-
try {
|
|
2627
|
-
this.db.prepare(
|
|
2628
|
-
`INSERT INTO ${FTS_TABLE} (text, id, path, source, model, start_line, end_line)
|
|
2629
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
2630
|
-
).run(
|
|
2631
|
-
chunk.text,
|
|
2632
|
-
chunkId,
|
|
2633
|
-
entry.path,
|
|
2634
|
-
"memory",
|
|
2635
|
-
this.provider.model,
|
|
2636
|
-
chunk.startLine,
|
|
2637
|
-
chunk.endLine
|
|
2638
|
-
);
|
|
2639
|
-
} catch (err) {
|
|
2640
|
-
logError2("insertFtsChunk", err, this.debug);
|
|
2641
|
-
}
|
|
2642
|
-
}
|
|
2643
|
-
}
|
|
2644
|
-
if (links && knowledgeId) {
|
|
2645
|
-
const upsertLink = this.db.prepare(
|
|
2646
|
-
`INSERT OR REPLACE INTO knowledge_links (from_id, to_id, relation, layer, weight, source_path, created_at)
|
|
2647
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
2648
|
-
);
|
|
2649
|
-
for (const link of links) {
|
|
2650
|
-
upsertLink.run(
|
|
2651
|
-
knowledgeId,
|
|
2652
|
-
link.target,
|
|
2653
|
-
link.relation,
|
|
2654
|
-
link.layer ?? null,
|
|
2655
|
-
0.5,
|
|
2656
|
-
entry.path,
|
|
2657
|
-
now
|
|
2658
|
-
);
|
|
2659
|
-
}
|
|
2660
|
-
}
|
|
2661
|
-
}
|
|
2662
|
-
async embedChunks(chunks) {
|
|
2663
|
-
if (chunks.length === 0) return [];
|
|
2664
|
-
const hashes = chunks.map((c) => c.hash);
|
|
2665
|
-
const cached = this.loadEmbeddingCache(hashes);
|
|
2666
|
-
const missing = [];
|
|
2667
|
-
for (let i = 0; i < chunks.length; i++) {
|
|
2668
|
-
if (!cached.has(hashes[i])) {
|
|
2669
|
-
missing.push({ index: i, chunk: chunks[i] });
|
|
2670
|
-
}
|
|
2671
|
-
}
|
|
2672
|
-
if (missing.length > 0) {
|
|
2673
|
-
const texts = missing.map((m) => m.chunk.text);
|
|
2674
|
-
const newEmbeddings = await this.embedBatchWithRetry(texts);
|
|
2675
|
-
for (let i = 0; i < missing.length; i++) {
|
|
2676
|
-
const hash = missing[i].chunk.hash;
|
|
2677
|
-
const embedding = newEmbeddings[i] ?? [];
|
|
2678
|
-
cached.set(hash, embedding);
|
|
2679
|
-
this.upsertEmbeddingCache(hash, embedding);
|
|
2680
|
-
}
|
|
2681
|
-
}
|
|
2682
|
-
return hashes.map((h) => cached.get(h) ?? []);
|
|
2683
|
-
}
|
|
2684
|
-
async embedBatchWithRetry(texts) {
|
|
2685
|
-
if (texts.length === 0) return [];
|
|
2686
|
-
if (this.batchConfig.enabled) {
|
|
2687
|
-
try {
|
|
2688
|
-
return await this.embedWithBatchApi(texts);
|
|
2689
|
-
} catch (err) {
|
|
2690
|
-
this.debug?.(`batch embedding failed, falling back to direct: ${String(err)}`);
|
|
2691
|
-
}
|
|
2692
|
-
}
|
|
2693
|
-
let lastError = null;
|
|
2694
|
-
for (let attempt = 0; attempt < EMBEDDING_RETRY_MAX_ATTEMPTS; attempt++) {
|
|
2695
|
-
try {
|
|
2696
|
-
return await this.provider.embedBatch(texts);
|
|
2697
|
-
} catch (err) {
|
|
2698
|
-
lastError = err instanceof Error ? err : new Error(String(err));
|
|
2699
|
-
if (attempt < EMBEDDING_RETRY_MAX_ATTEMPTS - 1) {
|
|
2700
|
-
const delay = Math.min(
|
|
2701
|
-
EMBEDDING_RETRY_MAX_DELAY_MS,
|
|
2702
|
-
EMBEDDING_RETRY_BASE_DELAY_MS * Math.pow(2, attempt)
|
|
2703
|
-
);
|
|
2704
|
-
await new Promise((resolve3) => setTimeout(resolve3, delay));
|
|
2705
|
-
}
|
|
2706
|
-
}
|
|
2707
|
-
}
|
|
2708
|
-
throw lastError;
|
|
2709
|
-
}
|
|
2710
|
-
async embedWithBatchApi(texts) {
|
|
2711
|
-
if (this.openAi) {
|
|
2712
|
-
const requests = texts.map((text, i) => ({
|
|
2713
|
-
custom_id: `chunk-${i}`,
|
|
2714
|
-
method: "POST",
|
|
2715
|
-
url: OPENAI_BATCH_ENDPOINT,
|
|
2716
|
-
body: { model: this.openAi.model, input: text }
|
|
2717
|
-
}));
|
|
2718
|
-
const results = await runOpenAiEmbeddingBatches({
|
|
2719
|
-
openAi: this.openAi,
|
|
2720
|
-
source: "minimem",
|
|
2721
|
-
requests,
|
|
2722
|
-
wait: this.batchConfig.wait,
|
|
2723
|
-
pollIntervalMs: this.batchConfig.pollIntervalMs,
|
|
2724
|
-
timeoutMs: this.batchConfig.timeoutMs,
|
|
2725
|
-
concurrency: this.batchConfig.concurrency,
|
|
2726
|
-
debug: this.debug
|
|
2727
|
-
});
|
|
2728
|
-
return texts.map((_, i) => results.get(`chunk-${i}`) ?? []);
|
|
2729
|
-
}
|
|
2730
|
-
if (this.gemini) {
|
|
2731
|
-
const requests = texts.map((text, i) => ({
|
|
2732
|
-
custom_id: `chunk-${i}`,
|
|
2733
|
-
content: { parts: [{ text }] },
|
|
2734
|
-
taskType: "RETRIEVAL_DOCUMENT"
|
|
2735
|
-
}));
|
|
2736
|
-
const results = await runGeminiEmbeddingBatches({
|
|
2737
|
-
gemini: this.gemini,
|
|
2738
|
-
source: "minimem",
|
|
2739
|
-
requests,
|
|
2740
|
-
wait: this.batchConfig.wait,
|
|
2741
|
-
pollIntervalMs: this.batchConfig.pollIntervalMs,
|
|
2742
|
-
timeoutMs: this.batchConfig.timeoutMs,
|
|
2743
|
-
concurrency: this.batchConfig.concurrency,
|
|
2744
|
-
debug: this.debug
|
|
2745
|
-
});
|
|
2746
|
-
return texts.map((_, i) => results.get(`chunk-${i}`) ?? []);
|
|
2747
|
-
}
|
|
2748
|
-
throw new Error("Batch API not available for local embeddings");
|
|
2749
|
-
}
|
|
2750
|
-
async embedQueryWithTimeout(text) {
|
|
2751
|
-
const timeout = this.provider.id === "local" ? EMBEDDING_QUERY_TIMEOUT_LOCAL_MS : EMBEDDING_QUERY_TIMEOUT_REMOTE_MS;
|
|
2752
|
-
const ac = new AbortController();
|
|
2753
|
-
const timer = setTimeout(() => ac.abort(), timeout);
|
|
2754
|
-
try {
|
|
2755
|
-
const result = await Promise.race([
|
|
2756
|
-
this.provider.embedQuery(text),
|
|
2757
|
-
new Promise((_, reject) => {
|
|
2758
|
-
ac.signal.addEventListener(
|
|
2759
|
-
"abort",
|
|
2760
|
-
() => reject(new Error("embedding query timeout"))
|
|
2761
|
-
);
|
|
2762
|
-
})
|
|
2763
|
-
]);
|
|
2764
|
-
return result;
|
|
2765
|
-
} finally {
|
|
2766
|
-
clearTimeout(timer);
|
|
2767
|
-
}
|
|
2768
|
-
}
|
|
2769
|
-
loadEmbeddingCache(hashes) {
|
|
2770
|
-
const result = /* @__PURE__ */ new Map();
|
|
2771
|
-
if (!this.cache.enabled || hashes.length === 0) return result;
|
|
2772
|
-
const placeholders = hashes.map(() => "?").join(",");
|
|
2773
|
-
const rows = this.db.prepare(
|
|
2774
|
-
`SELECT hash, embedding FROM ${EMBEDDING_CACHE_TABLE}
|
|
2775
|
-
WHERE provider = ? AND model = ? AND provider_key = ? AND hash IN (${placeholders})`
|
|
2776
|
-
).all(this.provider.id, this.provider.model, this.providerKey, ...hashes);
|
|
2777
|
-
const now = Date.now();
|
|
2778
|
-
for (const row of rows) {
|
|
2779
|
-
result.set(row.hash, parseEmbedding(row.embedding));
|
|
2780
|
-
this.db.prepare(
|
|
2781
|
-
`UPDATE ${EMBEDDING_CACHE_TABLE} SET updated_at = ?
|
|
2782
|
-
WHERE provider = ? AND model = ? AND provider_key = ? AND hash = ?`
|
|
2783
|
-
).run(now, this.provider.id, this.provider.model, this.providerKey, row.hash);
|
|
2784
|
-
}
|
|
2785
|
-
return result;
|
|
2786
|
-
}
|
|
2787
|
-
upsertEmbeddingCache(hash, embedding) {
|
|
2788
|
-
if (!this.cache.enabled) return;
|
|
2789
|
-
const now = Date.now();
|
|
2790
|
-
this.db.prepare(
|
|
2791
|
-
`INSERT OR REPLACE INTO ${EMBEDDING_CACHE_TABLE}
|
|
2792
|
-
(provider, model, provider_key, hash, embedding, dims, updated_at)
|
|
2793
|
-
VALUES (?, ?, ?, ?, ?, ?, ?)`
|
|
2794
|
-
).run(
|
|
2795
|
-
this.provider.id,
|
|
2796
|
-
this.provider.model,
|
|
2797
|
-
this.providerKey,
|
|
2798
|
-
hash,
|
|
2799
|
-
JSON.stringify(embedding),
|
|
2800
|
-
embedding.length,
|
|
2801
|
-
now
|
|
2802
|
-
);
|
|
2803
|
-
}
|
|
2804
|
-
pruneEmbeddingCacheIfNeeded() {
|
|
2805
|
-
if (!this.cache.enabled) return;
|
|
2806
|
-
const row = this.db.prepare(`SELECT COUNT(*) as count FROM ${EMBEDDING_CACHE_TABLE}`).get();
|
|
2807
|
-
if (row.count <= this.cache.maxEntries) return;
|
|
2808
|
-
const excess = row.count - this.cache.maxEntries;
|
|
2809
|
-
this.db.prepare(
|
|
2810
|
-
`DELETE FROM ${EMBEDDING_CACHE_TABLE}
|
|
2811
|
-
WHERE rowid IN (
|
|
2812
|
-
SELECT rowid FROM ${EMBEDDING_CACHE_TABLE}
|
|
2813
|
-
ORDER BY updated_at ASC
|
|
2814
|
-
LIMIT ?
|
|
2815
|
-
)`
|
|
2816
|
-
).run(excess);
|
|
2817
|
-
}
|
|
2818
|
-
async ensureVectorReady(dimensions) {
|
|
2819
|
-
if (this.vector.available === true) return true;
|
|
2820
|
-
if (this.vector.available === false) return false;
|
|
2821
|
-
if (!this.vectorReady) {
|
|
2822
|
-
this.vectorReady = this.loadVectorExtension();
|
|
2823
|
-
}
|
|
2824
|
-
const ready = await this.vectorReady;
|
|
2825
|
-
if (ready && dimensions && !this.vector.dims) {
|
|
2826
|
-
this.vector.dims = dimensions;
|
|
2827
|
-
this.ensureVectorTable(dimensions);
|
|
2828
|
-
}
|
|
2829
|
-
return ready;
|
|
2830
|
-
}
|
|
2831
|
-
async loadVectorExtension() {
|
|
2832
|
-
const result = await loadSqliteVecExtension({
|
|
2833
|
-
db: this.db,
|
|
2834
|
-
extensionPath: this.vectorExtensionPath
|
|
2835
|
-
});
|
|
2836
|
-
this.vector.available = result.ok;
|
|
2837
|
-
if (result.error) {
|
|
2838
|
-
this.vector.loadError = result.error;
|
|
2839
|
-
this.debug?.(`sqlite-vec load failed: ${result.error}`);
|
|
2840
|
-
}
|
|
2841
|
-
if (result.extensionPath) {
|
|
2842
|
-
this.vector.extensionPath = result.extensionPath;
|
|
2843
|
-
}
|
|
2844
|
-
return result.ok;
|
|
340
|
+
}
|
|
341
|
+
function formatPath(filePath) {
|
|
342
|
+
const home = os2.homedir();
|
|
343
|
+
if (filePath.startsWith(home)) {
|
|
344
|
+
return "~" + filePath.slice(home.length);
|
|
2845
345
|
}
|
|
2846
|
-
|
|
2847
|
-
|
|
2848
|
-
|
|
2849
|
-
|
|
2850
|
-
|
|
2851
|
-
|
|
2852
|
-
|
|
2853
|
-
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
346
|
+
return filePath;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// src/store/manifest.ts
|
|
350
|
+
import fs3 from "fs/promises";
|
|
351
|
+
import fsSync2 from "fs";
|
|
352
|
+
import path3 from "path";
|
|
353
|
+
import os3 from "os";
|
|
354
|
+
var GLOBAL_MANIFEST_PATH = path3.join(
|
|
355
|
+
os3.homedir(),
|
|
356
|
+
".config",
|
|
357
|
+
"minimem",
|
|
358
|
+
"stores.json"
|
|
359
|
+
);
|
|
360
|
+
var LINKS_FILENAME = "links.json";
|
|
361
|
+
async function loadManifest(manifestPath) {
|
|
362
|
+
const filePath = manifestPath ?? GLOBAL_MANIFEST_PATH;
|
|
363
|
+
try {
|
|
364
|
+
const content = await fs3.readFile(filePath, "utf-8");
|
|
365
|
+
const parsed = JSON.parse(content);
|
|
366
|
+
for (const [name, def] of Object.entries(parsed.stores ?? {})) {
|
|
367
|
+
parsed.stores[name] = {
|
|
368
|
+
...def,
|
|
369
|
+
path: expandHome(def.path)
|
|
370
|
+
};
|
|
2857
371
|
}
|
|
372
|
+
return { stores: parsed.stores ?? {} };
|
|
373
|
+
} catch {
|
|
374
|
+
return { stores: {} };
|
|
2858
375
|
}
|
|
2859
|
-
|
|
2860
|
-
|
|
376
|
+
}
|
|
377
|
+
async function saveManifest(manifest, manifestPath) {
|
|
378
|
+
const filePath = manifestPath ?? GLOBAL_MANIFEST_PATH;
|
|
379
|
+
await fs3.mkdir(path3.dirname(filePath), { recursive: true });
|
|
380
|
+
await fs3.writeFile(filePath, JSON.stringify(manifest, null, 2), "utf-8");
|
|
381
|
+
}
|
|
382
|
+
async function loadStoreLinks(memoryDir) {
|
|
383
|
+
const candidates = [
|
|
384
|
+
path3.join(memoryDir, ".minimem", LINKS_FILENAME),
|
|
385
|
+
path3.join(memoryDir, ".swarm", "minimem", LINKS_FILENAME),
|
|
386
|
+
path3.join(memoryDir, LINKS_FILENAME)
|
|
387
|
+
];
|
|
388
|
+
for (const candidate of candidates) {
|
|
2861
389
|
try {
|
|
2862
|
-
|
|
390
|
+
const content = await fs3.readFile(candidate, "utf-8");
|
|
391
|
+
const parsed = JSON.parse(content);
|
|
392
|
+
return { links: parsed.links ?? [] };
|
|
2863
393
|
} catch {
|
|
2864
|
-
|
|
394
|
+
continue;
|
|
2865
395
|
}
|
|
2866
396
|
}
|
|
2867
|
-
|
|
2868
|
-
|
|
2869
|
-
|
|
2870
|
-
|
|
2871
|
-
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
|
|
2875
|
-
|
|
2876
|
-
|
|
2877
|
-
|
|
2878
|
-
|
|
2879
|
-
|
|
2880
|
-
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
|
|
2893
|
-
|
|
2894
|
-
|
|
2895
|
-
|
|
2896
|
-
|
|
2897
|
-
|
|
2898
|
-
* Append content to a memory file (creates if doesn't exist)
|
|
2899
|
-
*/
|
|
2900
|
-
async appendFile(relativePath, content) {
|
|
2901
|
-
this.validateMemoryPath(relativePath);
|
|
2902
|
-
const absPath = path6.join(this.memoryDir, relativePath);
|
|
2903
|
-
const dir = path6.dirname(absPath);
|
|
2904
|
-
await fs4.mkdir(dir, { recursive: true });
|
|
2905
|
-
let toAppend = content;
|
|
2906
|
-
try {
|
|
2907
|
-
const existing = await fs4.readFile(absPath, "utf-8");
|
|
2908
|
-
if (existing.length > 0 && !existing.endsWith("\n")) {
|
|
2909
|
-
toAppend = "\n" + content;
|
|
2910
|
-
}
|
|
2911
|
-
} catch {
|
|
397
|
+
return { links: [] };
|
|
398
|
+
}
|
|
399
|
+
async function saveStoreLinks(memoryDir, links) {
|
|
400
|
+
const candidates = [
|
|
401
|
+
path3.join(memoryDir, ".minimem"),
|
|
402
|
+
path3.join(memoryDir, ".swarm", "minimem"),
|
|
403
|
+
memoryDir
|
|
404
|
+
// contained layout
|
|
405
|
+
];
|
|
406
|
+
let targetDir = candidates[0];
|
|
407
|
+
for (const candidate of candidates) {
|
|
408
|
+
if (fsSync2.existsSync(candidate)) {
|
|
409
|
+
targetDir = candidate;
|
|
410
|
+
break;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
await fs3.mkdir(targetDir, { recursive: true });
|
|
414
|
+
await fs3.writeFile(
|
|
415
|
+
path3.join(targetDir, LINKS_FILENAME),
|
|
416
|
+
JSON.stringify(links, null, 2),
|
|
417
|
+
"utf-8"
|
|
418
|
+
);
|
|
419
|
+
}
|
|
420
|
+
function resolveStore(manifest, storeName) {
|
|
421
|
+
return manifest.stores[storeName] ?? null;
|
|
422
|
+
}
|
|
423
|
+
function resolveStoreName(manifest, dirPath) {
|
|
424
|
+
const resolved = path3.resolve(dirPath);
|
|
425
|
+
for (const [name, def] of Object.entries(manifest.stores)) {
|
|
426
|
+
if (path3.resolve(def.path) === resolved) {
|
|
427
|
+
return name;
|
|
2912
428
|
}
|
|
2913
|
-
await fs4.appendFile(absPath, toAppend, "utf-8");
|
|
2914
|
-
this.dirty = true;
|
|
2915
|
-
this.debug?.(`memory append: ${relativePath}`);
|
|
2916
429
|
}
|
|
2917
|
-
|
|
2918
|
-
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
|
|
430
|
+
return null;
|
|
431
|
+
}
|
|
432
|
+
async function getLinkedStoreNames(manifest, storeName) {
|
|
433
|
+
const storeDef = manifest.stores[storeName];
|
|
434
|
+
if (!storeDef) return [];
|
|
435
|
+
const storeLinks = await loadStoreLinks(storeDef.path);
|
|
436
|
+
return [...new Set(storeLinks.links)];
|
|
437
|
+
}
|
|
438
|
+
function getManifestPath() {
|
|
439
|
+
return GLOBAL_MANIFEST_PATH;
|
|
440
|
+
}
|
|
441
|
+
function expandHome(filePath) {
|
|
442
|
+
if (filePath.startsWith("~/")) {
|
|
443
|
+
return path3.join(os3.homedir(), filePath.slice(2));
|
|
2925
444
|
}
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
445
|
+
return filePath;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
// src/store/materialize.ts
|
|
449
|
+
import fs4 from "fs/promises";
|
|
450
|
+
import fsSync3 from "fs";
|
|
451
|
+
import path4 from "path";
|
|
452
|
+
import os4 from "os";
|
|
453
|
+
import { execFile } from "child_process";
|
|
454
|
+
import { promisify } from "util";
|
|
455
|
+
var execFileAsync = promisify(execFile);
|
|
456
|
+
var CACHE_BASE = path4.join(os4.homedir(), ".cache", "minimem", "stores");
|
|
457
|
+
async function materializeStore(storeName, storeDef, opts) {
|
|
458
|
+
if (fsSync3.existsSync(storeDef.path)) {
|
|
459
|
+
return materializeLocal(storeName, storeDef.path);
|
|
2932
460
|
}
|
|
2933
|
-
|
|
2934
|
-
|
|
2935
|
-
|
|
2936
|
-
|
|
2937
|
-
|
|
2938
|
-
|
|
2939
|
-
|
|
461
|
+
if (storeDef.remote) {
|
|
462
|
+
return materializeRemote(storeName, storeDef, opts?.refresh ?? false);
|
|
463
|
+
}
|
|
464
|
+
return null;
|
|
465
|
+
}
|
|
466
|
+
async function materializeLocal(storeName, storePath) {
|
|
467
|
+
const tmpBase = await fs4.mkdtemp(
|
|
468
|
+
path4.join(os4.tmpdir(), "minimem-stores-")
|
|
469
|
+
);
|
|
470
|
+
const symlinkDir = path4.join(tmpBase, storeName);
|
|
471
|
+
await fs4.symlink(storePath, symlinkDir, "dir");
|
|
472
|
+
return {
|
|
473
|
+
// The Minimem instance should use the original path for its DB,
|
|
474
|
+
// but the symlink path can be used for discovery/resolution.
|
|
475
|
+
// We return the original path since Minimem works directly with it.
|
|
476
|
+
path: storePath,
|
|
477
|
+
strategy: "symlink",
|
|
478
|
+
cleanup: async () => {
|
|
479
|
+
try {
|
|
480
|
+
await fs4.unlink(symlinkDir);
|
|
481
|
+
await fs4.rmdir(tmpBase);
|
|
482
|
+
} catch {
|
|
483
|
+
}
|
|
2940
484
|
}
|
|
2941
|
-
|
|
2942
|
-
|
|
2943
|
-
|
|
485
|
+
};
|
|
486
|
+
}
|
|
487
|
+
async function materializeRemote(storeName, storeDef, refresh) {
|
|
488
|
+
const cacheDir = path4.join(CACHE_BASE, storeName);
|
|
489
|
+
try {
|
|
490
|
+
if (fsSync3.existsSync(path4.join(cacheDir, ".git"))) {
|
|
491
|
+
if (refresh) {
|
|
492
|
+
await gitFetch(cacheDir);
|
|
2944
493
|
}
|
|
2945
|
-
|
|
494
|
+
} else {
|
|
495
|
+
await gitClone(storeDef.remote, cacheDir);
|
|
2946
496
|
}
|
|
2947
|
-
throw new Error(
|
|
2948
|
-
`Invalid memory path: ${relativePath}. Must be MEMORY.md or memory/*.md`
|
|
2949
|
-
);
|
|
2950
|
-
}
|
|
2951
|
-
async status() {
|
|
2952
|
-
const fileRow = this.db.prepare(`SELECT COUNT(*) as count FROM files`).get();
|
|
2953
|
-
const chunkRow = this.db.prepare(`SELECT COUNT(*) as count FROM chunks`).get();
|
|
2954
|
-
const cacheRow = this.db.prepare(`SELECT COUNT(*) as count FROM ${EMBEDDING_CACHE_TABLE}`).get();
|
|
2955
497
|
return {
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
vectorAvailable: this.vector.available === true,
|
|
2961
|
-
ftsAvailable: this.fts.available,
|
|
2962
|
-
bm25Only: this.provider.id === "none",
|
|
2963
|
-
fallbackReason: this.providerFallbackReason,
|
|
2964
|
-
fileCount: fileRow.count,
|
|
2965
|
-
chunkCount: chunkRow.count,
|
|
2966
|
-
cacheCount: cacheRow.count
|
|
498
|
+
path: cacheDir,
|
|
499
|
+
strategy: "remote",
|
|
500
|
+
cleanup: async () => {
|
|
501
|
+
}
|
|
2967
502
|
};
|
|
503
|
+
} catch {
|
|
504
|
+
return null;
|
|
2968
505
|
}
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
2992
|
-
|
|
2993
|
-
|
|
2994
|
-
|
|
2995
|
-
|
|
2996
|
-
|
|
2997
|
-
|
|
2998
|
-
|
|
2999
|
-
|
|
3000
|
-
|
|
3001
|
-
|
|
3002
|
-
|
|
3003
|
-
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
}
|
|
3009
|
-
}
|
|
3010
|
-
return filtered;
|
|
506
|
+
}
|
|
507
|
+
async function gitClone(remote, targetDir) {
|
|
508
|
+
await fs4.mkdir(path4.dirname(targetDir), { recursive: true });
|
|
509
|
+
await execFileAsync("git", ["clone", "--depth", "1", remote, targetDir], {
|
|
510
|
+
timeout: 6e4
|
|
511
|
+
});
|
|
512
|
+
}
|
|
513
|
+
async function gitFetch(cacheDir) {
|
|
514
|
+
await execFileAsync("git", ["fetch", "--depth", "1", "origin"], {
|
|
515
|
+
cwd: cacheDir,
|
|
516
|
+
timeout: 6e4
|
|
517
|
+
});
|
|
518
|
+
await execFileAsync("git", ["reset", "--hard", "origin/HEAD"], {
|
|
519
|
+
cwd: cacheDir,
|
|
520
|
+
timeout: 3e4
|
|
521
|
+
});
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
// src/cli/commands/init.ts
|
|
525
|
+
var MEMORY_TEMPLATE = `# Memory
|
|
526
|
+
|
|
527
|
+
This is your memory file. Add notes, decisions, and context here.
|
|
528
|
+
|
|
529
|
+
## Quick Start
|
|
530
|
+
|
|
531
|
+
- Add daily logs in the \`memory/\` directory (e.g., \`memory/2024-01-15.md\`)
|
|
532
|
+
- Use \`minimem search <query>\` to find relevant memories
|
|
533
|
+
- Use \`minimem append <text>\` to quickly add to today's log
|
|
534
|
+
|
|
535
|
+
## Notes
|
|
536
|
+
|
|
537
|
+
`;
|
|
538
|
+
async function init(dir, options) {
|
|
539
|
+
const memoryDir = resolveMemoryDir({ dir, global: options.global });
|
|
540
|
+
const displayPath = formatPath(memoryDir);
|
|
541
|
+
if (!options.force && await isInitialized(memoryDir)) {
|
|
542
|
+
console.log(`Already initialized: ${displayPath}`);
|
|
543
|
+
console.log("Use --force to reinitialize");
|
|
544
|
+
return;
|
|
3011
545
|
}
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3019
|
-
|
|
546
|
+
console.log(`Initializing minimem in ${displayPath}...`);
|
|
547
|
+
await fs5.mkdir(memoryDir, { recursive: true });
|
|
548
|
+
await fs5.mkdir(path5.join(memoryDir, "memory"), { recursive: true });
|
|
549
|
+
const memoryFilePath = path5.join(memoryDir, "MEMORY.md");
|
|
550
|
+
try {
|
|
551
|
+
await fs5.access(memoryFilePath);
|
|
552
|
+
console.log(" MEMORY.md already exists, skipping");
|
|
553
|
+
} catch {
|
|
554
|
+
await fs5.writeFile(memoryFilePath, MEMORY_TEMPLATE, "utf-8");
|
|
555
|
+
console.log(" Created MEMORY.md");
|
|
3020
556
|
}
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
557
|
+
const config2 = getInitConfig();
|
|
558
|
+
const configPath = path5.join(memoryDir, "config.json");
|
|
559
|
+
await fs5.writeFile(configPath, JSON.stringify(config2, null, 2), "utf-8");
|
|
560
|
+
console.log(" Created config.json");
|
|
561
|
+
const gitignorePath = path5.join(memoryDir, ".gitignore");
|
|
562
|
+
await fs5.writeFile(gitignorePath, "index.db\nindex.db-*\n", "utf-8");
|
|
563
|
+
console.log(" Created .gitignore");
|
|
564
|
+
await materializeLinkedStores(memoryDir);
|
|
565
|
+
if (!options.skipSync) {
|
|
566
|
+
await initialSync(memoryDir);
|
|
3026
567
|
}
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
568
|
+
console.log();
|
|
569
|
+
console.log("Done! Your memory directory is ready.");
|
|
570
|
+
console.log(`Search your memories with: minimem search "your query"${dir ? ` --dir ${dir}` : ""}`);
|
|
571
|
+
}
|
|
572
|
+
async function initialSync(memoryDir) {
|
|
573
|
+
console.log();
|
|
574
|
+
console.log("Indexing memory files...");
|
|
575
|
+
const cliConfig = await loadConfig(memoryDir);
|
|
576
|
+
const config2 = buildMinimemConfig(memoryDir, cliConfig, {
|
|
577
|
+
watch: false
|
|
578
|
+
});
|
|
579
|
+
const { Minimem: Minimem2 } = await import("./minimem-MQXSBGNG.js");
|
|
580
|
+
let minimem = null;
|
|
581
|
+
try {
|
|
582
|
+
minimem = await Minimem2.create(config2);
|
|
583
|
+
await minimem.sync({ force: false });
|
|
584
|
+
const status2 = await minimem.status();
|
|
585
|
+
console.log(` Indexed ${status2.fileCount} file(s), ${status2.chunkCount} chunk(s)`);
|
|
586
|
+
} catch (err) {
|
|
587
|
+
console.log(" Skipped indexing (set an embedding API key to enable)");
|
|
588
|
+
} finally {
|
|
589
|
+
minimem?.close();
|
|
3032
590
|
}
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3040
|
-
|
|
3041
|
-
|
|
3042
|
-
|
|
591
|
+
}
|
|
592
|
+
async function materializeLinkedStores(memoryDir) {
|
|
593
|
+
try {
|
|
594
|
+
const manifest = await loadManifest();
|
|
595
|
+
if (Object.keys(manifest.stores).length === 0) return;
|
|
596
|
+
const storeName = resolveStoreName(manifest, memoryDir);
|
|
597
|
+
if (!storeName) return;
|
|
598
|
+
const linkedNames = await getLinkedStoreNames(manifest, storeName);
|
|
599
|
+
if (linkedNames.length === 0) return;
|
|
600
|
+
let materialized = 0;
|
|
601
|
+
for (const linkedName of linkedNames) {
|
|
602
|
+
const linkedDef = resolveStore(manifest, linkedName);
|
|
603
|
+
if (!linkedDef) continue;
|
|
604
|
+
const result = await materializeStore(linkedName, linkedDef);
|
|
605
|
+
if (result) {
|
|
606
|
+
if (result.strategy === "remote") {
|
|
607
|
+
console.log(` Materialized linked store "${linkedName}" from remote`);
|
|
608
|
+
}
|
|
609
|
+
await result.cleanup();
|
|
610
|
+
materialized++;
|
|
611
|
+
}
|
|
3043
612
|
}
|
|
3044
|
-
|
|
3045
|
-
|
|
3046
|
-
} catch (err) {
|
|
3047
|
-
logError2("dbClose", err, this.debug);
|
|
613
|
+
if (materialized > 0) {
|
|
614
|
+
console.log(` ${materialized} linked store(s) ready`);
|
|
3048
615
|
}
|
|
616
|
+
} catch {
|
|
3049
617
|
}
|
|
3050
|
-
}
|
|
618
|
+
}
|
|
3051
619
|
|
|
3052
620
|
// src/cli/commands/search.ts
|
|
3053
621
|
async function search(query, options) {
|
|
@@ -3069,13 +637,14 @@ async function search(query, options) {
|
|
|
3069
637
|
if (validDirs.length === 0) {
|
|
3070
638
|
exitWithError("No valid initialized memory directories found.");
|
|
3071
639
|
}
|
|
640
|
+
const { dirs: allDirs, cleanups } = options.noLinks ? { dirs: validDirs, cleanups: [] } : await resolveLinkedDirs(validDirs);
|
|
3072
641
|
const maxResults = options.max ? parseInt(options.max, 10) : 10;
|
|
3073
642
|
const minScore = options.minScore ? parseFloat(options.minScore) : void 0;
|
|
3074
643
|
const allResults = [];
|
|
3075
644
|
const instances = [];
|
|
3076
645
|
let warnedBm25 = false;
|
|
3077
646
|
try {
|
|
3078
|
-
for (const memoryDir of
|
|
647
|
+
for (const memoryDir of allDirs) {
|
|
3079
648
|
const cliConfig = await loadConfig(memoryDir);
|
|
3080
649
|
const config2 = buildMinimemConfig(memoryDir, cliConfig, {
|
|
3081
650
|
provider: options.provider,
|
|
@@ -3110,7 +679,7 @@ async function search(query, options) {
|
|
|
3110
679
|
console.log(JSON.stringify(topResults, null, 2));
|
|
3111
680
|
return;
|
|
3112
681
|
}
|
|
3113
|
-
const showSource =
|
|
682
|
+
const showSource = allDirs.length > 1;
|
|
3114
683
|
for (const result of topResults) {
|
|
3115
684
|
const score = (result.score * 100).toFixed(1);
|
|
3116
685
|
const location = `${result.path}:${result.startLine}-${result.endLine}`;
|
|
@@ -3123,12 +692,46 @@ async function search(query, options) {
|
|
|
3123
692
|
console.log(formatSnippet(result.snippet));
|
|
3124
693
|
console.log();
|
|
3125
694
|
}
|
|
3126
|
-
const dirSummary =
|
|
695
|
+
const dirSummary = allDirs.length > 1 ? ` across ${allDirs.length} directories` : "";
|
|
3127
696
|
console.log(`Found ${topResults.length} result${topResults.length === 1 ? "" : "s"}${dirSummary}`);
|
|
3128
697
|
} finally {
|
|
3129
698
|
for (const instance of instances) {
|
|
3130
699
|
instance.close();
|
|
3131
700
|
}
|
|
701
|
+
for (const cleanup of cleanups) {
|
|
702
|
+
await cleanup().catch(() => {
|
|
703
|
+
});
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
async function resolveLinkedDirs(dirs) {
|
|
708
|
+
const cleanups = [];
|
|
709
|
+
try {
|
|
710
|
+
const manifest = await loadManifest();
|
|
711
|
+
if (Object.keys(manifest.stores).length === 0) {
|
|
712
|
+
return { dirs, cleanups };
|
|
713
|
+
}
|
|
714
|
+
const allDirs = new Set(dirs);
|
|
715
|
+
for (const dir of dirs) {
|
|
716
|
+
const storeName = resolveStoreName(manifest, dir);
|
|
717
|
+
if (!storeName) continue;
|
|
718
|
+
const linkedNames = await getLinkedStoreNames(manifest, storeName);
|
|
719
|
+
for (const linkedName of linkedNames) {
|
|
720
|
+
const linkedDef = resolveStore(manifest, linkedName);
|
|
721
|
+
if (!linkedDef) continue;
|
|
722
|
+
try {
|
|
723
|
+
const result = await materializeStore(linkedName, linkedDef);
|
|
724
|
+
if (result) {
|
|
725
|
+
allDirs.add(result.path);
|
|
726
|
+
cleanups.push(result.cleanup);
|
|
727
|
+
}
|
|
728
|
+
} catch {
|
|
729
|
+
}
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
return { dirs: [...allDirs], cleanups };
|
|
733
|
+
} catch {
|
|
734
|
+
return { dirs, cleanups };
|
|
3132
735
|
}
|
|
3133
736
|
}
|
|
3134
737
|
function formatSnippet(snippet) {
|
|
@@ -3277,8 +880,8 @@ ${text}`;
|
|
|
3277
880
|
}
|
|
3278
881
|
|
|
3279
882
|
// src/cli/commands/upsert.ts
|
|
3280
|
-
import * as
|
|
3281
|
-
import * as
|
|
883
|
+
import * as fs6 from "fs/promises";
|
|
884
|
+
import * as path6 from "path";
|
|
3282
885
|
async function upsert(file, content, options) {
|
|
3283
886
|
const memoryDir = resolveMemoryDir({ dir: options.dir, global: options.global });
|
|
3284
887
|
if (!await isInitialized(memoryDir)) {
|
|
@@ -3303,22 +906,22 @@ async function upsert(file, content, options) {
|
|
|
3303
906
|
project: process.cwd()
|
|
3304
907
|
} : void 0;
|
|
3305
908
|
const filePath = resolveFilePath(file, memoryDir);
|
|
3306
|
-
const resolvedPath =
|
|
3307
|
-
const resolvedMemoryDir =
|
|
3308
|
-
if (!resolvedPath.startsWith(resolvedMemoryDir +
|
|
909
|
+
const resolvedPath = path6.resolve(filePath);
|
|
910
|
+
const resolvedMemoryDir = path6.resolve(memoryDir);
|
|
911
|
+
if (!resolvedPath.startsWith(resolvedMemoryDir + path6.sep) && resolvedPath !== resolvedMemoryDir) {
|
|
3309
912
|
exitWithError(
|
|
3310
913
|
"File path must be within the memory directory.",
|
|
3311
914
|
`Memory dir: ${formatPath(memoryDir)}, File: ${formatPath(filePath)}`
|
|
3312
915
|
);
|
|
3313
916
|
}
|
|
3314
|
-
const parentDir =
|
|
3315
|
-
await
|
|
917
|
+
const parentDir = path6.dirname(filePath);
|
|
918
|
+
await fs6.mkdir(parentDir, { recursive: true });
|
|
3316
919
|
let isUpdate = false;
|
|
3317
920
|
let existingContent;
|
|
3318
921
|
try {
|
|
3319
|
-
await
|
|
922
|
+
await fs6.access(filePath);
|
|
3320
923
|
isUpdate = true;
|
|
3321
|
-
existingContent = await
|
|
924
|
+
existingContent = await fs6.readFile(filePath, "utf-8");
|
|
3322
925
|
} catch {
|
|
3323
926
|
}
|
|
3324
927
|
let contentToWrite = finalContent;
|
|
@@ -3346,8 +949,8 @@ async function upsert(file, content, options) {
|
|
|
3346
949
|
});
|
|
3347
950
|
}
|
|
3348
951
|
}
|
|
3349
|
-
await
|
|
3350
|
-
const relativePath =
|
|
952
|
+
await fs6.writeFile(filePath, contentToWrite, "utf-8");
|
|
953
|
+
const relativePath = path6.relative(memoryDir, filePath);
|
|
3351
954
|
const action = isUpdate ? "Updated" : "Created";
|
|
3352
955
|
console.log(`${action}: ${relativePath}`);
|
|
3353
956
|
console.log(` in ${formatPath(memoryDir)}`);
|
|
@@ -3371,10 +974,10 @@ async function upsert(file, content, options) {
|
|
|
3371
974
|
}
|
|
3372
975
|
}
|
|
3373
976
|
function resolveFilePath(file, memoryDir) {
|
|
3374
|
-
if (
|
|
977
|
+
if (path6.isAbsolute(file)) {
|
|
3375
978
|
return file;
|
|
3376
979
|
}
|
|
3377
|
-
return
|
|
980
|
+
return path6.join(memoryDir, file);
|
|
3378
981
|
}
|
|
3379
982
|
async function readStdin() {
|
|
3380
983
|
const chunks = [];
|
|
@@ -3386,8 +989,8 @@ async function readStdin() {
|
|
|
3386
989
|
}
|
|
3387
990
|
|
|
3388
991
|
// src/cli/commands/mcp.ts
|
|
3389
|
-
import * as
|
|
3390
|
-
import * as
|
|
992
|
+
import * as fs7 from "fs/promises";
|
|
993
|
+
import * as path7 from "path";
|
|
3391
994
|
|
|
3392
995
|
// src/server/mcp.ts
|
|
3393
996
|
import * as readline from "readline";
|
|
@@ -4019,9 +1622,10 @@ async function mcp(options) {
|
|
|
4019
1622
|
if (includesGlobal && !await isInitialized(globalDir)) {
|
|
4020
1623
|
await ensureGlobalInitialized(globalDir);
|
|
4021
1624
|
}
|
|
1625
|
+
const expandedDirs = await resolveLinkedDirsForMcp(directories);
|
|
4022
1626
|
const instances = [];
|
|
4023
1627
|
const minimemInstances = [];
|
|
4024
|
-
for (const memoryDir of
|
|
1628
|
+
for (const memoryDir of expandedDirs) {
|
|
4025
1629
|
const isGlobal = memoryDir === globalDir;
|
|
4026
1630
|
if (!await isInitialized(memoryDir)) {
|
|
4027
1631
|
warn(`${formatPath(memoryDir)} is not initialized, skipping.`);
|
|
@@ -4083,12 +1687,11 @@ async function mcp(options) {
|
|
|
4083
1687
|
}
|
|
4084
1688
|
async function ensureGlobalInitialized(globalDir) {
|
|
4085
1689
|
console.error(`Auto-initializing global memory directory (~/.minimem)...`);
|
|
4086
|
-
await
|
|
4087
|
-
await
|
|
4088
|
-
|
|
4089
|
-
const memoryFilePath = path8.join(globalDir, "MEMORY.md");
|
|
1690
|
+
await fs7.mkdir(globalDir, { recursive: true });
|
|
1691
|
+
await fs7.mkdir(path7.join(globalDir, "memory"), { recursive: true });
|
|
1692
|
+
const memoryFilePath = path7.join(globalDir, "MEMORY.md");
|
|
4090
1693
|
try {
|
|
4091
|
-
await
|
|
1694
|
+
await fs7.access(memoryFilePath);
|
|
4092
1695
|
} catch {
|
|
4093
1696
|
const template = `# Global Memory
|
|
4094
1697
|
|
|
@@ -4099,14 +1702,41 @@ Notes stored here are available across all projects.
|
|
|
4099
1702
|
## Notes
|
|
4100
1703
|
|
|
4101
1704
|
`;
|
|
4102
|
-
await
|
|
1705
|
+
await fs7.writeFile(memoryFilePath, template, "utf-8");
|
|
4103
1706
|
}
|
|
4104
1707
|
const config2 = getInitConfig();
|
|
4105
|
-
|
|
4106
|
-
|
|
4107
|
-
|
|
1708
|
+
const configPath = path7.join(globalDir, "config.json");
|
|
1709
|
+
await fs7.writeFile(configPath, JSON.stringify(config2, null, 2), "utf-8");
|
|
1710
|
+
const gitignorePath = path7.join(globalDir, ".gitignore");
|
|
1711
|
+
await fs7.writeFile(gitignorePath, "index.db\nindex.db-*\n", "utf-8");
|
|
4108
1712
|
console.error(` Created ~/.minimem with default configuration.`);
|
|
4109
1713
|
}
|
|
1714
|
+
async function resolveLinkedDirsForMcp(dirs) {
|
|
1715
|
+
try {
|
|
1716
|
+
const manifest = await loadManifest();
|
|
1717
|
+
if (Object.keys(manifest.stores).length === 0) return dirs;
|
|
1718
|
+
const allDirs = new Set(dirs);
|
|
1719
|
+
for (const dir of dirs) {
|
|
1720
|
+
const storeName = resolveStoreName(manifest, dir);
|
|
1721
|
+
if (!storeName) continue;
|
|
1722
|
+
const linkedNames = await getLinkedStoreNames(manifest, storeName);
|
|
1723
|
+
for (const linkedName of linkedNames) {
|
|
1724
|
+
const linkedDef = resolveStore(manifest, linkedName);
|
|
1725
|
+
if (!linkedDef) continue;
|
|
1726
|
+
try {
|
|
1727
|
+
const result = await materializeStore(linkedName, linkedDef);
|
|
1728
|
+
if (result) {
|
|
1729
|
+
allDirs.add(result.path);
|
|
1730
|
+
}
|
|
1731
|
+
} catch {
|
|
1732
|
+
}
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
return [...allDirs];
|
|
1736
|
+
} catch {
|
|
1737
|
+
return dirs;
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
4110
1740
|
|
|
4111
1741
|
// src/cli/commands/config.ts
|
|
4112
1742
|
async function config(options) {
|
|
@@ -4250,9 +1880,9 @@ async function handleXdgConfigEdit(options) {
|
|
|
4250
1880
|
}
|
|
4251
1881
|
}
|
|
4252
1882
|
async function loadConfigFile2(configPath) {
|
|
4253
|
-
const
|
|
1883
|
+
const fs19 = await import("fs/promises");
|
|
4254
1884
|
try {
|
|
4255
|
-
const content = await
|
|
1885
|
+
const content = await fs19.readFile(configPath, "utf-8");
|
|
4256
1886
|
return JSON.parse(content);
|
|
4257
1887
|
} catch {
|
|
4258
1888
|
return {};
|
|
@@ -4298,22 +1928,22 @@ function unsetConfigValue(config2, keyPath) {
|
|
|
4298
1928
|
}
|
|
4299
1929
|
|
|
4300
1930
|
// src/cli/commands/sync-init.ts
|
|
4301
|
-
import
|
|
4302
|
-
import
|
|
1931
|
+
import fs11 from "fs/promises";
|
|
1932
|
+
import path11 from "path";
|
|
4303
1933
|
|
|
4304
1934
|
// src/cli/sync/central.ts
|
|
4305
|
-
import
|
|
4306
|
-
import
|
|
1935
|
+
import fs10 from "fs/promises";
|
|
1936
|
+
import path10 from "path";
|
|
4307
1937
|
import { execSync } from "child_process";
|
|
4308
1938
|
|
|
4309
1939
|
// src/cli/sync/registry.ts
|
|
4310
|
-
import
|
|
4311
|
-
import
|
|
1940
|
+
import fs8 from "fs/promises";
|
|
1941
|
+
import path8 from "path";
|
|
4312
1942
|
import os5 from "os";
|
|
4313
|
-
import
|
|
1943
|
+
import crypto2 from "crypto";
|
|
4314
1944
|
var REGISTRY_FILENAME = ".minimem-registry.json";
|
|
4315
1945
|
function getRegistryPath(centralRepo) {
|
|
4316
|
-
return
|
|
1946
|
+
return path8.join(centralRepo, REGISTRY_FILENAME);
|
|
4317
1947
|
}
|
|
4318
1948
|
function createEmptyRegistry() {
|
|
4319
1949
|
return {
|
|
@@ -4324,7 +1954,7 @@ function createEmptyRegistry() {
|
|
|
4324
1954
|
async function readRegistry(centralRepo) {
|
|
4325
1955
|
const registryPath = getRegistryPath(centralRepo);
|
|
4326
1956
|
try {
|
|
4327
|
-
const content = await
|
|
1957
|
+
const content = await fs8.readFile(registryPath, "utf-8");
|
|
4328
1958
|
const registry = JSON.parse(content);
|
|
4329
1959
|
if (!registry.mappings || !Array.isArray(registry.mappings)) {
|
|
4330
1960
|
return createEmptyRegistry();
|
|
@@ -4336,23 +1966,23 @@ async function readRegistry(centralRepo) {
|
|
|
4336
1966
|
}
|
|
4337
1967
|
async function writeRegistry(centralRepo, registry) {
|
|
4338
1968
|
const registryPath = getRegistryPath(centralRepo);
|
|
4339
|
-
const tempPath = `${registryPath}.${
|
|
1969
|
+
const tempPath = `${registryPath}.${crypto2.randomBytes(4).toString("hex")}.tmp`;
|
|
4340
1970
|
registry.version = registry.version || 1;
|
|
4341
|
-
await
|
|
4342
|
-
await
|
|
1971
|
+
await fs8.writeFile(tempPath, JSON.stringify(registry, null, 2), "utf-8");
|
|
1972
|
+
await fs8.rename(tempPath, registryPath);
|
|
4343
1973
|
}
|
|
4344
1974
|
function normalizePath(filePath) {
|
|
4345
1975
|
if (filePath.startsWith("~/")) {
|
|
4346
|
-
return
|
|
1976
|
+
return path8.resolve(os5.homedir(), filePath.slice(2));
|
|
4347
1977
|
}
|
|
4348
1978
|
if (filePath === "~") {
|
|
4349
1979
|
return os5.homedir();
|
|
4350
1980
|
}
|
|
4351
|
-
return
|
|
1981
|
+
return path8.resolve(filePath);
|
|
4352
1982
|
}
|
|
4353
1983
|
function compressPath(filePath) {
|
|
4354
1984
|
const home = os5.homedir();
|
|
4355
|
-
const resolved =
|
|
1985
|
+
const resolved = path8.resolve(filePath);
|
|
4356
1986
|
if (resolved.startsWith(home)) {
|
|
4357
1987
|
return "~" + resolved.slice(home.length);
|
|
4358
1988
|
}
|
|
@@ -4417,28 +2047,28 @@ function updateLastSync(registry, centralPath, machineId) {
|
|
|
4417
2047
|
}
|
|
4418
2048
|
|
|
4419
2049
|
// src/cli/sync/detection.ts
|
|
4420
|
-
import
|
|
4421
|
-
import
|
|
2050
|
+
import fs9 from "fs/promises";
|
|
2051
|
+
import path9 from "path";
|
|
4422
2052
|
async function isInsideGitRepo(dir) {
|
|
4423
|
-
let current =
|
|
4424
|
-
const root =
|
|
2053
|
+
let current = path9.resolve(dir);
|
|
2054
|
+
const root = path9.parse(current).root;
|
|
4425
2055
|
while (current !== root) {
|
|
4426
2056
|
try {
|
|
4427
|
-
const gitPath =
|
|
4428
|
-
const stat = await
|
|
2057
|
+
const gitPath = path9.join(current, ".git");
|
|
2058
|
+
const stat = await fs9.stat(gitPath);
|
|
4429
2059
|
if (stat.isDirectory() || stat.isFile()) {
|
|
4430
2060
|
return true;
|
|
4431
2061
|
}
|
|
4432
2062
|
} catch {
|
|
4433
2063
|
}
|
|
4434
|
-
current =
|
|
2064
|
+
current = path9.dirname(current);
|
|
4435
2065
|
}
|
|
4436
2066
|
return false;
|
|
4437
2067
|
}
|
|
4438
2068
|
async function hasSyncConfig(dir) {
|
|
4439
2069
|
const configPath = getConfigPath(dir);
|
|
4440
2070
|
try {
|
|
4441
|
-
const content = await
|
|
2071
|
+
const content = await fs9.readFile(configPath, "utf-8");
|
|
4442
2072
|
const config2 = JSON.parse(content);
|
|
4443
2073
|
return config2.sync?.enabled === true || typeof config2.sync?.path === "string";
|
|
4444
2074
|
} catch {
|
|
@@ -4507,9 +2137,9 @@ minimem sync init --path <directory-name>/
|
|
|
4507
2137
|
`;
|
|
4508
2138
|
async function isWritable(dirPath) {
|
|
4509
2139
|
try {
|
|
4510
|
-
const testFile =
|
|
4511
|
-
await
|
|
4512
|
-
await
|
|
2140
|
+
const testFile = path10.join(dirPath, `.minimem-write-test-${Date.now()}`);
|
|
2141
|
+
await fs10.writeFile(testFile, "test");
|
|
2142
|
+
await fs10.unlink(testFile);
|
|
4513
2143
|
return true;
|
|
4514
2144
|
} catch {
|
|
4515
2145
|
return false;
|
|
@@ -4528,7 +2158,7 @@ async function initGitRepo(dirPath) {
|
|
|
4528
2158
|
}
|
|
4529
2159
|
async function initCentralRepo(repoPath) {
|
|
4530
2160
|
const expandedPath = expandPath(repoPath);
|
|
4531
|
-
const resolvedPath =
|
|
2161
|
+
const resolvedPath = path10.resolve(expandedPath);
|
|
4532
2162
|
if (!isGitAvailable()) {
|
|
4533
2163
|
return {
|
|
4534
2164
|
success: false,
|
|
@@ -4539,7 +2169,7 @@ async function initCentralRepo(repoPath) {
|
|
|
4539
2169
|
}
|
|
4540
2170
|
let dirExists = false;
|
|
4541
2171
|
try {
|
|
4542
|
-
const stat = await
|
|
2172
|
+
const stat = await fs10.stat(resolvedPath);
|
|
4543
2173
|
dirExists = stat.isDirectory();
|
|
4544
2174
|
} catch {
|
|
4545
2175
|
dirExists = false;
|
|
@@ -4547,7 +2177,7 @@ async function initCentralRepo(repoPath) {
|
|
|
4547
2177
|
let created = false;
|
|
4548
2178
|
if (!dirExists) {
|
|
4549
2179
|
try {
|
|
4550
|
-
await
|
|
2180
|
+
await fs10.mkdir(resolvedPath, { recursive: true });
|
|
4551
2181
|
created = true;
|
|
4552
2182
|
} catch (error) {
|
|
4553
2183
|
return {
|
|
@@ -4579,23 +2209,23 @@ async function initCentralRepo(repoPath) {
|
|
|
4579
2209
|
};
|
|
4580
2210
|
}
|
|
4581
2211
|
}
|
|
4582
|
-
const gitignorePath =
|
|
2212
|
+
const gitignorePath = path10.join(resolvedPath, ".gitignore");
|
|
4583
2213
|
try {
|
|
4584
|
-
await
|
|
2214
|
+
await fs10.access(gitignorePath);
|
|
4585
2215
|
} catch {
|
|
4586
|
-
await
|
|
2216
|
+
await fs10.writeFile(gitignorePath, GITIGNORE_CONTENT, "utf-8");
|
|
4587
2217
|
}
|
|
4588
2218
|
const registryPath = getRegistryPath(resolvedPath);
|
|
4589
2219
|
try {
|
|
4590
|
-
await
|
|
2220
|
+
await fs10.access(registryPath);
|
|
4591
2221
|
} catch {
|
|
4592
2222
|
await writeRegistry(resolvedPath, createEmptyRegistry());
|
|
4593
2223
|
}
|
|
4594
|
-
const readmePath =
|
|
2224
|
+
const readmePath = path10.join(resolvedPath, "README.md");
|
|
4595
2225
|
try {
|
|
4596
|
-
await
|
|
2226
|
+
await fs10.access(readmePath);
|
|
4597
2227
|
} catch {
|
|
4598
|
-
await
|
|
2228
|
+
await fs10.writeFile(readmePath, README_CONTENT, "utf-8");
|
|
4599
2229
|
}
|
|
4600
2230
|
const globalConfig = await loadXdgConfig();
|
|
4601
2231
|
globalConfig.centralRepo = repoPath;
|
|
@@ -4609,11 +2239,11 @@ async function initCentralRepo(repoPath) {
|
|
|
4609
2239
|
}
|
|
4610
2240
|
async function validateCentralRepo(repoPath) {
|
|
4611
2241
|
const expandedPath = expandPath(repoPath);
|
|
4612
|
-
const resolvedPath =
|
|
2242
|
+
const resolvedPath = path10.resolve(expandedPath);
|
|
4613
2243
|
const warnings = [];
|
|
4614
2244
|
const errors = [];
|
|
4615
2245
|
try {
|
|
4616
|
-
const stat = await
|
|
2246
|
+
const stat = await fs10.stat(resolvedPath);
|
|
4617
2247
|
if (!stat.isDirectory()) {
|
|
4618
2248
|
errors.push("Path is not a directory");
|
|
4619
2249
|
return { valid: false, warnings, errors };
|
|
@@ -4628,7 +2258,7 @@ async function validateCentralRepo(repoPath) {
|
|
|
4628
2258
|
}
|
|
4629
2259
|
const registryPath = getRegistryPath(resolvedPath);
|
|
4630
2260
|
try {
|
|
4631
|
-
await
|
|
2261
|
+
await fs10.access(registryPath);
|
|
4632
2262
|
const registry = await readRegistry(resolvedPath);
|
|
4633
2263
|
if (!registry.mappings) {
|
|
4634
2264
|
warnings.push("Registry file is malformed");
|
|
@@ -4636,9 +2266,9 @@ async function validateCentralRepo(repoPath) {
|
|
|
4636
2266
|
} catch {
|
|
4637
2267
|
warnings.push("Registry file is missing - will be created on first sync");
|
|
4638
2268
|
}
|
|
4639
|
-
const gitignorePath =
|
|
2269
|
+
const gitignorePath = path10.join(resolvedPath, ".gitignore");
|
|
4640
2270
|
try {
|
|
4641
|
-
const gitignore = await
|
|
2271
|
+
const gitignore = await fs10.readFile(gitignorePath, "utf-8");
|
|
4642
2272
|
if (!gitignore.includes("*.db")) {
|
|
4643
2273
|
warnings.push(".gitignore does not exclude database files");
|
|
4644
2274
|
}
|
|
@@ -4657,7 +2287,7 @@ async function validateCentralRepo(repoPath) {
|
|
|
4657
2287
|
async function getCentralRepoPath() {
|
|
4658
2288
|
const globalConfig = await loadXdgConfig();
|
|
4659
2289
|
if (globalConfig.centralRepo) {
|
|
4660
|
-
return
|
|
2290
|
+
return path10.resolve(expandPath(globalConfig.centralRepo));
|
|
4661
2291
|
}
|
|
4662
2292
|
return void 0;
|
|
4663
2293
|
}
|
|
@@ -4763,9 +2393,9 @@ async function syncInit(options) {
|
|
|
4763
2393
|
});
|
|
4764
2394
|
await writeRegistry(centralRepo, updatedRegistry);
|
|
4765
2395
|
console.log(" Registered mapping in central repository");
|
|
4766
|
-
const centralDir =
|
|
2396
|
+
const centralDir = path11.join(centralRepo, centralPath);
|
|
4767
2397
|
try {
|
|
4768
|
-
await
|
|
2398
|
+
await fs11.mkdir(centralDir, { recursive: true });
|
|
4769
2399
|
} catch {
|
|
4770
2400
|
}
|
|
4771
2401
|
console.log();
|
|
@@ -4838,23 +2468,23 @@ async function syncRemove(options) {
|
|
|
4838
2468
|
console.log(`Removed mapping from central registry`);
|
|
4839
2469
|
console.log();
|
|
4840
2470
|
console.log("Note: Files in the central repository were NOT deleted.");
|
|
4841
|
-
console.log(`They remain at: ${formatPath(
|
|
2471
|
+
console.log(`They remain at: ${formatPath(path11.join(centralRepo, centralPath))}`);
|
|
4842
2472
|
}
|
|
4843
2473
|
|
|
4844
2474
|
// src/cli/sync/operations.ts
|
|
4845
|
-
import
|
|
4846
|
-
import
|
|
4847
|
-
import
|
|
2475
|
+
import fs15 from "fs/promises";
|
|
2476
|
+
import path15 from "path";
|
|
2477
|
+
import crypto4 from "crypto";
|
|
4848
2478
|
|
|
4849
2479
|
// src/cli/sync/state.ts
|
|
4850
|
-
import
|
|
4851
|
-
import
|
|
4852
|
-
import
|
|
2480
|
+
import fs12 from "fs/promises";
|
|
2481
|
+
import path12 from "path";
|
|
2482
|
+
import crypto3 from "crypto";
|
|
4853
2483
|
import { minimatch } from "minimatch";
|
|
4854
2484
|
var STATE_FILENAME = "sync-state.json";
|
|
4855
|
-
var STATE_DIR = ".minimem";
|
|
4856
2485
|
function getSyncStatePath(dir) {
|
|
4857
|
-
|
|
2486
|
+
const subdir = resolveConfigSubdir(dir);
|
|
2487
|
+
return path12.join(dir, subdir, STATE_FILENAME);
|
|
4858
2488
|
}
|
|
4859
2489
|
function createEmptySyncState(centralPath) {
|
|
4860
2490
|
return {
|
|
@@ -4868,7 +2498,7 @@ function createEmptySyncState(centralPath) {
|
|
|
4868
2498
|
async function loadSyncState(dir, centralPath) {
|
|
4869
2499
|
const statePath = getSyncStatePath(dir);
|
|
4870
2500
|
try {
|
|
4871
|
-
const content = await
|
|
2501
|
+
const content = await fs12.readFile(statePath, "utf-8");
|
|
4872
2502
|
const state = JSON.parse(content);
|
|
4873
2503
|
if (!state.files || typeof state.files !== "object") {
|
|
4874
2504
|
return createEmptySyncState(centralPath);
|
|
@@ -4888,27 +2518,28 @@ async function loadSyncState(dir, centralPath) {
|
|
|
4888
2518
|
}
|
|
4889
2519
|
async function saveSyncState(dir, state) {
|
|
4890
2520
|
const statePath = getSyncStatePath(dir);
|
|
4891
|
-
const stateDir =
|
|
4892
|
-
const tempPath = `${statePath}.${
|
|
4893
|
-
await
|
|
2521
|
+
const stateDir = path12.dirname(statePath);
|
|
2522
|
+
const tempPath = `${statePath}.${crypto3.randomBytes(4).toString("hex")}.tmp`;
|
|
2523
|
+
await fs12.mkdir(stateDir, { recursive: true });
|
|
4894
2524
|
state.version = state.version || 2;
|
|
4895
|
-
await
|
|
4896
|
-
await
|
|
2525
|
+
await fs12.writeFile(tempPath, JSON.stringify(state, null, 2), "utf-8");
|
|
2526
|
+
await fs12.rename(tempPath, statePath);
|
|
4897
2527
|
}
|
|
4898
2528
|
async function computeFileHash(filePath) {
|
|
4899
|
-
const content = await
|
|
4900
|
-
return
|
|
2529
|
+
const content = await fs12.readFile(filePath);
|
|
2530
|
+
return crypto3.createHash("sha256").update(content).digest("hex");
|
|
4901
2531
|
}
|
|
4902
2532
|
async function listSyncableFiles(dir, include, exclude) {
|
|
4903
2533
|
const files = [];
|
|
4904
|
-
async function
|
|
4905
|
-
const entries = await
|
|
2534
|
+
async function walkDir(currentDir, relativePath = "") {
|
|
2535
|
+
const entries = await fs12.readdir(currentDir, { withFileTypes: true });
|
|
4906
2536
|
for (const entry of entries) {
|
|
4907
|
-
const entryPath =
|
|
2537
|
+
const entryPath = path12.join(currentDir, entry.name);
|
|
4908
2538
|
const relPath = relativePath ? `${relativePath}/${entry.name}` : entry.name;
|
|
4909
|
-
if (entry.name === ".minimem") continue;
|
|
2539
|
+
if (entry.name === ".minimem" || entry.name === ".swarm") continue;
|
|
2540
|
+
if (entry.name === "index.db" || entry.name.startsWith("index.db-")) continue;
|
|
4910
2541
|
if (entry.isDirectory()) {
|
|
4911
|
-
await
|
|
2542
|
+
await walkDir(entryPath, relPath);
|
|
4912
2543
|
} else if (entry.isFile()) {
|
|
4913
2544
|
const matchesInclude = include.some(
|
|
4914
2545
|
(pattern) => minimatch(relPath, pattern)
|
|
@@ -4923,7 +2554,7 @@ async function listSyncableFiles(dir, include, exclude) {
|
|
|
4923
2554
|
}
|
|
4924
2555
|
}
|
|
4925
2556
|
try {
|
|
4926
|
-
await
|
|
2557
|
+
await walkDir(dir);
|
|
4927
2558
|
} catch (error) {
|
|
4928
2559
|
if (error.code !== "ENOENT") {
|
|
4929
2560
|
throw error;
|
|
@@ -4933,7 +2564,7 @@ async function listSyncableFiles(dir, include, exclude) {
|
|
|
4933
2564
|
}
|
|
4934
2565
|
async function getFileHashInfo(filePath) {
|
|
4935
2566
|
try {
|
|
4936
|
-
const stat = await
|
|
2567
|
+
const stat = await fs12.stat(filePath);
|
|
4937
2568
|
const hash = await computeFileHash(filePath);
|
|
4938
2569
|
return {
|
|
4939
2570
|
exists: true,
|
|
@@ -4961,26 +2592,27 @@ function getFileSyncStatus(localHash, remoteHash) {
|
|
|
4961
2592
|
}
|
|
4962
2593
|
|
|
4963
2594
|
// src/cli/commands/conflicts.ts
|
|
4964
|
-
import
|
|
4965
|
-
import
|
|
2595
|
+
import fs14 from "fs/promises";
|
|
2596
|
+
import path14 from "path";
|
|
4966
2597
|
import { spawn } from "child_process";
|
|
4967
2598
|
|
|
4968
2599
|
// src/cli/sync/conflicts.ts
|
|
4969
|
-
import
|
|
4970
|
-
import
|
|
2600
|
+
import fs13 from "fs/promises";
|
|
2601
|
+
import path13 from "path";
|
|
4971
2602
|
var CONFLICTS_DIR = "conflicts";
|
|
4972
2603
|
function getConflictsDir(memoryDir) {
|
|
4973
|
-
|
|
2604
|
+
const subdir = resolveConfigSubdir(memoryDir);
|
|
2605
|
+
return path13.join(memoryDir, subdir, CONFLICTS_DIR);
|
|
4974
2606
|
}
|
|
4975
2607
|
async function listQuarantinedConflicts(memoryDir) {
|
|
4976
2608
|
const conflictsDir = getConflictsDir(memoryDir);
|
|
4977
2609
|
const conflicts = [];
|
|
4978
2610
|
try {
|
|
4979
|
-
const entries = await
|
|
2611
|
+
const entries = await fs13.readdir(conflictsDir, { withFileTypes: true });
|
|
4980
2612
|
for (const entry of entries) {
|
|
4981
2613
|
if (entry.isDirectory()) {
|
|
4982
|
-
const dirPath =
|
|
4983
|
-
const files = await
|
|
2614
|
+
const dirPath = path13.join(conflictsDir, entry.name);
|
|
2615
|
+
const files = await fs13.readdir(dirPath);
|
|
4984
2616
|
const uniqueFiles = /* @__PURE__ */ new Set();
|
|
4985
2617
|
for (const file of files) {
|
|
4986
2618
|
const baseName = file.replace(/\.(local|remote|base)$/, "");
|
|
@@ -5005,7 +2637,7 @@ async function detectChanges(memoryDir) {
|
|
|
5005
2637
|
if (!syncConfig.path) {
|
|
5006
2638
|
throw new Error("Directory is not configured for sync");
|
|
5007
2639
|
}
|
|
5008
|
-
const remotePath =
|
|
2640
|
+
const remotePath = path13.join(centralRepo, syncConfig.path);
|
|
5009
2641
|
const [localFiles, remoteFiles] = await Promise.all([
|
|
5010
2642
|
listSyncableFiles(memoryDir, syncConfig.include, syncConfig.exclude),
|
|
5011
2643
|
listSyncableFiles(remotePath, syncConfig.include, syncConfig.exclude)
|
|
@@ -5020,8 +2652,8 @@ async function detectChanges(memoryDir) {
|
|
|
5020
2652
|
localModified: 0
|
|
5021
2653
|
};
|
|
5022
2654
|
for (const file of allFiles) {
|
|
5023
|
-
const localPath =
|
|
5024
|
-
const remoteFilePath =
|
|
2655
|
+
const localPath = path13.join(memoryDir, file);
|
|
2656
|
+
const remoteFilePath = path13.join(remotePath, file);
|
|
5025
2657
|
const [localInfo, remoteInfo] = await Promise.all([
|
|
5026
2658
|
getFileHashInfo(localPath),
|
|
5027
2659
|
getFileHashInfo(remoteFilePath)
|
|
@@ -5101,9 +2733,9 @@ async function resolveCommand(timestamp, options) {
|
|
|
5101
2733
|
if (!await isInitialized(memoryDir)) {
|
|
5102
2734
|
exitWithError(`${formatPath(memoryDir)} is not initialized.`);
|
|
5103
2735
|
}
|
|
5104
|
-
const conflictDir =
|
|
2736
|
+
const conflictDir = path14.join(getConflictsDir(memoryDir), timestamp);
|
|
5105
2737
|
try {
|
|
5106
|
-
await
|
|
2738
|
+
await fs14.access(conflictDir);
|
|
5107
2739
|
} catch {
|
|
5108
2740
|
exitWithError(
|
|
5109
2741
|
`Conflict '${timestamp}' not found.`,
|
|
@@ -5111,7 +2743,7 @@ async function resolveCommand(timestamp, options) {
|
|
|
5111
2743
|
);
|
|
5112
2744
|
}
|
|
5113
2745
|
try {
|
|
5114
|
-
const files = await
|
|
2746
|
+
const files = await fs14.readdir(conflictDir);
|
|
5115
2747
|
const fileGroups = /* @__PURE__ */ new Map();
|
|
5116
2748
|
for (const file of files) {
|
|
5117
2749
|
const match = file.match(/^(.+)\.(local|remote|base)$/);
|
|
@@ -5121,7 +2753,7 @@ async function resolveCommand(timestamp, options) {
|
|
|
5121
2753
|
fileGroups.set(baseName, {});
|
|
5122
2754
|
}
|
|
5123
2755
|
const group = fileGroups.get(baseName);
|
|
5124
|
-
group[type] =
|
|
2756
|
+
group[type] = path14.join(conflictDir, file);
|
|
5125
2757
|
}
|
|
5126
2758
|
}
|
|
5127
2759
|
if (fileGroups.size === 0) {
|
|
@@ -5194,7 +2826,7 @@ async function cleanupCommand(options) {
|
|
|
5194
2826
|
const conflictsDir = getConflictsDir(memoryDir);
|
|
5195
2827
|
let entries = [];
|
|
5196
2828
|
try {
|
|
5197
|
-
entries = await
|
|
2829
|
+
entries = await fs14.readdir(conflictsDir);
|
|
5198
2830
|
} catch {
|
|
5199
2831
|
console.log("No conflicts directory found.");
|
|
5200
2832
|
return;
|
|
@@ -5208,12 +2840,12 @@ async function cleanupCommand(options) {
|
|
|
5208
2840
|
console.log(` Skipping invalid timestamp: ${entry}`);
|
|
5209
2841
|
continue;
|
|
5210
2842
|
}
|
|
5211
|
-
const entryPath =
|
|
2843
|
+
const entryPath = path14.join(conflictsDir, entry);
|
|
5212
2844
|
if (entryDate < cutoffDate) {
|
|
5213
2845
|
if (options.dryRun) {
|
|
5214
2846
|
console.log(` Would remove: ${entry}`);
|
|
5215
2847
|
} else {
|
|
5216
|
-
await
|
|
2848
|
+
await fs14.rm(entryPath, { recursive: true, force: true });
|
|
5217
2849
|
console.log(` Removed: ${entry}`);
|
|
5218
2850
|
}
|
|
5219
2851
|
cleaned++;
|
|
@@ -5236,15 +2868,16 @@ Removed ${cleaned} conflict(s), kept ${kept}`);
|
|
|
5236
2868
|
var SYNC_LOG_FILE = "sync.log";
|
|
5237
2869
|
var MAX_LOG_ENTRIES = 1e3;
|
|
5238
2870
|
function getSyncLogPath(memoryDir) {
|
|
5239
|
-
|
|
2871
|
+
const subdir = resolveConfigSubdir(memoryDir);
|
|
2872
|
+
return path14.join(memoryDir, subdir, SYNC_LOG_FILE);
|
|
5240
2873
|
}
|
|
5241
2874
|
async function appendSyncLog(memoryDir, entry) {
|
|
5242
2875
|
const logPath = getSyncLogPath(memoryDir);
|
|
5243
2876
|
try {
|
|
5244
|
-
await
|
|
2877
|
+
await fs14.mkdir(path14.dirname(logPath), { recursive: true });
|
|
5245
2878
|
let entries = [];
|
|
5246
2879
|
try {
|
|
5247
|
-
const content2 = await
|
|
2880
|
+
const content2 = await fs14.readFile(logPath, "utf-8");
|
|
5248
2881
|
entries = content2.split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
5249
2882
|
} catch {
|
|
5250
2883
|
}
|
|
@@ -5253,7 +2886,7 @@ async function appendSyncLog(memoryDir, entry) {
|
|
|
5253
2886
|
entries = entries.slice(-MAX_LOG_ENTRIES);
|
|
5254
2887
|
}
|
|
5255
2888
|
const content = entries.map((e) => JSON.stringify(e)).join("\n") + "\n";
|
|
5256
|
-
await
|
|
2889
|
+
await fs14.writeFile(logPath, content);
|
|
5257
2890
|
} catch (error) {
|
|
5258
2891
|
const message = error instanceof Error ? error.message : String(error);
|
|
5259
2892
|
warn(`Failed to write sync log: ${message}`);
|
|
@@ -5262,7 +2895,7 @@ async function appendSyncLog(memoryDir, entry) {
|
|
|
5262
2895
|
async function readSyncLog(memoryDir) {
|
|
5263
2896
|
const logPath = getSyncLogPath(memoryDir);
|
|
5264
2897
|
try {
|
|
5265
|
-
const content = await
|
|
2898
|
+
const content = await fs14.readFile(logPath, "utf-8");
|
|
5266
2899
|
return content.split("\n").filter(Boolean).map((line) => JSON.parse(line));
|
|
5267
2900
|
} catch {
|
|
5268
2901
|
return [];
|
|
@@ -5311,19 +2944,19 @@ async function logCommand(options) {
|
|
|
5311
2944
|
}
|
|
5312
2945
|
|
|
5313
2946
|
// src/cli/sync/operations.ts
|
|
5314
|
-
async function
|
|
5315
|
-
await
|
|
2947
|
+
async function ensureDir(dirPath) {
|
|
2948
|
+
await fs15.mkdir(dirPath, { recursive: true });
|
|
5316
2949
|
}
|
|
5317
2950
|
async function copyFileAtomic(src, dest) {
|
|
5318
|
-
const destDir =
|
|
5319
|
-
await
|
|
5320
|
-
const tempDest = `${dest}.${
|
|
2951
|
+
const destDir = path15.dirname(dest);
|
|
2952
|
+
await ensureDir(destDir);
|
|
2953
|
+
const tempDest = `${dest}.${crypto4.randomBytes(4).toString("hex")}.tmp`;
|
|
5321
2954
|
try {
|
|
5322
|
-
await
|
|
5323
|
-
await
|
|
2955
|
+
await fs15.copyFile(src, tempDest);
|
|
2956
|
+
await fs15.rename(tempDest, dest);
|
|
5324
2957
|
} catch (error) {
|
|
5325
2958
|
try {
|
|
5326
|
-
await
|
|
2959
|
+
await fs15.unlink(tempDest);
|
|
5327
2960
|
} catch {
|
|
5328
2961
|
}
|
|
5329
2962
|
throw error;
|
|
@@ -5349,7 +2982,7 @@ async function push(memoryDir, options = {}) {
|
|
|
5349
2982
|
result.errors.push("No central repository configured");
|
|
5350
2983
|
return result;
|
|
5351
2984
|
}
|
|
5352
|
-
const remotePath =
|
|
2985
|
+
const remotePath = path15.join(centralRepo, syncConfig.path);
|
|
5353
2986
|
const state = await loadSyncState(memoryDir, syncConfig.path);
|
|
5354
2987
|
const localFiles = await listSyncableFiles(
|
|
5355
2988
|
memoryDir,
|
|
@@ -5363,8 +2996,8 @@ async function push(memoryDir, options = {}) {
|
|
|
5363
2996
|
);
|
|
5364
2997
|
const allFiles = /* @__PURE__ */ new Set([...localFiles, ...remoteFiles]);
|
|
5365
2998
|
for (const file of allFiles) {
|
|
5366
|
-
const localPath =
|
|
5367
|
-
const remoteFilePath =
|
|
2999
|
+
const localPath = path15.join(memoryDir, file);
|
|
3000
|
+
const remoteFilePath = path15.join(remotePath, file);
|
|
5368
3001
|
try {
|
|
5369
3002
|
const [localInfo, remoteInfo] = await Promise.all([
|
|
5370
3003
|
getFileHashInfo(localPath),
|
|
@@ -5443,7 +3076,7 @@ async function pull(memoryDir, options = {}) {
|
|
|
5443
3076
|
result.errors.push("No central repository configured");
|
|
5444
3077
|
return result;
|
|
5445
3078
|
}
|
|
5446
|
-
const remotePath =
|
|
3079
|
+
const remotePath = path15.join(centralRepo, syncConfig.path);
|
|
5447
3080
|
const state = await loadSyncState(memoryDir, syncConfig.path);
|
|
5448
3081
|
const localFiles = await listSyncableFiles(
|
|
5449
3082
|
memoryDir,
|
|
@@ -5457,8 +3090,8 @@ async function pull(memoryDir, options = {}) {
|
|
|
5457
3090
|
);
|
|
5458
3091
|
const allFiles = /* @__PURE__ */ new Set([...localFiles, ...remoteFiles]);
|
|
5459
3092
|
for (const file of allFiles) {
|
|
5460
|
-
const localPath =
|
|
5461
|
-
const remoteFilePath =
|
|
3093
|
+
const localPath = path15.join(memoryDir, file);
|
|
3094
|
+
const remoteFilePath = path15.join(remotePath, file);
|
|
5462
3095
|
try {
|
|
5463
3096
|
const [localInfo, remoteInfo] = await Promise.all([
|
|
5464
3097
|
getFileHashInfo(localPath),
|
|
@@ -5696,21 +3329,21 @@ async function syncStatusCommand(options) {
|
|
|
5696
3329
|
}
|
|
5697
3330
|
|
|
5698
3331
|
// src/cli/commands/daemon.ts
|
|
5699
|
-
import
|
|
3332
|
+
import fs18 from "fs/promises";
|
|
5700
3333
|
|
|
5701
3334
|
// src/cli/sync/daemon.ts
|
|
5702
|
-
import
|
|
5703
|
-
import
|
|
3335
|
+
import fs17 from "fs/promises";
|
|
3336
|
+
import path18 from "path";
|
|
5704
3337
|
import os6 from "os";
|
|
5705
3338
|
|
|
5706
3339
|
// src/cli/sync/watcher.ts
|
|
5707
|
-
import
|
|
5708
|
-
import
|
|
3340
|
+
import chokidar from "chokidar";
|
|
3341
|
+
import path16 from "path";
|
|
5709
3342
|
import { EventEmitter } from "events";
|
|
5710
3343
|
var DEFAULT_DEBOUNCE_MS = 2e3;
|
|
5711
3344
|
var DEFAULT_POLL_INTERVAL = 1e3;
|
|
5712
3345
|
var DEFAULT_INCLUDE = ["MEMORY.md", "memory/**/*.md"];
|
|
5713
|
-
var DEFAULT_EXCLUDE = [".minimem/**", "node_modules/**", ".git/**"];
|
|
3346
|
+
var DEFAULT_EXCLUDE = [".minimem/**", "index.db", "index.db-*", "node_modules/**", ".git/**"];
|
|
5714
3347
|
function createFileWatcher(memoryDir, options = {}) {
|
|
5715
3348
|
const debounceMs = options.debounceMs ?? DEFAULT_DEBOUNCE_MS;
|
|
5716
3349
|
const include = options.include ?? DEFAULT_INCLUDE;
|
|
@@ -5735,8 +3368,11 @@ function createFileWatcher(memoryDir, options = {}) {
|
|
|
5735
3368
|
debounceTimer = setTimeout(flushChanges, debounceMs);
|
|
5736
3369
|
};
|
|
5737
3370
|
const handleEvent = (event, filePath) => {
|
|
5738
|
-
const relativePath =
|
|
5739
|
-
if (relativePath.startsWith(".minimem")) {
|
|
3371
|
+
const relativePath = path16.relative(memoryDir, filePath);
|
|
3372
|
+
if (relativePath.startsWith(".minimem") || relativePath.startsWith(".swarm")) {
|
|
3373
|
+
return;
|
|
3374
|
+
}
|
|
3375
|
+
if (relativePath === "index.db" || relativePath.startsWith("index.db-")) {
|
|
5740
3376
|
return;
|
|
5741
3377
|
}
|
|
5742
3378
|
const isIncluded = include.some((pattern) => {
|
|
@@ -5766,9 +3402,9 @@ function createFileWatcher(memoryDir, options = {}) {
|
|
|
5766
3402
|
pendingChanges.set(relativePath, { event, file: relativePath });
|
|
5767
3403
|
scheduleFlush();
|
|
5768
3404
|
};
|
|
5769
|
-
const watchPaths = include.map((pattern) =>
|
|
5770
|
-
watcher =
|
|
5771
|
-
ignored: exclude.map((pattern) =>
|
|
3405
|
+
const watchPaths = include.map((pattern) => path16.join(memoryDir, pattern));
|
|
3406
|
+
watcher = chokidar.watch(watchPaths, {
|
|
3407
|
+
ignored: exclude.map((pattern) => path16.join(memoryDir, pattern)),
|
|
5772
3408
|
persistent: true,
|
|
5773
3409
|
ignoreInitial: true,
|
|
5774
3410
|
usePolling: options.usePolling ?? false,
|
|
@@ -5808,8 +3444,8 @@ function createFileWatcher(memoryDir, options = {}) {
|
|
|
5808
3444
|
}
|
|
5809
3445
|
|
|
5810
3446
|
// src/cli/sync/validation.ts
|
|
5811
|
-
import
|
|
5812
|
-
import
|
|
3447
|
+
import fs16 from "fs/promises";
|
|
3448
|
+
import path17 from "path";
|
|
5813
3449
|
var STALE_THRESHOLD_DAYS = 30;
|
|
5814
3450
|
async function validateRegistry() {
|
|
5815
3451
|
const result = {
|
|
@@ -5893,7 +3529,7 @@ async function validateRegistry() {
|
|
|
5893
3529
|
if (mapping.machineId === currentMachineId) {
|
|
5894
3530
|
const localPath = expandPath3(mapping.localPath);
|
|
5895
3531
|
try {
|
|
5896
|
-
await
|
|
3532
|
+
await fs16.access(localPath);
|
|
5897
3533
|
} catch {
|
|
5898
3534
|
result.issues.push({
|
|
5899
3535
|
type: "missing",
|
|
@@ -5911,7 +3547,7 @@ async function validateRegistry() {
|
|
|
5911
3547
|
}
|
|
5912
3548
|
function expandPath3(p) {
|
|
5913
3549
|
if (p.startsWith("~/")) {
|
|
5914
|
-
return
|
|
3550
|
+
return path17.join(process.env.HOME || "", p.slice(2));
|
|
5915
3551
|
}
|
|
5916
3552
|
return p;
|
|
5917
3553
|
}
|
|
@@ -5951,18 +3587,18 @@ var DAEMON_LOG_FILE = "daemon.log";
|
|
|
5951
3587
|
var PID_FILE = "daemon.pid";
|
|
5952
3588
|
var MAX_LOG_SIZE = 1024 * 1024;
|
|
5953
3589
|
function getDaemonDir() {
|
|
5954
|
-
return
|
|
3590
|
+
return path18.join(os6.homedir(), ".minimem");
|
|
5955
3591
|
}
|
|
5956
3592
|
function getPidFilePath() {
|
|
5957
|
-
return
|
|
3593
|
+
return path18.join(getDaemonDir(), PID_FILE);
|
|
5958
3594
|
}
|
|
5959
3595
|
function getDaemonLogPath() {
|
|
5960
|
-
return
|
|
3596
|
+
return path18.join(getDaemonDir(), DAEMON_LOG_FILE);
|
|
5961
3597
|
}
|
|
5962
3598
|
async function isDaemonRunning() {
|
|
5963
3599
|
const pidFile = getPidFilePath();
|
|
5964
3600
|
try {
|
|
5965
|
-
const content = await
|
|
3601
|
+
const content = await fs17.readFile(pidFile, "utf-8");
|
|
5966
3602
|
const pid = parseInt(content.trim(), 10);
|
|
5967
3603
|
if (isNaN(pid)) {
|
|
5968
3604
|
return false;
|
|
@@ -5971,7 +3607,7 @@ async function isDaemonRunning() {
|
|
|
5971
3607
|
process.kill(pid, 0);
|
|
5972
3608
|
return true;
|
|
5973
3609
|
} catch {
|
|
5974
|
-
await
|
|
3610
|
+
await fs17.unlink(pidFile).catch(() => {
|
|
5975
3611
|
});
|
|
5976
3612
|
return false;
|
|
5977
3613
|
}
|
|
@@ -5985,7 +3621,7 @@ async function getDaemonStatus() {
|
|
|
5985
3621
|
return { running: false };
|
|
5986
3622
|
}
|
|
5987
3623
|
const pidFile = getPidFilePath();
|
|
5988
|
-
const content = await
|
|
3624
|
+
const content = await fs17.readFile(pidFile, "utf-8");
|
|
5989
3625
|
const pid = parseInt(content.trim(), 10);
|
|
5990
3626
|
return {
|
|
5991
3627
|
running: true,
|
|
@@ -5998,15 +3634,15 @@ async function writeLog(message, level = "info") {
|
|
|
5998
3634
|
const line = `[${timestamp}] [${level.toUpperCase()}] ${message}
|
|
5999
3635
|
`;
|
|
6000
3636
|
try {
|
|
6001
|
-
await
|
|
3637
|
+
await fs17.mkdir(path18.dirname(logPath), { recursive: true });
|
|
6002
3638
|
try {
|
|
6003
|
-
const stats = await
|
|
3639
|
+
const stats = await fs17.stat(logPath);
|
|
6004
3640
|
if (stats.size > MAX_LOG_SIZE) {
|
|
6005
|
-
await
|
|
3641
|
+
await fs17.rename(logPath, `${logPath}.old`);
|
|
6006
3642
|
}
|
|
6007
3643
|
} catch {
|
|
6008
3644
|
}
|
|
6009
|
-
await
|
|
3645
|
+
await fs17.appendFile(logPath, line);
|
|
6010
3646
|
} catch (error) {
|
|
6011
3647
|
console.error(`Failed to write log: ${error}`);
|
|
6012
3648
|
}
|
|
@@ -6041,8 +3677,8 @@ async function startDaemon(options = {}) {
|
|
|
6041
3677
|
throw new Error("Daemon is already running");
|
|
6042
3678
|
}
|
|
6043
3679
|
const daemonDir = getDaemonDir();
|
|
6044
|
-
await
|
|
6045
|
-
await
|
|
3680
|
+
await fs17.mkdir(daemonDir, { recursive: true });
|
|
3681
|
+
await fs17.writeFile(getPidFilePath(), String(process.pid));
|
|
6046
3682
|
await writeLog("Daemon starting");
|
|
6047
3683
|
const centralRepo = await getCentralRepoPath();
|
|
6048
3684
|
if (!centralRepo) {
|
|
@@ -6133,7 +3769,7 @@ async function startDaemon(options = {}) {
|
|
|
6133
3769
|
}
|
|
6134
3770
|
watchedDirs.clear();
|
|
6135
3771
|
try {
|
|
6136
|
-
await
|
|
3772
|
+
await fs17.unlink(getPidFilePath());
|
|
6137
3773
|
} catch {
|
|
6138
3774
|
}
|
|
6139
3775
|
await writeLog("Daemon stopped");
|
|
@@ -6257,7 +3893,7 @@ async function daemonStatusCommand() {
|
|
|
6257
3893
|
async function daemonLogsCommand(options) {
|
|
6258
3894
|
const logPath = getDaemonLogPath();
|
|
6259
3895
|
try {
|
|
6260
|
-
const content = await
|
|
3896
|
+
const content = await fs18.readFile(logPath, "utf-8");
|
|
6261
3897
|
const lines = content.split("\n").filter(Boolean);
|
|
6262
3898
|
const numLines = options.lines ?? 50;
|
|
6263
3899
|
const displayLines = lines.slice(-numLines);
|
|
@@ -6266,12 +3902,12 @@ async function daemonLogsCommand(options) {
|
|
|
6266
3902
|
}
|
|
6267
3903
|
if (options.follow) {
|
|
6268
3904
|
console.log("\n--- Following log (Ctrl+C to stop) ---\n");
|
|
6269
|
-
let lastSize = (await
|
|
3905
|
+
let lastSize = (await fs18.stat(logPath)).size;
|
|
6270
3906
|
const poll = async () => {
|
|
6271
3907
|
try {
|
|
6272
|
-
const stats = await
|
|
3908
|
+
const stats = await fs18.stat(logPath);
|
|
6273
3909
|
if (stats.size > lastSize) {
|
|
6274
|
-
const fd = await
|
|
3910
|
+
const fd = await fs18.open(logPath, "r");
|
|
6275
3911
|
const buffer = Buffer.alloc(stats.size - lastSize);
|
|
6276
3912
|
await fd.read(buffer, 0, buffer.length, lastSize);
|
|
6277
3913
|
await fd.close();
|
|
@@ -6300,6 +3936,129 @@ async function daemonLogsCommand(options) {
|
|
|
6300
3936
|
}
|
|
6301
3937
|
}
|
|
6302
3938
|
|
|
3939
|
+
// src/cli/commands/store.ts
|
|
3940
|
+
import path19 from "path";
|
|
3941
|
+
async function storeList(options) {
|
|
3942
|
+
const manifest = await loadManifest();
|
|
3943
|
+
const storeNames = Object.keys(manifest.stores);
|
|
3944
|
+
if (storeNames.length === 0) {
|
|
3945
|
+
console.log("No stores registered.");
|
|
3946
|
+
console.log(`
|
|
3947
|
+
Register a store with: minimem store:add <name> <path>`);
|
|
3948
|
+
return;
|
|
3949
|
+
}
|
|
3950
|
+
if (options.json) {
|
|
3951
|
+
const result = {};
|
|
3952
|
+
for (const name of storeNames) {
|
|
3953
|
+
const def = manifest.stores[name];
|
|
3954
|
+
const links = await getLinkedStoreNames(manifest, name);
|
|
3955
|
+
result[name] = { path: def.path, remote: def.remote, links };
|
|
3956
|
+
}
|
|
3957
|
+
console.log(JSON.stringify(result, null, 2));
|
|
3958
|
+
return;
|
|
3959
|
+
}
|
|
3960
|
+
console.log(`Registered stores (${getManifestPath()}):
|
|
3961
|
+
`);
|
|
3962
|
+
for (const name of storeNames) {
|
|
3963
|
+
const def = manifest.stores[name];
|
|
3964
|
+
const links = await getLinkedStoreNames(manifest, name);
|
|
3965
|
+
console.log(` ${name}`);
|
|
3966
|
+
console.log(` path: ${formatPath(def.path)}`);
|
|
3967
|
+
if (def.remote) {
|
|
3968
|
+
console.log(` remote: ${def.remote}`);
|
|
3969
|
+
}
|
|
3970
|
+
if (def.description) {
|
|
3971
|
+
console.log(` description: ${def.description}`);
|
|
3972
|
+
}
|
|
3973
|
+
if (links.length > 0) {
|
|
3974
|
+
console.log(` links: ${links.join(", ")}`);
|
|
3975
|
+
}
|
|
3976
|
+
console.log();
|
|
3977
|
+
}
|
|
3978
|
+
}
|
|
3979
|
+
async function storeAdd(name, storePath, options) {
|
|
3980
|
+
const manifest = await loadManifest();
|
|
3981
|
+
const absPath = path19.resolve(storePath);
|
|
3982
|
+
if (manifest.stores[name]) {
|
|
3983
|
+
exitWithError(
|
|
3984
|
+
`Store "${name}" already exists.`,
|
|
3985
|
+
`Use a different name or remove it first with: minimem store:remove ${name}`
|
|
3986
|
+
);
|
|
3987
|
+
}
|
|
3988
|
+
const def = {
|
|
3989
|
+
path: absPath,
|
|
3990
|
+
remote: options.remote,
|
|
3991
|
+
description: options.description
|
|
3992
|
+
};
|
|
3993
|
+
manifest.stores[name] = def;
|
|
3994
|
+
await saveManifest(manifest);
|
|
3995
|
+
console.log(`Registered store "${name}" at ${formatPath(absPath)}`);
|
|
3996
|
+
if (options.remote) {
|
|
3997
|
+
console.log(` remote: ${options.remote}`);
|
|
3998
|
+
}
|
|
3999
|
+
const result = await materializeStore(name, def);
|
|
4000
|
+
if (result) {
|
|
4001
|
+
if (result.strategy === "remote") {
|
|
4002
|
+
console.log(` Cloned remote into ${formatPath(result.path)}`);
|
|
4003
|
+
}
|
|
4004
|
+
await result.cleanup();
|
|
4005
|
+
} else if (options.remote) {
|
|
4006
|
+
console.log(` Warning: could not materialize store (remote clone failed)`);
|
|
4007
|
+
}
|
|
4008
|
+
}
|
|
4009
|
+
async function storeRemove(name) {
|
|
4010
|
+
const manifest = await loadManifest();
|
|
4011
|
+
if (!manifest.stores[name]) {
|
|
4012
|
+
exitWithError(`Store "${name}" not found.`);
|
|
4013
|
+
}
|
|
4014
|
+
delete manifest.stores[name];
|
|
4015
|
+
await saveManifest(manifest);
|
|
4016
|
+
console.log(`Removed store "${name}"`);
|
|
4017
|
+
}
|
|
4018
|
+
async function storeLink(storeName, targetName) {
|
|
4019
|
+
const manifest = await loadManifest();
|
|
4020
|
+
const storeDef = manifest.stores[storeName];
|
|
4021
|
+
if (!storeDef) {
|
|
4022
|
+
exitWithError(
|
|
4023
|
+
`Store "${storeName}" not found.`,
|
|
4024
|
+
`Register it first with: minimem store:add ${storeName} <path>`
|
|
4025
|
+
);
|
|
4026
|
+
}
|
|
4027
|
+
if (!manifest.stores[targetName]) {
|
|
4028
|
+
exitWithError(
|
|
4029
|
+
`Target store "${targetName}" not found.`,
|
|
4030
|
+
`Register it first with: minimem store:add ${targetName} <path>`
|
|
4031
|
+
);
|
|
4032
|
+
}
|
|
4033
|
+
if (storeName === targetName) {
|
|
4034
|
+
exitWithError("Cannot link a store to itself.");
|
|
4035
|
+
}
|
|
4036
|
+
const links = await loadStoreLinks(storeDef.path);
|
|
4037
|
+
if (links.links.includes(targetName)) {
|
|
4038
|
+
console.log(`Store "${storeName}" already links to "${targetName}"`);
|
|
4039
|
+
return;
|
|
4040
|
+
}
|
|
4041
|
+
links.links.push(targetName);
|
|
4042
|
+
await saveStoreLinks(storeDef.path, links);
|
|
4043
|
+
console.log(`Linked "${storeName}" \u2192 "${targetName}"`);
|
|
4044
|
+
console.log(` When searching "${storeName}", "${targetName}" will be included automatically.`);
|
|
4045
|
+
}
|
|
4046
|
+
async function storeUnlink(storeName, targetName) {
|
|
4047
|
+
const manifest = await loadManifest();
|
|
4048
|
+
const storeDef = manifest.stores[storeName];
|
|
4049
|
+
if (!storeDef) {
|
|
4050
|
+
exitWithError(`Store "${storeName}" not found.`);
|
|
4051
|
+
}
|
|
4052
|
+
const links = await loadStoreLinks(storeDef.path);
|
|
4053
|
+
if (!links.links.includes(targetName)) {
|
|
4054
|
+
console.log(`Store "${storeName}" does not link to "${targetName}"`);
|
|
4055
|
+
return;
|
|
4056
|
+
}
|
|
4057
|
+
links.links = links.links.filter((l) => l !== targetName);
|
|
4058
|
+
await saveStoreLinks(storeDef.path, links);
|
|
4059
|
+
console.log(`Unlinked "${storeName}" from "${targetName}"`);
|
|
4060
|
+
}
|
|
4061
|
+
|
|
6303
4062
|
// src/cli/version.ts
|
|
6304
4063
|
import { readFileSync } from "fs";
|
|
6305
4064
|
import { dirname as dirname3, join as join4 } from "path";
|
|
@@ -6320,7 +4079,7 @@ var VERSION = getPackageVersion();
|
|
|
6320
4079
|
// src/cli/index.ts
|
|
6321
4080
|
program.name("minimem").description("File-based memory system with vector search for AI agents").version(VERSION);
|
|
6322
4081
|
program.command("init [dir]").description("Initialize a memory directory").option("-g, --global", "Use ~/.minimem as global memory directory").option("-f, --force", "Reinitialize even if already initialized").action(init);
|
|
6323
|
-
program.command("search <query>").description("Semantic search through memory files").option("-d, --dir <path...>", "Memory directories (can specify multiple)").option("-g, --global", "Include ~/.minimem in search").option("-n, --max <number>", "Maximum results (default: 10)").option("-s, --min-score <number>", "Minimum score threshold 0-1 (default: 0.3)").option("-p, --provider <name>", "Embedding provider (openai, gemini, local, auto)").option("--json", "Output results as JSON").action(search);
|
|
4082
|
+
program.command("search <query>").description("Semantic search through memory files").option("-d, --dir <path...>", "Memory directories (can specify multiple)").option("-g, --global", "Include ~/.minimem in search").option("-n, --max <number>", "Maximum results (default: 10)").option("-s, --min-score <number>", "Minimum score threshold 0-1 (default: 0.3)").option("-p, --provider <name>", "Embedding provider (openai, gemini, local, auto)").option("--json", "Output results as JSON").option("--no-links", "Disable linked store resolution").action(search);
|
|
6324
4083
|
program.command("sync").description("Force re-index memory files").option("-d, --dir <path>", "Memory directory").option("-g, --global", "Use ~/.minimem").option("-f, --force", "Force full re-index").option("-p, --provider <name>", "Embedding provider (openai, gemini, local, auto)").action(sync);
|
|
6325
4084
|
program.command("status").description("Show index stats and provider info").option("-d, --dir <path>", "Memory directory").option("-g, --global", "Use ~/.minimem").option("-p, --provider <name>", "Embedding provider (openai, gemini, local, auto)").option("--json", "Output as JSON").action(status);
|
|
6326
4085
|
program.command("append <text>").description("Append text to today's daily log (memory/YYYY-MM-DD.md)").option("-d, --dir <path>", "Memory directory").option("-g, --global", "Use ~/.minimem").option("-f, --file <path>", "Append to specific file instead of today's log").option("-p, --provider <name>", "Embedding provider (openai, gemini, local, auto)").option("-s, --session <id>", "Session ID to associate with this memory").option("--session-source <name>", "Session source (claude-code, vscode, etc.)").action(append);
|
|
@@ -6358,5 +4117,10 @@ program.command("sync:validate").description("Validate registry for collisions a
|
|
|
6358
4117
|
exitWithError(message);
|
|
6359
4118
|
}
|
|
6360
4119
|
});
|
|
4120
|
+
program.command("store:list").description("List all registered stores and their links").option("--json", "Output as JSON").action(storeList);
|
|
4121
|
+
program.command("store:add <name> <path>").description("Register a store in the global manifest").option("-r, --remote <url>", "Git remote URL for remote materialization").option("--description <text>", "Human-readable description").action(storeAdd);
|
|
4122
|
+
program.command("store:remove <name>").description("Remove a store from the global manifest").action(storeRemove);
|
|
4123
|
+
program.command("store:link <store> <target>").description("Link a store to another (searches include linked stores)").action(storeLink);
|
|
4124
|
+
program.command("store:unlink <store> <target>").description("Remove a link between stores").action(storeUnlink);
|
|
6361
4125
|
program.parse();
|
|
6362
4126
|
//# sourceMappingURL=index.js.map
|