monomind 2.3.3 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -4
- package/package.json +1 -1
- package/packages/@monomind/cli/.claude/commands/mastermind/{approve.md → approvev1.md} +12 -9
- package/packages/@monomind/cli/.claude/commands/mastermind/help.md +18 -0
- package/packages/@monomind/cli/.claude/commands/mastermind/master.md +3 -3
- package/packages/@monomind/cli/.claude/commands/mastermind/runorg.md +19 -107
- package/packages/@monomind/cli/.claude/commands/mastermind/runorgv1.md +159 -0
- package/packages/@monomind/cli/.claude/helpers/handlers/route-handler.cjs +50 -4
- package/packages/@monomind/cli/.claude/helpers/handlers/session-restore-handler.cjs +25 -18
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/agents.md +5 -1
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/{approve.md → approvev1.md} +8 -5
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/env.md +7 -2
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/{heartbeat.md → heartbeatv1.md} +7 -4
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/inbox.md +14 -2
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/instance.md +3 -1
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/org-settings.md +4 -1
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/orgs.md +14 -13
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/orgstatus.md +19 -8
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/projects.md +3 -0
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/runorg.md +37 -725
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/runorgv1.md +731 -0
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/stoporg.md +10 -3
- package/packages/@monomind/cli/.claude/skills/mastermind-skills/tasks.md +4 -1
- package/packages/@monomind/cli/README.md +37 -4
- package/packages/@monomind/cli/dist/src/commands/cleanup.js +30 -8
- package/packages/@monomind/cli/dist/src/commands/doc.js +36 -8
- package/packages/@monomind/cli/dist/src/commands/doctor-project-checks.js +23 -10
- package/packages/@monomind/cli/dist/src/commands/org-observe.js +30 -26
- package/packages/@monomind/cli/dist/src/commands/org.js +52 -2
- package/packages/@monomind/cli/dist/src/index.js +7 -0
- package/packages/@monomind/cli/dist/src/knowledge/document-pipeline.d.ts +2 -0
- package/packages/@monomind/cli/dist/src/knowledge/document-pipeline.js +148 -45
- package/packages/@monomind/cli/dist/src/memory/memory-bridge.d.ts +9 -0
- package/packages/@monomind/cli/dist/src/memory/memory-bridge.js +115 -68
- package/packages/@monomind/cli/dist/src/orgrt/daemon.js +45 -9
- package/packages/@monomind/cli/dist/src/orgrt/mailbox.d.ts +23 -0
- package/packages/@monomind/cli/dist/src/orgrt/mailbox.js +33 -1
- package/packages/@monomind/cli/dist/src/orgrt/migrate.d.ts +26 -0
- package/packages/@monomind/cli/dist/src/orgrt/migrate.js +111 -0
- package/packages/@monomind/cli/dist/src/orgrt/reporting.js +8 -1
- package/packages/@monomind/cli/dist/src/orgrt/session.js +8 -2
- package/packages/@monomind/cli/dist/src/ui/dashboard.html +44 -0
- package/packages/@monomind/cli/dist/src/ui/orgs.html +2 -2
- package/packages/@monomind/cli/dist/src/ui/server.mjs +25 -4
- package/packages/@monomind/cli/package.json +2 -2
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
import * as fs from 'node:fs';
|
|
8
8
|
import * as path from 'node:path';
|
|
9
9
|
import * as crypto from 'node:crypto';
|
|
10
|
+
import * as os from 'node:os';
|
|
10
11
|
import { DOC_EXTENSIONS, extractText } from '../capabilities/cap-documents.js';
|
|
11
12
|
const DEFAULT_CHUNK_SIZE = 3200;
|
|
12
13
|
const DEFAULT_OVERLAP = 400;
|
|
@@ -14,23 +15,52 @@ const DEFAULT_OVERLAP = 400;
|
|
|
14
15
|
// used only if the dynamic import below fails (package not installed/built).
|
|
15
16
|
// Keep in sync if the shared chunker's boundary-snapping logic changes.
|
|
16
17
|
const HEADING_LINE_RE = /^#{1,6} /;
|
|
17
|
-
|
|
18
|
+
const FENCE_LINE_RE = /^\s{0,3}(`{3,}|~{3,})/;
|
|
19
|
+
function fenceTogglesInline(text) {
|
|
20
|
+
const toggles = [];
|
|
21
|
+
let lineStart = 0;
|
|
22
|
+
while (lineStart <= text.length) {
|
|
23
|
+
const eol = text.indexOf('\n', lineStart);
|
|
24
|
+
const line = text.slice(lineStart, eol === -1 ? undefined : eol);
|
|
25
|
+
if (FENCE_LINE_RE.test(line))
|
|
26
|
+
toggles.push(lineStart);
|
|
27
|
+
if (eol === -1)
|
|
28
|
+
break;
|
|
29
|
+
lineStart = eol + 1;
|
|
30
|
+
}
|
|
31
|
+
return toggles;
|
|
32
|
+
}
|
|
33
|
+
function inFenceInline(toggles, pos) {
|
|
34
|
+
let lo = 0, hi = toggles.length;
|
|
35
|
+
while (lo < hi) {
|
|
36
|
+
const mid = (lo + hi) >> 1;
|
|
37
|
+
if (toggles[mid] <= pos)
|
|
38
|
+
lo = mid + 1;
|
|
39
|
+
else
|
|
40
|
+
hi = mid;
|
|
41
|
+
}
|
|
42
|
+
return (lo & 1) === 1;
|
|
43
|
+
}
|
|
44
|
+
function lastHeadingBefore(text, pos, toggles) {
|
|
18
45
|
let i = text.lastIndexOf('\n#', pos - 1);
|
|
19
46
|
while (i !== -1) {
|
|
20
47
|
const eol = text.indexOf('\n', i + 1);
|
|
21
48
|
const line = text.slice(i + 1, eol === -1 ? undefined : eol);
|
|
22
|
-
if (HEADING_LINE_RE.test(line))
|
|
49
|
+
if (HEADING_LINE_RE.test(line) && !inFenceInline(toggles, i + 1))
|
|
23
50
|
return line.replace(/^#+ /, '').trim();
|
|
24
|
-
i = text.lastIndexOf('\n#', i - 1);
|
|
51
|
+
i = i > 0 ? text.lastIndexOf('\n#', i - 1) : -1; // fromIndex -1 clamps to 0 — would loop on a match at 0
|
|
25
52
|
}
|
|
26
53
|
const firstEol = text.indexOf('\n');
|
|
27
54
|
const firstLine = firstEol === -1 ? text : text.slice(0, firstEol);
|
|
28
|
-
return HEADING_LINE_RE.test(firstLine) && firstEol !== -1 && firstEol < pos
|
|
55
|
+
return HEADING_LINE_RE.test(firstLine) && !inFenceInline(toggles, 0) && firstEol !== -1 && firstEol < pos
|
|
29
56
|
? firstLine.replace(/^#+ /, '').trim() : null;
|
|
30
57
|
}
|
|
31
58
|
function chunkDocumentInline(docId, text) {
|
|
59
|
+
if (text.includes('\r\n'))
|
|
60
|
+
text = text.replace(/\r\n/g, '\n');
|
|
32
61
|
if (text.length === 0)
|
|
33
62
|
return [];
|
|
63
|
+
const toggles = fenceTogglesInline(text);
|
|
34
64
|
const chunks = [];
|
|
35
65
|
let startChar = 0;
|
|
36
66
|
let chunkIndex = 0;
|
|
@@ -44,22 +74,27 @@ function chunkDocumentInline(docId, text) {
|
|
|
44
74
|
while (h !== -1) {
|
|
45
75
|
const eol = window.indexOf('\n', h + 1);
|
|
46
76
|
const line = window.slice(h + 1, eol === -1 ? undefined : eol);
|
|
47
|
-
if (HEADING_LINE_RE.test(line) && windowStart + h > startChar)
|
|
77
|
+
if (HEADING_LINE_RE.test(line) && windowStart + h > startChar && !inFenceInline(toggles, windowStart + h + 1))
|
|
48
78
|
break;
|
|
49
|
-
h = window.lastIndexOf('\n#', h - 1);
|
|
79
|
+
h = h > 0 ? window.lastIndexOf('\n#', h - 1) : -1;
|
|
50
80
|
}
|
|
51
81
|
if (h !== -1 && windowStart + h > startChar) {
|
|
52
82
|
endChar = windowStart + h + 1;
|
|
53
83
|
brokeAtHeading = true;
|
|
54
84
|
}
|
|
55
85
|
else {
|
|
56
|
-
|
|
86
|
+
let lastParagraph = window.lastIndexOf('\n\n');
|
|
87
|
+
while (lastParagraph > 0 && inFenceInline(toggles, windowStart + lastParagraph + 1)) {
|
|
88
|
+
lastParagraph = window.lastIndexOf('\n\n', lastParagraph - 1);
|
|
89
|
+
}
|
|
90
|
+
if (lastParagraph === 0 && inFenceInline(toggles, windowStart + 1))
|
|
91
|
+
lastParagraph = -1;
|
|
57
92
|
if (lastParagraph !== -1)
|
|
58
93
|
endChar = windowStart + lastParagraph + 2;
|
|
59
94
|
}
|
|
60
95
|
}
|
|
61
96
|
let chunkText = text.slice(startChar, endChar);
|
|
62
|
-
const heading = lastHeadingBefore(text, startChar + 1);
|
|
97
|
+
const heading = lastHeadingBefore(text, startChar + 1, toggles);
|
|
63
98
|
if (heading && !HEADING_LINE_RE.test(chunkText.trimStart()))
|
|
64
99
|
chunkText = `§ ${heading}\n${chunkText}`;
|
|
65
100
|
chunks.push({ chunkId: `${docId}:${chunkIndex}`, docId, text: chunkText, startChar, endChar, chunkIndex });
|
|
@@ -82,6 +117,15 @@ async function chunkDocument(docId, text) {
|
|
|
82
117
|
// ── Constants ──────────────────────────────────────────────────────
|
|
83
118
|
const KNOWLEDGE_NS_PREFIX = 'knowledge:';
|
|
84
119
|
const METADATA_FILE = 'doc-metadata.jsonl';
|
|
120
|
+
// Global brain constants — canonical definitions live in memory-bridge.ts
|
|
121
|
+
// (GLOBAL_BRAIN / GLOBAL_BRAIN_DIR); duplicated here because the bridge is
|
|
122
|
+
// imported lazily and these are needed synchronously.
|
|
123
|
+
const GLOBAL_BRAIN_SENTINEL = '@global';
|
|
124
|
+
const globalBrainRoot = () => process.env.MONOMIND_GLOBAL_BRAIN_DIR || path.join(os.homedir(), '.monomind', 'global-brain');
|
|
125
|
+
/** scope 'global' routes to the personal cross-project store. */
|
|
126
|
+
const isGlobalScope = (scope) => scope === 'global';
|
|
127
|
+
const effectiveRoot = (scope, rootDir) => isGlobalScope(scope) ? globalBrainRoot() : rootDir;
|
|
128
|
+
const storeDbPath = (scope) => isGlobalScope(scope) ? GLOBAL_BRAIN_SENTINEL : undefined;
|
|
85
129
|
const IGNORE_DIRS = new Set(['node_modules', '.git', 'dist', '.monomind', '.claude', '.next', '__pycache__', '.venv', 'vendor']);
|
|
86
130
|
const MAX_FILE_SIZE = 50 * 1024 * 1024; // 50MB
|
|
87
131
|
// ── Helpers ────────────────────────────────────────────────────────
|
|
@@ -100,7 +144,34 @@ function readMetadata(rootDir) {
|
|
|
100
144
|
const file = metadataPath(rootDir);
|
|
101
145
|
if (!fs.existsSync(file))
|
|
102
146
|
return [];
|
|
103
|
-
|
|
147
|
+
// Last-wins per (filePath, scope): the file is append-only under concurrent
|
|
148
|
+
// ingests (session-start detached reindex + a manual `doc ingest` can
|
|
149
|
+
// overlap), so duplicates are expected and the newest record is truth.
|
|
150
|
+
// Corrupt lines (torn concurrent writes) are skipped, not fatal.
|
|
151
|
+
const latest = new Map();
|
|
152
|
+
for (const l of fs.readFileSync(file, 'utf-8').split('\n')) {
|
|
153
|
+
if (!l.trim())
|
|
154
|
+
continue;
|
|
155
|
+
try {
|
|
156
|
+
const m = JSON.parse(l);
|
|
157
|
+
latest.set(`${m.filePath} ${m.scope}`, m);
|
|
158
|
+
}
|
|
159
|
+
catch { /* torn line */ }
|
|
160
|
+
}
|
|
161
|
+
// chunkCount -1 records are removal tombstones (see removeMetadataEntry)
|
|
162
|
+
const live = [...latest.values()].filter(m => m.chunkCount >= 0);
|
|
163
|
+
// Occasional compaction: append-only + tombstones grow without bound; when
|
|
164
|
+
// the log gets big, rewrite it deduped (atomic rename — a concurrent append
|
|
165
|
+
// in the tiny window loses only its own record and self-heals on re-ingest).
|
|
166
|
+
try {
|
|
167
|
+
if (fs.statSync(file).size > 1024 * 1024) {
|
|
168
|
+
const tmp = `${file}.${process.pid}.compact`;
|
|
169
|
+
fs.writeFileSync(tmp, live.map(r => JSON.stringify(r)).join('\n') + (live.length ? '\n' : ''), 'utf-8');
|
|
170
|
+
fs.renameSync(tmp, file);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
catch { /* compaction is best-effort */ }
|
|
174
|
+
return live;
|
|
104
175
|
}
|
|
105
176
|
function appendMetadata(rootDir, meta) {
|
|
106
177
|
fs.appendFileSync(metadataPath(rootDir), JSON.stringify(meta) + '\n', 'utf-8');
|
|
@@ -109,8 +180,12 @@ function removeMetadataEntry(rootDir, filePath, scope) {
|
|
|
109
180
|
const file = metadataPath(rootDir);
|
|
110
181
|
if (!fs.existsSync(file))
|
|
111
182
|
return;
|
|
112
|
-
|
|
113
|
-
|
|
183
|
+
// Tombstone by APPEND (chunkCount -1) instead of read-filter-rewrite — the
|
|
184
|
+
// rewrite raced concurrent appends and silently dropped them.
|
|
185
|
+
appendMetadata(rootDir, {
|
|
186
|
+
filePath, scope, contentHash: '', chunkCount: -1,
|
|
187
|
+
indexedAt: new Date().toISOString(), size: 0,
|
|
188
|
+
});
|
|
114
189
|
}
|
|
115
190
|
function toFileEntry(filePath) {
|
|
116
191
|
const stat = fs.statSync(filePath);
|
|
@@ -153,6 +228,7 @@ export async function ingestDocument(filePath, scope = 'shared', rootDir = proce
|
|
|
153
228
|
if (stat.size > MAX_FILE_SIZE) {
|
|
154
229
|
return { filePath: resolved, chunksIndexed: 0, scope, skipped: true, error: 'file too large (>50MB)' };
|
|
155
230
|
}
|
|
231
|
+
rootDir = effectiveRoot(scope, rootDir);
|
|
156
232
|
const meta = _metadataCache ?? readMetadata(rootDir);
|
|
157
233
|
const existing = meta.find(m => m.filePath === resolved && m.scope === scope);
|
|
158
234
|
let fullContent;
|
|
@@ -189,6 +265,7 @@ export async function ingestDocument(filePath, scope = 'shared', rootDir = proce
|
|
|
189
265
|
generateEmbeddingFlag: true,
|
|
190
266
|
tags: ['document', ext, `src:${resolved}`],
|
|
191
267
|
upsert: true,
|
|
268
|
+
dbPath: storeDbPath(scope),
|
|
192
269
|
});
|
|
193
270
|
if (storeResult?.success)
|
|
194
271
|
indexed++;
|
|
@@ -199,16 +276,26 @@ export async function ingestDocument(filePath, scope = 'shared', rootDir = proce
|
|
|
199
276
|
}
|
|
200
277
|
}
|
|
201
278
|
}
|
|
202
|
-
// Persist metadata
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
279
|
+
// Persist metadata — but ONLY when something was actually stored (or the
|
|
280
|
+
// document legitimately produced zero chunks). Recording the content hash
|
|
281
|
+
// after a total store failure (bridge unavailable, every store rejected)
|
|
282
|
+
// made the hash check skip the file on every future ingest: a permanent,
|
|
283
|
+
// silent search miss.
|
|
284
|
+
if (indexed > 0 || chunks.length === 0) {
|
|
285
|
+
appendMetadata(rootDir, {
|
|
286
|
+
filePath: resolved,
|
|
287
|
+
contentHash: hash,
|
|
288
|
+
chunkCount: indexed,
|
|
289
|
+
indexedAt: new Date().toISOString(),
|
|
290
|
+
scope,
|
|
291
|
+
size: stat.size,
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
const storeFailed = chunks.length > 0 && indexed === 0;
|
|
295
|
+
return {
|
|
296
|
+
filePath: resolved, chunksIndexed: indexed, scope, skipped: false,
|
|
297
|
+
...(storeFailed ? { error: bridge ? 'all chunk stores failed' : 'memory bridge unavailable — nothing indexed' } : {}),
|
|
298
|
+
};
|
|
212
299
|
}
|
|
213
300
|
export async function ingestDirectory(dirPath, scope = 'shared', opts) {
|
|
214
301
|
const scanDir = path.resolve(dirPath);
|
|
@@ -266,6 +353,9 @@ export async function ingestDirectory(dirPath, scope = 'shared', opts) {
|
|
|
266
353
|
return result;
|
|
267
354
|
}
|
|
268
355
|
// ── Search ─────────────────────────────────────────────────────────
|
|
356
|
+
/** Small additive boost so project knowledge wins ties against the global
|
|
357
|
+
* brain — local context is more likely to be what the user means. */
|
|
358
|
+
const PROJECT_SCOPE_BOOST = 0.05;
|
|
269
359
|
export async function searchKnowledge(query, opts) {
|
|
270
360
|
const bridge = await getBridge();
|
|
271
361
|
if (!bridge)
|
|
@@ -273,31 +363,44 @@ export async function searchKnowledge(query, opts) {
|
|
|
273
363
|
const scope = opts?.scope ?? 'shared';
|
|
274
364
|
const limit = opts?.limit ?? 10;
|
|
275
365
|
const minScore = opts?.minScore ?? 0.3;
|
|
276
|
-
const
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
threshold: minScore,
|
|
281
|
-
});
|
|
282
|
-
if (!result?.success || !result.results.length)
|
|
283
|
-
return [];
|
|
284
|
-
const meta = readMetadata(opts?.rootDir ?? process.cwd());
|
|
285
|
-
const hashToFile = new Map();
|
|
286
|
-
for (const m of meta) {
|
|
287
|
-
hashToFile.set(m.contentHash, m.filePath);
|
|
366
|
+
const store = opts?.store ?? 'all';
|
|
367
|
+
const targets = [];
|
|
368
|
+
if (store !== 'global') {
|
|
369
|
+
targets.push({ ns: namespace(scope), root: opts?.rootDir ?? process.cwd(), label: scope, boost: PROJECT_SCOPE_BOOST });
|
|
288
370
|
}
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
371
|
+
if (store !== 'project') {
|
|
372
|
+
targets.push({ ns: namespace('global'), dbPath: GLOBAL_BRAIN_SENTINEL, root: globalBrainRoot(), label: 'global', boost: 0 });
|
|
373
|
+
}
|
|
374
|
+
const perTarget = await Promise.all(targets.map(async (t) => {
|
|
375
|
+
const result = await bridge.bridgeSearchEntries({
|
|
376
|
+
query, namespace: t.ns, limit, threshold: minScore, dbPath: t.dbPath,
|
|
377
|
+
}).catch(() => null);
|
|
378
|
+
if (!result?.success || !result.results.length)
|
|
379
|
+
return [];
|
|
380
|
+
const meta = readMetadata(t.root);
|
|
381
|
+
const hashToFile = new Map();
|
|
382
|
+
for (const m of meta)
|
|
383
|
+
hashToFile.set(m.contentHash, m.filePath);
|
|
384
|
+
return result.results.map((r) => {
|
|
385
|
+
const parts = r.key.startsWith('doc:') ? r.key.split(':') : [];
|
|
386
|
+
const hash = parts[1] ?? '';
|
|
387
|
+
const idx = parseInt(parts[2] ?? '0', 10);
|
|
388
|
+
// The src: tag stored at ingest is the chunk's OWN provenance — the
|
|
389
|
+
// hash→file map can misattribute when two documents share identical
|
|
390
|
+
// content, and goes empty when a re-ingested file's hash changed.
|
|
391
|
+
const srcTag = (r.tags ?? []).find((tag) => tag.startsWith('src:'));
|
|
392
|
+
return {
|
|
393
|
+
filePath: srcTag ? srcTag.slice(4) : hashToFile.get(hash) ?? '',
|
|
394
|
+
text: r.content,
|
|
395
|
+
similarity: r.score + t.boost,
|
|
396
|
+
chunkIndex: isNaN(idx) ? 0 : idx,
|
|
397
|
+
scope: t.label,
|
|
398
|
+
};
|
|
399
|
+
});
|
|
400
|
+
}));
|
|
401
|
+
return perTarget.flat()
|
|
402
|
+
.sort((a, b) => b.similarity - a.similarity)
|
|
403
|
+
.slice(0, limit);
|
|
301
404
|
}
|
|
302
405
|
// ── List / Remove ──────────────────────────────────────────────────
|
|
303
406
|
export function listDocuments(rootDir = process.cwd(), scope) {
|
|
@@ -7,6 +7,14 @@
|
|
|
7
7
|
* @module v1/cli/memory-bridge
|
|
8
8
|
*/
|
|
9
9
|
export declare function safeParseEmbedding(raw: string | null | undefined): number[] | null;
|
|
10
|
+
/** The personal, cross-project knowledge store. Deliberately a SIBLING of
|
|
11
|
+
* ~/.monomind/projects (never inside it) so per-project pruning heuristics
|
|
12
|
+
* (`cleanup --data`) can never touch it. Env-overridable for tests and for
|
|
13
|
+
* users who keep their brain on a synced/external location. Resolved lazily
|
|
14
|
+
* so the override works regardless of import order. */
|
|
15
|
+
export declare function getGlobalBrainDir(): string;
|
|
16
|
+
/** Sentinel callers pass as dbPath to address the global brain. */
|
|
17
|
+
export declare const GLOBAL_BRAIN = "@global";
|
|
10
18
|
/** Resolve the real on-disk LanceDB path for a given custom path (or the default). */
|
|
11
19
|
export declare function bridgeGetDbPath(customPath?: string): string;
|
|
12
20
|
export declare function bridgeStoreEntry(options: {
|
|
@@ -46,6 +54,7 @@ export declare function bridgeSearchEntries(options: {
|
|
|
46
54
|
score: number;
|
|
47
55
|
namespace: string;
|
|
48
56
|
provenance?: string;
|
|
57
|
+
tags?: string[];
|
|
49
58
|
}[];
|
|
50
59
|
searchTime: number;
|
|
51
60
|
searchMethod?: string;
|
|
@@ -70,22 +70,37 @@ function realOrResolved(p) {
|
|
|
70
70
|
return p;
|
|
71
71
|
}
|
|
72
72
|
}
|
|
73
|
+
/** The personal, cross-project knowledge store. Deliberately a SIBLING of
|
|
74
|
+
* ~/.monomind/projects (never inside it) so per-project pruning heuristics
|
|
75
|
+
* (`cleanup --data`) can never touch it. Env-overridable for tests and for
|
|
76
|
+
* users who keep their brain on a synced/external location. Resolved lazily
|
|
77
|
+
* so the override works regardless of import order. */
|
|
78
|
+
export function getGlobalBrainDir() {
|
|
79
|
+
return process.env.MONOMIND_GLOBAL_BRAIN_DIR || path.join(os.homedir(), '.monomind', 'global-brain');
|
|
80
|
+
}
|
|
81
|
+
/** Sentinel callers pass as dbPath to address the global brain. */
|
|
82
|
+
export const GLOBAL_BRAIN = '@global';
|
|
73
83
|
function getDbPath(customPath) {
|
|
74
84
|
const defaultDir = path.join(projectDataDir(), 'lancedb');
|
|
75
85
|
if (!customPath || customPath === ':memory:')
|
|
76
86
|
return defaultDir;
|
|
87
|
+
if (customPath === GLOBAL_BRAIN)
|
|
88
|
+
return getGlobalBrainDir();
|
|
77
89
|
// Treat legacy .db paths (and the legacy .swarm dir) as a signal to use the default
|
|
78
90
|
if (customPath.endsWith('.db'))
|
|
79
91
|
return defaultDir;
|
|
80
92
|
const resolved = realOrResolved(path.resolve(customPath));
|
|
81
93
|
// Guard against path traversal from MCP inputs: only allow paths inside the
|
|
82
|
-
// project
|
|
94
|
+
// project, the per-project home data dir, or the global brain.
|
|
83
95
|
const relCwd = path.relative(realOrResolved(process.cwd()), resolved);
|
|
84
96
|
const relHome = path.relative(realOrResolved(projectDataDir()), resolved);
|
|
97
|
+
const relGlobal = path.relative(realOrResolved(getGlobalBrainDir()), resolved);
|
|
85
98
|
if (!relCwd.startsWith('..') && !path.isAbsolute(relCwd))
|
|
86
99
|
return resolved;
|
|
87
100
|
if (!relHome.startsWith('..') && !path.isAbsolute(relHome))
|
|
88
101
|
return resolved;
|
|
102
|
+
if (!relGlobal.startsWith('..') && !path.isAbsolute(relGlobal))
|
|
103
|
+
return resolved;
|
|
89
104
|
return defaultDir;
|
|
90
105
|
}
|
|
91
106
|
/** Resolve the real on-disk LanceDB path for a given custom path (or the default). */
|
|
@@ -114,13 +129,10 @@ function getAutomemConfig() {
|
|
|
114
129
|
function generateId(prefix) {
|
|
115
130
|
return `${prefix}_${Date.now()}_${crypto.randomBytes(8).toString('hex')}`;
|
|
116
131
|
}
|
|
117
|
-
|
|
118
|
-
let backendPromise = null;
|
|
119
|
-
let backendInstance = null;
|
|
120
|
-
let bridgeAvailable = null;
|
|
132
|
+
const backendSlots = new Map();
|
|
121
133
|
let _embedder = null;
|
|
134
|
+
let _embedderPromise = null;
|
|
122
135
|
const MAX_INIT_ATTEMPTS = 3;
|
|
123
|
-
let initAttempts = 0;
|
|
124
136
|
/** Flush after mutations: the sql.js fallback backend is in-memory WASM and
|
|
125
137
|
* only reaches disk via persist(); the CLI process is short-lived, so waiting
|
|
126
138
|
* for an auto-persist interval would lose writes. No-op on better-sqlite3. */
|
|
@@ -130,59 +142,73 @@ async function flushBackend(backend) {
|
|
|
130
142
|
}
|
|
131
143
|
catch { /* best effort */ }
|
|
132
144
|
}
|
|
145
|
+
async function loadEmbedder() {
|
|
146
|
+
if (_embedder)
|
|
147
|
+
return;
|
|
148
|
+
if (!_embedderPromise) {
|
|
149
|
+
_embedderPromise = (async () => {
|
|
150
|
+
try {
|
|
151
|
+
const hf = await import('@huggingface/transformers');
|
|
152
|
+
// revision must be a git ref — 'main' is the HF default; 'default' 404s and
|
|
153
|
+
// silently killed embeddings (every search degraded to keyword matching)
|
|
154
|
+
// dtype pinned explicitly: transformers.js logs a "dtype not specified"
|
|
155
|
+
// warning to the console on every load otherwise (leaks into CLI output).
|
|
156
|
+
const extractor = await hf.pipeline('feature-extraction', BRIDGE_EMBEDDING_MODEL, { revision: 'main', dtype: 'fp32' });
|
|
157
|
+
_embedder = async (text) => {
|
|
158
|
+
const output = await extractor(text, { pooling: 'mean', normalize: true });
|
|
159
|
+
return new Float32Array(output.data);
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
catch (e) {
|
|
163
|
+
_embedderPromise = null; // allow retry (e.g. first call offline)
|
|
164
|
+
if (process.env.DEBUG || process.env.MONOMIND_DEBUG)
|
|
165
|
+
console.error('[memory-bridge] embedding model failed to load — store and search without vectors:', e);
|
|
166
|
+
}
|
|
167
|
+
})();
|
|
168
|
+
}
|
|
169
|
+
await _embedderPromise;
|
|
170
|
+
}
|
|
133
171
|
async function getBackend(dbPath) {
|
|
134
|
-
|
|
172
|
+
const dir = getDbPath(dbPath);
|
|
173
|
+
let slot = backendSlots.get(dir);
|
|
174
|
+
if (!slot) {
|
|
175
|
+
slot = { promise: null, instance: null, available: null, attempts: 0 };
|
|
176
|
+
backendSlots.set(dir, slot);
|
|
177
|
+
}
|
|
178
|
+
if (slot.available === false)
|
|
135
179
|
return null;
|
|
136
|
-
if (
|
|
137
|
-
|
|
180
|
+
if (slot.attempts >= MAX_INIT_ATTEMPTS) {
|
|
181
|
+
slot.available = false;
|
|
138
182
|
return null;
|
|
139
183
|
}
|
|
140
|
-
if (
|
|
141
|
-
return
|
|
142
|
-
if (!
|
|
143
|
-
|
|
184
|
+
if (slot.instance)
|
|
185
|
+
return slot.instance;
|
|
186
|
+
if (!slot.promise) {
|
|
187
|
+
slot.promise = (async () => {
|
|
144
188
|
try {
|
|
145
189
|
const mod = await import('@monoes/memory');
|
|
146
|
-
|
|
147
|
-
let embeddingGenerator;
|
|
148
|
-
try {
|
|
149
|
-
const hf = await import('@huggingface/transformers');
|
|
150
|
-
// revision must be a git ref — 'main' is the HF default; 'default' 404s and
|
|
151
|
-
// silently killed embeddings (every search degraded to keyword matching)
|
|
152
|
-
// dtype pinned explicitly: transformers.js logs a "dtype not specified"
|
|
153
|
-
// warning to the console on every load otherwise (leaks into CLI output).
|
|
154
|
-
const extractor = await hf.pipeline('feature-extraction', BRIDGE_EMBEDDING_MODEL, { revision: 'main', dtype: 'fp32' });
|
|
155
|
-
embeddingGenerator = async (text) => {
|
|
156
|
-
const output = await extractor(text, { pooling: 'mean', normalize: true });
|
|
157
|
-
return new Float32Array(output.data);
|
|
158
|
-
};
|
|
159
|
-
_embedder = embeddingGenerator;
|
|
160
|
-
}
|
|
161
|
-
catch (e) {
|
|
162
|
-
if (process.env.DEBUG || process.env.MONOMIND_DEBUG)
|
|
163
|
-
console.error('[memory-bridge] embedding model failed to load — store and search without vectors:', e);
|
|
164
|
-
}
|
|
190
|
+
await loadEmbedder();
|
|
165
191
|
// Local SQLite engine (LanceDB replaced 2026-07): better-sqlite3 when its
|
|
166
192
|
// native binding loads, sql.js (pure WASM) otherwise — both persist text
|
|
167
193
|
// AND embeddings, so vectors are always recomputable/derivable data.
|
|
168
|
-
// getDbPath keeps its traversal guard and directory semantics; the SQLite
|
|
169
|
-
// file lives inside that directory.
|
|
170
|
-
const dir = getDbPath(dbPath);
|
|
171
194
|
fs.mkdirSync(dir, { recursive: true });
|
|
172
195
|
// Origin marker: records which project this data dir belongs to, so
|
|
173
196
|
// `monomind cleanup --data` can verifiably prune dirs whose project
|
|
174
|
-
// no longer exists (the dir-name hash is one-way). Best-effort
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
197
|
+
// no longer exists (the dir-name hash is one-way). Best-effort; never
|
|
198
|
+
// written for the global brain (it has no single origin project).
|
|
199
|
+
if (dir !== getGlobalBrainDir()) {
|
|
200
|
+
try {
|
|
201
|
+
const originFile = path.join(projectDataDir(), 'origin.json');
|
|
202
|
+
fs.writeFileSync(originFile, JSON.stringify({ path: path.resolve(process.cwd()), updatedAt: new Date().toISOString() }) + '\n', 'utf-8');
|
|
203
|
+
}
|
|
204
|
+
catch { /* non-fatal */ }
|
|
178
205
|
}
|
|
179
|
-
catch { /* non-fatal */ }
|
|
180
206
|
const cfg = {
|
|
181
207
|
databasePath: path.join(dir, 'memory.db'),
|
|
182
208
|
walMode: true,
|
|
183
209
|
optimize: true,
|
|
184
210
|
defaultNamespace: 'default',
|
|
185
|
-
embeddingGenerator,
|
|
211
|
+
embeddingGenerator: _embedder ?? undefined,
|
|
186
212
|
};
|
|
187
213
|
const origLog = console.log;
|
|
188
214
|
console.log = (...args) => {
|
|
@@ -207,20 +233,20 @@ async function getBackend(dbPath) {
|
|
|
207
233
|
finally {
|
|
208
234
|
console.log = origLog;
|
|
209
235
|
}
|
|
210
|
-
|
|
211
|
-
|
|
236
|
+
slot.instance = backend;
|
|
237
|
+
slot.available = true;
|
|
212
238
|
return backend;
|
|
213
239
|
}
|
|
214
240
|
catch {
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
if (
|
|
218
|
-
|
|
241
|
+
slot.attempts++;
|
|
242
|
+
slot.promise = null;
|
|
243
|
+
if (slot.attempts >= MAX_INIT_ATTEMPTS)
|
|
244
|
+
slot.available = false;
|
|
219
245
|
return null;
|
|
220
246
|
}
|
|
221
247
|
})();
|
|
222
248
|
}
|
|
223
|
-
return
|
|
249
|
+
return slot.promise;
|
|
224
250
|
}
|
|
225
251
|
// ===== Core CRUD =====
|
|
226
252
|
export async function bridgeStoreEntry(options) {
|
|
@@ -263,23 +289,26 @@ export async function bridgeStoreEntry(options) {
|
|
|
263
289
|
entry.id = id;
|
|
264
290
|
if (embedding)
|
|
265
291
|
entry.embedding = embedding;
|
|
266
|
-
// Upsert:
|
|
292
|
+
// Upsert: find any existing entry with the same key+namespace — deleted
|
|
293
|
+
// only AFTER the new entry stores successfully, so a failed store() can't
|
|
294
|
+
// destroy the existing data (old order was delete-then-store).
|
|
295
|
+
let upsertVictim = null;
|
|
267
296
|
if (options.upsert) {
|
|
268
297
|
try {
|
|
269
|
-
|
|
270
|
-
if (existing)
|
|
271
|
-
await backend.delete(existing.id);
|
|
272
|
-
}
|
|
273
|
-
catch (e) {
|
|
274
|
-
if (process.env.DEBUG || process.env.MONOMIND_DEBUG)
|
|
275
|
-
console.error('[memory-bridge] upsert failed to delete existing entry — may create a duplicate:', e);
|
|
298
|
+
upsertVictim = await backend.getByKey(namespace, key);
|
|
276
299
|
}
|
|
300
|
+
catch { /* treat as no existing entry */ }
|
|
277
301
|
}
|
|
278
|
-
// Dedup gate: skip if a near-duplicate already exists
|
|
302
|
+
// Dedup gate: skip if a near-duplicate already exists IN THIS NAMESPACE —
|
|
303
|
+
// an unscoped search let a similar entry in some other namespace swallow
|
|
304
|
+
// the store entirely (returned duplicate:true, nothing written where asked).
|
|
279
305
|
const automemCfg = getAutomemConfig();
|
|
280
306
|
if (embedding && !options.upsert) {
|
|
281
307
|
try {
|
|
282
|
-
const similar = await backend.search(embedding, {
|
|
308
|
+
const similar = await backend.search(embedding, {
|
|
309
|
+
k: 1, threshold: automemCfg.dedupThreshold,
|
|
310
|
+
filters: { type: 'exact', namespace },
|
|
311
|
+
});
|
|
283
312
|
if (similar.length > 0 && similar[0].score >= automemCfg.dedupThreshold) {
|
|
284
313
|
return { success: true, id: similar[0].entry.id, duplicate: true };
|
|
285
314
|
}
|
|
@@ -287,6 +316,15 @@ export async function bridgeStoreEntry(options) {
|
|
|
287
316
|
catch { /* non-fatal — store anyway */ }
|
|
288
317
|
}
|
|
289
318
|
await backend.store(entry);
|
|
319
|
+
if (upsertVictim && upsertVictim.id !== id) {
|
|
320
|
+
try {
|
|
321
|
+
await backend.delete(upsertVictim.id);
|
|
322
|
+
}
|
|
323
|
+
catch (e) {
|
|
324
|
+
if (process.env.DEBUG || process.env.MONOMIND_DEBUG)
|
|
325
|
+
console.error('[memory-bridge] upsert stored new entry but failed to delete the old one — duplicate may remain:', e);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
290
328
|
await flushBackend(backend);
|
|
291
329
|
return { success: true, id, embedding: embeddingInfo };
|
|
292
330
|
}
|
|
@@ -320,6 +358,7 @@ export async function bridgeSearchEntries(options) {
|
|
|
320
358
|
score: r.score,
|
|
321
359
|
namespace: r.entry.namespace,
|
|
322
360
|
provenance: `semantic:${r.score.toFixed(3)}`,
|
|
361
|
+
tags: r.entry.tags ?? [],
|
|
323
362
|
_createdAt: r.entry.createdAt || 0,
|
|
324
363
|
}));
|
|
325
364
|
searchMethod = 'semantic';
|
|
@@ -329,9 +368,11 @@ export async function bridgeSearchEntries(options) {
|
|
|
329
368
|
// Keyword fallback — scan all entries in namespace (not just first 100)
|
|
330
369
|
// to avoid missing documents that were ingested later in the batch.
|
|
331
370
|
if (results.length === 0) {
|
|
371
|
+
// No namespace filter means ALL namespaces — collapsing to 'default'
|
|
372
|
+
// made "search everything" silently miss every non-default entry.
|
|
332
373
|
const entries = await backend.query({
|
|
333
374
|
type: 'exact',
|
|
334
|
-
namespace
|
|
375
|
+
...(namespace ? { namespace } : {}),
|
|
335
376
|
limit: 50000,
|
|
336
377
|
});
|
|
337
378
|
// Token-based matching, not whole-phrase substring: "semantic test" must
|
|
@@ -356,6 +397,7 @@ export async function bridgeSearchEntries(options) {
|
|
|
356
397
|
score: Math.min(0.9, 0.3 + score * 0.6),
|
|
357
398
|
namespace: e.namespace,
|
|
358
399
|
provenance: `keyword:${score.toFixed(2)}`,
|
|
400
|
+
tags: e.tags ?? [],
|
|
359
401
|
_createdAt: e.createdAt || 0,
|
|
360
402
|
}));
|
|
361
403
|
}
|
|
@@ -363,11 +405,16 @@ export async function bridgeSearchEntries(options) {
|
|
|
363
405
|
}
|
|
364
406
|
// Filter stale entries based on automem config — skip for knowledge
|
|
365
407
|
// namespaces (documents should remain searchable indefinitely)
|
|
408
|
+
// Stale filtering is per-RESULT namespace (documents stay searchable
|
|
409
|
+
// forever) — keying it on the query's namespace filter meant an
|
|
410
|
+
// all-namespace search silently dropped knowledge:* results past the
|
|
411
|
+
// stale cutoff.
|
|
366
412
|
const isKnowledgeNs = namespace?.startsWith('knowledge:');
|
|
367
413
|
if (!isKnowledgeNs) {
|
|
368
414
|
const { staleDays } = getAutomemConfig();
|
|
369
415
|
const staleCutoff = Date.now() - staleDays * 86400000;
|
|
370
|
-
results = results.filter((r) =>
|
|
416
|
+
results = results.filter((r) => String(r.namespace ?? '').startsWith('knowledge:')
|
|
417
|
+
|| !r._createdAt || r._createdAt > staleCutoff);
|
|
371
418
|
}
|
|
372
419
|
results.forEach((r) => delete r._createdAt);
|
|
373
420
|
return {
|
|
@@ -579,17 +626,17 @@ export async function getControllerRegistry(dbPath) {
|
|
|
579
626
|
return getBackend(dbPath);
|
|
580
627
|
}
|
|
581
628
|
export async function shutdownBridge() {
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
629
|
+
for (const slot of backendSlots.values()) {
|
|
630
|
+
if (slot.instance) {
|
|
631
|
+
try {
|
|
632
|
+
await slot.instance.shutdown();
|
|
633
|
+
}
|
|
634
|
+
catch { /* ignore */ }
|
|
585
635
|
}
|
|
586
|
-
catch { /* ignore */ }
|
|
587
636
|
}
|
|
588
|
-
|
|
589
|
-
backendPromise = null;
|
|
590
|
-
bridgeAvailable = null;
|
|
637
|
+
backendSlots.clear();
|
|
591
638
|
_embedder = null;
|
|
592
|
-
|
|
639
|
+
_embedderPromise = null;
|
|
593
640
|
}
|
|
594
641
|
// ===== Pattern store =====
|
|
595
642
|
export async function bridgeStorePattern(options) {
|