memoir-cli 3.11.2 → 3.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -6
- package/bin/memoir.js +22 -0
- package/package.json +1 -1
- package/src/cloud/auth.js +12 -15
- package/src/cloud/constants.js +6 -2
- package/src/commands/activate.js +25 -2
- package/src/commands/cloud.js +1 -1
- package/src/commands/forget.js +100 -0
- package/src/commands/push.js +31 -12
- package/src/commands/recall.js +42 -0
- package/src/commands/restore.js +12 -4
- package/src/commands/session.js +8 -3
- package/src/commands/upgrade.js +2 -2
- package/src/commands/validate.js +13 -0
- package/src/context/capture.js +14 -2
- package/src/mcp.js +54 -139
- package/src/memory/search.js +503 -0
- package/src/providers/index.js +21 -0
- package/src/security/scanner.js +12 -4
- package/src/session/lock.js +36 -2
- package/src/session/state.js +122 -11
|
@@ -0,0 +1,503 @@
|
|
|
1
|
+
// Memory retrieval — the read side of memoir.
|
|
2
|
+
//
|
|
3
|
+
// Before 3.12 `memoir_recall` was `content.toLowerCase().includes(term)`
|
|
4
|
+
// per term, ranked by how many terms hit, and it returned the FIRST 500
|
|
5
|
+
// characters of each file. For a spec-shaped entry that is ~440 chars of
|
|
6
|
+
// YAML frontmatter and one truncated line of prose: the model asked a
|
|
7
|
+
// question, got ten headers back, and learned nothing. It also re-crawled
|
|
8
|
+
// ~1,000 directories under $HOME on every call to find project CLAUDE.md
|
|
9
|
+
// files (measured: 6,462 stat probes / call on a real machine).
|
|
10
|
+
//
|
|
11
|
+
// This module fixes the read path without adding a dependency:
|
|
12
|
+
// • matched PASSAGES, frontmatter stripped, with a line of context;
|
|
13
|
+
// • field-weighted scoring — aliases > name > description > headings >
|
|
14
|
+
// body — with per-term saturation and a coverage multiplier so a file
|
|
15
|
+
// that mentions all three query terms outranks one that hammers one;
|
|
16
|
+
// • light morphology: plural/-ing/-ed folding plus prefix matching from
|
|
17
|
+
// 4 chars, so "auth" finds "authentication" and "deploys" finds
|
|
18
|
+
// "deploy" (a real stemmer over-merges; this is deliberately timid);
|
|
19
|
+
// • an mtime-keyed parse cache and a TTL'd project-file index, so a
|
|
20
|
+
// long-lived MCP process stops re-reading and re-walking the disk.
|
|
21
|
+
//
|
|
22
|
+
// What it does NOT do: semantic/concept matching. "tiktok" still won't
|
|
23
|
+
// find a file that only ever says "vertical swipe feed". The honest,
|
|
24
|
+
// dependency-free answer to that is the `aliases:` frontmatter field
|
|
25
|
+
// (SPEC.md 3.2) written at save time and weighted heaviest here — the
|
|
26
|
+
// model that saves a memory knows what else it might be called.
|
|
27
|
+
|
|
28
|
+
import fs from 'fs-extra';
|
|
29
|
+
import path from 'path';
|
|
30
|
+
import os from 'os';
|
|
31
|
+
import { adapters } from '../adapters/index.js';
|
|
32
|
+
import { parseFrontmatter } from '../commands/validate.js';
|
|
33
|
+
|
|
34
|
+
const home = os.homedir();
|
|
35
|
+
|
|
36
|
+
// ── Tokenizing ───────────────────────────────────────────────────
|
|
37
|
+
|
|
38
|
+
const STOPWORDS = new Set([
|
|
39
|
+
'the', 'a', 'an', 'of', 'to', 'in', 'on', 'for', 'and', 'or', 'is', 'it',
|
|
40
|
+
'this', 'that', 'what', 'how', 'do', 'does', 'did', 'we', 'i', 'my', 'our',
|
|
41
|
+
'with', 'about', 'was', 'were', 'be', 'are', 'at', 'by', 'from', 'as',
|
|
42
|
+
'into', 'up', 'out', 'so', 'if', 'not', 'no', 'me', 'you', 'your', 'us',
|
|
43
|
+
'why', 'when', 'where', 'which', 'who', 'can', 'should', 'would', 'could',
|
|
44
|
+
'have', 'has', 'had', 'been', 'being', 'there', 'here', 'than', 'then',
|
|
45
|
+
]);
|
|
46
|
+
|
|
47
|
+
// Minimum length at which a token may match another by prefix. Below this
|
|
48
|
+
// "in" would match "index" and "at" would match "attribution".
|
|
49
|
+
const PREFIX_MIN = 4;
|
|
50
|
+
|
|
51
|
+
export function tokenize(str) {
|
|
52
|
+
return String(str || '')
|
|
53
|
+
.toLowerCase()
|
|
54
|
+
.split(/[^a-z0-9_$./-]+/)
|
|
55
|
+
.flatMap((t) => t.split(/[./-]+/))
|
|
56
|
+
.map((t) => t.replace(/^[_$]+|[_$]+$/g, ''))
|
|
57
|
+
.filter((t) => t.length >= 2);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// Fold the commonest English inflections. Conservative on purpose: every
|
|
61
|
+
// rule keeps a stem of at least 4 characters so short words are untouched
|
|
62
|
+
// ("was" -> "was", not "wa"), and -ing/-ed only strip from words long
|
|
63
|
+
// enough that the residue is still a word ("deploying" -> "deploy",
|
|
64
|
+
// "ring" -> "ring").
|
|
65
|
+
export function normalize(token) {
|
|
66
|
+
let t = token;
|
|
67
|
+
if (t.length >= 5 && t.endsWith('ies')) t = t.slice(0, -3) + 'y';
|
|
68
|
+
else if (t.length >= 6 && t.endsWith('sses')) t = t.slice(0, -2);
|
|
69
|
+
else if (t.length >= 4 && t.endsWith('s') && !t.endsWith('ss')) t = t.slice(0, -1);
|
|
70
|
+
if (t.length >= 7 && t.endsWith('ing')) t = t.slice(0, -3);
|
|
71
|
+
else if (t.length >= 6 && t.endsWith('ed')) t = t.slice(0, -2);
|
|
72
|
+
return t;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Does a normalized query term match a normalized document token?
|
|
76
|
+
// Returns 1 for an exact match, PREFIX_WEIGHT for a prefix match, 0 otherwise.
|
|
77
|
+
const PREFIX_WEIGHT = 0.6;
|
|
78
|
+
export function termMatch(term, token) {
|
|
79
|
+
if (term === token) return 1;
|
|
80
|
+
if (term.length >= PREFIX_MIN && token.startsWith(term)) return PREFIX_WEIGHT;
|
|
81
|
+
if (token.length >= PREFIX_MIN && term.startsWith(token)) return PREFIX_WEIGHT;
|
|
82
|
+
return 0;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export function queryTerms(query) {
|
|
86
|
+
const raw = tokenize(query);
|
|
87
|
+
const kept = raw.filter((t) => !STOPWORDS.has(t));
|
|
88
|
+
// If the whole query was stopwords ("what is it"), search on it anyway
|
|
89
|
+
// rather than returning nothing.
|
|
90
|
+
const terms = (kept.length ? kept : raw).map(normalize);
|
|
91
|
+
return Array.from(new Set(terms));
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// ── Document model ───────────────────────────────────────────────
|
|
95
|
+
|
|
96
|
+
// Field weights. Aliases win because they exist for exactly one reason —
|
|
97
|
+
// they are the other names someone might search under. Then the entry's
|
|
98
|
+
// own name, its one-line description, headings, and finally prose.
|
|
99
|
+
const W = { aliases: 6, name: 4, description: 3, headings: 2, body: 1 };
|
|
100
|
+
// Body term-frequency saturates: 1 hit = 1.0, 3 = 1.5, 7 = 2.0. A file that
|
|
101
|
+
// says "deploy" forty times is not forty times more relevant.
|
|
102
|
+
const BODY_TF_CAP = 2;
|
|
103
|
+
|
|
104
|
+
function listField(v) {
|
|
105
|
+
if (Array.isArray(v)) return v.map(String);
|
|
106
|
+
if (typeof v === 'string' && v.trim()) return v.split(',').map((s) => s.trim()).filter(Boolean);
|
|
107
|
+
return [];
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function fieldTokens(strings) {
|
|
111
|
+
const out = new Map(); // normalized token -> count
|
|
112
|
+
for (const s of strings) {
|
|
113
|
+
for (const t of tokenize(s)) {
|
|
114
|
+
const n = normalize(t);
|
|
115
|
+
out.set(n, (out.get(n) || 0) + 1);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
return out;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Parse a raw memory file into the shape the scorer wants. Cheap enough to
|
|
123
|
+
* run on every file, but cached by (path, mtime, size) in readMemoryFiles.
|
|
124
|
+
*/
|
|
125
|
+
export function buildDoc({ path: relPath, content, tool, absPath, mtimeMs }) {
|
|
126
|
+
const isMarkdown = /\.(md|markdown)$/i.test(relPath);
|
|
127
|
+
const { fields, body } = isMarkdown ? parseFrontmatter(content) : { fields: {}, body: content };
|
|
128
|
+
const bodyLines = body.split(/\r?\n/);
|
|
129
|
+
const headings = bodyLines.filter((l) => /^\s{0,3}#{1,6}\s/.test(l));
|
|
130
|
+
|
|
131
|
+
const nameStrings = [path.basename(relPath).replace(/\.[^.]+$/, '')];
|
|
132
|
+
if (fields.name) nameStrings.push(String(fields.name));
|
|
133
|
+
|
|
134
|
+
return {
|
|
135
|
+
path: relPath,
|
|
136
|
+
absPath,
|
|
137
|
+
tool,
|
|
138
|
+
mtimeMs: mtimeMs || 0,
|
|
139
|
+
isMarkdown,
|
|
140
|
+
type: fields.type || fields.metadata?.type || null,
|
|
141
|
+
description: fields.description ? String(fields.description) : '',
|
|
142
|
+
aliases: listField(fields.aliases),
|
|
143
|
+
tags: listField(fields.tags),
|
|
144
|
+
body,
|
|
145
|
+
bodyLines,
|
|
146
|
+
// Token maps per field
|
|
147
|
+
tf: {
|
|
148
|
+
aliases: fieldTokens([...listField(fields.aliases), ...listField(fields.tags)]),
|
|
149
|
+
name: fieldTokens(nameStrings),
|
|
150
|
+
description: fieldTokens([fields.description || '']),
|
|
151
|
+
headings: fieldTokens(headings),
|
|
152
|
+
body: fieldTokens(bodyLines),
|
|
153
|
+
},
|
|
154
|
+
// Kept for legacy callers (memoir_list sizes, memoir_read) — the raw file.
|
|
155
|
+
content,
|
|
156
|
+
};
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// ── Scoring ──────────────────────────────────────────────────────
|
|
160
|
+
|
|
161
|
+
function fieldScore(tfMap, term) {
|
|
162
|
+
// Best match across the field's tokens: exact beats prefix; tf saturates.
|
|
163
|
+
let best = 0;
|
|
164
|
+
let count = 0;
|
|
165
|
+
for (const [token, n] of tfMap) {
|
|
166
|
+
const m = termMatch(term, token);
|
|
167
|
+
if (m > best) { best = m; count = n; }
|
|
168
|
+
else if (m === best && m > 0) count += n;
|
|
169
|
+
}
|
|
170
|
+
if (!best) return 0;
|
|
171
|
+
const tf = 0.5 + 0.5 * Math.log2(1 + count);
|
|
172
|
+
return best * Math.min(BODY_TF_CAP, tf);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// A single term's evidence in one document is its best field hit plus a
|
|
176
|
+
// quarter of the rest, capped. Without the cap, a term that appears in the
|
|
177
|
+
// alias list AND the name AND the description AND a heading (which is
|
|
178
|
+
// exactly what a well-formed entry looks like) stacked to ~16 and let a
|
|
179
|
+
// one-term match outrank a file that covered every term in the query.
|
|
180
|
+
const TERM_CAP = 8;
|
|
181
|
+
|
|
182
|
+
export function scoreDoc(doc, terms) {
|
|
183
|
+
let sum = 0;
|
|
184
|
+
let matched = 0;
|
|
185
|
+
const perTerm = {};
|
|
186
|
+
for (const term of terms) {
|
|
187
|
+
const fieldScores = Object.keys(W).map((f) => W[f] * fieldScore(doc.tf[f], term)).sort((a, b) => b - a);
|
|
188
|
+
const s = Math.min(TERM_CAP, fieldScores[0] + 0.25 * fieldScores.slice(1).reduce((x, y) => x + y, 0));
|
|
189
|
+
if (s > 0) matched++;
|
|
190
|
+
perTerm[term] = s;
|
|
191
|
+
sum += s;
|
|
192
|
+
}
|
|
193
|
+
if (!matched) return { score: 0, matched: 0, perTerm };
|
|
194
|
+
const coverage = matched / terms.length;
|
|
195
|
+
// Coverage-squared: a file that covers all the terms beats a file that
|
|
196
|
+
// covers a third of them unless the partial match is overwhelming
|
|
197
|
+
// (2/3 coverage keeps 44% of its raw score, 1/3 keeps 11%). This is the
|
|
198
|
+
// "over-recall" fix — a common word no longer drags in every file that
|
|
199
|
+
// mentions it once. Non-markdown (settings.json etc.) is de-weighted:
|
|
200
|
+
// it is config, not memory, and matches on it are usually noise.
|
|
201
|
+
const score = sum * coverage * coverage * (doc.isMarkdown ? 1 : 0.5);
|
|
202
|
+
return { score, matched, coverage, perTerm };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
// ── Passage extraction ───────────────────────────────────────────
|
|
206
|
+
|
|
207
|
+
const PASSAGE_BUDGET = 700; // chars per result
|
|
208
|
+
const CONTEXT_LINES = 1; // lines of context either side of a hit
|
|
209
|
+
const MAX_LINE = 240; // clamp very long lines (pasted JSON, tables)
|
|
210
|
+
|
|
211
|
+
function lineMatches(line, terms) {
|
|
212
|
+
const toks = tokenize(line).map(normalize);
|
|
213
|
+
if (!toks.length) return 0;
|
|
214
|
+
let hits = 0;
|
|
215
|
+
for (const term of terms) {
|
|
216
|
+
if (toks.some((t) => termMatch(term, t) > 0)) hits++;
|
|
217
|
+
}
|
|
218
|
+
return hits;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function clampLine(l) {
|
|
222
|
+
const s = l.replace(/\s+$/, '');
|
|
223
|
+
return s.length > MAX_LINE ? s.slice(0, MAX_LINE - 1) + '…' : s;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* The passage(s) of a document that best answer the query: matched lines
|
|
228
|
+
* with a line of context either side, best windows first, within a byte
|
|
229
|
+
* budget. Frontmatter is never included — callers get the description
|
|
230
|
+
* separately for the header line.
|
|
231
|
+
*/
|
|
232
|
+
export function extractPassage(doc, terms, budget = PASSAGE_BUDGET) {
|
|
233
|
+
const lines = doc.bodyLines;
|
|
234
|
+
const hits = [];
|
|
235
|
+
for (let i = 0; i < lines.length; i++) {
|
|
236
|
+
const l = lines[i];
|
|
237
|
+
if (!l.trim()) continue;
|
|
238
|
+
const h = lineMatches(l, terms);
|
|
239
|
+
if (h) hits.push({ i, h });
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
if (!hits.length) {
|
|
243
|
+
// Match lived in frontmatter only (name/description/aliases). Show the
|
|
244
|
+
// opening of the body so the reader still gets substance.
|
|
245
|
+
const opening = lines.filter((l) => l.trim()).slice(0, 6).map(clampLine).join('\n');
|
|
246
|
+
return opening.slice(0, budget);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Merge hits into windows [start, end] with context, then rank windows by
|
|
250
|
+
// (distinct terms matched desc, position asc).
|
|
251
|
+
const windows = [];
|
|
252
|
+
for (const { i, h } of hits) {
|
|
253
|
+
const start = Math.max(0, i - CONTEXT_LINES);
|
|
254
|
+
const end = Math.min(lines.length - 1, i + CONTEXT_LINES);
|
|
255
|
+
const last = windows[windows.length - 1];
|
|
256
|
+
if (last && start <= last.end + 1) {
|
|
257
|
+
last.end = Math.max(last.end, end);
|
|
258
|
+
last.h = Math.max(last.h, h);
|
|
259
|
+
last.hits++;
|
|
260
|
+
} else {
|
|
261
|
+
windows.push({ start, end, h, hits: 1 });
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
windows.sort((a, b) => b.h - a.h || b.hits - a.hits || a.start - b.start);
|
|
265
|
+
|
|
266
|
+
const chosen = [];
|
|
267
|
+
let used = 0;
|
|
268
|
+
for (const w of windows) {
|
|
269
|
+
const text = lines.slice(w.start, w.end + 1).filter((l) => l.trim()).map(clampLine).join('\n');
|
|
270
|
+
if (!text) continue;
|
|
271
|
+
if (used && used + text.length > budget) continue;
|
|
272
|
+
chosen.push({ ...w, text: used + text.length > budget ? text.slice(0, budget - used - 1) + '…' : text });
|
|
273
|
+
used += text.length + 2;
|
|
274
|
+
if (used >= budget) break;
|
|
275
|
+
}
|
|
276
|
+
// Present in document order so the excerpt reads naturally.
|
|
277
|
+
chosen.sort((a, b) => a.start - b.start);
|
|
278
|
+
return chosen.map((c) => c.text).join('\n⋯\n');
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
// ── Reading + caching ────────────────────────────────────────────
|
|
282
|
+
|
|
283
|
+
// (absPath) -> { mtimeMs, size, doc }. The MCP server is a long-lived
|
|
284
|
+
// process; memory files change rarely relative to how often recall runs.
|
|
285
|
+
const docCache = new Map();
|
|
286
|
+
|
|
287
|
+
async function readDoc(absPath, relPath, tool) {
|
|
288
|
+
let st;
|
|
289
|
+
try { st = await fs.stat(absPath); } catch { return null; }
|
|
290
|
+
const hit = docCache.get(absPath);
|
|
291
|
+
if (hit && hit.mtimeMs === st.mtimeMs && hit.size === st.size) return hit.doc;
|
|
292
|
+
let content;
|
|
293
|
+
try { content = await fs.readFile(absPath, 'utf8'); } catch { return null; }
|
|
294
|
+
const doc = buildDoc({ path: relPath, content, tool, absPath, mtimeMs: st.mtimeMs });
|
|
295
|
+
docCache.set(absPath, { mtimeMs: st.mtimeMs, size: st.size, doc });
|
|
296
|
+
return doc;
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
/** Test hook — drop every cached parse. */
|
|
300
|
+
export function clearSearchCache() {
|
|
301
|
+
docCache.clear();
|
|
302
|
+
projectIndex.at = 0;
|
|
303
|
+
projectIndex.files = [];
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
const MEMORY_EXT = /\.(md|json|ya?ml)$/i;
|
|
307
|
+
|
|
308
|
+
/**
|
|
309
|
+
* Read every memory file an adapter owns, as parsed docs. Cached by mtime.
|
|
310
|
+
*/
|
|
311
|
+
export async function readMemoryFiles(adapter) {
|
|
312
|
+
const files = [];
|
|
313
|
+
|
|
314
|
+
if (adapter.customExtract) {
|
|
315
|
+
for (const file of adapter.files) {
|
|
316
|
+
const abs = path.join(adapter.source, file);
|
|
317
|
+
const doc = await readDoc(abs, file, adapter.name);
|
|
318
|
+
if (doc) files.push(doc);
|
|
319
|
+
}
|
|
320
|
+
return files;
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
if (!(await fs.pathExists(adapter.source))) return files;
|
|
324
|
+
|
|
325
|
+
const walk = async (dir, prefix = '') => {
|
|
326
|
+
let entries;
|
|
327
|
+
try { entries = await fs.readdir(dir, { withFileTypes: true }); } catch { return; }
|
|
328
|
+
for (const entry of entries) {
|
|
329
|
+
const fullPath = path.join(dir, entry.name);
|
|
330
|
+
const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
331
|
+
if (entry.isDirectory()) {
|
|
332
|
+
if (adapter.filter(fullPath)) await walk(fullPath, relPath);
|
|
333
|
+
} else if (MEMORY_EXT.test(entry.name) && adapter.filter(fullPath)) {
|
|
334
|
+
const doc = await readDoc(fullPath, relPath, adapter.name);
|
|
335
|
+
if (doc) files.push(doc);
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
};
|
|
339
|
+
|
|
340
|
+
await walk(adapter.source);
|
|
341
|
+
return files;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// Per-project AI config files. Discovery (the directory walk) is the
|
|
345
|
+
// expensive part and changes rarely, so the found-path list is cached for
|
|
346
|
+
// PROJECT_INDEX_TTL_MS; the files themselves go through readDoc's mtime cache.
|
|
347
|
+
const PROJECT_FILES = ['CLAUDE.md', 'GEMINI.md', 'CHATGPT.md', 'AGENTS.md', '.cursorrules', '.windsurfrules', '.clinerules'];
|
|
348
|
+
const SKIP_DIRS = new Set(['node_modules', '.git', '.next', '.vercel', 'dist', 'build', '__pycache__', '.venv', 'venv', '.cache', 'Library', '.Trash', 'Applications', 'Downloads', 'Movies', 'Music', 'Pictures']);
|
|
349
|
+
const PROJECT_INDEX_TTL_MS = 60_000;
|
|
350
|
+
const PROJECT_SCAN_DEPTH = 3;
|
|
351
|
+
const projectIndex = { at: 0, files: [] };
|
|
352
|
+
|
|
353
|
+
async function discoverProjectFiles(root) {
|
|
354
|
+
const found = [];
|
|
355
|
+
const scan = async (dir, depth) => {
|
|
356
|
+
if (depth > PROJECT_SCAN_DEPTH) return;
|
|
357
|
+
let entries;
|
|
358
|
+
try { entries = await fs.readdir(dir, { withFileTypes: true }); } catch { return; }
|
|
359
|
+
const names = new Set(entries.filter((e) => e.isFile()).map((e) => e.name));
|
|
360
|
+
for (const f of PROJECT_FILES) {
|
|
361
|
+
if (names.has(f)) found.push({ abs: path.join(dir, f), rel: `${path.basename(dir)}/${f}`, project: path.basename(dir) });
|
|
362
|
+
}
|
|
363
|
+
for (const entry of entries) {
|
|
364
|
+
if (!entry.isDirectory()) continue;
|
|
365
|
+
if (entry.name.startsWith('.') && entry.name !== '.github') continue;
|
|
366
|
+
if (SKIP_DIRS.has(entry.name)) continue;
|
|
367
|
+
await scan(path.join(dir, entry.name), depth + 1);
|
|
368
|
+
}
|
|
369
|
+
};
|
|
370
|
+
await scan(root, 0);
|
|
371
|
+
return found;
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
async function projectDocs(root = home) {
|
|
375
|
+
const now = Date.now();
|
|
376
|
+
if (now - projectIndex.at > PROJECT_INDEX_TTL_MS || projectIndex.root !== root) {
|
|
377
|
+
projectIndex.files = await discoverProjectFiles(root);
|
|
378
|
+
projectIndex.at = now;
|
|
379
|
+
projectIndex.root = root;
|
|
380
|
+
}
|
|
381
|
+
const docs = [];
|
|
382
|
+
for (const f of projectIndex.files) {
|
|
383
|
+
const doc = await readDoc(f.abs, f.rel, `Project: ${f.project}`);
|
|
384
|
+
if (doc) docs.push(doc);
|
|
385
|
+
}
|
|
386
|
+
return docs;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// ── Write-side helper: aliases/tags into frontmatter ─────────────
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* Merge `aliases` / `tags` lists into a markdown entry's frontmatter.
|
|
393
|
+
* If the content has frontmatter, lists are added (or extended, deduped)
|
|
394
|
+
* in the SPEC.md 3.1 `- item` shape. If it has none, a minimal block is
|
|
395
|
+
* created — the entry becomes findable without forcing the caller to
|
|
396
|
+
* author YAML. Existing unrelated fields are preserved verbatim.
|
|
397
|
+
*/
|
|
398
|
+
export function withFrontmatterLists(content, lists = {}) {
|
|
399
|
+
const wanted = {};
|
|
400
|
+
for (const [k, v] of Object.entries(lists)) {
|
|
401
|
+
const arr = Array.isArray(v) ? v.map((s) => String(s).trim()).filter(Boolean) : [];
|
|
402
|
+
if (arr.length) wanted[k] = arr;
|
|
403
|
+
}
|
|
404
|
+
if (!Object.keys(wanted).length) return content;
|
|
405
|
+
|
|
406
|
+
const src = String(content || '');
|
|
407
|
+
const lines = src.split(/\r?\n/);
|
|
408
|
+
const hasFm = (lines[0] || '').trim() === '---' && lines.slice(1).some((l) => l.trim() === '---');
|
|
409
|
+
|
|
410
|
+
const renderList = (key, arr) => [`${key}:`, ...arr.map((a) => ` - ${JSON.stringify(a)}`)];
|
|
411
|
+
|
|
412
|
+
if (!hasFm) {
|
|
413
|
+
const fm = ['---'];
|
|
414
|
+
for (const [k, arr] of Object.entries(wanted)) fm.push(...renderList(k, arr));
|
|
415
|
+
fm.push('---', '');
|
|
416
|
+
return fm.join('\n') + src;
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
const end = lines.findIndex((l, i) => i > 0 && l.trim() === '---');
|
|
420
|
+
const head = lines.slice(1, end);
|
|
421
|
+
const { fields } = parseFrontmatter(src);
|
|
422
|
+
const out = [];
|
|
423
|
+
const done = new Set();
|
|
424
|
+
for (let i = 0; i < head.length; i++) {
|
|
425
|
+
const line = head[i];
|
|
426
|
+
const m = line.match(/^([^:\s][^:]*):\s*(.*)$/);
|
|
427
|
+
const key = m ? m[1].trim() : null;
|
|
428
|
+
if (key && wanted[key]) {
|
|
429
|
+
// Replace this key (and any nested/list lines under it) with the merged list.
|
|
430
|
+
const existing = listField(fields[key]);
|
|
431
|
+
const merged = Array.from(new Set([...existing, ...wanted[key]].map((s) => s.trim()).filter(Boolean)));
|
|
432
|
+
out.push(...renderList(key, merged));
|
|
433
|
+
done.add(key);
|
|
434
|
+
while (i + 1 < head.length && /^\s/.test(head[i + 1]) && head[i + 1].trim()) i++;
|
|
435
|
+
continue;
|
|
436
|
+
}
|
|
437
|
+
out.push(line);
|
|
438
|
+
}
|
|
439
|
+
for (const [k, arr] of Object.entries(wanted)) {
|
|
440
|
+
if (!done.has(k)) out.push(...renderList(k, arr));
|
|
441
|
+
}
|
|
442
|
+
return ['---', ...out, '---', ...lines.slice(end + 1)].join('\n');
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
// ── Search ───────────────────────────────────────────────────────
|
|
446
|
+
|
|
447
|
+
/**
|
|
448
|
+
* Search every memory file (all adapters + per-project configs).
|
|
449
|
+
* Returns ranked results with a passage each. Never throws on a bad file.
|
|
450
|
+
*/
|
|
451
|
+
export async function searchMemories(query, { limit = 10, root = home } = {}) {
|
|
452
|
+
const terms = queryTerms(query);
|
|
453
|
+
if (!terms.length) return { terms, results: [], total: 0 };
|
|
454
|
+
|
|
455
|
+
const docs = [];
|
|
456
|
+
for (const adapter of adapters) {
|
|
457
|
+
try { docs.push(...(await readMemoryFiles(adapter))); } catch {}
|
|
458
|
+
}
|
|
459
|
+
try { docs.push(...(await projectDocs(root))); } catch {}
|
|
460
|
+
|
|
461
|
+
const scored = [];
|
|
462
|
+
for (const doc of docs) {
|
|
463
|
+
const s = scoreDoc(doc, terms);
|
|
464
|
+
if (s.score > 0) scored.push({ doc, ...s });
|
|
465
|
+
}
|
|
466
|
+
scored.sort((a, b) => b.score - a.score || b.doc.mtimeMs - a.doc.mtimeMs);
|
|
467
|
+
|
|
468
|
+
const top = scored.slice(0, limit).map((r) => ({
|
|
469
|
+
tool: r.doc.tool,
|
|
470
|
+
path: r.doc.path,
|
|
471
|
+
type: r.doc.type,
|
|
472
|
+
description: r.doc.description,
|
|
473
|
+
score: r.score,
|
|
474
|
+
coverage: r.coverage,
|
|
475
|
+
matched: r.matched,
|
|
476
|
+
passage: extractPassage(r.doc, terms),
|
|
477
|
+
}));
|
|
478
|
+
|
|
479
|
+
return { terms, results: top, total: scored.length };
|
|
480
|
+
}
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Plain-text rendering shared by the MCP tool and `memoir recall`.
|
|
484
|
+
*/
|
|
485
|
+
export function formatRecallResults(query, { terms, results, total }) {
|
|
486
|
+
if (!results.length) {
|
|
487
|
+
return `No memories found matching "${query}"${terms.length ? ` (terms: ${terms.join(', ')})` : ''}.`;
|
|
488
|
+
}
|
|
489
|
+
const blocks = results.map((r, i) => {
|
|
490
|
+
const meta = [r.type, r.description].filter(Boolean).join(' · ');
|
|
491
|
+
const cov = r.matched < terms.length ? ` · ${r.matched}/${terms.length} terms` : '';
|
|
492
|
+
return [
|
|
493
|
+
`── ${i + 1}. ${r.tool} / ${r.path}${cov} ──`,
|
|
494
|
+
meta ? ` ${meta}` : null,
|
|
495
|
+
r.passage,
|
|
496
|
+
].filter(Boolean).join('\n');
|
|
497
|
+
});
|
|
498
|
+
const shown = results.length;
|
|
499
|
+
const head = total > shown
|
|
500
|
+
? `Found ${total} memories matching "${query}" — showing the top ${shown}. Use memoir_read for a full file.`
|
|
501
|
+
: `Found ${total} memor${total === 1 ? 'y' : 'ies'} matching "${query}":`;
|
|
502
|
+
return `${head}\n\n${blocks.join('\n\n')}`;
|
|
503
|
+
}
|
package/src/providers/index.js
CHANGED
|
@@ -23,6 +23,27 @@ export async function syncToLocal(config, stagingDir, spinner) {
|
|
|
23
23
|
await fs.ensureDir(resolvedDest);
|
|
24
24
|
|
|
25
25
|
await fs.copy(stagingDir, resolvedDest);
|
|
26
|
+
|
|
27
|
+
// Prune orphaned encrypted blobs. Each encrypted push derives a fresh salt
|
|
28
|
+
// and therefore fresh HMAC filenames, so without this every push leaves the
|
|
29
|
+
// previous push's data/*.enc behind forever and localPath grows without
|
|
30
|
+
// bound. Only runs for a full encrypted sync (manifest.enc present in what
|
|
31
|
+
// we just wrote) — `memoir snapshot` also calls syncToLocal with a staging
|
|
32
|
+
// dir of a single handoff file, and blanket-emptying the destination there
|
|
33
|
+
// would delete the user's backup.
|
|
34
|
+
try {
|
|
35
|
+
const stagedManifest = path.join(stagingDir, 'manifest.enc');
|
|
36
|
+
const destData = path.join(resolvedDest, 'data');
|
|
37
|
+
if (await fs.pathExists(stagedManifest) && await fs.pathExists(destData)) {
|
|
38
|
+
const keep = new Set(await fs.readdir(path.join(stagingDir, 'data')).catch(() => []));
|
|
39
|
+
for (const f of await fs.readdir(destData)) {
|
|
40
|
+
if (!keep.has(f)) await fs.remove(path.join(destData, f)).catch(() => {});
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
} catch {
|
|
44
|
+
// Pruning is housekeeping — never fail a completed backup over it.
|
|
45
|
+
}
|
|
46
|
+
|
|
26
47
|
spinner.succeed(chalk.green('Sync complete! ') + chalk.gray(`(Saved to ${resolvedDest})`));
|
|
27
48
|
await appendEvent('sync_pushed', { provider: 'local' });
|
|
28
49
|
}
|
package/src/security/scanner.js
CHANGED
|
@@ -31,7 +31,7 @@ const SECRET_PATTERNS = [
|
|
|
31
31
|
|
|
32
32
|
// Generic secrets in env/config patterns
|
|
33
33
|
{ regex: /(?:^|[\s;])(?:export\s+)?(?:API_KEY|SECRET_KEY|AUTH_TOKEN|ACCESS_TOKEN|PRIVATE_KEY|DB_PASSWORD|DATABASE_URL|JWT_SECRET|ENCRYPTION_KEY|MASTER_KEY)\s*=\s*["']?([^\s'"]{8,})/gmi, label: 'Environment variable secret' },
|
|
34
|
-
{ regex: /(?:password|passwd|pwd)\s*[:=]\s*["']?([^\s'"]{6,})/gi, label: 'Password' },
|
|
34
|
+
{ regex: /(?:password|passwd|pwd)\s*[:=]\s*["']?([^\s'"]{6,})/gi, label: 'Password', minLength: 6 },
|
|
35
35
|
|
|
36
36
|
// Private keys
|
|
37
37
|
{ regex: /(-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----)/g, label: 'Private key' },
|
|
@@ -55,10 +55,18 @@ export function scanForSecrets(text) {
|
|
|
55
55
|
let match;
|
|
56
56
|
while ((match = pattern.regex.exec(text)) !== null) {
|
|
57
57
|
const secret = match[1] || match[0];
|
|
58
|
-
//
|
|
59
|
-
|
|
58
|
+
// Per-pattern floor. A global 8 threw away 6-7 char matches that the
|
|
59
|
+
// Password pattern ({6,}) was written to catch: `password: s3cr3t`
|
|
60
|
+
// survived verbatim into the handoff and the backup while the scan
|
|
61
|
+
// reported "no secrets detected" — a silent miss is worse than a
|
|
62
|
+
// false positive in a tool that promises redaction.
|
|
63
|
+
if (secret.length < (pattern.minLength ?? 8)) continue;
|
|
60
64
|
|
|
61
|
-
|
|
65
|
+
// For short secrets, slice(0,4)+slice(-4) can reproduce the whole
|
|
66
|
+
// thing (a 6-char secret would show 4+4 of 6 characters).
|
|
67
|
+
const redacted = secret.length >= 12
|
|
68
|
+
? secret.slice(0, 4) + '****' + secret.slice(-4)
|
|
69
|
+
: secret.slice(0, 2) + '****';
|
|
62
70
|
findings.push({
|
|
63
71
|
label: pattern.label,
|
|
64
72
|
match: secret,
|
package/src/session/lock.js
CHANGED
|
@@ -68,7 +68,29 @@ export async function withSessionLock(lockPath, fn) {
|
|
|
68
68
|
try {
|
|
69
69
|
const stat = fs.statSync(lockPath);
|
|
70
70
|
if (Date.now() - stat.mtimeMs > STALE_MS) {
|
|
71
|
-
|
|
71
|
+
// Steal by rename, not unlink: two processes racing an unlink can
|
|
72
|
+
// both "win" and both proceed. rename() is atomic, so exactly one
|
|
73
|
+
// wins and the loser simply retries.
|
|
74
|
+
let stolen = false;
|
|
75
|
+
try {
|
|
76
|
+
const graveyard = `${lockPath}.stale-${process.pid}-${Date.now()}`;
|
|
77
|
+
fs.renameSync(lockPath, graveyard);
|
|
78
|
+
stolen = true;
|
|
79
|
+
// The rename is only there to make the steal atomic; the file
|
|
80
|
+
// itself is debris. Remove it immediately — best-effort, and
|
|
81
|
+
// harmless to leave behind if this fails.
|
|
82
|
+
try { fs.unlinkSync(graveyard); } catch {}
|
|
83
|
+
} catch {}
|
|
84
|
+
if (stolen) {
|
|
85
|
+
continue; // we removed it; retry the acquire immediately
|
|
86
|
+
}
|
|
87
|
+
// Could not remove it (read-only dir, permissions). Fall through
|
|
88
|
+
// to the deadline + backoff below instead of spinning forever.
|
|
89
|
+
if (Date.now() - start > MAX_WAIT_MS) {
|
|
90
|
+
fd = null;
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
await sleep(RETRY_DELAY_MS);
|
|
72
94
|
continue;
|
|
73
95
|
}
|
|
74
96
|
} catch {
|
|
@@ -95,8 +117,20 @@ export async function withSessionLock(lockPath, fn) {
|
|
|
95
117
|
return await fn();
|
|
96
118
|
} finally {
|
|
97
119
|
if (fd !== null) {
|
|
120
|
+
// Only unlink if the file at lockPath is still OURS. If our lock was
|
|
121
|
+
// stolen as stale and another process now holds a NEW file at the same
|
|
122
|
+
// path, unlinking by path would delete the current holder's lock and
|
|
123
|
+
// let a third process in. Compare inode via the fd we still hold.
|
|
124
|
+
let ours = false;
|
|
125
|
+
try {
|
|
126
|
+
const byFd = fs.fstatSync(fd);
|
|
127
|
+
const byPath = fs.statSync(lockPath);
|
|
128
|
+
ours = byFd.ino === byPath.ino && byFd.dev === byPath.dev;
|
|
129
|
+
} catch {
|
|
130
|
+
ours = false; // path gone or unreadable — nothing safe to remove
|
|
131
|
+
}
|
|
98
132
|
try { fs.closeSync(fd); } catch {}
|
|
99
|
-
try { fs.unlinkSync(lockPath); } catch {}
|
|
133
|
+
if (ours) { try { fs.unlinkSync(lockPath); } catch {} }
|
|
100
134
|
}
|
|
101
135
|
}
|
|
102
136
|
}
|