memoir-cli 3.11.2 → 3.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,503 @@
1
+ // Memory retrieval — the read side of memoir.
2
+ //
3
+ // Before 3.12 `memoir_recall` was `content.toLowerCase().includes(term)`
4
+ // per term, ranked by how many terms hit, and it returned the FIRST 500
5
+ // characters of each file. For a spec-shaped entry that is ~440 chars of
6
+ // YAML frontmatter and one truncated line of prose: the model asked a
7
+ // question, got ten headers back, and learned nothing. It also re-crawled
8
+ // ~1,000 directories under $HOME on every call to find project CLAUDE.md
9
+ // files (measured: 6,462 stat probes / call on a real machine).
10
+ //
11
+ // This module fixes the read path without adding a dependency:
12
+ // • matched PASSAGES, frontmatter stripped, with a line of context;
13
+ // • field-weighted scoring — aliases > name > description > headings >
14
+ // body — with per-term saturation and a coverage multiplier so a file
15
+ // that mentions all three query terms outranks one that hammers one;
16
+ // • light morphology: plural/-ing/-ed folding plus prefix matching from
17
+ // 4 chars, so "auth" finds "authentication" and "deploys" finds
18
+ // "deploy" (a real stemmer over-merges; this is deliberately timid);
19
+ // • an mtime-keyed parse cache and a TTL'd project-file index, so a
20
+ // long-lived MCP process stops re-reading and re-walking the disk.
21
+ //
22
+ // What it does NOT do: semantic/concept matching. "tiktok" still won't
23
+ // find a file that only ever says "vertical swipe feed". The honest,
24
+ // dependency-free answer to that is the `aliases:` frontmatter field
25
+ // (SPEC.md 3.2) written at save time and weighted heaviest here — the
26
+ // model that saves a memory knows what else it might be called.
27
+
28
+ import fs from 'fs-extra';
29
+ import path from 'path';
30
+ import os from 'os';
31
+ import { adapters } from '../adapters/index.js';
32
+ import { parseFrontmatter } from '../commands/validate.js';
33
+
34
+ const home = os.homedir();
35
+
36
+ // ── Tokenizing ───────────────────────────────────────────────────
37
+
38
+ const STOPWORDS = new Set([
39
+ 'the', 'a', 'an', 'of', 'to', 'in', 'on', 'for', 'and', 'or', 'is', 'it',
40
+ 'this', 'that', 'what', 'how', 'do', 'does', 'did', 'we', 'i', 'my', 'our',
41
+ 'with', 'about', 'was', 'were', 'be', 'are', 'at', 'by', 'from', 'as',
42
+ 'into', 'up', 'out', 'so', 'if', 'not', 'no', 'me', 'you', 'your', 'us',
43
+ 'why', 'when', 'where', 'which', 'who', 'can', 'should', 'would', 'could',
44
+ 'have', 'has', 'had', 'been', 'being', 'there', 'here', 'than', 'then',
45
+ ]);
46
+
47
+ // Minimum length at which a token may match another by prefix. Below this
48
+ // "in" would match "index" and "at" would match "attribution".
49
+ const PREFIX_MIN = 4;
50
+
51
+ export function tokenize(str) {
52
+ return String(str || '')
53
+ .toLowerCase()
54
+ .split(/[^a-z0-9_$./-]+/)
55
+ .flatMap((t) => t.split(/[./-]+/))
56
+ .map((t) => t.replace(/^[_$]+|[_$]+$/g, ''))
57
+ .filter((t) => t.length >= 2);
58
+ }
59
+
60
+ // Fold the commonest English inflections. Conservative on purpose: every
61
+ // rule keeps a stem of at least 4 characters so short words are untouched
62
+ // ("was" -> "was", not "wa"), and -ing/-ed only strip from words long
63
+ // enough that the residue is still a word ("deploying" -> "deploy",
64
+ // "ring" -> "ring").
65
+ export function normalize(token) {
66
+ let t = token;
67
+ if (t.length >= 5 && t.endsWith('ies')) t = t.slice(0, -3) + 'y';
68
+ else if (t.length >= 6 && t.endsWith('sses')) t = t.slice(0, -2);
69
+ else if (t.length >= 4 && t.endsWith('s') && !t.endsWith('ss')) t = t.slice(0, -1);
70
+ if (t.length >= 7 && t.endsWith('ing')) t = t.slice(0, -3);
71
+ else if (t.length >= 6 && t.endsWith('ed')) t = t.slice(0, -2);
72
+ return t;
73
+ }
74
+
75
+ // Does a normalized query term match a normalized document token?
76
+ // Returns 1 for an exact match, PREFIX_WEIGHT for a prefix match, 0 otherwise.
77
+ const PREFIX_WEIGHT = 0.6;
78
+ export function termMatch(term, token) {
79
+ if (term === token) return 1;
80
+ if (term.length >= PREFIX_MIN && token.startsWith(term)) return PREFIX_WEIGHT;
81
+ if (token.length >= PREFIX_MIN && term.startsWith(token)) return PREFIX_WEIGHT;
82
+ return 0;
83
+ }
84
+
85
+ export function queryTerms(query) {
86
+ const raw = tokenize(query);
87
+ const kept = raw.filter((t) => !STOPWORDS.has(t));
88
+ // If the whole query was stopwords ("what is it"), search on it anyway
89
+ // rather than returning nothing.
90
+ const terms = (kept.length ? kept : raw).map(normalize);
91
+ return Array.from(new Set(terms));
92
+ }
93
+
94
+ // ── Document model ───────────────────────────────────────────────
95
+
96
+ // Field weights. Aliases win because they exist for exactly one reason —
97
+ // they are the other names someone might search under. Then the entry's
98
+ // own name, its one-line description, headings, and finally prose.
99
+ const W = { aliases: 6, name: 4, description: 3, headings: 2, body: 1 };
100
+ // Body term-frequency saturates: 1 hit = 1.0, 3 = 1.5, 7 = 2.0. A file that
101
+ // says "deploy" forty times is not forty times more relevant.
102
+ const BODY_TF_CAP = 2;
103
+
104
+ function listField(v) {
105
+ if (Array.isArray(v)) return v.map(String);
106
+ if (typeof v === 'string' && v.trim()) return v.split(',').map((s) => s.trim()).filter(Boolean);
107
+ return [];
108
+ }
109
+
110
+ function fieldTokens(strings) {
111
+ const out = new Map(); // normalized token -> count
112
+ for (const s of strings) {
113
+ for (const t of tokenize(s)) {
114
+ const n = normalize(t);
115
+ out.set(n, (out.get(n) || 0) + 1);
116
+ }
117
+ }
118
+ return out;
119
+ }
120
+
121
+ /**
122
+ * Parse a raw memory file into the shape the scorer wants. Cheap enough to
123
+ * run on every file, but cached by (path, mtime, size) in readMemoryFiles.
124
+ */
125
+ export function buildDoc({ path: relPath, content, tool, absPath, mtimeMs }) {
126
+ const isMarkdown = /\.(md|markdown)$/i.test(relPath);
127
+ const { fields, body } = isMarkdown ? parseFrontmatter(content) : { fields: {}, body: content };
128
+ const bodyLines = body.split(/\r?\n/);
129
+ const headings = bodyLines.filter((l) => /^\s{0,3}#{1,6}\s/.test(l));
130
+
131
+ const nameStrings = [path.basename(relPath).replace(/\.[^.]+$/, '')];
132
+ if (fields.name) nameStrings.push(String(fields.name));
133
+
134
+ return {
135
+ path: relPath,
136
+ absPath,
137
+ tool,
138
+ mtimeMs: mtimeMs || 0,
139
+ isMarkdown,
140
+ type: fields.type || fields.metadata?.type || null,
141
+ description: fields.description ? String(fields.description) : '',
142
+ aliases: listField(fields.aliases),
143
+ tags: listField(fields.tags),
144
+ body,
145
+ bodyLines,
146
+ // Token maps per field
147
+ tf: {
148
+ aliases: fieldTokens([...listField(fields.aliases), ...listField(fields.tags)]),
149
+ name: fieldTokens(nameStrings),
150
+ description: fieldTokens([fields.description || '']),
151
+ headings: fieldTokens(headings),
152
+ body: fieldTokens(bodyLines),
153
+ },
154
+ // Kept for legacy callers (memoir_list sizes, memoir_read) — the raw file.
155
+ content,
156
+ };
157
+ }
158
+
159
+ // ── Scoring ──────────────────────────────────────────────────────
160
+
161
+ function fieldScore(tfMap, term) {
162
+ // Best match across the field's tokens: exact beats prefix; tf saturates.
163
+ let best = 0;
164
+ let count = 0;
165
+ for (const [token, n] of tfMap) {
166
+ const m = termMatch(term, token);
167
+ if (m > best) { best = m; count = n; }
168
+ else if (m === best && m > 0) count += n;
169
+ }
170
+ if (!best) return 0;
171
+ const tf = 0.5 + 0.5 * Math.log2(1 + count);
172
+ return best * Math.min(BODY_TF_CAP, tf);
173
+ }
174
+
175
+ // A single term's evidence in one document is its best field hit plus a
176
+ // quarter of the rest, capped. Without the cap, a term that appears in the
177
+ // alias list AND the name AND the description AND a heading (which is
178
+ // exactly what a well-formed entry looks like) stacked to ~16 and let a
179
+ // one-term match outrank a file that covered every term in the query.
180
+ const TERM_CAP = 8;
181
+
182
+ export function scoreDoc(doc, terms) {
183
+ let sum = 0;
184
+ let matched = 0;
185
+ const perTerm = {};
186
+ for (const term of terms) {
187
+ const fieldScores = Object.keys(W).map((f) => W[f] * fieldScore(doc.tf[f], term)).sort((a, b) => b - a);
188
+ const s = Math.min(TERM_CAP, fieldScores[0] + 0.25 * fieldScores.slice(1).reduce((x, y) => x + y, 0));
189
+ if (s > 0) matched++;
190
+ perTerm[term] = s;
191
+ sum += s;
192
+ }
193
+ if (!matched) return { score: 0, matched: 0, perTerm };
194
+ const coverage = matched / terms.length;
195
+ // Coverage-squared: a file that covers all the terms beats a file that
196
+ // covers a third of them unless the partial match is overwhelming
197
+ // (2/3 coverage keeps 44% of its raw score, 1/3 keeps 11%). This is the
198
+ // "over-recall" fix — a common word no longer drags in every file that
199
+ // mentions it once. Non-markdown (settings.json etc.) is de-weighted:
200
+ // it is config, not memory, and matches on it are usually noise.
201
+ const score = sum * coverage * coverage * (doc.isMarkdown ? 1 : 0.5);
202
+ return { score, matched, coverage, perTerm };
203
+ }
204
+
205
+ // ── Passage extraction ───────────────────────────────────────────
206
+
207
+ const PASSAGE_BUDGET = 700; // chars per result
208
+ const CONTEXT_LINES = 1; // lines of context either side of a hit
209
+ const MAX_LINE = 240; // clamp very long lines (pasted JSON, tables)
210
+
211
+ function lineMatches(line, terms) {
212
+ const toks = tokenize(line).map(normalize);
213
+ if (!toks.length) return 0;
214
+ let hits = 0;
215
+ for (const term of terms) {
216
+ if (toks.some((t) => termMatch(term, t) > 0)) hits++;
217
+ }
218
+ return hits;
219
+ }
220
+
221
+ function clampLine(l) {
222
+ const s = l.replace(/\s+$/, '');
223
+ return s.length > MAX_LINE ? s.slice(0, MAX_LINE - 1) + '…' : s;
224
+ }
225
+
226
+ /**
227
+ * The passage(s) of a document that best answer the query: matched lines
228
+ * with a line of context either side, best windows first, within a byte
229
+ * budget. Frontmatter is never included — callers get the description
230
+ * separately for the header line.
231
+ */
232
+ export function extractPassage(doc, terms, budget = PASSAGE_BUDGET) {
233
+ const lines = doc.bodyLines;
234
+ const hits = [];
235
+ for (let i = 0; i < lines.length; i++) {
236
+ const l = lines[i];
237
+ if (!l.trim()) continue;
238
+ const h = lineMatches(l, terms);
239
+ if (h) hits.push({ i, h });
240
+ }
241
+
242
+ if (!hits.length) {
243
+ // Match lived in frontmatter only (name/description/aliases). Show the
244
+ // opening of the body so the reader still gets substance.
245
+ const opening = lines.filter((l) => l.trim()).slice(0, 6).map(clampLine).join('\n');
246
+ return opening.slice(0, budget);
247
+ }
248
+
249
+ // Merge hits into windows [start, end] with context, then rank windows by
250
+ // (distinct terms matched desc, position asc).
251
+ const windows = [];
252
+ for (const { i, h } of hits) {
253
+ const start = Math.max(0, i - CONTEXT_LINES);
254
+ const end = Math.min(lines.length - 1, i + CONTEXT_LINES);
255
+ const last = windows[windows.length - 1];
256
+ if (last && start <= last.end + 1) {
257
+ last.end = Math.max(last.end, end);
258
+ last.h = Math.max(last.h, h);
259
+ last.hits++;
260
+ } else {
261
+ windows.push({ start, end, h, hits: 1 });
262
+ }
263
+ }
264
+ windows.sort((a, b) => b.h - a.h || b.hits - a.hits || a.start - b.start);
265
+
266
+ const chosen = [];
267
+ let used = 0;
268
+ for (const w of windows) {
269
+ const text = lines.slice(w.start, w.end + 1).filter((l) => l.trim()).map(clampLine).join('\n');
270
+ if (!text) continue;
271
+ if (used && used + text.length > budget) continue;
272
+ chosen.push({ ...w, text: used + text.length > budget ? text.slice(0, budget - used - 1) + '…' : text });
273
+ used += text.length + 2;
274
+ if (used >= budget) break;
275
+ }
276
+ // Present in document order so the excerpt reads naturally.
277
+ chosen.sort((a, b) => a.start - b.start);
278
+ return chosen.map((c) => c.text).join('\n⋯\n');
279
+ }
280
+
281
+ // ── Reading + caching ────────────────────────────────────────────
282
+
283
+ // (absPath) -> { mtimeMs, size, doc }. The MCP server is a long-lived
284
+ // process; memory files change rarely relative to how often recall runs.
285
+ const docCache = new Map();
286
+
287
+ async function readDoc(absPath, relPath, tool) {
288
+ let st;
289
+ try { st = await fs.stat(absPath); } catch { return null; }
290
+ const hit = docCache.get(absPath);
291
+ if (hit && hit.mtimeMs === st.mtimeMs && hit.size === st.size) return hit.doc;
292
+ let content;
293
+ try { content = await fs.readFile(absPath, 'utf8'); } catch { return null; }
294
+ const doc = buildDoc({ path: relPath, content, tool, absPath, mtimeMs: st.mtimeMs });
295
+ docCache.set(absPath, { mtimeMs: st.mtimeMs, size: st.size, doc });
296
+ return doc;
297
+ }
298
+
299
+ /** Test hook — drop every cached parse. */
300
+ export function clearSearchCache() {
301
+ docCache.clear();
302
+ projectIndex.at = 0;
303
+ projectIndex.files = [];
304
+ }
305
+
306
+ const MEMORY_EXT = /\.(md|json|ya?ml)$/i;
307
+
308
+ /**
309
+ * Read every memory file an adapter owns, as parsed docs. Cached by mtime.
310
+ */
311
+ export async function readMemoryFiles(adapter) {
312
+ const files = [];
313
+
314
+ if (adapter.customExtract) {
315
+ for (const file of adapter.files) {
316
+ const abs = path.join(adapter.source, file);
317
+ const doc = await readDoc(abs, file, adapter.name);
318
+ if (doc) files.push(doc);
319
+ }
320
+ return files;
321
+ }
322
+
323
+ if (!(await fs.pathExists(adapter.source))) return files;
324
+
325
+ const walk = async (dir, prefix = '') => {
326
+ let entries;
327
+ try { entries = await fs.readdir(dir, { withFileTypes: true }); } catch { return; }
328
+ for (const entry of entries) {
329
+ const fullPath = path.join(dir, entry.name);
330
+ const relPath = prefix ? `${prefix}/${entry.name}` : entry.name;
331
+ if (entry.isDirectory()) {
332
+ if (adapter.filter(fullPath)) await walk(fullPath, relPath);
333
+ } else if (MEMORY_EXT.test(entry.name) && adapter.filter(fullPath)) {
334
+ const doc = await readDoc(fullPath, relPath, adapter.name);
335
+ if (doc) files.push(doc);
336
+ }
337
+ }
338
+ };
339
+
340
+ await walk(adapter.source);
341
+ return files;
342
+ }
343
+
344
+ // Per-project AI config files. Discovery (the directory walk) is the
345
+ // expensive part and changes rarely, so the found-path list is cached for
346
+ // PROJECT_INDEX_TTL_MS; the files themselves go through readDoc's mtime cache.
347
+ const PROJECT_FILES = ['CLAUDE.md', 'GEMINI.md', 'CHATGPT.md', 'AGENTS.md', '.cursorrules', '.windsurfrules', '.clinerules'];
348
+ const SKIP_DIRS = new Set(['node_modules', '.git', '.next', '.vercel', 'dist', 'build', '__pycache__', '.venv', 'venv', '.cache', 'Library', '.Trash', 'Applications', 'Downloads', 'Movies', 'Music', 'Pictures']);
349
+ const PROJECT_INDEX_TTL_MS = 60_000;
350
+ const PROJECT_SCAN_DEPTH = 3;
351
+ const projectIndex = { at: 0, files: [] };
352
+
353
+ async function discoverProjectFiles(root) {
354
+ const found = [];
355
+ const scan = async (dir, depth) => {
356
+ if (depth > PROJECT_SCAN_DEPTH) return;
357
+ let entries;
358
+ try { entries = await fs.readdir(dir, { withFileTypes: true }); } catch { return; }
359
+ const names = new Set(entries.filter((e) => e.isFile()).map((e) => e.name));
360
+ for (const f of PROJECT_FILES) {
361
+ if (names.has(f)) found.push({ abs: path.join(dir, f), rel: `${path.basename(dir)}/${f}`, project: path.basename(dir) });
362
+ }
363
+ for (const entry of entries) {
364
+ if (!entry.isDirectory()) continue;
365
+ if (entry.name.startsWith('.') && entry.name !== '.github') continue;
366
+ if (SKIP_DIRS.has(entry.name)) continue;
367
+ await scan(path.join(dir, entry.name), depth + 1);
368
+ }
369
+ };
370
+ await scan(root, 0);
371
+ return found;
372
+ }
373
+
374
+ async function projectDocs(root = home) {
375
+ const now = Date.now();
376
+ if (now - projectIndex.at > PROJECT_INDEX_TTL_MS || projectIndex.root !== root) {
377
+ projectIndex.files = await discoverProjectFiles(root);
378
+ projectIndex.at = now;
379
+ projectIndex.root = root;
380
+ }
381
+ const docs = [];
382
+ for (const f of projectIndex.files) {
383
+ const doc = await readDoc(f.abs, f.rel, `Project: ${f.project}`);
384
+ if (doc) docs.push(doc);
385
+ }
386
+ return docs;
387
+ }
388
+
389
+ // ── Write-side helper: aliases/tags into frontmatter ─────────────
390
+
391
+ /**
392
+ * Merge `aliases` / `tags` lists into a markdown entry's frontmatter.
393
+ * If the content has frontmatter, lists are added (or extended, deduped)
394
+ * in the SPEC.md 3.1 `- item` shape. If it has none, a minimal block is
395
+ * created — the entry becomes findable without forcing the caller to
396
+ * author YAML. Existing unrelated fields are preserved verbatim.
397
+ */
398
+ export function withFrontmatterLists(content, lists = {}) {
399
+ const wanted = {};
400
+ for (const [k, v] of Object.entries(lists)) {
401
+ const arr = Array.isArray(v) ? v.map((s) => String(s).trim()).filter(Boolean) : [];
402
+ if (arr.length) wanted[k] = arr;
403
+ }
404
+ if (!Object.keys(wanted).length) return content;
405
+
406
+ const src = String(content || '');
407
+ const lines = src.split(/\r?\n/);
408
+ const hasFm = (lines[0] || '').trim() === '---' && lines.slice(1).some((l) => l.trim() === '---');
409
+
410
+ const renderList = (key, arr) => [`${key}:`, ...arr.map((a) => ` - ${JSON.stringify(a)}`)];
411
+
412
+ if (!hasFm) {
413
+ const fm = ['---'];
414
+ for (const [k, arr] of Object.entries(wanted)) fm.push(...renderList(k, arr));
415
+ fm.push('---', '');
416
+ return fm.join('\n') + src;
417
+ }
418
+
419
+ const end = lines.findIndex((l, i) => i > 0 && l.trim() === '---');
420
+ const head = lines.slice(1, end);
421
+ const { fields } = parseFrontmatter(src);
422
+ const out = [];
423
+ const done = new Set();
424
+ for (let i = 0; i < head.length; i++) {
425
+ const line = head[i];
426
+ const m = line.match(/^([^:\s][^:]*):\s*(.*)$/);
427
+ const key = m ? m[1].trim() : null;
428
+ if (key && wanted[key]) {
429
+ // Replace this key (and any nested/list lines under it) with the merged list.
430
+ const existing = listField(fields[key]);
431
+ const merged = Array.from(new Set([...existing, ...wanted[key]].map((s) => s.trim()).filter(Boolean)));
432
+ out.push(...renderList(key, merged));
433
+ done.add(key);
434
+ while (i + 1 < head.length && /^\s/.test(head[i + 1]) && head[i + 1].trim()) i++;
435
+ continue;
436
+ }
437
+ out.push(line);
438
+ }
439
+ for (const [k, arr] of Object.entries(wanted)) {
440
+ if (!done.has(k)) out.push(...renderList(k, arr));
441
+ }
442
+ return ['---', ...out, '---', ...lines.slice(end + 1)].join('\n');
443
+ }
444
+
445
+ // ── Search ───────────────────────────────────────────────────────
446
+
447
+ /**
448
+ * Search every memory file (all adapters + per-project configs).
449
+ * Returns ranked results with a passage each. Never throws on a bad file.
450
+ */
451
+ export async function searchMemories(query, { limit = 10, root = home } = {}) {
452
+ const terms = queryTerms(query);
453
+ if (!terms.length) return { terms, results: [], total: 0 };
454
+
455
+ const docs = [];
456
+ for (const adapter of adapters) {
457
+ try { docs.push(...(await readMemoryFiles(adapter))); } catch {}
458
+ }
459
+ try { docs.push(...(await projectDocs(root))); } catch {}
460
+
461
+ const scored = [];
462
+ for (const doc of docs) {
463
+ const s = scoreDoc(doc, terms);
464
+ if (s.score > 0) scored.push({ doc, ...s });
465
+ }
466
+ scored.sort((a, b) => b.score - a.score || b.doc.mtimeMs - a.doc.mtimeMs);
467
+
468
+ const top = scored.slice(0, limit).map((r) => ({
469
+ tool: r.doc.tool,
470
+ path: r.doc.path,
471
+ type: r.doc.type,
472
+ description: r.doc.description,
473
+ score: r.score,
474
+ coverage: r.coverage,
475
+ matched: r.matched,
476
+ passage: extractPassage(r.doc, terms),
477
+ }));
478
+
479
+ return { terms, results: top, total: scored.length };
480
+ }
481
+
482
+ /**
483
+ * Plain-text rendering shared by the MCP tool and `memoir recall`.
484
+ */
485
+ export function formatRecallResults(query, { terms, results, total }) {
486
+ if (!results.length) {
487
+ return `No memories found matching "${query}"${terms.length ? ` (terms: ${terms.join(', ')})` : ''}.`;
488
+ }
489
+ const blocks = results.map((r, i) => {
490
+ const meta = [r.type, r.description].filter(Boolean).join(' · ');
491
+ const cov = r.matched < terms.length ? ` · ${r.matched}/${terms.length} terms` : '';
492
+ return [
493
+ `── ${i + 1}. ${r.tool} / ${r.path}${cov} ──`,
494
+ meta ? ` ${meta}` : null,
495
+ r.passage,
496
+ ].filter(Boolean).join('\n');
497
+ });
498
+ const shown = results.length;
499
+ const head = total > shown
500
+ ? `Found ${total} memories matching "${query}" — showing the top ${shown}. Use memoir_read for a full file.`
501
+ : `Found ${total} memor${total === 1 ? 'y' : 'ies'} matching "${query}":`;
502
+ return `${head}\n\n${blocks.join('\n\n')}`;
503
+ }
@@ -23,6 +23,27 @@ export async function syncToLocal(config, stagingDir, spinner) {
23
23
  await fs.ensureDir(resolvedDest);
24
24
 
25
25
  await fs.copy(stagingDir, resolvedDest);
26
+
27
+ // Prune orphaned encrypted blobs. Each encrypted push derives a fresh salt
28
+ // and therefore fresh HMAC filenames, so without this every push leaves the
29
+ // previous push's data/*.enc behind forever and localPath grows without
30
+ // bound. Only runs for a full encrypted sync (manifest.enc present in what
31
+ // we just wrote) — `memoir snapshot` also calls syncToLocal with a staging
32
+ // dir of a single handoff file, and blanket-emptying the destination there
33
+ // would delete the user's backup.
34
+ try {
35
+ const stagedManifest = path.join(stagingDir, 'manifest.enc');
36
+ const destData = path.join(resolvedDest, 'data');
37
+ if (await fs.pathExists(stagedManifest) && await fs.pathExists(destData)) {
38
+ const keep = new Set(await fs.readdir(path.join(stagingDir, 'data')).catch(() => []));
39
+ for (const f of await fs.readdir(destData)) {
40
+ if (!keep.has(f)) await fs.remove(path.join(destData, f)).catch(() => {});
41
+ }
42
+ }
43
+ } catch {
44
+ // Pruning is housekeeping — never fail a completed backup over it.
45
+ }
46
+
26
47
  spinner.succeed(chalk.green('Sync complete! ') + chalk.gray(`(Saved to ${resolvedDest})`));
27
48
  await appendEvent('sync_pushed', { provider: 'local' });
28
49
  }
@@ -31,7 +31,7 @@ const SECRET_PATTERNS = [
31
31
 
32
32
  // Generic secrets in env/config patterns
33
33
  { regex: /(?:^|[\s;])(?:export\s+)?(?:API_KEY|SECRET_KEY|AUTH_TOKEN|ACCESS_TOKEN|PRIVATE_KEY|DB_PASSWORD|DATABASE_URL|JWT_SECRET|ENCRYPTION_KEY|MASTER_KEY)\s*=\s*["']?([^\s'"]{8,})/gmi, label: 'Environment variable secret' },
34
- { regex: /(?:password|passwd|pwd)\s*[:=]\s*["']?([^\s'"]{6,})/gi, label: 'Password' },
34
+ { regex: /(?:password|passwd|pwd)\s*[:=]\s*["']?([^\s'"]{6,})/gi, label: 'Password', minLength: 6 },
35
35
 
36
36
  // Private keys
37
37
  { regex: /(-----BEGIN (?:RSA |EC |DSA |OPENSSH )?PRIVATE KEY-----)/g, label: 'Private key' },
@@ -55,10 +55,18 @@ export function scanForSecrets(text) {
55
55
  let match;
56
56
  while ((match = pattern.regex.exec(text)) !== null) {
57
57
  const secret = match[1] || match[0];
58
- // Skip very short matches (likely false positives)
59
- if (secret.length < 8) continue;
58
+ // Per-pattern floor. A global 8 threw away 6-7 char matches that the
59
+ // Password pattern ({6,}) was written to catch: `password: s3cr3t`
60
+ // survived verbatim into the handoff and the backup while the scan
61
+ // reported "no secrets detected" — a silent miss is worse than a
62
+ // false positive in a tool that promises redaction.
63
+ if (secret.length < (pattern.minLength ?? 8)) continue;
60
64
 
61
- const redacted = secret.slice(0, 4) + '****' + secret.slice(-4);
65
+ // For short secrets, slice(0,4)+slice(-4) can reproduce the whole
66
+ // thing (a 6-char secret would show 4+4 of 6 characters).
67
+ const redacted = secret.length >= 12
68
+ ? secret.slice(0, 4) + '****' + secret.slice(-4)
69
+ : secret.slice(0, 2) + '****';
62
70
  findings.push({
63
71
  label: pattern.label,
64
72
  match: secret,
@@ -68,7 +68,29 @@ export async function withSessionLock(lockPath, fn) {
68
68
  try {
69
69
  const stat = fs.statSync(lockPath);
70
70
  if (Date.now() - stat.mtimeMs > STALE_MS) {
71
- try { fs.unlinkSync(lockPath); } catch {}
71
+ // Steal by rename, not unlink: two processes racing an unlink can
72
+ // both "win" and both proceed. rename() is atomic, so exactly one
73
+ // wins and the loser simply retries.
74
+ let stolen = false;
75
+ try {
76
+ const graveyard = `${lockPath}.stale-${process.pid}-${Date.now()}`;
77
+ fs.renameSync(lockPath, graveyard);
78
+ stolen = true;
79
+ // The rename is only there to make the steal atomic; the file
80
+ // itself is debris. Remove it immediately — best-effort, and
81
+ // harmless to leave behind if this fails.
82
+ try { fs.unlinkSync(graveyard); } catch {}
83
+ } catch {}
84
+ if (stolen) {
85
+ continue; // we removed it; retry the acquire immediately
86
+ }
87
+ // Could not remove it (read-only dir, permissions). Fall through
88
+ // to the deadline + backoff below instead of spinning forever.
89
+ if (Date.now() - start > MAX_WAIT_MS) {
90
+ fd = null;
91
+ break;
92
+ }
93
+ await sleep(RETRY_DELAY_MS);
72
94
  continue;
73
95
  }
74
96
  } catch {
@@ -95,8 +117,20 @@ export async function withSessionLock(lockPath, fn) {
95
117
  return await fn();
96
118
  } finally {
97
119
  if (fd !== null) {
120
+ // Only unlink if the file at lockPath is still OURS. If our lock was
121
+ // stolen as stale and another process now holds a NEW file at the same
122
+ // path, unlinking by path would delete the current holder's lock and
123
+ // let a third process in. Compare inode via the fd we still hold.
124
+ let ours = false;
125
+ try {
126
+ const byFd = fs.fstatSync(fd);
127
+ const byPath = fs.statSync(lockPath);
128
+ ours = byFd.ino === byPath.ino && byFd.dev === byPath.dev;
129
+ } catch {
130
+ ours = false; // path gone or unreadable — nothing safe to remove
131
+ }
98
132
  try { fs.closeSync(fd); } catch {}
99
- try { fs.unlinkSync(lockPath); } catch {}
133
+ if (ours) { try { fs.unlinkSync(lockPath); } catch {} }
100
134
  }
101
135
  }
102
136
  }