memoir-cli 3.11.3 → 3.14.0

package/src/memory/store.js

@@ -0,0 +1,141 @@
+import fs from 'fs-extra';
+import path from 'path';
+import os from 'os';
+import crypto from 'crypto';
+import { parseFrontmatter } from '../commands/validate.js';
+import { memoryFilename, readSafeFile, writeSafeFile, listSafeFiles, safePath, MAX_FILE_BYTES } from '../security/files.js';
+import { withSessionLock } from '../session/lock.js';
+import { projectIdentity } from './scope.js';
+
+export const memoryRoot = path.join(os.homedir(), '.config', 'memoir', 'memories');
+
+export async function rememberMemory({ filename, content, project, scope = 'project', tool = 'memoir', aliases = [], tags = [] }) {
+  const name = memoryFilename(filename);
+  if (Buffer.byteLength(content) > MAX_FILE_BYTES) throw new Error('Memory exceeds the size limit');
+  const projectId = scope === 'shared' ? 'shared' : projectIdentity(project);
+  const id = crypto.createHash('sha256').update(projectId + '\0' + name).digest('hex');
+  return withSessionLock(path.join(memoryRoot, '.write.lock'), async () => {
+    const rel = id + '.md';
+    let prior = null;
+    try { prior = parseFrontmatter((await readSafeFile(memoryRoot, rel)).toString('utf8')); }
+    catch (err) { if (err.code !== 'ENOENT') throw err; }
+    if (prior?.fields.hidden === true) throw new Error('This memory identity was forgotten. Choose a new filename for a new memory.');
+    const parsed = parseFrontmatter(content);
+    if (parsed.error) throw new Error(parsed.error);
+    const now = new Date().toISOString();
+    const fields = {
+      ...parsed.fields,
+      id, name: parsed.fields.name || name.replace(/\.md$/, ''),
+      type: parsed.fields.type || 'fact',
+      project: projectId, source_tool: tool, updated: now,
+      created: prior?.fields.created || now,
+      revision: Number(prior?.fields.revision || 0) + 1,
+      verification: 'unverified',
+      aliases: [...new Set([...(Array.isArray(parsed.fields.aliases) ? parsed.fields.aliases : []), ...aliases])],
+      tags: [...new Set([...(Array.isArray(parsed.fields.tags) ? parsed.fields.tags : []), ...tags])],
+    };
+    const lines = ['---'];
+    for (const [key, value] of Object.entries(fields)) {
+      if (!/^[a-z][a-z0-9_]*$/i.test(key)) continue;
+      if (Array.isArray(value)) lines.push(key + ':', ...value.map(v => '  - ' + JSON.stringify(String(v))));
+      else if (value != null && typeof value !== 'object') lines.push(key + ': ' + JSON.stringify(value));
+    }
+    const rendered = lines.concat(['---', parsed.body]).join('\n');
+    // Keep previous revisions separate from the searchable current record.
+    if (prior) await writeSafeFile(memoryRoot, 'history/' + id + '/' + crypto.randomUUID() + '.md', await readSafeFile(memoryRoot, rel));
+    await writeSafeFile(memoryRoot, rel, rendered);
+    return { id, revision: fields.revision, project: projectId, path: rel };
+  });
+}
+
+export async function readStoredMemories() {
+  if (!await fs.pathExists(memoryRoot)) return [];
+  const docs = [];
+  for (const entry of await fs.readdir(memoryRoot, { withFileTypes: true })) {
+    if (!entry.isFile() || !/^[a-f0-9]{64}\.md$/.test(entry.name)) continue;
+    docs.push({ path: entry.name, absPath: path.join(memoryRoot, entry.name), content: (await readSafeFile(memoryRoot, entry.name)).toString('utf8'), tool: 'Memoir' });
+  }
+  return docs;
+}
+
+export async function stageMemories(dest) {
+  if (!await fs.pathExists(memoryRoot)) return 0;
+  let count = 0;
+  for (const rel of await listSafeFiles(memoryRoot)) {
+    if (rel.endsWith('.lock') || rel.includes('.memoir-write-')) continue;
+    await writeSafeFile(dest, 'memoir-memories/' + rel, await readSafeFile(memoryRoot, rel));
+    count++;
+  }
+  return count;
+}
+
+export async function restoreStoredMemories(source) {
+  const dir = path.join(source, 'memoir-memories');
+  if (!await fs.pathExists(dir)) return 0;
+  return withSessionLock(path.join(memoryRoot, '.write.lock'), async () => {
+    const entries = [], purged = new Set();
+    // Parse and validate the entire incoming set before modifying records.
+    for (const rel of await listSafeFiles(dir)) {
+      if (!/^(?:[a-f0-9]{64}\.md|history\/[a-f0-9]{64}\/[a-f0-9-]+\.md)$/.test(rel)) throw new Error('Invalid canonical memory path');
+      const id = rel.startsWith('history/') ? rel.split('/')[1] : rel.slice(0, -3);
+      const incoming = await readSafeFile(dir, rel);
+      const parsed = parseFrontmatter(incoming.toString());
+      if (parsed.error || parsed.fields.id !== id || !/^(?:shared|(?:git|local):[a-f0-9]{32})$/.test(parsed.fields.project || '')) throw new Error('Invalid canonical memory record');
+      await safePath(memoryRoot, rel);
+      let local;
+      try { local = await readSafeFile(memoryRoot, rel); } catch (err) { if (err.code !== 'ENOENT') throw err; }
+      if (!rel.startsWith('history/') && (parsed.fields.purged === true || (local && parseFrontmatter(local.toString()).fields.purged === true))) purged.add(id);
+      entries.push({ rel, id, incoming, parsed, local });
+    }
+    // A stale backup might contain history without its current record.
+    for (const entry of await readStoredMemories()) {
+      const fields = parseFrontmatter(entry.content).fields;
+      if (fields.purged === true) purged.add(fields.id);
+    }
+    let count = 0;
+    for (const { rel, id, incoming, parsed, local } of entries) {
+      if (rel.startsWith('history/') && purged.has(id)) continue;
+      if (local && !local.equals(incoming) && !rel.startsWith('history/')) {
+        const a = parseFrontmatter(local.toString()).fields, b = parsed.fields;
+        const rank = fields => fields.purged === true ? 2 : fields.hidden === true ? 1 : 0;
+        const digest = bytes => crypto.createHash('sha256').update(bytes).digest('hex');
+        const incomingWins = rank(b) > rank(a) || (rank(b) === rank(a) &&
+          (String(b.updated || '') > String(a.updated || '') ||
+          (String(b.updated || '') === String(a.updated || '') && digest(incoming) > digest(local))));
+        if (!purged.has(id)) await writeSafeFile(memoryRoot, 'history/' + id + '/' + crypto.randomUUID() + '.md', incomingWins ? local : incoming);
+        if (!incomingWins) continue;
+      }
+      await writeSafeFile(memoryRoot, rel, incoming);
+      count++;
+    }
+    for (const id of purged) await fs.remove(path.join(memoryRoot, 'history', id));
+    return count;
+  });
+}
+
+// Hiding is durable and syncable. Purge affects this device's current record
+// and its local revision history, not older remote snapshots or Git history.
+export async function forgetStoredMemory(id, { purge = false, project } = {}) {
+  if (!/^[a-f0-9]{64}$/.test(id)) throw new Error('Invalid memory ID');
+  return withSessionLock(path.join(memoryRoot, '.write.lock'), async () => {
+    const rel = id + '.md';
+    const raw = await readSafeFile(memoryRoot, rel);
+    const { fields, body } = parseFrontmatter(raw.toString());
+    const { visibleMemory } = await import('./scope.js');
+    if (!visibleMemory(fields, { project })) throw new Error('Memory is hidden or outside this project');
+    const metadata = purge ? { id, project: fields.project, type: fields.type || 'fact' } : fields;
+    metadata.hidden = true;
+    if (purge) metadata.purged = true;
+    metadata.hidden_at = new Date().toISOString();
+    metadata.updated = metadata.hidden_at;
+    metadata.revision = Number(fields.revision || 0) + 1;
+    const lines = ['---'];
+    for (const [key, value] of Object.entries(metadata)) {
+      if (Array.isArray(value)) lines.push(key + ':', ...value.map(v => '  - ' + JSON.stringify(v)));
+      else if (value != null && typeof value !== 'object') lines.push(key + ': ' + JSON.stringify(value));
+    }
+    await writeSafeFile(memoryRoot, rel, lines.concat(['---', purge ? '[purged]' : body]).join('\n'));
+    if (purge) await fs.remove(path.join(memoryRoot, 'history', id));
+    return { id, hidden: true, purged: purge };
+  });
+}

package/src/providers/index.js

@@ -4,6 +4,8 @@ import os from 'os';
 import chalk from 'chalk';
 import { execFileSync } from 'child_process';
 import { appendEvent } from '../events/log.js';
+import { listSafeFiles, readSafeFile, writeSafeFile, restoreFileSet } from '../security/files.js';
+import { withSessionLock } from '../session/lock.js';
 
 function sanitizeUrl(url) {
   // Reject URLs with shell metacharacters
@@ -13,99 +15,228 @@ function sanitizeUrl(url) {
   return url;
 }
 
-export async function syncToLocal(config, stagingDir, spinner) {
-  const destDir = config.localPath;
-  if (!destDir) throw new Error('Local path is not configured.');
+export async function withLocalBackupLock(config, fn) {
+  const dest = path.resolve(config.localPath.replace(/^~/, os.homedir()));
+  return withSessionLock(dest + '.memoir-lock', fn, { maxWaitMs: 5000 });
+}
 
-  const resolvedDest = destDir.replace(/^~/, os.homedir());
+export async function syncToLocal(config, stagingDir, spinner, options = {}) {
+  if (!config.localPath) throw new Error('Local path is not configured.');
+  const dest = path.resolve(config.localPath.replace(/^~/, os.homedir()));
+  if (dest === path.parse(dest).root || dest === path.resolve(os.homedir()) || path.resolve(stagingDir).startsWith(dest + path.sep)) throw new Error('Choose a dedicated backup directory');
+  const sync = async () => {
+    const stat = await fs.lstat(dest).catch(err => { if (err.code !== 'ENOENT') throw err; return null; });
+    if (stat?.isSymbolicLink()) throw new Error('Backup destination must not be a symlink');
+    const encrypted = await fs.pathExists(path.join(stagingDir, 'manifest.enc'));
+    const exists = await fs.pathExists(dest);
+    const wasEncrypted = exists && await fs.pathExists(path.join(dest, 'manifest.enc'));
+    if (encrypted) {
+      if (exists && !wasEncrypted && (await fs.readdir(dest)).length && !options.verifiedReplacement) throw new Error('Use memoir push to verify and migrate this plaintext backup before replacing it.');
+      const parent = path.dirname(dest);
+      await fs.ensureDir(parent);
+      const next = await fs.mkdtemp(path.join(parent, '.memoir-sync-'));
+      const old = next + '-previous';
+      try {
+        await listSafeFiles(stagingDir);
+        await fs.copy(stagingDir, next);
+        if (exists) await fs.rename(dest, old);
+        try { await fs.rename(next, dest); }
+        catch (err) { if (exists) await fs.rename(old, dest); throw err; }
+        if (exists) await fs.remove(old);
+      } finally { await fs.remove(next).catch(() => {}); }
+    } else {
+      if (wasEncrypted) throw new Error('Cannot append plaintext to an encrypted backup. Use memoir push with encryption enabled.');
+      const files = [];
+      for (const rel of await listSafeFiles(stagingDir)) files.push({ path: rel, content: await readSafeFile(stagingDir, rel) });
+      await restoreFileSet(dest, files);
+    }
+  };
+  if (options.lockHeld) await sync();
+  else await withLocalBackupLock(config, sync);
+  spinner.succeed(chalk.green('Sync complete! ') + chalk.gray('(Saved to ' + dest + ')'));
+  await appendEvent('sync_pushed', { provider: 'local' });
+}
 
-  spinner.text = `Syncing files to local directory: ${chalk.cyan(resolvedDest)}`;
-  await fs.ensureDir(resolvedDest);
+// ── Git sync ─────────────────────────────────────────────────────
+//
+// One cheap clone per push. `--depth 1 --filter=blob:none --no-checkout`
+// fetches commits and trees only — no file contents, no working tree. The
+// remote of the author's own store is 720MB with ~40MB of workspace bundles
+// at HEAD; the old full shallow clone, done TWICE per push (the merge peek in
+// push.js and then this function), pulled all of it through on every Stop
+// hook and a quarter of pushes died on the 60s timeout (183 sync_failed in
+// 15 days, none with a reason). Blobs are now fetched on demand only for the
+// files a caller actually reads (`git checkout HEAD -- session.json`).
+// Remotes that don't support filters (plain file:// paths, old hosts) print
+// a warning and fall back to a normal shallow clone — never worse than before.
+//
+// Mirror semantics are unchanged and come for free: after a --no-checkout
+// clone the index is empty and every remote file is a staged deletion until
+// something re-adds it, so `copy staging → git add -A` yields exactly
+// "staging + the files the caller asked to preserve".
+
+export const CLONE_TIMEOUT_MS = 60000;
+const GIT_QUIET = ['ignore', 'ignore', 'pipe']; // stderr kept for classifyGitError
+
+export function cloneForSync(repoUrl, dir, { timeout = CLONE_TIMEOUT_MS } = {}) {
+  execFileSync('git', ['clone', '--depth', '1', '--filter=blob:none', '--no-checkout', repoUrl, '.'], {
+    cwd: dir, stdio: GIT_QUIET, timeout,
+  });
+  // Memoir writes main; read that same branch even when remote HEAD is master.
+  const main = execFileSync('git', ['ls-remote', '--heads', 'origin', 'refs/heads/main'], {
+    cwd: dir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'], timeout,
+  }).trim();
+  if (main) {
+    execFileSync('git', ['fetch', '--depth', '1', 'origin', 'refs/heads/main'], { cwd: dir, stdio: GIT_QUIET, timeout });
+    execFileSync('git', ['update-ref', 'refs/heads/main', 'FETCH_HEAD'], { cwd: dir, stdio: GIT_QUIET, timeout });
+    execFileSync('git', ['symbolic-ref', 'HEAD', 'refs/heads/main'], { cwd: dir, stdio: GIT_QUIET, timeout });
+  }
+}
 
-  await fs.copy(stagingDir, resolvedDest);
+// Does the remote HEAD carry `file`? Reads the tree only — no blob fetch.
+export function remoteHasFile(dir, file) {
+  try {
+    const out = execFileSync('git', ['ls-tree', '--name-only', 'HEAD', '--', file], {
+      cwd: dir, encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'], timeout: 10000,
+    });
+    return out.trim().length > 0;
+  } catch {
+    return false; // unborn HEAD (empty remote) or not a repo
+  }
+}
 
-  // Prune orphaned encrypted blobs. Each encrypted push derives a fresh salt
-  // and therefore fresh HMAC filenames, so without this every push leaves the
-  // previous push's data/*.enc behind forever and localPath grows without
-  // bound. Only runs for a full encrypted sync (manifest.enc present in what
-  // we just wrote) — `memoir snapshot` also calls syncToLocal with a staging
-  // dir of a single handoff file, and blanket-emptying the destination there
-  // would delete the user's backup.
+// Materialise one remote file into the working tree (fetches its blob).
+export function checkoutFromRemote(dir, file) {
   try {
-    const stagedManifest = path.join(stagingDir, 'manifest.enc');
-    const destData = path.join(resolvedDest, 'data');
-    if (await fs.pathExists(stagedManifest) && await fs.pathExists(destData)) {
-      const keep = new Set(await fs.readdir(path.join(stagingDir, 'data')).catch(() => []));
-      for (const f of await fs.readdir(destData)) {
-        if (!keep.has(f)) await fs.remove(path.join(destData, f)).catch(() => {});
-      }
-    }
+    execFileSync('git', ['checkout', 'HEAD', '--', file], { cwd: dir, stdio: 'ignore', timeout: 30000 });
+    return true;
   } catch {
-    // Pruning is housekeeping — never fail a completed backup over it.
+    return false;
   }
+}
 
-  spinner.succeed(chalk.green('Sync complete! ') + chalk.gray(`(Saved to ${resolvedDest})`));
-  await appendEvent('sync_pushed', { provider: 'local' });
+// A short enum for the event log — never the raw stderr (it can carry the
+// repo URL, a username, a local path). Enough to tell "the network is down"
+// from "two pushes raced" from "the token expired" when reading events.jsonl.
+export function classifyGitError(err) {
+  const text = `${err?.message || ''}\n${err?.stderr || ''}`.toLowerCase();
+  if (err?.code === 'ETIMEDOUT' || err?.signal === 'SIGTERM' || /timed? ?out/.test(text)) return 'timeout';
+  if (/non-fast-forward|fetch first|\[rejected\]/.test(text)) return 'non_fast_forward';
+  // Match HTTP diagnostics, not a coincidental 403 in a path or timestamp.
+  if (/authentication failed|could not read username|could not read password|permission denied|terminal prompts disabled|invalid credentials|requested url returned error: 403\b|http(?:\/[\d.]+)?(?: error| status(?: code)?)?[: ]+403\b/.test(text)) return 'auth';
+  if (/could not resolve host|unable to access|connection (?:refused|reset|timed)|network is unreachable|early eof|remote end hung up/.test(text)) return 'network';
+  if (/repository not found|does not appear to be a git repository|does not exist|no such file/.test(text)) return 'not_found';
+  if (/invalid characters/.test(text)) return 'bad_url';
+  return 'unknown';
 }
 
+/**
+ * Push `stagingDir` to the git remote as the new HEAD:main.
+ *
+ * options:
+ *   cloneDir  — a directory already prepared by cloneForSync (push.js's merge
+ *               peek). Reused instead of cloning again; removed when done.
+ *   preserve  — remote files to keep even though staging lacks them
+ *               (an unreadable session.json the caller declined to overwrite).
+ *   additive  — overlay staging onto the remote tree instead of mirroring it
+ *               (`memoir snapshot` uploads ONE handoff file; mirroring would
+ *               have wiped every other file in the backup).
+ */
 export async function syncToGit(config, stagingDir, spinner, options = {}) {
   const repoUrl = sanitizeUrl(config.gitRepo);
   if (!repoUrl) throw new Error('Git repository is not configured.');
 
   spinner.text = `Authenticating and syncing with Git remote: ${chalk.cyan(repoUrl)}`;
 
-  const gitDir = path.join(os.tmpdir(), `memoir-git-${Date.now()}`);
+  const reuse = Boolean(options.cloneDir) && await fs.pathExists(path.join(options.cloneDir, '.git'));
+  const gitDir = reuse ? options.cloneDir : path.join(os.tmpdir(), `memoir-git-${Date.now()}`);
   await fs.ensureDir(gitDir);
+  const started = Date.now();
 
   try {
-    try {
-      execFileSync('git', ['clone', '--depth', '1', repoUrl, '.'], { cwd: gitDir, stdio: 'ignore', timeout: 60000 });
-      const preserve = new Set(options.preserve || []);
-      const files = await fs.readdir(gitDir);
-      for (const f of files) {
-        // preserve: files the caller knows exist remotely but could not
-        // merge (unreadable session.json) — deleting them here would be
-        // the mirror-clobber the push guard just declined to commit.
-        if (f !== '.git' && !preserve.has(f)) await fs.remove(path.join(gitDir, f));
+    if (!reuse) {
+      try {
+        cloneForSync(repoUrl, gitDir);
+      } catch (err) {
+        // Cloning an EMPTY remote succeeds (git warns and continues), so a
+        // failure here is a real one. Unreachable / unauthorised / timed out
+        // must surface as such — the old unconditional `git init` fallback
+        // turned every one of them into a later, misleading
+        // "non-fast-forward" or "credentials" failure. Only a not_found /
+        // unknown failure (a host that refuses to clone an empty repo) still
+        // falls back to init so a first push can create the history.
+        const reason = classifyGitError(err);
+        if (reason === 'timeout' || reason === 'auth' || reason === 'network') throw err;
+        execFileSync('git', ['init'], { cwd: gitDir, stdio: 'ignore' });
+        execFileSync('git', ['branch', '-m', 'main'], { cwd: gitDir, stdio: 'ignore' });
       }
-    } catch {
-      execFileSync('git', ['init'], { cwd: gitDir, stdio: 'ignore' });
-      execFileSync('git', ['branch', '-m', 'main'], { cwd: gitDir, stdio: 'ignore' });
     }
 
-    await fs.copy(stagingDir, gitDir);
+    // Files the caller wants kept although staging lacks them: check them
+    // out (one blob each) so `git add -A` below sees them in the tree.
+    const preserved = [];
+    for (const p of options.preserve || []) {
+      if (!checkoutFromRemote(gitDir, p)) throw new Error('Could not preserve remote file');
+      preserved.push({ path: p, content: await readSafeFile(gitDir, p) });
+    }
+    if (!options.additive) {
+      // The merge peek may have materialized plaintext. A complete encrypted
+      // replacement must remove it from the current tree before adding blobs.
+      for (const entry of await fs.readdir(gitDir)) {
+        if (entry !== '.git') await fs.remove(path.join(gitDir, entry));
+      }
+    }
 
-    execFileSync('git', ['add', '-A'], { cwd: gitDir, stdio: 'ignore', timeout: 30000 });
+
+    if (options.additive) {
+      // Populate the index from HEAD (trees only, no blobs) so everything
+      // the remote already has stays in the commit; then add ONLY what
+      // staging brought. `git add -A` over an empty working tree would
+      // stage every other remote file as deleted.
+      try { execFileSync('git', ['read-tree', 'HEAD'], { cwd: gitDir, stdio: 'ignore', timeout: 30000 }); } catch {}
+    }
+
+    const stagedFiles = await listSafeFiles(stagingDir);
+    await fs.copy(stagingDir, gitDir);
+    for (const entry of preserved) await writeSafeFile(gitDir, entry.path, entry.content);
+
+    if (options.additive) {
+      if (stagedFiles.length) execFileSync('git', ['add', '--pathspec-from-file=-', '--pathspec-file-nul'], {
+        cwd: gitDir, input: Buffer.from(stagedFiles.join('\0') + '\0'), stdio: ['pipe', 'ignore', 'pipe'], timeout: 30000,
+      });
+    } else {
+      execFileSync('git', ['add', '-A'], { cwd: gitDir, stdio: 'ignore', timeout: 30000 });
+    }
     execFileSync('git', ['config', 'user.name', 'memoir'], { cwd: gitDir, stdio: 'ignore', timeout: 5000 });
     execFileSync('git', ['config', 'user.email', 'bot@memoir.dev'], { cwd: gitDir, stdio: 'ignore', timeout: 5000 });
 
     const timestamp = new Date().toISOString().split('T')[0];
-    try {
-      execFileSync('git', ['commit', '-m', `memoir backup ${timestamp}`], { cwd: gitDir, stdio: 'ignore', timeout: 30000 });
-    } catch {
+    const changed = execFileSync('git', ['diff', '--cached', '--name-only'], { cwd: gitDir, encoding: 'utf8', timeout: 30000 });
+    if (!changed.trim()) {
       spinner.succeed(chalk.green('Already up to date! ') + chalk.gray('No changes to push.'));
       return;
     }
+    execFileSync('git', ['commit', '-q', '-m', 'memoir backup ' + timestamp], { cwd: gitDir, stdio: GIT_QUIET, timeout: 30000 });
 
     spinner.text = `Pushing data to ${chalk.cyan(repoUrl)}...`;
     // HEAD:main pushes whatever branch the clone checked out (a master-
     // default remote used to make `push main` fail silently under autopush
     // with a misleading credentials error, while doctor reported green).
-    execFileSync('git', ['push', repoUrl, 'HEAD:main'], { cwd: gitDir, stdio: 'ignore', timeout: 120000 });
+    execFileSync('git', ['push', '-q', repoUrl, 'HEAD:main'], { cwd: gitDir, stdio: GIT_QUIET, timeout: 120000 });
 
     spinner.succeed(chalk.green('Sync complete! ') + chalk.gray('(Uploaded securely to GitHub)'));
-    await appendEvent('sync_pushed', { provider: 'git' });
+    await appendEvent('sync_pushed', { provider: 'git', ms: Date.now() - started, reused_clone: reuse });
   } catch (err) {
     // Makes a silently-swallowed push failure (a non-fast-forward rejection
     // from two racing pushes, a network error, bad credentials, etc.)
     // visible in the event log instead of vanishing into the detached
-    // autopush child's ignored stdio. Deliberately no raw error text/repo
-    // URL in the payload — those can contain usernames/paths; type+provider
-    // is enough to know "pushes are failing" without leaking anything.
-    await appendEvent('sync_failed', { provider: 'git' });
+    // autopush child's ignored stdio. `reason` is a short enum, never the
+    // raw error text/repo URL — those can contain usernames/paths.
+    const reason = classifyGitError(err);
+    await appendEvent('sync_failed', { provider: 'git', reason, ms: Date.now() - started });
     if (err.message.includes('invalid characters')) throw err;
-    throw new Error('Failed to push to git repository. Ensure your credentials are configured and the repository exists.');
+    throw new Error(`Failed to push to git repository (${reason}). Ensure your credentials are configured and the repository exists.`);
   } finally {
-    await fs.remove(gitDir);
+    await fs.remove(gitDir).catch(() => {});
   }
 }

package/src/providers/restore.js

@@ -3,6 +3,7 @@ import fs from 'fs-extra';
 import path from 'path';
 import os from 'os';
 import { execFileSync } from 'child_process';
+import { cloneForSync, checkoutFromRemote } from './index.js';
 import { restoreMemories } from '../adapters/restore.js';
 
 export async function fetchFromLocal(config, stagingDir, spinner, onlyFilter = null, autoYes = false) {
@@ -18,6 +19,7 @@ export async function fetchFromLocal(config, stagingDir, spinner, onlyFilter = n
   spinner.text = `Fetching data from local directory: ${chalk.cyan(resolvedSource)}`;
   await fs.copy(resolvedSource, stagingDir);
 
+  if (await fs.pathExists(path.join(stagingDir, 'manifest.enc'))) return false;
   return await restoreMemories(stagingDir, spinner, onlyFilter, autoYes);
 }
 
@@ -28,10 +30,12 @@ export async function fetchFromGit(config, stagingDir, spinner, onlyFilter = nul
   spinner.text = `Cloning memory from Git remote: ${chalk.cyan(repoUrl)}`;
 
   try {
-    execFileSync('git', ['clone', '--depth', '1', repoUrl, '.'], { cwd: stagingDir, stdio: 'ignore' });
+    cloneForSync(repoUrl, stagingDir);
+    if (!checkoutFromRemote(stagingDir, '.')) throw new Error('Could not read backup main branch');
   } catch (err) {
     throw new Error('Failed to pull from git repository. Ensure your SSH keys are configured and the repository is accessible.');
   }
 
+  if (await fs.pathExists(path.join(stagingDir, 'manifest.enc'))) return false;
   return await restoreMemories(stagingDir, spinner, onlyFilter, autoYes);
 }

package/src/security/encryption.js

@@ -2,6 +2,7 @@ import crypto from 'crypto';
 import { promisify } from 'util';
 import fs from 'fs-extra';
 import path from 'path';
+import { listSafeFiles, readSafeFile, restoreFileSet, relativeFile, MAX_SNAPSHOT_FILES, MAX_SNAPSHOT_BYTES, MAX_FILE_BYTES } from './files.js';
 
 const scryptAsync = promisify(crypto.scrypt);
 
@@ -50,6 +51,7 @@ export async function encryptBuffer(plaintext, passphrase) {
  * Decrypt a buffer. Throws on wrong passphrase or tampered data.
  */
 export async function decryptBuffer(data, passphrase) {
+  if (!Buffer.isBuffer(data) || data.length < 68) throw new Error('Truncated encrypted file');
   const magic = data.subarray(0, 8);
   if (!magic.equals(MAGIC)) {
     throw new Error('Not a memoir-encrypted file (bad header)');
@@ -88,23 +90,16 @@ export async function encryptDirectory(srcDir, destDir, passphrase, spinner = nu
   // Phase 2: Index files
   if (spinner) spinner.text = 'Indexing files...';
   const fileList = [];
-  async function index(dir, relBase = '') {
-    const entries = await fs.readdir(dir, { withFileTypes: true });
-    for (const entry of entries) {
-      const fullPath = path.join(dir, entry.name);
-      const relPath = path.join(relBase, entry.name);
-      if (entry.isDirectory()) {
-        await index(fullPath, relPath);
-      } else {
-        const stat = await fs.stat(fullPath);
-        fileList.push({ fullPath, relPath, size: stat.size });
-      }
-    }
+  for (const relPath of await listSafeFiles(srcDir)) {
+    const fullPath = path.join(srcDir, relPath);
+    const stat = await fs.stat(fullPath);
+    if (stat.size > MAX_FILE_BYTES) throw new Error('Snapshot file size limit exceeded');
+    fileList.push({ fullPath, relPath, size: stat.size });
   }
-  await index(srcDir);
 
   const totalFiles = fileList.length;
   const totalBytes = fileList.reduce((sum, f) => sum + f.size, 0);
+  if (totalBytes > MAX_SNAPSHOT_BYTES) throw new Error('Snapshot size limit exceeded');
 
   // Phase 3: Encrypt files
   const manifest = {};
@@ -118,9 +113,10 @@ export async function encryptDirectory(srcDir, destDir, passphrase, spinner = nu
       .digest('hex')
       .slice(0, 24);
 
-    const plaintext = await fs.readFile(fullPath);
+    const plaintext = await readSafeFile(srcDir, relPath);
     const iv = crypto.randomBytes(IV_LENGTH);
     const cipher = crypto.createCipheriv(ALGORITHM, key, iv, { authTagLength: TAG_LENGTH });
+    cipher.setAAD(Buffer.from(relPath, 'utf8'));
     const encrypted = Buffer.concat([cipher.update(plaintext), cipher.final()]);
     const tag = cipher.getAuthTag();
 
@@ -143,7 +139,7 @@ export async function encryptDirectory(srcDir, destDir, passphrase, spinner = nu
 
   // Phase 4: Encrypt manifest
   if (spinner) spinner.text = 'Encrypting file manifest...';
-  const manifestJson = Buffer.from(JSON.stringify(manifest));
+  const manifestJson = Buffer.from(JSON.stringify({ version: 2, files: manifest }));
   const manifestEncrypted = await encryptBuffer(manifestJson, passphrase);
   await fs.writeFile(path.join(destDir, 'manifest.enc'), manifestEncrypted);
 
@@ -166,53 +162,31 @@ function formatBytes(bytes) {
  * Decrypt an encrypted directory back to plaintext.
  */
 export async function decryptDirectory(encDir, destDir, passphrase, spinner = null) {
-  const startTime = Date.now();
-
-  // Phase 1: Decrypt manifest
-  if (spinner) spinner.text = 'Decrypting file manifest...';
-  const manifestData = await fs.readFile(path.join(encDir, 'manifest.enc'));
-  const manifestJson = await decryptBuffer(manifestData, passphrase);
-  const manifest = JSON.parse(manifestJson.toString('utf8'));
-
-  // Phase 2: Derive key
-  if (spinner) spinner.text = 'Deriving decryption key (scrypt)...';
-  const salt = await fs.readFile(path.join(encDir, 'salt'));
+  const manifestData = await readSafeFile(encDir, 'manifest.enc');
+  const decoded = JSON.parse((await decryptBuffer(manifestData, passphrase)).toString('utf8'));
+  const version = decoded.version === 2 ? 2 : 1;
+  const manifest = version === 2 ? decoded.files : decoded;
+  if (!manifest || Array.isArray(manifest) || typeof manifest !== 'object' || Object.keys(manifest).length > MAX_SNAPSHOT_FILES) throw new Error('Invalid encrypted manifest');
+  const salt = await readSafeFile(encDir, 'salt');
+  if (salt.length !== SALT_LENGTH) throw new Error('Invalid encryption salt');
   const { key } = await deriveKey(passphrase, salt);
-
-  const dataDir = path.join(encDir, 'data');
-  const totalFiles = Object.keys(manifest).length;
-  let count = 0;
-  let bytesProcessed = 0;
-
-  // Phase 3: Decrypt files
-  for (const [hashedName, relPath] of Object.entries(manifest)) {
-    const encFilePath = path.join(dataDir, `${hashedName}.enc`);
-    if (!(await fs.pathExists(encFilePath))) continue;
-
-    const data = await fs.readFile(encFilePath);
-    const iv = data.subarray(0, IV_LENGTH);
-    const tag = data.subarray(IV_LENGTH, IV_LENGTH + TAG_LENGTH);
-    const ciphertext = data.subarray(IV_LENGTH + TAG_LENGTH);
-
-    const decipher = crypto.createDecipheriv(ALGORITHM, key, iv, { authTagLength: TAG_LENGTH });
-    decipher.setAuthTag(tag);
-    const decrypted = Buffer.concat([decipher.update(ciphertext), decipher.final()]);
-
-    const outPath = path.join(destDir, relPath);
-    await fs.ensureDir(path.dirname(outPath));
-    await fs.writeFile(outPath, decrypted);
-    count++;
-    bytesProcessed += decrypted.length;
-
-    if (spinner) {
-      const pct = Math.round((count / totalFiles) * 100);
-      spinner.text = `Decrypting ${count}/${totalFiles} files — ${formatBytes(bytesProcessed)} [${pct}%]`;
-    }
+  const files = [];
+  let bytes = 0;
+  for (const [hashedName, storedPath] of Object.entries(manifest)) {
+    if (!/^[a-f0-9]{24}$/.test(hashedName)) throw new Error('Invalid encrypted blob name');
+    const relPath = relativeFile(storedPath);
+    const data = await readSafeFile(encDir, 'data/' + hashedName + '.enc', { maxBytes: MAX_FILE_BYTES + IV_LENGTH + TAG_LENGTH });
+    if (data.length < IV_LENGTH + TAG_LENGTH) throw new Error('Truncated encrypted blob');
+    const decipher = crypto.createDecipheriv(ALGORITHM, key, data.subarray(0, IV_LENGTH), { authTagLength: TAG_LENGTH });
+    if (version === 2) decipher.setAAD(Buffer.from(storedPath, 'utf8'));
+    decipher.setAuthTag(data.subarray(IV_LENGTH, IV_LENGTH + TAG_LENGTH));
+    const content = Buffer.concat([decipher.update(data.subarray(IV_LENGTH + TAG_LENGTH)), decipher.final()]);
+    bytes += content.length;
+    if (bytes > MAX_SNAPSHOT_BYTES) throw new Error('Snapshot size limit exceeded');
+    files.push({ path: relPath, content });
   }
-
-  const elapsed = ((Date.now() - startTime) / 1000).toFixed(1);
-  if (spinner) spinner.text = `Decrypted ${count} files (${formatBytes(bytesProcessed)}) in ${elapsed}s`;
-
+  const count = await restoreFileSet(destDir, files);
+  if (spinner) spinner.text = 'Decrypted and verified ' + count + ' files (' + formatBytes(bytes) + ')';
   return count;
 }