memoir-cli 3.12.0 → 3.15.0

package/src/workspace/tracker.js

@@ -1,350 +1,102 @@
 import fs from 'fs-extra';
 import path from 'path';
 import os from 'os';
+import crypto from 'crypto';
 import { execFileSync } from 'child_process';
+import { relativeFile, readSafeFile, writeSafeFile, restoreFileSet } from '../security/files.js';
+import { scanForSecrets } from '../security/scanner.js';
+import { projectIdentity } from '../memory/scope.js';
+import { repositoryState } from '../memory/repository.js';
 
-const home = os.homedir();
+const skippedDirectories = new Set(['.git', 'node_modules', '.next', '.cache', '.venv', 'venv', 'dist', 'build', '__pycache__']);
+const sensitive = /(^|\/)(\.env(?:\..*)?|\.npmrc|\.pypirc|credentials[^/]*|secrets?[^/]*|id_rsa|id_ed25519)$|\.(pem|key|p12|pfx)$/i;
 
-/**
- * Scan home directory for git projects and build a workspace manifest.
- * Tracks: git remote URL, branch, last commit, uncommitted changes (as patch).
- * For non-git projects with AI configs, bundles them as tar.gz.
- */
+// Explicit opt-in captures only the active project. No home-wide discovery,
+// external archive extraction, or remote repository instructions are executed.
 export async function scanWorkspace(stagingDir, spinner, opts = {}) {
-  const maxDepth = opts.maxDepth || 3;
-  const maxBundleSize = opts.maxBundleSize || 50 * 1024 * 1024; // 50MB
-
-  const skipDirs = new Set([
-    'node_modules', '.git', '.next', '.vercel', 'dist', 'build',
-    '__pycache__', '.venv', 'venv', '.cache', '.npm', '.bun',
-    'Library', '.Trash', 'Applications', 'Pictures', 'Music',
-    'Movies', 'Public', 'Downloads', '.local', '.cargo', '.rustup',
-    '.docker', '.ssh', '.config', '.claude', '.gemini'
-  ]);
-
-  // Project markers — files that indicate "this is a project"
-  const projectMarkers = [
-    'package.json', 'Cargo.toml', 'go.mod', 'pyproject.toml',
-    'requirements.txt', 'Gemfile', 'pom.xml', 'build.gradle',
-    'Makefile', 'CMakeLists.txt', '.project', 'CLAUDE.md',
-    'GEMINI.md', 'AGENTS.md', 'README.md',
-    // Also detect dirs with .git or multiple content files
-    '.gitignore', 'index.html', 'main.py', 'app.py', 'index.js',
-  ];
-
-  // Also detect dirs with multiple markdown/code files as potential projects
-  const isContentProject = (entries) => {
-    const mdFiles = entries.filter(e => !e.isDirectory() && e.name.endsWith('.md'));
-    return mdFiles.length >= 2; // 2+ markdown files = likely a writing project
-  };
-
-  const projects = [];
-
-  const scanDir = async (dir, depth = 0) => {
-    if (depth > maxDepth) return;
-    let entries;
-    try {
-      entries = await fs.readdir(dir, { withFileTypes: true });
-    } catch { return; }
-
-    // Check if this dir is a project
-    const hasMarker = entries.some(e => !e.isDirectory() && projectMarkers.includes(e.name));
-    const hasContent = isContentProject(entries);
-
-    if ((hasMarker || hasContent) && dir !== home) {
-      const info = await getProjectInfo(dir);
-      if (info) projects.push(info);
-      // Don't recurse into sub-projects deeper than this
-      return;
-    }
-
-    // Recurse into subdirectories
-    for (const entry of entries) {
-      if (!entry.isDirectory()) continue;
-      if (entry.name.startsWith('.') && entry.name !== '.github') continue;
-      if (skipDirs.has(entry.name)) continue;
-      await scanDir(path.join(dir, entry.name), depth + 1);
-    }
-  };
-
-  if (spinner) spinner.text = 'Scanning workspace for projects...';
-  await scanDir(home);
-
-  // Build manifest
-  const manifest = {
-    version: 1,
-    machine: os.hostname(),
-    platform: process.platform,
-    home: home,
-    scannedAt: new Date().toISOString(),
-    projects: []
-  };
-
-  const bundleDir = path.join(stagingDir, 'workspace-bundles');
-
-  for (const proj of projects) {
-    const entry = {
-      name: proj.name,
-      relativePath: proj.relativePath,
-      originalPath: proj.path,
-      type: proj.hasGit ? 'git' : 'bundle',
-    };
-
-    if (proj.hasGit) {
-      entry.gitRemote = proj.gitRemote;
-      entry.branch = proj.branch;
-      entry.lastCommit = proj.lastCommit;
-      entry.lastCommitMessage = proj.lastCommitMessage;
-
-      // Save uncommitted changes as patch
-      if (proj.hasDirtyWork && proj.lastCommit) {
-        try {
-          const patchDir = path.join(stagingDir, 'workspace-patches');
-          await fs.ensureDir(patchDir);
-          const diff = execFileSync('git', ['diff', 'HEAD'], {
-            cwd: proj.path,
-            maxBuffer: 10 * 1024 * 1024,
-            timeout: 10000
-          }).toString();
-          if (diff.trim()) {
-            const patchFile = `${proj.name}.patch`;
-            await fs.writeFile(path.join(patchDir, patchFile), diff);
-            entry.patchFile = patchFile;
-          }
-        } catch {
-          // Patch capture is best-effort
-        }
-      }
-    } else {
-      // Bundle non-git project (if small enough)
-      const size = await getDirSize(proj.path);
-      if (size <= maxBundleSize) {
-        try {
-          await fs.ensureDir(bundleDir);
-          const bundleName = `${proj.name}.tar.gz`;
-          execFileSync('tar', [
-            'czf', path.join(bundleDir, bundleName),
-            '-C', path.dirname(proj.path),
-            '--exclude', 'node_modules',
-            '--exclude', '.git',
-            '--exclude', '__pycache__',
-            '--exclude', '.venv',
-            '--exclude', 'dist',
-            '--exclude', 'build',
-            proj.name
-          ], { stdio: 'ignore', timeout: 30000 });
-          entry.bundleFile = bundleName;
-          entry.bundleSize = (await fs.stat(path.join(bundleDir, bundleName))).size;
-        } catch {
-          // Bundle is best-effort
-          entry.bundleFailed = true;
-        }
-      } else {
-        entry.tooLarge = true;
-        entry.size = size;
+  const project = await fs.realpath(path.resolve(opts.project || process.env.MEMOIR_PROJECT_ROOT || process.cwd()));
+  const repository = repositoryState(project);
+  let candidates = [];
+  if (repository.head) {
+    candidates = execFileSync('git', ['ls-files', '-z', '--cached', '--others', '--exclude-standard'], {
+      cwd: project, encoding: 'utf8', timeout: 10000, maxBuffer: 16 * 1024 * 1024,
+    }).split('\0').filter(Boolean);
+  } else {
+    const walk = async (dir, prefix = '') => {
+      for (const entry of await fs.readdir(dir, { withFileTypes: true })) {
+        if (skippedDirectories.has(entry.name)) continue;
+        const rel = prefix + entry.name;
+        if (entry.isDirectory()) await walk(path.join(dir, entry.name), rel + '/');
+        else candidates.push(rel);
+        if (candidates.length > 50000) throw new Error('Workspace file count limit exceeded');
       }
+    };
+    await walk(project);
+  }
+  if (candidates.length > 50000) throw new Error('Workspace file count limit exceeded');
+  const projectId = projectIdentity(project);
+  const key = crypto.createHash('sha256').update(projectId).digest('hex').slice(0, 16);
+  const files = [], omitted = [];
+  let bytes = 0;
+  for (const candidate of [...new Set(candidates)].sort()) {
+    const rel = relativeFile(candidate);
+    if (rel.split('/').some(p => skippedDirectories.has(p)) || sensitive.test(rel)) { omitted.push({ path: rel, reason: 'excluded' }); continue; }
+    let content;
+    try { content = await readSafeFile(project, rel); }
+    catch (err) {
+      if (err.code === 'ENOENT') { omitted.push({ path: rel, reason: 'deleted' }); continue; }
+      throw err;
     }
-
-    manifest.projects.push(entry);
+    // A heuristic, not a guarantee: omitted matches remain visible in the manifest.
+    if (scanForSecrets(content.toString('utf8')).found.length) { omitted.push({ path: rel, reason: 'potential-secret' }); continue; }
+    bytes += content.length;
+    if (bytes > (opts.maxBundleSize || 50 * 1024 * 1024)) throw new Error('Workspace size limit exceeded');
+    const stored = 'workspace-files/' + key + '/' + rel;
+    await writeSafeFile(stagingDir, stored, content);
+    files.push({ path: rel, stored, sha256: crypto.createHash('sha256').update(content).digest('hex') });
   }
-
-  // Save manifest
-  await fs.writeFile(
-    path.join(stagingDir, 'workspace.json'),
-    JSON.stringify(manifest, null, 2)
-  );
-
+  const manifest = { version: 2, scannedAt: new Date().toISOString(), projects: [{
+    name: path.basename(project), identity: projectId, key, type: 'files',
+    repository, files, omitted, size: bytes,
+  }] };
+  await writeSafeFile(stagingDir, 'workspace.json', JSON.stringify(manifest, null, 2));
   return manifest;
 }
 
-/**
- * Restore workspace on this machine from a manifest.
- * Clones git projects, unpacks bundles, applies patches.
- */
+// Recovery produces a separate directory for inspection, never overlays an
+// existing checkout. The recorded commit is evidence, not a claim of Git history.
 export async function restoreWorkspace(sourceDir, spinner, autoYes = false) {
-  const manifestPath = path.join(sourceDir, 'workspace.json');
-  if (!await fs.pathExists(manifestPath)) return null;
-
-  const manifest = await fs.readJson(manifestPath);
-  if (!manifest.projects || manifest.projects.length === 0) return null;
-
-  const results = { cloned: [], unpacked: [], patched: [], skipped: [] };
-
-  for (const proj of manifest.projects) {
-    // Determine where to put this project on the local machine
-    const localPath = resolveLocalPath(proj);
-
-    // Skip if project already exists locally
-    if (await fs.pathExists(localPath)) {
-      // But check if we should apply a patch
-      if (proj.patchFile) {
-        const patchPath = path.join(sourceDir, 'workspace-patches', proj.patchFile);
-        if (await fs.pathExists(patchPath)) {
-          try {
-            execFileSync('git', ['apply', '--check', patchPath], {
-              cwd: localPath, stdio: 'ignore'
-            });
-            execFileSync('git', ['apply', patchPath], {
-              cwd: localPath, stdio: 'ignore'
-            });
-            results.patched.push({ name: proj.name, path: localPath });
-          } catch {
-            // Patch didn't apply cleanly — skip
-          }
-        }
-      }
-      results.skipped.push({ name: proj.name, path: localPath, reason: 'exists' });
-      continue;
-    }
-
-    if (proj.type === 'git' && proj.gitRemote) {
-      // Clone the repo
-      if (spinner) spinner.text = `Cloning ${proj.name}...`;
-      try {
-        await fs.ensureDir(path.dirname(localPath));
-        execFileSync('git', ['clone', proj.gitRemote, localPath], {
-          stdio: 'ignore',
-          timeout: 120000
-        });
-
-        // Checkout the right branch
-        if (proj.branch && proj.branch !== 'main' && proj.branch !== 'master') {
-          try {
-            execFileSync('git', ['checkout', proj.branch], {
-              cwd: localPath, stdio: 'ignore'
-            });
-          } catch {}
-        }
-
-        // Apply patch if available
-        if (proj.patchFile) {
-          const patchPath = path.join(sourceDir, 'workspace-patches', proj.patchFile);
-          if (await fs.pathExists(patchPath)) {
-            try {
-              execFileSync('git', ['apply', patchPath], {
-                cwd: localPath, stdio: 'ignore'
-              });
-              results.patched.push({ name: proj.name, path: localPath });
-            } catch {}
-          }
-        }
-
-        results.cloned.push({ name: proj.name, path: localPath, remote: proj.gitRemote });
-      } catch (err) {
-        results.skipped.push({ name: proj.name, reason: `clone failed: ${err.message}` });
-      }
-    } else if (proj.bundleFile) {
-      // Unpack bundle
-      const bundlePath = path.join(sourceDir, 'workspace-bundles', proj.bundleFile);
-      if (await fs.pathExists(bundlePath)) {
-        if (spinner) spinner.text = `Unpacking ${proj.name}...`;
-        try {
-          await fs.ensureDir(path.dirname(localPath));
-          execFileSync('tar', ['xzf', bundlePath, '-C', path.dirname(localPath)], {
-            stdio: 'ignore',
-            timeout: 60000
-          });
-          results.unpacked.push({ name: proj.name, path: localPath });
-        } catch (err) {
-          results.skipped.push({ name: proj.name, reason: `unpack failed: ${err.message}` });
-        }
-      }
-    } else {
-      results.skipped.push({ name: proj.name, reason: proj.tooLarge ? 'too large' : 'no source' });
+  let raw;
+  try { raw = await readSafeFile(sourceDir, 'workspace.json'); }
+  catch (err) { if (err.code === 'ENOENT') return null; throw err; }
+  const manifest = JSON.parse(raw.toString());
+  if (manifest.version !== 2) throw new Error('Legacy workspace archives cannot be safely auto-restored. Keep the backup and inspect its archive separately, or create a new workspace snapshot.');
+  if (!Array.isArray(manifest.projects) || manifest.projects.length > 100) throw new Error('Invalid workspace manifest');
+  const plans = [];
+  let totalBytes = 0, totalFiles = 0;
+  for (const project of manifest.projects) {
+    if (!/^[a-f0-9]{16}$/.test(project.key) || !Array.isArray(project.files)) throw new Error('Invalid workspace project');
+    const entries = [];
+    for (const file of project.files) {
+      if (++totalFiles > 50000) throw new Error('Workspace file count limit exceeded');
+      const rel = relativeFile(file.path);
+      if (file.stored !== 'workspace-files/' + project.key + '/' + rel) throw new Error('Invalid workspace source path');
+      const content = await readSafeFile(sourceDir, file.stored);
+      totalBytes += content.length;
+      if (totalBytes > 256 * 1024 * 1024) throw new Error('Workspace size limit exceeded');
+      if (crypto.createHash('sha256').update(content).digest('hex') !== file.sha256) throw new Error('Workspace content failed verification');
+      entries.push({ path: rel, content });
     }
+    plans.push({ project, entries });
   }
-
-  return results;
-}
-
-/**
- * Figure out where a project should live on this machine.
- */
-function resolveLocalPath(proj) {
-  // If the project had a relative path from home, use that
-  if (proj.relativePath) {
-    return path.join(home, proj.relativePath);
-  }
-  // Default: put it in home directory
-  return path.join(home, proj.name);
-}
-
-/**
- * Get info about a project directory.
- */
-async function getProjectInfo(dir) {
-  const name = path.basename(dir);
-  const relativePath = path.relative(home, dir);
-  const info = {
-    name,
-    path: dir,
-    relativePath,
-    hasGit: false,
-    gitRemote: null,
-    branch: null,
-    lastCommit: null,
-    lastCommitMessage: null,
-    hasDirtyWork: false,
-  };
-
-  // Check for git
-  const gitDir = path.join(dir, '.git');
-  if (await fs.pathExists(gitDir)) {
-    info.hasGit = true;
-    try {
-      const remote = execFileSync('git', ['remote', 'get-url', 'origin'], {
-        cwd: dir, stdio: ['pipe', 'pipe', 'ignore'], timeout: 5000
-      }).toString().trim();
-      info.gitRemote = remote;
-    } catch {}
-
-    try {
-      info.branch = execFileSync('git', ['branch', '--show-current'], {
-        cwd: dir, stdio: ['pipe', 'pipe', 'ignore'], timeout: 5000
-      }).toString().trim();
-    } catch {}
-
-    try {
-      info.lastCommit = execFileSync('git', ['log', '-1', '--format=%H'], {
-        cwd: dir, stdio: ['pipe', 'pipe', 'ignore'], timeout: 5000
-      }).toString().trim();
-      info.lastCommitMessage = execFileSync('git', ['log', '-1', '--format=%s'], {
-        cwd: dir, stdio: ['pipe', 'pipe', 'ignore'], timeout: 5000
-      }).toString().trim();
-    } catch {}
-
-    try {
-      const status = execFileSync('git', ['status', '--porcelain'], {
-        cwd: dir, stdio: ['pipe', 'pipe', 'ignore'], timeout: 5000
-      }).toString().trim();
-      info.hasDirtyWork = status.length > 0;
-    } catch {}
+  const results = { cloned: [], unpacked: [], patched: [], skipped: [] };
+  for (const { project, entries } of plans) {
+    const parent = path.join(os.homedir(), 'memoir-restored');
+    await fs.ensureDir(parent);
+    const destination = await fs.mkdtemp(path.join(parent, project.key + '-'));
+    try { await restoreFileSet(destination, entries); }
+    catch (err) { await fs.remove(destination); throw err; }
+    results.unpacked.push({ name: project.name, path: destination, omitted: project.omitted?.length || 0 });
   }
-
-  return info;
-}
-
-/**
- * Get total size of a directory (excluding common heavy dirs).
- */
-async function getDirSize(dir) {
-  let size = 0;
-  const skip = new Set(['node_modules', '.git', '__pycache__', '.venv', 'dist', 'build']);
-
-  const walk = async (d) => {
-    let entries;
-    try { entries = await fs.readdir(d, { withFileTypes: true }); } catch { return; }
-    for (const e of entries) {
-      if (e.isDirectory()) {
-        if (!skip.has(e.name)) await walk(path.join(d, e.name));
-      } else {
-        try {
-          const stat = await fs.stat(path.join(d, e.name));
-          size += stat.size;
-        } catch {}
-      }
-    }
-  };
-  await walk(dir);
-  return size;
+  return results;
 }

package/supabase/migrations/202609050001_backup_versions.sql

@@ -0,0 +1,50 @@
+-- Review and apply to the existing Memoir Supabase project before deploying
+-- the new cloud writer. No old backups are deleted or renumbered.
+-- If duplicate (user_id, version) rows exist, the unique index fails; inspect
+-- and reconcile those records before retrying this transactional migration.
+begin;
+
+alter table public.backups add column if not exists encryption_format text not null default 'legacy';
+alter table public.backups add column if not exists source_backup_id uuid;
+
+create unique index if not exists memoir_backups_user_version
+  on public.backups (user_id, version);
+
+create table if not exists public.memoir_backup_counters (
+  user_id uuid primary key references auth.users(id) on delete cascade,
+  next_version bigint not null check (next_version > 0)
+);
+alter table public.memoir_backup_counters enable row level security;
+revoke all on public.memoir_backup_counters from public, anon, authenticated;
+
+create or replace function public.memoir_next_backup_version()
+returns bigint
+language plpgsql
+security definer
+set search_path = ''
+as $$
+declare
+  actor uuid := auth.uid();
+  allocated bigint;
+begin
+  if actor is null then
+    raise exception 'Authentication required';
+  end if;
+
+  insert into public.memoir_backup_counters as counter (user_id, next_version)
+  select actor, coalesce(max(b.version), 0) + 1
+    from public.backups b where b.user_id = actor
+  on conflict (user_id) do update
+    set next_version = greatest(
+      counter.next_version + 1,
+      (select coalesce(max(b.version), 0) + 1 from public.backups b where b.user_id = actor)
+    )
+  returning next_version into allocated;
+
+  return allocated;
+end;
+$$;
+
+revoke all on function public.memoir_next_backup_version() from public, anon;
+grant execute on function public.memoir_next_backup_version() to authenticated;
+commit;