mrmd-sync 0.3.0 → 0.3.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "mrmd-sync",
-  "version": "0.3.0",
+  "version": "0.3.2",
   "description": "Production-ready sync server for mrmd - real-time collaboration with file persistence",
   "type": "module",
   "main": "src/index.js",

package/src/index.js

@@ -24,6 +24,7 @@ import {
   mkdirSync,
   statSync,
   rmSync,
+  readdirSync,
 } from 'fs';
 import { join, dirname, relative, resolve } from 'path';
 import { createHash } from 'crypto';
@@ -279,6 +280,89 @@ const DEFAULT_CONFIG = {
 // Paths that require dangerouslyAllowSystemPaths: true
 const DANGEROUS_PATHS = ['/', '/etc', '/usr', '/var', '/bin', '/sbin', '/root', '/home'];
 
+// =============================================================================
+// DATA LOSS PREVENTION - Memory Monitoring
+// =============================================================================
+// Added after investigating unexplained data loss on 2026-01-16.
+// The sync server crashed with OOM after ~9 minutes, consuming 4GB for a 2.5KB
+// document. User lost ~2.5 hours of work because the editor gave no warning.
+//
+// These safeguards ensure:
+// 1. Memory usage is monitored and warnings are logged
+// 2. Y.Doc compaction runs periodically to bound memory growth
+// 3. Server fails fast (512MB limit in electron main.js) rather than OOM later
+// =============================================================================
+
+const MEMORY_WARNING_MB = 200;   // Warn at 200MB
+// DISABLED: Compaction causes document duplication!
+// When clients reconnect after compaction, Yjs merges their state with the
+// server's fresh state, causing content to double. Need a different approach.
+// Keeping memory monitoring for warnings only.
+const MEMORY_COMPACT_MB = Infinity;   // Disabled
+const MEMORY_CHECK_INTERVAL_MS = 30000; // Check every 30 seconds
+const COMPACTION_INTERVAL_MS = Infinity; // Disabled
+
+/**
+ * Get current memory usage in MB
+ */
+function getMemoryUsageMB() {
+  const usage = process.memoryUsage();
+  return {
+    heapUsed: Math.round(usage.heapUsed / 1024 / 1024),
+    heapTotal: Math.round(usage.heapTotal / 1024 / 1024),
+    rss: Math.round(usage.rss / 1024 / 1024),
+    external: Math.round(usage.external / 1024 / 1024),
+  };
+}
+
+/**
+ * Compact a Y.Doc by creating a fresh doc with only current content.
+ * This discards all operation history and tombstones, dramatically reducing memory.
+ *
+ * NOTE: This will disconnect all clients, who will need to reconnect.
+ * The content itself is preserved via the file and snapshot.
+ *
+ * @param {Object} docData - The document data object from getDoc()
+ * @param {Object} log - Logger instance
+ * @returns {Object} - New Y.Doc and Y.Text
+ */
+function compactYDoc(docData, log) {
+  const { ydoc, ytext, docName } = docData;
+
+  // Get current content before compaction
+  const currentContent = ytext.toString();
+  const oldStateSize = Y.encodeStateAsUpdate(ydoc).length;
+
+  log.info('Compacting Y.Doc', {
+    doc: docName,
+    contentLength: currentContent.length,
+    oldStateSize,
+  });
+
+  // Create fresh Y.Doc
+  const newYdoc = new Y.Doc();
+  newYdoc.name = docName;
+  const newYtext = newYdoc.getText('content');
+
+  // Insert current content into fresh doc
+  if (currentContent.length > 0) {
+    newYdoc.transact(() => {
+      newYtext.insert(0, currentContent);
+    });
+  }
+
+  const newStateSize = Y.encodeStateAsUpdate(newYdoc).length;
+
+  log.info('Y.Doc compacted', {
+    doc: docName,
+    oldStateSize,
+    newStateSize,
+    reduction: `${Math.round((1 - newStateSize / oldStateSize) * 100)}%`,
+  });
+
+  return { newYdoc, newYtext };
+}
+
 // =============================================================================
 // UTILITY FUNCTIONS
 // =============================================================================
@@ -352,6 +436,87 @@ function atomicWriteFile(filePath, content) {
   }
 }
 
+/**
+ * Clean up stale temp files left behind by crashed processes.
+ * Temp files have pattern: {filename}.tmp.{pid}.{timestamp}
+ * A file is stale if:
+ *   - The PID no longer exists (process died)
+ *   - OR the timestamp is older than maxAgeMs (fallback safety)
+ */
+function cleanupStaleTempFiles(dir, log, maxAgeMs = 3600000) {
+  const tmpPattern = /\.tmp\.(\d+)\.(\d+)$/;
+  let cleaned = 0;
+  let errors = 0;
+
+  function walkDir(currentDir) {
+    let entries;
+    try {
+      entries = readdirSync(currentDir, { withFileTypes: true });
+    } catch {
+      return; // Skip directories we can't read
+    }
+
+    for (const entry of entries) {
+      const fullPath = join(currentDir, entry.name);
+
+      if (entry.isDirectory()) {
+        // Skip hidden directories and node_modules
+        if (!entry.name.startsWith('.') && entry.name !== 'node_modules') {
+          walkDir(fullPath);
+        }
+        continue;
+      }
+
+      // Check if this is a temp file
+      const match = entry.name.match(tmpPattern);
+      if (!match) continue;
+
+      const pid = parseInt(match[1], 10);
+      const timestamp = parseInt(match[2], 10);
+      const age = Date.now() - timestamp;
+
+      // Check if process is dead
+      let processIsDead = false;
+      try {
+        process.kill(pid, 0); // Signal 0 = check if process exists
+      } catch (err) {
+        if (err.code === 'ESRCH') {
+          processIsDead = true; // Process doesn't exist
+        }
+        // EPERM means process exists but we can't signal it
+      }
+
+      // Remove if process is dead OR file is older than maxAgeMs
+      if (processIsDead || age > maxAgeMs) {
+        try {
+          unlinkSync(fullPath);
+          cleaned++;
+          log.info('Removed stale temp file', {
+            path: fullPath,
+            pid,
+            processIsDead,
+            ageMs: age,
+          });
+        } catch (err) {
+          errors++;
+          log.warn('Failed to remove stale temp file', {
+            path: fullPath,
+            error: err.message,
+          });
+        }
+      }
+    }
+  }
+
+  walkDir(dir);
+
+  if (cleaned > 0 || errors > 0) {
+    log.info('Temp file cleanup complete', { cleaned, errors });
+  }
+
+  return { cleaned, errors };
+}
+
 /**
  * Save Yjs document state for crash recovery
  */
@@ -495,6 +660,9 @@ export function createServer(options = {}) {
   // Acquire PID lock to prevent multiple instances on same directory
   const releasePidLock = acquirePidLock(snapshotDir, port, log);
 
+  // Clean up stale temp files from previous crashed processes
+  cleanupStaleTempFiles(resolvedDir, log);
+
   // Document storage
   const docs = new Map();
 
@@ -539,6 +707,21 @@ export function createServer(options = {}) {
       : null;
 
     const ytext = ydoc.getText('content');
+    const docData = {
+      docName,
+      ydoc,
+      ytext,
+      awareness,
+      conns,
+      mutex,
+      filePath,
+      snapshotPath,
+      applyFileChange: null,
+      flushWrite: null,
+      scheduleCleanup: null,
+      cancelCleanup: null,
+      scheduleWrite: null,
+    };
 
     // Track state
     let lastFileHash = null;
@@ -604,7 +787,7 @@ export function createServer(options = {}) {
           if (isShuttingDown) return;
 
           isWritingToFile = true;
-          const content = ytext.toString();
+          const content = docData.ytext.toString();
           const hash = contentHash(content);
 
           // Skip if content unchanged
@@ -614,7 +797,7 @@ export function createServer(options = {}) {
             return;
           }
 
-          const { success, error } = atomicWriteFile(filePath, content);
+          const { success, error } = atomicWriteFile(docData.filePath, content);
           if (error) {
             log.error('Error saving file', { path: filePath, error });
             metrics.errorOccurred();
@@ -630,15 +813,17 @@ export function createServer(options = {}) {
       }, debounceMs);
     };
 
+    docData.scheduleWrite = scheduleWrite;
+
     // Listen for Yjs updates
-    ydoc.on('update', scheduleWrite);
+    docData.ydoc.on('update', scheduleWrite);
 
     // Schedule Yjs snapshot saves
     if (persistYjsState && snapshotPath) {
       const scheduleSnapshot = () => {
         clearTimeout(snapshotTimeout);
         snapshotTimeout = setTimeout(() => {
-          const { success, error } = saveYjsSnapshot(snapshotPath, ydoc);
+          const { success, error } = saveYjsSnapshot(snapshotPath, docData.ydoc);
           if (error) {
             log.warn('Failed to save Yjs snapshot', { doc: docName, error });
           }
@@ -656,7 +841,7 @@ export function createServer(options = {}) {
         const newHash = contentHash(newContent);
         if (newHash === lastFileHash) return;
 
-        const oldContent = ytext.toString();
+        const oldContent = docData.ytext.toString();
         if (oldContent === newContent) {
           lastFileHash = newHash;
           return;
@@ -665,14 +850,14 @@ export function createServer(options = {}) {
         isWritingToYjs = true;
         const changes = diffChars(oldContent, newContent);
 
-        ydoc.transact(() => {
+        docData.ydoc.transact(() => {
           let pos = 0;
           for (const change of changes) {
             if (change.added) {
-              ytext.insert(pos, change.value);
+              docData.ytext.insert(pos, change.value);
               pos += change.value.length;
             } else if (change.removed) {
-              ytext.delete(pos, change.value.length);
+              docData.ytext.delete(pos, change.value.length);
             } else {
               pos += change.value.length;
             }
@@ -689,10 +874,10 @@ export function createServer(options = {}) {
     const flushWrite = async () => {
       clearTimeout(writeTimeout);
       await mutex.withLock(async () => {
-        const content = ytext.toString();
+        const content = docData.ytext.toString();
         const hash = contentHash(content);
         if (hash !== lastFileHash) {
-          const { error } = atomicWriteFile(filePath, content);
+          const { error } = atomicWriteFile(docData.filePath, content);
           if (error) {
             log.error('Error flushing file', { path: filePath, error });
           } else {
@@ -701,7 +886,7 @@ export function createServer(options = {}) {
         }
         // Save final snapshot
         if (snapshotPath) {
-          saveYjsSnapshot(snapshotPath, ydoc);
+          saveYjsSnapshot(snapshotPath, docData.ydoc);
         }
       });
       pendingWrites.delete(docName);
@@ -714,8 +899,8 @@ export function createServer(options = {}) {
         if (conns.size === 0) {
           await flushWrite();
           clearTimeout(snapshotTimeout);
-          awareness.destroy();
-          ydoc.destroy();
+          docData.awareness.destroy();
+          docData.ydoc.destroy();
           docs.delete(docName);
           log.info('Cleaned up document', { doc: docName });
         }
@@ -726,19 +911,10 @@ export function createServer(options = {}) {
       clearTimeout(cleanupTimeout);
     };
 
-    const docData = {
-      ydoc,
-      ytext,
-      awareness,
-      conns,
-      mutex,
-      filePath,
-      snapshotPath,
-      applyFileChange,
-      flushWrite,
-      scheduleCleanup,
-      cancelCleanup,
-    };
+    docData.applyFileChange = applyFileChange;
+    docData.flushWrite = flushWrite;
+    docData.scheduleCleanup = scheduleCleanup;
+    docData.cancelCleanup = cancelCleanup;
 
     docs.set(docName, docData);
     return docData;
@@ -751,6 +927,12 @@ export function createServer(options = {}) {
   const watcher = watch(join(resolvedDir, '**/*.md'), {
     ignoreInitial: true,
     awaitWriteFinish: { stabilityThreshold: 300 },
+    ignored: [
+      '**/node_modules/**',
+      '**/.venv/**',
+      '**/__pycache__/**',
+      '**/.git/**',
+    ],
   });
 
   watcher.on('change', async (filePath) => {
@@ -1045,7 +1227,7 @@ export function createServer(options = {}) {
   // GRACEFUL SHUTDOWN
   // =============================================================================
 
-  const gracefulShutdown = async (signal) => {
+  let gracefulShutdown = async (signal) => {
     if (isShuttingDown) return;
     isShuttingDown = true;
 
@@ -1100,6 +1282,117 @@ export function createServer(options = {}) {
     });
   });
 
+  // =============================================================================
+  // DATA LOSS PREVENTION - Memory Monitoring
+  // =============================================================================
+  // Added after investigating unexplained data loss on 2026-01-16.
+  // Monitors memory usage and triggers compaction if needed.
+  // =============================================================================
+
+  let lastCompactionTime = Date.now();
+  let memoryWarningLogged = false;
+  let compactionInProgress = false;
+
+  const memoryMonitorInterval = setInterval(async () => {
+    if (isShuttingDown) return;
+
+    const mem = getMemoryUsageMB();
+
+    // Log warning if memory is getting high
+    if (mem.heapUsed >= MEMORY_WARNING_MB && !memoryWarningLogged) {
+      log.warn('High memory usage detected', {
+        heapUsed: `${mem.heapUsed}MB`,
+        heapTotal: `${mem.heapTotal}MB`,
+        rss: `${mem.rss}MB`,
+        threshold: `${MEMORY_WARNING_MB}MB`,
+      });
+      memoryWarningLogged = true;
+    } else if (mem.heapUsed < MEMORY_WARNING_MB) {
+      memoryWarningLogged = false;
+    }
+
+    if (compactionInProgress) return;
+
+    // Trigger compaction if memory is critical OR if enough time has passed
+    const timeSinceLastCompaction = Date.now() - lastCompactionTime;
+    const shouldCompact =
+      mem.heapUsed >= MEMORY_COMPACT_MB ||
+      timeSinceLastCompaction >= COMPACTION_INTERVAL_MS;
+
+    if (shouldCompact && docs.size > 0) {
+      compactionInProgress = true;
+      try {
+        log.info('Triggering Y.Doc compaction', {
+          reason: mem.heapUsed >= MEMORY_COMPACT_MB ? 'memory-pressure' : 'periodic',
+          heapUsed: `${mem.heapUsed}MB`,
+          docsCount: docs.size,
+        });
+
+        // Compact all documents
+        for (const [docName, docData] of docs) {
+          try {
+            await docData.flushWrite();
+
+            // Disconnect all clients (they will reconnect and get fresh state)
+            for (const ws of docData.conns) {
+              try {
+                ws.close(4000, 'Document compacted - please reconnect');
+              } catch (e) {
+                // Ignore close errors
+              }
+            }
+            docData.conns.clear();
+
+            // Destroy old doc and create fresh one
+            const oldYdoc = docData.ydoc;
+            const oldAwareness = docData.awareness;
+            const { newYdoc, newYtext } = compactYDoc(docData, log);
+            const newAwareness = new awarenessProtocol.Awareness(newYdoc);
+
+            oldYdoc.off('update', docData.scheduleWrite);
+
+            // Update the document entry
+            docData.ydoc = newYdoc;
+            docData.ytext = newYtext;
+            docData.awareness = newAwareness;
+
+            // Re-register the update listener
+            newYdoc.on('update', docData.scheduleWrite);
+
+            // Clean up old doc
+            oldAwareness.destroy();
+            oldYdoc.destroy();
+
+            log.info('Document compacted successfully', { doc: docName });
+          } catch (e) {
+            log.error('Error compacting document', { doc: docName, error: e.message });
+          }
+        }
+
+        lastCompactionTime = Date.now();
+
+        // Force garbage collection if available (--expose-gc flag)
+        if (global.gc) {
+          global.gc();
+          const afterMem = getMemoryUsageMB();
+          log.info('GC completed', {
+            heapUsed: `${afterMem.heapUsed}MB`,
+            freed: `${mem.heapUsed - afterMem.heapUsed}MB`,
+          });
+        }
+      } finally {
+        compactionInProgress = false;
+      }
+    }
+  }, MEMORY_CHECK_INTERVAL_MS);
+
+  // Clean up interval on shutdown
+  const originalGracefulShutdown = gracefulShutdown;
+  gracefulShutdown = async (signal) => {
+    clearInterval(memoryMonitorInterval);
+    return originalGracefulShutdown(signal);
+  };
+
   // =============================================================================
   // PUBLIC API
   // =============================================================================