@myrialabs/clopen 0.2.11 → 0.2.13

package/backend/snapshot/blob-store.ts

@@ -4,7 +4,6 @@
  *
  * Structure:
  *   ~/.clopen/snapshots/blobs/{hash[0:2]}/{hash}.gz  - compressed file blobs
- *   ~/.clopen/snapshots/trees/{snapshotId}.json       - tree maps (filepath -> hash)
  *
  * Deduplication: Same file content across any snapshot is stored only once.
  * Compression: All blobs are gzip compressed to minimize disk usage.
@@ -18,7 +17,6 @@ import { getClopenDir } from '../utils/index.js';
 
 const SNAPSHOTS_DIR = join(getClopenDir(), 'snapshots');
 const BLOBS_DIR = join(SNAPSHOTS_DIR, 'blobs');
-const TREES_DIR = join(SNAPSHOTS_DIR, 'trees');
 
 export interface TreeMap {
 	[filepath: string]: string; // filepath -> blob hash
@@ -45,7 +43,6 @@ class BlobStore {
 	async init(): Promise<void> {
 		if (this.initialized) return;
 		await fs.mkdir(BLOBS_DIR, { recursive: true });
-		await fs.mkdir(TREES_DIR, { recursive: true });
 		this.initialized = true;
 	}
 
@@ -112,69 +109,6 @@ class BlobStore {
 		return gunzipSync(compressed);
 	}
 
-	/**
-	 * Store a tree (snapshot state) as a JSON file.
-	 * Returns the tree hash for reference.
-	 */
-	async storeTree(snapshotId: string, tree: TreeMap): Promise<string> {
-		await this.init();
-		const treePath = join(TREES_DIR, `${snapshotId}.json`);
-		const content = JSON.stringify(tree);
-		const treeHash = this.hashContent(Buffer.from(content, 'utf-8'));
-		await fs.writeFile(treePath, content, 'utf-8');
-		return treeHash;
-	}
-
-	/**
-	 * Read a tree by snapshot ID
-	 */
-	async readTree(snapshotId: string): Promise<TreeMap> {
-		const treePath = join(TREES_DIR, `${snapshotId}.json`);
-		const content = await fs.readFile(treePath, 'utf-8');
-		return JSON.parse(content) as TreeMap;
-	}
-
-	/**
-	 * Check if a tree exists
-	 */
-	async hasTree(snapshotId: string): Promise<boolean> {
-		try {
-			await fs.access(join(TREES_DIR, `${snapshotId}.json`));
-			return true;
-		} catch {
-			return false;
-		}
-	}
-
-	/**
-	 * Resolve a tree to full file contents (as Buffers).
-	 * Reads all blobs in parallel for performance.
-	 * Returns { filepath: Buffer } map for binary-safe handling.
-	 */
-	async resolveTree(tree: TreeMap): Promise<Record<string, Buffer>> {
-		const result: Record<string, Buffer> = {};
-
-		const entries = Object.entries(tree);
-		const blobPromises = entries.map(async ([filepath, hash]) => {
-			try {
-				const content = await this.readBlob(hash);
-				return { filepath, content };
-			} catch (err) {
-				debug.warn('snapshot', `Could not read blob ${hash} for ${filepath}:`, err);
-				return null;
-			}
-		});
-
-		const results = await Promise.all(blobPromises);
-		for (const r of results) {
-			if (r) {
-				result[r.filepath] = r.content;
-			}
-		}
-
-		return result;
-	}
-
 	/**
 	 * Hash a file using mtime cache. Returns { hash, content? }.
 	 * If the file hasn't changed (same mtime+size), returns cached hash without reading content.
@@ -191,10 +125,15 @@ class BlobStore {
 		// Check mtime cache
 		const cached = this.fileHashCache.get(filepath);
 		if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size) {
-			return { hash: cached.hash, content: null, cached: true };
+			// Verify blob still exists on disk (could have been cleaned up)
+			if (await this.hasBlob(cached.hash)) {
+				return { hash: cached.hash, content: null, cached: true };
+			}
+			// Blob was deleted — invalidate cache, fall through to re-read and re-store
+			this.fileHashCache.delete(filepath);
 		}
 
-		// File changed - read as Buffer (binary-safe, no encoding conversion)
+		// File changed or cache miss - read as Buffer (binary-safe, no encoding conversion)
 		const content = await fs.readFile(fullPath);
 		const hash = this.hashContent(content);
 
@@ -212,14 +151,55 @@ class BlobStore {
 	}
 
 	/**
-	 * Delete a tree file (cleanup)
+	 * Delete multiple blobs by hash.
+	 * Also invalidates fileHashCache entries whose hash matches a deleted blob.
 	 */
-	async deleteTree(snapshotId: string): Promise<void> {
+	async deleteBlobs(hashes: string[]): Promise<number> {
+		const hashSet = new Set(hashes);
+		let deleted = 0;
+		for (const hash of hashes) {
+			try {
+				await fs.unlink(this.getBlobPath(hash));
+				deleted++;
+			} catch {
+				// Ignore - might not exist
+			}
+		}
+
+		// Invalidate fileHashCache entries pointing to deleted blobs
+		for (const [filepath, entry] of this.fileHashCache) {
+			if (hashSet.has(entry.hash)) {
+				this.fileHashCache.delete(filepath);
+			}
+		}
+
+		return deleted;
+	}
+
+	/**
+	 * Scan all blob files on disk and return their hashes.
+	 * Used for full garbage collection — compare with DB references to find orphans.
+	 */
+	async scanAllBlobHashes(): Promise<Set<string>> {
+		const hashes = new Set<string>();
 		try {
-			await fs.unlink(join(TREES_DIR, `${snapshotId}.json`));
+			const prefixDirs = await fs.readdir(BLOBS_DIR);
+			for (const prefix of prefixDirs) {
+				const prefixPath = join(BLOBS_DIR, prefix);
+				const stat = await fs.stat(prefixPath);
+				if (!stat.isDirectory()) continue;
+
+				const files = await fs.readdir(prefixPath);
+				for (const file of files) {
+					if (file.endsWith('.gz')) {
+						hashes.add(file.slice(0, -3)); // Remove .gz suffix
+					}
+				}
+			}
 		} catch {
-			// Ignore - might not exist
+			// Directory might not exist yet
 		}
+		return hashes;
 	}
 }
 

package/backend/snapshot/snapshot-service.ts

@@ -691,6 +691,30 @@ export class SnapshotService {
 		return calculateFileChangeStats(previousSnapshot, currentSnapshot);
 	}
 
+	/**
+	 * Get all blob hashes from the in-memory baseline for a session.
+	 * Must be called BEFORE clearSessionBaseline.
+	 */
+	getSessionBaselineHashes(sessionId: string): Set<string> {
+		const baseline = this.sessionBaselines.get(sessionId);
+		if (!baseline) return new Set();
+		return new Set(Object.values(baseline));
+	}
+
+	/**
+	 * Get all blob hashes from ALL in-memory baselines (all active sessions).
+	 * Used to protect blobs still needed by other sessions during cleanup.
+	 */
+	getAllBaselineHashes(): Set<string> {
+		const hashes = new Set<string>();
+		for (const baseline of this.sessionBaselines.values()) {
+			for (const hash of Object.values(baseline)) {
+				hashes.add(hash);
+			}
+		}
+		return hashes;
+	}
+
 	/**
 	 * Clean up session baseline cache when session is no longer active.
 	 */

package/backend/terminal/stream-manager.ts

@@ -1,11 +1,12 @@
 /**
  * Terminal Stream Manager
  * Manages background terminal streams and their state
+ * Uses @xterm/headless to maintain accurate terminal state in-memory
  */
 
 import type { IPty } from 'bun-pty';
-import { existsSync, mkdirSync, readFileSync, unlinkSync } from 'fs';
-import { join } from 'path';
+import { Terminal } from '@xterm/headless';
+import { SerializeAddon } from '@xterm/addon-serialize';
 
 interface TerminalStream {
   streamId: string;
@@ -17,23 +18,30 @@ interface TerminalStream {
   workingDirectory?: string;
   projectPath?: string;
   projectId?: string;
-  output: string[];
   processId?: number;
-  outputStartIndex?: number; // Track where new output starts (for background output)
+  headlessTerminal: Terminal;
+  serializeAddon: SerializeAddon;
 }
 
 class TerminalStreamManager {
   private streams: Map<string, TerminalStream> = new Map();
   private sessionToStream: Map<string, string> = new Map();
-  private tempDir: string = '.terminal-output-cache';
 
-  constructor() {
-    // Create temp directory for output caching
-    if (!existsSync(this.tempDir)) {
-      mkdirSync(this.tempDir, { recursive: true });
-    }
+  /**
+   * Create a headless terminal instance with serialize addon
+   */
+  private createHeadlessTerminal(cols: number, rows: number): { terminal: Terminal; serializeAddon: SerializeAddon } {
+    const terminal = new Terminal({
+      scrollback: 1000,
+      cols,
+      rows,
+      allowProposedApi: true
+    });
+    const serializeAddon = new SerializeAddon();
+    terminal.loadAddon(serializeAddon);
+    return { terminal, serializeAddon };
   }
-  
+
   /**
    * Create a new terminal stream
    */
@@ -45,26 +53,45 @@ class TerminalStreamManager {
     projectPath?: string,
     projectId?: string,
     predefinedStreamId?: string,
-    outputStartIndex?: number
+    terminalSize?: { cols: number; rows: number }
   ): string {
+    const cols = terminalSize?.cols || 80;
+    const rows = terminalSize?.rows || 24;
+
     // Check if there's already a stream for this session
     const existingStreamId = this.sessionToStream.get(sessionId);
-    let preservedOutput: string[] = [];
     if (existingStreamId) {
       const existingStream = this.streams.get(existingStreamId);
       if (existingStream) {
-        if (existingStream.pty && existingStream.pty !== pty) {
-          // Different PTY, kill the old one
+        if (existingStream.pty === pty) {
+          // Same PTY (reconnection) - reuse existing headless terminal as-is
+          // The headless terminal already has all accumulated output
+          const newStreamId = predefinedStreamId || existingStreamId;
+
+          // Resize headless terminal if dimensions changed
+          existingStream.headlessTerminal.resize(cols, rows);
+
+          // Update stream ID if changed
+          if (newStreamId !== existingStreamId) {
+            this.streams.delete(existingStreamId);
+            existingStream.streamId = newStreamId;
+            this.streams.set(newStreamId, existingStream);
+            this.sessionToStream.set(sessionId, newStreamId);
+          }
+
+          return newStreamId;
+        }
+
+        // Different PTY, kill the old one and dispose headless terminal
+        if (existingStream.pty) {
           try {
             existingStream.pty.kill();
-          } catch (error) {
+          } catch {
             // Ignore error if PTY already killed
           }
-        } else if (existingStream.pty === pty) {
-          // Same PTY (reconnection after browser refresh) - preserve output buffer
-          preservedOutput = [...existingStream.output];
         }
-        // Remove the old stream
+        existingStream.serializeAddon.dispose();
+        existingStream.headlessTerminal.dispose();
         this.streams.delete(existingStreamId);
       }
     }
@@ -72,6 +99,9 @@ class TerminalStreamManager {
     // Use provided streamId or generate a new one
     const streamId = predefinedStreamId || `terminal-stream-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
 
+    // Create headless terminal
+    const { terminal: headlessTerminal, serializeAddon } = this.createHeadlessTerminal(cols, rows);
+
     const stream: TerminalStream = {
       streamId,
       sessionId,
@@ -82,9 +112,9 @@ class TerminalStreamManager {
       workingDirectory,
       projectPath,
       projectId,
-      output: preservedOutput,
       processId: pty.pid,
-      outputStartIndex: outputStartIndex || 0
+      headlessTerminal,
+      serializeAddon
     };
 
     this.streams.set(streamId, stream);
@@ -92,14 +122,14 @@ class TerminalStreamManager {
 
     return streamId;
   }
-  
+
   /**
    * Get stream by ID
    */
   getStream(streamId: string): TerminalStream | undefined {
     return this.streams.get(streamId);
   }
-  
+
   /**
    * Get stream by session ID
    */
@@ -110,121 +140,65 @@ class TerminalStreamManager {
     }
     return undefined;
   }
-  
+
   /**
-   * Add output to stream
+   * Add output to stream (writes to headless terminal)
    */
   addOutput(streamId: string, output: string): void {
     const stream = this.streams.get(streamId);
     if (stream) {
-      stream.output.push(output);
-
-      // Keep only last 2000 entries to prevent memory overflow
-      if (stream.output.length > 2000) {
-        stream.output = stream.output.slice(-2000);
-      }
-
-      // Also persist output to disk for background persistence
-      this.persistOutputToDisk(stream);
+      stream.headlessTerminal.write(output);
     }
   }
 
-  /** Pending write flag to coalesce rapid writes */
-  private pendingWrites = new Set<string>();
-
   /**
-   * Persist output to disk for cross-project persistence (async, coalesced)
+   * Get serialized terminal output for a stream
    */
-  private persistOutputToDisk(stream: TerminalStream): void {
-    // Coalesce rapid writes - only schedule one write per session per microtask
-    if (this.pendingWrites.has(stream.sessionId)) return;
-    this.pendingWrites.add(stream.sessionId);
-
-    queueMicrotask(() => {
-      this.pendingWrites.delete(stream.sessionId);
-
-      try {
-        const cacheFile = join(this.tempDir, `${stream.sessionId}.json`);
-
-        // Only save new output (from outputStartIndex onwards)
-        const newOutput = stream.outputStartIndex !== undefined
-          ? stream.output.slice(stream.outputStartIndex)
-          : stream.output;
-
-        const cacheData = {
-          streamId: stream.streamId,
-          sessionId: stream.sessionId,
-          command: stream.command,
-          projectId: stream.projectId,
-          projectPath: stream.projectPath,
-          workingDirectory: stream.workingDirectory,
-          startedAt: stream.startedAt,
-          status: stream.status,
-          output: newOutput,
-          outputStartIndex: stream.outputStartIndex || 0,
-          lastUpdated: new Date().toISOString()
-        };
-
-        // Use Bun.write for non-blocking async disk write
-        Bun.write(cacheFile, JSON.stringify(cacheData)).catch(() => {
-          // Silently handle write errors
-        });
-      } catch {
-        // Silently handle errors
-      }
-    });
+  getSerializedOutput(streamId: string): string {
+    const stream = this.streams.get(streamId);
+    if (stream) {
+      return stream.serializeAddon.serialize();
+    }
+    return '';
   }
 
   /**
-   * Load cached output from disk (public method for API access)
+   * Get serialized terminal output by session ID
    */
-  loadCachedOutput(sessionId: string): string[] | null {
-    try {
-      const cacheFile = join(this.tempDir, `${sessionId}.json`);
-      if (existsSync(cacheFile)) {
-        const data = JSON.parse(readFileSync(cacheFile, 'utf-8'));
-        return data.output || [];
-      }
-    } catch (error) {
-      // Silently handle read errors
+  getSerializedOutputBySession(sessionId: string): string {
+    const streamId = this.sessionToStream.get(sessionId);
+    if (streamId) {
+      return this.getSerializedOutput(streamId);
     }
-    return null;
+    return '';
   }
-  
+
   /**
-   * Get output from index
+   * Clear headless terminal buffer (sync with frontend clear)
    */
-  getOutput(streamId: string, fromIndex: number = 0): string[] {
-    const stream = this.streams.get(streamId);
-    if (stream) {
-      return stream.output.slice(fromIndex);
-    }
-
-    // If stream not in memory, try to load from cache
-    // This handles cases where server restarts or stream is cleaned from memory
-    const sessionId = this.getSessionIdByStreamId(streamId);
-    if (sessionId) {
-      const cachedOutput = this.loadCachedOutput(sessionId);
-      if (cachedOutput) {
-        return cachedOutput.slice(fromIndex);
+  clearHeadlessTerminal(sessionId: string): void {
+    const streamId = this.sessionToStream.get(sessionId);
+    if (streamId) {
+      const stream = this.streams.get(streamId);
+      if (stream) {
+        stream.headlessTerminal.clear();
       }
     }
-
-    return [];
   }
 
   /**
-   * Get session ID from stream ID (helper method)
+   * Resize headless terminal to match PTY dimensions
    */
-  private getSessionIdByStreamId(streamId: string): string | null {
-    for (const [sessionId, sid] of this.sessionToStream.entries()) {
-      if (sid === streamId) {
-        return sessionId;
+  resizeHeadlessTerminal(sessionId: string, cols: number, rows: number): void {
+    const streamId = this.sessionToStream.get(sessionId);
+    if (streamId) {
+      const stream = this.streams.get(streamId);
+      if (stream) {
+        stream.headlessTerminal.resize(cols, rows);
       }
     }
-    return null;
   }
-  
+
   /**
    * Update stream status
    */
@@ -232,7 +206,7 @@ class TerminalStreamManager {
     const stream = this.streams.get(streamId);
     if (stream) {
       stream.status = status;
-      
+
       // Clean up completed/cancelled streams after a delay
       if (status === 'completed' || status === 'cancelled' || status === 'error') {
         // Keep stream for 5 minutes for reconnection attempts
@@ -242,9 +216,9 @@ class TerminalStreamManager {
       }
     }
   }
-  
+
   /**
-   * Remove stream
+   * Remove stream and dispose headless terminal
    */
   removeStream(streamId: string): void {
     const stream = this.streams.get(streamId);
@@ -253,33 +227,27 @@ class TerminalStreamManager {
       if (stream.status === 'active' && stream.pty) {
         try {
           stream.pty.kill();
-        } catch (error) {
+        } catch {
           // Silently handle error
         }
       }
 
-      // Clean up cache file
-      try {
-        const cacheFile = join(this.tempDir, `${stream.sessionId}.json`);
-        if (existsSync(cacheFile)) {
-          unlinkSync(cacheFile);
-        }
-      } catch (error) {
-        // Silently handle error
-      }
+      // Dispose headless terminal
+      stream.serializeAddon.dispose();
+      stream.headlessTerminal.dispose();
 
       // Remove from maps
       this.streams.delete(streamId);
       this.sessionToStream.delete(stream.sessionId);
     }
   }
-  
+
   /**
    * Get stream status info
    */
   getStreamStatus(streamId: string): {
     status: string;
-    messagesCount: number;
+    bufferLength: number;
     startedAt: Date;
     processId?: number;
   } | null {
@@ -287,15 +255,37 @@ class TerminalStreamManager {
     if (!stream) {
       return null;
     }
-    
+
     return {
       status: stream.status,
-      messagesCount: stream.output.length,
+      bufferLength: stream.headlessTerminal.buffer.active.length,
       startedAt: stream.startedAt,
       processId: stream.processId
     };
   }
-  
+
+  /**
+   * Clean up terminal streams for a specific project
+   */
+  cleanupProjectStreams(projectId: string): number {
+    let cleaned = 0;
+
+    for (const [streamId, stream] of this.streams) {
+      if (stream.projectId === projectId) {
+        if (stream.status === 'active' && stream.pty) {
+          try { stream.pty.kill(); } catch {}
+        }
+        stream.serializeAddon.dispose();
+        stream.headlessTerminal.dispose();
+        this.streams.delete(streamId);
+        this.sessionToStream.delete(stream.sessionId);
+        cleaned++;
+      }
+    }
+
+    return cleaned;
+  }
+
   /**
    * Clean up all streams
    */
@@ -307,4 +297,4 @@ class TerminalStreamManager {
 }
 
 // Export singleton instance
-export const terminalStreamManager = new TerminalStreamManager();
+export const terminalStreamManager = new TerminalStreamManager();

package/backend/ws/chat/stream.ts

@@ -22,11 +22,11 @@ import { sessionQueries, messageQueries } from '../../database/queries';
 // exists (e.g., after browser refresh when user is on a different project).
 // Ensures cross-project notifications (presence update, sound, push) always work.
 // ============================================================================
-streamManager.on('stream:lifecycle', (event: { status: string; streamId: string; projectId?: string; chatSessionId?: string; timestamp: string }) => {
-	const { status, projectId, chatSessionId, timestamp } = event;
+streamManager.on('stream:lifecycle', (event: { status: string; streamId: string; projectId?: string; chatSessionId?: string; timestamp: string; reason?: string }) => {
+	const { status, projectId, chatSessionId, timestamp, reason } = event;
 	if (!projectId) return;
 
-	debug.log('chat', `Stream lifecycle: ${status} for project ${projectId} session ${chatSessionId}`);
+	debug.log('chat', `Stream lifecycle: ${status} for project ${projectId} session ${chatSessionId}${reason ? ` (reason: ${reason})` : ''}`);
 
 	// Mark any tool_use blocks that never got a tool_result as interrupted (persisted to DB)
 	if (chatSessionId) {
@@ -42,7 +42,8 @@ streamManager.on('stream:lifecycle', (event: { status: string; streamId: string;
 		projectId,
 		chatSessionId: chatSessionId || '',
 		status: status as 'completed' | 'error' | 'cancelled',
-		timestamp
+		timestamp,
+		reason
 	});
 
 	// Broadcast updated presence (status indicators for all projects)
@@ -254,8 +255,11 @@ export const streamHandler = createRouter()
 							break;
 					}
 				} catch (err) {
+					// Log but do NOT unsubscribe — one bad event must not kill the
+					// entire stream subscription. The bridge between StreamManager
+					// and the WS room would be permanently broken, causing the UI
+					// to stop receiving stream output while the WS stays connected.
 					debug.error('chat', 'Error handling stream event:', err);
-					unsubscribe();
 				}
 			};
 
@@ -393,8 +397,10 @@ export const streamHandler = createRouter()
 							break;
 					}
 				} catch (err) {
+					// Log but do NOT unsubscribe — same rationale as the initial
+					// stream handler: a transient error must not permanently break
+					// the EventEmitter → WS room bridge.
 					debug.error('chat', 'Error handling reconnected stream event:', err);
-					unsubscribe();
 				}
 			};
 
@@ -789,7 +795,8 @@ export const streamHandler = createRouter()
 		projectId: t.String(),
 		chatSessionId: t.String(),
 		status: t.Union([t.Literal('completed'), t.Literal('error'), t.Literal('cancelled')]),
-		timestamp: t.String()
+		timestamp: t.String(),
+		reason: t.Optional(t.String())
 	}))
 
 	.emit('chat:waiting-input', t.Object({

package/backend/ws/engine/claude/accounts.ts

@@ -33,15 +33,13 @@ function stripAnsi(str: string): string {
 		.replace(/\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])/g, '');
 }
 
+// Extracts the first https:// URL from PTY output.
+// Known formats (may change across Claude Code versions):
+//   - https://claude.ai/oauth/authorize?...
+//   - https://claude.com/cai/oauth/authorize?...
 function extractAuthUrl(clean: string): string | null {
-	const urlPrefix = 'https://claude.ai/oauth/authorize?';
-	const urlStart = clean.indexOf(urlPrefix);
-	if (urlStart === -1) return null;
-
-	const pasteIdx = clean.indexOf('Paste', urlStart);
-	if (pasteIdx === -1) return null;
-
-	return clean.substring(urlStart, pasteIdx).replace(/\s/g, '');
+	const match = clean.match(/https:\/\/\S+/);
+	return match ? match[0] : null;
 }
 
 function extractOAuthToken(clean: string): string | null {