@myrialabs/clopen 0.0.1

package/backend/lib/terminal/stream-manager.ts

@@ -0,0 +1,293 @@
+/**
+ * Terminal Stream Manager
+ * Manages background terminal streams and their state
+ */
+
+import type { IPty } from 'bun-pty';
+import { existsSync, mkdirSync, writeFileSync, readFileSync, unlinkSync } from 'fs';
+import { join } from 'path';
+
+interface TerminalStream {
+  streamId: string;
+  sessionId: string;
+  command: string;
+  pty: IPty;
+  status: 'active' | 'completed' | 'error' | 'cancelled';
+  startedAt: Date;
+  workingDirectory?: string;
+  projectPath?: string;
+  projectId?: string;
+  output: string[];
+  processId?: number;
+  outputStartIndex?: number; // Track where new output starts (for background output)
+}
+
+class TerminalStreamManager {
+  private streams: Map<string, TerminalStream> = new Map();
+  private sessionToStream: Map<string, string> = new Map();
+  private tempDir: string = '.terminal-output-cache';
+
+  constructor() {
+    // Create temp directory for output caching
+    if (!existsSync(this.tempDir)) {
+      mkdirSync(this.tempDir, { recursive: true });
+    }
+  }
+  
+  /**
+   * Create a new terminal stream
+   */
+  createStream(
+    sessionId: string,
+    command: string,
+    pty: IPty,
+    workingDirectory?: string,
+    projectPath?: string,
+    projectId?: string,
+    predefinedStreamId?: string,
+    outputStartIndex?: number
+  ): string {
+    // Check if there's already a stream for this session
+    const existingStreamId = this.sessionToStream.get(sessionId);
+    if (existingStreamId) {
+      const existingStream = this.streams.get(existingStreamId);
+      if (existingStream) {
+        // Clean up existing stream first
+        if (existingStream.pty && existingStream.pty !== pty) {
+          // If it's a different PTY, kill the old one
+          try {
+            existingStream.pty.kill();
+          } catch (error) {
+            // Ignore error if PTY already killed
+          }
+        }
+        // Remove the old stream
+        this.streams.delete(existingStreamId);
+      }
+    }
+
+    // Use provided streamId or generate a new one
+    const streamId = predefinedStreamId || `terminal-stream-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+
+    const stream: TerminalStream = {
+      streamId,
+      sessionId,
+      command,
+      pty,
+      status: 'active',
+      startedAt: new Date(),
+      workingDirectory,
+      projectPath,
+      projectId,
+      output: [],
+      processId: pty.pid,
+      outputStartIndex: outputStartIndex || 0
+    };
+
+    this.streams.set(streamId, stream);
+    this.sessionToStream.set(sessionId, streamId);
+
+    return streamId;
+  }
+  
+  /**
+   * Get stream by ID
+   */
+  getStream(streamId: string): TerminalStream | undefined {
+    return this.streams.get(streamId);
+  }
+  
+  /**
+   * Get stream by session ID
+   */
+  getStreamBySession(sessionId: string): TerminalStream | undefined {
+    const streamId = this.sessionToStream.get(sessionId);
+    if (streamId) {
+      return this.streams.get(streamId);
+    }
+    return undefined;
+  }
+  
+  /**
+   * Add output to stream
+   */
+  addOutput(streamId: string, output: string): void {
+    const stream = this.streams.get(streamId);
+    if (stream) {
+      stream.output.push(output);
+
+      // Keep only last 2000 entries to prevent memory overflow
+      if (stream.output.length > 2000) {
+        stream.output = stream.output.slice(-2000);
+      }
+
+      // Also persist output to disk for background persistence
+      this.persistOutputToDisk(stream);
+    }
+  }
+
+  /**
+   * Persist output to disk for cross-project persistence
+   */
+  private persistOutputToDisk(stream: TerminalStream): void {
+    try {
+      const cacheFile = join(this.tempDir, `${stream.sessionId}.json`);
+
+      // Only save new output (from outputStartIndex onwards)
+      // This prevents duplicating old output that was already displayed
+      const newOutput = stream.outputStartIndex !== undefined
+        ? stream.output.slice(stream.outputStartIndex)
+        : stream.output;
+
+      const cacheData = {
+        streamId: stream.streamId,
+        sessionId: stream.sessionId,
+        command: stream.command,
+        projectId: stream.projectId,
+        projectPath: stream.projectPath,
+        workingDirectory: stream.workingDirectory,
+        startedAt: stream.startedAt,
+        status: stream.status,
+        output: newOutput, // Only save new output
+        outputStartIndex: stream.outputStartIndex || 0,
+        lastUpdated: new Date().toISOString()
+      };
+      writeFileSync(cacheFile, JSON.stringify(cacheData));
+    } catch (error) {
+      // Silently handle write errors
+    }
+  }
+
+  /**
+   * Load cached output from disk (public method for API access)
+   */
+  loadCachedOutput(sessionId: string): string[] | null {
+    try {
+      const cacheFile = join(this.tempDir, `${sessionId}.json`);
+      if (existsSync(cacheFile)) {
+        const data = JSON.parse(readFileSync(cacheFile, 'utf-8'));
+        return data.output || [];
+      }
+    } catch (error) {
+      // Silently handle read errors
+    }
+    return null;
+  }
+  
+  /**
+   * Get output from index
+   */
+  getOutput(streamId: string, fromIndex: number = 0): string[] {
+    const stream = this.streams.get(streamId);
+    if (stream) {
+      return stream.output.slice(fromIndex);
+    }
+
+    // If stream not in memory, try to load from cache
+    // This handles cases where server restarts or stream is cleaned from memory
+    const sessionId = this.getSessionIdByStreamId(streamId);
+    if (sessionId) {
+      const cachedOutput = this.loadCachedOutput(sessionId);
+      if (cachedOutput) {
+        return cachedOutput.slice(fromIndex);
+      }
+    }
+
+    return [];
+  }
+
+  /**
+   * Get session ID from stream ID (helper method)
+   */
+  private getSessionIdByStreamId(streamId: string): string | null {
+    for (const [sessionId, sid] of this.sessionToStream.entries()) {
+      if (sid === streamId) {
+        return sessionId;
+      }
+    }
+    return null;
+  }
+  
+  /**
+   * Update stream status
+   */
+  updateStatus(streamId: string, status: TerminalStream['status']): void {
+    const stream = this.streams.get(streamId);
+    if (stream) {
+      stream.status = status;
+      
+      // Clean up completed/cancelled streams after a delay
+      if (status === 'completed' || status === 'cancelled' || status === 'error') {
+        // Keep stream for 5 minutes for reconnection attempts
+        setTimeout(() => {
+          this.removeStream(streamId);
+        }, 5 * 60 * 1000);
+      }
+    }
+  }
+  
+  /**
+   * Remove stream
+   */
+  removeStream(streamId: string): void {
+    const stream = this.streams.get(streamId);
+    if (stream) {
+      // Kill PTY if still active
+      if (stream.status === 'active' && stream.pty) {
+        try {
+          stream.pty.kill();
+        } catch (error) {
+          // Silently handle error
+        }
+      }
+
+      // Clean up cache file
+      try {
+        const cacheFile = join(this.tempDir, `${stream.sessionId}.json`);
+        if (existsSync(cacheFile)) {
+          unlinkSync(cacheFile);
+        }
+      } catch (error) {
+        // Silently handle error
+      }
+
+      // Remove from maps
+      this.streams.delete(streamId);
+      this.sessionToStream.delete(stream.sessionId);
+    }
+  }
+  
+  /**
+   * Get stream status info
+   */
+  getStreamStatus(streamId: string): {
+    status: string;
+    messagesCount: number;
+    startedAt: Date;
+    processId?: number;
+  } | null {
+    const stream = this.streams.get(streamId);
+    if (!stream) {
+      return null;
+    }
+    
+    return {
+      status: stream.status,
+      messagesCount: stream.output.length,
+      startedAt: stream.startedAt,
+      processId: stream.processId
+    };
+  }
+  
+  /**
+   * Clean up all streams
+   */
+  cleanup(): void {
+    for (const streamId of this.streams.keys()) {
+      this.removeStream(streamId);
+    }
+  }
+}
+
+// Export singleton instance
+export const terminalStreamManager = new TerminalStreamManager();

package/backend/lib/tunnel/global-tunnel-manager.ts

@@ -0,0 +1,243 @@
+import { Tunnel, bin, install } from 'cloudflared';
+import { debug } from '$shared/utils/logger';
+import { existsSync } from 'fs';
+
+interface TunnelInstance {
+	tunnel: any;
+	publicUrl: string;
+	localPort: number;
+	startedAt: Date;
+	autoStopTimer?: NodeJS.Timeout;
+}
+
+class GlobalTunnelManager {
+	// Key format: "port" for global tunnels
+	private activeTunnels = new Map<number, TunnelInstance>();
+	private binaryInstalled = false;
+
+	/**
+	 * Ensure cloudflared binary is installed
+	 * Downloads binary on first use (~66MB, takes 30-60 seconds)
+	 */
+	private async ensureBinaryInstalled(
+		onProgress?: (stage: string, data?: any) => void
+	): Promise<{
+		needsDownload: boolean;
+		downloadTime?: number;
+	}> {
+		// Check if already verified in this session
+		if (this.binaryInstalled) {
+			onProgress?.('binary-ready', { cached: true });
+			return { needsDownload: false };
+		}
+
+		// Check if binary file exists
+		if (!existsSync(bin)) {
+			debug.log('tunnel', '[PROGRESS:DOWNLOADING_BINARY] Cloudflared binary not found, downloading...');
+			debug.log('tunnel', 'This may take 30-60 seconds on first use (downloading ~66MB)');
+			onProgress?.('downloading-binary', { size: '~66MB' });
+
+			const startTime = Date.now();
+
+			try {
+				// Call install() from cloudflared package
+				// This downloads the binary from Cloudflare's CDN
+				await install(bin);
+
+				const downloadTime = Date.now() - startTime;
+				debug.log('tunnel', `[PROGRESS:BINARY_READY] Cloudflared binary installed successfully in ${downloadTime}ms at: ${bin}`);
+
+				// Verify the binary was actually created
+				if (!existsSync(bin)) {
+					throw new Error('Binary file was not created after installation');
+				}
+
+				this.binaryInstalled = true;
+				onProgress?.('binary-ready', { downloaded: true, time: downloadTime });
+				return { needsDownload: true, downloadTime };
+			} catch (error) {
+				debug.error('tunnel', 'Failed to install cloudflared binary:', error);
+				throw new Error(
+					'Failed to download cloudflared binary. Please check your internet connection and try again. ' +
+						'The tunnel feature requires downloading ~66MB on first use.'
+				);
+			}
+		} else {
+			debug.log('tunnel', '[PROGRESS:BINARY_READY] Cloudflared binary already installed at:', bin);
+			this.binaryInstalled = true;
+			onProgress?.('binary-ready', { cached: true });
+			return { needsDownload: false };
+		}
+	}
+
+	/**
+	 * Start cloudflared tunnel for a port
+	 */
+	async startTunnel(
+		port: number,
+		autoStopMinutes: number = 60,
+		onProgress?: (stage: string, data?: any) => void
+	): Promise<{ publicUrl: string; status: 'active'; binaryDownloaded: boolean; timings: any }> {
+		debug.log('tunnel', '[PROGRESS:CHECKING_BINARY] Checking if binary is installed...');
+		onProgress?.('checking-binary');
+
+		// Ensure binary is installed
+		const binaryInfo = await this.ensureBinaryInstalled(onProgress);
+		const timings: any = {};
+
+		if (binaryInfo.needsDownload) {
+			timings.binaryDownload = binaryInfo.downloadTime;
+		}
+
+		// Check if tunnel already exists for this port
+		if (this.activeTunnels.has(port)) {
+			const existing = this.activeTunnels.get(port)!;
+			debug.log('tunnel', `Tunnel already exists for port ${port}, returning existing URL`);
+			return {
+				publicUrl: existing.publicUrl,
+				status: 'active',
+				binaryDownloaded: binaryInfo.needsDownload,
+				timings
+			};
+		}
+
+		debug.log('tunnel', `[PROGRESS:STARTING_TUNNEL] Starting cloudflared tunnel for port ${port}...`);
+		onProgress?.('starting-tunnel', { port });
+
+		const tunnelStartTime = Date.now();
+
+		try {
+			// Create tunnel instance
+			const tunnel = new Tunnel({
+				// Bind to specific port
+				url: `http://localhost:${port}`
+			});
+
+			// Wait for tunnel to be ready
+			debug.log('tunnel', '[PROGRESS:GENERATING_URL] Waiting for public URL...');
+			onProgress?.('generating-url');
+
+			const publicUrl: string = await new Promise((resolve, reject) => {
+				const timeout = setTimeout(() => {
+					reject(new Error('Tunnel connection timeout (30s)'));
+				}, 30000); // 30 second timeout
+
+				tunnel.on('url', (url: string) => {
+					debug.log('tunnel', `[PROGRESS:CONNECTED] ✅ Tunnel connected! Public URL: ${url}`);
+					clearTimeout(timeout);
+					resolve(url);
+				});
+
+				tunnel.on('error', (error: Error) => {
+					debug.error('tunnel', '[PROGRESS:FAILED] Tunnel error:', error);
+					clearTimeout(timeout);
+					reject(error);
+				});
+			});
+
+			timings.tunnelStart = Date.now() - tunnelStartTime;
+			timings.total = Date.now() - tunnelStartTime + (binaryInfo.downloadTime || 0);
+
+			// Setup auto-stop timer
+			const autoStopMs = autoStopMinutes * 60 * 1000;
+			const autoStopTimer = setTimeout(
+				() => {
+					debug.log('tunnel', `Auto-stopping tunnel on port ${port} after ${autoStopMinutes} minutes`);
+					this.stopTunnel(port);
+				},
+				autoStopMs
+			);
+
+			// Store tunnel instance
+			const instance: TunnelInstance = {
+				tunnel,
+				publicUrl,
+				localPort: port,
+				startedAt: new Date(),
+				autoStopTimer
+			};
+
+			this.activeTunnels.set(port, instance);
+
+			debug.log('tunnel', `✅ Tunnel started successfully on port ${port}`);
+			debug.log('tunnel', `   Public URL: ${publicUrl}`);
+			debug.log('tunnel', `   Auto-stop: ${autoStopMinutes} minutes`);
+			debug.log('tunnel', `   Timings:`, timings);
+			onProgress?.('connected', { publicUrl, timings });
+
+			return {
+				publicUrl,
+				status: 'active',
+				binaryDownloaded: binaryInfo.needsDownload,
+				timings
+			};
+		} catch (error) {
+			debug.error('tunnel', `Failed to start tunnel on port ${port}:`, error);
+			throw error;
+		}
+	}
+
+	/**
+	 * Stop tunnel for a port
+	 */
+	async stopTunnel(port: number): Promise<void> {
+		const instance = this.activeTunnels.get(port);
+
+		if (!instance) {
+			debug.log('tunnel', `No active tunnel found for port ${port}`);
+			return;
+		}
+
+		debug.log('tunnel', `Stopping tunnel on port ${port}...`);
+
+		try {
+			// Clear auto-stop timer
+			if (instance.autoStopTimer) {
+				clearTimeout(instance.autoStopTimer);
+			}
+
+			// Stop tunnel
+			if (instance.tunnel && typeof instance.tunnel.stop === 'function') {
+				await instance.tunnel.stop();
+			}
+
+			// Remove from active tunnels
+			this.activeTunnels.delete(port);
+
+			debug.log('tunnel', `✅ Tunnel stopped on port ${port}`);
+		} catch (error) {
+			debug.error('tunnel', `Error stopping tunnel on port ${port}:`, error);
+			// Still remove from active tunnels even if stop failed
+			this.activeTunnels.delete(port);
+			throw error;
+		}
+	}
+
+	/**
+	 * Get all active tunnels
+	 */
+	getActiveTunnels(): Array<{
+		port: number;
+		publicUrl: string;
+		startedAt: string;
+	}> {
+		return Array.from(this.activeTunnels.entries()).map(([port, instance]) => ({
+			port,
+			publicUrl: instance.publicUrl,
+			startedAt: instance.startedAt.toISOString()
+		}));
+	}
+
+	/**
+	 * Stop all tunnels
+	 */
+	async stopAllTunnels(): Promise<void> {
+		debug.log('tunnel', 'Stopping all tunnels...');
+		const ports = Array.from(this.activeTunnels.keys());
+		await Promise.all(ports.map((port) => this.stopTunnel(port)));
+		debug.log('tunnel', '✅ All tunnels stopped');
+	}
+}
+
+// Singleton instance
+export const globalTunnelManager = new GlobalTunnelManager();