@myrialabs/clopen 0.1.10 → 0.2.0

package/backend/lib/mcp/remote-server.ts

@@ -0,0 +1,132 @@
+/**
+ * Remote MCP HTTP Server for Open Code
+ *
+ * Serves custom MCP tools over HTTP (Streamable HTTP transport) so Open Code
+ * can connect via `type: 'remote'` config instead of spawning a stdio subprocess.
+ *
+ * Tool handlers execute directly in the main Clopen process — no subprocess,
+ * no WebSocket bridge. This is architecturally identical to how Claude Code
+ * uses in-process MCP servers via createSdkMcpServer().
+ *
+ * Transport: WebStandardStreamableHTTPServerTransport (works natively with Bun)
+ */
+
+import { WebStandardStreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/webStandardStreamableHttp.js';
+import { isInitializeRequest } from '@modelcontextprotocol/sdk/types.js';
+import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
+import { createRemoteMcpServer } from './servers/helper';
+import { debug } from '$shared/utils/logger';
+
+// Lazy imports to avoid circular dependencies at module load time
+let _allServers: Parameters<typeof createRemoteMcpServer>[0] | null = null;
+let _enabledConfig: Parameters<typeof createRemoteMcpServer>[1] | null = null;
+
+async function getServerDeps() {
+	if (!_allServers || !_enabledConfig) {
+		const { allServers } = await import('./servers/index');
+		const { mcpServersConfig } = await import('./config');
+		_allServers = allServers;
+		_enabledConfig = mcpServersConfig;
+	}
+	return { allServers: _allServers, enabledConfig: _enabledConfig };
+}
+
+// ============================================================================
+// Session Management
+// ============================================================================
+
+/** Active transports keyed by MCP session ID */
+const transports = new Map<string, WebStandardStreamableHTTPServerTransport>();
+
+/** Active MCP servers keyed by MCP session ID */
+const servers = new Map<string, McpServer>();
+
+/**
+ * Handle an incoming MCP HTTP request (GET/POST/DELETE).
+ *
+ * Mounted at /mcp on the main Elysia server.
+ * Follows the Streamable HTTP transport protocol:
+ * - POST without session: initialization → create new transport + server
+ * - POST with session: route to existing transport
+ * - GET with session: SSE stream for server notifications
+ * - DELETE with session: close session
+ */
+export async function handleMcpRequest(request: Request): Promise<Response> {
+	const sessionId = request.headers.get('mcp-session-id');
+
+	// Existing session — route to its transport
+	if (sessionId && transports.has(sessionId)) {
+		const transport = transports.get(sessionId)!;
+		return transport.handleRequest(request);
+	}
+
+	// New initialization request — create transport + MCP server
+	if (request.method === 'POST') {
+		// Parse body to check if it's an init request
+		const body = await request.json();
+
+		if (isInitializeRequest(body)) {
+			const { allServers, enabledConfig } = await getServerDeps();
+
+			const transport = new WebStandardStreamableHTTPServerTransport({
+				sessionIdGenerator: () => crypto.randomUUID(),
+				onsessioninitialized: (sid) => {
+					transports.set(sid, transport);
+					debug.log('mcp', `🌐 Remote MCP session initialized: ${sid}`);
+				},
+				onsessionclosed: (sid) => {
+					transports.delete(sid);
+					servers.delete(sid);
+					debug.log('mcp', `🌐 Remote MCP session closed: ${sid}`);
+				},
+			});
+
+			transport.onclose = () => {
+				if (transport.sessionId) {
+					transports.delete(transport.sessionId);
+					servers.delete(transport.sessionId);
+				}
+			};
+
+			// Create a fresh MCP server with all enabled tools (in-process handlers)
+			const mcpServer = createRemoteMcpServer(allServers, enabledConfig);
+			await mcpServer.connect(transport);
+
+			// Store server reference for cleanup
+			if (transport.sessionId) {
+				servers.set(transport.sessionId, mcpServer);
+			}
+
+			// Handle the initialization request with pre-parsed body
+			return transport.handleRequest(request, { parsedBody: body });
+		}
+	}
+
+	// Invalid request
+	return new Response(JSON.stringify({
+		jsonrpc: '2.0',
+		error: { code: -32000, message: 'Bad Request: No valid session ID provided' },
+		id: null,
+	}), {
+		status: 400,
+		headers: { 'Content-Type': 'application/json' },
+	});
+}
+
+/**
+ * Close all active MCP sessions and transports.
+ * Called during graceful server shutdown.
+ */
+export async function closeMcpServer(): Promise<void> {
+	for (const [sessionId, transport] of transports) {
+		try {
+			await transport.close();
+			debug.log('mcp', `🌐 Remote MCP transport closed: ${sessionId}`);
+		} catch (error) {
+			debug.error('mcp', `Error closing MCP transport ${sessionId}:`, error);
+		}
+	}
+	transports.clear();
+	servers.clear();
+	debug.log('mcp', '🌐 All remote MCP sessions closed');
+}

package/backend/lib/mcp/servers/helper.ts

@@ -3,10 +3,14 @@
  *
  * Stores both the Claude SDK server instance AND raw tool definitions
  * so the same source can be used for Claude Code (in-process) and
- * Open Code (stdio subprocess via @modelcontextprotocol/sdk).
+ * Open Code (remote HTTP MCP via @modelcontextprotocol/sdk).
+ *
+ * Claude Code: createSdkMcpServer() → in-process MCP server
+ * Open Code:   createRemoteMcpServer() → HTTP MCP server (same process, same handlers)
  */
 
 import { createSdkMcpServer, tool } from "@anthropic-ai/claude-agent-sdk";
+import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import type { z } from "zod";
 
 /**
@@ -35,8 +39,7 @@ type ToolHandler<TSchema extends Record<string, z.ZodType<any>> | undefined> =
  * Raw tool definition — schema, description, and handler.
  * Single source of truth used by:
  * - Claude Code: in-process via createSdkMcpServer
- * - Open Code stdio: schema/description for registration, handler via bridge
- * - MCP bridge: handler for in-process execution
+ * - Open Code: remote HTTP MCP via createRemoteMcpServer (in-process handlers)
  */
 export interface RawToolDef {
 	description: string;
@@ -152,3 +155,46 @@ export function buildServerRegistries<
 		}
 	};
 }
+
+// ============================================================================
+// Remote MCP Server for Open Code (HTTP transport, in-process execution)
+// ============================================================================
+
+/**
+ * Create a McpServer instance (from @modelcontextprotocol/sdk) with tools registered
+ * from the same RawToolDef definitions used by Claude Code.
+ *
+ * This is the Open Code equivalent of createSdkMcpServer() for Claude Code.
+ * Handlers execute directly in-process — no subprocess, no bridge.
+ *
+ * @param servers - Server definitions from defineServer()
+ * @param enabledConfig - Which servers/tools are enabled (from mcpServersConfig)
+ */
+export function createRemoteMcpServer(
+	servers: readonly ServerWithMeta<string, readonly string[]>[],
+	enabledConfig: Record<string, { enabled: boolean; tools: readonly string[] }>
+): McpServer {
+	const mcpServer = new McpServer({
+		name: 'clopen-mcp',
+		version: '1.0.0',
+	});
+
+	for (const srv of servers) {
+		const config = enabledConfig[srv.meta.name];
+		if (!config?.enabled) continue;
+
+		for (const toolName of config.tools) {
+			const def = srv.meta.toolDefs[toolName as string];
+			if (!def) continue;
+
+			mcpServer.registerTool(toolName as string, {
+				description: def.description,
+				inputSchema: def.schema,
+			}, async (args: Record<string, unknown>) => {
+				return await def.handler(args) as any;
+			});
+		}
+	}
+
+	return mcpServer;
+}

package/backend/lib/mcp/servers/index.ts

@@ -14,8 +14,9 @@ import weather from './weather/index';
 import browserAutomation from './browser-automation/index';
 import { buildServerRegistries } from './helper';
 
-// Re-export types for stdio server
+// Re-export types and remote server factory
 export type { RawToolDef } from './helper';
+export { createRemoteMcpServer } from './helper';
 
 /**
  * All MCP Servers
@@ -23,7 +24,7 @@ export type { RawToolDef } from './helper';
  * Simply import and add new servers to this array.
  * Metadata and registry will be automatically built.
  */
-const allServers = [
+export const allServers = [
 	weather,
 	browserAutomation,
 	// Add more servers here...

package/backend/lib/preview/browser/browser-audio-capture.ts

@@ -11,14 +11,31 @@ import { audioCaptureScript } from './scripts/audio-stream';
 
 export class BrowserAudioCapture {
 	/**
-	 * Setup audio capture for a page
-	 * Injects audio capture script that intercepts AudioContext before page loads
+	 * Setup audio capture for a page (pre-navigation).
+	 * WARNING: Uses evaluateOnNewDocument which patches AudioContext BEFORE page
+	 * scripts run. This is detected by Cloudflare's fingerprinting.
+	 * Prefer injectAudioCapture() for post-navigation injection.
 	 */
 	async setupAudioCapture(page: Page, config: StreamingConfig['audio']): Promise<void> {
-		// Inject audio capture script BEFORE page loads to intercept AudioContext
 		await page.evaluateOnNewDocument(audioCaptureScript, config);
 	}
 
+	/**
+	 * Inject audio capture into the current page context (post-navigation).
+	 * Uses page.evaluate() instead of evaluateOnNewDocument() to avoid
+	 * Cloudflare detection — AudioContext constructor patching before page
+	 * load is heavily flagged by CF's fingerprinting algorithms.
+	 * Call this AFTER navigation completes and CF challenges pass.
+	 */
+	async injectAudioCapture(page: Page, config: StreamingConfig['audio']): Promise<boolean> {
+		try {
+			await page.evaluate(audioCaptureScript, config);
+			return true;
+		} catch {
+			return false;
+		}
+	}
+
 	/**
 	 * Check if audio encoder is supported in the page
 	 */

package/backend/lib/preview/browser/browser-navigation-tracker.ts

@@ -62,6 +62,8 @@ export class BrowserNavigationTracker extends EventEmitter {
 		});
 
 		// Track hash changes (fragment identifier changes like #contact-us)
+		// Temporarily disabled URL tracking injection to test CloudFlare evasion
+		/*
 		await page.evaluateOnNewDocument(() => {
 			let lastUrl = window.location.href;
 
@@ -86,6 +88,7 @@ export class BrowserNavigationTracker extends EventEmitter {
 			// Periodically check for URL changes (for SPA navigation)
 			setInterval(checkUrlChange, 500);
 		});
+		*/
 	}
 
 	async navigateSession(sessionId: string, session: BrowserTab, url: string): Promise<string> {

package/backend/lib/preview/browser/browser-pool.ts

@@ -1,35 +1,32 @@
 /**
- * Browser Pool Module using puppeteer-cluster
+ * Browser Pool Module
  *
- * Uses puppeteer-cluster for efficient browser management with isolated contexts.
+ * Uses puppeteer-extra with StealthPlugin for Cloudflare bypass.
+ * Architecture mirrors the working test-cf.ts approach:
+ * - Single shared browser launched directly via puppeteer.launch()
+ * - Isolated BrowserContext per session (separate cookies, storage, cache)
+ * - StealthPlugin applied at launch time (via puppeteer-extra hooks)
  *
- * Architecture:
- * - puppeteer-cluster manages browser lifecycle and crash recovery
- * - CONCURRENCY_CONTEXT mode: shared browser, isolated contexts per worker
- * - Each user session gets its own isolated BrowserContext
- *
- * Isolation per session:
- * - Separate cookies
- * - Separate localStorage/sessionStorage
- * - Separate cache
- * - Separate service workers
- * - No data leakage between users
- *
- * Memory Usage:
- * - 1 user: ~300MB (browser) + ~20MB (context) = ~320MB
- * - 10 users: ~300MB (browser) + ~200MB (contexts) = ~500MB
- * - vs. old: 10 users = 10 browsers = 2-5GB
+ * Why not puppeteer-cluster?
+ * - Cluster's CONCURRENCY_CONTEXT mode accesses the browser via the raw
+ *   underlying reference, bypassing puppeteer-extra's page creation hooks.
+ * - This causes a race condition where stealth evasions (evaluateOnNewDocument)
+ *   may not be registered before the first navigation, breaking Cloudflare bypass.
+ * - Direct launch() ensures puppeteer-extra wraps ALL page creation correctly.
  */
 
-import { Cluster } from 'puppeteer-cluster';
 import type { Browser, BrowserContext, Page } from 'puppeteer';
 import { debug } from '$shared/utils/logger';
+import puppeteer from 'puppeteer-extra';
+import StealthPlugin from 'puppeteer-extra-plugin-stealth';
+
+puppeteer.use(StealthPlugin());
 
 export interface PoolConfig {
-	maxConcurrency: number; // Maximum concurrent isolated contexts
-	timeout: number; // Task timeout
-	retryLimit: number; // Number of retries on failure
-	retryDelay: number; // Delay between retries
+	maxConcurrency: number;
+	timeout: number;
+	retryLimit: number;
+	retryDelay: number;
 }
 
 export interface PooledSession {
@@ -40,150 +37,89 @@ export interface PooledSession {
 }
 
 const DEFAULT_CONFIG: PoolConfig = {
-	maxConcurrency: 50, // Support up to 50 concurrent users
-	timeout: 60000, // 60 second timeout
-	retryLimit: 3, // Retry 3 times on failure
-	retryDelay: 1000 // 1 second delay between retries
+	maxConcurrency: 50,
+	timeout: 60000,
+	retryLimit: 3,
+	retryDelay: 1000
 };
 
 /**
- * Optimized Chromium launch arguments for low-resource usage
+ * Chromium launch arguments for stealth (matches test-cf.ts exactly)
  */
 const CHROMIUM_ARGS = [
-	// === CORE STABILITY (Windows compatible) ===
 	'--no-sandbox',
-	'--disable-dev-shm-usage',
-	'--disable-gpu',
-
-	// === PREVENT THROTTLING ===
-	'--disable-background-timer-throttling',
-	'--disable-backgrounding-occluded-windows',
-	'--disable-renderer-backgrounding',
-
-	// === DISABLE UNNECESSARY FEATURES ===
-	'--no-first-run',
-	'--no-default-browser-check',
-	'--disable-extensions',
-	'--disable-popup-blocking',
-
-	// === LOW-END DEVICE OPTIMIZATIONS ===
-	'--memory-pressure-off',
-	'--disable-features=TranslateUI',
-	'--disable-sync',
-	'--disable-domain-reliability',
-	'--disable-client-side-phishing-detection',
-	'--disable-software-rasterizer',
-	'--disable-smooth-scrolling',
-	'--disable-threaded-animation',
-	'--disable-threaded-scrolling',
-	'--disable-composited-antialiasing',
-	'--disable-webgl',
-	'--disable-webgl2',
-	'--disable-accelerated-2d-canvas',
-	'--disable-gpu-vsync',
-	'--disable-ipc-flooding-protection',
-
-	// === AUDIO SUPPORT ===
-	'--autoplay-policy=no-user-gesture-required',
-	'--use-fake-ui-for-media-stream'
+	'--disable-blink-features=AutomationControlled',
+	'--window-size=1366,768'
 ];
 
 class BrowserPool {
-	private cluster: Cluster | null = null;
+	private browser: Browser | null = null;
 	private sessions = new Map<string, PooledSession>();
 	private config: PoolConfig;
-	private isInitializing = false;
-	private initPromise: Promise<Cluster> | null = null;
+	private isLaunching = false;
+	private launchPromise: Promise<Browser> | null = null;
 
 	constructor(config: Partial<PoolConfig> = {}) {
 		this.config = { ...DEFAULT_CONFIG, ...config };
 	}
 
 	/**
-	 * Get or create the puppeteer-cluster instance
-	 * Thread-safe: multiple calls during initialization will wait for the same promise
+	 * Get or create the shared browser instance.
+	 * Uses puppeteer-extra directly (same as test-cf.ts) to ensure
+	 * StealthPlugin hooks fire for every page created.
 	 */
-	async getCluster(): Promise<Cluster> {
-		if (this.cluster) {
-			return this.cluster;
+	async getBrowser(): Promise<Browser> {
+		if (this.browser?.connected) {
+			return this.browser;
 		}
 
-		if (this.isInitializing && this.initPromise) {
-			return this.initPromise;
+		if (this.isLaunching && this.launchPromise) {
+			return this.launchPromise;
 		}
 
-		this.isInitializing = true;
-		this.initPromise = this.launchCluster();
+		this.isLaunching = true;
+		this.launchPromise = this.launchBrowser();
 
 		try {
-			this.cluster = await this.initPromise;
-			return this.cluster;
+			this.browser = await this.launchPromise;
+			return this.browser;
 		} finally {
-			this.isInitializing = false;
-			this.initPromise = null;
+			this.isLaunching = false;
+			this.launchPromise = null;
 		}
 	}
 
 	/**
-	 * Launch puppeteer-cluster with CONCURRENCY_CONTEXT for isolation
+	 * Launch browser via puppeteer-extra (with StealthPlugin already registered).
+	 * This matches test-cf.ts which successfully bypasses Cloudflare.
 	 */
-	private async launchCluster(): Promise<Cluster> {
-		debug.log('preview', '🚀 Launching puppeteer-cluster...');
-
-		const cluster = await Cluster.launch({
-			// CONCURRENCY_CONTEXT: shared browser, isolated context per worker
-			// Each context has its own cookies, localStorage, sessionStorage, cache
-			concurrency: Cluster.CONCURRENCY_CONTEXT,
-			maxConcurrency: this.config.maxConcurrency,
-			timeout: this.config.timeout,
-			retryLimit: this.config.retryLimit,
-			retryDelay: this.config.retryDelay,
-
-			puppeteerOptions: {
-				headless: true,
-				args: CHROMIUM_ARGS
-			},
-
-			// Monitor events
-			monitor: false // Disable built-in monitoring, we use our own logging
+	private async launchBrowser(): Promise<Browser> {
+		debug.log('preview', '🚀 Launching browser with puppeteer-extra + StealthPlugin...');
+
+		const browser = await puppeteer.launch({
+			headless: true,
+			channel: 'chrome',
+			args: CHROMIUM_ARGS
+		}) as unknown as Browser;
+
+		debug.log('preview', '✅ Browser launched successfully');
+
+		// Handle browser disconnection
+		browser.on('disconnected', () => {
+			debug.warn('preview', '⚠️ Browser disconnected');
+			this.browser = null;
+			// Close all sessions since browser is gone
+			this.sessions.clear();
 		});
 
-		// Handle cluster errors
-		cluster.on('taskerror', (err, data) => {
-			debug.error('preview', `Task error for session ${data?.sessionId}:`, err.message);
-		});
-
-		debug.log('preview', '✅ puppeteer-cluster launched successfully');
-		debug.log('preview', `📊 Max concurrency: ${this.config.maxConcurrency}`);
-
-		return cluster;
-	}
-
-	/**
-	 * Get the shared browser instance from the cluster
-	 * Note: This accesses the internal browser - use with caution
-	 */
-	async getBrowser(): Promise<Browser> {
-		const cluster = await this.getCluster();
-
-		// Access the browser through a dummy task
-		// This is a workaround since cluster doesn't expose browser directly
-		return new Promise((resolve, reject) => {
-			cluster
-				.execute(async ({ page }: { page: Page }) => {
-					const browser = page.browser();
-					resolve(browser);
-				})
-				.catch(reject);
-		});
+		return browser;
 	}
 
 	/**
-	 * Create an isolated session with its own BrowserContext
-	 * Uses puppeteer-cluster's task execution for proper resource management
+	 * Create an isolated session with its own BrowserContext.
+	 * Each context has separate cookies, localStorage, sessionStorage, and cache.
 	 */
 	async createSession(sessionId: string): Promise<PooledSession> {
-		// Check if session already exists
 		const existing = this.sessions.get(sessionId);
 		if (existing) {
 			debug.log('preview', `♻️ Reusing existing session: ${sessionId}`);
@@ -192,13 +128,10 @@ class BrowserPool {
 
 		debug.log('preview', `🔒 Creating isolated session: ${sessionId}`);
 
-		const cluster = await this.getCluster();
 		const browser = await this.getBrowser();
 
-		// Create isolated BrowserContext for this session
-		// This provides FULL ISOLATION: cookies, localStorage, sessionStorage, cache
-		// When sessionId is prefixed with projectId (e.g., "project123:tab-1"),
-		// this ensures complete isolation between projects
+		// Create isolated context — puppeteer-extra wraps this correctly
+		// so StealthPlugin's onPageCreated fires for every page in this context
 		const context = await browser.createBrowserContext();
 		const page = await context.newPage();
 
@@ -242,14 +175,12 @@ class BrowserPool {
 		debug.log('preview', `🗑️ Destroying session: ${sessionId}`);
 
 		try {
-			// Close the page first
 			if (session.page && !session.page.isClosed()) {
 				await session.page.close().catch((err: Error) => {
 					debug.warn('preview', `Error closing page: ${err.message}`);
 				});
 			}
 
-			// Close the context (this clears all cookies, storage, cache)
 			await session.context.close().catch((err: Error) => {
 				debug.warn('preview', `Error closing context: ${err.message}`);
 			});
@@ -261,35 +192,13 @@ class BrowserPool {
 		debug.log('preview', `✅ Session destroyed (remaining: ${this.sessions.size})`);
 	}
 
-	/**
-	 * Execute a task in the cluster (for one-off operations)
-	 */
-	async execute<T>(
-		taskFunction: (opts: { page: Page; data: any }) => Promise<T>,
-		data?: any
-	): Promise<T> {
-		const cluster = await this.getCluster();
-		return cluster.execute(data, taskFunction);
-	}
-
-	/**
-	 * Queue a task in the cluster
-	 */
-	async queue(taskFunction: (opts: { page: Page; data: any }) => Promise<void>, data?: any): Promise<void> {
-		const cluster = await this.getCluster();
-		cluster.queue(data, taskFunction);
-	}
-
 	/**
 	 * Check if a session is valid
 	 */
 	isSessionValid(sessionId: string): boolean {
 		const session = this.sessions.get(sessionId);
 		if (!session) return false;
-
-		// Check if page is still open
 		if (session.page.isClosed()) return false;
-
 		return true;
 	}
 
@@ -298,7 +207,7 @@ class BrowserPool {
 	 */
 	getStats() {
 		return {
-			clusterActive: this.cluster !== null,
+			browserConnected: this.browser?.connected ?? false,
 			activeSessions: this.sessions.size,
 			maxConcurrency: this.config.maxConcurrency,
 			sessions: Array.from(this.sessions.entries()).map(([id, session]) => ({
@@ -310,34 +219,22 @@ class BrowserPool {
 		};
 	}
 
-	/**
-	 * Wait for all queued tasks to complete
-	 */
-	async idle(): Promise<void> {
-		if (this.cluster) {
-			await this.cluster.idle();
-		}
-	}
-
 	/**
 	 * Clean up all resources
 	 */
 	async cleanup(): Promise<void> {
 		debug.log('preview', '🧹 Cleaning up browser pool...');
 
-		// Destroy all sessions
 		const sessionIds = Array.from(this.sessions.keys());
 		await Promise.all(sessionIds.map((id) => this.destroySession(id)));
 
-		// Close the cluster
-		if (this.cluster) {
+		if (this.browser) {
 			try {
-				await this.cluster.idle();
-				await this.cluster.close();
+				await this.browser.close();
 			} catch (error) {
-				debug.warn('preview', `⚠️ Error closing cluster: ${error}`);
+				debug.warn('preview', `⚠️ Error closing browser: ${error}`);
 			}
-			this.cluster = null;
+			this.browser = null;
 		}
 
 		debug.log('preview', '✅ Browser pool cleaned up');

package/backend/lib/preview/browser/browser-preview-service.ts

@@ -219,8 +219,9 @@ export class BrowserPreviewService extends EventEmitter {
 		await this.navigationTracker.setupNavigationTracking(tab.id, tab.page, tab);
 
 		// Setup dialog bindings and handling
-		await this.dialogHandler.setupDialogBindings(tab.id, tab.page);
-		await this.dialogHandler.setupDialogHandling(tab.id, tab.page, tab);
+		// Temporarily disable dialog injection to test CloudFlare evasion
+		// await this.dialogHandler.setupDialogBindings(tab.id, tab.page);
+		// await this.dialogHandler.setupDialogHandling(tab.id, tab.page, tab);
 
 		return tab;
 	}