opencode-webfetch-plugin 0.1.1 → 0.1.3

package/src/BrowserManager.ts

@@ -1,19 +1,16 @@
 import type { ToolContext } from "@opencode-ai/plugin";
 
-import * as os from 'os';
-import * as path from 'path';
-import * as fs from 'fs';
 import { HumanInteractor } from './HumanInteractor.js';
 import { Extractor } from './Extractor.js';
-import type { BrowserContext, Page } from 'playwright';
+import { BrowserServer } from './BrowserServer.js';
+import type { Page } from 'playwright';
 
 type PlaywrightModule = typeof import('playwright');
 
 export class BrowserManager {
-  private context: BrowserContext | null = null;
-  private page: Page | null = null;
   private readonly playwright: PlaywrightModule;
   private readonly client: any;
+  private browserServer: BrowserServer | null = null;
 
   constructor(playwright: PlaywrightModule, client: any) {
     this.playwright = playwright;
@@ -21,125 +18,71 @@ export class BrowserManager {
   }
 
   /**
-   * Initializes the persistent context if not already done.
+   * Ensures the browser server is initialized.
    */
-  public async ensureContext(): Promise<void> {
-    let isConnected = false;
-    if (this.context && this.page) {
-      try {
-        // Simple check to see if the page is still open and connected
-        isConnected = !this.page.isClosed();
-      } catch (e) {
-        isConnected = false;
-      }
-    }
-
-    if (isConnected) {
-      return;
-    }
-
-    // Clean up just in case
-    await this.dispose();
-
-    const userDataDir = path.resolve(os.homedir(), '.cache/opencode/user-data');
-    if (!fs.existsSync(userDataDir)) {
-      fs.mkdirSync(userDataDir, { recursive: true });
+  private async ensureBrowserServer(): Promise<BrowserServer> {
+    if (!this.browserServer) {
+      this.browserServer = await BrowserServer.getInstance(this.playwright, this.client);
     }
-
-    // Launch a persistent context with extension support
-    const extensionPath = path.resolve(os.homedir(), '.cache/opencode/extensions');
-    const extensions: string[] = [];
-    if (fs.existsSync(extensionPath)) {
-      const dirs = fs.readdirSync(extensionPath).map(d => path.join(extensionPath, d));
-      extensions.push(...dirs.filter(d => fs.statSync(d).isDirectory()));
-    }
-
-    const launchOptions: Parameters<typeof this.playwright.chromium.launchPersistentContext>[1] = {
-      headless: false,
-      args: [
-        '--no-sandbox',
-        '--disable-dev-shm-usage',
-        '--disable-blink-features=AutomationControlled',
-        '--disable-features=VizDisplayCompositor',
-        '--window-size=1280,720',
-        ...(extensions.length > 0 ? [
-          `--disable-extensions-except=${extensions.join(',')}`,
-          `--load-extension=${extensions.join(',')}`
-        ] : []),
-      ],
-      viewport: { width: 1280, height: 720 },
-    };
-
-    this.context = await this.playwright.chromium.launchPersistentContext(userDataDir, launchOptions);
-
-    // Create a new page or use the default one created by launchPersistentContext
-    const pages = this.context.pages();
-    if (pages.length > 0) {
-      this.page = pages[0];
-    } else {
-      this.page = await this.context.newPage();
-    }
-
-    // Mask webdriver
-    await this.page.addInitScript(() => {
-      Object.defineProperty(navigator, 'webdriver', {
-        get: () => false,
-      });
-
-      const chrome = (window as any).chrome;
-      if (chrome && chrome.runtime && chrome.runtime.onConnect) {
-        delete chrome.runtime.onConnect;
-      }
-    });
+    return this.browserServer;
   }
 
   /**
    * Navigates to a URL and tries to extract the content.
    * Prompts the user via terminal if it encounters a captcha or login screen.
+   * For concurrent calls, creates new pages in the same context.
    */
   public async fetchWebpage(url: string, timeout: number, ctx: ToolContext): Promise<string> {
-    await this.ensureContext();
-    if (!this.page) throw new Error('Page not initialized');
+    const browserServer = await this.ensureBrowserServer();
+    const context = browserServer.getContext();
+    
+    if (!context) throw new Error('Browser context not initialized');
 
-    console.log(`\nNavigating to: ${url}`);
+    // Create a new page for each concurrent request
+    const page = await context.newPage();
+
+    this.client?.logger?.info(`Navigating to: ${url}`);
     
     // Add a listener to handle abortions
     const onAbort = () => {
-      console.log('Operation aborted by user or timeout.');
+      this.client?.logger?.info('Operation aborted by user or timeout.');
+      page.close().catch(() => undefined);
     };
     ctx.abort.addEventListener('abort', onAbort);
 
     try {
       // Go to the requested URL
-      await this.page.goto(url, { waitUntil: 'domcontentloaded', timeout }).catch((e) => {
-        console.warn(`Navigation might have timed out or failed partially: ${e.message}`);
+      await page.goto(url, { waitUntil: 'domcontentloaded', timeout }).catch((e) => {
+        this.client?.logger?.warn(`Navigation might have timed out or failed partially: ${e.message}`);
       });
 
       // too fast
-      await this.page.waitForTimeout(2000);
+      await page.waitForTimeout(2000);
 
       // Basic heuristic to check if human intervention is needed
-      const needsHelp = await this.detectBlockers(this.page);
+      const needsHelp = await this.detectBlockers(page);
       
       if (needsHelp.blocked) {
-        await HumanInteractor.askForHumanHelp(`The page appears to be blocked or requires login.\nReason: ${needsHelp.reason}\nURL: ${this.page.url()}`, ctx, this.client, () => this.detectBlockers(this.page));
+        await HumanInteractor.askForHumanHelp(`The page appears to be blocked or requires login.\nReason: ${needsHelp.reason}\nURL: ${page.url()}`, ctx, this.client, () => this.detectBlockers(page));
       } else {
         // Wait a little bit for dynamic content if not blocked
-        await this.page.waitForTimeout(2000);
+        await page.waitForTimeout(2000);
       }
 
       // Allow one more check in case the user didn't fully resolve it, or if it redirected
-      const needsHelpAgain = await this.detectBlockers(this.page);
+      const needsHelpAgain = await this.detectBlockers(page);
       if (needsHelpAgain.blocked) {
-        await HumanInteractor.askForHumanHelp(`Still detected a blocker.\nReason: ${needsHelpAgain.reason}\nPlease complete the action and try again.`, ctx, this.client, () => this.detectBlockers(this.page));
+        await HumanInteractor.askForHumanHelp(`Still detected a blocker.\nReason: ${needsHelpAgain.reason}\nPlease complete the action and try again.`, ctx, this.client, () => this.detectBlockers(page));
       }
 
       // Extract content as Markdown
-      console.log('Extracting page content...');
-      const markdown = await Extractor.extractMarkdown(this.page, this.page.url());
+      this.client?.logger?.info('Extracting page content...');
+      const markdown = await Extractor.extractMarkdown(page, page.url());
       return markdown;
     } finally {
       ctx.abort.removeEventListener('abort', onAbort);
+      // Close the page after extraction to free resources
+      await page.close().catch(() => undefined);
     }
   }
 
@@ -199,25 +142,19 @@ export class BrowserManager {
       }
 
     } catch (e) {
-      console.error('Error detecting blockers:', e);
+      this.client?.logger?.error('Error detecting blockers:', e);
     }
 
     return { blocked: false };
   }
 
   /**
-   * Close the browser context safely.
+   * Close the browser if this process owns it.
    */
   public async dispose(): Promise<void> {
-    try {
-      if (this.context) {
-        await this.context.close().catch(() => {});
-      }
-    } catch (e) {
-      // Ignore errors on close
-    } finally {
-      this.context = null;
-      this.page = null;
+    if (this.browserServer) {
+      await this.browserServer.dispose();
     }
+    this.browserServer = null;
   }
 }

package/src/BrowserServer.ts

@@ -0,0 +1,226 @@
+import * as os from 'os';
+import * as path from 'path';
+import * as fs from 'fs';
+import * as http from 'http';
+import { spawn } from 'child_process';
+import { fileURLToPath } from 'url';
+import type { BrowserContext } from 'playwright';
+
+type PlaywrightModule = typeof import('playwright');
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const PROJECT_DIR = path.resolve(__dirname, '..');
+
+const CDP_PORT = parseInt(process.env.WEBFETCH_CDP_PORT || '9222', 10);
+const USER_DATA_DIR = path.resolve(os.homedir(), `.cache/opencode/user-data-${CDP_PORT}`);
+const LAUNCH_SCRIPT = path.resolve(os.homedir(), '.cache/opencode/launch-browser.ts');
+
+export class BrowserServer {
+  private static instance: BrowserServer | null = null;
+  private static initPromise: Promise<BrowserServer> | null = null;
+  private context: BrowserContext | null = null;
+  private readonly playwright: PlaywrightModule;
+  private readonly client: any;
+
+  private constructor(playwright: PlaywrightModule, client: any) {
+    this.playwright = playwright;
+    this.client = client;
+  }
+
+  static async getInstance(playwright: PlaywrightModule, client: any): Promise<BrowserServer> {
+    if (BrowserServer.initPromise) {
+      return BrowserServer.initPromise;
+    }
+
+    if (!BrowserServer.instance) {
+      BrowserServer.initPromise = (async () => {
+        BrowserServer.instance = new BrowserServer(playwright, client);
+        await BrowserServer.instance.initialize();
+        return BrowserServer.instance;
+      })();
+      
+      try {
+        const instance = await BrowserServer.initPromise;
+        return instance;
+      } finally {
+        BrowserServer.initPromise = null;
+      }
+    }
+    return BrowserServer.instance;
+  }
+
+  private async initialize(): Promise<void> {
+    if (!fs.existsSync(USER_DATA_DIR)) {
+      fs.mkdirSync(USER_DATA_DIR, { recursive: true });
+    }
+
+    this.ensureLaunchScript();
+
+    for (let attempt = 1; attempt <= 3; attempt++) {
+      if (await this.tryConnect()) {
+        return;
+      }
+
+      this.client?.logger?.info(`Attempt ${attempt}/3: Starting independent browser process...`);
+      
+      await this.spawnIndependentBrowser();
+      
+      const waitTime = Math.floor(Math.random() * 3000) + 2000;
+      this.client?.logger?.info(`Waiting ${waitTime}ms for browser to start...`);
+      await this.sleep(waitTime);
+
+      if (await this.tryConnect()) {
+        return;
+      }
+    }
+
+    throw new Error('Failed to start or connect to browser after 3 attempts');
+  }
+
+  private ensureLaunchScript(): void {
+    const scriptContent = `import { chromium } from 'playwright';
+import * as fs from 'fs';
+import * as path from 'path';
+import * as os from 'os';
+
+const CDP_PORT = ${CDP_PORT};
+const USER_DATA_DIR = path.resolve(os.homedir(), '.cache/opencode/user-data-${CDP_PORT}');
+
+async function main() {
+  const singletonLock = path.join(USER_DATA_DIR, 'SingletonLock');
+  if (fs.existsSync(singletonLock)) {
+    fs.unlinkSync(singletonLock);
+  }
+
+  const extensionPath = path.resolve(os.homedir(), '.cache/opencode/extensions');
+  const extensions: string[] = [];
+  if (fs.existsSync(extensionPath)) {
+    const dirs = fs.readdirSync(extensionPath).map(d => path.join(extensionPath, d));
+    extensions.push(...dirs.filter(d => fs.statSync(d).isDirectory()));
+  }
+
+  const context = await chromium.launchPersistentContext(USER_DATA_DIR, {
+    headless: false,
+    ignoreDefaultArgs: ['--remote-debugging-pipe'],
+    args: [
+      '--no-sandbox',
+      '--disable-dev-shm-usage',
+      '--remote-allow-origins="*"',
+      // '--disable-blink-features=AutomationControlled',
+      // '--disable-features=VizDisplayCompositor',
+      // '--window-size=1280,720',
+      '--remote-debugging-port=' + CDP_PORT,
+      // ...(extensions.length > 0 ? [
+      //   '--disable-extensions-except=' + extensions.join(','),
+      //   '--load-extension=' + extensions.join(',')
+      // ] : []),
+    ],
+    viewport: { width: 1280, height: 720 },
+  });
+
+  context.on('page', async (page) => {
+    await page.addInitScript(() => {
+      Object.defineProperty(navigator, 'webdriver', { get: () => false });
+      const chrome = (window as any).chrome;
+      if (chrome && chrome.runtime && chrome.runtime.onConnect) {
+        delete chrome.runtime.onConnect;
+      }
+    });
+  });
+
+  console.log('Browser launched on port ' + CDP_PORT);
+}
+
+main().catch(console.error);
+`;
+
+    const scriptDir = path.dirname(LAUNCH_SCRIPT);
+    if (!fs.existsSync(scriptDir)) {
+      fs.mkdirSync(scriptDir, { recursive: true });
+    }
+    fs.writeFileSync(LAUNCH_SCRIPT, scriptContent);
+  }
+
+  private async spawnIndependentBrowser(): Promise<void> {
+    return new Promise((resolve) => {
+      const child = spawn('npx', ['tsx', LAUNCH_SCRIPT], {
+        cwd: PROJECT_DIR,
+        detached: true,
+        stdio: 'ignore',
+        windowsHide: true,
+        shell: true,
+      });
+      
+      child.unref();
+      
+      child.on('error', (e) => {
+        this.client?.logger?.warn(`Failed to spawn browser: ${e.message}`);
+      });
+
+      resolve();
+    });
+  }
+
+  private async tryConnect(): Promise<boolean> {
+    if (!(await this.isPortInUse(CDP_PORT))) {
+      return false;
+    }
+
+    try {
+      await this.connectToBrowser(`http://localhost:${CDP_PORT}`);
+      return true;
+    } catch (e) {
+      this.client?.logger?.warn(`Failed to connect: ${e}`);
+      return false;
+    }
+  }
+
+  private async isPortInUse(port: number): Promise<boolean> {
+    return new Promise((resolve) => {
+      const req = http.request({
+        hostname: 'localhost',
+        port: port,
+        path: '/json/version',
+        method: 'GET',
+        timeout: 1000
+      }, (res) => {
+        resolve(res.statusCode === 200);
+      });
+      
+      req.on('error', () => resolve(false));
+      req.on('timeout', () => {
+        req.destroy();
+        resolve(false);
+      });
+      
+      req.end();
+    });
+  }
+
+  private async connectToBrowser(wsEndpoint: string): Promise<void> {
+    this.client?.logger?.info(`Connecting to browser at ${wsEndpoint}...`);
+
+    const browser = await this.playwright.chromium.connectOverCDP(wsEndpoint);
+    
+    const contexts = browser.contexts();
+    if (contexts.length > 0) {
+      this.context = contexts[0];
+      this.client?.logger?.info('Connected to existing browser');
+    } else {
+      throw new Error('No context available in connected browser');
+    }
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise(resolve => setTimeout(resolve, ms));
+  }
+
+  getContext(): BrowserContext | null {
+    return this.context;
+  }
+
+  async dispose(): Promise<void> {
+    this.context = null;
+  }
+}

package/src/BrowserWorkerManager.ts

@@ -0,0 +1,184 @@
+import { fork, type ChildProcess } from 'child_process';
+import { fileURLToPath } from 'url';
+import * as path from 'path';
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+const MAX_TIMEOUT = 120_000;
+
+interface WorkerRequest {
+  id: string;
+  type: 'fetch' | 'dispose';
+  url?: string;
+  timeout?: number;
+}
+
+interface WorkerResponse {
+  id: string;
+  success: boolean;
+  data?: string;
+  error?: string;
+}
+
+export class BrowserWorkerManager {
+  private worker: ChildProcess | null = null;
+  private requestId = 0;
+  private pendingRequests = new Map<string, { resolve: (value: any) => void; reject: (error: any) => void }>();
+  private client: any;
+  private initPromise: Promise<void> | null = null;
+
+  constructor(client: any) {
+    this.client = client;
+  }
+
+  private async ensureWorker(): Promise<void> {
+    if (this.initPromise) {
+      return this.initPromise;
+    }
+
+    if (this.worker) {
+      return;
+    }
+
+    this.initPromise = this.startWorker();
+    try {
+      await this.initPromise;
+    } finally {
+      this.initPromise = null;
+    }
+  }
+
+  private async startWorker(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      const workerPath = path.resolve(__dirname, 'browser-worker.ts');
+      
+      this.client?.logger?.info(`Starting browser worker: ${workerPath}`);
+      
+      // Use tsx directly as the executable (it's in node_modules/.bin)
+      const tsxBin = path.resolve(__dirname, '../node_modules/.bin/tsx');
+      
+      // Use tsx to execute TypeScript directly
+      this.worker = fork(workerPath, [], {
+        stdio: ['ignore', 'pipe', 'pipe', 'ipc'],
+        execPath: tsxBin,
+        execArgv: [],
+      });
+
+      // Forward stdout to logger for debugging
+      if (this.worker.stdout) {
+        this.worker.stdout.on('data', (data) => {
+          this.client?.logger?.info(`[Worker stdout] ${data.toString().trim()}`);
+        });
+      }
+
+      // Forward stderr to logger
+      if (this.worker.stderr) {
+        this.worker.stderr.on('data', (data) => {
+          this.client?.logger?.error(`[Worker stderr] ${data.toString().trim()}`);
+        });
+      }
+
+      // Forward stderr to logger
+      if (this.worker.stderr) {
+        this.worker.stderr.on('data', (data) => {
+          this.client?.logger?.error(`[Worker stderr] ${data.toString().trim()}`);
+        });
+      }
+
+      const onReady = (message: any) => {
+        if (message && message.type === 'ready') {
+          this.worker?.off('message', onReady);
+          resolve();
+        }
+      };
+
+      this.worker.on('message', onReady);
+
+      this.worker.on('message', (message: any) => {
+        if (message.type === 'toast') {
+          // Forward toast to client
+          this.client?.tui?.showToast(message.data);
+        } else if (message.id) {
+          // Handle response
+          const pending = this.pendingRequests.get(message.id);
+          if (pending) {
+            this.pendingRequests.delete(message.id);
+            if (message.success) {
+              pending.resolve(message.data);
+            } else {
+              pending.reject(new Error(message.error || 'Unknown worker error'));
+            }
+          }
+        }
+      });
+
+      this.worker.on('error', (error) => {
+        this.client?.logger?.error('Worker process error:', error);
+        reject(error);
+      });
+
+      this.worker.on('exit', (code) => {
+        this.client?.logger?.warn(`Worker process exited with code ${code}`);
+        this.worker = null;
+        // Reject all pending requests
+        for (const [id, pending] of this.pendingRequests.entries()) {
+          pending.reject(new Error('Worker process exited'));
+        }
+        this.pendingRequests.clear();
+      });
+
+      // Timeout for worker startup
+      setTimeout(() => {
+        if (this.initPromise) {
+          reject(new Error('Worker startup timeout'));
+        }
+      }, 10000);
+    });
+  }
+
+  async sendRequest(request: Omit<WorkerRequest, 'id'>): Promise<any> {
+    await this.ensureWorker();
+
+    if (!this.worker) {
+      throw new Error('Worker process not available');
+    }
+
+    const id = `req-${++this.requestId}`;
+    const fullRequest: WorkerRequest = { id, ...request };
+
+    return new Promise((resolve, reject) => {
+      this.pendingRequests.set(id, { resolve, reject });
+      
+      this.worker!.send(fullRequest, (error) => {
+        if (error) {
+          this.pendingRequests.delete(id);
+          reject(error);
+        }
+      });
+
+      // Request timeout
+      setTimeout(() => {
+        if (this.pendingRequests.has(id)) {
+          this.pendingRequests.delete(id);
+          reject(new Error('Request timeout'));
+        }
+      }, MAX_TIMEOUT + 5000);
+    });
+  }
+
+  async dispose(): Promise<void> {
+    if (this.worker) {
+      try {
+        await this.sendRequest({ type: 'dispose' });
+      } catch (e) {
+        this.client?.logger?.warn('Error disposing worker:', e);
+      }
+      
+      this.worker.kill();
+      this.worker = null;
+    }
+    
+    this.pendingRequests.clear();
+  }
+}

package/src/HumanInteractor.ts

@@ -45,7 +45,7 @@ export class HumanInteractor {
         }
       }
     } catch (e) {
-      console.error(e)
+      client?.logger?.error('Error in human interaction:', e);
     }
   }
 }