opencode-webfetch-plugin 0.1.1 → 0.1.3

package/src/browser-worker.ts

@@ -0,0 +1,136 @@
+#!/usr/bin/env node
+/**
+ * Browser Worker Process
+ * Runs in Node.js to avoid Bun's connectOverCDP issues
+ * Communicates with parent process via IPC
+ */
+
+import { BrowserManager } from './BrowserManager.js';
+
+type PlaywrightModule = typeof import('playwright');
+
+interface WorkerRequest {
+  id: string;
+  type: 'fetch' | 'dispose';
+  url?: string;
+  timeout?: number;
+}
+
+interface WorkerResponse {
+  id: string;
+  success: boolean;
+  data?: string;
+  error?: string;
+}
+
+let globalManager: BrowserManager | null = null;
+let playwright: PlaywrightModule | null = null;
+
+// Mock client for worker process
+const workerClient = {
+  logger: {
+    info: (msg: string) => console.error(`[Worker Info] ${msg}`),
+    warn: (msg: string) => console.error(`[Worker Warn] ${msg}`),
+    error: (msg: string, err?: any) => console.error(`[Worker Error] ${msg}`, err),
+  },
+  tui: {
+    showToast: (opts: any) => {
+      // Send toast request to parent
+      sendToParent({ type: 'toast', data: opts });
+    }
+  }
+};
+
+// Mock ToolContext for worker
+function createMockContext(): any {
+  const abortController = new AbortController();
+  return {
+    abort: abortController.signal,
+    metadata: () => {},
+  };
+}
+
+async function loadPlaywright(): Promise<PlaywrightModule> {
+  try {
+    return await import('playwright');
+  } catch (error) {
+    throw new Error(
+      'webfetch plugin requires Playwright. Install it with: npm install playwright && npx playwright install chromium',
+      { cause: error },
+    );
+  }
+}
+
+async function handleRequest(req: WorkerRequest): Promise<WorkerResponse> {
+  try {
+    if (req.type === 'fetch') {
+      if (!req.url) {
+        throw new Error('URL is required for fetch request');
+      }
+
+      if (!globalManager) {
+        playwright = await loadPlaywright();
+        globalManager = new BrowserManager(playwright, workerClient);
+      }
+
+      const timeout = req.timeout || 30000;
+      const ctx = createMockContext();
+      const markdown = await globalManager.fetchWebpage(req.url, timeout, ctx);
+
+      return {
+        id: req.id,
+        success: true,
+        data: markdown,
+      };
+    } else if (req.type === 'dispose') {
+      if (globalManager) {
+        await globalManager.dispose();
+        globalManager = null;
+      }
+      return {
+        id: req.id,
+        success: true,
+      };
+    } else {
+      throw new Error(`Unknown request type: ${(req as any).type}`);
+    }
+  } catch (error: any) {
+    return {
+      id: req.id,
+      success: false,
+      error: error.message || String(error),
+    };
+  }
+}
+
+function sendToParent(message: any): void {
+  if (process.send) {
+    process.send(message);
+  }
+}
+
+// Listen for messages from parent process
+process.on('message', async (message: any) => {
+  if (message && typeof message === 'object' && message.id) {
+    const response = await handleRequest(message as WorkerRequest);
+    sendToParent(response);
+  }
+});
+
+// Handle graceful shutdown
+process.on('SIGTERM', async () => {
+  if (globalManager) {
+    await globalManager.dispose().catch(() => {});
+  }
+  process.exit(0);
+});
+
+process.on('SIGINT', async () => {
+  if (globalManager) {
+    await globalManager.dispose().catch(() => {});
+  }
+  process.exit(0);
+});
+
+// Signal ready
+sendToParent({ type: 'ready' });

package/src/index.ts

@@ -1,32 +1,23 @@
 import { type Plugin, tool } from "@opencode-ai/plugin";
-import { BrowserManager } from "./BrowserManager.js";
-
-type PlaywrightModule = typeof import("playwright");
+import { BrowserWorkerManager } from './BrowserWorkerManager.js';
 
 const DEFAULT_TIMEOUT = 30_000;
 const MAX_TIMEOUT = 120_000;
 
-let globalManager: BrowserManager | null = null;
+let globalWorkerManager: BrowserWorkerManager | null = null;
 
 export const WebfetchPlugin: Plugin = async ({ client }) => {
   const WebfetchTool = tool({
     description: "Fetch a webpage's main content in markdown.",
     args: {
       url: tool.schema.string().describe("The URL to fetch."),
-      // timeout: tool.schema
-      //   .number()
-      //   .min(5)
-      //   .max(120)
-      //   .optional()
-      //   .describe("Timeout in seconds (default: 30, max: 120)"),
     },
     async execute(params: any, ctx: any) {
-      if (!globalManager) {
-        const playwright = await loadPlaywright();
-        globalManager = new BrowserManager(playwright, client);
+      if (!globalWorkerManager) {
+        globalWorkerManager = new BrowserWorkerManager(client);
       }
       
-      const manager = globalManager;
+      const manager = globalWorkerManager;
       const timeoutMs = Math.min((params.timeout ?? DEFAULT_TIMEOUT / 1000) * 1000, MAX_TIMEOUT);
 
       const abortHandler = () => {
@@ -41,7 +32,11 @@ export const WebfetchPlugin: Plugin = async ({ client }) => {
           targetUrl = `https://www.google.com/search?q=${encodeURIComponent(targetUrl)}`;
         }
 
-        const markdownResult = await manager.fetchWebpage(targetUrl, timeoutMs, ctx);
+        const markdownResult = await manager.sendRequest({
+          type: 'fetch',
+          url: targetUrl,
+          timeout: timeoutMs,
+        });
 
         ctx.metadata({
           title: `Webfetch: ${targetUrl}`,
@@ -67,20 +62,4 @@ export const WebfetchPlugin: Plugin = async ({ client }) => {
   };
 };
 
-async function loadPlaywright(): Promise<PlaywrightModule> {
-  try {
-    return await import("playwright");
-  } catch (error) {
-    try {
-      // @ts-ignore
-      return await import("/tmp/node_modules/playwright");
-    } catch {
-      throw new Error(
-        "webfetch plugin requires Playwright. Install it with: bun install playwright && bunx playwright install chromium",
-        { cause: error },
-      );
-    }
-  }
-}
-
 export default WebfetchPlugin;

package/test.md

@@ -0,0 +1 @@
+webfetch https://z.weixin.qq.com/ https://bailian.console.aliyun.com/#/home 两个页面，并分别做总结

package/tests/BrowserWorkerManager.test.ts

@@ -0,0 +1,49 @@
+import { describe, it, expect, afterEach } from 'vitest';
+import { BrowserWorkerManager } from '../src/BrowserWorkerManager.js';
+
+describe('BrowserWorkerManager', () => {
+  let manager: BrowserWorkerManager | null = null;
+
+  afterEach(async () => {
+    if (manager) {
+      await manager.dispose().catch(() => {});
+      manager = null;
+    }
+  });
+
+  it('should handle worker lifecycle and concurrent requests', async () => {
+    const mockClient = {
+      logger: {
+        info: () => {},
+        warn: () => {},
+        error: () => {},
+      },
+      tui: {
+        showToast: () => {},
+      },
+    };
+
+    manager = new BrowserWorkerManager(mockClient);
+
+    // Send two concurrent fetch requests
+    const [result1, result2] = await Promise.all([
+      manager.sendRequest({
+        type: 'fetch',
+        url: 'https://example.com',
+        timeout: 30000,
+      }),
+      manager.sendRequest({
+        type: 'fetch',
+        url: 'https://httpbin.org/html',
+        timeout: 30000,
+      }),
+    ]);
+
+    // Verify both requests returned markdown content
+    expect(result1).toContain('Example Domain');
+    expect(result2).toContain('httpbin');
+    
+    // Verify worker can be disposed
+    await manager.dispose();
+  }, 60000);
+});

package/tests/browser.test.ts

@@ -0,0 +1,84 @@
+import { describe, it, expect, beforeAll, afterAll } from 'vitest';
+import { BrowserServer } from '../src/BrowserServer.js';
+import { BrowserManager } from '../src/BrowserManager.js';
+import { fork } from 'child_process';
+import * as path from 'path';
+
+async function getPlaywright() {
+  return await import('playwright');
+}
+
+function resetBrowserServerSingleton() {
+  (BrowserServer as any).instance = null;
+  (BrowserServer as any).initPromise = null;
+}
+
+describe('Browser Tests', () => {
+  beforeAll(() => {
+    resetBrowserServerSingleton();
+  });
+
+  afterAll(async () => {
+    resetBrowserServerSingleton();
+  });
+
+  it('should fetch two pages concurrently in the same browser instance', async () => {
+    const playwright = await getPlaywright();
+    const client = { logger: console };
+    
+    const manager = new BrowserManager(playwright, client);
+    
+    const [result1, result2] = await Promise.all([
+      manager.fetchWebpage('https://example.com', 30000, { abort: new AbortController().signal } as any),
+      manager.fetchWebpage('https://httpbin.org/html', 30000, { abort: new AbortController().signal } as any),
+    ]);
+    
+    expect(result1).toContain('Example Domain');
+    expect(result2).toContain('httpbin.org');
+  });
+
+  it('should share browser between two independent processes', async () => {
+    const workerScript = path.join(__dirname, 'helpers', 'cookie-worker.ts');
+    
+    const runWorker = (action: string): Promise<{ content: string }> => {
+      return new Promise((resolve, reject) => {
+        const child = fork(workerScript, [action], {
+          execArgv: ['--import', 'tsx'],
+          stdio: ['pipe', 'pipe', 'pipe', 'ipc']
+        });
+        
+        let stdout = '';
+        child.stdout?.on('data', (data) => { stdout += data; });
+        child.stderr?.on('data', (data) => { console.log(`[worker stderr] ${data}`); });
+        
+        child.on('message', (result: { content: string }) => {
+          resolve(result);
+          child.disconnect();
+        });
+        
+        child.on('error', reject);
+        child.on('exit', (code) => {
+          if (code !== 0) {
+            reject(new Error(`Worker exited with code ${code}: ${stdout}`));
+          }
+        });
+      });
+    };
+    
+    const res = await runWorker('https://example.com');
+    expect(res.content).toContain('Example Domain');
+
+    const [result1, result2] = await Promise.all([runWorker('https://example.com'), runWorker('https://httpbin.org/html')]); 
+
+    // console.log(result1)
+    // console.log(result2)
+    expect(result1.content).toContain('Example Domain');
+    expect(result2.content).toContain('httpbin.org');
+    // expect(result1.port).toBe(9222);
+    // expect(result1.cookies).toHaveProperty('test_cookie', 'test_value');
+    
+    // // const result2 = await runWorker('get');
+    // expect(result2.port).toBe(9222);
+    // expect(result2.cookies).toHaveProperty('test_cookie', 'test_value');
+  });
+});

package/tests/helpers/cookie-worker.ts

@@ -0,0 +1,51 @@
+import { BrowserManager } from '../../src/BrowserManager.js'
+
+async function main () {
+  const action = process.argv[2]
+
+  const playwright = await import('playwright')
+  const client = { logger: console }
+
+  const manager = new BrowserManager(playwright, client)
+  // const server = await (manager as any).ensureBrowserServer();
+  // const context = server.getContext();
+
+  // if (!context) {
+  //   process.send?.({ error: 'No context' });
+  //   process.exit(1);
+  // }
+
+  // const page = await context.newPage();
+
+  // let cookies: Record<string, string> = {};
+
+  let content = await manager.fetchWebpage(action, 3000, {
+    abort: new AbortController().signal
+  } as any)
+
+  // await page.goto('https://httpbin.org/cookies', { waitUntil: 'domcontentloaded' });
+  // await page.waitForTimeout(1000);
+
+  // const content = await page.content();
+  // const match = content.match(/{\s*"cookies":\s*({[^}]*})\s*}/)
+  // if (match) {
+  //   try {
+  //     cookies = JSON.parse(match[1])
+  //   } catch (e) {
+  //     // ignore
+  //   }
+  // }
+
+  // await page.close()
+
+  process.send?.({
+    content
+  })
+
+  process.exit(0)
+}
+
+main().catch(e => {
+  console.error(e)
+  process.exit(1)
+})

package/vitest.config.ts

@@ -0,0 +1,17 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    testTimeout: 60000,
+    hookTimeout: 60000,
+    teardownTimeout: 5000,
+    isolate: false,
+    maxConcurrency: 1,
+    fileParallelism: false,
+    sequence: {
+      concurrent: false,
+    },
+  },
+});

package/tsconfig.json

@@ -1,17 +0,0 @@
-{
-  "$schema": "https://json.schemastore.org/tsconfig",
-  "compilerOptions": {
-    "target": "ES2022",
-    "module": "ES2022",
-    "moduleResolution": "Bundler",
-    "lib": ["ES2022", "DOM"],
-    "strict": true,
-    "declaration": true,
-    "declarationMap": true,
-    "sourceMap": true,
-    "outDir": "dist",
-    "esModuleInterop": true,
-    "skipLibCheck": true
-  },
-  "include": ["src/**/*"]
-}