@jackwener/opencli 0.5.1 → 0.5.2

package/src/browser.test.ts

@@ -49,8 +49,27 @@ describe('browser helpers', () => {
     expect(__test__.appendLimited('12345', '67890', 8)).toBe('34567890');
   });
 
+  it('builds Playwright MCP args with kebab-case executable path', () => {
+    expect(__test__.buildMcpArgs({
+      mcpPath: '/tmp/cli.js',
+      executablePath: '/mnt/c/Program Files/Google/Chrome/Application/chrome.exe',
+    })).toEqual([
+      '/tmp/cli.js',
+      '--extension',
+      '--executable-path',
+      '/mnt/c/Program Files/Google/Chrome/Application/chrome.exe',
+    ]);
+
+    expect(__test__.buildMcpArgs({
+      mcpPath: '/tmp/cli.js',
+    })).toEqual([
+      '/tmp/cli.js',
+      '--extension',
+    ]);
+  });
+
   it('times out slow promises', async () => {
-    await expect(__test__.withTimeout(new Promise(() => {}), 10, 'timeout')).rejects.toThrow('timeout');
+    await expect(__test__.withTimeoutMs(new Promise(() => {}), 10, 'timeout')).rejects.toThrow('timeout');
   });
 });
 

package/src/browser.ts

@@ -10,10 +10,10 @@ import * as fs from 'node:fs';
 import * as os from 'node:os';
 import * as path from 'node:path';
 import { formatSnapshot } from './snapshotFormatter.js';
-
-// Read version from package.json (single source of truth)
-const __browser_dirname = path.dirname(fileURLToPath(import.meta.url));
-const PKG_VERSION = (() => { try { return JSON.parse(fs.readFileSync(path.resolve(__browser_dirname, '..', 'package.json'), 'utf-8')).version; } catch { return '0.0.0'; } })();
+import { PKG_VERSION } from './version.js';
+import { normalizeEvaluateSource } from './pipeline/template.js';
+import { generateInterceptorJs, generateReadInterceptedJs } from './interceptor.js';
+import { withTimeoutMs } from './runtime.js';
 
 const CONNECT_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_CONNECT_TIMEOUT ?? '30', 10);
 const STDERR_BUFFER_LIMIT = 16 * 1024;
@@ -158,23 +158,10 @@ export class Page implements IPage {
 
   async evaluate(js: string): Promise<any> {
     // Normalize IIFE format to function format expected by MCP browser_evaluate
-    const normalized = this.normalizeEval(js);
+    const normalized = normalizeEvaluateSource(js);
     return this.call('tools/call', { name: 'browser_evaluate', arguments: { function: normalized } });
   }
 
-  private normalizeEval(source: string): string {
-    const s = source.trim();
-    if (!s) return '() => undefined';
-    // IIFE: (async () => {...})()  →  wrap as () => (...)
-    if (s.startsWith('(') && s.endsWith(')()')) return `() => (${s})`;
-    // Already a function/arrow
-    if (/^(async\s+)?\([^)]*\)\s*=>/.test(s)) return s;
-    if (/^(async\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=>/.test(s)) return s;
-    if (s.startsWith('function ') || s.startsWith('async function ')) return s;
-    // Raw expression → wrap
-    return `() => (${s})`;
-  }
-
   async snapshot(opts: { interactive?: boolean; compact?: boolean; maxDepth?: number; raw?: boolean } = {}): Promise<any> {
     const raw = await this.call('tools/call', { name: 'browser_snapshot', arguments: {} });
     if (opts.raw) return raw;
@@ -263,57 +250,15 @@ export class Page implements IPage {
   }
 
   async installInterceptor(pattern: string): Promise<void> {
-    const js = `
-      () => {
-        window.__opencli_xhr = window.__opencli_xhr || [];
-        window.__opencli_patterns = window.__opencli_patterns || [];
-        if (!window.__opencli_patterns.includes('${pattern}')) {
-          window.__opencli_patterns.push('${pattern}');
-        }
-        
-        if (!window.__patched_xhr) {
-          const checkMatch = (url) => window.__opencli_patterns.some(p => url.includes(p));
-
-          const XHR = XMLHttpRequest.prototype;
-          const open = XHR.open;
-          const send = XHR.send;
-          XHR.open = function(method, url) {
-            this._url = url;
-            return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
-          };
-          XHR.send = function() {
-            this.addEventListener('load', function() {
-              if (checkMatch(this._url)) {
-                try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
-              }
-            });
-            return send.apply(this, arguments);
-          };
-
-          const origFetch = window.fetch;
-          window.fetch = async function(...args) {
-            let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
-            const res = await origFetch.apply(this, args);
-            setTimeout(async () => {
-              try {
-                if (checkMatch(u)) {
-                  const clone = res.clone();
-                  const j = await clone.json();
-                  window.__opencli_xhr.push({url: u, data: j});
-                }
-              } catch(e) {}
-            }, 0);
-            return res;
-          };
-          window.__patched_xhr = true;
-        }
-      }
-    `;
-    await this.evaluate(js);
+    await this.evaluate(generateInterceptorJs(JSON.stringify(pattern), {
+      arrayName: '__opencli_xhr',
+      patchGuard: '__opencli_interceptor_patched',
+    }));
   }
 
   async getInterceptedRequests(): Promise<any[]> {
-    return (await this.evaluate('() => window.__opencli_xhr')) || [];
+    const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr'));
+    return result || [];
   }
 }
 
@@ -442,13 +387,13 @@ export class PlaywrightMCP {
         }));
       }, timeout * 1000);
 
-      const mcpArgs: string[] = [mcpPath, '--extension'];
+      const mcpArgs = buildMcpArgs({
+        mcpPath,
+        executablePath: process.env.OPENCLI_BROWSER_EXECUTABLE_PATH,
+      });
       if (process.env.OPENCLI_VERBOSE) {
         console.error(`[opencli] Extension token: ${extensionToken ? `configured (fingerprint ${tokenFingerprint})` : 'missing'}`);
       }
-      if (process.env.OPENCLI_BROWSER_EXECUTABLE_PATH) {
-        mcpArgs.push('--executablePath', process.env.OPENCLI_BROWSER_EXECUTABLE_PATH);
-      }
       debugLog(`Spawning node ${mcpArgs.join(' ')}`);
 
       this._proc = spawn('node', mcpArgs, {
@@ -530,7 +475,7 @@ export class PlaywrightMCP {
 
         // Use tabs as a readiness probe and for tab cleanup bookkeeping.
         debugLog('Fetching initial tabs count...');
-        withTimeout(page.tabs(), INITIAL_TABS_TIMEOUT_MS, 'Timed out fetching initial tabs').then((tabs: any) => {
+        withTimeoutMs(page.tabs(), INITIAL_TABS_TIMEOUT_MS, 'Timed out fetching initial tabs').then((tabs: any) => {
           debugLog(`Tabs response: ${typeof tabs === 'string' ? tabs : JSON.stringify(tabs)}`);
           this._initialTabIdentities = extractTabIdentities(tabs);
           settleSuccess(page);
@@ -555,7 +500,7 @@ export class PlaywrightMCP {
         // Extension mode opens bridge/session tabs that we can clean up best-effort.
         if (this._page && this._proc && !this._proc.killed) {
           try {
-            const tabs = await withTimeout(this._page.tabs(), TAB_CLEANUP_TIMEOUT_MS, 'Timed out fetching tabs during cleanup');
+            const tabs = await withTimeoutMs(this._page.tabs(), TAB_CLEANUP_TIMEOUT_MS, 'Timed out fetching tabs during cleanup');
             const tabEntries = extractTabEntries(tabs);
             const tabsToClose = diffTabIndexes(this._initialTabIdentities, tabEntries);
             for (const index of tabsToClose) {
@@ -664,20 +609,12 @@ function appendLimited(current: string, chunk: string, limit: number): string {
   return next.slice(-limit);
 }
 
-function withTimeout<T>(promise: Promise<T>, timeoutMs: number, message: string): Promise<T> {
-  return new Promise<T>((resolve, reject) => {
-    const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
-    promise.then(
-      (value) => {
-        clearTimeout(timer);
-        resolve(value);
-      },
-      (error) => {
-        clearTimeout(timer);
-        reject(error);
-      },
-    );
-  });
+function buildMcpArgs(input: { mcpPath: string; executablePath?: string | null }): string[] {
+  const args = [input.mcpPath, '--extension'];
+  if (input.executablePath) {
+    args.push('--executable-path', input.executablePath);
+  }
+  return args;
 }
 
 export const __test__ = {
@@ -685,7 +622,8 @@ export const __test__ = {
   extractTabEntries,
   diffTabIndexes,
   appendLimited,
-  withTimeout,
+  buildMcpArgs,
+  withTimeoutMs,
 };
 
 function findMcpServerPath(): string | null {

package/src/cascade.ts

@@ -37,6 +37,49 @@ interface CascadeResult {
   confidence: number;
 }
 
+/**
+ * Build the JavaScript source for a fetch probe.
+ * Shared logic for PUBLIC, COOKIE, and HEADER strategies.
+ */
+function buildFetchProbeJs(url: string, opts: {
+  credentials?: boolean;
+  extractCsrf?: boolean;
+}): string {
+  const credentialsLine = opts.credentials ? `credentials: 'include',` : '';
+  const headerSetup = opts.extractCsrf
+    ? `
+      const cookies = document.cookie.split(';').map(c => c.trim());
+      const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('=');
+      const headers = {};
+      if (csrf) { headers['X-Csrf-Token'] = csrf; headers['X-XSRF-Token'] = csrf; }
+    `
+    : 'const headers = {};';
+
+  return `
+    async () => {
+      try {
+        ${headerSetup}
+        const resp = await fetch(${JSON.stringify(url)}, {
+          ${credentialsLine}
+          headers
+        });
+        const status = resp.status;
+        if (!resp.ok) return { status, ok: false };
+        const text = await resp.text();
+        let hasData = false;
+        try {
+          const json = JSON.parse(text);
+          hasData = !!json && (Array.isArray(json) ? json.length > 0 :
+            typeof json === 'object' && Object.keys(json).length > 0);
+          // Check for API-level error codes (common in Chinese sites)
+          if (json.code !== undefined && json.code !== 0) hasData = false;
+        } catch {}
+        return { status, ok: true, hasData, preview: text.slice(0, 200) };
+      } catch (e) { return { ok: false, error: e.message }; }
+    }
+  `;
+}
+
 /**
  * Probe an endpoint with a specific strategy.
  * Returns whether the probe succeeded and basic response info.
@@ -45,32 +88,14 @@ export async function probeEndpoint(
   page: IPage,
   url: string,
   strategy: Strategy,
-  opts: { timeout?: number } = {},
+  _opts: { timeout?: number } = {},
 ): Promise<ProbeResult> {
   const result: ProbeResult = { strategy, success: false };
 
   try {
     switch (strategy) {
       case Strategy.PUBLIC: {
-        // Try direct fetch without browser (no credentials)
-        const js = `
-          async () => {
-            try {
-              const resp = await fetch(${JSON.stringify(url)});
-              const status = resp.status;
-              if (!resp.ok) return { status, ok: false };
-              const text = await resp.text();
-              let hasData = false;
-              try {
-                const json = JSON.parse(text);
-                hasData = !!json && (Array.isArray(json) ? json.length > 0 :
-                  typeof json === 'object' && Object.keys(json).length > 0);
-              } catch {}
-              return { status, ok: true, hasData, preview: text.slice(0, 200) };
-            } catch (e) { return { ok: false, error: e.message }; }
-          }
-        `;
-        const resp = await page.evaluate(js);
+        const resp = await page.evaluate(buildFetchProbeJs(url, {}));
         result.statusCode = resp?.status;
         result.success = resp?.ok && resp?.hasData;
         result.hasData = resp?.hasData;
@@ -79,27 +104,7 @@ export async function probeEndpoint(
       }
 
       case Strategy.COOKIE: {
-        // Fetch with credentials: 'include' (uses browser cookies)
-        const js = `
-          async () => {
-            try {
-              const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
-              const status = resp.status;
-              if (!resp.ok) return { status, ok: false };
-              const text = await resp.text();
-              let hasData = false;
-              try {
-                const json = JSON.parse(text);
-                hasData = !!json && (Array.isArray(json) ? json.length > 0 :
-                  typeof json === 'object' && Object.keys(json).length > 0);
-                // Check for API-level error codes (common in Chinese sites)
-                if (json.code !== undefined && json.code !== 0) hasData = false;
-              } catch {}
-              return { status, ok: true, hasData, preview: text.slice(0, 200) };
-            } catch (e) { return { ok: false, error: e.message }; }
-          }
-        `;
-        const resp = await page.evaluate(js);
+        const resp = await page.evaluate(buildFetchProbeJs(url, { credentials: true }));
         result.statusCode = resp?.status;
         result.success = resp?.ok && resp?.hasData;
         result.hasData = resp?.hasData;
@@ -108,39 +113,7 @@ export async function probeEndpoint(
       }
 
       case Strategy.HEADER: {
-        // Fetch with credentials + try to extract common auth headers
-        const js = `
-          async () => {
-            try {
-              // Try to extract CSRF tokens from cookies
-              const cookies = document.cookie.split(';').map(c => c.trim());
-              const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('=');
-              
-              const headers = {};
-              if (csrf) {
-                headers['X-Csrf-Token'] = csrf;
-                headers['X-XSRF-Token'] = csrf;
-              }
-              
-              const resp = await fetch(${JSON.stringify(url)}, {
-                credentials: 'include',
-                headers
-              });
-              const status = resp.status;
-              if (!resp.ok) return { status, ok: false };
-              const text = await resp.text();
-              let hasData = false;
-              try {
-                const json = JSON.parse(text);
-                hasData = !!json && (Array.isArray(json) ? json.length > 0 :
-                  typeof json === 'object' && Object.keys(json).length > 0);
-                if (json.code !== undefined && json.code !== 0) hasData = false;
-              } catch {}
-              return { status, ok: true, hasData, preview: text.slice(0, 200) };
-            } catch (e) { return { ok: false, error: e.message }; }
-          }
-        `;
-        const resp = await page.evaluate(js);
+        const resp = await page.evaluate(buildFetchProbeJs(url, { credentials: true, extractCsrf: true }));
         result.statusCode = resp?.status;
         result.success = resp?.ok && resp?.hasData;
         result.hasData = resp?.hasData;
@@ -151,7 +124,6 @@ export async function probeEndpoint(
       case Strategy.INTERCEPT:
       case Strategy.UI:
         // These require specific implementation per-site
-        // Mark as needing manual implementation
         result.success = false;
         result.error = `Strategy ${strategy} requires site-specific implementation`;
         break;

package/src/constants.ts

@@ -0,0 +1,35 @@
+/**
+ * Shared constants used across explore, synthesize, and pipeline modules.
+ */
+
+/** URL query params that are volatile/ephemeral and should be stripped from patterns */
+export const VOLATILE_PARAMS = new Set([
+  'w_rid', 'wts', '_', 'callback', 'timestamp', 't', 'nonce', 'sign',
+]);
+
+/** Search-related query parameter names */
+export const SEARCH_PARAMS = new Set([
+  'q', 'query', 'keyword', 'search', 'wd', 'kw', 'search_query', 'w',
+]);
+
+/** Pagination-related query parameter names */
+export const PAGINATION_PARAMS = new Set([
+  'page', 'pn', 'offset', 'cursor', 'next', 'page_num',
+]);
+
+/** Limit/page-size query parameter names */
+export const LIMIT_PARAMS = new Set([
+  'limit', 'count', 'size', 'per_page', 'page_size', 'ps', 'num',
+]);
+
+/** Field role → common API field names mapping */
+export const FIELD_ROLES: Record<string, string[]> = {
+  title:    ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'],
+  url:      ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'share_url'],
+  author:   ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'],
+  score:    ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'play', 'favorite_count', 'reply_count'],
+  time:     ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'],
+  id:       ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'],
+  cover:    ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'],
+  category: ['category', 'tag', 'type', 'tname', 'channel', 'section'],
+};

package/src/engine.test.ts

@@ -0,0 +1,77 @@
+/**
+ * Tests for engine.ts: CLI discovery and command execution.
+ */
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { discoverClis, executeCommand } from './engine.js';
+import { getRegistry, cli, Strategy } from './registry.js';
+
+describe('discoverClis', () => {
+  it('handles non-existent directories gracefully', async () => {
+    // Should not throw for missing directories
+    await expect(discoverClis('/tmp/nonexistent-opencli-test-dir')).resolves.not.toThrow();
+  });
+});
+
+describe('executeCommand', () => {
+  it('executes a command with func', async () => {
+    const cmd = cli({
+      site: 'test-engine',
+      name: 'func-test',
+      description: 'test command with func',
+      browser: false,
+      strategy: Strategy.PUBLIC,
+      func: async (_page, kwargs) => {
+        return [{ title: kwargs.query ?? 'default' }];
+      },
+    });
+
+    const result = await executeCommand(cmd, null, { query: 'hello' });
+    expect(result).toEqual([{ title: 'hello' }]);
+  });
+
+  it('executes a command with pipeline', async () => {
+    const cmd = cli({
+      site: 'test-engine',
+      name: 'pipe-test',
+      description: 'test command with pipeline',
+      browser: false,
+      strategy: Strategy.PUBLIC,
+      pipeline: [
+        { evaluate: '() => [{ n: 1 }, { n: 2 }, { n: 3 }]' },
+        { limit: '2' },
+      ],
+    });
+
+    // Pipeline commands require page for evaluate step, so we'll test the error path
+    await expect(executeCommand(cmd, null, {})).rejects.toThrow();
+  });
+
+  it('throws for command with no func or pipeline', async () => {
+    const cmd = cli({
+      site: 'test-engine',
+      name: 'empty-test',
+      description: 'empty command',
+      browser: false,
+    });
+
+    await expect(executeCommand(cmd, null, {})).rejects.toThrow('has no func or pipeline');
+  });
+
+  it('passes debug flag to func', async () => {
+    let receivedDebug = false;
+    const cmd = cli({
+      site: 'test-engine',
+      name: 'debug-test',
+      description: 'debug test',
+      browser: false,
+      func: async (_page, _kwargs, debug) => {
+        receivedDebug = debug ?? false;
+        return [];
+      },
+    });
+
+    await executeCommand(cmd, null, {}, true);
+    expect(receivedDebug).toBe(true);
+  });
+});

package/src/engine.ts

@@ -11,7 +11,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import yaml from 'js-yaml';
-import { type CliCommand, type Arg, Strategy, registerCommand } from './registry.js';
+import { type CliCommand, type InternalCliCommand, type Arg, Strategy, registerCommand } from './registry.js';
 import type { IPage } from './types.js';
 import { executePipeline } from './pipeline.js';
 
@@ -66,7 +66,7 @@ function loadFromManifest(manifestPath: string, clisDir: string): void {
         // The actual module is loaded lazily on first executeCommand().
         const strategy = (Strategy as any)[(entry.strategy ?? 'cookie').toUpperCase()] ?? Strategy.COOKIE;
         const modulePath = path.resolve(clisDir, entry.modulePath);
-        const cmd: CliCommand = {
+        const cmd: InternalCliCommand = {
           site: entry.site,
           name: entry.name,
           description: entry.description ?? '',
@@ -77,7 +77,6 @@ function loadFromManifest(manifestPath: string, clisDir: string): void {
           columns: entry.columns,
           timeoutSeconds: entry.timeout,
           source: modulePath,
-          // Mark as lazy — executeCommand will load the module before running
           _lazy: true,
           _modulePath: modulePath,
         };
@@ -170,8 +169,9 @@ export async function executeCommand(
   debug: boolean = false,
 ): Promise<any> {
   // Lazy-load TS module on first execution
-  if ((cmd as any)._lazy && (cmd as any)._modulePath) {
-    const modulePath = (cmd as any)._modulePath;
+  const internal = cmd as InternalCliCommand;
+  if (internal._lazy && internal._modulePath) {
+    const modulePath = internal._modulePath;
     if (!_loadedModules.has(modulePath)) {
       try {
         await import(`file://${modulePath}`);

package/src/explore.ts

@@ -9,6 +9,7 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
 import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
+import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES } from './constants.js';
 
 // ── Site name detection ────────────────────────────────────────────────────
 
@@ -43,21 +44,7 @@ export function slugify(value: string): string {
 
 // ── Field & capability inference ───────────────────────────────────────────
 
-const FIELD_ROLES: Record<string, string[]> = {
-  title:    ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'],
-  url:      ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'share_url'],
-  author:   ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'],
-  score:    ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'play', 'favorite_count', 'reply_count'],
-  time:     ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'],
-  id:       ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'],
-  cover:    ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'],
-  category: ['category', 'tag', 'type', 'tname', 'channel', 'section'],
-};
-
-const SEARCH_PARAMS = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'search_query', 'w']);
-const PAGINATION_PARAMS = new Set(['page', 'pn', 'offset', 'cursor', 'next', 'page_num']);
-const LIMIT_PARAMS = new Set(['limit', 'count', 'size', 'per_page', 'page_size', 'ps', 'num']);
-const VOLATILE_PARAMS = new Set(['w_rid', 'wts', '_', 'callback', 'timestamp', 't', 'nonce', 'sign']);
+// (constants now imported from constants.ts)
 
 // ── Network analysis ───────────────────────────────────────────────────────