@jackwener/opencli 0.6.1 → 0.6.3

package/.github/actions/setup-chrome/action.yml

@@ -0,0 +1,26 @@
+name: Setup Chrome + xvfb
+description: Install real Chrome and xvfb virtual display for headed browser testing
+
+outputs:
+  chrome-path:
+    description: Path to the installed Chrome binary
+    value: ${{ steps.setup-chrome.outputs.chrome-path }}
+
+runs:
+  using: composite
+  steps:
+    - name: Install real Chrome (stable)
+      uses: browser-actions/setup-chrome@v1
+      id: setup-chrome
+      with:
+        chrome-version: stable
+
+    - name: Verify Chrome installation
+      shell: bash
+      run: |
+        echo "Chrome path: ${{ steps.setup-chrome.outputs.chrome-path }}"
+        ${{ steps.setup-chrome.outputs.chrome-path }} --version
+
+    - name: Install xvfb for headed mode
+      shell: bash
+      run: sudo apt-get install -y xvfb

package/.github/workflows/ci.yml

@@ -2,12 +2,16 @@ name: CI
 
 on:
   push:
-    branches: [main]
+    branches: [main, dev]
   pull_request:
-    branches: [main]
+    branches: [main, dev]
+  schedule:
+    - cron: '0 8 * * 1'  # Weekly Monday 08:00 UTC — smoke tests
+  workflow_dispatch:
 
 jobs:
-  check:
+  # ── Fast gate: typecheck + build ──
+  build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
@@ -15,6 +19,7 @@ jobs:
       - uses: actions/setup-node@v4
         with:
           node-version: '22'
+          cache: 'npm'
 
       - name: Install dependencies
         run: npm ci
@@ -24,3 +29,54 @@ jobs:
 
       - name: Build
         run: npm run build
+
+  # ── Unit tests (vitest shard) ──
+  unit-test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        shard: [1, 2]
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run unit tests (shard ${{ matrix.shard }}/2)
+        run: npx vitest run src/ --reporter=verbose --shard=${{ matrix.shard }}/2
+
+  # ── Smoke tests (scheduled / manual only) ──
+  smoke-test:
+    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
+    needs: build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Setup Chrome + xvfb
+        uses: ./.github/actions/setup-chrome
+        id: setup-chrome
+
+      - name: Build
+        run: npm run build
+
+      - name: Run smoke tests
+        run: |
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" \
+            npx vitest run tests/smoke/ --reporter=verbose
+        env:
+          OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
+    timeout-minutes: 15

package/.github/workflows/e2e-headed.yml

@@ -0,0 +1,37 @@
+name: E2E Headed Chrome
+
+on:
+  push:
+    branches: [main, dev]
+  pull_request:
+    branches: [main, dev]
+  workflow_dispatch:
+
+jobs:
+  e2e-headed:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Setup Chrome + xvfb
+        uses: ./.github/actions/setup-chrome
+        id: setup-chrome
+
+      - name: Build
+        run: npm run build
+
+      - name: Run E2E tests (headed Chrome + xvfb)
+        run: |
+          xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" \
+            npx vitest run tests/e2e/ --reporter=verbose
+        env:
+          OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}

package/README.md

@@ -21,6 +21,7 @@ A CLI tool that turns **any website** into a command-line interface — bilibili
 - [Built-in Commands](#built-in-commands)
 - [Output Formats](#output-formats)
 - [For AI Agents (Developer Guide)](#for-ai-agents-developer-guide)
+- [Testing](#testing)
 - [Troubleshooting](#troubleshooting)
 - [Releasing New Versions](#releasing-new-versions)
 - [License](#license)
@@ -138,7 +139,7 @@ npm install -g @jackwener/opencli@latest
 | **twitter** | `trending` `bookmarks` `profile` `search` `timeline` `following` `followers` `notifications` `post` `reply` `delete` `like` | 🔐 Browser |
 | **reddit** | `hot` `frontpage` `search` `subreddit` | 🔐 Browser |
 | **weibo** | `hot` | 🔐 Browser |
-| **boss** | `search` | 🔐 Browser |
+| **boss** | `search` `detail` | 🔐 Browser |
 | **coupang** | `search` `add-to-cart` | 🔐 Browser |
 | **youtube** | `search` | 🔐 Browser |
 | **yahoo-finance** | `quote` | 🔐 Browser |
@@ -189,6 +190,24 @@ opencli cascade https://api.example.com/data
 
 Explore outputs to `.opencli/explore/<site>/` (manifest.json, endpoints.json, capabilities.json, auth.json).
 
+## Testing
+
+See **[TESTING.md](./TESTING.md)** for the full testing guide, including:
+
+- Current test coverage (unit + ~52 E2E tests across all 18 sites)
+- How to run tests locally
+- How to add tests when creating new adapters
+- CI/CD pipeline with sharding
+- Headless browser mode (`OPENCLI_HEADLESS=1`)
+
+```bash
+# Quick start
+npm run build
+npx vitest run                              # All tests
+npx vitest run src/                          # Unit tests only
+npx vitest run tests/e2e/                    # E2E tests
+```
+
 ## Troubleshooting
 
 - **"Failed to connect to Playwright MCP Bridge"**

package/README.zh-CN.md

@@ -138,7 +138,7 @@ npm install -g @jackwener/opencli@latest
 | **twitter** | `trending` `bookmarks` `profile` `search` `timeline` `following` `followers` `notifications` `post` `reply` `delete` `like` | 🔐 浏览器 |
 | **reddit** | `hot` `frontpage` `search` `subreddit` | 🔐 浏览器 |
 | **weibo** | `hot` | 🔐 浏览器 |
-| **boss** | `search` | 🔐 浏览器 |
+| **boss** | `search` `detail` | 🔐 浏览器 |
 | **coupang** | `search` `add-to-cart` | 🔐 浏览器 |
 | **youtube** | `search` | 🔐 浏览器 |
 | **yahoo-finance** | `quote` | 🔐 浏览器 |

package/TESTING.md

@@ -0,0 +1,233 @@
+# Testing Guide
+
+> 面向开发者和 AI Agent 的测试参考手册。
+
+## 目录
+
+- [测试架构](#测试架构)
+- [当前覆盖范围](#当前覆盖范围)
+- [本地运行测试](#本地运行测试)
+- [如何添加新测试](#如何添加新测试)
+- [CI/CD 流水线](#cicd-流水线)
+- [浏览器模式](#浏览器模式)
+- [站点兼容性](#站点兼容性)
+
+---
+
+## 测试架构
+
+测试分为三层，全部使用 **vitest** 运行：
+
+```
+tests/
+├── e2e/                           # E2E 集成测试（子进程运行真实 CLI）
+│   ├── helpers.ts                 # runCli() 共享工具
+│   ├── public-commands.test.ts    # 公开 API 命令（无需浏览器）
+│   ├── browser-public.test.ts     # 浏览器命令（公开数据）
+│   ├── browser-auth.test.ts       # 需登录命令（graceful failure 测试）
+│   ├── management.test.ts         # 管理命令（list, validate, verify, help）
+│   └── output-formats.test.ts     # 输出格式（json/yaml/csv/md）
+├── smoke/                         # 烟雾测试（仅定时 / 手动触发）
+│   └── api-health.test.ts         # 外部 API 可用性检测
+src/
+├── *.test.ts                      # 单元测试（已有 8 个）
+```
+
+| 层 | 位置 | 运行方式 | 用途 |
+|---|---|---|---|
+| 单元测试 | `src/**/*.test.ts` | `npx vitest run src/` | 内部模块逻辑 |
+| E2E 测试 | `tests/e2e/*.test.ts` | `npx vitest run tests/e2e/` | 真实 CLI 命令执行 |
+| 烟雾测试 | `tests/smoke/*.test.ts` | `npx vitest run tests/smoke/` | 外部 API 健康 |
+
+---
+
+## 当前覆盖范围
+
+### 单元测试（8 个文件）
+
+| 文件 | 覆盖内容 |
+|---|---|
+| `browser.test.ts` | JSON-RPC、tab 管理、extension/standalone 模式切换 |
+| `engine.test.ts` | 命令发现与执行 |
+| `registry.test.ts` | 命令注册与策略分配 |
+| `output.test.ts` | 输出格式渲染 |
+| `doctor.test.ts` | Token 诊断 |
+| `coupang.test.ts` | 数据归一化 |
+| `pipeline/template.test.ts` | 模板表达式求值 |
+| `pipeline/transform.test.ts` | 数据变换步骤 |
+
+### E2E 测试（~52 个用例）
+
+| 文件 | 覆盖站点/功能 | 测试数 |
+|---|---|---|
+| `public-commands.test.ts` | hackernews/top, v2ex/hot, v2ex/latest, v2ex/topic | 5 |
+| `browser-public.test.ts` | bbc, bilibili×3, weibo, zhihu×2, reddit×2, twitter, xueqiu×2, reuters, youtube, smzdm, boss, ctrip, coupang, xiaohongshu, yahoo-finance, v2ex/daily | 21 |
+| `browser-auth.test.ts` | bilibili/me,dynamic,favorite,history,following + twitter/bookmarks,timeline,notifications + v2ex/me,notifications + xueqiu/feed,watchlist + xiaohongshu/feed,notifications | 14 |
+| `management.test.ts` | list×5 格式, validate×3 级别, verify, --version, --help, unknown cmd | 12 |
+| `output-formats.test.ts` | json, yaml, csv, md 格式验证 | 5 |
+
+### 烟雾测试
+
+公开 API 可用性（hackernews, v2ex×2, v2ex/topic）+ 全站点注册完整性检查。
+
+---
+
+## 本地运行测试
+
+### 前置条件
+
+```bash
+npm ci                # 安装依赖
+npm run build         # 编译（E2E 测试需要 dist/main.js）
+```
+
+### 运行命令
+
+```bash
+# 全部单元测试
+npx vitest run src/
+
+# 全部 E2E 测试（会真实调用外部 API）
+npx vitest run tests/e2e/
+
+# 单个测试文件
+npx vitest run tests/e2e/management.test.ts
+
+# 全部测试（单元 + E2E）
+npx vitest run
+
+# 烟雾测试
+npx vitest run tests/smoke/
+
+# watch 模式（开发时推荐）
+npx vitest src/
+```
+
+### 浏览器命令本地测试须知
+
+- 无 `PLAYWRIGHT_MCP_EXTENSION_TOKEN` 时，opencli 自动启动一个独立浏览器实例
+- `browser-public.test.ts` 使用 `tryBrowserCommand()`，站点反爬导致空数据时 warn + pass
+- `browser-auth.test.ts` 验证 **graceful failure**（不 crash 不 hang 即通过）
+- 如需测试完整登录态，保持 Chrome 登录态 + 设置 `PLAYWRIGHT_MCP_EXTENSION_TOKEN`，手动跑对应测试
+
+---
+
+## 如何添加新测试
+
+### 新增 YAML Adapter（如 `src/clis/producthunt/trending.yaml`）
+
+1. **无需额外操作**：`validate` 测试会自动覆盖 YAML 结构验证
+2. 根据 adapter 类型，在对应文件加一个 `it()` block：
+
+```typescript
+// 如果 browser: false（公开 API）→ tests/e2e/public-commands.test.ts
+it('producthunt trending returns data', async () => {
+  const { stdout, code } = await runCli(['producthunt', 'trending', '--limit', '3', '-f', 'json']);
+  expect(code).toBe(0);
+  const data = parseJsonOutput(stdout);
+  expect(Array.isArray(data)).toBe(true);
+  expect(data.length).toBeGreaterThanOrEqual(1);
+  expect(data[0]).toHaveProperty('title');
+}, 30_000);
+```
+
+```typescript
+// 如果 browser: true 但可公开访问 → tests/e2e/browser-public.test.ts
+it('producthunt trending returns data', async () => {
+  const data = await tryBrowserCommand(['producthunt', 'trending', '--limit', '3', '-f', 'json']);
+  expectDataOrSkip(data, 'producthunt trending');
+}, 60_000);
+```
+
+```typescript
+// 如果 browser: true 且需登录 → tests/e2e/browser-auth.test.ts
+it('producthunt me fails gracefully without login', async () => {
+  await expectGracefulAuthFailure(['producthunt', 'me', '-f', 'json'], 'producthunt me');
+}, 60_000);
+```
+
+### 新增管理命令（如 `opencli export`）
+
+在 `tests/e2e/management.test.ts` 添加测试。
+
+### 新增内部模块
+
+在 `src/` 下对应位置创建 `*.test.ts`。
+
+### 决策流程图
+
+```
+新增功能 → 是内部模块？ → 是 → src/ 下加 *.test.ts
+                ↓ 否
+         是 CLI 命令？ → browser: false? → tests/e2e/public-commands.test.ts
+                              ↓ true
+                        公开数据？ → tests/e2e/browser-public.test.ts
+                              ↓ 需登录
+                        tests/e2e/browser-auth.test.ts
+```
+
+---
+
+## CI/CD 流水线
+
+### ci.yml（主流水线）
+
+| Job | 触发条件 | 内容 |
+|---|---|---|
+| **build** | push/PR to main,dev | typecheck + build |
+| **unit-test** | push/PR to main,dev | 单元测试，2 shard 并行 |
+| **smoke-test** | 每周一 08:00 UTC / 手动 | xvfb + real Chrome，外部 API 健康检查 |
+
+### e2e-headed.yml（E2E 测试）
+
+| Job | 触发条件 | 内容 |
+|---|---|---|
+| **e2e-headed** | push/PR to main,dev | xvfb + real Chrome，全部 E2E 测试 |
+
+E2E 使用 `browser-actions/setup-chrome` 安装真实 Chrome，配合 `xvfb-run` 提供虚拟显示器，以 headed 模式运行浏览器。
+
+### Sharding
+
+单元测试使用 vitest 内置 shard：
+
+```yaml
+strategy:
+  matrix:
+    shard: [1, 2]
+steps:
+  - run: npx vitest run src/ --shard=${{ matrix.shard }}/2
+```
+
+---
+
+## 浏览器模式
+
+opencli 根据 `PLAYWRIGHT_MCP_EXTENSION_TOKEN` 环境变量自动选择模式：
+
+| 条件 | 模式 | MCP 参数 | 使用场景 |
+|---|---|---|---|
+| Token 已设置 | Extension 模式 | `--extension` | 本地用户，连接已登录的 Chrome |
+| Token 未设置 | Standalone 模式 | （无特殊 flag） | CI 或无扩展环境，自启浏览器 |
+
+CI 中使用 `OPENCLI_BROWSER_EXECUTABLE_PATH` 指定真实 Chrome 路径：
+
+```yaml
+env:
+  OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
+```
+
+---
+
+## 站点兼容性
+
+在 GitHub Actions 美国 runner 上，部分站点因地域限制或登录要求返回空数据。E2E 测试对这些站点使用 warn + pass 策略，不影响 CI 绿灯。
+
+| 站点 | CI 状态 | 限制原因 |
+|---|---|---|
+| hackernews, bbc, v2ex | ✅ 返回数据 | 无限制 |
+| yahoo-finance | ✅ 返回数据 | 无限制 |
+| bilibili, zhihu, weibo, xiaohongshu | ⚠️ 空数据 | 地域限制（中国站点） |
+| reddit, twitter, youtube | ⚠️ 空数据 | 需登录或 cookie |
+| smzdm, boss, ctrip, coupang, xueqiu | ⚠️ 空数据 | 地域限制 / 需登录 |
+
+> 使用 self-hosted runner（国内服务器）可解决地域限制问题。

package/dist/bilibili.js

@@ -63,10 +63,10 @@ export async function apiGet(page, path, opts = {}) {
     return fetchJson(page, url);
 }
 export async function fetchJson(page, url) {
-    const escapedUrl = url.replace(/"/g, '\\"');
+    const urlJs = JSON.stringify(url);
     return page.evaluate(`
     async () => {
-      const res = await fetch("${escapedUrl}", { credentials: "include" });
+      const res = await fetch(${urlJs}, { credentials: "include" });
       return await res.json();
     }
   `);

package/dist/browser.d.ts

@@ -50,7 +50,7 @@ export declare class Page implements IPage {
     selectTab(index: number): Promise<void>;
     networkRequests(includeStatic?: boolean): Promise<any>;
     consoleMessages(level?: string): Promise<any>;
-    scroll(direction?: string, amount?: number): Promise<void>;
+    scroll(direction?: string, _amount?: number): Promise<void>;
     autoScroll(options?: {
         times?: number;
         delayMs?: number;

package/dist/browser.js

@@ -169,7 +169,7 @@ export class Page {
     async consoleMessages(level = 'info') {
         return this.call('tools/call', { name: 'browser_console_messages', arguments: { level } });
     }
-    async scroll(direction = 'down', amount = 500) {
+    async scroll(direction = 'down', _amount = 500) {
         await this.call('tools/call', { name: 'browser_press_key', arguments: { key: direction === 'down' ? 'PageDown' : 'PageUp' } });
     }
     async autoScroll(options = {}) {
@@ -303,6 +303,7 @@ export class PlaywrightMCP {
         return new Promise((resolve, reject) => {
             const isDebug = process.env.DEBUG?.includes('opencli:mcp');
             const debugLog = (msg) => isDebug && console.error(`[opencli:mcp] ${msg}`);
+            const useExtension = !!process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN;
             const extensionToken = process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN;
             const tokenFingerprint = getTokenFingerprint(extensionToken);
             let stderrBuffer = '';
@@ -344,7 +345,9 @@ export class PlaywrightMCP {
                 executablePath: process.env.OPENCLI_BROWSER_EXECUTABLE_PATH,
             });
             if (process.env.OPENCLI_VERBOSE) {
-                console.error(`[opencli] Extension token: ${extensionToken ? `configured (fingerprint ${tokenFingerprint})` : 'missing'}`);
+                console.error(`[opencli] Mode: ${useExtension ? 'extension' : 'standalone'}`);
+                if (useExtension)
+                    console.error(`[opencli] Extension token: fingerprint ${tokenFingerprint}`);
             }
             debugLog(`Spawning node ${mcpArgs.join(' ')}`);
             this._proc = spawn('node', mcpArgs, {
@@ -559,7 +562,13 @@ function appendLimited(current, chunk, limit) {
     return next.slice(-limit);
 }
 function buildMcpArgs(input) {
-    const args = [input.mcpPath, '--extension'];
+    const args = [input.mcpPath];
+    if (!process.env.CI) {
+        // Local: always connect to user's running Chrome via MCP Bridge extension
+        args.push('--extension');
+    }
+    // CI: standalone mode — @playwright/mcp launches its own browser (headed by default).
+    // xvfb provides a virtual display for headed mode in GitHub Actions.
     if (input.executablePath) {
         args.push('--executable-path', input.executablePath);
     }

package/dist/browser.test.js

@@ -34,22 +34,62 @@ describe('browser helpers', () => {
     it('keeps only the tail of stderr buffers', () => {
         expect(__test__.appendLimited('12345', '67890', 8)).toBe('34567890');
     });
-    it('builds Playwright MCP args with kebab-case executable path', () => {
-        expect(__test__.buildMcpArgs({
-            mcpPath: '/tmp/cli.js',
-            executablePath: '/mnt/c/Program Files/Google/Chrome/Application/chrome.exe',
-        })).toEqual([
-            '/tmp/cli.js',
-            '--extension',
-            '--executable-path',
-            '/mnt/c/Program Files/Google/Chrome/Application/chrome.exe',
-        ]);
-        expect(__test__.buildMcpArgs({
-            mcpPath: '/tmp/cli.js',
-        })).toEqual([
-            '/tmp/cli.js',
-            '--extension',
-        ]);
+    it('builds extension MCP args in local mode (no CI)', () => {
+        const savedCI = process.env.CI;
+        delete process.env.CI;
+        try {
+            expect(__test__.buildMcpArgs({
+                mcpPath: '/tmp/cli.js',
+                executablePath: '/mnt/c/Program Files/Google/Chrome/Application/chrome.exe',
+            })).toEqual([
+                '/tmp/cli.js',
+                '--extension',
+                '--executable-path',
+                '/mnt/c/Program Files/Google/Chrome/Application/chrome.exe',
+            ]);
+            expect(__test__.buildMcpArgs({
+                mcpPath: '/tmp/cli.js',
+            })).toEqual([
+                '/tmp/cli.js',
+                '--extension',
+            ]);
+        }
+        finally {
+            if (savedCI !== undefined) {
+                process.env.CI = savedCI;
+            }
+            else {
+                delete process.env.CI;
+            }
+        }
+    });
+    it('builds standalone MCP args in CI mode', () => {
+        const savedCI = process.env.CI;
+        process.env.CI = 'true';
+        try {
+            // CI mode: no --extension — browser launches in standalone headed mode
+            expect(__test__.buildMcpArgs({
+                mcpPath: '/tmp/cli.js',
+            })).toEqual([
+                '/tmp/cli.js',
+            ]);
+            expect(__test__.buildMcpArgs({
+                mcpPath: '/tmp/cli.js',
+                executablePath: '/usr/bin/chromium',
+            })).toEqual([
+                '/tmp/cli.js',
+                '--executable-path',
+                '/usr/bin/chromium',
+            ]);
+        }
+        finally {
+            if (savedCI !== undefined) {
+                process.env.CI = savedCI;
+            }
+            else {
+                delete process.env.CI;
+            }
+        }
     });
     it('times out slow promises', async () => {
         await expect(__test__.withTimeoutMs(new Promise(() => { }), 10, 'timeout')).rejects.toThrow('timeout');

package/dist/cli-manifest.json

@@ -392,6 +392,44 @@
       "url"
     ]
   },
+  {
+    "site": "boss",
+    "name": "detail",
+    "description": "BOSS直聘查看职位详情",
+    "strategy": "cookie",
+    "browser": true,
+    "args": [
+      {
+        "name": "security_id",
+        "type": "str",
+        "required": true,
+        "help": "Security ID from search results (securityId field)"
+      }
+    ],
+    "type": "ts",
+    "modulePath": "boss/detail.js",
+    "domain": "www.zhipin.com",
+    "columns": [
+      "name",
+      "salary",
+      "experience",
+      "degree",
+      "city",
+      "district",
+      "description",
+      "skills",
+      "welfare",
+      "boss_name",
+      "boss_title",
+      "active_time",
+      "company",
+      "industry",
+      "scale",
+      "stage",
+      "address",
+      "url"
+    ]
+  },
   {
     "site": "boss",
     "name": "search",
@@ -467,6 +505,7 @@
       "degree",
       "skills",
       "boss",
+      "security_id",
       "url"
     ]
   },

package/dist/clis/boss/detail.d.ts

@@ -0,0 +1 @@
+export {};