@dyyz1993/agent-browser 0.11.5 → 0.12.0

package/scripts/generate-skill.cjs

@@ -0,0 +1,303 @@
+#!/usr/bin/env node
+
+const { execSync } = require('child_process');
+const fs = require('fs');
+const path = require('path');
+
+const SKILL_DIR = path.join(__dirname, '..', 'skills', 'agent-browser');
+const REFS_DIR = path.join(SKILL_DIR, 'references');
+
+function run(cmd) {
+  try {
+    return execSync(cmd, { encoding: 'utf-8', timeout: 10000 }).trim();
+  } catch {
+    return null;
+  }
+}
+
+function getSections(helpText) {
+  if (!helpText) return {};
+  const sections = {};
+  let current = null;
+  for (const line of helpText.split('\n')) {
+    const headerMatch = line.match(/^([A-Z][A-Za-z &/\-]+):\s*$/);
+    if (headerMatch) {
+      current = headerMatch[1];
+      sections[current] = [];
+    } else if (current) {
+      sections[current].push(line);
+    }
+  }
+  return sections;
+}
+
+function extractCommands(sections) {
+  const cmds = [];
+  for (const [name, lines] of Object.entries(sections)) {
+    for (const line of lines) {
+      const m = line.match(/^\s{2}(\S+)\s+(.*)/);
+      if (m) cmds.push({ group: name, name: m[1], desc: m[2].trim() });
+    }
+  }
+  return cmds;
+}
+
+console.log('Scanning agent-browser CLI...');
+
+const helpText = run('agent-browser --help 2>&1');
+if (!helpText) {
+  console.error('ERROR: agent-browser not found or failed.');
+  process.exit(1);
+}
+
+const sections = getSections(helpText);
+const commands = extractCommands(sections);
+
+console.log(`Found ${Object.keys(sections).length} sections, ${commands.length} commands`);
+
+const existingRefs = fs.readdirSync(REFS_DIR)
+  .filter(f => f.endsWith('.md'))
+  .map(f => f.replace('.md', ''));
+
+console.log(`Existing reference docs: ${existingRefs.join(', ')}`);
+
+const subSkills = [
+  {
+    title: 'Page Navigation & Interaction',
+    match: c => ['Core Commands', 'Navigation'].includes(c.group) &&
+      !['screenshot', 'pdf', 'snapshot', 'eval', 'connect', 'close'].includes(c.name),
+    summary: 'open, click, fill, type, press, hover, drag, scroll, check/uncheck, select',
+    ref: null,
+  },
+  {
+    title: 'Snapshot & Element Inspection',
+    match: c => c.name === 'snapshot' || c.group === 'Get Info' || c.group === 'Check State',
+    summary: 'snapshot -i (refs), get text/url/title/box/styles, is visible/enabled/checked',
+    ref: 'snapshot-refs',
+  },
+  {
+    title: 'Finding Elements',
+    match: c => c.group === 'Find Elements',
+    summary: 'find by role, text, label, placeholder, testid, nth',
+    ref: 'commands',
+  },
+  {
+    title: 'Data Extraction',
+    match: c => c.name === 'eval',
+    summary: 'JS evaluation, DOM scraping, API interception, infinite scroll',
+    ref: 'data-extraction',
+  },
+  {
+    title: 'Network Control',
+    match: c => c.group === 'Network',
+    summary: 'request monitoring, API mocking, URL blocking',
+    ref: 'network-monitoring',
+  },
+  {
+    title: 'Session & State',
+    match: c => ['Sessions', 'Storage', 'Tabs'].includes(c.group) ||
+      ['connect', 'close'].includes(c.name),
+    summary: 'named sessions, state save/load, cookies, tabs, parallel sessions',
+    ref: 'session-management',
+  },
+  {
+    title: 'Authentication',
+    match: () => false,
+    summary: 'login flows, OAuth, 2FA, state reuse, cookie persistence',
+    ref: 'authentication',
+  },
+  {
+    title: 'Recording & Replay',
+    match: c => c.group === 'Debug' && (c.name.startsWith('record') || c.name.startsWith('recorder')),
+    summary: 'step recorder (YAML), video recording (WebM), trace',
+    ref: 'recorder',
+  },
+  {
+    title: 'Visual Remote Control (Viewer)',
+    match: c => c.group === 'Remote' && c.name === 'viewer',
+    summary: 'real-time frame streaming, element crop mode, WebSocket viewer',
+    ref: 'viewer-mode',
+  },
+  {
+    title: 'Mobile Remote Control',
+    match: () => false,
+    summary: 'touchpad gestures, input panel, IME/CJK, DeviceMode auto-switch',
+    ref: 'mobile-viewer',
+  },
+  {
+    title: 'iOS Simulator (Appium)',
+    match: () => false,
+    summary: 'native iOS automation via Xcode + Appium',
+    ref: null,
+  },
+  {
+    title: 'Cloud Browser Providers',
+    match: () => false,
+    summary: 'browserbase, kernel, browseruse',
+    ref: null,
+  },
+  {
+    title: 'Proxy & Network Config',
+    match: c => c.name === 'install',
+    summary: 'HTTP/SOCKS5 proxy, geo-testing, rotating proxies',
+    ref: 'proxy-support',
+  },
+];
+
+for (const sk of subSkills) {
+  const seen = new Set();
+  sk.matched = commands.filter(sk.match).filter(c => {
+    if (seen.has(c.name)) return false;
+    seen.add(c.name);
+    return true;
+  });
+}
+
+const envSection = sections['Environment'] || [];
+const envVars = envSection
+  .map(l => l.match(/^\s+(\S+)\s+(.*)/))
+  .filter(Boolean)
+  .map(m => ({ name: m[1], desc: m[2].trim() }));
+
+const optSection = sections['Options'] || [];
+const opts = optSection
+  .map(l => {
+    const m = l.match(/^\s+(-{1,2}[\w-]+(?:,\s*-{1,2}[\w-]+)?)\s+<([^>]+)>\s+(.*)/);
+    if (m) return { flag: `${m[1]} <${m[2]}>`, desc: m[3].trim() };
+    const m2 = l.match(/^\s+(-{1,2}[\w-]+(?:,\s*-{1,2}[\w-]+)?)\s+(.*)/);
+    if (m2) return { flag: m2[1], desc: m2[2].trim() };
+    return null;
+  })
+  .filter(Boolean);
+
+console.log(`Parsed ${envVars.length} env vars, ${opts.length} flags`);
+
+const refTableRows = [
+  { file: 'commands.md', label: 'Complete Command Reference', desc: 'All commands with options and examples' },
+  { file: 'snapshot-refs.md', label: 'Snapshot & Refs', desc: 'Ref lifecycle, invalidation rules, shell scripts' },
+  { file: 'data-extraction.md', label: 'Data Extraction', desc: 'DOM scraping, JS eval, API interception, infinite scroll' },
+  { file: 'session-management.md', label: 'Session & State', desc: 'Parallel sessions, state persistence, concurrent scraping' },
+  { file: 'authentication.md', label: 'Authentication', desc: 'Login flows, OAuth, 2FA, state reuse' },
+  { file: 'network-monitoring.md', label: 'Network Control', desc: 'Request monitoring, API mocking, URL blocking' },
+  { file: 'recorder.md', label: 'Recording & Replay', desc: 'Step recorder, video recording, trace' },
+  { file: 'proxy-support.md', label: 'Proxy Config', desc: 'HTTP/SOCKS5 proxy, geo-testing, rotating proxies' },
+  { file: 'viewer-mode.md', label: 'Viewer / Streaming', desc: 'Frame streaming, element crop, architecture' },
+  { file: 'mobile-viewer.md', label: 'Mobile Remote Control', desc: 'Touchpad, input panel, IME/CJK, DeviceMode' },
+  { file: 'video-recording.md', label: 'Video Recording', desc: 'WebM video capture for debugging' },
+];
+
+const output = [];
+const w = (s = '') => output.push(s);
+
+w('---');
+w('name: agent-browser');
+w('description: Browser automation CLI for AI agents. Use when the user needs to interact with websites, including navigating pages, filling forms, clicking buttons, taking screenshots, extracting data, testing web apps, viewer/streaming mode, mobile remote control, or automating any browser task. Triggers include requests to "open a website", "fill out a form", "click a button", "take a screenshot", "scrape data from a page", "test this web app", "login to a site", "automate browser actions", "view remote browser", "mobile browsing", or any task requiring programmatic web interaction.');
+w('allowed-tools: Bash(agent-browser:*)');
+w('---');
+w();
+w('# Browser Automation with agent-browser');
+w();
+w('Fast CLI for browser automation. Works headlessly by default, supports named sessions, proxy, and remote streaming.');
+w();
+w('## Browser Setup (macOS)');
+w();
+w('Set the browser path to avoid Playwright downloading Chromium:');
+w();
+w('```bash');
+w('export AGENT_BROWSER_EXECUTABLE_PATH=/Applications/Chromium.app/Contents/MacOS/Chromium');
+w('```');
+w();
+w('Or per-command: `agent-browser --executable-path /Applications/Chromium.app/Contents/MacOS/Chromium open <url>`');
+w();
+w('Verify: `agent-browser config`');
+w();
+w('## Quick Start');
+w();
+w('```bash');
+w('agent-browser open https://example.com');
+w('agent-browser snapshot -i                        # Get refs: @e1, @e2, ...');
+w('agent-browser fill @e1 "user@example.com"        # Interact via refs');
+w('agent-browser click @e2');
+w('agent-browser snapshot -i                        # Re-snapshot after page change');
+w('```');
+w();
+w('## Discovering Commands');
+w();
+w('```bash');
+w('agent-browser --help                             # All commands & options');
+w('agent-browser snapshot --help                    # Command-specific help');
+w('agent-browser config                             # Current config & env vars');
+w('```');
+w();
+w('The CLI is self-documenting. When unsure about a command, run `--help` first.');
+w();
+w('## Capabilities');
+w();
+w('| Area | Key Commands | Deep Dive |');
+w('|------|-------------|-----------|');
+
+for (const sk of subSkills) {
+  const cmdList = sk.matched.length > 0
+    ? sk.matched.slice(0, 5).map(c => `\`${c.name}\``).join(', ')
+    : sk.summary.split(',').slice(0, 3).map(s => `\`${s.trim()}\``).join(', ');
+  const refLink = sk.ref
+    ? `[${sk.ref}](references/${sk.ref}.md)`
+    : sk.title;
+  w(`| ${sk.title} | ${cmdList} | ${refLink} |`);
+}
+
+w();
+w('### Core Workflow Pattern');
+w();
+w('1. `open <url>` → navigate');
+w('2. `snapshot -i` → get element refs (`@e1`, `@e2`, ...)');
+w('3. `fill` / `click` / `select` → interact using refs');
+w('4. Re-`snapshot` after any page change (refs are invalidated)');
+w();
+w('### Refs');
+w();
+w('Refs (`@e1`, `@e2`) are **short-lived** — invalidated by any page change. Always re-snapshot after navigation or DOM mutations. See [snapshot-refs.md](references/snapshot-refs.md).');
+w();
+w('### Iframes');
+w();
+w('```bash');
+w('agent-browser snapshot --in-frame "#my-iframe"           # Single iframe');
+w('agent-browser click @e1 --in-frame "#outer/inner"        # Nested');
+w('```');
+w();
+w('### Semantic Locators (No Refs Needed)');
+w();
+w('```bash');
+w('agent-browser find text "Sign In" click');
+w('agent-browser find label "Email" fill "user@test.com"');
+w('agent-browser find role button click --name "Submit"');
+w('```');
+w();
+w('## Key Flags');
+w();
+for (const opt of opts) {
+  w(`- \`${opt.flag}\` — ${opt.desc}`);
+}
+w();
+w('## Environment Variables');
+w();
+for (const ev of envVars) {
+  w(`- \`${ev.name}\` — ${ev.desc}`);
+}
+w();
+w('## Reference Docs');
+w();
+w('| Doc | Content |');
+w('|-----|---------|');
+for (const r of refTableRows) {
+  w(`| [${r.label}](references/${r.file}) | ${r.desc} |`);
+}
+w();
+
+const outPath = path.join(SKILL_DIR, 'SKILL.md');
+fs.writeFileSync(outPath, output.join('\n'));
+console.log(`\nGenerated: ${outPath} (${output.length} lines)`);
+console.log('\nSummary:');
+for (const sk of subSkills) {
+  console.log(`  ${sk.title}: ${sk.matched.length} commands -> ${sk.ref || '(inline)'}`);
+}

package/scripts/get_jd_product_detail.js

@@ -0,0 +1,16 @@
+(() => {
+  const reviews = {};
+  
+  const goodEl = document.querySelector('.filter-items a[data-tab="good"]');
+  const neutralEl = document.querySelector('.filter-items a[data-tab="neutral"]');
+  const badEl = document.querySelector('.filter-items a[data-tab="bad"]');
+  
+  reviews.good = goodEl?.textContent?.trim() || '';
+  reviews.neutral = neutralEl?.textContent?.trim() || '';
+  reviews.bad = badEl?.textContent?.trim() || '';
+  
+  const regionEl = document.querySelector('.address-item');
+  const region = regionEl?.textContent?.trim() || '';
+  
+  return { reviews, region };
+})()

package/scripts/get_jd_products.js

@@ -0,0 +1,23 @@
+(() => {
+  const items = document.querySelectorAll('.gl-item');
+  const products = [];
+  
+  items.forEach((item, index) => {
+    const titleEl = item.querySelector('.p-name a');
+    const priceEl = item.querySelector('.p-price');
+    const shopEl = item.querySelector('.p-shop a');
+    const salesEl = item.querySelector('.p-commit');
+    const linkEl = item.querySelector('.p-name a');
+    
+    products.push({
+      index: index,
+      title: titleEl?.textContent?.trim() || '',
+      price: priceEl?.textContent?.trim() || '',
+      shop: shopEl?.textContent?.trim() || '',
+      sales: salesEl?.textContent?.trim() || '',
+      link: linkEl?.href || ''
+    });
+  });
+  
+  return products.slice(0, 10);
+})()

package/scripts/get_jd_seafood_products.js

@@ -0,0 +1,44 @@
+(async () => {
+  const result = {
+    products: []
+  };
+  
+  window.scrollTo(0, 1000);
+  await new Promise(resolve => setTimeout(resolve, 3000));
+  
+  window.scrollTo(0, 3000);
+  await new Promise(resolve => setTimeout(resolve, 3000));
+  
+  const productCards = document.querySelectorAll('[data-point-id], .youLikeItem, [class*="item"]');
+  
+  productCards.forEach((card, index) => {
+    if (index >= 15) return;
+    
+    const product = {
+      index,
+      className: card.className,
+      title: '',
+      price: '',
+      shop: '',
+      link: ''
+    };
+    
+    const titleEl = card.querySelector('[class*="title"], [class*="name"], h3, h4, h5');
+    const priceEl = card.querySelector('[class*="price"], .price');
+    const shopEl = card.querySelector('[class*="shop"], [class*="store"], [class*="brand"]');
+    const linkEl = card.querySelector('a[href*="jd"]');
+    const imgEl = card.querySelector('img');
+    
+    if (titleEl) product.title = titleEl.textContent?.trim() || '';
+    if (priceEl) product.price = priceEl.textContent?.trim() || '';
+    if (shopEl) product.shop = shopEl.textContent?.trim() || '';
+    if (linkEl) product.link = linkEl.href || '';
+    if (imgEl && imgEl.src) product.image = imgEl.src.substring(0, 100);
+    
+    if (product.title || product.price || product.link) {
+      result.products.push(product);
+    }
+  });
+  
+  return result;
+})()

package/scripts/get_product_details_from_images.js

@@ -0,0 +1,54 @@
+(async () => {
+  const result = {
+    products: []
+  };
+  
+  const wrappers = document.querySelectorAll('._wrapper_8g5fc_1');
+  
+  wrappers.forEach((wrapper, index) => {
+    if (index >= 10) return;
+    
+    const product = {
+      index,
+      className: wrapper.className
+    };
+    
+    const img = wrapper.querySelector('img');
+    if (img) {
+      product.img = img.getAttribute('data-src')?.substring(0, 100) || img.src?.substring(0, 100);
+    }
+    
+    const parentLink = wrapper.closest('a');
+    if (parentLink) {
+      product.link = parentLink.href;
+    }
+    
+    const allText = wrapper.textContent?.trim() || '';
+    product.text = allText.substring(0, 150);
+    
+    const priceElements = wrapper.querySelectorAll('*');
+    for (const el of priceElements) {
+      const text = el.textContent?.trim() || '';
+      if (text.match(/[¥￥]\s*\d+/)) {
+        product.price = text;
+        break;
+      }
+    }
+    
+    const greatGrandParent = wrapper.parentElement?.parentElement?.parentElement;
+    if (greatGrandParent) {
+      product.greatGrandParentClass = greatGrandParent.className?.substring(0, 100);
+      const greatGrandParentText = greatGrandParent.textContent?.trim() || '';
+      product.greatGrandParentText = greatGrandParentText.substring(0, 200);
+      
+      const greatGrandParentLink = greatGrandParent.querySelector('a[href*="jd"]');
+      if (greatGrandParentLink) {
+        product.link = greatGrandParentLink.href;
+      }
+    }
+    
+    result.products.push(product);
+  });
+  
+  return result;
+})()

package/scripts/verify-form.sh

@@ -0,0 +1,67 @@
+#!/bin/bash
+# 场景2: 表单填写验证脚本
+# 从 form-test.yaml 生成的脚本
+
+set -e  # 遇到错误立即退出
+
+# 启用人工模拟模式
+export AGENT_BROWSER_HUMAN=bezier
+SESSION="form-verify-$(date +%s)"
+
+# 测试页面路径
+TEST_PAGE="file:///Users/xuyingzhou/Project/temporary/agent-browser/test-pages/form-test.html"
+
+echo "=== 开始执行表单填写脚本 ==="
+echo "Session: $SESSION"
+echo "AGENT_BROWSER_HUMAN: $AGENT_BROWSER_HUMAN"
+
+# 打开测试页面
+echo "1. 打开表单测试页面..."
+agent-browser --session $SESSION --allow-file-access --headed open "$TEST_PAGE"
+agent-browser --session $SESSION wait --load networkidle
+
+# 填写表单
+echo "2. 填写表单..."
+agent-browser --session $SESSION fill "#name" "测试用户"
+agent-browser --session $SESSION fill "#email" "test@example.com"
+agent-browser --session $SESSION fill "#password" "password123"
+agent-browser --session $SESSION select "#country" "cn"
+agent-browser --session $SESSION check "#agree"
+
+# 提交表单
+echo "3. 提交表单..."
+agent-browser --session $SESSION click "#submitBtn"
+
+# 等待结果显示
+agent-browser --session $SESSION wait 500
+
+# 验证结果
+echo "4. 验证结果..."
+URL=$(agent-browser --session $SESSION get url)
+echo "当前URL: $URL"
+
+# 检查URL是否正确
+if [[ "$URL" == *"form-test.html"* ]]; then
+    echo "✓ URL验证成功"
+else
+    echo "✗ URL验证失败"
+    agent-browser --session $SESSION close
+    exit 1
+fi
+
+# 检查表单数据是否显示
+RESULT=$(agent-browser --session $SESSION get text "#result")
+if [[ "$RESULT" == *"测试用户"* ]]; then
+    echo "✓ 表单数据验证成功"
+else
+    echo "✗ 表单数据验证失败"
+    agent-browser --session $SESSION close
+    exit 1
+fi
+
+# 清理
+echo "5. 关闭浏览器..."
+agent-browser --session $SESSION close
+
+echo "=== 脚本执行成功 ==="
+exit 0

package/scripts/verify-login.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+# 场景8: 登录功能验证脚本（含验证码识别）
+
+set -e
+export AGENT_BROWSER_HUMAN=bezier
+SESSION="login-verify-$(date +%s)"
+TEST_PAGE="file:///Users/xuyingzhou/Project/temporary/agent-browser/test-pages/login-test.html"
+
+echo "=== 开始执行登录验证脚本 ==="
+echo "Session: $SESSION"
+
+# 打开测试页面
+echo "1. 打开登录测试页面..."
+agent-browser --session $SESSION --allow-file-access --headed open "$TEST_PAGE"
+agent-browser --session $SESSION wait --load networkidle
+
+# 检查未登录标识
+echo "2. 检查未登录标识..."
+UNLOGGED=$(agent-browser --session $SESSION get text "#loggedOutIndicator")
+if [[ "$UNLOGGED" == *"尚未登录"* ]]; then
+    echo "✓ 检测到未登录标识"
+else
+    echo "✗ 未检测到未登录标识"
+fi
+
+# 获取验证码并立即填写（使用链式命令减少延迟）
+echo "3. 获取验证码并填写..."
+agent-browser --session $SESSION eval --stdin <<'EOF'
+const captcha = document.getElementById("captchaCode").textContent;
+document.getElementById("username").value = "testuser";
+document.getElementById("password").value = "password123";
+document.getElementById("captcha").value = captcha;
+"验证码: " + captcha;
+EOF
+
+echo "4. 提交登录..."
+agent-browser --session $SESSION click "#loginBtn"
+agent-browser --session $SESSION wait 500
+
+# 验证登录结果
+echo "6. 验证登录结果..."
+STATUS=$(agent-browser --session $SESSION get text "#loginStatus")
+if [[ "$STATUS" == *"登录成功"* ]]; then
+    echo "✓ 登录成功"
+else
+    echo "✗ 登录失败: $STATUS"
+    agent-browser --session $SESSION close
+    exit 1
+fi
+
+# 检查用户信息显示
+echo "7. 检查用户信息..."
+USERINFO=$(agent-browser --session $SESSION get text "#userInfo")
+if [[ "$USERINFO" == *"testuser"* ]]; then
+    echo "✓ 用户信息显示正确"
+else
+    echo "✗ 用户信息显示错误"
+fi
+
+# 清理
+echo "8. 关闭浏览器..."
+agent-browser --session $SESSION close
+
+echo "=== 脚本执行成功 ==="
+exit 0

package/scripts/verify-recording.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+# 从 complex-scene-5.yaml 生成的验证脚本
+# 用于验证录制转脚本的可靠性
+
+set -e  # 遇到错误立即退出
+
+export AGENT_BROWSER_HUMAN=bezier
+SESSION="verify-$(date +%s)"
+
+# 测试页面路径
+TEST_PAGE="file:///Users/xuyingzhou/Project/temporary/agent-browser/test-pages/complex-scene-5.html"
+
+echo "=== 开始执行脚本 ==="
+echo "Session: $SESSION"
+
+# 打开测试页面
+echo "1. 打开测试页面..."
+agent-browser --session $SESSION --allow-file-access --headed open "$TEST_PAGE"
+agent-browser --session $SESSION wait --load networkidle
+
+# 执行表单填写
+echo "2. 填写表单..."
+agent-browser --session $SESSION fill "#username" "测试用户001"
+agent-browser --session $SESSION fill "#email" "test@example.com"
+agent-browser --session $SESSION fill "#comments" "这是一个自动化测试备注信息"
+
+# 点击按钮
+echo "3. 点击按钮..."
+agent-browser --session $SESSION click "#btn-primary"
+agent-browser --session $SESSION click "#btn-secondary"
+agent-browser --session $SESSION click "#btn-success"
+agent-browser --session $SESSION click "#btn-danger"
+
+# 下拉选择
+echo "4. 选择下拉选项..."
+agent-browser --session $SESSION select "#country" "cn"
+agent-browser --session $SESSION select "#city" "北京"
+
+# 复选框
+echo "5. 勾选复选框..."
+agent-browser --session $SESSION check "#chk1"
+agent-browser --session $SESSION check "#chk2"
+agent-browser --session $SESSION check "#chk4"
+
+# 单选框
+echo "6. 选择单选项..."
+agent-browser --session $SESSION click "#rd1"
+
+# 计数器
+echo "7. 点击计数器..."
+agent-browser --session $SESSION click "#counter-increase"
+
+# 开关
+echo "8. 切换开关..."
+agent-browser --session $SESSION click "#dark-mode-toggle"
+
+# 收集数据
+echo "9. 点击收集数据..."
+agent-browser --session $SESSION click "#collect-data"
+
+# 验证结果
+echo "10. 验证结果..."
+URL=$(agent-browser --session $SESSION get url)
+echo "当前URL: $URL"
+
+# 检查URL是否正确
+if [[ "$URL" == *"complex-scene-5.html"* ]]; then
+    echo "✓ URL验证成功"
+else
+    echo "✗ URL验证失败"
+    agent-browser --session $SESSION close
+    exit 1
+fi
+
+# 清理
+echo "11. 关闭浏览器..."
+agent-browser --session $SESSION close
+
+echo "=== 脚本执行成功 ==="
+exit 0