@jackwener/opencli 1.3.3 → 1.4.1

package/src/snapshotFormatter.ts

@@ -1,21 +1,14 @@
 /**
  * Aria snapshot formatter: parses Playwright MCP snapshot text into clean format.
  *
- * Multi-pass pipeline:
- * 1. Parse & filter: strip annotations, metadata, noise roles, ads, decorators
- * 2. Deduplicate: generic/text child matching parent label
- * 3. Deduplicate: heading + link with identical labels
- * 4. Deduplicate: nested identical links
- * 5. Prune: empty containers (iterative bottom-up)
- * 6. Collapse: single-child containers
+ * 4-pass pipeline:
+ * 1. Parse & filter: strip annotations, metadata, noise, ads, boilerplate subtrees
+ * 2. Deduplicate: generic/text parent match, heading+link, nested identical links
+ * 3. Prune: empty containers (iterative bottom-up)
+ * 4. Collapse: single-child containers
  */
 
-export interface FormatOptions {
-  interactive?: boolean;
-  compact?: boolean;
-  maxDepth?: number;
-  maxTextLength?: number;
-}
+import type { SnapshotOptions } from './types.js';
 
 const DEFAULT_MAX_TEXT_LENGTH = 200;
 
@@ -199,19 +192,18 @@ interface Entry {
   trailingText: string;
   isInteractive: boolean;
   isLandmark: boolean;
-  isSubtreeSkip: boolean; // ad nodes or boilerplate — skip entire subtree
 }
 
-export function formatSnapshot(raw: string, opts: FormatOptions = {}): string {
+export function formatSnapshot(raw: string, opts: SnapshotOptions = {}): string {
   if (!raw || typeof raw !== 'string') return '';
 
   const maxTextLen = opts.maxTextLength ?? DEFAULT_MAX_TEXT_LENGTH;
   const lines = raw.split('\n');
 
-  // === Pass 1: Parse, filter, and collect entries ===
-  const entries: Entry[] = [];
+  // === Pass 1: Parse, filter, and collect entries (merged with ad/boilerplate subtree skip) ===
+  const parsed: Entry[] = [];
   let refCounter = 0;
-  let skipUntilDepth = -1; // When >= 0, skip all nodes at depth > this value
+  let skipUntilDepth = -1;
 
   for (let i = 0; i < lines.length; i++) {
     const line = lines[i];
@@ -220,148 +212,88 @@ export function formatSnapshot(raw: string, opts: FormatOptions = {}): string {
     const indent = line.length - line.trimStart().length;
     const depth = Math.floor(indent / 2);
 
-    // If we're in a subtree skip zone, check depth
+    // Subtree skip zone (noise roles, ads, boilerplate)
     if (skipUntilDepth >= 0) {
-      if (depth > skipUntilDepth) continue; // still inside subtree
-      skipUntilDepth = -1; // exited subtree
+      if (depth > skipUntilDepth) continue;
+      skipUntilDepth = -1;
     }
 
     let content = line.trimStart();
-
-    // Strip leading "- "
-    if (content.startsWith('- ')) {
-      content = content.slice(2);
-    }
-
-    // Skip metadata lines
+    if (content.startsWith('- ')) content = content.slice(2);
     if (isMetadataLine(content)) continue;
-
-    // Apply maxDepth filter
     if (opts.maxDepth !== undefined && depth > opts.maxDepth) continue;
 
     const { role, text, hasText, trailingText } = parseLine(content);
 
-    // Skip noise nodes
     if (isNoiseNode(role, hasText, text, trailingText)) continue;
 
-    // Skip subtree noise roles (contentinfo footer, etc.) — skip entire subtree
-    if (SUBTREE_NOISE_ROLES.has(role)) {
-      skipUntilDepth = depth;
-      continue;
-    }
+    // Subtree noise roles (contentinfo footer, etc.)
+    if (SUBTREE_NOISE_ROLES.has(role)) { skipUntilDepth = depth; continue; }
 
-    // Strip annotations
-    content = stripAnnotations(content);
+    // Ads and boilerplate — skip entire subtree (merged from old Pass 2)
+    if (isAdNode(text, trailingText) || isBoilerplateNode(text)) { skipUntilDepth = depth; continue; }
 
-    // Check if node should trigger subtree skip (ads, boilerplate)
-    const isSubtreeSkip = isAdNode(text, trailingText) || isBoilerplateNode(text);
+    content = stripAnnotations(content);
 
-    // Interactive mode filter
     const isInteractive = INTERACTIVE_ROLES.has(role);
     const isLandmark = LANDMARK_ROLES.has(role);
-
     if (opts.interactive && !isInteractive && !isLandmark && !hasText) continue;
 
-    // Compact mode
     if (opts.compact) {
-      content = content
-        .replace(/\s*\[.*?\]\s*/g, ' ')
-        .replace(/\s+/g, ' ')
-        .trim();
+      content = content.replace(/\s*\[.*?\]\s*/g, ' ').replace(/\s+/g, ' ').trim();
     }
-
-    // Text truncation
     if (maxTextLen > 0 && content.length > maxTextLen) {
       content = content.slice(0, maxTextLen) + '…';
     }
-
-    // Assign refs to interactive elements
     if (isInteractive) {
       refCounter++;
       content = `[@${refCounter}] ${content}`;
     }
 
-    entries.push({ depth, content, role, text, trailingText, isInteractive, isLandmark, isSubtreeSkip });
-  }
-
-  // === Pass 2: Remove subtree-skip nodes (ads, boilerplate, contentinfo) ===
-  let noAds: Entry[] = [];
-  for (let i = 0; i < entries.length; i++) {
-    const entry = entries[i];
-    if (entry.isSubtreeSkip) {
-      const skipDepth = entry.depth;
-      i++;
-      while (i < entries.length && entries[i].depth > skipDepth) {
-        i++;
-      }
-      i--;
-      continue;
-    }
-    noAds.push(entry);
+    parsed.push({ depth, content, role, text, trailingText, isInteractive, isLandmark });
   }
 
-  // === Pass 3: Deduplicate child generic/text matching parent label ===
-  let deduped: Entry[] = [];
-  for (let i = 0; i < noAds.length; i++) {
-    const entry = noAds[i];
+  // === Pass 2: Deduplicate (merged: generic/text parent match + heading+link + nested links) ===
+  const deduped: Entry[] = [];
+  for (let i = 0; i < parsed.length; i++) {
+    const entry = parsed[i];
 
+    // Dedup: generic/text child matching parent label
     if (entry.role === 'generic' || entry.role === 'text') {
       let parent: Entry | undefined;
       for (let j = deduped.length - 1; j >= 0; j--) {
-        if (deduped[j].depth < entry.depth) {
-          parent = deduped[j];
-          break;
-        }
+        if (deduped[j].depth < entry.depth) { parent = deduped[j]; break; }
         if (deduped[j].depth === entry.depth) break;
       }
-
       if (parent) {
         const childText = entry.trailingText || entry.text;
-        if (childText && parent.text && childText === parent.text) {
-          continue;
-        }
+        if (childText && parent.text && childText === parent.text) continue;
       }
     }
 
-    deduped.push(entry);
-  }
-
-  // === Pass 4: Deduplicate heading + child link with identical label ===
-  // Pattern: heading "Title": → link "Title": (same text) → skip the link
-  const deduped2: Entry[] = [];
-  for (let i = 0; i < deduped.length; i++) {
-    const entry = deduped[i];
-
+    // Dedup: heading + child link with identical label
     if (entry.role === 'heading' && entry.text) {
-      const next = deduped[i + 1];
+      const next = parsed[i + 1];
       if (next && next.role === 'link' && next.text === entry.text && next.depth === entry.depth + 1) {
-        // Keep the heading, skip the link. But preserve link's children re-parented.
-        deduped2.push(entry);
-        i++; // skip the link
+        deduped.push(entry);
+        i++; // skip the link, preserve its children
         continue;
       }
     }
 
-    deduped2.push(entry);
-  }
-
-  // === Pass 5: Deduplicate nested identical links ===
-  const deduped3: Entry[] = [];
-  for (let i = 0; i < deduped2.length; i++) {
-    const entry = deduped2[i];
-
+    // Dedup: nested identical links (skip parent, keep child)
     if (entry.role === 'link' && entry.text) {
-      const next = deduped2[i + 1];
+      const next = parsed[i + 1];
       if (next && next.role === 'link' && next.text === entry.text && next.depth === entry.depth + 1) {
-        continue; // Skip parent, keep child
+        continue;
       }
     }
 
-    deduped3.push(entry);
+    deduped.push(entry);
   }
 
-  // === Pass 6: Iteratively prune empty containers (bottom-up) ===
-  let current = deduped3;
+  // === Pass 3: Iteratively prune empty containers (bottom-up) ===
+  let current = deduped;
   let changed = true;
   while (changed) {
     changed = false;
@@ -372,22 +304,16 @@ export function formatSnapshot(raw: string, opts: FormatOptions = {}): string {
         let hasChildren = false;
         for (let j = i + 1; j < current.length; j++) {
           if (current[j].depth <= entry.depth) break;
-          if (current[j].depth > entry.depth) {
-            hasChildren = true;
-            break;
-          }
-        }
-        if (!hasChildren) {
-          changed = true;
-          continue;
+          if (current[j].depth > entry.depth) { hasChildren = true; break; }
         }
+        if (!hasChildren) { changed = true; continue; }
       }
       next.push(entry);
     }
     current = next;
   }
 
-  // === Pass 7: Collapse single-child containers ===
+  // === Pass 4: Collapse single-child containers ===
   const collapsed: Entry[] = [];
   for (let i = 0; i < current.length; i++) {
     const entry = current[i];
@@ -408,17 +334,13 @@ export function formatSnapshot(raw: string, opts: FormatOptions = {}): string {
         let hasGrandchildren = false;
         for (let j = childIdx + 1; j < current.length; j++) {
           if (current[j].depth <= child.depth) break;
-          if (current[j].depth > child.depth) {
-            hasGrandchildren = true;
-            break;
-          }
+          if (current[j].depth > child.depth) { hasGrandchildren = true; break; }
         }
 
         if (!hasGrandchildren) {
-          const mergedContent = entry.content.replace(/:$/, '') + ' > ' + child.content;
           collapsed.push({
             ...entry,
-            content: mergedContent,
+            content: entry.content.replace(/:$/, '') + ' > ' + child.content,
             role: child.role,
             text: child.text,
             trailingText: child.trailingText,

package/src/utils.ts

@@ -0,0 +1,39 @@
+/**
+ * Shared utility functions used across the codebase.
+ */
+
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+
+/** Type guard: checks if a value is a non-null, non-array object. */
+export function isRecord(value: unknown): value is Record<string, unknown> {
+  return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+
+/** Simple async concurrency limiter. */
+export async function mapConcurrent<T, R>(
+  items: T[],
+  limit: number,
+  fn: (item: T, index: number) => Promise<R>,
+): Promise<R[]> {
+  const results: R[] = new Array(items.length);
+  let index = 0;
+
+  async function worker() {
+    while (index < items.length) {
+      const i = index++;
+      results[i] = await fn(items[i], i);
+    }
+  }
+
+  const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
+  await Promise.all(workers);
+  return results;
+}
+
+/** Save a base64-encoded string to a file, creating parent directories as needed. */
+export async function saveBase64ToFile(base64: string, filePath: string): Promise<void> {
+  const dir = path.dirname(filePath);
+  await fs.promises.mkdir(dir, { recursive: true });
+  await fs.promises.writeFile(filePath, Buffer.from(base64, 'base64'));
+}

package/src/validate.ts

@@ -6,10 +6,10 @@ import { getErrorMessage } from './errors.js';
 
 /** All recognized pipeline step names */
 const KNOWN_STEP_NAMES = new Set([
-  'navigate', 'click', 'type', 'wait', 'press', 'snapshot', 'scroll',
+  'navigate', 'click', 'type', 'wait', 'press', 'snapshot',
   'fetch', 'evaluate',
   'select', 'map', 'filter', 'sort', 'limit',
-  'intercept', 'tap',
+  'intercept', 'tap', 'download',
 ]);
 
 export interface FileValidationResult {
@@ -34,9 +34,7 @@ interface ValidatedYamlCliDefinition {
   args?: Record<string, unknown>;
 }
 
-function isRecord(value: unknown): value is Record<string, unknown> {
-  return typeof value === 'object' && value !== null && !Array.isArray(value);
-}
+import { isRecord } from './utils.js';
 
 
 export function validateClisWithTarget(dirs: string[], target?: string): ValidationReport {

package/src/weread-search-regression.test.ts

@@ -0,0 +1,44 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { getRegistry } from './registry.js';
+import './clis/weread/search.js';
+
+describe('weread/search regression', () => {
+  beforeEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('uses the query argument for the search API and returns urls', async () => {
+    const command = getRegistry().get('weread/search');
+    expect(command?.func).toBeTypeOf('function');
+
+    const fetchMock = vi.fn().mockResolvedValue({
+      ok: true,
+      json: () => Promise.resolve({
+        books: [
+          {
+            bookInfo: {
+              title: 'Deep Work',
+              author: 'Cal Newport',
+              bookId: 'abc123',
+            },
+          },
+        ],
+      }),
+    });
+    vi.stubGlobal('fetch', fetchMock);
+
+    const result = await command!.func!(null as any, { query: 'deep work', limit: 5 });
+
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+    expect(String(fetchMock.mock.calls[0][0])).toContain('keyword=deep+work');
+    expect(result).toEqual([
+      {
+        rank: 1,
+        title: 'Deep Work',
+        author: 'Cal Newport',
+        bookId: 'abc123',
+        url: 'https://weread.qq.com/web/bookDetail/abc123',
+      },
+    ]);
+  });
+});

package/src/yaml-schema.ts

@@ -0,0 +1,28 @@
+/**
+ * Shared YAML CLI definition types.
+ * Used by both discovery.ts (runtime) and build-manifest.ts (build-time).
+ */
+
+export interface YamlArgDefinition {
+  type?: string;
+  default?: unknown;
+  required?: boolean;
+  positional?: boolean;
+  description?: string;
+  help?: string;
+  choices?: string[];
+}
+
+export interface YamlCliDefinition {
+  site?: string;
+  name?: string;
+  description?: string;
+  domain?: string;
+  strategy?: string;
+  browser?: boolean;
+  args?: Record<string, YamlArgDefinition>;
+  columns?: string[];
+  pipeline?: Record<string, unknown>[];
+  timeout?: number;
+  navigateBefore?: boolean | string;
+}

package/tests/e2e/browser-auth.test.ts

@@ -113,6 +113,31 @@ describe('login-required commands — graceful failure', () => {
     await expectGracefulAuthFailure(['xiaohongshu', 'notifications', '--limit', '3', '-f', 'json'], 'xiaohongshu notifications');
   }, 60_000);
 
+  // ── pixiv (requires login) ──
+  it('pixiv ranking fails gracefully without login', async () => {
+    await expectGracefulAuthFailure(['pixiv', 'ranking', '--limit', '3', '-f', 'json'], 'pixiv ranking');
+  }, 60_000);
+
+  it('pixiv search fails gracefully without login', async () => {
+    await expectGracefulAuthFailure(['pixiv', 'search', '初音ミク', '--limit', '3', '-f', 'json'], 'pixiv search');
+  }, 60_000);
+
+  it('pixiv user fails gracefully without login', async () => {
+    await expectGracefulAuthFailure(['pixiv', 'user', '11', '-f', 'json'], 'pixiv user');
+  }, 60_000);
+
+  it('pixiv illusts fails gracefully without login', async () => {
+    await expectGracefulAuthFailure(['pixiv', 'illusts', '11', '--limit', '3', '-f', 'json'], 'pixiv illusts');
+  }, 60_000);
+
+  it('pixiv detail fails gracefully without login', async () => {
+    await expectGracefulAuthFailure(['pixiv', 'detail', '123456', '-f', 'json'], 'pixiv detail');
+  }, 60_000);
+
+  it('pixiv download fails gracefully without login', async () => {
+    await expectGracefulAuthFailure(['pixiv', 'download', '123456', '--output', '/tmp/pixiv-e2e-test', '-f', 'json'], 'pixiv download');
+  }, 60_000);
+
   // ── yollomi (requires login session) ──
   it('yollomi generate fails gracefully without login', async () => {
     await expectGracefulAuthFailure(['yollomi', 'generate', 'a cute cat', '--no-download', '-f', 'json'], 'yollomi generate');

package/tests/e2e/browser-public-extended.test.ts

@@ -0,0 +1,162 @@
+/**
+ * Extended E2E tests for all other browser commands.
+ * Opt-in only: OPENCLI_E2E=1 npx vitest run
+ */
+
+import { describe, it, expect } from 'vitest';
+import { runCli, parseJsonOutput } from './helpers.js';
+
+async function tryBrowserCommand(args: string[]): Promise<any[] | null> {
+  const { stdout, code } = await runCli(args, { timeout: 60_000 });
+  if (code !== 0) return null;
+  try {
+    const data = parseJsonOutput(stdout);
+    return Array.isArray(data) ? data : null;
+  } catch {
+    return null;
+  }
+}
+
+function expectDataOrSkip(data: any[] | null, label: string) {
+  if (data === null || data.length === 0) {
+    console.warn(`${label}: skipped — no data returned (likely bot detection or geo-blocking)`);
+    return;
+  }
+  expect(data.length).toBeGreaterThanOrEqual(1);
+}
+
+describe('browser extended public-data commands E2E', () => {
+
+  // ── bbc ──
+  it('bbc news returns headlines', async () => {
+    const data = await tryBrowserCommand(['bbc', 'news', '--limit', '3', '-f', 'json']);
+    expectDataOrSkip(data, 'bbc news');
+    if (data) {
+      expect(data[0]).toHaveProperty('title');
+    }
+  }, 60_000);
+
+  // ── bloomberg ──
+  it('bloomberg news returns article detail when the article page is accessible', async () => {
+    const feedResult = await runCli(['bloomberg', 'tech', '--limit', '1', '-f', 'json']);
+    if (feedResult.code !== 0) {
+      console.warn('bloomberg news: skipped — could not load Bloomberg tech feed');
+      return;
+    }
+
+    const feedItems = parseJsonOutput(feedResult.stdout);
+    const link = Array.isArray(feedItems) ? feedItems[0]?.link : null;
+    if (!link) {
+      console.warn('bloomberg news: skipped — tech feed returned no link');
+      return;
+    }
+
+    const data = await tryBrowserCommand(['bloomberg', 'news', link, '-f', 'json']);
+    expectDataOrSkip(data, 'bloomberg news');
+    if (data) {
+      expect(data[0]).toHaveProperty('title');
+      expect(data[0]).toHaveProperty('summary');
+      expect(data[0]).toHaveProperty('link');
+      expect(data[0]).toHaveProperty('mediaLinks');
+      expect(data[0]).toHaveProperty('content');
+    }
+  }, 60_000);
+
+  // ── weibo ──
+  it('weibo hot returns trending topics', async () => {
+    const data = await tryBrowserCommand(['weibo', 'hot', '--limit', '5', '-f', 'json']);
+    expectDataOrSkip(data, 'weibo hot');
+  }, 60_000);
+
+  it('weibo search returns results', async () => {
+    const data = await tryBrowserCommand(['weibo', 'search', 'openai', '--limit', '3', '-f', 'json']);
+    expectDataOrSkip(data, 'weibo search');
+  }, 60_000);
+
+  // ── reddit ──
+  it('reddit hot returns posts', async () => {
+    const data = await tryBrowserCommand(['reddit', 'hot', '--limit', '5', '-f', 'json']);
+    expectDataOrSkip(data, 'reddit hot');
+  }, 60_000);
+
+  it('reddit frontpage returns posts', async () => {
+    const data = await tryBrowserCommand(['reddit', 'frontpage', '--limit', '5', '-f', 'json']);
+    expectDataOrSkip(data, 'reddit frontpage');
+  }, 60_000);
+
+  // ── twitter ──
+  it('twitter trending returns trends', async () => {
+    const data = await tryBrowserCommand(['twitter', 'trending', '--limit', '5', '-f', 'json']);
+    expectDataOrSkip(data, 'twitter trending');
+  }, 60_000);
+
+  // ── xueqiu ──
+  it('xueqiu hot returns hot posts', async () => {
+    const data = await tryBrowserCommand(['xueqiu', 'hot', '--limit', '5', '-f', 'json']);
+    expectDataOrSkip(data, 'xueqiu hot');
+  }, 60_000);
+
+  it('xueqiu hot-stock returns stocks', async () => {
+    const data = await tryBrowserCommand(['xueqiu', 'hot-stock', '--limit', '5', '-f', 'json']);
+    expectDataOrSkip(data, 'xueqiu hot-stock');
+  }, 60_000);
+
+  // ── reuters ──
+  it('reuters search returns articles', async () => {
+    const data = await tryBrowserCommand(['reuters', 'search', 'technology', '--limit', '3', '-f', 'json']);
+    expectDataOrSkip(data, 'reuters search');
+  }, 60_000);
+
+  // ── youtube ──
+  it('youtube search returns videos', async () => {
+    const data = await tryBrowserCommand(['youtube', 'search', 'typescript tutorial', '--limit', '3', '-f', 'json']);
+    expectDataOrSkip(data, 'youtube search');
+  }, 60_000);
+
+  // ── smzdm ──
+  it('smzdm search returns deals', async () => {
+    const data = await tryBrowserCommand(['smzdm', 'search', '键盘', '--limit', '3', '-f', 'json']);
+    expectDataOrSkip(data, 'smzdm search');
+  }, 60_000);
+
+  // ── boss ──
+  it('boss search returns jobs', async () => {
+    const data = await tryBrowserCommand(['boss', 'search', 'golang', '--limit', '3', '-f', 'json']);
+    expectDataOrSkip(data, 'boss search');
+  }, 60_000);
+
+  // ── ctrip ──
+  it('ctrip search returns flights', async () => {
+    const data = await tryBrowserCommand(['ctrip', 'search', '-f', 'json']);
+    expectDataOrSkip(data, 'ctrip search');
+  }, 60_000);
+
+  // ── coupang ──
+  it('coupang search returns products', async () => {
+    const data = await tryBrowserCommand(['coupang', 'search', 'laptop', '--limit', '3', '-f', 'json']);
+    expectDataOrSkip(data, 'coupang search');
+  }, 60_000);
+
+  // ── xiaohongshu ──
+  it('xiaohongshu search returns notes', async () => {
+    const data = await tryBrowserCommand(['xiaohongshu', 'search', '美食', '--limit', '3', '-f', 'json']);
+    expectDataOrSkip(data, 'xiaohongshu search');
+  }, 60_000);
+
+  // ── google ──
+  it('google search returns results', async () => {
+    const data = await tryBrowserCommand(['google', 'search', 'typescript', '--limit', '5', '-f', 'json']);
+    expectDataOrSkip(data, 'google search');
+    if (data) {
+      expect(data[0]).toHaveProperty('type');
+      expect(data[0]).toHaveProperty('title');
+      expect(data[0]).toHaveProperty('url');
+    }
+  }, 60_000);
+
+  // ── yahoo-finance ──
+  it('yahoo-finance quote returns stock data', async () => {
+    const data = await tryBrowserCommand(['yahoo-finance', 'quote', '--symbol', 'AAPL', '-f', 'json']);
+    expectDataOrSkip(data, 'yahoo-finance quote');
+  }, 60_000);
+});