@jackwener/opencli 1.3.3 → 1.4.0

package/src/clis/twitter/search.test.ts

@@ -0,0 +1,113 @@
+import { describe, expect, it, vi } from 'vitest';
+import { getRegistry } from '../../registry.js';
+import './search.js';
+
+describe('twitter search command', () => {
+  it('retries transient SPA navigation failures before giving up', async () => {
+    const command = getRegistry().get('twitter/search');
+    expect(command?.func).toBeTypeOf('function');
+
+    const evaluate = vi.fn()
+      .mockResolvedValueOnce(undefined)
+      .mockResolvedValueOnce('/explore')
+      .mockResolvedValueOnce(undefined)
+      .mockResolvedValueOnce('/search');
+
+    const page = {
+      goto: vi.fn().mockResolvedValue(undefined),
+      wait: vi.fn().mockResolvedValue(undefined),
+      installInterceptor: vi.fn().mockResolvedValue(undefined),
+      evaluate,
+      autoScroll: vi.fn().mockResolvedValue(undefined),
+      getInterceptedRequests: vi.fn().mockResolvedValue([
+        {
+          data: {
+            search_by_raw_query: {
+              search_timeline: {
+                timeline: {
+                  instructions: [
+                    {
+                      type: 'TimelineAddEntries',
+                      entries: [
+                        {
+                          entryId: 'tweet-1',
+                          content: {
+                            itemContent: {
+                              tweet_results: {
+                                result: {
+                                  rest_id: '1',
+                                  legacy: {
+                                    full_text: 'hello world',
+                                    favorite_count: 7,
+                                  },
+                                  core: {
+                                    user_results: {
+                                      result: {
+                                        core: {
+                                          screen_name: 'alice',
+                                        },
+                                      },
+                                    },
+                                  },
+                                  views: {
+                                    count: '12',
+                                  },
+                                },
+                              },
+                            },
+                          },
+                        },
+                      ],
+                    },
+                  ],
+                },
+              },
+            },
+          },
+        },
+      ]),
+    };
+
+    const result = await command!.func!(page as any, { query: 'from:alice', limit: 5 });
+
+    expect(result).toEqual([
+      {
+        id: '1',
+        author: 'alice',
+        text: 'hello world',
+        likes: 7,
+        views: '12',
+        url: 'https://x.com/i/status/1',
+      },
+    ]);
+    expect(page.installInterceptor).toHaveBeenCalledWith('SearchTimeline');
+    expect(evaluate).toHaveBeenCalledTimes(4);
+  });
+
+  it('throws with the final path after both attempts fail', async () => {
+    const command = getRegistry().get('twitter/search');
+    expect(command?.func).toBeTypeOf('function');
+
+    const evaluate = vi.fn()
+      .mockResolvedValueOnce(undefined)
+      .mockResolvedValueOnce('/explore')
+      .mockResolvedValueOnce(undefined)
+      .mockResolvedValueOnce('/login');
+
+    const page = {
+      goto: vi.fn().mockResolvedValue(undefined),
+      wait: vi.fn().mockResolvedValue(undefined),
+      installInterceptor: vi.fn().mockResolvedValue(undefined),
+      evaluate,
+      autoScroll: vi.fn().mockResolvedValue(undefined),
+      getInterceptedRequests: vi.fn(),
+    };
+
+    await expect(command!.func!(page as any, { query: 'from:alice', limit: 5 }))
+      .rejects
+      .toThrow('Final path: /login');
+    expect(page.autoScroll).not.toHaveBeenCalled();
+    expect(page.getInterceptedRequests).not.toHaveBeenCalled();
+    expect(evaluate).toHaveBeenCalledTimes(4);
+  });
+});

package/src/clis/twitter/search.ts

@@ -1,4 +1,41 @@
+import { CommandExecutionError } from '../../errors.js';
 import { cli, Strategy } from '../../registry.js';
+import type { IPage } from '../../types.js';
+
+/**
+ * Trigger Twitter search SPA navigation and retry once on transient failures.
+ *
+ * Twitter/X sometimes keeps the page on /explore for a short period even after
+ * pushState + popstate. A second attempt is enough for the intermittent cases
+ * reported in issue #353 while keeping the flow narrowly scoped.
+ */
+async function navigateToSearch(page: Pick<IPage, 'evaluate' | 'wait'>, query: string): Promise<void> {
+  const searchUrl = JSON.stringify(`/search?q=${encodeURIComponent(query)}&f=top`);
+  let lastPath = '';
+
+  for (let attempt = 1; attempt <= 2; attempt++) {
+    await page.evaluate(`
+      (() => {
+        window.history.pushState({}, '', ${searchUrl});
+        window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
+      })()
+    `);
+    await page.wait(5);
+
+    lastPath = String(await page.evaluate('() => window.location.pathname') || '');
+    if (lastPath.startsWith('/search')) {
+      return;
+    }
+
+    if (attempt < 2) {
+      await page.wait(1);
+    }
+  }
+
+  throw new CommandExecutionError(
+    `SPA navigation to /search failed. Final path: ${lastPath || '(empty)'}. Twitter may have changed its routing.`,
+  );
+}
 
 cli({
   site: 'twitter',
@@ -29,20 +66,7 @@ cli({
     //    a full page reload, so the interceptor stays alive.
     //    Note: the previous approach (nativeSetter + Enter keydown on the
     //    search input) does not reliably trigger Twitter's form submission.
-    const searchUrl = JSON.stringify(`/search?q=${encodeURIComponent(query)}&f=top`);
-    await page.evaluate(`
-      (() => {
-        window.history.pushState({}, '', ${searchUrl});
-        window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
-      })()
-    `);
-    await page.wait(5);
-
-    // Verify SPA navigation succeeded
-    const currentPath = await page.evaluate('() => window.location.pathname');
-    if (!currentPath?.startsWith('/search')) {
-        throw new Error('SPA navigation to /search failed. Twitter may have changed its routing.');
-    }
+    await navigateToSearch(page, query);
 
     // 4. Scroll to trigger additional pagination
     await page.autoScroll({ times: 3, delayMs: 2000 });

package/src/clis/twitter/thread.ts

@@ -1,5 +1,5 @@
 import { cli, Strategy } from '../../registry.js';
-import { AuthRequiredError } from '../../errors.js';
+import { AuthRequiredError, CommandExecutionError } from '../../errors.js';
 
 // ── Twitter GraphQL constants ──────────────────────────────────────────
 
@@ -165,7 +165,7 @@ cli({
       }`);
 
       if (data?.error) {
-        if (allTweets.length === 0) throw new Error(`HTTP ${data.error}: Tweet not found or queryId expired`);
+        if (allTweets.length === 0) throw new CommandExecutionError(`HTTP ${data.error}: Tweet not found or queryId expired`);
         break;
       }
 

package/src/clis/twitter/timeline.ts

@@ -1,3 +1,4 @@
+import { AuthRequiredError, CommandExecutionError } from '../../errors.js';
 import { cli, Strategy } from '../../registry.js';
 
 // ── Twitter GraphQL constants ──────────────────────────────────────────
@@ -194,7 +195,7 @@ cli({
     const ct0 = await page.evaluate(`() => {
       return document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1] || null;
     }`);
-    if (!ct0) throw new Error('Not logged into x.com (no ct0 cookie)');
+    if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
 
     // Dynamically resolve queryId for the selected endpoint
     const resolved = await page.evaluate(`async () => {
@@ -236,7 +237,7 @@ cli({
 
       if (data?.error) {
         if (allTweets.length === 0)
-          throw new Error(`HTTP ${data.error}: Failed to fetch timeline. queryId may have expired.`);
+          throw new CommandExecutionError(`HTTP ${data.error}: Failed to fetch timeline. queryId may have expired.`);
         break;
       }
 

package/src/clis/twitter/trending.ts

@@ -1,4 +1,5 @@
 import { cli, Strategy } from '../../registry.js';
+import { AuthRequiredError, EmptyResultError } from '../../errors.js';
 
 // ── Twitter GraphQL constants ──────────────────────────────────────────
 
@@ -37,7 +38,7 @@ cli({
     const ct0 = await page.evaluate(`(() => {
       return document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1] || null;
     })()`);
-    if (!ct0) throw new Error('Not logged into x.com (no ct0 cookie)');
+    if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
 
     // Try legacy guide.json API first (faster than DOM scraping)
     let trends: TrendItem[] = [];
@@ -105,7 +106,7 @@ cli({
     }
 
     if (trends.length === 0) {
-      throw new Error('No trending data found. API may have changed or login may be required.');
+      throw new EmptyResultError('twitter trending', 'API may have changed or login may be required.');
     }
 
     return trends.slice(0, limit);

package/src/clis/twitter/unblock.ts

@@ -1,3 +1,4 @@
+import { CommandExecutionError } from '../../errors.js';
 import { cli, Strategy } from '../../registry.js';
 import type { IPage } from '../../types.js';
 
@@ -13,7 +14,7 @@ cli({
   ],
   columns: ['status', 'message'],
   func: async (page: IPage | null, kwargs: any) => {
-    if (!page) throw new Error('Requires browser');
+    if (!page) throw new CommandExecutionError('Browser session required for twitter unblock');
     const username = kwargs.username.replace(/^@/, '');
 
     await page.goto(`https://x.com/${username}`);

package/src/clis/twitter/unbookmark.ts

@@ -1,3 +1,4 @@
+import { CommandExecutionError } from '../../errors.js';
 import { cli, Strategy } from '../../registry.js';
 import type { IPage } from '../../types.js';
 
@@ -13,7 +14,7 @@ cli({
   ],
   columns: ['status', 'message'],
   func: async (page: IPage | null, kwargs: any) => {
-    if (!page) throw new Error('Requires browser');
+    if (!page) throw new CommandExecutionError('Browser session required for twitter unbookmark');
 
     await page.goto(kwargs.url);
     await page.wait(5);

package/src/clis/twitter/unfollow.ts

@@ -1,4 +1,5 @@
 import { cli, Strategy } from '../../registry.js';
+import { CommandExecutionError } from '../../errors.js';
 import type { IPage } from '../../types.js';
 
 cli({
@@ -13,7 +14,7 @@ cli({
   ],
   columns: ['status', 'message'],
   func: async (page: IPage | null, kwargs: any) => {
-    if (!page) throw new Error('Requires browser');
+    if (!page) throw new CommandExecutionError('Browser session required for twitter unfollow');
     const username = kwargs.username.replace(/^@/, '');
 
     await page.goto(`https://x.com/${username}`);

package/src/clis/v2ex/daily.ts

@@ -1,6 +1,7 @@
 /**
  * V2EX Daily Check-in adapter.
  */
+import { CommandExecutionError } from '../../errors.js';
 import { cli, Strategy } from '../../registry.js';
 import type { IPage } from '../../types.js';
 
@@ -15,7 +16,7 @@ cli({
   args: [],
   columns: ['status', 'message'],
   func: async (page: IPage | null) => {
-    if (!page) throw new Error('Browser page required');
+    if (!page) throw new CommandExecutionError('Browser page required');
 
     if (process.env.OPENCLI_VERBOSE) {
       console.error('[opencli:v2ex] Navigating to /mission/daily');
@@ -60,7 +61,7 @@ cli({
         console.error(`[opencli:v2ex:debug] Page Title: ${checkResult.debug_title}`);
         console.error(`[opencli:v2ex:debug] Page Body: ${checkResult.debug_body}`);
       }
-      throw new Error(checkResult.error);
+      throw new CommandExecutionError(checkResult.error);
     }
 
     if (checkResult.claimed) {

package/src/clis/v2ex/me.ts

@@ -1,6 +1,7 @@
 /**
  * V2EX Me (Profile/Balance) adapter.
  */
+import { CommandExecutionError } from '../../errors.js';
 import { cli, Strategy } from '../../registry.js';
 import type { IPage } from '../../types.js';
 
@@ -15,7 +16,7 @@ cli({
   args: [],
   columns: ['username', 'balance', 'unread_notifications', 'daily_reward_ready'],
   func: async (page: IPage | null) => {
-    if (!page) throw new Error('Browser page required');
+    if (!page) throw new CommandExecutionError('Browser page required');
 
     if (process.env.OPENCLI_VERBOSE) {
       console.error('[opencli:v2ex] Navigating to /');
@@ -95,7 +96,7 @@ cli({
         console.error(`[opencli:v2ex:debug] Page Title: ${data.debug_title}`);
         console.error(`[opencli:v2ex:debug] Page Body: ${data.debug_body}`);
       }
-      throw new Error(data.error);
+      throw new CommandExecutionError(data.error);
     }
 
     return [data];

package/src/clis/v2ex/notifications.ts

@@ -1,6 +1,7 @@
 /**
  * V2EX Notifications adapter.
  */
+import { CommandExecutionError } from '../../errors.js';
 import { cli, Strategy } from '../../registry.js';
 import type { IPage } from '../../types.js';
 
@@ -17,7 +18,7 @@ cli({
   ],
   columns: ['type', 'content', 'time'],
   func: async (page: IPage | null, kwargs) => {
-    if (!page) throw new Error('Browser page required');
+    if (!page) throw new CommandExecutionError('Browser page required');
 
     if (process.env.OPENCLI_VERBOSE) {
       console.error('[opencli:v2ex] Navigating to /notifications');
@@ -67,9 +68,7 @@ cli({
       }
     `);
 
-    if (!Array.isArray(data)) {
-      throw new Error('Failed to parse notifications data');
-    }
+    if (!Array.isArray(data)) throw new CommandExecutionError('Failed to parse notifications data');
 
     const limit = kwargs.limit || 20;
     return data.slice(0, limit);

package/src/clis/web/read.ts

@@ -0,0 +1,210 @@
+/**
+ * Generic web page reader — fetch any URL and export as Markdown.
+ *
+ * Uses browser-side DOM heuristics to extract the main content:
+ *   1. <article> element
+ *   2. [role="main"] element
+ *   3. <main> element
+ *   4. Largest text-dense block as fallback
+ *
+ * Pipes through the shared article-download pipeline (Turndown + image download).
+ *
+ * Usage:
+ *   opencli web read --url "https://www.anthropic.com/research/..." --output ./articles
+ *   opencli web read --url "https://..." --download-images false
+ */
+
+import { cli, Strategy } from '../../registry.js';
+import { downloadArticle } from '../../download/article-download.js';
+
+cli({
+  site: 'web',
+  name: 'read',
+  description: 'Fetch any web page and export as Markdown',
+  strategy: Strategy.COOKIE,
+  navigateBefore: false, // we handle navigation ourselves
+  args: [
+    { name: 'url', required: true, help: 'Any web page URL' },
+    { name: 'output', default: './web-articles', help: 'Output directory' },
+    { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
+    { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' },
+  ],
+  columns: ['title', 'author', 'publish_time', 'status', 'size'],
+  func: async (page, kwargs) => {
+    const url = kwargs.url;
+    const waitSeconds = kwargs.wait ?? 3;
+
+    // Navigate to the target URL
+    await page.goto(url);
+    await page.wait(waitSeconds);
+
+    // Extract article content using browser-side heuristics
+    const data = await page.evaluate(`
+      (() => {
+        const result = {
+          title: '',
+          author: '',
+          publishTime: '',
+          contentHtml: '',
+          imageUrls: []
+        };
+
+        // --- Title extraction ---
+        // Priority: og:title > <title> > first <h1>
+        const ogTitle = document.querySelector('meta[property="og:title"]');
+        if (ogTitle) {
+          result.title = ogTitle.getAttribute('content')?.trim() || '';
+        }
+        if (!result.title) {
+          result.title = document.title?.trim() || '';
+        }
+        if (!result.title) {
+          const h1 = document.querySelector('h1');
+          result.title = h1?.textContent?.trim() || 'untitled';
+        }
+        // Strip site suffix (e.g. " | Anthropic", " - Blog")
+        result.title = result.title.replace(/\\s*[|\\-–—]\\s*[^|\\-–—]{1,30}$/, '').trim();
+
+        // --- Author extraction ---
+        const authorMeta = document.querySelector(
+          'meta[name="author"], meta[property="article:author"], meta[name="twitter:creator"]'
+        );
+        result.author = authorMeta?.getAttribute('content')?.trim() || '';
+
+        // --- Publish time extraction ---
+        const timeMeta = document.querySelector(
+          'meta[property="article:published_time"], meta[name="date"], meta[name="publishdate"], time[datetime]'
+        );
+        if (timeMeta) {
+          result.publishTime = timeMeta.getAttribute('content')
+            || timeMeta.getAttribute('datetime')
+            || timeMeta.textContent?.trim()
+            || '';
+        }
+
+        // --- Content extraction ---
+        // Strategy: try semantic elements first, then fall back to largest text block
+        let contentEl = null;
+
+        // 1. <article>
+        const articles = document.querySelectorAll('article');
+        if (articles.length === 1) {
+          contentEl = articles[0];
+        } else if (articles.length > 1) {
+          // Pick the largest article by text length
+          let maxLen = 0;
+          articles.forEach(a => {
+            const len = a.textContent?.length || 0;
+            if (len > maxLen) { maxLen = len; contentEl = a; }
+          });
+        }
+
+        // 2. [role="main"]
+        if (!contentEl) {
+          contentEl = document.querySelector('[role="main"]');
+        }
+
+        // 3. <main>
+        if (!contentEl) {
+          contentEl = document.querySelector('main');
+        }
+
+        // 4. Largest text-dense block fallback
+        if (!contentEl) {
+          const candidates = document.querySelectorAll(
+            'div[class*="content"], div[class*="article"], div[class*="post"], ' +
+            'div[class*="entry"], div[class*="body"], div[id*="content"], ' +
+            'div[id*="article"], div[id*="post"], section'
+          );
+          let maxLen = 0;
+          candidates.forEach(c => {
+            const len = c.textContent?.length || 0;
+            if (len > maxLen) { maxLen = len; contentEl = c; }
+          });
+        }
+
+        // 5. Last resort: document.body
+        if (!contentEl || (contentEl.textContent?.length || 0) < 200) {
+          contentEl = document.body;
+        }
+
+        // Clean up noise elements before extraction
+        const clone = contentEl.cloneNode(true);
+        const noise = 'nav, header, footer, aside, .sidebar, .nav, .menu, .footer, ' +
+          '.header, .comments, .comment, .ad, .ads, .advertisement, .social-share, ' +
+          '.related-posts, .newsletter, .cookie-banner, script, style, noscript, iframe';
+        clone.querySelectorAll(noise).forEach(el => el.remove());
+
+        // Deduplicate: some sites (e.g. Anthropic) render each paragraph twice
+        // (a visible version + a line-broken animation version with missing spaces).
+        // Compare by stripping ALL whitespace so "Hello world" matches "Helloworld".
+        const stripWS = (s) => (s || '').replace(/\\s+/g, '');
+        const dedup = (parent) => {
+          const children = Array.from(parent.children || []);
+          for (let i = children.length - 1; i >= 1; i--) {
+            const curRaw = children[i].textContent || '';
+            const prevRaw = children[i - 1].textContent || '';
+            const cur = stripWS(curRaw);
+            const prev = stripWS(prevRaw);
+            if (cur.length < 20 || prev.length < 20) continue;
+            // Exact match after whitespace strip, or >90% overlap
+            if (cur === prev) {
+              // Keep the one with more proper spacing (more spaces = better formatted)
+              const curSpaces = (curRaw.match(/ /g) || []).length;
+              const prevSpaces = (prevRaw.match(/ /g) || []).length;
+              if (curSpaces >= prevSpaces) children[i - 1].remove();
+              else children[i].remove();
+            } else if (prev.includes(cur) && cur.length / prev.length > 0.8) {
+              children[i].remove();
+            } else if (cur.includes(prev) && prev.length / cur.length > 0.8) {
+              children[i - 1].remove();
+            }
+          }
+        };
+        dedup(clone);
+        clone.querySelectorAll('section, div').forEach(el => {
+          if (el.children && el.children.length > 2) dedup(el);
+        });
+
+        result.contentHtml = clone.innerHTML;
+
+        // --- Image extraction ---
+        const seen = new Set();
+        clone.querySelectorAll('img').forEach(img => {
+          const src = img.getAttribute('data-src')
+            || img.getAttribute('data-original')
+            || img.getAttribute('src');
+          if (src && !src.startsWith('data:') && !seen.has(src)) {
+            seen.add(src);
+            result.imageUrls.push(src);
+          }
+        });
+
+        return result;
+      })()
+    `);
+
+    // Determine Referer from URL for image downloads
+    let referer = '';
+    try {
+      const parsed = new URL(url);
+      referer = parsed.origin + '/';
+    } catch { /* ignore */ }
+
+    return downloadArticle(
+      {
+        title: data?.title || 'untitled',
+        author: data?.author,
+        publishTime: data?.publishTime,
+        sourceUrl: url,
+        contentHtml: data?.contentHtml || '',
+        imageUrls: data?.imageUrls,
+      },
+      {
+        output: kwargs.output,
+        downloadImages: kwargs['download-images'],
+        imageHeaders: referer ? { Referer: referer } : undefined,
+      },
+    );
+  },
+});

package/src/clis/xueqiu/danjuan-utils.test.ts

@@ -0,0 +1,49 @@
+import { describe, expect, it, vi } from 'vitest';
+import { fetchDanjuanAll } from './danjuan-utils.js';
+
+describe('fetchDanjuanAll', () => {
+  it('throws when no Danjuan accounts are visible', async () => {
+    const mockPage = {
+      evaluate: vi.fn().mockResolvedValue({ _emptyAccounts: true }),
+    } as any;
+
+    await expect(fetchDanjuanAll(mockPage)).rejects.toThrow('No fund accounts found');
+  });
+
+  it('throws when any account detail request fails', async () => {
+    const mockPage = {
+      evaluate: vi.fn().mockResolvedValue({
+        detailErrors: [
+          { accountName: '默认账户', accountId: 'acc-1', error: 403 },
+        ],
+      }),
+    } as any;
+
+    await expect(fetchDanjuanAll(mockPage)).rejects.toThrow(
+      'Failed to fetch Danjuan account details: 默认账户 (acc-1): 403',
+    );
+  });
+
+  it('returns the combined snapshot when all account details succeed', async () => {
+    const snapshot = {
+      asOf: '2026-03-25',
+      totalAssetAmount: 100,
+      totalAssetDailyGain: 1,
+      totalAssetHoldGain: 2,
+      totalAssetTotalGain: 3,
+      totalFundMarketValue: 80,
+      accounts: [{ accountId: 'acc-1', accountName: '默认账户' }],
+      holdings: [{ accountId: 'acc-1', fdCode: '000001', fdName: '示例基金' }],
+      detailErrors: [],
+    };
+    const mockPage = {
+      evaluate: vi.fn().mockResolvedValue(snapshot),
+    } as any;
+
+    await expect(fetchDanjuanAll(mockPage)).resolves.toMatchObject({
+      asOf: '2026-03-25',
+      accounts: [{ accountId: 'acc-1', accountName: '默认账户' }],
+      holdings: [{ accountId: 'acc-1', fdCode: '000001', fdName: '示例基金' }],
+    });
+  });
+});