@jackwener/opencli 1.7.8 → 1.7.10

package/clis/web/read.test.js

@@ -1,4 +1,5 @@
 import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { JSDOM } from 'jsdom';
 
 const { mockDownloadArticle } = vi.hoisted(() => ({
     mockDownloadArticle: vi.fn(),
@@ -12,19 +13,30 @@ const { __test__ } = await import('./read.js');
 
 describe('web/read stdout behavior', () => {
     const read = __test__.command;
-    const page = {
-        goto: vi.fn().mockResolvedValue(undefined),
-        wait: vi.fn().mockResolvedValue(undefined),
-        evaluate: vi.fn().mockResolvedValue({
+    const extractedArticle = {
             title: 'Example Article',
             author: 'Author',
             publishTime: '2026-04-22',
             contentHtml: '<p>hello</p>',
             imageUrls: ['https://example.com/a.jpg'],
-        }),
+            diagnostics: {
+                url: 'https://example.com/article',
+                frames: [],
+                emptyContainers: [],
+                includedFrameCount: 0,
+            },
+        };
+    const page = {
+        goto: vi.fn().mockResolvedValue(undefined),
+        wait: vi.fn().mockResolvedValue(undefined),
+        evaluate: vi.fn().mockResolvedValue(extractedArticle),
+        startNetworkCapture: vi.fn().mockResolvedValue(true),
+        readNetworkCapture: vi.fn().mockResolvedValue([]),
     };
 
     beforeEach(() => {
+        vi.restoreAllMocks();
+        vi.useRealTimers();
         mockDownloadArticle.mockReset();
         mockDownloadArticle.mockResolvedValue([{
             title: 'Example Article',
@@ -37,6 +49,11 @@ describe('web/read stdout behavior', () => {
         page.goto.mockClear();
         page.wait.mockClear();
         page.evaluate.mockClear();
+        page.evaluate.mockResolvedValue(extractedArticle);
+        page.startNetworkCapture.mockClear();
+        page.startNetworkCapture.mockResolvedValue(true);
+        page.readNetworkCapture.mockClear();
+        page.readNetworkCapture.mockResolvedValue([]);
     });
 
     it('returns null in --stdout mode so the CLI does not append result rows to stdout', async () => {
@@ -58,6 +75,7 @@ describe('web/read stdout behavior', () => {
                 stdout: true,
             }),
         );
+        expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "same-origin"');
     });
 
     it('still returns the saved-row payload when writing to disk', async () => {
@@ -73,4 +91,202 @@ describe('web/read stdout behavior', () => {
 
         expect(result).toBe(rows);
     });
+
+    it('waits for a selector in the main document or same-origin iframes before extracting', async () => {
+        page.evaluate
+            .mockResolvedValueOnce({ ok: true, scope: 'iframe', url: 'https://example.com/frame' })
+            .mockResolvedValueOnce(extractedArticle);
+
+        await read.func(page, {
+            url: 'https://example.com/article',
+            output: '/tmp/out',
+            'download-images': false,
+            'wait-for': '#gridDatas li',
+            wait: 7,
+            stdout: false,
+        });
+
+        expect(page.wait).not.toHaveBeenCalled();
+        expect(page.evaluate).toHaveBeenCalledTimes(2);
+        expect(page.evaluate.mock.calls[0]?.[0]).toContain('"#gridDatas li"');
+        expect(page.evaluate.mock.calls[0]?.[0]).toContain('sameOriginFrameDocs');
+        expect(page.evaluate.mock.calls[1]?.[0]).toContain('const frameMode = "same-origin"');
+    });
+
+    it('throws a clear error when --wait-for times out', async () => {
+        page.evaluate.mockResolvedValueOnce({ ok: false, timedOut: true, selector: '#missing' });
+
+        await expect(read.func(page, {
+            url: 'https://example.com/article',
+            output: '/tmp/out',
+            'download-images': false,
+            'wait-for': '#missing',
+            wait: 1,
+            stdout: false,
+        })).rejects.toThrow('Timed out waiting for selector "#missing"');
+
+        expect(mockDownloadArticle).not.toHaveBeenCalled();
+    });
+
+    it('starts network capture and writes diagnostics in diagnose mode', async () => {
+        const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true);
+        page.readNetworkCapture.mockResolvedValueOnce([{
+            method: 'POST',
+            url: 'https://example.com/api/data',
+            responseStatus: 200,
+            responseContentType: 'application/json',
+            responsePreview: '{"ok":true}',
+        }]);
+
+        await read.func(page, {
+            url: 'https://example.com/article',
+            output: '/tmp/out',
+            'download-images': false,
+            diagnose: true,
+            wait: 0,
+            stdout: false,
+        });
+
+        expect(page.startNetworkCapture).toHaveBeenCalledWith('');
+        expect(page.readNetworkCapture).toHaveBeenCalledTimes(1);
+        expect(stderr).toHaveBeenCalledWith(expect.stringContaining('[web-read diagnose]'));
+        expect(stderr).toHaveBeenCalledWith(expect.stringContaining('POST 200 application/json https://example.com/api/data'));
+    });
+
+    it('passes --frames none into the extractor', async () => {
+        await read.func(page, {
+            url: 'https://example.com/article',
+            output: '/tmp/out',
+            'download-images': false,
+            frames: 'none',
+            stdout: false,
+        });
+
+        expect(page.evaluate.mock.calls[0]?.[0]).toContain('const frameMode = "none"');
+    });
+
+    it('fails fast when --wait-until networkidle is requested but capture is unavailable', async () => {
+        page.startNetworkCapture.mockResolvedValue(false);
+
+        await expect(read.func(page, {
+            url: 'https://example.com/article',
+            output: '/tmp/out',
+            'download-images': false,
+            'wait-until': 'networkidle',
+            wait: 2,
+            stdout: false,
+        })).rejects.toThrow('Network capture is unavailable');
+
+        expect(page.wait).not.toHaveBeenCalled();
+        expect(mockDownloadArticle).not.toHaveBeenCalled();
+    });
+
+    it('fails fast when network traffic never settles before the networkidle timeout', async () => {
+        vi.useFakeTimers();
+        page.readNetworkCapture.mockResolvedValue([{
+            method: 'POST',
+            url: 'https://example.com/api/data',
+            responseStatus: 200,
+            responseContentType: 'application/json',
+            responsePreview: '{"ok":true}',
+        }]);
+
+        const pending = expect(read.func(page, {
+            url: 'https://example.com/article',
+            output: '/tmp/out',
+            'download-images': false,
+            'wait-until': 'networkidle',
+            wait: 1,
+            stdout: false,
+        })).rejects.toThrow('Timed out waiting for network idle after 1s');
+
+        await vi.advanceTimersByTimeAsync(2000);
+
+        await pending;
+        expect(mockDownloadArticle).not.toHaveBeenCalled();
+    });
+});
+
+describe('web/read render-aware helpers', () => {
+    it('merges accessible same-origin iframe bodies into the extracted HTML', () => {
+        const dom = new JSDOM(`
+          <main>
+            <h1>Shell</h1>
+            <iframe id="MF" src="/frame.html"></iframe>
+          </main>
+        `, { url: 'https://example.com/main.html', runScripts: 'outside-only' });
+        const frame = dom.window.document.querySelector('iframe');
+        frame.contentDocument.open();
+        frame.contentDocument.write('<body><table id="gridHd"><tr><th>Name</th></tr></table><ul id="gridDatas"><li>Station A 42</li></ul></body>');
+        frame.contentDocument.close();
+
+        const result = dom.window.eval(__test__.buildRenderAwareExtractorJs({ frames: 'same-origin' }));
+
+        expect(result.diagnostics.includedFrameCount).toBe(1);
+        expect(result.contentHtml).toContain('data-opencli-iframe-source="https://example.com/frame.html"');
+        expect(result.contentHtml).toContain('来自 iframe: https://example.com/frame.html');
+        expect(result.contentHtml).toContain('Station A 42');
+    });
+
+    it('marks API-like network entries as interesting and ignores static assets', () => {
+        expect(__test__.isInterestingNetworkEntry({
+            method: 'POST',
+            url: 'https://example.com/GJZ/Ajax/Publish.ashx',
+            status: 200,
+            contentType: 'text/html',
+            size: 100,
+            bodyTruncated: false,
+        })).toBe(true);
+        expect(__test__.isInterestingNetworkEntry({
+            method: 'POST',
+            url: 'https://example.com/GJZ/Ajax/Publish.ashx',
+            status: 200,
+            contentType: 'application/json',
+            size: 100,
+            bodyTruncated: false,
+        })).toBe(true);
+        expect(__test__.isInterestingNetworkEntry({
+            method: 'GET',
+            url: 'https://example.com/app.js',
+            status: 200,
+            contentType: 'application/javascript',
+            size: 100,
+            bodyTruncated: false,
+        })).toBe(false);
+    });
+
+    it('formats frame and XHR diagnostics for shell pages', () => {
+        const output = __test__.formatDiagnostics({
+            diagnostics: {
+                url: 'https://example.com/main.html',
+                includedFrameCount: 1,
+                frames: [{
+                    index: 0,
+                    src: 'https://example.com/frame.html',
+                    sameOrigin: true,
+                    accessible: true,
+                    textLength: 42,
+                }],
+                emptyContainers: [{
+                    scope: 'iframe',
+                    url: 'https://example.com/frame.html',
+                    tag: 'ul',
+                    id: 'gridDatas',
+                    className: '',
+                }],
+            },
+        }, [{
+            method: 'POST',
+            url: 'https://example.com/GJZ/Ajax/Publish.ashx',
+            status: 200,
+            contentType: 'application/json',
+            size: 64,
+            bodyTruncated: false,
+        }], true);
+
+        expect(output).toContain('frames: 1, included_same_origin: 1');
+        expect(output).toContain('[frame 0] same-origin accessible text=42 https://example.com/frame.html');
+        expect(output).toContain('iframe: ul#gridDatas (https://example.com/frame.html)');
+        expect(output).toContain('POST 200 application/json https://example.com/GJZ/Ajax/Publish.ashx');
+    });
 });

package/clis/weibo/favorites.js

@@ -0,0 +1,169 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { ArgumentError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
+import { getSelfUid } from './utils.js';
+
+const DEFAULT_LIMIT = 20;
+const MAX_LIMIT = 50;
+
+function parsePositiveInt(value, name, defaultValue) {
+  const raw = value ?? defaultValue;
+  const number = Number(raw);
+  if (!Number.isInteger(number) || number <= 0) {
+    throw new ArgumentError(`weibo favorites ${name} must be a positive integer`);
+  }
+  if (number > MAX_LIMIT) {
+    throw new ArgumentError(`weibo favorites ${name} must be <= ${MAX_LIMIT}`);
+  }
+  return number;
+}
+
+function parseFavoriteCard(card, favUrl) {
+  const raw = String(card?.text ?? '');
+  const lines = raw.split('\n');
+
+  let author = '';
+  let time = '';
+  let source = '';
+  let content = '';
+  let likes = '0';
+  let comments = '0';
+  let reposts = '0';
+
+  for (const line of lines) {
+    const t = line.trim();
+    if (!t || t === '添加') continue;
+
+    if (!time && /\d+小时前|\d+分钟前|\d+秒前|昨天|前天|\d{1,2}:\d{2}/.test(t)) {
+      time = t;
+      continue;
+    }
+
+    if (t.startsWith('来自')) {
+      source = t;
+      continue;
+    }
+
+    if (content) {
+      const n = Number.parseInt(t, 10);
+      if (!Number.isNaN(n) && n > 0 && n < 1_000_000 && t === String(n)) {
+        if (likes === '0') likes = t;
+        else if (comments === '0') comments = t;
+        else if (reposts === '0') reposts = t;
+        continue;
+      }
+    }
+
+    if (!author && t.length < 40) {
+      author = t;
+      continue;
+    }
+
+    if (!content && author) {
+      content = t;
+      continue;
+    }
+
+    if (content) content += ` ${t}`;
+  }
+
+  if (!content || !author) return null;
+
+  return {
+    author,
+    text: content.substring(0, 300),
+    time,
+    source,
+    likes,
+    comments,
+    reposts,
+    url: card?.url || favUrl,
+  };
+}
+
+function dedupeFavorites(items, favUrl) {
+  const seen = new Set();
+  const result = [];
+  for (const item of items) {
+    const key = item.url && item.url !== favUrl
+      ? item.url
+      : `${item.author}\n${item.text}\n${item.time}`;
+    if (seen.has(key)) continue;
+    seen.add(key);
+    result.push(item);
+  }
+  return result;
+}
+
+cli({
+  site: 'weibo',
+  name: 'favorites',
+  description: '我的微博收藏列表',
+  domain: 'weibo.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'limit', type: 'int', default: 20, help: '数量（最多50）' },
+  ],
+  columns: ['author', 'text', 'time', 'source', 'likes', 'comments', 'reposts', 'url'],
+  func: async (page, kwargs) => {
+    const limit = parsePositiveInt(kwargs.limit, 'limit', DEFAULT_LIMIT);
+
+    await page.goto('https://weibo.com');
+    await page.wait(2);
+
+    const uid = await getSelfUid(page);
+
+    const favUrl = 'https://www.weibo.com/u/page/fav/' + uid;
+
+    await page.goto(favUrl);
+    await page.wait(4);
+
+    for (let i = 0; i < 3; i++) {
+      await page.evaluate('() => window.scrollBy(0, 800)');
+      await page.wait(1);
+    }
+
+    const rawData = await page.evaluate(`
+      (() => {
+        const scrollers = document.querySelectorAll('.wbpro-scroller-item, .vue-recycle-scroller__item-view');
+        const out = [];
+        for (const s of scrollers) {
+          // Use textContent to preserve newlines, then split by \n
+          const bodyEl = s.querySelector('[class*="_body_"]') || s.querySelector('.wbpro-item-body') || s;
+          // innerText preserves newlines between block elements (unlike textContent)
+          const rawText = bodyEl.innerText || s.innerText || '';
+
+          let postUrl = '';
+          const anchors = s.querySelectorAll('a[href]');
+          for (const a of anchors) {
+            const m = String(a.href).match(/weibo\\.com\\/(\\d+)\\/([a-zA-Z0-9]+)/);
+            if (m) { postUrl = 'https://weibo.com/' + m[1] + '/' + m[2]; break; }
+          }
+
+          if (rawText.length > 20) out.push({ text: rawText, url: postUrl });
+          if (out.length >= ${limit}) break;
+        }
+        return out;
+      })()
+    `);
+
+    if (!Array.isArray(rawData) || rawData.length === 0) {
+      throw new EmptyResultError('weibo favorites', 'No favorites were visible on the favorites page');
+    }
+
+    const items = rawData
+      .map(card => parseFavoriteCard(card, favUrl))
+      .filter(Boolean);
+
+    const uniqueItems = dedupeFavorites(items, favUrl);
+    if (uniqueItems.length === 0) {
+      throw new CommandExecutionError('Failed to parse visible Weibo favorites');
+    }
+    return uniqueItems.slice(0, limit);
+  },
+});
+
+export const __test__ = {
+  parseFavoriteCard,
+  parsePositiveInt,
+  dedupeFavorites,
+};

package/clis/weibo/favorites.test.js

@@ -0,0 +1,114 @@
+import { describe, expect, it, vi } from 'vitest';
+import { getRegistry } from '@jackwener/opencli/registry';
+import { ArgumentError, AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
+
+import './favorites.js';
+
+function makePage(evaluateResults = []) {
+  const queue = [...evaluateResults];
+  const evaluate = vi.fn(async (script) => {
+    if (String(script).includes('window.scrollBy')) return undefined;
+    return queue.length ? queue.shift() : [];
+  });
+
+  return {
+    goto: vi.fn().mockResolvedValue(undefined),
+    wait: vi.fn().mockResolvedValue(undefined),
+    evaluate,
+  };
+}
+
+describe('weibo favorites command', () => {
+  const getCommand = () => getRegistry().get('weibo/favorites');
+
+  it('registers as a JS adapter and parses visible favorites', async () => {
+    const command = getCommand();
+    expect(command?.func).toBeTypeOf('function');
+
+    const page = makePage([
+      '123456',
+      [
+        {
+          text: [
+            '作者A',
+            '昨天 12:00',
+            '来自 iPhone',
+            '这是一条收藏微博',
+            '12',
+            '3',
+            '2',
+          ].join('\n'),
+          url: 'https://weibo.com/123/AbCd1',
+        },
+      ],
+    ]);
+
+    const result = await command.func(page, { limit: 10 });
+
+    expect(result).toEqual([
+      {
+        author: '作者A',
+        text: '这是一条收藏微博',
+        time: '昨天 12:00',
+        source: '来自 iPhone',
+        likes: '12',
+        comments: '3',
+        reposts: '2',
+        url: 'https://weibo.com/123/AbCd1',
+      },
+    ]);
+    expect(page.goto).toHaveBeenCalledWith('https://weibo.com');
+    expect(page.goto).toHaveBeenCalledWith('https://www.weibo.com/u/page/fav/123456');
+  });
+
+  it('throws AuthRequiredError when uid cannot be resolved', async () => {
+    const command = getCommand();
+    const page = makePage([null, null]);
+
+    await expect(command.func(page, { limit: 10 })).rejects.toBeInstanceOf(AuthRequiredError);
+  });
+
+  it('validates limit before navigation', async () => {
+    const command = getCommand();
+    const page = makePage();
+
+    await expect(command.func(page, { limit: 0 })).rejects.toBeInstanceOf(ArgumentError);
+    await expect(command.func(page, { limit: 51 })).rejects.toBeInstanceOf(ArgumentError);
+    expect(page.goto).not.toHaveBeenCalled();
+  });
+
+  it('throws EmptyResultError when no favorite cards are visible', async () => {
+    const command = getCommand();
+    const page = makePage(['123456', []]);
+
+    await expect(command.func(page, { limit: 10 })).rejects.toBeInstanceOf(EmptyResultError);
+  });
+
+  it('throws CommandExecutionError when visible cards cannot be parsed', async () => {
+    const command = getCommand();
+    const page = makePage(['123456', [{ text: '添加\n昨天', url: '' }]]);
+
+    await expect(command.func(page, { limit: 10 })).rejects.toBeInstanceOf(CommandExecutionError);
+  });
+
+  it('deduplicates repeated cards and applies the requested limit', async () => {
+    const command = getCommand();
+    const rawCard = {
+      text: '作者A\n内容A',
+      url: 'https://weibo.com/123/AbCd1',
+    };
+    const page = makePage([
+      '123456',
+      [
+        rawCard,
+        rawCard,
+        { text: '作者B\n内容B', url: 'https://weibo.com/123/AbCd2' },
+      ],
+    ]);
+
+    const result = await command.func(page, { limit: 1 });
+
+    expect(result).toHaveLength(1);
+    expect(result[0].author).toBe('作者A');
+  });
+});