@jackwener/opencli 1.7.4 → 1.7.5

package/clis/twitter/lists.js

@@ -1,62 +1,149 @@
-import { AuthRequiredError, SelectorError } from '@jackwener/opencli/errors';
 import { cli, Strategy } from '@jackwener/opencli/registry';
-import { isEmptyListsState, parseListCards } from './lists-parser.js';
+import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
 
-cli({
+const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
+const LISTS_QUERY_ID = '78UbkyXwXBD98IgUWXOy9g';
+const OPERATION_NAME = 'ListsManagementPageTimeline';
+
+const FEATURES = {
+    rweb_video_screen_enabled: false,
+    profile_label_improvements_pcf_label_in_post_enabled: true,
+    rweb_tipjar_consumption_enabled: true,
+    verified_phone_label_enabled: false,
+    creator_subscriptions_tweet_preview_api_enabled: true,
+    responsive_web_graphql_timeline_navigation_enabled: true,
+    responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+    premium_content_api_read_enabled: false,
+    communities_web_enable_tweet_community_results_fetch: true,
+    c9s_tweet_anatomy_moderator_badge_enabled: true,
+    responsive_web_grok_analyze_button_fetch_trends_enabled: false,
+    responsive_web_grok_analyze_post_followups_enabled: true,
+    responsive_web_jetfuel_frame: false,
+    responsive_web_grok_share_attachment_enabled: true,
+    articles_preview_enabled: true,
+    responsive_web_edit_tweet_api_enabled: true,
+    graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+    view_counts_everywhere_api_enabled: true,
+    longform_notetweets_consumption_enabled: true,
+    responsive_web_twitter_article_tweet_consumption_enabled: true,
+    tweet_awards_web_tipping_enabled: false,
+    responsive_web_grok_show_grok_translated_post: false,
+    responsive_web_grok_analysis_button_from_backend: false,
+    creator_subscriptions_quote_tweet_preview_enabled: false,
+    freedom_of_speech_not_reach_fetch_enabled: true,
+    standardized_nudges_misinfo: true,
+    tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+    longform_notetweets_rich_text_read_enabled: true,
+    longform_notetweets_inline_media_enabled: true,
+    responsive_web_grok_image_annotation_enabled: true,
+    responsive_web_enhance_cards_enabled: false,
+};
+
+function buildUrl(queryId) {
+    return `/i/api/graphql/${queryId}/${OPERATION_NAME}`
+        + `?features=${encodeURIComponent(JSON.stringify(FEATURES))}`;
+}
+
+export function extractListEntry(entry, seen) {
+    const list = entry?.content?.itemContent?.list
+        || entry?.content?.list
+        || entry?.item?.itemContent?.list;
+    if (!list) return null;
+    const id = list.id_str || list.id || '';
+    if (!id || seen.has(id)) return null;
+    seen.add(id);
+    const mode = typeof list.mode === 'string' && /private/i.test(list.mode) ? 'private' : 'public';
+    return {
+        id: String(id),
+        name: list.name || '',
+        members: String(list.member_count ?? 0),
+        followers: String(list.subscriber_count ?? 0),
+        mode,
+    };
+}
+
+export function parseListsManagement(data, seen) {
+    const lists = [];
+    const instructions = data?.data?.viewer?.list_management_timeline?.timeline?.instructions
+        || data?.data?.viewer_v2?.user_results?.result?.list_management_timeline?.timeline?.instructions
+        || data?.data?.list_management_timeline?.timeline?.instructions
+        || [];
+    for (const inst of instructions) {
+        for (const entry of inst.entries || []) {
+            const direct = extractListEntry(entry, seen);
+            if (direct) {
+                lists.push(direct);
+                continue;
+            }
+            for (const item of entry?.content?.items || []) {
+                const nested = extractListEntry(item, seen);
+                if (nested) lists.push(nested);
+            }
+        }
+    }
+    return lists;
+}
+
+export const command = cli({
     site: 'twitter',
     name: 'lists',
-    description: 'Get Twitter/X lists for a user',
+    description: 'Get Twitter/X lists for the logged-in user (owned + subscribed)',
     domain: 'x.com',
     strategy: Strategy.COOKIE,
     browser: true,
     args: [
-        { name: 'user', positional: true, type: 'string', required: false },
         { name: 'limit', type: 'int', default: 50 },
     ],
-    columns: ['name', 'members', 'followers', 'mode'],
+    columns: ['id', 'name', 'members', 'followers', 'mode'],
     func: async (page, kwargs) => {
-        let targetUser = kwargs.user;
-        if (!targetUser) {
-            await page.goto('https://x.com/home');
-            await page.wait({ selector: '[data-testid="primaryColumn"]' });
-            const href = await page.evaluate(`() => {
-            const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
-            return link ? link.getAttribute('href') : null;
-        }`);
-            if (!href) {
-                throw new AuthRequiredError('x.com', 'Could not find logged-in user profile link. Are you logged in?');
-            }
-            targetUser = href.replace('/', '');
-        }
-        await page.goto(`https://x.com/${targetUser}/lists`);
+        const limit = kwargs.limit || 50;
+        await page.goto('https://x.com');
         await page.wait(3);
-        const pageData = await page.evaluate(`() => {
-            const cards = [];
-            const seen = new Set();
-            for (const anchor of Array.from(document.querySelectorAll('a[href*="/i/lists/"]'))) {
-                const href = anchor.getAttribute('href') || '';
-                if (!/\\/i\\/lists\\/\\d+/.test(href) || seen.has(href)) continue;
-                seen.add(href);
-                const container = anchor.closest('[data-testid="cellInnerDiv"]') || anchor;
-                const text = (container.innerText || anchor.innerText || '').trim();
-                if (!text) continue;
-                cards.push({ href, text });
-            }
-            return {
-                cards,
-                pageText: document.body.innerText || '',
-            };
+        const ct0 = await page.evaluate(`() => {
+            return document.cookie.split(';').map(c => c.trim()).find(c => c.startsWith('ct0='))?.split('=')[1] || null;
         }`);
-        if (!pageData?.pageText) {
-            throw new SelectorError('Twitter lists', 'Empty page text');
-        }
-        const results = parseListCards(pageData.cards);
-        if (results.length === 0) {
-            if (isEmptyListsState(pageData.pageText)) {
-                return [];
-            }
-            throw new SelectorError('Twitter lists', `Could not parse list data`);
+        if (!ct0)
+            throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
+        const queryId = await page.evaluate(`async () => {
+            try {
+                const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json');
+                if (ghResp.ok) {
+                    const data = await ghResp.json();
+                    const entry = data['${OPERATION_NAME}'];
+                    if (entry && entry.queryId) return entry.queryId;
+                }
+            } catch {}
+            try {
+                const scripts = performance.getEntriesByType('resource')
+                    .filter(r => r.name.includes('client-web') && r.name.endsWith('.js'))
+                    .map(r => r.name);
+                for (const scriptUrl of scripts.slice(0, 15)) {
+                    try {
+                        const text = await (await fetch(scriptUrl)).text();
+                        const re = /queryId:"([A-Za-z0-9_-]+)"[^}]{0,200}operationName:"${OPERATION_NAME}"/;
+                        const m = text.match(re);
+                        if (m) return m[1];
+                    } catch {}
+                }
+            } catch {}
+            return null;
+        }`) || LISTS_QUERY_ID;
+        const headers = JSON.stringify({
+            'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`,
+            'X-Csrf-Token': ct0,
+            'X-Twitter-Auth-Type': 'OAuth2Session',
+            'X-Twitter-Active-User': 'yes',
+        });
+        const apiUrl = buildUrl(queryId);
+        const data = await page.evaluate(`async () => {
+            const r = await fetch(${JSON.stringify(apiUrl)}, { headers: ${headers}, credentials: 'include' });
+            return r.ok ? await r.json() : { error: r.status };
+        }`);
+        if (data?.error) {
+            throw new CommandExecutionError(`HTTP ${data.error}: Failed to fetch lists. queryId may have expired.`);
         }
-        return results.slice(0, kwargs.limit);
-    }
+        const seen = new Set();
+        const lists = parseListsManagement(data, seen);
+        return lists.slice(0, limit);
+    },
 });

package/clis/twitter/lists.test.js

@@ -1,50 +1,117 @@
 import { describe, expect, it } from 'vitest';
-import { isEmptyListsState, parseListCards } from './lists-parser.js';
+import { extractListEntry, parseListsManagement } from './lists.js';
 
 describe('twitter lists parser', () => {
-    it('parses english list cards without relying on page locale', () => {
-        const result = parseListCards([
-            {
-                href: '/i/lists/123',
-                text: `AI Researchers
-@jack
-124 Members 3.4K Followers
-Private`,
+    it('extracts a list entry with full metadata', () => {
+        const entry = {
+            content: {
+                itemContent: {
+                    list: {
+                        id_str: '1597593475389984769',
+                        name: 'Crypto',
+                        member_count: 44,
+                        subscriber_count: 8747,
+                        mode: 'Public',
+                    },
+                },
             },
-        ]);
-        expect(result).toEqual([
-            {
-                name: 'AI Researchers',
-                members: '124',
-                followers: '3.4K',
-                mode: 'private',
-            },
-        ]);
+        };
+        expect(extractListEntry(entry, new Set())).toEqual({
+            id: '1597593475389984769',
+            name: 'Crypto',
+            members: '44',
+            followers: '8747',
+            mode: 'public',
+        });
     });
 
-    it('parses chinese list cards without scanning document.body.innerText', () => {
-        const result = parseListCards([
-            {
-                href: '/i/lists/456',
-                text: `AI观察
-@jack
-321 位成员 8.8K 位关注者
-锁定列表`,
+    it('maps Private mode to private', () => {
+        const entry = {
+            content: {
+                itemContent: {
+                    list: {
+                        id_str: '2044679538156912976',
+                        name: 'AI & Agents',
+                        member_count: 15,
+                        subscriber_count: 0,
+                        mode: 'Private',
+                    },
+                },
             },
-        ]);
-        expect(result).toEqual([
-            {
-                name: 'AI观察',
-                members: '321',
-                followers: '8.8K',
-                mode: 'private',
+        };
+        expect(extractListEntry(entry, new Set())?.mode).toBe('private');
+    });
+
+    it('deduplicates by list id', () => {
+        const entry = {
+            content: { itemContent: { list: { id_str: '1', name: 'X' } } },
+        };
+        const seen = new Set();
+        expect(extractListEntry(entry, seen)).not.toBeNull();
+        expect(extractListEntry(entry, seen)).toBeNull();
+    });
+
+    it('returns null when no list payload is present', () => {
+        expect(extractListEntry({}, new Set())).toBeNull();
+        expect(extractListEntry({ content: { itemContent: {} } }, new Set())).toBeNull();
+    });
+
+    it('parses ListsManagementPageTimeline payload instructions', () => {
+        const payload = {
+            data: {
+                viewer: {
+                    list_management_timeline: {
+                        timeline: {
+                            instructions: [
+                                {
+                                    entries: [
+                                        {
+                                            entryId: 'owned-list-1',
+                                            content: {
+                                                itemContent: {
+                                                    list: { id_str: '1', name: 'Crypto', member_count: 44, subscriber_count: 8747, mode: 'Public' },
+                                                },
+                                            },
+                                        },
+                                        {
+                                            entryId: 'subscribed-list-2',
+                                            content: {
+                                                itemContent: {
+                                                    list: { id_str: '2', name: 'AI', member_count: 15, subscriber_count: 0, mode: 'Private' },
+                                                },
+                                            },
+                                        },
+                                    ],
+                                },
+                            ],
+                        },
+                    },
+                },
             },
-        ]);
+        };
+        const result = parseListsManagement(payload, new Set());
+        expect(result).toHaveLength(2);
+        expect(result[0]).toMatchObject({ id: '1', name: 'Crypto', mode: 'public' });
+        expect(result[1]).toMatchObject({ id: '2', name: 'AI', mode: 'private' });
     });
 
-    it('detects empty state text in english and chinese', () => {
-        expect(isEmptyListsState(`@jack hasn't created any Lists yet`)).toBe(true);
-        expect(isEmptyListsState('这个账号还没有创建任何列表')).toBe(true);
-        expect(isEmptyListsState('AI Researchers 124 Members')).toBe(false);
+    it('returns empty list for malformed payload', () => {
+        expect(parseListsManagement({}, new Set())).toEqual([]);
+        expect(parseListsManagement({ data: {} }, new Set())).toEqual([]);
+    });
+
+    it('dedupes across repeated entries', () => {
+        const entryA = { content: { itemContent: { list: { id_str: '1', name: 'A' } } } };
+        const payload = {
+            data: {
+                viewer: {
+                    list_management_timeline: {
+                        timeline: { instructions: [{ entries: [entryA, entryA] }] },
+                    },
+                },
+            },
+        };
+        const result = parseListsManagement(payload, new Set());
+        expect(result).toHaveLength(1);
     });
 });

package/clis/wanfang/search.js

@@ -0,0 +1,66 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { clampInt, requireNonEmptyQuery } from '../_shared/common.js';
+
+cli({
+    site: 'wanfang',
+    name: 'search',
+    description: '万方数据论文搜索',
+    domain: 's.wanfangdata.com.cn',
+    strategy: Strategy.PUBLIC,
+    browser: true,
+    args: [
+        { name: 'query', positional: true, required: true, help: '搜索关键词' },
+        { name: 'limit', type: 'int', default: 10, help: '返回结果数量 (max 20)' },
+    ],
+    columns: ['rank', 'title', 'authors', 'source', 'year', 'type', 'cited', 'url'],
+    navigateBefore: false,
+    func: async (page, kwargs) => {
+        const limit = clampInt(kwargs.limit, 10, 1, 20);
+        const query = requireNonEmptyQuery(kwargs.query);
+        await page.goto(`https://s.wanfangdata.com.cn/paper?q=${encodeURIComponent(query)}`);
+        await page.wait(5);
+        const data = await page.evaluate(`
+      (async () => {
+        const normalize = v => (v || '').replace(/\\s+/g, ' ').trim();
+        for (let i = 0; i < 30; i++) {
+          if (document.querySelectorAll('span.title').length > 0) break;
+          await new Promise(r => setTimeout(r, 500));
+        }
+        const results = [];
+        for (const titleSpan of document.querySelectorAll('span.title')) {
+          const title = normalize(titleSpan.textContent);
+          if (!title || title.length < 3) continue;
+
+          let container = titleSpan.parentElement;
+          for (let i = 0; i < 6; i++) {
+            if (!container?.parentElement || container.parentElement.tagName === 'BODY') break;
+            if (container.querySelectorAll('span.title').length >= 1 && container.querySelectorAll('span.authors').length >= 1) break;
+            container = container.parentElement;
+          }
+          if (!container) continue;
+
+          const id = normalize(container.querySelector('span.title-id-hidden')?.textContent);
+          const url = id ? 'https://d.wanfangdata.com.cn/' + id : '';
+          const authors = Array.from(container.querySelectorAll('span.authors'))
+            .map((item) => normalize(item.textContent))
+            .filter(Boolean)
+            .join(', ')
+            .slice(0, 80);
+          const type = normalize(container.querySelector('span.essay-type')?.textContent);
+          const source = normalize(container.querySelector('span.periodical, span.source')?.textContent);
+
+          let year = normalize(container.querySelector('span.year, span.date')?.textContent);
+          if (!year) year = (container.textContent || '').match(/(19|20)\\d{2}/)?.[0] || '';
+
+          const citedText = normalize(container.querySelector('.stat-item.quote, [class*=\"quote\"]')?.textContent);
+          const cited = citedText.match(/(\\d+)/)?.[1] || '0';
+
+          results.push({ rank: results.length + 1, title, authors, source, year, type, cited, url });
+          if (results.length >= ${limit}) break;
+        }
+        return results;
+      })()
+    `);
+        return Array.isArray(data) ? data : [];
+    },
+});

package/clis/wanfang/search.test.js

@@ -0,0 +1,23 @@
+import { describe, expect, it, vi } from 'vitest';
+import { getRegistry } from '@jackwener/opencli/registry';
+import './search.js';
+
+describe('wanfang search command', () => {
+    const command = getRegistry().get('wanfang/search');
+
+    it('registers as a public browser command', () => {
+        expect(command).toBeDefined();
+        expect(command.site).toBe('wanfang');
+        expect(command.strategy).toBe('public');
+        expect(command.browser).toBe(true);
+    });
+
+    it('rejects empty queries before browser navigation', async () => {
+        const page = { goto: vi.fn() };
+        await expect(command.func(page, { query: '   ' })).rejects.toMatchObject({
+            name: 'ArgumentError',
+            code: 'ARGUMENT',
+        });
+        expect(page.goto).not.toHaveBeenCalled();
+    });
+});

package/clis/web/read.js

@@ -27,7 +27,7 @@ cli({
         { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
         { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' },
     ],
-    columns: ['title', 'author', 'publish_time', 'status', 'size'],
+    columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'],
     func: async (page, kwargs) => {
         const url = kwargs.url;
         const waitSeconds = kwargs.wait ?? 3;

package/clis/weixin/download.js

@@ -179,12 +179,12 @@ cli({
         { name: 'output', default: './weixin-articles', help: 'Output directory' },
         { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
     ],
-    columns: ['title', 'author', 'publish_time', 'status', 'size'],
+    columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'],
     func: async (page, kwargs) => {
         const rawUrl = kwargs.url;
         const url = normalizeWechatUrl(rawUrl);
         if (!url.startsWith('https://mp.weixin.qq.com/')) {
-            return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-' }];
+            return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-', saved: '-' }];
         }
         // Navigate and wait for content to load
         await page.goto(url);
@@ -297,6 +297,7 @@ cli({
                     publish_time: '-',
                     status: 'failed — verification required in WeChat browser page',
                     size: '-',
+                    saved: '-',
                 }];
         }
         return downloadArticle({