@jackwener/opencli 1.7.4 → 1.7.6

package/clis/twitter/tweets.js

@@ -0,0 +1,218 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { AuthRequiredError, CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
+import { resolveTwitterQueryId, sanitizeQueryId } from './shared.js';
+
+const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
+const USER_TWEETS_QUERY_ID = '6fWQaBPK51aGyC_VC7t9GQ';
+const USER_BY_SCREEN_NAME_QUERY_ID = 'IGgvgiOx4QZndDHuD3x9TQ';
+
+const USER_TWEETS_FEATURES = {
+    rweb_video_screen_enabled: false,
+    payments_enabled: false,
+    profile_label_improvements_pcf_label_in_post_enabled: true,
+    rweb_tipjar_consumption_enabled: true,
+    verified_phone_label_enabled: false,
+    creator_subscriptions_tweet_preview_api_enabled: true,
+    responsive_web_graphql_timeline_navigation_enabled: true,
+    responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+    premium_content_api_read_enabled: false,
+    communities_web_enable_tweet_community_results_fetch: true,
+    c9s_tweet_anatomy_moderator_badge_enabled: true,
+    responsive_web_grok_analyze_button_fetch_trends_enabled: false,
+    responsive_web_grok_analyze_post_followups_enabled: true,
+    responsive_web_jetfuel_frame: true,
+    responsive_web_grok_share_attachment_enabled: true,
+    responsive_web_grok_annotations_enabled: true,
+    articles_preview_enabled: true,
+    responsive_web_edit_tweet_api_enabled: true,
+    graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+    view_counts_everywhere_api_enabled: true,
+    longform_notetweets_consumption_enabled: true,
+    responsive_web_twitter_article_tweet_consumption_enabled: true,
+    tweet_awards_web_tipping_enabled: false,
+    content_disclosure_indicator_enabled: true,
+    content_disclosure_ai_generated_indicator_enabled: true,
+    responsive_web_grok_show_grok_translated_post: false,
+    responsive_web_grok_analysis_button_from_backend: true,
+    post_ctas_fetch_enabled: false,
+    freedom_of_speech_not_reach_fetch_enabled: true,
+    standardized_nudges_misinfo: true,
+    tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+    longform_notetweets_rich_text_read_enabled: true,
+    longform_notetweets_inline_media_enabled: true,
+    responsive_web_grok_image_annotation_enabled: true,
+    responsive_web_grok_imagine_annotation_enabled: true,
+    responsive_web_grok_community_note_auto_translation_is_enabled: false,
+    responsive_web_enhance_cards_enabled: false,
+};
+
+const USER_BY_SCREEN_NAME_FEATURES = {
+    hidden_profile_subscriptions_enabled: true,
+    rweb_tipjar_consumption_enabled: true,
+    responsive_web_graphql_exclude_directive_enabled: true,
+    verified_phone_label_enabled: false,
+    subscriptions_verification_info_is_identity_verified_enabled: true,
+    subscriptions_verification_info_verified_since_enabled: true,
+    highlights_tweets_tab_ui_enabled: true,
+    responsive_web_twitter_article_notes_tab_enabled: true,
+    subscriptions_feature_can_gift_premium: true,
+    creator_subscriptions_tweet_preview_api_enabled: true,
+    responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+    responsive_web_graphql_timeline_navigation_enabled: true,
+};
+
+function buildUserTweetsUrl(queryId, userId, count, cursor) {
+    const vars = {
+        userId,
+        count,
+        includePromotedContent: false,
+        withQuickPromoteEligibilityTweetFields: true,
+        withVoice: true,
+    };
+    if (cursor) vars.cursor = cursor;
+    return `/i/api/graphql/${queryId}/UserTweets`
+        + `?variables=${encodeURIComponent(JSON.stringify(vars))}`
+        + `&features=${encodeURIComponent(JSON.stringify(USER_TWEETS_FEATURES))}`;
+}
+
+function buildUserByScreenNameUrl(queryId, screenName) {
+    const vars = { screen_name: screenName, withSafetyModeUserFields: true };
+    return `/i/api/graphql/${queryId}/UserByScreenName`
+        + `?variables=${encodeURIComponent(JSON.stringify(vars))}`
+        + `&features=${encodeURIComponent(JSON.stringify(USER_BY_SCREEN_NAME_FEATURES))}`;
+}
+
+function extractTweet(result, seen) {
+    if (!result) return null;
+    const tw = result.tweet || result;
+    const legacy = tw.legacy || {};
+    if (!tw.rest_id || seen.has(tw.rest_id)) return null;
+    seen.add(tw.rest_id);
+    const user = tw.core?.user_results?.result;
+    const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';
+    const displayName = user?.legacy?.name || user?.core?.name || '';
+    const noteText = tw.note_tweet?.note_tweet_results?.result?.text;
+    const isRetweet = Boolean(legacy.retweeted_status_result || legacy.full_text?.startsWith('RT @'));
+    return {
+        id: tw.rest_id,
+        author: screenName,
+        name: displayName,
+        text: noteText || legacy.full_text || '',
+        likes: legacy.favorite_count || 0,
+        retweets: legacy.retweet_count || 0,
+        replies: legacy.reply_count || 0,
+        views: Number(tw.views?.count) || 0,
+        is_retweet: isRetweet,
+        created_at: legacy.created_at || '',
+        url: `https://x.com/${screenName}/status/${tw.rest_id}`,
+    };
+}
+
+function parseUserTweets(data, seen) {
+    const tweets = [];
+    let nextCursor = null;
+    const instructions = data?.data?.user?.result?.timeline_v2?.timeline?.instructions
+        || data?.data?.user?.result?.timeline?.timeline?.instructions
+        || [];
+    for (const inst of instructions) {
+        if (inst.type === 'TimelinePinEntry') continue;
+        for (const entry of inst.entries || []) {
+            const content = entry.content;
+            if (content?.entryType === 'TimelineTimelineCursor' || content?.__typename === 'TimelineTimelineCursor') {
+                if (content.cursorType === 'Bottom' || content.cursorType === 'ShowMore') nextCursor = content.value;
+                continue;
+            }
+            if (entry.entryId?.startsWith('cursor-bottom-') || entry.entryId?.startsWith('cursor-showMore-')) {
+                nextCursor = content?.value || content?.itemContent?.value || nextCursor;
+                continue;
+            }
+            const direct = extractTweet(content?.itemContent?.tweet_results?.result, seen);
+            if (direct) {
+                tweets.push(direct);
+                continue;
+            }
+            for (const item of content?.items || []) {
+                const nested = extractTweet(item.item?.itemContent?.tweet_results?.result, seen);
+                if (nested) tweets.push(nested);
+            }
+        }
+    }
+    return { tweets, nextCursor };
+}
+
+cli({
+    site: 'twitter',
+    name: 'tweets',
+    description: "Fetch a Twitter user's most recent tweets (chronological, excludes pinned)",
+    domain: 'x.com',
+    strategy: Strategy.COOKIE,
+    browser: true,
+    args: [
+        { name: 'username', type: 'string', positional: true, required: true, help: 'Twitter screen name (with or without @)' },
+        { name: 'limit', type: 'int', default: 20, help: 'Max tweets to return' },
+    ],
+    columns: ['author', 'created_at', 'is_retweet', 'text', 'likes', 'retweets', 'replies', 'views', 'url'],
+    func: async (page, kwargs) => {
+        const limit = Math.max(1, Math.min(200, kwargs.limit || 20));
+        const username = String(kwargs.username || '').replace(/^@/, '').trim();
+        if (!username) throw new CommandExecutionError('username is required');
+
+        await page.goto('https://x.com');
+        await page.wait(3);
+
+        const ct0 = await page.evaluate(`() => {
+      return document.cookie.split(';').map(c => c.trim()).find(c => c.startsWith('ct0='))?.split('=')[1] || null;
+    }`);
+        if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
+
+        const userTweetsQueryId = await resolveTwitterQueryId(page, 'UserTweets', USER_TWEETS_QUERY_ID);
+        const userByScreenNameQueryId = await resolveTwitterQueryId(page, 'UserByScreenName', USER_BY_SCREEN_NAME_QUERY_ID);
+
+        const headers = JSON.stringify({
+            'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`,
+            'X-Csrf-Token': ct0,
+            'X-Twitter-Auth-Type': 'OAuth2Session',
+            'X-Twitter-Active-User': 'yes',
+        });
+
+        const ubsUrl = buildUserByScreenNameUrl(userByScreenNameQueryId, username);
+        const userId = await page.evaluate(`async () => {
+      const resp = await fetch("${ubsUrl}", { headers: ${headers}, credentials: 'include' });
+      if (!resp.ok) return null;
+      const d = await resp.json();
+      return d?.data?.user?.result?.rest_id || null;
+    }`);
+        if (!userId) throw new CommandExecutionError(`Could not resolve @${username}`);
+
+        const seen = new Set();
+        const all = [];
+        let cursor = null;
+        for (let i = 0; i < 5 && all.length < limit; i++) {
+            const fetchCount = Math.min(100, limit - all.length + 10);
+            const url = buildUserTweetsUrl(userTweetsQueryId, userId, fetchCount, cursor);
+            const data = await page.evaluate(`async () => {
+        const r = await fetch("${url}", { headers: ${headers}, credentials: 'include' });
+        return r.ok ? await r.json() : { error: r.status };
+      }`);
+            if (data?.error) {
+                if (all.length === 0) throw new CommandExecutionError(`HTTP ${data.error}: UserTweets fetch failed — queryId may have expired`);
+                break;
+            }
+            const { tweets, nextCursor } = parseUserTweets(data, seen);
+            all.push(...tweets);
+            if (!nextCursor || nextCursor === cursor) break;
+            cursor = nextCursor;
+        }
+
+        if (all.length === 0) throw new EmptyResultError(`@${username} has no recent tweets`, 'Account may be private or suspended');
+        return all.slice(0, limit);
+    },
+});
+
+export const __test__ = {
+    sanitizeQueryId,
+    buildUserTweetsUrl,
+    buildUserByScreenNameUrl,
+    extractTweet,
+    parseUserTweets,
+};

package/clis/twitter/tweets.test.js

@@ -0,0 +1,125 @@
+import { describe, expect, it } from 'vitest';
+import { getRegistry } from '@jackwener/opencli/registry';
+import { __test__ } from './tweets.js';
+
+describe('twitter tweets helpers', () => {
+    it('registers is_retweet in the default columns', () => {
+        const cmd = getRegistry().get('twitter/tweets');
+        expect(cmd?.columns).toEqual(['author', 'created_at', 'is_retweet', 'text', 'likes', 'retweets', 'replies', 'views', 'url']);
+    });
+
+    it('falls back when queryId contains unsafe characters', () => {
+        expect(__test__.sanitizeQueryId('safe_Query-123', 'fallback')).toBe('safe_Query-123');
+        expect(__test__.sanitizeQueryId('bad"id', 'fallback')).toBe('fallback');
+        expect(__test__.sanitizeQueryId('bad/id', 'fallback')).toBe('fallback');
+        expect(__test__.sanitizeQueryId(null, 'fallback')).toBe('fallback');
+    });
+
+    it('builds UserTweets url with cursor and features', () => {
+        const url = __test__.buildUserTweetsUrl('query123', '42', 20, 'cursor-1');
+        expect(url).toContain('/i/api/graphql/query123/UserTweets');
+        const decoded = decodeURIComponent(url);
+        expect(decoded).toContain('"userId":"42"');
+        expect(decoded).toContain('"count":20');
+        expect(decoded).toContain('"cursor":"cursor-1"');
+        expect(decoded).toContain('longform_notetweets_consumption_enabled');
+    });
+
+    it('builds UserByScreenName url for the given handle', () => {
+        const url = __test__.buildUserByScreenNameUrl('uquery', 'jakevin7');
+        expect(url).toContain('/i/api/graphql/uquery/UserByScreenName');
+        expect(decodeURIComponent(url)).toContain('"screen_name":"jakevin7"');
+    });
+
+    it('prefers note_tweet text over legacy.full_text for long posts', () => {
+        const seen = new Set();
+        const tweet = __test__.extractTweet({
+            rest_id: '99',
+            legacy: { full_text: 'short truncated…', favorite_count: 1, retweet_count: 0, reply_count: 0, created_at: 'now' },
+            note_tweet: { note_tweet_results: { result: { text: 'full long-form body' } } },
+            core: { user_results: { result: { legacy: { screen_name: 'bob', name: 'Bob' } } } },
+            views: { count: '42' },
+        }, seen);
+        expect(tweet.text).toBe('full long-form body');
+        expect(tweet.views).toBe(42);
+    });
+
+    it('flags retweets via RT prefix or retweeted_status_result', () => {
+        const a = __test__.extractTweet({
+            rest_id: '1',
+            legacy: { full_text: 'RT @foo: hi', favorite_count: 0, retweet_count: 0, reply_count: 0, created_at: '' },
+            core: { user_results: { result: { legacy: { screen_name: 'u', name: 'U' } } } },
+        }, new Set());
+        expect(a.is_retweet).toBe(true);
+
+        const b = __test__.extractTweet({
+            rest_id: '2',
+            legacy: { full_text: 'hello', favorite_count: 0, retweet_count: 0, reply_count: 0, created_at: '', retweeted_status_result: { result: {} } },
+            core: { user_results: { result: { legacy: { screen_name: 'u', name: 'U' } } } },
+        }, new Set());
+        expect(b.is_retweet).toBe(true);
+    });
+
+    it('parses chronological tweets and skips pinned instruction', () => {
+        const chronEntry = {
+            entryId: 'tweet-1',
+            content: {
+                itemContent: {
+                    tweet_results: {
+                        result: {
+                            rest_id: '1',
+                            legacy: { full_text: 'chronological post', favorite_count: 5, retweet_count: 1, reply_count: 2, created_at: 'now' },
+                            core: { user_results: { result: { legacy: { screen_name: 'alice', name: 'Alice' } } } },
+                            views: { count: '100' },
+                        },
+                    },
+                },
+            },
+        };
+        const cursorEntry = {
+            entryId: 'cursor-bottom-1',
+            content: { entryType: 'TimelineTimelineCursor', cursorType: 'Bottom', value: 'cursor-next' },
+        };
+        const pinnedEntry = {
+            entryId: 'tweet-pinned-999',
+            content: {
+                itemContent: {
+                    tweet_results: {
+                        result: {
+                            rest_id: '999',
+                            legacy: { full_text: 'pinned post', favorite_count: 0, retweet_count: 0, reply_count: 0, created_at: 'old' },
+                            core: { user_results: { result: { legacy: { screen_name: 'alice', name: 'Alice' } } } },
+                        },
+                    },
+                },
+            },
+        };
+        const payload = {
+            data: {
+                user: {
+                    result: {
+                        timeline_v2: {
+                            timeline: {
+                                instructions: [
+                                    { type: 'TimelinePinEntry', entries: [pinnedEntry] },
+                                    { entries: [chronEntry, cursorEntry] },
+                                ],
+                            },
+                        },
+                    },
+                },
+            },
+        };
+        const result = __test__.parseUserTweets(payload, new Set());
+        expect(result.nextCursor).toBe('cursor-next');
+        expect(result.tweets).toHaveLength(1);
+        expect(result.tweets[0]).toMatchObject({
+            id: '1',
+            author: 'alice',
+            text: 'chronological post',
+            likes: 5,
+            views: 100,
+            url: 'https://x.com/alice/status/1',
+        });
+    });
+});

package/clis/wanfang/search.js

@@ -0,0 +1,66 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { clampInt, requireNonEmptyQuery } from '../_shared/common.js';
+
+cli({
+    site: 'wanfang',
+    name: 'search',
+    description: '万方数据论文搜索',
+    domain: 's.wanfangdata.com.cn',
+    strategy: Strategy.PUBLIC,
+    browser: true,
+    args: [
+        { name: 'query', positional: true, required: true, help: '搜索关键词' },
+        { name: 'limit', type: 'int', default: 10, help: '返回结果数量 (max 20)' },
+    ],
+    columns: ['rank', 'title', 'authors', 'source', 'year', 'type', 'cited', 'url'],
+    navigateBefore: false,
+    func: async (page, kwargs) => {
+        const limit = clampInt(kwargs.limit, 10, 1, 20);
+        const query = requireNonEmptyQuery(kwargs.query);
+        await page.goto(`https://s.wanfangdata.com.cn/paper?q=${encodeURIComponent(query)}`);
+        await page.wait(5);
+        const data = await page.evaluate(`
+      (async () => {
+        const normalize = v => (v || '').replace(/\\s+/g, ' ').trim();
+        for (let i = 0; i < 30; i++) {
+          if (document.querySelectorAll('span.title').length > 0) break;
+          await new Promise(r => setTimeout(r, 500));
+        }
+        const results = [];
+        for (const titleSpan of document.querySelectorAll('span.title')) {
+          const title = normalize(titleSpan.textContent);
+          if (!title || title.length < 3) continue;
+
+          let container = titleSpan.parentElement;
+          for (let i = 0; i < 6; i++) {
+            if (!container?.parentElement || container.parentElement.tagName === 'BODY') break;
+            if (container.querySelectorAll('span.title').length >= 1 && container.querySelectorAll('span.authors').length >= 1) break;
+            container = container.parentElement;
+          }
+          if (!container) continue;
+
+          const id = normalize(container.querySelector('span.title-id-hidden')?.textContent);
+          const url = id ? 'https://d.wanfangdata.com.cn/' + id : '';
+          const authors = Array.from(container.querySelectorAll('span.authors'))
+            .map((item) => normalize(item.textContent))
+            .filter(Boolean)
+            .join(', ')
+            .slice(0, 80);
+          const type = normalize(container.querySelector('span.essay-type')?.textContent);
+          const source = normalize(container.querySelector('span.periodical, span.source')?.textContent);
+
+          let year = normalize(container.querySelector('span.year, span.date')?.textContent);
+          if (!year) year = (container.textContent || '').match(/(19|20)\\d{2}/)?.[0] || '';
+
+          const citedText = normalize(container.querySelector('.stat-item.quote, [class*=\"quote\"]')?.textContent);
+          const cited = citedText.match(/(\\d+)/)?.[1] || '0';
+
+          results.push({ rank: results.length + 1, title, authors, source, year, type, cited, url });
+          if (results.length >= ${limit}) break;
+        }
+        return results;
+      })()
+    `);
+        return Array.isArray(data) ? data : [];
+    },
+});

package/clis/wanfang/search.test.js

@@ -0,0 +1,23 @@
+import { describe, expect, it, vi } from 'vitest';
+import { getRegistry } from '@jackwener/opencli/registry';
+import './search.js';
+
+describe('wanfang search command', () => {
+    const command = getRegistry().get('wanfang/search');
+
+    it('registers as a public browser command', () => {
+        expect(command).toBeDefined();
+        expect(command.site).toBe('wanfang');
+        expect(command.strategy).toBe('public');
+        expect(command.browser).toBe(true);
+    });
+
+    it('rejects empty queries before browser navigation', async () => {
+        const page = { goto: vi.fn() };
+        await expect(command.func(page, { query: '   ' })).rejects.toMatchObject({
+            name: 'ArgumentError',
+            code: 'ARGUMENT',
+        });
+        expect(page.goto).not.toHaveBeenCalled();
+    });
+});

package/clis/web/read.js

@@ -27,7 +27,7 @@ cli({
         { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
         { name: 'wait', type: 'int', default: 3, help: 'Seconds to wait after page load' },
     ],
-    columns: ['title', 'author', 'publish_time', 'status', 'size'],
+    columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'],
     func: async (page, kwargs) => {
         const url = kwargs.url;
         const waitSeconds = kwargs.wait ?? 3;

package/clis/weixin/download.js

@@ -179,12 +179,12 @@ cli({
         { name: 'output', default: './weixin-articles', help: 'Output directory' },
         { name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
     ],
-    columns: ['title', 'author', 'publish_time', 'status', 'size'],
+    columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'],
     func: async (page, kwargs) => {
         const rawUrl = kwargs.url;
         const url = normalizeWechatUrl(rawUrl);
         if (!url.startsWith('https://mp.weixin.qq.com/')) {
-            return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-' }];
+            return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-', saved: '-' }];
         }
         // Navigate and wait for content to load
         await page.goto(url);
@@ -297,6 +297,7 @@ cli({
                     publish_time: '-',
                     status: 'failed — verification required in WeChat browser page',
                     size: '-',
+                    saved: '-',
                 }];
         }
         return downloadArticle({