@jackwener/opencli 1.7.18 → 1.7.19

package/clis/twitter/reply.test.js

@@ -13,6 +13,8 @@ describe('twitter reply command', () => {
         const cmd = getRegistry().get('twitter/reply');
         expect(cmd?.func).toBeTypeOf('function');
         const page = createPageMock([
+            { ok: true },
+            { ok: true },
             { ok: true, message: 'Reply posted successfully.' },
         ]);
         const result = await cmd.func(page, {
@@ -38,6 +40,8 @@ describe('twitter reply command', () => {
         const setFileInput = vi.fn().mockResolvedValue(undefined);
         const page = createPageMock([
             { ok: true, previewCount: 1 },
+            { ok: true },
+            { ok: true },
             { ok: true, message: 'Reply posted successfully.' },
         ], {
             setFileInput,
@@ -74,6 +78,8 @@ describe('twitter reply command', () => {
         const setFileInput = vi.fn().mockResolvedValue(undefined);
         const page = createPageMock([
             { ok: true, previewCount: 1 },
+            { ok: true },
+            { ok: true },
             { ok: true, message: 'Reply posted successfully.' },
         ], {
             setFileInput,
@@ -102,6 +108,55 @@ describe('twitter reply command', () => {
         ]);
         vi.unstubAllGlobals();
     });
+    it('falls back to the target tweet page when the dedicated composer route does not expose a textarea', async () => {
+        const cmd = getRegistry().get('twitter/reply');
+        expect(cmd?.func).toBeTypeOf('function');
+        const wait = vi.fn()
+            .mockRejectedValueOnce(new Error('Selector not found: [data-testid="tweetTextarea_0"]'))
+            .mockResolvedValue(undefined);
+        const page = createPageMock([
+            { ok: true }, // click target tweet page Reply button
+            { ok: true }, // insert reply text
+            { ok: true }, // click composer Reply button
+            { ok: true, message: 'Reply posted successfully.' }, // submit completed
+        ], { wait });
+
+        const url = 'https://x.com/_kop6/status/2040254679301718161?s=20';
+        const result = await cmd.func(page, { url, text: 'fallback reply' });
+
+        expect(page.goto).toHaveBeenNthCalledWith(1, 'https://x.com/compose/post?in_reply_to=2040254679301718161', { waitUntil: 'load', settleMs: 2500 });
+        expect(page.goto).toHaveBeenNthCalledWith(2, url, { waitUntil: 'load', settleMs: 2500 });
+        expect(page.evaluate.mock.calls[0][0]).toContain('[data-testid="reply"]');
+        expect(wait).toHaveBeenLastCalledWith({ selector: '[data-testid="tweetTextarea_0"]', timeout: 15 });
+        expect(result).toEqual([{ status: 'success', message: 'Reply posted successfully.', text: 'fallback reply' }]);
+    });
+    it('treats an X success toast as success after a Promise was collected error', async () => {
+        const cmd = getRegistry().get('twitter/reply');
+        expect(cmd?.func).toBeTypeOf('function');
+        const evaluate = vi.fn()
+            .mockResolvedValueOnce({ ok: true }) // insert reply text
+            .mockResolvedValueOnce({ ok: true }) // click Reply
+            .mockRejectedValueOnce(new Error('{"code":-32000,"message":"Promise was collected"}'))
+            .mockResolvedValueOnce({
+                ok: true,
+                message: 'Reply posted successfully.',
+                url: 'https://x.com/me/status/123',
+            });
+        const page = createPageMock([], { evaluate });
+
+        const result = await cmd.func(page, {
+            url: 'https://x.com/_kop6/status/2040254679301718161?s=20',
+            text: 'toast recovery',
+        });
+
+        expect(page.wait).toHaveBeenCalledWith(2);
+        expect(result).toEqual([{
+            status: 'success',
+            message: 'Reply posted successfully.',
+            text: 'toast recovery',
+            url: 'https://x.com/me/status/123',
+        }]);
+    });
     it('rejects using --image and --image-url together', async () => {
         const cmd = getRegistry().get('twitter/reply');
         expect(cmd?.func).toBeTypeOf('function');

package/clis/twitter/search.js

@@ -1,7 +1,7 @@
-import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors';
+import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
 import { cli, Strategy } from '@jackwener/opencli/registry';
-import { extractMedia } from './shared.js';
-import { applyTopByEngagement } from './utils.js';
+import { extractMedia, normalizeTwitterGraphqlPayload, resolveTwitterOperationMetadata } from './shared.js';
+import { TWITTER_BEARER_TOKEN, applyTopByEngagement } from './utils.js';
 
 // ── Public-search operator surface ─────────────────────────────────────
 //
@@ -35,6 +35,68 @@ const PRODUCT_TO_F_PARAM = Object.freeze({
     videos: 'video',
 });
 
+const PRODUCT_TO_GRAPHQL_PRODUCT = Object.freeze({
+    top: 'Top',
+    live: 'Latest',
+    photos: 'Photos',
+    videos: 'Videos',
+});
+const MAX_PAGINATION_PAGES = 100;
+
+const SEARCH_TIMELINE_OPERATION = {
+    queryId: 'VhUd6vHVmLBcw0uX-6jMLA',
+    features: {
+    rweb_video_screen_enabled: true,
+    rweb_cashtags_enabled: true,
+    profile_label_improvements_pcf_label_in_post_enabled: true,
+    responsive_web_profile_redirect_enabled: true,
+    rweb_tipjar_consumption_enabled: true,
+    verified_phone_label_enabled: false,
+    creator_subscriptions_tweet_preview_api_enabled: true,
+    responsive_web_graphql_timeline_navigation_enabled: true,
+    responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+    premium_content_api_read_enabled: false,
+    communities_web_enable_tweet_community_results_fetch: true,
+    c9s_tweet_anatomy_moderator_badge_enabled: true,
+    responsive_web_grok_analyze_button_fetch_trends_enabled: false,
+    responsive_web_grok_analyze_post_followups_enabled: true,
+    rweb_cashtags_composer_attachment_enabled: true,
+    responsive_web_jetfuel_frame: true,
+    responsive_web_grok_share_attachment_enabled: true,
+    responsive_web_grok_annotations_enabled: true,
+    articles_preview_enabled: true,
+    responsive_web_edit_tweet_api_enabled: true,
+    graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+    view_counts_everywhere_api_enabled: true,
+    longform_notetweets_consumption_enabled: true,
+    responsive_web_twitter_article_tweet_consumption_enabled: true,
+    content_disclosure_indicator_enabled: true,
+    content_disclosure_ai_generated_indicator_enabled: true,
+    responsive_web_grok_show_grok_translated_post: false,
+    responsive_web_grok_analysis_button_from_backend: true,
+    post_ctas_fetch_enabled: false,
+    freedom_of_speech_not_reach_fetch_enabled: true,
+    standardized_nudges_misinfo: true,
+    tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+    longform_notetweets_rich_text_read_enabled: true,
+    longform_notetweets_inline_media_enabled: true,
+    responsive_web_grok_image_annotation_enabled: true,
+    responsive_web_grok_imagine_annotation_enabled: true,
+    responsive_web_grok_community_note_auto_translation_is_enabled: false,
+    responsive_web_enhance_cards_enabled: false,
+    },
+    fieldToggles: {
+        withPayments: true,
+        withAuxiliaryUserLabels: true,
+        withArticleRichContentState: true,
+        withArticlePlainText: true,
+        withArticleSummaryText: true,
+        withArticleVoiceOver: true,
+        withGrokAnalyze: true,
+        withDisallowedReplyControls: true,
+    },
+};
+
 const FROM_USER_PATTERN = /^[A-Za-z0-9_]{1,15}$/;
 
 const EXCLUDE_TO_OPERATOR = Object.freeze({
@@ -99,125 +161,96 @@ function resolveSearchFParam(kwargs) {
     return kwargs.filter === 'live' ? 'live' : 'top';
 }
 
-/**
- * Trigger Twitter search SPA navigation with fallback strategies.
- *
- * Primary: pushState + popstate (works in most environments).
- * Fallback: Type into the search input and press Enter when pushState fails
- *   intermittently (e.g. due to Twitter A/B tests or timing races — see #690).
- *
- * Both strategies preserve the JS context so the fetch interceptor stays alive.
- *
- * @param {object} page
- * @param {string} query  — final composed query (already merged with operators)
- * @param {string} fParam — Twitter URL `f=` value (top|live|image|video)
- */
-async function navigateToSearch(page, query, fParam) {
-    const searchUrl = JSON.stringify(`/search?q=${encodeURIComponent(query)}&f=${fParam}`);
-    let lastPath = '';
-    // Strategy 1 (primary): pushState + popstate with retry
-    for (let attempt = 1; attempt <= 2; attempt++) {
-        await page.evaluate(`
-      (() => {
-        window.history.pushState({}, '', ${searchUrl});
-        window.dispatchEvent(new PopStateEvent('popstate', { state: {} }));
-      })()
-    `);
-        try {
-            await page.wait({ selector: '[data-testid="primaryColumn"]' });
-        }
-        catch {
-            // selector timeout — fall through to path check or next attempt
-        }
-        lastPath = String(await page.evaluate('() => window.location.pathname') || '');
-        if (lastPath.startsWith('/search')) {
-            return;
-        }
-        if (attempt < 2) {
-            await page.wait(1);
-        }
+function resolveSearchProduct(kwargs) {
+    const product = kwargs.product || (kwargs.filter === 'live' ? 'live' : 'top');
+    return PRODUCT_TO_GRAPHQL_PRODUCT[product] || 'Top';
+}
+
+function normalizeOperation(operation) {
+    if (typeof operation === 'string') {
+        return {
+            queryId: operation,
+            features: SEARCH_TIMELINE_OPERATION.features,
+            fieldToggles: SEARCH_TIMELINE_OPERATION.fieldToggles,
+        };
     }
-    // Strategy 2 (fallback): Use the search input on /explore.
-    // The nativeSetter + Enter approach triggers Twitter's own form handler,
-    // performing SPA navigation without a full page reload.
-    const queryStr = JSON.stringify(query);
-    const navResult = await page.evaluate(`(async () => {
-    try {
-      const input = document.querySelector('[data-testid="SearchBox_Search_Input"]');
-      if (!input) return { ok: false };
+    return {
+        queryId: operation?.queryId || SEARCH_TIMELINE_OPERATION.queryId,
+        features: operation?.features || SEARCH_TIMELINE_OPERATION.features,
+        fieldToggles: operation?.fieldToggles || SEARCH_TIMELINE_OPERATION.fieldToggles,
+    };
+}
 
-      input.focus();
-      await new Promise(r => setTimeout(r, 300));
+function buildSearchTimelineRequest(operation, rawQuery, product, count, cursor) {
+    const normalized = normalizeOperation(operation);
+    const vars = {
+        rawQuery,
+        count,
+        querySource: 'typed_query',
+        product,
+    };
+    if (cursor) vars.cursor = cursor;
+    return [
+        `/i/api/graphql/${normalized.queryId}/SearchTimeline`,
+        {
+            variables: vars,
+            features: normalized.features,
+            fieldToggles: normalized.fieldToggles,
+        },
+    ];
+}
 
-      const nativeSetter = Object.getOwnPropertyDescriptor(
-        window.HTMLInputElement.prototype, 'value'
-      )?.set;
-      if (!nativeSetter) return { ok: false };
-      nativeSetter.call(input, ${queryStr});
-      input.dispatchEvent(new Event('input', { bubbles: true }));
-      input.dispatchEvent(new Event('change', { bubbles: true }));
-      await new Promise(r => setTimeout(r, 500));
+function unwrapTweetResult(result) {
+    if (!result) return null;
+    if (result.__typename === 'TweetWithVisibilityResults' && result.tweet) return result.tweet;
+    if (result.tweet) return result.tweet;
+    return result;
+}
 
-      input.dispatchEvent(new KeyboardEvent('keydown', {
-        key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
-      }));
+function tweetToRow(result, seen) {
+    const tweet = unwrapTweetResult(result);
+    if (!tweet?.rest_id || seen.has(tweet.rest_id)) return null;
+    seen.add(tweet.rest_id);
+    const tweetUser = tweet.core?.user_results?.result;
+    return {
+        id: tweet.rest_id,
+        author: tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown',
+        text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '',
+        created_at: tweet.legacy?.created_at || '',
+        likes: tweet.legacy?.favorite_count || 0,
+        views: tweet.views?.count || '0',
+        url: `https://x.com/i/status/${tweet.rest_id}`,
+        ...extractMedia(tweet.legacy),
+    };
+}
 
-      return { ok: true };
-    } catch {
-      return { ok: false };
-    }
-  })()`);
-    if (navResult?.ok) {
-        try {
-            await page.wait({ selector: '[data-testid="primaryColumn"]' });
+function parseSearchTimeline(data, seen) {
+    const rows = [];
+    let nextCursor = null;
+    const instructions = data?.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || [];
+    const visit = (value) => {
+        if (!value || typeof value !== 'object') return;
+        if (value.tweet_results?.result) {
+            const row = tweetToRow(value.tweet_results.result, seen);
+            if (row) rows.push(row);
         }
-        catch {
-            // fall through to path check
+        if (
+            (value.entryType === 'TimelineTimelineCursor' || value.__typename === 'TimelineTimelineCursor')
+            && (value.cursorType === 'Bottom' || value.cursorType === 'ShowMore')
+            && value.value
+        ) {
+            nextCursor = value.value;
         }
-        lastPath = String(await page.evaluate('() => window.location.pathname') || '');
-        if (lastPath.startsWith('/search')) {
-            // The fallback path doesn't carry the f= URL param, so click the
-            // matching tab to align with the requested product. Only `live`
-            // currently surfaces a distinct tab label — `image`/`video` tabs
-            // also need an explicit click, so try them all.
-            const tabClicked = await clickProductTabIfNeeded(page, fParam);
-            if (!tabClicked) {
-                throw new CommandExecutionError(`SPA fallback reached /search but could not select the requested product tab: ${fParam}`);
-            }
+        if (Array.isArray(value)) {
+            for (const item of value) visit(item);
             return;
         }
-    }
-    throw new CommandExecutionError(`SPA navigation to /search failed. Final path: ${lastPath || '(empty)'}. Twitter may have changed its routing.`);
-}
-
-/**
- * After the search-input fallback lands on /search, the f= param is missing
- * from the URL. Click the matching tab in the result page header so the
- * SearchTimeline call uses the right filter. No-op for fParam=top (default).
- */
-async function clickProductTabIfNeeded(page, fParam) {
-    if (fParam === 'top') return true;
-    const tabLabels = JSON.stringify({
-        live: ['Latest', '最新'],
-        image: ['Photos', 'Images', '照片', '图片'],
-        video: ['Videos', '视频'],
-    }[fParam] || []);
-    if (tabLabels === '[]') return true;
-    const clicked = await page.evaluate(`(() => {
-      const labels = ${tabLabels};
-      const tabs = document.querySelectorAll('[role="tab"]');
-      for (const tab of tabs) {
-        const txt = (tab.textContent || '').trim();
-        if (labels.some(l => txt.includes(l))) {
-          tab.click();
-          return true;
+        for (const child of Object.values(value)) {
+            if (child && typeof child === 'object') visit(child);
         }
-      }
-      return false;
-    })()`);
-    if (!clicked) return false;
-    await page.wait(2);
-    return true;
+    };
+    visit(instructions);
+    return { rows, nextCursor };
 }
 
 cli({
@@ -226,7 +259,7 @@ cli({
     access: 'read',
     description: 'Search Twitter/X for tweets, with optional --from / --has / --exclude / --product filters mapped to X\'s search operators',
     domain: 'x.com',
-    strategy: Strategy.INTERCEPT, // Use intercept strategy
+    strategy: Strategy.COOKIE,
     browser: true,
     siteSession: 'persistent',
     args: [
@@ -248,65 +281,47 @@ cli({
         if (!Number.isInteger(Number(kwargs.limit)) || Number(kwargs.limit) <= 0) {
             throw new ArgumentError('twitter search --limit must be a positive integer', 'Example: opencli twitter search opencli --limit 15');
         }
-        const fParam = resolveSearchFParam(kwargs);
-        // 1. Navigate to x.com/explore (has a search input at the top)
-        await page.goto('https://x.com/explore');
-        await page.wait(3);
-        // 2. Install interceptor BEFORE triggering search.
-        //    SPA navigation preserves the JS context, so the monkey-patched
-        //    fetch will capture the SearchTimeline API call.
-        await page.installInterceptor('SearchTimeline');
-        // 3. Trigger SPA navigation to search results via history API.
-        //    pushState + popstate triggers React Router's listener without
-        //    a full page reload, so the interceptor stays alive.
-        //    Note: the previous approach (nativeSetter + Enter keydown on the
-        //    search input) does not reliably trigger Twitter's form submission.
-        await navigateToSearch(page, finalQuery, fParam);
-        // 4. Scroll to trigger additional pagination
-        await page.autoScroll({ times: 3, delayMs: 2000 });
-        // 5. Retrieve captured data
-        const requests = await page.getInterceptedRequests();
-        if (!requests || requests.length === 0)
-            return [];
-        let results = [];
+        const cookies = await page.getCookies({ url: 'https://x.com' });
+        const ct0 = cookies.find((c) => c.name === 'ct0')?.value || null;
+        if (!ct0) throw new AuthRequiredError('x.com', 'Not logged into x.com (no ct0 cookie)');
+        await page.goto('https://x.com/home', { waitUntil: 'load', settleMs: 1000 });
+        const operation = await resolveTwitterOperationMetadata(page, 'SearchTimeline', SEARCH_TIMELINE_OPERATION);
+        const headers = JSON.stringify({
+            'Authorization': `Bearer ${decodeURIComponent(TWITTER_BEARER_TOKEN)}`,
+            'X-Csrf-Token': ct0,
+            'X-Twitter-Auth-Type': 'OAuth2Session',
+            'X-Twitter-Active-User': 'yes',
+            'Content-Type': 'application/json',
+        });
+        const product = resolveSearchProduct(kwargs);
+        const results = [];
         const seen = new Set();
-        for (const req of requests) {
-            try {
-                const insts = req?.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || [];
-                const addEntries = insts.find((i) => i.type === 'TimelineAddEntries')
-                    || insts.find((i) => i.entries && Array.isArray(i.entries));
-                if (!addEntries?.entries)
-                    continue;
-                for (const entry of addEntries.entries) {
-                    if (!entry.entryId.startsWith('tweet-'))
-                        continue;
-                    let tweet = entry.content?.itemContent?.tweet_results?.result;
-                    if (!tweet)
-                        continue;
-                    // Handle retweet wrapping
-                    if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
-                        tweet = tweet.tweet;
-                    }
-                    if (!tweet.rest_id || seen.has(tweet.rest_id))
-                        continue;
-                    seen.add(tweet.rest_id);
-                    // Twitter moved screen_name from legacy to core
-                    const tweetUser = tweet.core?.user_results?.result;
-                    results.push({
-                        id: tweet.rest_id,
-                        author: tweetUser?.core?.screen_name || tweetUser?.legacy?.screen_name || 'unknown',
-                        text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '',
-                        created_at: tweet.legacy?.created_at || '',
-                        likes: tweet.legacy?.favorite_count || 0,
-                        views: tweet.views?.count || '0',
-                        url: `https://x.com/i/status/${tweet.rest_id}`,
-                        ...extractMedia(tweet.legacy),
-                    });
-                }
-            }
-            catch (e) {
-                // ignore parsing errors for individual payloads
+        let cursor = null;
+        // Runaway guard only; --limit and cursor exhaustion control normal pagination.
+        for (let i = 0; i < MAX_PAGINATION_PAGES && results.length < kwargs.limit; i++) {
+            const fetchCount = Number(kwargs.limit) - results.length + 10;
+            const [requestUrl, requestPayload] = buildSearchTimelineRequest(operation, finalQuery, product, fetchCount, cursor);
+            const requestBody = JSON.stringify(requestPayload);
+            const data = normalizeTwitterGraphqlPayload(await page.evaluate(`async () => {
+        const options = {
+          method: 'POST',
+          headers: ${headers},
+          credentials: 'include',
+        };
+        options['body'] = ${JSON.stringify(requestBody)};
+        const r = await fetch(${JSON.stringify(requestUrl)}, {
+          ...options,
+        });
+        return r.ok ? await r.json() : { error: r.status };
+      }`));
+            if (data?.error) {
+                if (results.length === 0) throw new CommandExecutionError(`HTTP ${data.error}: SearchTimeline fetch failed — queryId may have expired`);
+                break;
             }
+            const { rows, nextCursor } = parseSearchTimeline(data, seen);
+            results.push(...rows);
+            if (!nextCursor || nextCursor === cursor) break;
+            cursor = nextCursor;
         }
         const trimmed = results.slice(0, kwargs.limit);
         return applyTopByEngagement(trimmed, kwargs['top-by-engagement']);
@@ -316,6 +331,9 @@ cli({
 export const __test__ = {
     buildSearchQuery,
     resolveSearchFParam,
+    resolveSearchProduct,
+    buildSearchTimelineRequest,
+    parseSearchTimeline,
     HAS_CHOICES,
     EXCLUDE_CHOICES,
     PRODUCT_CHOICES,