@jackwener/opencli 0.7.8 → 0.7.10

package/src/clis/twitter/search.ts

@@ -13,14 +13,17 @@ cli({
   ],
   columns: ['id', 'author', 'text', 'likes', 'views', 'url'],
   func: async (page, kwargs) => {
+    // Install the interceptor before opening the target page so we don't miss
+    // the initial SearchTimeline request fired during hydration.
+    await page.goto('https://x.com');
+    await page.wait(2);
+    await page.installInterceptor('SearchTimeline');
+
     // 1. Navigate to the search page
     const q = encodeURIComponent(kwargs.query);
     await page.goto(`https://x.com/search?q=${q}&f=top`);
     await page.wait(5);
 
-    // 2. Inject XHR interceptor
-    await page.installInterceptor('SearchTimeline');
-
     // 3. Trigger API by scrolling
     await page.autoScroll({ times: 3, delayMs: 2000 });
     
@@ -29,11 +32,13 @@ cli({
     if (!requests || requests.length === 0) return [];
 
     let results: any[] = [];
+    const seen = new Set<string>();
     for (const req of requests) {
       try {
-        const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
-        const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
-        if (!addEntries) continue;
+        const insts = req.data?.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || [];
+        const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries')
+          || insts.find((i: any) => i.entries && Array.isArray(i.entries));
+        if (!addEntries?.entries) continue;
 
         for (const entry of addEntries.entries) {
           if (!entry.entryId.startsWith('tweet-')) continue;
@@ -45,11 +50,13 @@ cli({
           if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
               tweet = tweet.tweet;
           }
+          if (!tweet.rest_id || seen.has(tweet.rest_id)) continue;
+          seen.add(tweet.rest_id);
 
           results.push({
             id: tweet.rest_id,
             author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
-            text: tweet.legacy?.full_text || '',
+            text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '',
             likes: tweet.legacy?.favorite_count || 0,
             views: tweet.views?.count || '0',
             url: `https://x.com/i/status/${tweet.rest_id}`

package/src/clis/twitter/timeline.ts

@@ -1,50 +1,218 @@
 import { cli, Strategy } from '../../registry.js';
 
+// ── Twitter GraphQL constants ──────────────────────────────────────────
+
+const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
+const HOME_TIMELINE_QUERY_ID = 'c-CzHF1LboFilMpsx4ZCrQ';
+
+const FEATURES = {
+  rweb_video_screen_enabled: false,
+  profile_label_improvements_pcf_label_in_post_enabled: true,
+  rweb_tipjar_consumption_enabled: true,
+  verified_phone_label_enabled: false,
+  creator_subscriptions_tweet_preview_api_enabled: true,
+  responsive_web_graphql_timeline_navigation_enabled: true,
+  responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+  premium_content_api_read_enabled: false,
+  communities_web_enable_tweet_community_results_fetch: true,
+  c9s_tweet_anatomy_moderator_badge_enabled: true,
+  responsive_web_grok_analyze_button_fetch_trends_enabled: false,
+  responsive_web_grok_analyze_post_followups_enabled: true,
+  responsive_web_jetfuel_frame: false,
+  responsive_web_grok_share_attachment_enabled: true,
+  articles_preview_enabled: true,
+  responsive_web_edit_tweet_api_enabled: true,
+  graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+  view_counts_everywhere_api_enabled: true,
+  longform_notetweets_consumption_enabled: true,
+  responsive_web_twitter_article_tweet_consumption_enabled: true,
+  tweet_awards_web_tipping_enabled: false,
+  responsive_web_grok_show_grok_translated_post: false,
+  responsive_web_grok_analysis_button_from_backend: false,
+  creator_subscriptions_quote_tweet_preview_enabled: false,
+  freedom_of_speech_not_reach_fetch_enabled: true,
+  standardized_nudges_misinfo: true,
+  tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+  longform_notetweets_rich_text_read_enabled: true,
+  longform_notetweets_inline_media_enabled: true,
+  responsive_web_grok_image_annotation_enabled: true,
+  responsive_web_enhance_cards_enabled: false,
+};
+
+// ── Pure functions (type-safe, testable) ───────────────────────────────
+
+interface TimelineTweet {
+  id: string;
+  author: string;
+  text: string;
+  likes: number;
+  retweets: number;
+  replies: number;
+  views: number;
+  created_at: string;
+  url: string;
+}
+
+function buildHomeTimelineUrl(count: number, cursor?: string | null): string {
+  const vars: Record<string, any> = {
+    count,
+    includePromotedContent: false,
+    latestControlAvailable: true,
+    requestContext: 'launch',
+    withCommunity: true,
+  };
+  if (cursor) vars.cursor = cursor;
+
+  return `/i/api/graphql/${HOME_TIMELINE_QUERY_ID}/HomeTimeline`
+    + `?variables=${encodeURIComponent(JSON.stringify(vars))}`
+    + `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`;
+}
+
+function extractTweet(result: any, seen: Set<string>): TimelineTweet | null {
+  if (!result) return null;
+  const tw = result.tweet || result;
+  const l = tw.legacy || {};
+  if (!tw.rest_id || seen.has(tw.rest_id)) return null;
+  seen.add(tw.rest_id);
+
+  const u = tw.core?.user_results?.result;
+  const screenName = u?.legacy?.screen_name || u?.core?.screen_name || 'unknown';
+  const noteText = tw.note_tweet?.note_tweet_results?.result?.text;
+  const views = tw.views?.count ? parseInt(tw.views.count, 10) : 0;
+
+  return {
+    id: tw.rest_id,
+    author: screenName,
+    text: noteText || l.full_text || '',
+    likes: l.favorite_count || 0,
+    retweets: l.retweet_count || 0,
+    replies: l.reply_count || 0,
+    views,
+    created_at: l.created_at || '',
+    url: `https://x.com/${screenName}/status/${tw.rest_id}`,
+  };
+}
+
+function parseHomeTimeline(data: any, seen: Set<string>): { tweets: TimelineTweet[]; nextCursor: string | null } {
+  const tweets: TimelineTweet[] = [];
+  let nextCursor: string | null = null;
+
+  const instructions =
+    data?.data?.home?.home_timeline_urt?.instructions || [];
+
+  for (const inst of instructions) {
+    for (const entry of inst.entries || []) {
+      const c = entry.content;
+
+      // Cursor entries
+      if (c?.entryType === 'TimelineTimelineCursor' || c?.__typename === 'TimelineTimelineCursor') {
+        if (c.cursorType === 'Bottom') nextCursor = c.value;
+        continue;
+      }
+      if (entry.entryId?.startsWith('cursor-bottom-')) {
+        nextCursor = c?.value || nextCursor;
+        continue;
+      }
+
+      // Single tweet entry
+      const tweetResult = c?.itemContent?.tweet_results?.result;
+      if (tweetResult) {
+        // Skip promoted content
+        if (c?.itemContent?.promotedMetadata) continue;
+        const tw = extractTweet(tweetResult, seen);
+        if (tw) tweets.push(tw);
+        continue;
+      }
+
+      // Conversation module (grouped tweets)
+      for (const item of c?.items || []) {
+        const nested = item.item?.itemContent?.tweet_results?.result;
+        if (nested) {
+          if (item.item?.itemContent?.promotedMetadata) continue;
+          const tw = extractTweet(nested, seen);
+          if (tw) tweets.push(tw);
+        }
+      }
+    }
+  }
+
+  return { tweets, nextCursor };
+}
+
+// ── CLI definition ────────────────────────────────────────────────────
+
 cli({
   site: 'twitter',
   name: 'timeline',
-  description: 'Twitter Home Timeline',
+  description: 'Fetch Twitter Home Timeline',
   domain: 'x.com',
   strategy: Strategy.COOKIE,
+  browser: true,
   args: [
     { name: 'limit', type: 'int', default: 20 },
   ],
-  columns: ['responseType', 'first'],
+  columns: ['id', 'author', 'text', 'likes', 'retweets', 'replies', 'views', 'created_at', 'url'],
   func: async (page, kwargs) => {
-    await page.goto('https://x.com/home');
-    await page.wait(5);
-    // Inject the fetch interceptor manually to see exactly what happens
-    await page.evaluate(`
-      () => {
-        window.__intercept_data = [];
-        const origFetch = window.fetch;
-        window.fetch = async function(...args) {
-          let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
-          const res = await origFetch.apply(this, args);
-          setTimeout(async () => {
-            try {
-              if (u.includes('HomeTimeline')) {
-                const clone = res.clone();
-                const j = await clone.json();
-                window.__intercept_data.push(j);
-              }
-            } catch(e) {}
-          }, 0);
-          return res;
-        };
+    const limit = kwargs.limit || 20;
+
+    // Navigate to x.com for cookie context
+    await page.goto('https://x.com');
+    await page.wait(3);
+
+    // Extract CSRF token
+    const ct0 = await page.evaluate(`() => {
+      return document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1] || null;
+    }`);
+    if (!ct0) throw new Error('Not logged into x.com (no ct0 cookie)');
+
+    // Dynamically resolve queryId
+    const queryId = await page.evaluate(`async () => {
+      try {
+        const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json');
+        if (ghResp.ok) {
+          const data = await ghResp.json();
+          const entry = data['HomeTimeline'];
+          if (entry && entry.queryId) return entry.queryId;
+        }
+      } catch {}
+      return null;
+    }`) || HOME_TIMELINE_QUERY_ID;
+
+    // Build auth headers
+    const headers = JSON.stringify({
+      'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`,
+      'X-Csrf-Token': ct0,
+      'X-Twitter-Auth-Type': 'OAuth2Session',
+      'X-Twitter-Active-User': 'yes',
+    });
+
+    // Paginate — fetch in browser, parse in TypeScript
+    const allTweets: TimelineTweet[] = [];
+    const seen = new Set<string>();
+    let cursor: string | null = null;
+
+    for (let i = 0; i < 5 && allTweets.length < limit; i++) {
+      const fetchCount = Math.min(40, limit - allTweets.length + 5); // over-fetch slightly for promoted filtering
+      const apiUrl = buildHomeTimelineUrl(fetchCount, cursor)
+        .replace(HOME_TIMELINE_QUERY_ID, queryId);
+
+      const data = await page.evaluate(`async () => {
+        const r = await fetch("${apiUrl}", { headers: ${headers}, credentials: 'include' });
+        return r.ok ? await r.json() : { error: r.status };
+      }`);
+
+      if (data?.error) {
+        if (allTweets.length === 0) throw new Error(`HTTP ${data.error}: Failed to fetch timeline. queryId may have expired.`);
+        break;
       }
-    `);
-    
-    // trigger scroll
-    for(let i=0; i<3; i++) {
-      await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
-      await page.wait(2);
+
+      const { tweets, nextCursor } = parseHomeTimeline(data, seen);
+      allTweets.push(...tweets);
+
+      if (!nextCursor || nextCursor === cursor) break;
+      cursor = nextCursor;
     }
-    
-    // extract
-    const data = await page.evaluate('() => window.__intercept_data');
-    if (!data || data.length === 0) return [{responseType: 'no data captured'}];
-    
-    return [{responseType: `captured ${data.length} responses`, first: JSON.stringify(data[0]).substring(0,300)}];
-  }
+
+    return allTweets.slice(0, limit);
+  },
 });

package/src/clis/twitter/trending.yaml

@@ -25,9 +25,15 @@ pipeline:
           credentials: 'include',
           headers: { 'x-twitter-active-user': 'yes', 'x-csrf-token': csrfToken, 'authorization': 'Bearer ' + bearerToken }
         });
+        if (!res.ok) throw new Error('HTTP ' + res.status + '. Hint: trending endpoint may require login or API shape changed.');
         const data = await res.json();
-        const trends = data?.timeline?.instructions?.[1]?.addEntries?.entries || [];
-        return trends.filter(e => e.content?.timelineModule).flatMap(e => e.content.timelineModule.items || []).map(t => t?.item?.content?.trend).filter(Boolean);
+        const instructions = data?.timeline?.instructions || [];
+        const entries = instructions.flatMap(inst => inst?.addEntries?.entries || inst?.entries || []);
+        return entries
+          .filter(e => e.content?.timelineModule)
+          .flatMap(e => e.content.timelineModule.items || [])
+          .map(t => t?.item?.content?.trend)
+          .filter(Boolean);
       })()
 
   - map:

package/src/setup.ts

@@ -184,11 +184,17 @@ export async function runSetup(opts: { cliVersion?: string; token?: string } = {
     if (result.ok) {
       console.log(`  ${chalk.green('✓')} Browser connected in ${(result.durationMs / 1000).toFixed(1)}s`);
     } else {
+      console.log(`  ${chalk.green('✓')} Token saved successfully.`);
       console.log(`  ${chalk.yellow('!')} Browser connectivity test failed: ${result.error ?? 'unknown'}`);
-      console.log(chalk.dim('    Make sure Chrome is running with the extension enabled.'));
+      console.log(chalk.dim('    Token configuration is complete. To use opencli, make sure Chrome'));
+      console.log(chalk.dim('    is running with the Playwright MCP Bridge extension enabled.'));
+      console.log(chalk.dim(`    Run ${chalk.bold('opencli doctor --live')} to re-test connectivity.`));
     }
   } catch {
-    console.log(`  ${chalk.yellow('!')} Could not verify connectivity (Chrome may not be running)`);
+    console.log(`  ${chalk.green('✓')} Token saved successfully.`);
+    console.log(`  ${chalk.yellow('!')} Browser connectivity test skipped (Chrome may not be running).`);
+    console.log(chalk.dim('    Token configuration is complete. Start Chrome to begin using opencli.'));
+    console.log(chalk.dim(`    Run ${chalk.bold('opencli doctor --live')} to re-test connectivity.`));
   }
   console.log();
 }

package/dist/clis/twitter/bookmarks.yaml

@@ -1,85 +0,0 @@
-site: twitter
-name: bookmarks
-description: 获取 Twitter 书签列表
-domain: x.com
-browser: true
-
-args:
-  limit:
-    type: int
-    default: 20
-    description: Number of bookmarks to return (default 20)
-
-pipeline:
-  - navigate: https://x.com/i/bookmarks
-  - wait: 2
-  - evaluate: |
-      (async () => {
-        const ct0 = document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1];
-        if (!ct0) throw new Error('No ct0 cookie. Hint: Not logged into x.com.');
-        const bearer = decodeURIComponent('AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA');
-        const _h = {'Authorization':'Bearer '+bearer, 'X-Csrf-Token':ct0, 'X-Twitter-Auth-Type':'OAuth2Session', 'X-Twitter-Active-User':'yes'};
-
-        const count = Math.min(${{ args.limit }}, 100);
-        const variables = JSON.stringify({count, includePromotedContent: false});
-        const features = JSON.stringify({
-          rweb_video_screen_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true,
-          responsive_web_profile_redirect_enabled: false, rweb_tipjar_consumption_enabled: false,
-          verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true,
-          responsive_web_graphql_timeline_navigation_enabled: true,
-          responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
-           premium_content_api_read_enabled: false, communities_web_enable_tweet_community_results_fetch: true,
-          c9s_tweet_anatomy_moderator_badge_enabled: true,
-          articles_preview_enabled: true, responsive_web_edit_tweet_api_enabled: true,
-          graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
-          view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true,
-          responsive_web_twitter_article_tweet_consumption_enabled: true,
-          tweet_awards_web_tipping_enabled: false,
-          content_disclosure_indicator_enabled: true, content_disclosure_ai_generated_indicator_enabled: true,
-          freedom_of_speech_not_reach_fetch_enabled: true, standardized_nudges_misinfo: true,
-          tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
-          longform_notetweets_rich_text_read_enabled: true, longform_notetweets_inline_media_enabled: false,
-          responsive_web_enhance_cards_enabled: false
-        });
-        const url = '/i/api/graphql/Fy0QMy4q_aZCpkO0PnyLYw/Bookmarks?variables=' + encodeURIComponent(variables) + '&features=' + encodeURIComponent(features);
-        const resp = await fetch(url, {headers: _h, credentials: 'include'});
-        if (!resp.ok) throw new Error('HTTP ' + resp.status + '. Hint: queryId may have changed.');
-        const d = await resp.json();
-
-        const instructions = d.data?.bookmark_timeline_v2?.timeline?.instructions || d.data?.bookmark_timeline?.timeline?.instructions || [];
-        let tweets = [], seen = new Set();
-        for (const inst of instructions) {
-          for (const entry of (inst.entries || [])) {
-            const r = entry.content?.itemContent?.tweet_results?.result;
-            if (!r) continue;
-            const tw = r.tweet || r;
-            const l = tw.legacy || {};
-            if (!tw.rest_id || seen.has(tw.rest_id)) continue;
-            seen.add(tw.rest_id);
-            const u = tw.core?.user_results?.result;
-            const nt = tw.note_tweet?.note_tweet_results?.result?.text;
-            const screenName = u?.legacy?.screen_name || u?.core?.screen_name;
-            tweets.push({
-              id: tw.rest_id, 
-              author: screenName,
-              name: u?.legacy?.name || u?.core?.name,
-              url: 'https://x.com/' + (screenName || '_') + '/status/' + tw.rest_id,
-              text: nt || l.full_text || '',
-              likes: l.favorite_count, 
-              retweets: l.retweet_count,
-              created_at: l.created_at
-            });
-          }
-        }
-        return tweets;
-      })()
-
-  - map:
-      author: ${{ item.author }}
-      text: ${{ item.text }}
-      likes: ${{ item.likes }}
-      url: ${{ item.url }}
-
-  - limit: ${{ args.limit }}
-
-columns: [author, text, likes, url]

package/src/clis/twitter/bookmarks.yaml

@@ -1,85 +0,0 @@
-site: twitter
-name: bookmarks
-description: 获取 Twitter 书签列表
-domain: x.com
-browser: true
-
-args:
-  limit:
-    type: int
-    default: 20
-    description: Number of bookmarks to return (default 20)
-
-pipeline:
-  - navigate: https://x.com/i/bookmarks
-  - wait: 2
-  - evaluate: |
-      (async () => {
-        const ct0 = document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1];
-        if (!ct0) throw new Error('No ct0 cookie. Hint: Not logged into x.com.');
-        const bearer = decodeURIComponent('AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA');
-        const _h = {'Authorization':'Bearer '+bearer, 'X-Csrf-Token':ct0, 'X-Twitter-Auth-Type':'OAuth2Session', 'X-Twitter-Active-User':'yes'};
-
-        const count = Math.min(${{ args.limit }}, 100);
-        const variables = JSON.stringify({count, includePromotedContent: false});
-        const features = JSON.stringify({
-          rweb_video_screen_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true,
-          responsive_web_profile_redirect_enabled: false, rweb_tipjar_consumption_enabled: false,
-          verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true,
-          responsive_web_graphql_timeline_navigation_enabled: true,
-          responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
-           premium_content_api_read_enabled: false, communities_web_enable_tweet_community_results_fetch: true,
-          c9s_tweet_anatomy_moderator_badge_enabled: true,
-          articles_preview_enabled: true, responsive_web_edit_tweet_api_enabled: true,
-          graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
-          view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true,
-          responsive_web_twitter_article_tweet_consumption_enabled: true,
-          tweet_awards_web_tipping_enabled: false,
-          content_disclosure_indicator_enabled: true, content_disclosure_ai_generated_indicator_enabled: true,
-          freedom_of_speech_not_reach_fetch_enabled: true, standardized_nudges_misinfo: true,
-          tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
-          longform_notetweets_rich_text_read_enabled: true, longform_notetweets_inline_media_enabled: false,
-          responsive_web_enhance_cards_enabled: false
-        });
-        const url = '/i/api/graphql/Fy0QMy4q_aZCpkO0PnyLYw/Bookmarks?variables=' + encodeURIComponent(variables) + '&features=' + encodeURIComponent(features);
-        const resp = await fetch(url, {headers: _h, credentials: 'include'});
-        if (!resp.ok) throw new Error('HTTP ' + resp.status + '. Hint: queryId may have changed.');
-        const d = await resp.json();
-
-        const instructions = d.data?.bookmark_timeline_v2?.timeline?.instructions || d.data?.bookmark_timeline?.timeline?.instructions || [];
-        let tweets = [], seen = new Set();
-        for (const inst of instructions) {
-          for (const entry of (inst.entries || [])) {
-            const r = entry.content?.itemContent?.tweet_results?.result;
-            if (!r) continue;
-            const tw = r.tweet || r;
-            const l = tw.legacy || {};
-            if (!tw.rest_id || seen.has(tw.rest_id)) continue;
-            seen.add(tw.rest_id);
-            const u = tw.core?.user_results?.result;
-            const nt = tw.note_tweet?.note_tweet_results?.result?.text;
-            const screenName = u?.legacy?.screen_name || u?.core?.screen_name;
-            tweets.push({
-              id: tw.rest_id, 
-              author: screenName,
-              name: u?.legacy?.name || u?.core?.name,
-              url: 'https://x.com/' + (screenName || '_') + '/status/' + tw.rest_id,
-              text: nt || l.full_text || '',
-              likes: l.favorite_count, 
-              retweets: l.retweet_count,
-              created_at: l.created_at
-            });
-          }
-        }
-        return tweets;
-      })()
-
-  - map:
-      author: ${{ item.author }}
-      text: ${{ item.text }}
-      likes: ${{ item.likes }}
-      url: ${{ item.url }}
-
-  - limit: ${{ args.limit }}
-
-columns: [author, text, likes, url]