@jackwener/opencli 0.7.8 → 0.7.9

package/src/clis/twitter/timeline.ts

@@ -1,50 +1,218 @@
 import { cli, Strategy } from '../../registry.js';
 
+// ── Twitter GraphQL constants ──────────────────────────────────────────
+
+const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
+const HOME_TIMELINE_QUERY_ID = 'c-CzHF1LboFilMpsx4ZCrQ';
+
+const FEATURES = {
+  rweb_video_screen_enabled: false,
+  profile_label_improvements_pcf_label_in_post_enabled: true,
+  rweb_tipjar_consumption_enabled: true,
+  verified_phone_label_enabled: false,
+  creator_subscriptions_tweet_preview_api_enabled: true,
+  responsive_web_graphql_timeline_navigation_enabled: true,
+  responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+  premium_content_api_read_enabled: false,
+  communities_web_enable_tweet_community_results_fetch: true,
+  c9s_tweet_anatomy_moderator_badge_enabled: true,
+  responsive_web_grok_analyze_button_fetch_trends_enabled: false,
+  responsive_web_grok_analyze_post_followups_enabled: true,
+  responsive_web_jetfuel_frame: false,
+  responsive_web_grok_share_attachment_enabled: true,
+  articles_preview_enabled: true,
+  responsive_web_edit_tweet_api_enabled: true,
+  graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+  view_counts_everywhere_api_enabled: true,
+  longform_notetweets_consumption_enabled: true,
+  responsive_web_twitter_article_tweet_consumption_enabled: true,
+  tweet_awards_web_tipping_enabled: false,
+  responsive_web_grok_show_grok_translated_post: false,
+  responsive_web_grok_analysis_button_from_backend: false,
+  creator_subscriptions_quote_tweet_preview_enabled: false,
+  freedom_of_speech_not_reach_fetch_enabled: true,
+  standardized_nudges_misinfo: true,
+  tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+  longform_notetweets_rich_text_read_enabled: true,
+  longform_notetweets_inline_media_enabled: true,
+  responsive_web_grok_image_annotation_enabled: true,
+  responsive_web_enhance_cards_enabled: false,
+};
+
+// ── Pure functions (type-safe, testable) ───────────────────────────────
+
+interface TimelineTweet {
+  id: string;
+  author: string;
+  text: string;
+  likes: number;
+  retweets: number;
+  replies: number;
+  views: number;
+  created_at: string;
+  url: string;
+}
+
+function buildHomeTimelineUrl(count: number, cursor?: string | null): string {
+  const vars: Record<string, any> = {
+    count,
+    includePromotedContent: false,
+    latestControlAvailable: true,
+    requestContext: 'launch',
+    withCommunity: true,
+  };
+  if (cursor) vars.cursor = cursor;
+
+  return `/i/api/graphql/${HOME_TIMELINE_QUERY_ID}/HomeTimeline`
+    + `?variables=${encodeURIComponent(JSON.stringify(vars))}`
+    + `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`;
+}
+
+function extractTweet(result: any, seen: Set<string>): TimelineTweet | null {
+  if (!result) return null;
+  const tw = result.tweet || result;
+  const l = tw.legacy || {};
+  if (!tw.rest_id || seen.has(tw.rest_id)) return null;
+  seen.add(tw.rest_id);
+
+  const u = tw.core?.user_results?.result;
+  const screenName = u?.legacy?.screen_name || u?.core?.screen_name || 'unknown';
+  const noteText = tw.note_tweet?.note_tweet_results?.result?.text;
+  const views = tw.views?.count ? parseInt(tw.views.count, 10) : 0;
+
+  return {
+    id: tw.rest_id,
+    author: screenName,
+    text: noteText || l.full_text || '',
+    likes: l.favorite_count || 0,
+    retweets: l.retweet_count || 0,
+    replies: l.reply_count || 0,
+    views,
+    created_at: l.created_at || '',
+    url: `https://x.com/${screenName}/status/${tw.rest_id}`,
+  };
+}
+
+function parseHomeTimeline(data: any, seen: Set<string>): { tweets: TimelineTweet[]; nextCursor: string | null } {
+  const tweets: TimelineTweet[] = [];
+  let nextCursor: string | null = null;
+
+  const instructions =
+    data?.data?.home?.home_timeline_urt?.instructions || [];
+
+  for (const inst of instructions) {
+    for (const entry of inst.entries || []) {
+      const c = entry.content;
+
+      // Cursor entries
+      if (c?.entryType === 'TimelineTimelineCursor' || c?.__typename === 'TimelineTimelineCursor') {
+        if (c.cursorType === 'Bottom') nextCursor = c.value;
+        continue;
+      }
+      if (entry.entryId?.startsWith('cursor-bottom-')) {
+        nextCursor = c?.value || nextCursor;
+        continue;
+      }
+
+      // Single tweet entry
+      const tweetResult = c?.itemContent?.tweet_results?.result;
+      if (tweetResult) {
+        // Skip promoted content
+        if (c?.itemContent?.promotedMetadata) continue;
+        const tw = extractTweet(tweetResult, seen);
+        if (tw) tweets.push(tw);
+        continue;
+      }
+
+      // Conversation module (grouped tweets)
+      for (const item of c?.items || []) {
+        const nested = item.item?.itemContent?.tweet_results?.result;
+        if (nested) {
+          if (item.item?.itemContent?.promotedMetadata) continue;
+          const tw = extractTweet(nested, seen);
+          if (tw) tweets.push(tw);
+        }
+      }
+    }
+  }
+
+  return { tweets, nextCursor };
+}
+
+// ── CLI definition ────────────────────────────────────────────────────
+
 cli({
   site: 'twitter',
   name: 'timeline',
-  description: 'Twitter Home Timeline',
+  description: 'Fetch Twitter Home Timeline',
   domain: 'x.com',
   strategy: Strategy.COOKIE,
+  browser: true,
   args: [
     { name: 'limit', type: 'int', default: 20 },
   ],
-  columns: ['responseType', 'first'],
+  columns: ['id', 'author', 'text', 'likes', 'retweets', 'replies', 'views', 'created_at', 'url'],
   func: async (page, kwargs) => {
-    await page.goto('https://x.com/home');
-    await page.wait(5);
-    // Inject the fetch interceptor manually to see exactly what happens
-    await page.evaluate(`
-      () => {
-        window.__intercept_data = [];
-        const origFetch = window.fetch;
-        window.fetch = async function(...args) {
-          let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
-          const res = await origFetch.apply(this, args);
-          setTimeout(async () => {
-            try {
-              if (u.includes('HomeTimeline')) {
-                const clone = res.clone();
-                const j = await clone.json();
-                window.__intercept_data.push(j);
-              }
-            } catch(e) {}
-          }, 0);
-          return res;
-        };
+    const limit = kwargs.limit || 20;
+
+    // Navigate to x.com for cookie context
+    await page.goto('https://x.com');
+    await page.wait(3);
+
+    // Extract CSRF token
+    const ct0 = await page.evaluate(`() => {
+      return document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1] || null;
+    }`);
+    if (!ct0) throw new Error('Not logged into x.com (no ct0 cookie)');
+
+    // Dynamically resolve queryId
+    const queryId = await page.evaluate(`async () => {
+      try {
+        const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json');
+        if (ghResp.ok) {
+          const data = await ghResp.json();
+          const entry = data['HomeTimeline'];
+          if (entry && entry.queryId) return entry.queryId;
+        }
+      } catch {}
+      return null;
+    }`) || HOME_TIMELINE_QUERY_ID;
+
+    // Build auth headers
+    const headers = JSON.stringify({
+      'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`,
+      'X-Csrf-Token': ct0,
+      'X-Twitter-Auth-Type': 'OAuth2Session',
+      'X-Twitter-Active-User': 'yes',
+    });
+
+    // Paginate — fetch in browser, parse in TypeScript
+    const allTweets: TimelineTweet[] = [];
+    const seen = new Set<string>();
+    let cursor: string | null = null;
+
+    for (let i = 0; i < 5 && allTweets.length < limit; i++) {
+      const fetchCount = Math.min(40, limit - allTweets.length + 5); // over-fetch slightly for promoted filtering
+      const apiUrl = buildHomeTimelineUrl(fetchCount, cursor)
+        .replace(HOME_TIMELINE_QUERY_ID, queryId);
+
+      const data = await page.evaluate(`async () => {
+        const r = await fetch("${apiUrl}", { headers: ${headers}, credentials: 'include' });
+        return r.ok ? await r.json() : { error: r.status };
+      }`);
+
+      if (data?.error) {
+        if (allTweets.length === 0) throw new Error(`HTTP ${data.error}: Failed to fetch timeline. queryId may have expired.`);
+        break;
       }
-    `);
-    
-    // trigger scroll
-    for(let i=0; i<3; i++) {
-      await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
-      await page.wait(2);
+
+      const { tweets, nextCursor } = parseHomeTimeline(data, seen);
+      allTweets.push(...tweets);
+
+      if (!nextCursor || nextCursor === cursor) break;
+      cursor = nextCursor;
     }
-    
-    // extract
-    const data = await page.evaluate('() => window.__intercept_data');
-    if (!data || data.length === 0) return [{responseType: 'no data captured'}];
-    
-    return [{responseType: `captured ${data.length} responses`, first: JSON.stringify(data[0]).substring(0,300)}];
-  }
+
+    return allTweets.slice(0, limit);
+  },
 });

package/src/setup.ts

@@ -184,11 +184,17 @@ export async function runSetup(opts: { cliVersion?: string; token?: string } = {
     if (result.ok) {
       console.log(`  ${chalk.green('✓')} Browser connected in ${(result.durationMs / 1000).toFixed(1)}s`);
     } else {
+      console.log(`  ${chalk.green('✓')} Token saved successfully.`);
       console.log(`  ${chalk.yellow('!')} Browser connectivity test failed: ${result.error ?? 'unknown'}`);
-      console.log(chalk.dim('    Make sure Chrome is running with the extension enabled.'));
+      console.log(chalk.dim('    Token configuration is complete. To use opencli, make sure Chrome'));
+      console.log(chalk.dim('    is running with the Playwright MCP Bridge extension enabled.'));
+      console.log(chalk.dim(`    Run ${chalk.bold('opencli doctor --live')} to re-test connectivity.`));
     }
   } catch {
-    console.log(`  ${chalk.yellow('!')} Could not verify connectivity (Chrome may not be running)`);
+    console.log(`  ${chalk.green('✓')} Token saved successfully.`);
+    console.log(`  ${chalk.yellow('!')} Browser connectivity test skipped (Chrome may not be running).`);
+    console.log(chalk.dim('    Token configuration is complete. Start Chrome to begin using opencli.'));
+    console.log(chalk.dim(`    Run ${chalk.bold('opencli doctor --live')} to re-test connectivity.`));
   }
   console.log();
 }