@jackwener/opencli 0.7.8 → 0.7.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1526,8 +1526,7 @@
1526
1526
  {
1527
1527
  "site": "twitter",
1528
1528
  "name": "bookmarks",
1529
- "description": "获取 Twitter 书签列表",
1530
- "domain": "x.com",
1529
+ "description": "Fetch Twitter/X bookmarks",
1531
1530
  "strategy": "cookie",
1532
1531
  "browser": true,
1533
1532
  "args": [
@@ -1536,38 +1535,18 @@
1536
1535
  "type": "int",
1537
1536
  "default": 20,
1538
1537
  "required": false,
1539
- "help": "Number of bookmarks to return (default 20)"
1538
+ "help": ""
1540
1539
  }
1541
1540
  ],
1541
+ "type": "ts",
1542
+ "modulePath": "twitter/bookmarks.js",
1543
+ "domain": "x.com",
1542
1544
  "columns": [
1543
1545
  "author",
1544
1546
  "text",
1545
1547
  "likes",
1546
1548
  "url"
1547
- ],
1548
- "pipeline": [
1549
- {
1550
- "navigate": "https://x.com/i/bookmarks"
1551
- },
1552
- {
1553
- "wait": 2
1554
- },
1555
- {
1556
- "evaluate": "(async () => {\n const ct0 = document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1];\n if (!ct0) throw new Error('No ct0 cookie. Hint: Not logged into x.com.');\n const bearer = decodeURIComponent('AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA');\n const _h = {'Authorization':'Bearer '+bearer, 'X-Csrf-Token':ct0, 'X-Twitter-Auth-Type':'OAuth2Session', 'X-Twitter-Active-User':'yes'};\n\n const count = Math.min(${{ args.limit }}, 100);\n const variables = JSON.stringify({count, includePromotedContent: false});\n const features = JSON.stringify({\n rweb_video_screen_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true,\n responsive_web_profile_redirect_enabled: false, rweb_tipjar_consumption_enabled: false,\n verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true,\n responsive_web_graphql_timeline_navigation_enabled: true,\n responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,\n premium_content_api_read_enabled: false, communities_web_enable_tweet_community_results_fetch: true,\n c9s_tweet_anatomy_moderator_badge_enabled: true,\n articles_preview_enabled: true, responsive_web_edit_tweet_api_enabled: true,\n graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,\n view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true,\n responsive_web_twitter_article_tweet_consumption_enabled: true,\n tweet_awards_web_tipping_enabled: false,\n content_disclosure_indicator_enabled: true, content_disclosure_ai_generated_indicator_enabled: true,\n freedom_of_speech_not_reach_fetch_enabled: true, standardized_nudges_misinfo: true,\n tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,\n longform_notetweets_rich_text_read_enabled: true, longform_notetweets_inline_media_enabled: false,\n responsive_web_enhance_cards_enabled: false\n });\n const url = '/i/api/graphql/Fy0QMy4q_aZCpkO0PnyLYw/Bookmarks?variables=' + encodeURIComponent(variables) + '&features=' + encodeURIComponent(features);\n const resp = await fetch(url, {headers: _h, credentials: 'include'});\n if (!resp.ok) throw new Error('HTTP ' + resp.status + '. Hint: queryId may have changed.');\n const d = await resp.json();\n\n const instructions = d.data?.bookmark_timeline_v2?.timeline?.instructions || d.data?.bookmark_timeline?.timeline?.instructions || [];\n let tweets = [], seen = new Set();\n for (const inst of instructions) {\n for (const entry of (inst.entries || [])) {\n const r = entry.content?.itemContent?.tweet_results?.result;\n if (!r) continue;\n const tw = r.tweet || r;\n const l = tw.legacy || {};\n if (!tw.rest_id || seen.has(tw.rest_id)) continue;\n seen.add(tw.rest_id);\n const u = tw.core?.user_results?.result;\n const nt = tw.note_tweet?.note_tweet_results?.result?.text;\n const screenName = u?.legacy?.screen_name || u?.core?.screen_name;\n tweets.push({\n id: tw.rest_id, \n author: screenName,\n name: u?.legacy?.name || u?.core?.name,\n url: 'https://x.com/' + (screenName || '_') + '/status/' + tw.rest_id,\n text: nt || l.full_text || '',\n likes: l.favorite_count, \n retweets: l.retweet_count,\n created_at: l.created_at\n });\n }\n }\n return tweets;\n})()\n"
1557
- },
1558
- {
1559
- "map": {
1560
- "author": "${{ item.author }}",
1561
- "text": "${{ item.text }}",
1562
- "likes": "${{ item.likes }}",
1563
- "url": "${{ item.url }}"
1564
- }
1565
- },
1566
- {
1567
- "limit": "${{ args.limit }}"
1568
- }
1569
- ],
1570
- "type": "yaml"
1549
+ ]
1571
1550
  },
1572
1551
  {
1573
1552
  "site": "twitter",
@@ -1877,7 +1856,7 @@
1877
1856
  {
1878
1857
  "site": "twitter",
1879
1858
  "name": "timeline",
1880
- "description": "Twitter Home Timeline",
1859
+ "description": "Fetch Twitter Home Timeline",
1881
1860
  "strategy": "cookie",
1882
1861
  "browser": true,
1883
1862
  "args": [
@@ -1893,8 +1872,15 @@
1893
1872
  "modulePath": "twitter/timeline.js",
1894
1873
  "domain": "x.com",
1895
1874
  "columns": [
1896
- "responseType",
1897
- "first"
1875
+ "id",
1876
+ "author",
1877
+ "text",
1878
+ "likes",
1879
+ "retweets",
1880
+ "replies",
1881
+ "views",
1882
+ "created_at",
1883
+ "url"
1898
1884
  ]
1899
1885
  },
1900
1886
  {
@@ -1923,7 +1909,7 @@
1923
1909
  "navigate": "https://x.com/explore/tabs/trending"
1924
1910
  },
1925
1911
  {
1926
- "evaluate": "(async () => {\n const cookies = document.cookie.split(';').reduce((acc, c) => {\n const [k, v] = c.trim().split('=');\n acc[k] = v;\n return acc;\n }, {});\n const csrfToken = cookies['ct0'] || '';\n const bearerToken = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';\n const res = await fetch('/i/api/2/guide.json?include_page_configuration=true', {\n credentials: 'include',\n headers: { 'x-twitter-active-user': 'yes', 'x-csrf-token': csrfToken, 'authorization': 'Bearer ' + bearerToken }\n });\n const data = await res.json();\n const trends = data?.timeline?.instructions?.[1]?.addEntries?.entries || [];\n return trends.filter(e => e.content?.timelineModule).flatMap(e => e.content.timelineModule.items || []).map(t => t?.item?.content?.trend).filter(Boolean);\n})()\n"
1912
+ "evaluate": "(async () => {\n const cookies = document.cookie.split(';').reduce((acc, c) => {\n const [k, v] = c.trim().split('=');\n acc[k] = v;\n return acc;\n }, {});\n const csrfToken = cookies['ct0'] || '';\n const bearerToken = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';\n const res = await fetch('/i/api/2/guide.json?include_page_configuration=true', {\n credentials: 'include',\n headers: { 'x-twitter-active-user': 'yes', 'x-csrf-token': csrfToken, 'authorization': 'Bearer ' + bearerToken }\n });\n if (!res.ok) throw new Error('HTTP ' + res.status + '. Hint: trending endpoint may require login or API shape changed.');\n const data = await res.json();\n const instructions = data?.timeline?.instructions || [];\n const entries = instructions.flatMap(inst => inst?.addEntries?.entries || inst?.entries || []);\n return entries\n .filter(e => e.content?.timelineModule)\n .flatMap(e => e.content.timelineModule.items || [])\n .map(t => t?.item?.content?.trend)\n .filter(Boolean);\n})()\n"
1927
1913
  },
1928
1914
  {
1929
1915
  "map": {
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,171 @@
1
+ import { cli, Strategy } from '../../registry.js';
2
+ const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
3
+ const BOOKMARKS_QUERY_ID = 'Fy0QMy4q_aZCpkO0PnyLYw';
4
+ const FEATURES = {
5
+ rweb_video_screen_enabled: false,
6
+ profile_label_improvements_pcf_label_in_post_enabled: true,
7
+ responsive_web_profile_redirect_enabled: false,
8
+ rweb_tipjar_consumption_enabled: false,
9
+ verified_phone_label_enabled: false,
10
+ creator_subscriptions_tweet_preview_api_enabled: true,
11
+ responsive_web_graphql_timeline_navigation_enabled: true,
12
+ responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
13
+ premium_content_api_read_enabled: false,
14
+ communities_web_enable_tweet_community_results_fetch: true,
15
+ c9s_tweet_anatomy_moderator_badge_enabled: true,
16
+ articles_preview_enabled: true,
17
+ responsive_web_edit_tweet_api_enabled: true,
18
+ graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
19
+ view_counts_everywhere_api_enabled: true,
20
+ longform_notetweets_consumption_enabled: true,
21
+ responsive_web_twitter_article_tweet_consumption_enabled: true,
22
+ tweet_awards_web_tipping_enabled: false,
23
+ content_disclosure_indicator_enabled: true,
24
+ content_disclosure_ai_generated_indicator_enabled: true,
25
+ freedom_of_speech_not_reach_fetch_enabled: true,
26
+ standardized_nudges_misinfo: true,
27
+ tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
28
+ longform_notetweets_rich_text_read_enabled: true,
29
+ longform_notetweets_inline_media_enabled: false,
30
+ responsive_web_enhance_cards_enabled: false,
31
+ };
32
+ function buildBookmarksUrl(count, cursor) {
33
+ const vars = {
34
+ count,
35
+ includePromotedContent: false,
36
+ };
37
+ if (cursor)
38
+ vars.cursor = cursor;
39
+ return `/i/api/graphql/${BOOKMARKS_QUERY_ID}/Bookmarks`
40
+ + `?variables=${encodeURIComponent(JSON.stringify(vars))}`
41
+ + `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`;
42
+ }
43
+ function extractBookmarkTweet(result, seen) {
44
+ if (!result)
45
+ return null;
46
+ const tw = result.tweet || result;
47
+ const legacy = tw.legacy || {};
48
+ if (!tw.rest_id || seen.has(tw.rest_id))
49
+ return null;
50
+ seen.add(tw.rest_id);
51
+ const user = tw.core?.user_results?.result;
52
+ const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';
53
+ const displayName = user?.legacy?.name || user?.core?.name || '';
54
+ const noteText = tw.note_tweet?.note_tweet_results?.result?.text;
55
+ return {
56
+ id: tw.rest_id,
57
+ author: screenName,
58
+ name: displayName,
59
+ text: noteText || legacy.full_text || '',
60
+ likes: legacy.favorite_count || 0,
61
+ retweets: legacy.retweet_count || 0,
62
+ created_at: legacy.created_at || '',
63
+ url: `https://x.com/${screenName}/status/${tw.rest_id}`,
64
+ };
65
+ }
66
+ function parseBookmarks(data, seen) {
67
+ const tweets = [];
68
+ let nextCursor = null;
69
+ const instructions = data?.data?.bookmark_timeline_v2?.timeline?.instructions
70
+ || data?.data?.bookmark_timeline?.timeline?.instructions
71
+ || [];
72
+ for (const inst of instructions) {
73
+ for (const entry of inst.entries || []) {
74
+ const content = entry.content;
75
+ if (content?.entryType === 'TimelineTimelineCursor' || content?.__typename === 'TimelineTimelineCursor') {
76
+ if (content.cursorType === 'Bottom' || content.cursorType === 'ShowMore')
77
+ nextCursor = content.value;
78
+ continue;
79
+ }
80
+ if (entry.entryId?.startsWith('cursor-bottom-') || entry.entryId?.startsWith('cursor-showMore-')) {
81
+ nextCursor = content?.value || content?.itemContent?.value || nextCursor;
82
+ continue;
83
+ }
84
+ const direct = extractBookmarkTweet(content?.itemContent?.tweet_results?.result, seen);
85
+ if (direct) {
86
+ tweets.push(direct);
87
+ continue;
88
+ }
89
+ for (const item of content?.items || []) {
90
+ const nested = extractBookmarkTweet(item.item?.itemContent?.tweet_results?.result, seen);
91
+ if (nested)
92
+ tweets.push(nested);
93
+ }
94
+ }
95
+ }
96
+ return { tweets, nextCursor };
97
+ }
98
+ cli({
99
+ site: 'twitter',
100
+ name: 'bookmarks',
101
+ description: 'Fetch Twitter/X bookmarks',
102
+ domain: 'x.com',
103
+ strategy: Strategy.COOKIE,
104
+ browser: true,
105
+ args: [
106
+ { name: 'limit', type: 'int', default: 20 },
107
+ ],
108
+ columns: ['author', 'text', 'likes', 'url'],
109
+ func: async (page, kwargs) => {
110
+ const limit = kwargs.limit || 20;
111
+ await page.goto('https://x.com');
112
+ await page.wait(3);
113
+ const ct0 = await page.evaluate(`() => {
114
+ return document.cookie.split(';').map(c => c.trim()).find(c => c.startsWith('ct0='))?.split('=')[1] || null;
115
+ }`);
116
+ if (!ct0)
117
+ throw new Error('Not logged into x.com (no ct0 cookie)');
118
+ const queryId = await page.evaluate(`async () => {
119
+ try {
120
+ const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json');
121
+ if (ghResp.ok) {
122
+ const data = await ghResp.json();
123
+ const entry = data['Bookmarks'];
124
+ if (entry && entry.queryId) return entry.queryId;
125
+ }
126
+ } catch {}
127
+ try {
128
+ const scripts = performance.getEntriesByType('resource')
129
+ .filter(r => r.name.includes('client-web') && r.name.endsWith('.js'))
130
+ .map(r => r.name);
131
+ for (const scriptUrl of scripts.slice(0, 15)) {
132
+ try {
133
+ const text = await (await fetch(scriptUrl)).text();
134
+ const re = /queryId:"([A-Za-z0-9_-]+)"[^}]{0,200}operationName:"Bookmarks"/;
135
+ const m = text.match(re);
136
+ if (m) return m[1];
137
+ } catch {}
138
+ }
139
+ } catch {}
140
+ return null;
141
+ }`) || BOOKMARKS_QUERY_ID;
142
+ const headers = JSON.stringify({
143
+ 'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`,
144
+ 'X-Csrf-Token': ct0,
145
+ 'X-Twitter-Auth-Type': 'OAuth2Session',
146
+ 'X-Twitter-Active-User': 'yes',
147
+ });
148
+ const allTweets = [];
149
+ const seen = new Set();
150
+ let cursor = null;
151
+ for (let i = 0; i < 5 && allTweets.length < limit; i++) {
152
+ const fetchCount = Math.min(100, limit - allTweets.length + 10);
153
+ const apiUrl = buildBookmarksUrl(fetchCount, cursor).replace(BOOKMARKS_QUERY_ID, queryId);
154
+ const data = await page.evaluate(`async () => {
155
+ const r = await fetch("${apiUrl}", { headers: ${headers}, credentials: 'include' });
156
+ return r.ok ? await r.json() : { error: r.status };
157
+ }`);
158
+ if (data?.error) {
159
+ if (allTweets.length === 0)
160
+ throw new Error(`HTTP ${data.error}: Failed to fetch bookmarks. queryId may have expired.`);
161
+ break;
162
+ }
163
+ const { tweets, nextCursor } = parseBookmarks(data, seen);
164
+ allTweets.push(...tweets);
165
+ if (!nextCursor || nextCursor === cursor)
166
+ break;
167
+ cursor = nextCursor;
168
+ }
169
+ return allTweets.slice(0, limit);
170
+ },
171
+ });
@@ -13,7 +13,6 @@ cli({
13
13
  func: async (page, kwargs) => {
14
14
  if (!page)
15
15
  throw new Error('Requires browser');
16
- console.log(`Navigating to tweet: ${kwargs.url}`);
17
16
  await page.goto(kwargs.url);
18
17
  await page.wait(5); // Wait for tweet to load completely
19
18
  const result = await page.evaluate(`(async () => {
@@ -30,8 +30,8 @@ cli({
30
30
  // 1. Navigate to user profile page
31
31
  await page.goto(`https://x.com/${targetUser}`);
32
32
  await page.wait(3);
33
- // 2. Inject interceptor for Followers GraphQL API (or user_flow.json)
34
- await page.installInterceptor('graphql');
33
+ // 2. Inject interceptor for the followers GraphQL API
34
+ await page.installInterceptor('Followers');
35
35
  // 3. Click the followers link inside the profile page
36
36
  await page.evaluate(`() => {
37
37
  const target = '${targetUser}';
@@ -43,23 +43,12 @@ cli({
43
43
  await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 });
44
44
  // 4. Retrieve data from opencli's registered interceptors
45
45
  const allRequests = await page.getInterceptedRequests();
46
- // Debug: Force dump all intercepted XHRs that match followers
47
- if (!allRequests || allRequests.length === 0) {
48
- console.log('No GraphQL requests captured by the interceptor backend.');
46
+ const requestList = Array.isArray(allRequests) ? allRequests : [];
47
+ if (requestList.length === 0) {
49
48
  return [];
50
49
  }
51
- console.log('Intercepted keys:', allRequests.map((r) => {
52
- try {
53
- const u = new URL(r.url);
54
- return u.pathname;
55
- }
56
- catch (e) {
57
- return r.url;
58
- }
59
- }));
60
- const requests = allRequests.filter((r) => r.url.includes('Followers'));
50
+ const requests = requestList.filter((r) => r?.url?.includes('Followers'));
61
51
  if (!requests || requests.length === 0) {
62
- console.log('No specific Followers requests captured. Check keys printed above.');
63
52
  return [];
64
53
  }
65
54
  let results = [];
@@ -43,13 +43,12 @@ cli({
43
43
  await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 });
44
44
  // 4. Retrieve data from opencli's registered interceptors
45
45
  const requests = await page.getInterceptedRequests();
46
- // Debug: Force dump all intercepted XHRs that match following
47
- if (!requests || requests.length === 0) {
48
- console.log('No Following requests captured by the interceptor backend.');
46
+ const requestList = Array.isArray(requests) ? requests : [];
47
+ if (requestList.length === 0) {
49
48
  return [];
50
49
  }
51
50
  let results = [];
52
- for (const req of requests) {
51
+ for (const req of requestList) {
53
52
  try {
54
53
  let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions;
55
54
  if (!instructions)
@@ -13,7 +13,6 @@ cli({
13
13
  func: async (page, kwargs) => {
14
14
  if (!page)
15
15
  throw new Error('Requires browser');
16
- console.log(`Navigating to tweet: ${kwargs.url}`);
17
16
  await page.goto(kwargs.url);
18
17
  await page.wait(5); // Wait for tweet to load completely
19
18
  const result = await page.evaluate(`(async () => {
@@ -11,11 +11,14 @@ cli({
11
11
  ],
12
12
  columns: ['id', 'action', 'author', 'text', 'url'],
13
13
  func: async (page, kwargs) => {
14
+ // Install the interceptor before loading the notifications page so we
15
+ // capture the initial timeline request triggered during page load.
16
+ await page.goto('https://x.com');
17
+ await page.wait(2);
18
+ await page.installInterceptor('NotificationsTimeline');
14
19
  // 1. Navigate to notifications
15
20
  await page.goto('https://x.com/notifications');
16
21
  await page.wait(5);
17
- // 2. Inject interceptor
18
- await page.installInterceptor('NotificationsTimeline');
19
22
  // 3. Trigger API by scrolling (if we need to load more)
20
23
  await page.autoScroll({ times: 2, delayMs: 2000 });
21
24
  // 4. Retrieve data
@@ -23,6 +26,7 @@ cli({
23
26
  if (!requests || requests.length === 0)
24
27
  return [];
25
28
  let results = [];
29
+ const seen = new Set();
26
30
  for (const req of requests) {
27
31
  try {
28
32
  let instructions = [];
@@ -65,13 +69,15 @@ cli({
65
69
  if (item.__typename === 'TimelineNotification') {
66
70
  // Greet likes, retweet, mentions
67
71
  text = item.rich_message?.text || item.message?.text || '';
68
- author = item.template?.from_users?.[0]?.user_results?.result?.core?.screen_name || 'unknown';
72
+ const fromUser = item.template?.from_users?.[0]?.user_results?.result;
73
+ author = fromUser?.legacy?.screen_name || fromUser?.core?.screen_name || 'unknown';
69
74
  urlStr = item.notification_url?.url || '';
70
75
  actionText = item.notification_icon || 'Activity';
71
76
  // If there's an attached tweet
72
77
  const targetTweet = item.template?.target_objects?.[0]?.tweet_results?.result;
73
78
  if (targetTweet) {
74
- text += ' | ' + (targetTweet.legacy?.full_text || '');
79
+ const targetText = targetTweet.note_tweet?.note_tweet_results?.result?.text || targetTweet.legacy?.full_text || '';
80
+ text += text && targetText ? ' | ' + targetText : targetText;
75
81
  if (!urlStr) {
76
82
  urlStr = `https://x.com/i/status/${targetTweet.rest_id}`;
77
83
  }
@@ -81,18 +87,22 @@ cli({
81
87
  // Direct mention/reply
82
88
  const tweet = item.tweet_result?.result;
83
89
  author = tweet?.core?.user_results?.result?.legacy?.screen_name || 'unknown';
84
- text = tweet?.legacy?.full_text || item.message?.text || '';
90
+ text = tweet?.note_tweet?.note_tweet_results?.result?.text || tweet?.legacy?.full_text || item.message?.text || '';
85
91
  actionText = 'Mention/Reply';
86
92
  urlStr = `https://x.com/i/status/${tweet?.rest_id}`;
87
93
  }
88
94
  else if (item.__typename === 'Tweet') {
89
95
  author = item.core?.user_results?.result?.legacy?.screen_name || 'unknown';
90
- text = item.legacy?.full_text || '';
96
+ text = item.note_tweet?.note_tweet_results?.result?.text || item.legacy?.full_text || '';
91
97
  actionText = 'Mention';
92
98
  urlStr = `https://x.com/i/status/${item.rest_id}`;
93
99
  }
100
+ const id = item.id || item.rest_id || entryId;
101
+ if (seen.has(id))
102
+ return;
103
+ seen.add(id);
94
104
  results.push({
95
- id: item.id || item.rest_id || entryId,
105
+ id,
96
106
  action: actionText,
97
107
  author: author,
98
108
  text: text,
@@ -12,12 +12,15 @@ cli({
12
12
  ],
13
13
  columns: ['id', 'author', 'text', 'likes', 'views', 'url'],
14
14
  func: async (page, kwargs) => {
15
+ // Install the interceptor before opening the target page so we don't miss
16
+ // the initial SearchTimeline request fired during hydration.
17
+ await page.goto('https://x.com');
18
+ await page.wait(2);
19
+ await page.installInterceptor('SearchTimeline');
15
20
  // 1. Navigate to the search page
16
21
  const q = encodeURIComponent(kwargs.query);
17
22
  await page.goto(`https://x.com/search?q=${q}&f=top`);
18
23
  await page.wait(5);
19
- // 2. Inject XHR interceptor
20
- await page.installInterceptor('SearchTimeline');
21
24
  // 3. Trigger API by scrolling
22
25
  await page.autoScroll({ times: 3, delayMs: 2000 });
23
26
  // 4. Retrieve data
@@ -25,11 +28,13 @@ cli({
25
28
  if (!requests || requests.length === 0)
26
29
  return [];
27
30
  let results = [];
31
+ const seen = new Set();
28
32
  for (const req of requests) {
29
33
  try {
30
- const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
31
- const addEntries = insts.find((i) => i.type === 'TimelineAddEntries');
32
- if (!addEntries)
34
+ const insts = req.data?.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || [];
35
+ const addEntries = insts.find((i) => i.type === 'TimelineAddEntries')
36
+ || insts.find((i) => i.entries && Array.isArray(i.entries));
37
+ if (!addEntries?.entries)
33
38
  continue;
34
39
  for (const entry of addEntries.entries) {
35
40
  if (!entry.entryId.startsWith('tweet-'))
@@ -41,10 +46,13 @@ cli({
41
46
  if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
42
47
  tweet = tweet.tweet;
43
48
  }
49
+ if (!tweet.rest_id || seen.has(tweet.rest_id))
50
+ continue;
51
+ seen.add(tweet.rest_id);
44
52
  results.push({
45
53
  id: tweet.rest_id,
46
54
  author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
47
- text: tweet.legacy?.full_text || '',
55
+ text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '',
48
56
  likes: tweet.legacy?.favorite_count || 0,
49
57
  views: tweet.views?.count || '0',
50
58
  url: `https://x.com/i/status/${tweet.rest_id}`