@jackwener/opencli 0.7.9 → 0.7.11

package/src/clis/barchart/options.ts

@@ -0,0 +1,110 @@
+/**
+ * Barchart options chain — strike, bid/ask, volume, OI, greeks, IV.
+ * Auth: CSRF token from <meta name="csrf-token"> + session cookies.
+ */
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'barchart',
+  name: 'options',
+  description: 'Barchart options chain with greeks, IV, volume, and open interest',
+  domain: 'www.barchart.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'symbol', required: true, help: 'Stock ticker (e.g. AAPL)' },
+    { name: 'type', type: 'str', default: 'Call', help: 'Option type: Call or Put', choices: ['Call', 'Put'] },
+    { name: 'limit', type: 'int', default: 20, help: 'Max number of strikes to return' },
+  ],
+  columns: [
+    'strike', 'bid', 'ask', 'last', 'change', 'volume', 'openInterest',
+    'iv', 'delta', 'gamma', 'theta', 'vega', 'expiration',
+  ],
+  func: async (page, kwargs) => {
+    const symbol = kwargs.symbol.toUpperCase().trim();
+    const optType = kwargs.type || 'Call';
+    const limit = kwargs.limit ?? 20;
+
+    await page.goto(`https://www.barchart.com/stocks/quotes/${encodeURIComponent(symbol)}/options`);
+    await page.wait(4);
+
+    const data = await page.evaluate(`
+      (async () => {
+        const sym = '${symbol}';
+        const type = '${optType}';
+        const limit = ${limit};
+        const csrf = document.querySelector('meta[name="csrf-token"]')?.content || '';
+        const headers = { 'X-CSRF-TOKEN': csrf };
+
+        // API: options chain with greeks
+        try {
+          const fields = [
+            'strikePrice','bidPrice','askPrice','lastPrice','priceChange',
+            'volume','openInterest','volatility',
+            'delta','gamma','theta','vega',
+            'expirationDate','optionType','percentFromLast',
+          ].join(',');
+
+          const url = '/proxies/core-api/v1/options/chain?symbol=' + encodeURIComponent(sym)
+            + '&fields=' + fields + '&raw=1';
+          const resp = await fetch(url, { credentials: 'include', headers });
+          if (resp.ok) {
+            const d = await resp.json();
+            let items = d?.data || [];
+
+            // Filter by type
+            items = items.filter(i => {
+              const t = (i.raw || i).optionType || '';
+              return t.toLowerCase() === type.toLowerCase();
+            });
+
+            // Sort by closeness to current price
+            items.sort((a, b) => {
+              const aD = Math.abs((a.raw || a).percentFromLast || 999);
+              const bD = Math.abs((b.raw || b).percentFromLast || 999);
+              return aD - bD;
+            });
+
+            return items.slice(0, limit).map(i => {
+              const r = i.raw || i;
+              return {
+                strike: r.strikePrice,
+                bid: r.bidPrice,
+                ask: r.askPrice,
+                last: r.lastPrice,
+                change: r.priceChange,
+                volume: r.volume,
+                openInterest: r.openInterest,
+                iv: r.volatility,
+                delta: r.delta,
+                gamma: r.gamma,
+                theta: r.theta,
+                vega: r.vega,
+                expiration: r.expirationDate,
+              };
+            });
+          }
+        } catch(e) {}
+
+        return [];
+      })()
+    `);
+
+    if (!data || !Array.isArray(data)) return [];
+
+    return data.map(r => ({
+      strike: r.strike,
+      bid: r.bid != null ? Number(Number(r.bid).toFixed(2)) : null,
+      ask: r.ask != null ? Number(Number(r.ask).toFixed(2)) : null,
+      last: r.last != null ? Number(Number(r.last).toFixed(2)) : null,
+      change: r.change != null ? Number(Number(r.change).toFixed(2)) : null,
+      volume: r.volume,
+      openInterest: r.openInterest,
+      iv: r.iv != null ? Number(Number(r.iv).toFixed(2)) + '%' : null,
+      delta: r.delta != null ? Number(Number(r.delta).toFixed(4)) : null,
+      gamma: r.gamma != null ? Number(Number(r.gamma).toFixed(4)) : null,
+      theta: r.theta != null ? Number(Number(r.theta).toFixed(4)) : null,
+      vega: r.vega != null ? Number(Number(r.vega).toFixed(4)) : null,
+      expiration: r.expiration ?? null,
+    }));
+  },
+});

package/src/clis/barchart/quote.ts

@@ -0,0 +1,137 @@
+/**
+ * Barchart stock quote — price, volume, market cap, P/E, EPS, and key metrics.
+ * Auth: CSRF token from <meta name="csrf-token"> + session cookies.
+ */
+import { cli, Strategy } from '../../registry.js';
+
+cli({
+  site: 'barchart',
+  name: 'quote',
+  description: 'Barchart stock quote with price, volume, and key metrics',
+  domain: 'www.barchart.com',
+  strategy: Strategy.COOKIE,
+  args: [
+    { name: 'symbol', required: true, help: 'Stock ticker (e.g. AAPL, MSFT, TSLA)' },
+  ],
+  columns: [
+    'symbol', 'name', 'price', 'change', 'changePct',
+    'open', 'high', 'low', 'prevClose', 'volume',
+    'avgVolume', 'marketCap', 'peRatio', 'eps',
+  ],
+  func: async (page, kwargs) => {
+    const symbol = kwargs.symbol.toUpperCase().trim();
+    await page.goto(`https://www.barchart.com/stocks/quotes/${encodeURIComponent(symbol)}/overview`);
+    await page.wait(4);
+
+    const data = await page.evaluate(`
+      (async () => {
+        const sym = '${symbol}';
+        const csrf = document.querySelector('meta[name="csrf-token"]')?.content || '';
+
+        // Strategy 1: internal proxy API with CSRF token
+        try {
+          const fields = [
+            'symbol','symbolName','lastPrice','priceChange','percentChange',
+            'highPrice','lowPrice','openPrice','previousPrice','volume','averageVolume',
+            'marketCap','peRatio','earningsPerShare','tradeTime',
+          ].join(',');
+          const url = '/proxies/core-api/v1/quotes/get?symbol=' + encodeURIComponent(sym) + '&fields=' + fields;
+          const resp = await fetch(url, {
+            credentials: 'include',
+            headers: { 'X-CSRF-TOKEN': csrf },
+          });
+          if (resp.ok) {
+            const d = await resp.json();
+            const row = d?.data?.[0] || null;
+            if (row) {
+              return { source: 'api', row };
+            }
+          }
+        } catch(e) {}
+
+        // Strategy 2: parse from DOM
+        try {
+          const priceEl = document.querySelector('span.last-change');
+          const price = priceEl ? priceEl.textContent.trim() : null;
+
+          // Change values are sibling spans inside .pricechangerow > .last-change
+          const changeParent = priceEl?.parentElement;
+          const changeSpans = changeParent ? changeParent.querySelectorAll('span') : [];
+          let change = null;
+          let changePct = null;
+          for (const s of changeSpans) {
+            const t = s.textContent.trim();
+            if (s === priceEl) continue;
+            if (t.includes('%')) changePct = t.replace(/[()]/g, '');
+            else if (t.match(/^[+-]?[\\d.]+$/)) change = t;
+          }
+
+          // Financial data rows
+          const rows = document.querySelectorAll('.financial-data-row');
+          const fdata = {};
+          for (const row of rows) {
+            const spans = row.querySelectorAll('span');
+            if (spans.length >= 2) {
+              const label = spans[0].textContent.trim();
+              const valSpan = row.querySelector('span.right span:not(.ng-hide)');
+              fdata[label] = valSpan ? valSpan.textContent.trim() : '';
+            }
+          }
+
+          // Day high/low from row chart
+          const dayLow = document.querySelector('.bc-quote-row-chart .small-6:first-child .inline:not(.ng-hide)');
+          const dayHigh = document.querySelector('.bc-quote-row-chart .text-right .inline:not(.ng-hide)');
+          const openEl = document.querySelector('.mark span');
+          const openText = openEl ? openEl.textContent.trim().replace('Open ', '') : null;
+
+          const name = document.querySelector('h1 span.symbol');
+
+          return {
+            source: 'dom',
+            row: {
+              symbol: sym,
+              symbolName: name ? name.textContent.trim() : sym,
+              lastPrice: price,
+              priceChange: change,
+              percentChange: changePct,
+              open: openText,
+              highPrice: dayHigh ? dayHigh.textContent.trim() : null,
+              lowPrice: dayLow ? dayLow.textContent.trim() : null,
+              previousClose: fdata['Previous Close'] || null,
+              volume: fdata['Volume'] || null,
+              averageVolume: fdata['Average Volume'] || null,
+              marketCap: null,
+              peRatio: null,
+              earningsPerShare: null,
+            }
+          };
+        } catch(e) {
+          return { error: 'Could not fetch quote for ' + sym + ': ' + e.message };
+        }
+      })()
+    `);
+
+    if (!data || data.error) return [];
+
+    const r = data.row || {};
+    // API returns formatted strings like "+1.41" and "+0.56%"; use raw if available
+    const raw = r.raw || {};
+
+    return [{
+      symbol: r.symbol || symbol,
+      name: r.symbolName || r.name || symbol,
+      price: r.lastPrice ?? null,
+      change: r.priceChange ?? null,
+      changePct: r.percentChange ?? null,
+      open: r.openPrice ?? r.open ?? null,
+      high: r.highPrice ?? null,
+      low: r.lowPrice ?? null,
+      prevClose: r.previousPrice ?? r.previousClose ?? null,
+      volume: r.volume ?? null,
+      avgVolume: r.averageVolume ?? null,
+      marketCap: r.marketCap ?? null,
+      peRatio: r.peRatio ?? null,
+      eps: r.earningsPerShare ?? null,
+    }];
+  },
+});

package/src/clis/twitter/bookmarks.ts

@@ -0,0 +1,201 @@
+import { cli, Strategy } from '../../registry.js';
+
+const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
+const BOOKMARKS_QUERY_ID = 'Fy0QMy4q_aZCpkO0PnyLYw';
+
+const FEATURES = {
+  rweb_video_screen_enabled: false,
+  profile_label_improvements_pcf_label_in_post_enabled: true,
+  responsive_web_profile_redirect_enabled: false,
+  rweb_tipjar_consumption_enabled: false,
+  verified_phone_label_enabled: false,
+  creator_subscriptions_tweet_preview_api_enabled: true,
+  responsive_web_graphql_timeline_navigation_enabled: true,
+  responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+  premium_content_api_read_enabled: false,
+  communities_web_enable_tweet_community_results_fetch: true,
+  c9s_tweet_anatomy_moderator_badge_enabled: true,
+  articles_preview_enabled: true,
+  responsive_web_edit_tweet_api_enabled: true,
+  graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+  view_counts_everywhere_api_enabled: true,
+  longform_notetweets_consumption_enabled: true,
+  responsive_web_twitter_article_tweet_consumption_enabled: true,
+  tweet_awards_web_tipping_enabled: false,
+  content_disclosure_indicator_enabled: true,
+  content_disclosure_ai_generated_indicator_enabled: true,
+  freedom_of_speech_not_reach_fetch_enabled: true,
+  standardized_nudges_misinfo: true,
+  tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+  longform_notetweets_rich_text_read_enabled: true,
+  longform_notetweets_inline_media_enabled: false,
+  responsive_web_enhance_cards_enabled: false,
+};
+
+interface BookmarkTweet {
+  id: string;
+  author: string;
+  name: string;
+  text: string;
+  likes: number;
+  retweets: number;
+  created_at: string;
+  url: string;
+}
+
+function buildBookmarksUrl(count: number, cursor?: string | null): string {
+  const vars: Record<string, any> = {
+    count,
+    includePromotedContent: false,
+  };
+  if (cursor) vars.cursor = cursor;
+
+  return `/i/api/graphql/${BOOKMARKS_QUERY_ID}/Bookmarks`
+    + `?variables=${encodeURIComponent(JSON.stringify(vars))}`
+    + `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`;
+}
+
+function extractBookmarkTweet(result: any, seen: Set<string>): BookmarkTweet | null {
+  if (!result) return null;
+  const tw = result.tweet || result;
+  const legacy = tw.legacy || {};
+  if (!tw.rest_id || seen.has(tw.rest_id)) return null;
+  seen.add(tw.rest_id);
+
+  const user = tw.core?.user_results?.result;
+  const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';
+  const displayName = user?.legacy?.name || user?.core?.name || '';
+  const noteText = tw.note_tweet?.note_tweet_results?.result?.text;
+
+  return {
+    id: tw.rest_id,
+    author: screenName,
+    name: displayName,
+    text: noteText || legacy.full_text || '',
+    likes: legacy.favorite_count || 0,
+    retweets: legacy.retweet_count || 0,
+    created_at: legacy.created_at || '',
+    url: `https://x.com/${screenName}/status/${tw.rest_id}`,
+  };
+}
+
+function parseBookmarks(data: any, seen: Set<string>): { tweets: BookmarkTweet[]; nextCursor: string | null } {
+  const tweets: BookmarkTweet[] = [];
+  let nextCursor: string | null = null;
+
+  const instructions =
+    data?.data?.bookmark_timeline_v2?.timeline?.instructions
+    || data?.data?.bookmark_timeline?.timeline?.instructions
+    || [];
+
+  for (const inst of instructions) {
+    for (const entry of inst.entries || []) {
+      const content = entry.content;
+
+      if (content?.entryType === 'TimelineTimelineCursor' || content?.__typename === 'TimelineTimelineCursor') {
+        if (content.cursorType === 'Bottom' || content.cursorType === 'ShowMore') nextCursor = content.value;
+        continue;
+      }
+      if (entry.entryId?.startsWith('cursor-bottom-') || entry.entryId?.startsWith('cursor-showMore-')) {
+        nextCursor = content?.value || content?.itemContent?.value || nextCursor;
+        continue;
+      }
+
+      const direct = extractBookmarkTweet(content?.itemContent?.tweet_results?.result, seen);
+      if (direct) {
+        tweets.push(direct);
+        continue;
+      }
+
+      for (const item of content?.items || []) {
+        const nested = extractBookmarkTweet(item.item?.itemContent?.tweet_results?.result, seen);
+        if (nested) tweets.push(nested);
+      }
+    }
+  }
+
+  return { tweets, nextCursor };
+}
+
+cli({
+  site: 'twitter',
+  name: 'bookmarks',
+  description: 'Fetch Twitter/X bookmarks',
+  domain: 'x.com',
+  strategy: Strategy.COOKIE,
+  browser: true,
+  args: [
+    { name: 'limit', type: 'int', default: 20 },
+  ],
+  columns: ['author', 'text', 'likes', 'url'],
+  func: async (page, kwargs) => {
+    const limit = kwargs.limit || 20;
+
+    await page.goto('https://x.com');
+    await page.wait(3);
+
+    const ct0 = await page.evaluate(`() => {
+      return document.cookie.split(';').map(c => c.trim()).find(c => c.startsWith('ct0='))?.split('=')[1] || null;
+    }`);
+    if (!ct0) throw new Error('Not logged into x.com (no ct0 cookie)');
+
+    const queryId = await page.evaluate(`async () => {
+      try {
+        const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json');
+        if (ghResp.ok) {
+          const data = await ghResp.json();
+          const entry = data['Bookmarks'];
+          if (entry && entry.queryId) return entry.queryId;
+        }
+      } catch {}
+      try {
+        const scripts = performance.getEntriesByType('resource')
+          .filter(r => r.name.includes('client-web') && r.name.endsWith('.js'))
+          .map(r => r.name);
+        for (const scriptUrl of scripts.slice(0, 15)) {
+          try {
+            const text = await (await fetch(scriptUrl)).text();
+            const re = /queryId:"([A-Za-z0-9_-]+)"[^}]{0,200}operationName:"Bookmarks"/;
+            const m = text.match(re);
+            if (m) return m[1];
+          } catch {}
+        }
+      } catch {}
+      return null;
+    }`) || BOOKMARKS_QUERY_ID;
+
+    const headers = JSON.stringify({
+      'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`,
+      'X-Csrf-Token': ct0,
+      'X-Twitter-Auth-Type': 'OAuth2Session',
+      'X-Twitter-Active-User': 'yes',
+    });
+
+    const allTweets: BookmarkTweet[] = [];
+    const seen = new Set<string>();
+    let cursor: string | null = null;
+
+    for (let i = 0; i < 5 && allTweets.length < limit; i++) {
+      const fetchCount = Math.min(100, limit - allTweets.length + 10);
+      const apiUrl = buildBookmarksUrl(fetchCount, cursor).replace(BOOKMARKS_QUERY_ID, queryId);
+
+      const data = await page.evaluate(`async () => {
+        const r = await fetch("${apiUrl}", { headers: ${headers}, credentials: 'include' });
+        return r.ok ? await r.json() : { error: r.status };
+      }`);
+
+      if (data?.error) {
+        if (allTweets.length === 0) throw new Error(`HTTP ${data.error}: Failed to fetch bookmarks. queryId may have expired.`);
+        break;
+      }
+
+      const { tweets, nextCursor } = parseBookmarks(data, seen);
+      allTweets.push(...tweets);
+
+      if (!nextCursor || nextCursor === cursor) break;
+      cursor = nextCursor;
+    }
+
+    return allTweets.slice(0, limit);
+  },
+});

package/src/clis/twitter/delete.ts

@@ -15,7 +15,6 @@ cli({
   func: async (page: IPage | null, kwargs: any) => {
     if (!page) throw new Error('Requires browser');
 
-    console.log(`Navigating to tweet: ${kwargs.url}`);
     await page.goto(kwargs.url);
     await page.wait(5); // Wait for tweet to load completely
 

package/src/clis/twitter/followers.ts

@@ -1,5 +1,4 @@
 import { cli, Strategy } from '../../registry.js';
-import * as fs from 'fs';
 
 cli({
   site: 'twitter',
@@ -37,8 +36,8 @@ cli({
     await page.goto(`https://x.com/${targetUser}`);
     await page.wait(3);
 
-    // 2. Inject interceptor for Followers GraphQL API (or user_flow.json)
-    await page.installInterceptor('graphql');
+    // 2. Inject interceptor for the followers GraphQL API
+    await page.installInterceptor('Followers');
     
     // 3. Click the followers link inside the profile page
     await page.evaluate(`() => {
@@ -53,24 +52,14 @@ cli({
 
     // 4. Retrieve data from opencli's registered interceptors
     const allRequests = await page.getInterceptedRequests();
+    const requestList = Array.isArray(allRequests) ? allRequests : [];
     
-    // Debug: Force dump all intercepted XHRs that match followers
-    if (!allRequests || allRequests.length === 0) {
-       console.log('No GraphQL requests captured by the interceptor backend.');
+    if (requestList.length === 0) {
        return [];
     }
-
-    console.log('Intercepted keys:', allRequests.map((r: any) => {
-      try {
-        const u = new URL(r.url); return u.pathname;
-      } catch (e) {
-        return r.url;
-      }
-    }));
     
-    const requests = allRequests.filter((r: any) => r.url.includes('Followers'));
+    const requests = requestList.filter((r: any) => r?.url?.includes('Followers'));
     if (!requests || requests.length === 0) {
-       console.log('No specific Followers requests captured. Check keys printed above.');
        return [];
     }
 

package/src/clis/twitter/following.ts

@@ -1,5 +1,4 @@
 import { cli, Strategy } from '../../registry.js';
-import * as fs from 'fs';
 
 cli({
   site: 'twitter',
@@ -53,15 +52,14 @@ cli({
 
     // 4. Retrieve data from opencli's registered interceptors
     const requests = await page.getInterceptedRequests();
+    const requestList = Array.isArray(requests) ? requests : [];
     
-    // Debug: Force dump all intercepted XHRs that match following
-    if (!requests || requests.length === 0) {
-       console.log('No Following requests captured by the interceptor backend.');
+    if (requestList.length === 0) {
        return [];
     }
 
     let results: any[] = [];
-    for (const req of requests) {
+    for (const req of requestList) {
       try {
         let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions;
         if (!instructions) continue;

package/src/clis/twitter/like.ts

@@ -15,7 +15,6 @@ cli({
   func: async (page: IPage | null, kwargs: any) => {
     if (!page) throw new Error('Requires browser');
 
-    console.log(`Navigating to tweet: ${kwargs.url}`);
     await page.goto(kwargs.url);
     await page.wait(5); // Wait for tweet to load completely
 

package/src/clis/twitter/notifications.ts

@@ -1,5 +1,4 @@
 import { cli, Strategy } from '../../registry.js';
-import * as fs from 'fs';
 
 cli({
   site: 'twitter',
@@ -13,13 +12,16 @@ cli({
   ],
   columns: ['id', 'action', 'author', 'text', 'url'],
   func: async (page, kwargs) => {
+    // Install the interceptor before loading the notifications page so we
+    // capture the initial timeline request triggered during page load.
+    await page.goto('https://x.com');
+    await page.wait(2);
+    await page.installInterceptor('NotificationsTimeline');
+
     // 1. Navigate to notifications
     await page.goto('https://x.com/notifications');
     await page.wait(5);
 
-    // 2. Inject interceptor
-    await page.installInterceptor('NotificationsTimeline');
-
     // 3. Trigger API by scrolling (if we need to load more)
     await page.autoScroll({ times: 2, delayMs: 2000 });
 
@@ -28,6 +30,7 @@ cli({
     if (!requests || requests.length === 0) return [];
 
     let results: any[] = [];
+    const seen = new Set<string>();
     for (const req of requests) {
       try {
         let instructions: any[] = [];
@@ -75,14 +78,16 @@ cli({
             if (item.__typename === 'TimelineNotification') {
                  // Greet likes, retweet, mentions
                  text = item.rich_message?.text || item.message?.text || '';
-                 author = item.template?.from_users?.[0]?.user_results?.result?.core?.screen_name || 'unknown';
+                 const fromUser = item.template?.from_users?.[0]?.user_results?.result;
+                 author = fromUser?.legacy?.screen_name || fromUser?.core?.screen_name || 'unknown';
                  urlStr = item.notification_url?.url || '';
                  actionText = item.notification_icon || 'Activity';
                  
                  // If there's an attached tweet
                  const targetTweet = item.template?.target_objects?.[0]?.tweet_results?.result;
                  if (targetTweet) {
-                    text += ' | ' + (targetTweet.legacy?.full_text || '');
+                    const targetText = targetTweet.note_tweet?.note_tweet_results?.result?.text || targetTweet.legacy?.full_text || '';
+                    text += text && targetText ? ' | ' + targetText : targetText;
                     if (!urlStr) {
                          urlStr = `https://x.com/i/status/${targetTweet.rest_id}`;
                     }
@@ -91,18 +96,22 @@ cli({
                  // Direct mention/reply
                  const tweet = item.tweet_result?.result;
                  author = tweet?.core?.user_results?.result?.legacy?.screen_name || 'unknown';
-                 text = tweet?.legacy?.full_text || item.message?.text || '';
+                 text = tweet?.note_tweet?.note_tweet_results?.result?.text || tweet?.legacy?.full_text || item.message?.text || '';
                  actionText = 'Mention/Reply';
                  urlStr = `https://x.com/i/status/${tweet?.rest_id}`;
             } else if (item.__typename === 'Tweet') {
                  author = item.core?.user_results?.result?.legacy?.screen_name || 'unknown';
-                 text = item.legacy?.full_text || '';
+                 text = item.note_tweet?.note_tweet_results?.result?.text || item.legacy?.full_text || '';
                  actionText = 'Mention';
                  urlStr = `https://x.com/i/status/${item.rest_id}`;
             }
 
+            const id = item.id || item.rest_id || entryId;
+            if (seen.has(id)) return;
+            seen.add(id);
+
             results.push({
-              id: item.id || item.rest_id || entryId,
+              id,
               action: actionText,
               author: author,
               text: text,

package/src/clis/twitter/search.ts

@@ -13,14 +13,17 @@ cli({
   ],
   columns: ['id', 'author', 'text', 'likes', 'views', 'url'],
   func: async (page, kwargs) => {
+    // Install the interceptor before opening the target page so we don't miss
+    // the initial SearchTimeline request fired during hydration.
+    await page.goto('https://x.com');
+    await page.wait(2);
+    await page.installInterceptor('SearchTimeline');
+
     // 1. Navigate to the search page
     const q = encodeURIComponent(kwargs.query);
     await page.goto(`https://x.com/search?q=${q}&f=top`);
     await page.wait(5);
 
-    // 2. Inject XHR interceptor
-    await page.installInterceptor('SearchTimeline');
-
     // 3. Trigger API by scrolling
     await page.autoScroll({ times: 3, delayMs: 2000 });
     
@@ -29,11 +32,13 @@ cli({
     if (!requests || requests.length === 0) return [];
 
     let results: any[] = [];
+    const seen = new Set<string>();
     for (const req of requests) {
       try {
-        const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
-        const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
-        if (!addEntries) continue;
+        const insts = req.data?.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || [];
+        const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries')
+          || insts.find((i: any) => i.entries && Array.isArray(i.entries));
+        if (!addEntries?.entries) continue;
 
         for (const entry of addEntries.entries) {
           if (!entry.entryId.startsWith('tweet-')) continue;
@@ -45,11 +50,13 @@ cli({
           if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
               tweet = tweet.tweet;
           }
+          if (!tweet.rest_id || seen.has(tweet.rest_id)) continue;
+          seen.add(tweet.rest_id);
 
           results.push({
             id: tweet.rest_id,
             author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
-            text: tweet.legacy?.full_text || '',
+            text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '',
             likes: tweet.legacy?.favorite_count || 0,
             views: tweet.views?.count || '0',
             url: `https://x.com/i/status/${tweet.rest_id}`