@jackwener/opencli 0.7.9 → 0.7.10

package/src/clis/twitter/bookmarks.ts

@@ -0,0 +1,201 @@
+import { cli, Strategy } from '../../registry.js';
+
+const BEARER_TOKEN = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA';
+const BOOKMARKS_QUERY_ID = 'Fy0QMy4q_aZCpkO0PnyLYw';
+
+const FEATURES = {
+  rweb_video_screen_enabled: false,
+  profile_label_improvements_pcf_label_in_post_enabled: true,
+  responsive_web_profile_redirect_enabled: false,
+  rweb_tipjar_consumption_enabled: false,
+  verified_phone_label_enabled: false,
+  creator_subscriptions_tweet_preview_api_enabled: true,
+  responsive_web_graphql_timeline_navigation_enabled: true,
+  responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+  premium_content_api_read_enabled: false,
+  communities_web_enable_tweet_community_results_fetch: true,
+  c9s_tweet_anatomy_moderator_badge_enabled: true,
+  articles_preview_enabled: true,
+  responsive_web_edit_tweet_api_enabled: true,
+  graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+  view_counts_everywhere_api_enabled: true,
+  longform_notetweets_consumption_enabled: true,
+  responsive_web_twitter_article_tweet_consumption_enabled: true,
+  tweet_awards_web_tipping_enabled: false,
+  content_disclosure_indicator_enabled: true,
+  content_disclosure_ai_generated_indicator_enabled: true,
+  freedom_of_speech_not_reach_fetch_enabled: true,
+  standardized_nudges_misinfo: true,
+  tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
+  longform_notetweets_rich_text_read_enabled: true,
+  longform_notetweets_inline_media_enabled: false,
+  responsive_web_enhance_cards_enabled: false,
+};
+
+interface BookmarkTweet {
+  id: string;
+  author: string;
+  name: string;
+  text: string;
+  likes: number;
+  retweets: number;
+  created_at: string;
+  url: string;
+}
+
+function buildBookmarksUrl(count: number, cursor?: string | null): string {
+  const vars: Record<string, any> = {
+    count,
+    includePromotedContent: false,
+  };
+  if (cursor) vars.cursor = cursor;
+
+  return `/i/api/graphql/${BOOKMARKS_QUERY_ID}/Bookmarks`
+    + `?variables=${encodeURIComponent(JSON.stringify(vars))}`
+    + `&features=${encodeURIComponent(JSON.stringify(FEATURES))}`;
+}
+
+function extractBookmarkTweet(result: any, seen: Set<string>): BookmarkTweet | null {
+  if (!result) return null;
+  const tw = result.tweet || result;
+  const legacy = tw.legacy || {};
+  if (!tw.rest_id || seen.has(tw.rest_id)) return null;
+  seen.add(tw.rest_id);
+
+  const user = tw.core?.user_results?.result;
+  const screenName = user?.legacy?.screen_name || user?.core?.screen_name || 'unknown';
+  const displayName = user?.legacy?.name || user?.core?.name || '';
+  const noteText = tw.note_tweet?.note_tweet_results?.result?.text;
+
+  return {
+    id: tw.rest_id,
+    author: screenName,
+    name: displayName,
+    text: noteText || legacy.full_text || '',
+    likes: legacy.favorite_count || 0,
+    retweets: legacy.retweet_count || 0,
+    created_at: legacy.created_at || '',
+    url: `https://x.com/${screenName}/status/${tw.rest_id}`,
+  };
+}
+
+function parseBookmarks(data: any, seen: Set<string>): { tweets: BookmarkTweet[]; nextCursor: string | null } {
+  const tweets: BookmarkTweet[] = [];
+  let nextCursor: string | null = null;
+
+  const instructions =
+    data?.data?.bookmark_timeline_v2?.timeline?.instructions
+    || data?.data?.bookmark_timeline?.timeline?.instructions
+    || [];
+
+  for (const inst of instructions) {
+    for (const entry of inst.entries || []) {
+      const content = entry.content;
+
+      if (content?.entryType === 'TimelineTimelineCursor' || content?.__typename === 'TimelineTimelineCursor') {
+        if (content.cursorType === 'Bottom' || content.cursorType === 'ShowMore') nextCursor = content.value;
+        continue;
+      }
+      if (entry.entryId?.startsWith('cursor-bottom-') || entry.entryId?.startsWith('cursor-showMore-')) {
+        nextCursor = content?.value || content?.itemContent?.value || nextCursor;
+        continue;
+      }
+
+      const direct = extractBookmarkTweet(content?.itemContent?.tweet_results?.result, seen);
+      if (direct) {
+        tweets.push(direct);
+        continue;
+      }
+
+      for (const item of content?.items || []) {
+        const nested = extractBookmarkTweet(item.item?.itemContent?.tweet_results?.result, seen);
+        if (nested) tweets.push(nested);
+      }
+    }
+  }
+
+  return { tweets, nextCursor };
+}
+
+cli({
+  site: 'twitter',
+  name: 'bookmarks',
+  description: 'Fetch Twitter/X bookmarks',
+  domain: 'x.com',
+  strategy: Strategy.COOKIE,
+  browser: true,
+  args: [
+    { name: 'limit', type: 'int', default: 20 },
+  ],
+  columns: ['author', 'text', 'likes', 'url'],
+  func: async (page, kwargs) => {
+    const limit = kwargs.limit || 20;
+
+    await page.goto('https://x.com');
+    await page.wait(3);
+
+    const ct0 = await page.evaluate(`() => {
+      return document.cookie.split(';').map(c => c.trim()).find(c => c.startsWith('ct0='))?.split('=')[1] || null;
+    }`);
+    if (!ct0) throw new Error('Not logged into x.com (no ct0 cookie)');
+
+    const queryId = await page.evaluate(`async () => {
+      try {
+        const ghResp = await fetch('https://raw.githubusercontent.com/fa0311/twitter-openapi/refs/heads/main/src/config/placeholder.json');
+        if (ghResp.ok) {
+          const data = await ghResp.json();
+          const entry = data['Bookmarks'];
+          if (entry && entry.queryId) return entry.queryId;
+        }
+      } catch {}
+      try {
+        const scripts = performance.getEntriesByType('resource')
+          .filter(r => r.name.includes('client-web') && r.name.endsWith('.js'))
+          .map(r => r.name);
+        for (const scriptUrl of scripts.slice(0, 15)) {
+          try {
+            const text = await (await fetch(scriptUrl)).text();
+            const re = /queryId:"([A-Za-z0-9_-]+)"[^}]{0,200}operationName:"Bookmarks"/;
+            const m = text.match(re);
+            if (m) return m[1];
+          } catch {}
+        }
+      } catch {}
+      return null;
+    }`) || BOOKMARKS_QUERY_ID;
+
+    const headers = JSON.stringify({
+      'Authorization': `Bearer ${decodeURIComponent(BEARER_TOKEN)}`,
+      'X-Csrf-Token': ct0,
+      'X-Twitter-Auth-Type': 'OAuth2Session',
+      'X-Twitter-Active-User': 'yes',
+    });
+
+    const allTweets: BookmarkTweet[] = [];
+    const seen = new Set<string>();
+    let cursor: string | null = null;
+
+    for (let i = 0; i < 5 && allTweets.length < limit; i++) {
+      const fetchCount = Math.min(100, limit - allTweets.length + 10);
+      const apiUrl = buildBookmarksUrl(fetchCount, cursor).replace(BOOKMARKS_QUERY_ID, queryId);
+
+      const data = await page.evaluate(`async () => {
+        const r = await fetch("${apiUrl}", { headers: ${headers}, credentials: 'include' });
+        return r.ok ? await r.json() : { error: r.status };
+      }`);
+
+      if (data?.error) {
+        if (allTweets.length === 0) throw new Error(`HTTP ${data.error}: Failed to fetch bookmarks. queryId may have expired.`);
+        break;
+      }
+
+      const { tweets, nextCursor } = parseBookmarks(data, seen);
+      allTweets.push(...tweets);
+
+      if (!nextCursor || nextCursor === cursor) break;
+      cursor = nextCursor;
+    }
+
+    return allTweets.slice(0, limit);
+  },
+});

package/src/clis/twitter/delete.ts

@@ -15,7 +15,6 @@ cli({
   func: async (page: IPage | null, kwargs: any) => {
     if (!page) throw new Error('Requires browser');
 
-    console.log(`Navigating to tweet: ${kwargs.url}`);
     await page.goto(kwargs.url);
     await page.wait(5); // Wait for tweet to load completely
 

package/src/clis/twitter/followers.ts

@@ -1,5 +1,4 @@
 import { cli, Strategy } from '../../registry.js';
-import * as fs from 'fs';
 
 cli({
   site: 'twitter',
@@ -37,8 +36,8 @@ cli({
     await page.goto(`https://x.com/${targetUser}`);
     await page.wait(3);
 
-    // 2. Inject interceptor for Followers GraphQL API (or user_flow.json)
-    await page.installInterceptor('graphql');
+    // 2. Inject interceptor for the followers GraphQL API
+    await page.installInterceptor('Followers');
     
     // 3. Click the followers link inside the profile page
     await page.evaluate(`() => {
@@ -53,24 +52,14 @@ cli({
 
     // 4. Retrieve data from opencli's registered interceptors
     const allRequests = await page.getInterceptedRequests();
+    const requestList = Array.isArray(allRequests) ? allRequests : [];
     
-    // Debug: Force dump all intercepted XHRs that match followers
-    if (!allRequests || allRequests.length === 0) {
-       console.log('No GraphQL requests captured by the interceptor backend.');
+    if (requestList.length === 0) {
        return [];
     }
-
-    console.log('Intercepted keys:', allRequests.map((r: any) => {
-      try {
-        const u = new URL(r.url); return u.pathname;
-      } catch (e) {
-        return r.url;
-      }
-    }));
     
-    const requests = allRequests.filter((r: any) => r.url.includes('Followers'));
+    const requests = requestList.filter((r: any) => r?.url?.includes('Followers'));
     if (!requests || requests.length === 0) {
-       console.log('No specific Followers requests captured. Check keys printed above.');
        return [];
     }
 

package/src/clis/twitter/following.ts

@@ -1,5 +1,4 @@
 import { cli, Strategy } from '../../registry.js';
-import * as fs from 'fs';
 
 cli({
   site: 'twitter',
@@ -53,15 +52,14 @@ cli({
 
     // 4. Retrieve data from opencli's registered interceptors
     const requests = await page.getInterceptedRequests();
+    const requestList = Array.isArray(requests) ? requests : [];
     
-    // Debug: Force dump all intercepted XHRs that match following
-    if (!requests || requests.length === 0) {
-       console.log('No Following requests captured by the interceptor backend.');
+    if (requestList.length === 0) {
        return [];
     }
 
     let results: any[] = [];
-    for (const req of requests) {
+    for (const req of requestList) {
       try {
         let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions;
         if (!instructions) continue;

package/src/clis/twitter/like.ts

@@ -15,7 +15,6 @@ cli({
   func: async (page: IPage | null, kwargs: any) => {
     if (!page) throw new Error('Requires browser');
 
-    console.log(`Navigating to tweet: ${kwargs.url}`);
     await page.goto(kwargs.url);
     await page.wait(5); // Wait for tweet to load completely
 

package/src/clis/twitter/notifications.ts

@@ -1,5 +1,4 @@
 import { cli, Strategy } from '../../registry.js';
-import * as fs from 'fs';
 
 cli({
   site: 'twitter',
@@ -13,13 +12,16 @@ cli({
   ],
   columns: ['id', 'action', 'author', 'text', 'url'],
   func: async (page, kwargs) => {
+    // Install the interceptor before loading the notifications page so we
+    // capture the initial timeline request triggered during page load.
+    await page.goto('https://x.com');
+    await page.wait(2);
+    await page.installInterceptor('NotificationsTimeline');
+
     // 1. Navigate to notifications
     await page.goto('https://x.com/notifications');
     await page.wait(5);
 
-    // 2. Inject interceptor
-    await page.installInterceptor('NotificationsTimeline');
-
     // 3. Trigger API by scrolling (if we need to load more)
     await page.autoScroll({ times: 2, delayMs: 2000 });
 
@@ -28,6 +30,7 @@ cli({
     if (!requests || requests.length === 0) return [];
 
     let results: any[] = [];
+    const seen = new Set<string>();
     for (const req of requests) {
       try {
         let instructions: any[] = [];
@@ -75,14 +78,16 @@ cli({
             if (item.__typename === 'TimelineNotification') {
                  // Greet likes, retweet, mentions
                  text = item.rich_message?.text || item.message?.text || '';
-                 author = item.template?.from_users?.[0]?.user_results?.result?.core?.screen_name || 'unknown';
+                 const fromUser = item.template?.from_users?.[0]?.user_results?.result;
+                 author = fromUser?.legacy?.screen_name || fromUser?.core?.screen_name || 'unknown';
                  urlStr = item.notification_url?.url || '';
                  actionText = item.notification_icon || 'Activity';
                  
                  // If there's an attached tweet
                  const targetTweet = item.template?.target_objects?.[0]?.tweet_results?.result;
                  if (targetTweet) {
-                    text += ' | ' + (targetTweet.legacy?.full_text || '');
+                    const targetText = targetTweet.note_tweet?.note_tweet_results?.result?.text || targetTweet.legacy?.full_text || '';
+                    text += text && targetText ? ' | ' + targetText : targetText;
                     if (!urlStr) {
                          urlStr = `https://x.com/i/status/${targetTweet.rest_id}`;
                     }
@@ -91,18 +96,22 @@ cli({
                  // Direct mention/reply
                  const tweet = item.tweet_result?.result;
                  author = tweet?.core?.user_results?.result?.legacy?.screen_name || 'unknown';
-                 text = tweet?.legacy?.full_text || item.message?.text || '';
+                 text = tweet?.note_tweet?.note_tweet_results?.result?.text || tweet?.legacy?.full_text || item.message?.text || '';
                  actionText = 'Mention/Reply';
                  urlStr = `https://x.com/i/status/${tweet?.rest_id}`;
             } else if (item.__typename === 'Tweet') {
                  author = item.core?.user_results?.result?.legacy?.screen_name || 'unknown';
-                 text = item.legacy?.full_text || '';
+                 text = item.note_tweet?.note_tweet_results?.result?.text || item.legacy?.full_text || '';
                  actionText = 'Mention';
                  urlStr = `https://x.com/i/status/${item.rest_id}`;
             }
 
+            const id = item.id || item.rest_id || entryId;
+            if (seen.has(id)) return;
+            seen.add(id);
+
             results.push({
-              id: item.id || item.rest_id || entryId,
+              id,
               action: actionText,
               author: author,
               text: text,

package/src/clis/twitter/search.ts

@@ -13,14 +13,17 @@ cli({
   ],
   columns: ['id', 'author', 'text', 'likes', 'views', 'url'],
   func: async (page, kwargs) => {
+    // Install the interceptor before opening the target page so we don't miss
+    // the initial SearchTimeline request fired during hydration.
+    await page.goto('https://x.com');
+    await page.wait(2);
+    await page.installInterceptor('SearchTimeline');
+
     // 1. Navigate to the search page
     const q = encodeURIComponent(kwargs.query);
     await page.goto(`https://x.com/search?q=${q}&f=top`);
     await page.wait(5);
 
-    // 2. Inject XHR interceptor
-    await page.installInterceptor('SearchTimeline');
-
     // 3. Trigger API by scrolling
     await page.autoScroll({ times: 3, delayMs: 2000 });
     
@@ -29,11 +32,13 @@ cli({
     if (!requests || requests.length === 0) return [];
 
     let results: any[] = [];
+    const seen = new Set<string>();
     for (const req of requests) {
       try {
-        const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
-        const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
-        if (!addEntries) continue;
+        const insts = req.data?.data?.search_by_raw_query?.search_timeline?.timeline?.instructions || [];
+        const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries')
+          || insts.find((i: any) => i.entries && Array.isArray(i.entries));
+        if (!addEntries?.entries) continue;
 
         for (const entry of addEntries.entries) {
           if (!entry.entryId.startsWith('tweet-')) continue;
@@ -45,11 +50,13 @@ cli({
           if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
               tweet = tweet.tweet;
           }
+          if (!tweet.rest_id || seen.has(tweet.rest_id)) continue;
+          seen.add(tweet.rest_id);
 
           results.push({
             id: tweet.rest_id,
             author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
-            text: tweet.legacy?.full_text || '',
+            text: tweet.note_tweet?.note_tweet_results?.result?.text || tweet.legacy?.full_text || '',
             likes: tweet.legacy?.favorite_count || 0,
             views: tweet.views?.count || '0',
             url: `https://x.com/i/status/${tweet.rest_id}`

package/src/clis/twitter/trending.yaml

@@ -25,9 +25,15 @@ pipeline:
           credentials: 'include',
           headers: { 'x-twitter-active-user': 'yes', 'x-csrf-token': csrfToken, 'authorization': 'Bearer ' + bearerToken }
         });
+        if (!res.ok) throw new Error('HTTP ' + res.status + '. Hint: trending endpoint may require login or API shape changed.');
         const data = await res.json();
-        const trends = data?.timeline?.instructions?.[1]?.addEntries?.entries || [];
-        return trends.filter(e => e.content?.timelineModule).flatMap(e => e.content.timelineModule.items || []).map(t => t?.item?.content?.trend).filter(Boolean);
+        const instructions = data?.timeline?.instructions || [];
+        const entries = instructions.flatMap(inst => inst?.addEntries?.entries || inst?.entries || []);
+        return entries
+          .filter(e => e.content?.timelineModule)
+          .flatMap(e => e.content.timelineModule.items || [])
+          .map(t => t?.item?.content?.trend)
+          .filter(Boolean);
       })()
 
   - map:

package/dist/_debug.js

@@ -1,7 +0,0 @@
-import { render, evalExpr } from '../pipeline/template.js';
-const ctx = { item: { first: 'X', second: 'Y' } };
-console.log('evalExpr item.first:', JSON.stringify(evalExpr('item.first', ctx)));
-console.log('evalExpr item.second:', JSON.stringify(evalExpr('item.second', ctx)));
-const template = '$' + '{{ item.first }}-$' + '{{ item.second }}';
-console.log('template:', JSON.stringify(template));
-console.log('render result:', JSON.stringify(render(template, ctx)));

package/dist/browser-tab.d.ts

@@ -1,2 +0,0 @@
-import type { IPage } from './types.js';
-export declare function withTemporaryTab<T>(page: IPage, fn: () => Promise<T>): Promise<T>;

package/dist/browser-tab.js

@@ -1,30 +0,0 @@
-function tabCount(tabs) {
-    if (Array.isArray(tabs))
-        return tabs.length;
-    if (typeof tabs === 'string') {
-        const matches = tabs.match(/Tab \d+/g);
-        return matches ? matches.length : 0;
-    }
-    return 0;
-}
-export async function withTemporaryTab(page, fn) {
-    let closeIndex = null;
-    try {
-        const before = tabCount(await page.tabs());
-        await page.newTab();
-        const after = tabCount(await page.tabs());
-        closeIndex = Math.max(after - 1, before);
-        if (closeIndex >= 0) {
-            await page.selectTab(closeIndex);
-        }
-        return await fn();
-    }
-    finally {
-        if (closeIndex != null && closeIndex >= 0) {
-            try {
-                await page.closeTab(closeIndex);
-            }
-            catch { }
-        }
-    }
-}

package/dist/browser.d.ts

@@ -1,105 +0,0 @@
-/**
- * Browser interaction via Playwright MCP Bridge extension.
- * Connects to an existing Chrome browser through the extension.
- */
-import { withTimeoutMs } from './runtime.js';
-type ConnectFailureKind = 'missing-token' | 'extension-timeout' | 'extension-not-installed' | 'mcp-init' | 'process-exit' | 'unknown';
-type PlaywrightMCPState = 'idle' | 'connecting' | 'connected' | 'closing' | 'closed';
-type ConnectFailureInput = {
-    kind: ConnectFailureKind;
-    timeout: number;
-    hasExtensionToken: boolean;
-    tokenFingerprint?: string | null;
-    stderr?: string;
-    exitCode?: number | null;
-    rawMessage?: string;
-};
-export declare function getTokenFingerprint(token: string | undefined): string | null;
-export declare function formatBrowserConnectError(input: ConnectFailureInput): Error;
-declare function createJsonRpcRequest(method: string, params?: Record<string, any>): {
-    id: number;
-    message: string;
-};
-import type { IPage } from './types.js';
-/**
- * Page abstraction wrapping JSON-RPC calls to Playwright MCP.
- */
-export declare class Page implements IPage {
-    private _request;
-    constructor(_request: (method: string, params?: Record<string, any>) => Promise<any>);
-    call(method: string, params?: Record<string, any>): Promise<any>;
-    goto(url: string): Promise<void>;
-    evaluate(js: string): Promise<any>;
-    snapshot(opts?: {
-        interactive?: boolean;
-        compact?: boolean;
-        maxDepth?: number;
-        raw?: boolean;
-    }): Promise<any>;
-    click(ref: string): Promise<void>;
-    typeText(ref: string, text: string): Promise<void>;
-    pressKey(key: string): Promise<void>;
-    wait(options: number | {
-        text?: string;
-        time?: number;
-        timeout?: number;
-    }): Promise<void>;
-    tabs(): Promise<any>;
-    closeTab(index?: number): Promise<void>;
-    newTab(): Promise<void>;
-    selectTab(index: number): Promise<void>;
-    networkRequests(includeStatic?: boolean): Promise<any>;
-    consoleMessages(level?: string): Promise<any>;
-    scroll(direction?: string, _amount?: number): Promise<void>;
-    autoScroll(options?: {
-        times?: number;
-        delayMs?: number;
-    }): Promise<void>;
-    installInterceptor(pattern: string): Promise<void>;
-    getInterceptedRequests(): Promise<any[]>;
-}
-/**
- * Playwright MCP process manager.
- */
-export declare class PlaywrightMCP {
-    private static _activeInsts;
-    private static _cleanupRegistered;
-    private static _registerGlobalCleanup;
-    private _proc;
-    private _buffer;
-    private _pending;
-    private _initialTabIdentities;
-    private _closingPromise;
-    private _state;
-    private _page;
-    get state(): PlaywrightMCPState;
-    private _sendRequest;
-    private _rejectPendingRequests;
-    private _resetAfterFailedConnect;
-    connect(opts?: {
-        timeout?: number;
-    }): Promise<Page>;
-    close(): Promise<void>;
-}
-declare function extractTabEntries(raw: any): Array<{
-    index: number;
-    identity: string;
-}>;
-declare function diffTabIndexes(initialIdentities: string[], currentTabs: Array<{
-    index: number;
-    identity: string;
-}>): number[];
-declare function appendLimited(current: string, chunk: string, limit: number): string;
-declare function buildMcpArgs(input: {
-    mcpPath: string;
-    executablePath?: string | null;
-}): string[];
-export declare const __test__: {
-    createJsonRpcRequest: typeof createJsonRpcRequest;
-    extractTabEntries: typeof extractTabEntries;
-    diffTabIndexes: typeof diffTabIndexes;
-    appendLimited: typeof appendLimited;
-    buildMcpArgs: typeof buildMcpArgs;
-    withTimeoutMs: typeof withTimeoutMs;
-};
-export {};