@the-convocation/twitter-scraper 0.16.1 → 0.16.2

package/dist/node/esm/index.mjs

@@ -1,3 +1,4 @@
+import debug from 'debug';
 import { Cookie, CookieJar } from 'tough-cookie';
 import setCookie from 'set-cookie-parser';
 import { Headers } from 'headers-polyfill';
@@ -14,7 +15,7 @@ class ApiError extends Error {
     super(
       `Response status: ${response.status} | headers: ${JSON.stringify(
         headersToString(response.headers)
-      )} | data: ${data}`
+      )} | data: ${typeof data === "string" ? data : JSON.stringify(data)}`
     );
     this.response = response;
     this.data = data;
@@ -50,10 +51,15 @@ class AuthenticationError extends Error {
   }
 }
 
+const log$2 = debug("twitter-scraper:rate-limit");
 class WaitingRateLimitStrategy {
   async onRateLimit({ response: res }) {
+    const xRateLimitLimit = res.headers.get("x-rate-limit-limit");
     const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
     const xRateLimitReset = res.headers.get("x-rate-limit-reset");
+    log$2(
+      `Rate limit event: limit=${xRateLimitLimit}, remaining=${xRateLimitRemaining}, reset=${xRateLimitReset}`
+    );
     if (xRateLimitRemaining == "0" && xRateLimitReset) {
       const currentTime = (/* @__PURE__ */ new Date()).valueOf() / 1e3;
       const timeDeltaMs = 1e3 * (parseInt(xRateLimitReset) - currentTime);
@@ -101,12 +107,14 @@ async function updateCookieJar(cookieJar, headers) {
   }
 }
 
+const log$1 = debug("twitter-scraper:api");
 const bearerToken = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF";
 async function jitter(maxMs) {
   const jitter2 = Math.random() * maxMs;
   await new Promise((resolve) => setTimeout(resolve, jitter2));
 }
 async function requestApi(url, auth, method = "GET", platform = new Platform()) {
+  log$1(`Making ${method} request to ${url}`);
   const headers = new Headers();
   await auth.installTo(headers, url);
   await platform.randomizeCiphers();
@@ -133,6 +141,7 @@ async function requestApi(url, auth, method = "GET", platform = new Platform())
     }
     await updateCookieJar(auth.cookieJar(), res.headers);
     if (res.status === 429) {
+      log$1("Rate limit hit, waiting for retry...");
       await auth.onRateLimit({
         fetchParameters,
         response: res
@@ -291,11 +300,17 @@ class TwitterGuestAuth {
     }
     headers.set("cookie", await this.getCookieString());
   }
-  getCookies() {
-    return this.jar.getCookies(this.getCookieJarUrl());
-  }
-  getCookieString() {
-    return this.jar.getCookieString(this.getCookieJarUrl());
+  async getCookies() {
+    const cookies = await Promise.all([
+      this.jar.getCookies(this.getCookieJarUrl()),
+      this.jar.getCookies("https://twitter.com"),
+      this.jar.getCookies("https://x.com")
+    ]);
+    return cookies.flat();
+  }
+  async getCookieString() {
+    const cookies = await this.getCookies();
+    return cookies.map((cookie) => `${cookie.key}=${cookie.value}`).join("; ");
   }
   async removeCookie(key) {
     const store = this.jar.store;
@@ -315,7 +330,7 @@ class TwitterGuestAuth {
    * Updates the authentication state with a new guest token from the Twitter API.
    */
   async updateGuestToken() {
-    const guestActivateUrl = "https://api.twitter.com/1.1/guest/activate.json";
+    const guestActivateUrl = "https://api.x.com/1.1/guest/activate.json";
     const headers = new Headers({
       Authorization: `Bearer ${this.bearerToken}`,
       Cookie: await this.getCookieString()
@@ -349,6 +364,7 @@ class TwitterGuestAuth {
   }
 }
 
+const log = debug("twitter-scraper:auth-user");
 const TwitterUserAuthSubtask = Type.Object({
   subtask_id: Type.String(),
   enter_text: Type.Optional(Type.Object({}))
@@ -400,7 +416,7 @@ class TwitterUserAuth extends TwitterGuestAuth {
   }
   async isLoggedIn() {
     const res = await requestApi(
-      "https://api.twitter.com/1.1/account/verify_credentials.json",
+      "https://api.x.com/1.1/account/verify_credentials.json",
       this
     );
     if (!res.success) {
@@ -444,7 +460,7 @@ class TwitterUserAuth extends TwitterGuestAuth {
     }
     try {
       await requestApi(
-        "https://api.twitter.com/1.1/account/logout.json",
+        "https://api.x.com/1.1/account/logout.json",
         this,
         "POST"
       );
@@ -480,15 +496,59 @@ class TwitterUserAuth extends TwitterGuestAuth {
     this.removeCookie("external_referer=");
     this.removeCookie("ct0=");
     this.removeCookie("aa_u=");
+    this.removeCookie("__cf_bm=");
     return await this.executeFlowTask({
       flow_name: "login",
       input_flow_data: {
         flow_context: {
           debug_overrides: {},
           start_location: {
-            location: "splash_screen"
+            location: "unknown"
           }
         }
+      },
+      subtask_versions: {
+        action_list: 2,
+        alert_dialog: 1,
+        app_download_cta: 1,
+        check_logged_in_account: 1,
+        choice_selection: 3,
+        contacts_live_sync_permission_prompt: 0,
+        cta: 7,
+        email_verification: 2,
+        end_flow: 1,
+        enter_date: 1,
+        enter_email: 2,
+        enter_password: 5,
+        enter_phone: 2,
+        enter_recaptcha: 1,
+        enter_text: 5,
+        enter_username: 2,
+        generic_urt: 3,
+        in_app_notification: 1,
+        interest_picker: 3,
+        js_instrumentation: 1,
+        menu_dialog: 1,
+        notifications_permission_prompt: 2,
+        open_account: 2,
+        open_home_timeline: 1,
+        open_link: 1,
+        phone_verification: 4,
+        privacy_options: 1,
+        security_key: 3,
+        select_avatar: 4,
+        select_banner: 2,
+        settings_list: 7,
+        show_code: 1,
+        sign_up: 2,
+        sign_up_review: 4,
+        tweet_selection_urt: 1,
+        update_users: 1,
+        upload_media: 1,
+        user_recommendations_list: 4,
+        user_recommendations_urt: 1,
+        wait_spinner: 3,
+        web_modal: 1
       }
     });
   }
@@ -621,7 +681,10 @@ class TwitterUserAuth extends TwitterGuestAuth {
     });
   }
   async executeFlowTask(data) {
-    const onboardingTaskUrl = "https://api.twitter.com/1.1/onboarding/task.json";
+    let onboardingTaskUrl = "https://api.x.com/1.1/onboarding/task.json";
+    if ("flow_name" in data) {
+      onboardingTaskUrl = `https://api.x.com/1.1/onboarding/task.json?flow_name=${data.flow_name}`;
+    }
     const token = this.guestToken;
     if (token == null) {
       throw new AuthenticationError(
@@ -639,13 +702,37 @@ class TwitterUserAuth extends TwitterGuestAuth {
       "x-twitter-client-language": "en"
     });
     await this.installCsrfToken(headers);
-    const res = await this.fetch(onboardingTaskUrl, {
-      credentials: "include",
-      method: "POST",
-      headers,
-      body: JSON.stringify(data)
-    });
-    await updateCookieJar(this.jar, res.headers);
+    let res;
+    do {
+      const fetchParameters = [
+        onboardingTaskUrl,
+        {
+          credentials: "include",
+          method: "POST",
+          headers,
+          body: JSON.stringify(data)
+        }
+      ];
+      try {
+        res = await this.fetch(...fetchParameters);
+      } catch (err) {
+        if (!(err instanceof Error)) {
+          throw err;
+        }
+        return {
+          status: "error",
+          err: new Error("Failed to perform request.")
+        };
+      }
+      await updateCookieJar(this.jar, res.headers);
+      if (res.status === 429) {
+        log("Rate limit hit, waiting before retrying...");
+        await this.onRateLimit({
+          fetchParameters,
+          response: res
+        });
+      }
+    } while (res.status === 429);
     if (!res.ok) {
       return { status: "error", err: await ApiError.fromResponse(res) };
     }
@@ -745,33 +832,33 @@ const apiRequestFactory = createApiRequestFactory(endpoints);
 function getAvatarOriginalSizeUrl(avatarUrl) {
   return avatarUrl ? avatarUrl.replace("_normal", "") : void 0;
 }
-function parseProfile(user, isBlueVerified) {
+function parseProfile(legacy, isBlueVerified) {
   const profile = {
-    avatar: getAvatarOriginalSizeUrl(user.profile_image_url_https),
-    banner: user.profile_banner_url,
-    biography: user.description,
-    followersCount: user.followers_count,
-    followingCount: user.friends_count,
-    friendsCount: user.friends_count,
-    mediaCount: user.media_count,
-    isPrivate: user.protected ?? false,
-    isVerified: user.verified,
-    likesCount: user.favourites_count,
-    listedCount: user.listed_count,
-    location: user.location,
-    name: user.name,
-    pinnedTweetIds: user.pinned_tweet_ids_str,
-    tweetsCount: user.statuses_count,
-    url: `https://twitter.com/${user.screen_name}`,
-    userId: user.id_str,
-    username: user.screen_name,
+    avatar: getAvatarOriginalSizeUrl(legacy.profile_image_url_https),
+    banner: legacy.profile_banner_url,
+    biography: legacy.description,
+    followersCount: legacy.followers_count,
+    followingCount: legacy.friends_count,
+    friendsCount: legacy.friends_count,
+    mediaCount: legacy.media_count,
+    isPrivate: legacy.protected ?? false,
+    isVerified: legacy.verified,
+    likesCount: legacy.favourites_count,
+    listedCount: legacy.listed_count,
+    location: legacy.location,
+    name: legacy.name,
+    pinnedTweetIds: legacy.pinned_tweet_ids_str,
+    tweetsCount: legacy.statuses_count,
+    url: `https://twitter.com/${legacy.screen_name}`,
+    userId: legacy.id_str,
+    username: legacy.screen_name,
     isBlueVerified: isBlueVerified ?? false,
-    canDm: user.can_dm
+    canDm: legacy.can_dm
   };
-  if (user.created_at != null) {
-    profile.joined = new Date(Date.parse(user.created_at));
+  if (legacy.created_at != null) {
+    profile.joined = new Date(Date.parse(legacy.created_at));
   }
-  const urls = user.entities?.url?.urls;
+  const urls = legacy.entities?.url?.urls;
   if (urls?.length != null && urls?.length > 0) {
     profile.website = urls[0].expanded_url;
   }
@@ -810,15 +897,20 @@ async function getProfile(username, auth) {
     };
   }
   legacy.id_str = user.rest_id;
+  legacy.screen_name ?? (legacy.screen_name = user.core?.screen_name);
+  legacy.profile_image_url_https ?? (legacy.profile_image_url_https = user.avatar?.image_url);
+  legacy.created_at ?? (legacy.created_at = user.core?.created_at);
+  legacy.location ?? (legacy.location = user.location?.location);
+  legacy.name ?? (legacy.name = user.core?.name);
   if (legacy.screen_name == null || legacy.screen_name.length === 0) {
     return {
       success: false,
-      err: new Error(`Either ${username} does not exist or is private.`)
+      err: new Error(`User ${username} does not exist or is private.`)
     };
   }
   return {
     success: true,
-    value: parseProfile(user.legacy, user.is_blue_verified)
+    value: parseProfile(legacy, user.is_blue_verified)
   };
 }
 const idCache = /* @__PURE__ */ new Map();