@the-convocation/twitter-scraper 0.18.3 → 0.19.0

package/dist/node/cjs/index.cjs

@@ -72,13 +72,13 @@ class AuthenticationError extends Error {
   }
 }
 
-const log$3 = debug("twitter-scraper:rate-limit");
+const log$4 = debug("twitter-scraper:rate-limit");
 class WaitingRateLimitStrategy {
   async onRateLimit({ response: res }) {
     const xRateLimitLimit = res.headers.get("x-rate-limit-limit");
     const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
     const xRateLimitReset = res.headers.get("x-rate-limit-reset");
-    log$3(
+    log$4(
       `Rate limit event: limit=${xRateLimitLimit}, remaining=${xRateLimitRemaining}, reset=${xRateLimitReset}`
     );
     if (xRateLimitRemaining == "0" && xRateLimitReset) {
@@ -107,16 +107,47 @@ class Platform {
   }
 }
 
+const log$3 = debug("twitter-scraper:requests");
 async function updateCookieJar(cookieJar, headers) {
-  const setCookieHeader = headers.get("set-cookie");
-  if (setCookieHeader) {
-    const cookies = setCookie.splitCookiesString(setCookieHeader);
-    for (const cookie of cookies.map((c) => toughCookie.Cookie.parse(c))) {
-      if (!cookie) continue;
-      await cookieJar.setCookie(
-        cookie,
-        `${cookie.secure ? "https" : "http"}://${cookie.domain}${cookie.path}`
-      );
+  let setCookieHeaders = [];
+  if (typeof headers.getSetCookie === "function") {
+    setCookieHeaders = headers.getSetCookie();
+  } else {
+    const setCookieHeader = headers.get("set-cookie");
+    if (setCookieHeader) {
+      setCookieHeaders = setCookie.splitCookiesString(setCookieHeader);
+    }
+  }
+  if (setCookieHeaders.length > 0) {
+    for (const cookieStr of setCookieHeaders) {
+      const cookie = toughCookie.Cookie.parse(cookieStr);
+      if (!cookie) {
+        log$3(`Failed to parse cookie: ${cookieStr.substring(0, 100)}`);
+        continue;
+      }
+      if (cookie.maxAge === 0 || cookie.expires && cookie.expires < /* @__PURE__ */ new Date()) {
+        if (cookie.key === "ct0") {
+          log$3(`Skipping deletion of ct0 cookie (Max-Age=0)`);
+        }
+        continue;
+      }
+      try {
+        const url = `${cookie.secure ? "https" : "http"}://${cookie.domain}${cookie.path}`;
+        await cookieJar.setCookie(cookie, url);
+        if (cookie.key === "ct0") {
+          log$3(
+            `Successfully set ct0 cookie with value: ${cookie.value.substring(
+              0,
+              20
+            )}...`
+          );
+        }
+      } catch (err) {
+        log$3(`Failed to set cookie ${cookie.key}: ${err}`);
+        if (cookie.key === "ct0") {
+          log$3(`FAILED to set ct0 cookie! Error: ${err}`);
+        }
+      }
     }
   } else if (typeof document !== "undefined") {
     for (const cookie of document.cookie.split(";")) {
@@ -134,9 +165,8 @@ async function jitter(maxMs) {
   const jitter2 = Math.random() * maxMs;
   await new Promise((resolve) => setTimeout(resolve, jitter2));
 }
-async function requestApi(url, auth, method = "GET", platform = new Platform()) {
+async function requestApi(url, auth, method = "GET", platform = new Platform(), headers = new headersPolyfill.Headers()) {
   log$2(`Making ${method} request to ${url}`);
-  const headers = new headersPolyfill.Headers();
   await auth.installTo(headers, url);
   await platform.randomizeCiphers();
   let res;
@@ -322,6 +352,16 @@ class TwitterGuestAuth {
     }
     headers.set("cookie", await this.getCookieString());
   }
+  async setCookie(key, value) {
+    const cookie = toughCookie.Cookie.parse(`${key}=${value}`);
+    if (!cookie) {
+      throw new Error("Failed to parse cookie.");
+    }
+    await this.jar.setCookie(cookie, this.getCookieJarUrl());
+    if (typeof document !== "undefined") {
+      document.cookie = cookie.toString();
+    }
+  }
   async getCookies() {
     return this.jar.getCookies(this.getCookieJarUrl());
   }
@@ -372,6 +412,8 @@ class TwitterGuestAuth {
     }
     this.guestToken = newGuestToken;
     this.guestCreatedAt = /* @__PURE__ */ new Date();
+    await this.setCookie("gt", newGuestToken);
+    log$1(`Updated guest token: ${newGuestToken}`);
   }
   /**
    * Returns if the authentication token needs to be updated or not.
@@ -498,7 +540,11 @@ class TwitterUserAuth extends TwitterGuestAuth {
   }
   async installTo(headers) {
     headers.set("authorization", `Bearer ${this.bearerToken}`);
-    headers.set("cookie", await this.getCookieString());
+    const cookie = await this.getCookieString();
+    headers.set("cookie", cookie);
+    if (this.guestToken) {
+      headers.set("x-guest-token", this.guestToken);
+    }
     await this.installCsrfToken(headers);
   }
   async initLogin() {
@@ -711,16 +757,27 @@ class TwitterUserAuth extends TwitterGuestAuth {
       );
     }
     const headers = new headersPolyfill.Headers({
-      authorization: `Bearer ${this.bearerToken}`,
-      cookie: await this.getCookieString(),
+      accept: "*/*",
+      "accept-language": "en-US,en;q=0.9",
       "content-type": "application/json",
-      "User-Agent": "Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36",
+      "cache-control": "no-cache",
+      origin: "https://x.com",
+      pragma: "no-cache",
+      priority: "u=1, i",
+      referer: "https://x.com/",
+      "sec-ch-ua": '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
+      "sec-ch-ua-mobile": "?0",
+      "sec-ch-ua-platform": '"Windows"',
+      "sec-fetch-dest": "empty",
+      "sec-fetch-mode": "cors",
+      "sec-fetch-site": "same-origin",
+      "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
       "x-guest-token": token,
       "x-twitter-auth-type": "OAuth2Client",
       "x-twitter-active-user": "yes",
       "x-twitter-client-language": "en"
     });
-    await this.installCsrfToken(headers);
+    await this.installTo(headers);
     let res;
     do {
       const fetchParameters = [