@the-convocation/twitter-scraper 0.16.2 → 0.16.4

package/dist/default/cjs/index.js

@@ -70,13 +70,13 @@ class AuthenticationError extends Error {
   }
 }
 
-const log$2 = debug("twitter-scraper:rate-limit");
+const log$3 = debug("twitter-scraper:rate-limit");
 class WaitingRateLimitStrategy {
   async onRateLimit({ response: res }) {
     const xRateLimitLimit = res.headers.get("x-rate-limit-limit");
     const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
     const xRateLimitReset = res.headers.get("x-rate-limit-reset");
-    log$2(
+    log$3(
       `Rate limit event: limit=${xRateLimitLimit}, remaining=${xRateLimitRemaining}, reset=${xRateLimitReset}`
     );
     if (xRateLimitRemaining == "0" && xRateLimitReset) {
@@ -129,14 +129,14 @@ async function updateCookieJar(cookieJar, headers) {
   }
 }
 
-const log$1 = debug("twitter-scraper:api");
+const log$2 = debug("twitter-scraper:api");
 const bearerToken = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF";
 async function jitter(maxMs) {
   const jitter2 = Math.random() * maxMs;
   await new Promise((resolve) => setTimeout(resolve, jitter2));
 }
 async function requestApi(url, auth, method = "GET", platform = new Platform()) {
-  log$1(`Making ${method} request to ${url}`);
+  log$2(`Making ${method} request to ${url}`);
   const headers = new headersPolyfill.Headers();
   await auth.installTo(headers, url);
   await platform.randomizeCiphers();
@@ -163,7 +163,7 @@ async function requestApi(url, auth, method = "GET", platform = new Platform())
     }
     await updateCookieJar(auth.cookieJar(), res.headers);
     if (res.status === 429) {
-      log$1("Rate limit hit, waiting for retry...");
+      log$2("Rate limit hit, waiting for retry...");
       await auth.onRateLimit({
         fetchParameters,
         response: res
@@ -254,6 +254,7 @@ function addApiParams(params, includeTweetReplies) {
   return params;
 }
 
+const log$1 = debug("twitter-scraper:auth");
 function withTransform(fetchFn, transform) {
   return async (input, init) => {
     const fetchArgs = await transform?.request?.(input, init) ?? [
@@ -357,6 +358,7 @@ class TwitterGuestAuth {
       Authorization: `Bearer ${this.bearerToken}`,
       Cookie: await this.getCookieString()
     });
+    log$1(`Making POST request to ${guestActivateUrl}`);
     const res = await this.fetch(guestActivateUrl, {
       method: "POST",
       headers,
@@ -707,6 +709,7 @@ class TwitterUserAuth extends TwitterGuestAuth {
     if ("flow_name" in data) {
       onboardingTaskUrl = `https://api.x.com/1.1/onboarding/task.json?flow_name=${data.flow_name}`;
     }
+    log(`Making POST request to ${onboardingTaskUrl}`);
     const token = this.guestToken;
     if (token == null) {
       throw new AuthenticationError(
@@ -743,7 +746,7 @@ class TwitterUserAuth extends TwitterGuestAuth {
         }
         return {
           status: "error",
-          err: new Error("Failed to perform request.")
+          err
         };
       }
       await updateCookieJar(this.jar, res.headers);
@@ -1218,7 +1221,8 @@ function parseLegacyTweet(coreUser, user, tweet, editControl) {
       const parsedResult = parseLegacyTweet(
         retweetedStatusResult?.core?.user_results?.result?.core,
         retweetedStatusResult?.core?.user_results?.result?.legacy,
-        retweetedStatusResult?.legacy
+        retweetedStatusResult?.legacy,
+        retweetedStatusResult?.edit_control?.edit_control_initial
       );
       if (parsedResult.success) {
         tw.retweetedStatus = parsedResult.tweet;
@@ -1246,7 +1250,8 @@ function parseResult(result) {
   const tweetResult = parseLegacyTweet(
     result?.core?.user_results?.result?.core,
     result?.core?.user_results?.result?.legacy,
-    result?.legacy
+    result?.legacy,
+    result?.edit_control?.edit_control_initial
   );
   if (!tweetResult.success) {
     return tweetResult;
@@ -1274,7 +1279,7 @@ function parseTimelineTweetsV2(timeline) {
   let bottomCursor;
   let topCursor;
   const tweets = [];
-  const instructions = timeline.data?.user?.result?.timeline_v2?.timeline?.instructions ?? [];
+  const instructions = timeline.data?.user?.result?.timeline?.timeline?.instructions ?? [];
   for (const instruction of instructions) {
     const entries = instruction.entries ?? [];
     for (const entry of entries) {