@the-convocation/twitter-scraper 0.14.1 → 0.15.0

package/dist/node/cjs/index.cjs

@@ -50,6 +50,23 @@ class ApiError extends Error {
   }
 }
 
+class WaitingRateLimitStrategy {
+  async onRateLimit({ response: res }) {
+    const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
+    const xRateLimitReset = res.headers.get("x-rate-limit-reset");
+    if (xRateLimitRemaining == "0" && xRateLimitReset) {
+      const currentTime = (/* @__PURE__ */ new Date()).valueOf() / 1e3;
+      const timeDeltaMs = 1e3 * (parseInt(xRateLimitReset) - currentTime);
+      await new Promise((resolve) => setTimeout(resolve, timeDeltaMs));
+    }
+  }
+}
+class ErrorRateLimitStrategy {
+  onRateLimit({ response: res }) {
+    throw ApiError.fromResponse(res);
+  }
+}
+
 class Platform {
   async randomizeCiphers() {
     const platform = await Platform.importPlatform();
@@ -91,12 +108,16 @@ async function requestApi(url, auth, method = "GET", platform = new Platform())
   await platform.randomizeCiphers();
   let res;
   do {
-    try {
-      res = await auth.fetch(url, {
+    const fetchParameters = [
+      url,
+      {
         method,
         headers,
         credentials: "include"
-      });
+      }
+    ];
+    try {
+      res = await auth.fetch(...fetchParameters);
     } catch (err) {
       if (!(err instanceof Error)) {
         throw err;
@@ -108,13 +129,10 @@ async function requestApi(url, auth, method = "GET", platform = new Platform())
     }
     await updateCookieJar(auth.cookieJar(), res.headers);
     if (res.status === 429) {
-      const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
-      const xRateLimitReset = res.headers.get("x-rate-limit-reset");
-      if (xRateLimitRemaining == "0" && xRateLimitReset) {
-        const currentTime = (/* @__PURE__ */ new Date()).valueOf() / 1e3;
-        const timeDeltaMs = 1e3 * (parseInt(xRateLimitReset) - currentTime);
-        await new Promise((resolve) => setTimeout(resolve, timeDeltaMs));
-      }
+      await auth.onRateLimit({
+        fetchParameters,
+        response: res
+      });
     }
   } while (res.status === 429);
   if (!res.ok) {
@@ -215,9 +233,13 @@ class TwitterGuestAuth {
   constructor(bearerToken, options) {
     this.options = options;
     this.fetch = withTransform(options?.fetch ?? fetch, options?.transform);
+    this.rateLimitStrategy = options?.rateLimitStrategy ?? new WaitingRateLimitStrategy();
     this.bearerToken = bearerToken;
     this.jar = new toughCookie.CookieJar();
   }
+  async onRateLimit(event) {
+    await this.rateLimitStrategy.onRateLimit(event);
+  }
   cookieJar() {
     return this.jar;
   }
@@ -2109,7 +2131,8 @@ class Scraper {
   getAuthOptions() {
     return {
       fetch: this.options?.fetch,
-      transform: this.options?.transform
+      transform: this.options?.transform,
+      rateLimitStrategy: this.options?.rateLimitStrategy
     };
   }
   handleResponse(res) {
@@ -2151,6 +2174,9 @@ var index = /*#__PURE__*/Object.freeze({
   platform: platform
 });
 
+exports.ApiError = ApiError;
+exports.ErrorRateLimitStrategy = ErrorRateLimitStrategy;
 exports.Scraper = Scraper;
 exports.SearchMode = SearchMode;
+exports.WaitingRateLimitStrategy = WaitingRateLimitStrategy;
 //# sourceMappingURL=index.cjs.map