@the-convocation/twitter-scraper 0.14.1 → 0.15.0

package/README.md

@@ -168,6 +168,38 @@ const scraper = new Scraper({
 });
 ```
 
+### Rate limiting
+The Twitter API heavily rate-limits clients, requiring that the scraper has its own
+rate-limit handling to behave predictably when rate-limiting occurs. By default, the
+scraper uses a rate-limiting strategy that rates for the current rate-limiting period
+to expire before resuming requests.
+
+**This has been known to take a very long time, in some cases (up to 13 minutes).**
+
+You may want to change how rate-limiting events are handled, potentially by pooling
+scrapers logged-in to different accounts (approach currently out of scope for this
+README). The rate-limit handling strategy can be configured by passing a custom
+implementation to the `rateLimitStrategy` option in the scraper constructor:
+
+```ts
+import { Scraper, RateLimitStrategy } from "@the-convocation/twitter-scraper";
+
+class CustomRateLimitStrategy implements RateLimitStrategy {
+  async onRateLimit(event: RateLimitEvent): Promise<void> {
+    // your own logic...
+  }
+}
+
+const scraper = new Scraper({
+  rateLimitStrategy: new CustomRateLimitStrategy(),
+});
+```
+
+More information on this interface can be found on the [`RateLimitStrategy`](https://the-convocation.github.io/twitter-scraper/interfaces/RateLimitStrategy.html)
+page in the documentation. The library provides two pre-written implementations to choose from:
+- `WaitingRateLimitStrategy`: The default, which waits for the limit to expire.
+- `ErrorRateLimitStrategy`: A strategy that throws if any rate-limit event occurs.
+
 ## Contributing
 
 ### Setup

package/dist/default/cjs/index.js

@@ -48,6 +48,23 @@ class ApiError extends Error {
   }
 }
 
+class WaitingRateLimitStrategy {
+  async onRateLimit({ response: res }) {
+    const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
+    const xRateLimitReset = res.headers.get("x-rate-limit-reset");
+    if (xRateLimitRemaining == "0" && xRateLimitReset) {
+      const currentTime = (/* @__PURE__ */ new Date()).valueOf() / 1e3;
+      const timeDeltaMs = 1e3 * (parseInt(xRateLimitReset) - currentTime);
+      await new Promise((resolve) => setTimeout(resolve, timeDeltaMs));
+    }
+  }
+}
+class ErrorRateLimitStrategy {
+  onRateLimit({ response: res }) {
+    throw ApiError.fromResponse(res);
+  }
+}
+
 const genericPlatform = new class {
   randomizeCiphers() {
     return Promise.resolve();
@@ -92,12 +109,16 @@ async function requestApi(url, auth, method = "GET", platform = new Platform())
   await platform.randomizeCiphers();
   let res;
   do {
-    try {
-      res = await auth.fetch(url, {
+    const fetchParameters = [
+      url,
+      {
         method,
         headers,
         credentials: "include"
-      });
+      }
+    ];
+    try {
+      res = await auth.fetch(...fetchParameters);
     } catch (err) {
       if (!(err instanceof Error)) {
         throw err;
@@ -109,13 +130,10 @@ async function requestApi(url, auth, method = "GET", platform = new Platform())
     }
     await updateCookieJar(auth.cookieJar(), res.headers);
     if (res.status === 429) {
-      const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
-      const xRateLimitReset = res.headers.get("x-rate-limit-reset");
-      if (xRateLimitRemaining == "0" && xRateLimitReset) {
-        const currentTime = (/* @__PURE__ */ new Date()).valueOf() / 1e3;
-        const timeDeltaMs = 1e3 * (parseInt(xRateLimitReset) - currentTime);
-        await new Promise((resolve) => setTimeout(resolve, timeDeltaMs));
-      }
+      await auth.onRateLimit({
+        fetchParameters,
+        response: res
+      });
     }
   } while (res.status === 429);
   if (!res.ok) {
@@ -216,9 +234,13 @@ class TwitterGuestAuth {
   constructor(bearerToken, options) {
     this.options = options;
     this.fetch = withTransform(options?.fetch ?? fetch, options?.transform);
+    this.rateLimitStrategy = options?.rateLimitStrategy ?? new WaitingRateLimitStrategy();
     this.bearerToken = bearerToken;
     this.jar = new toughCookie.CookieJar();
   }
+  async onRateLimit(event) {
+    await this.rateLimitStrategy.onRateLimit(event);
+  }
   cookieJar() {
     return this.jar;
   }
@@ -2110,7 +2132,8 @@ class Scraper {
   getAuthOptions() {
     return {
       fetch: this.options?.fetch,
-      transform: this.options?.transform
+      transform: this.options?.transform,
+      rateLimitStrategy: this.options?.rateLimitStrategy
     };
   }
   handleResponse(res) {
@@ -2121,6 +2144,9 @@ class Scraper {
   }
 }
 
+exports.ApiError = ApiError;
+exports.ErrorRateLimitStrategy = ErrorRateLimitStrategy;
 exports.Scraper = Scraper;
 exports.SearchMode = SearchMode;
+exports.WaitingRateLimitStrategy = WaitingRateLimitStrategy;
 //# sourceMappingURL=index.js.map