@the-convocation/twitter-scraper 0.14.1 → 0.15.0

package/dist/types/index.d.ts

@@ -2,6 +2,7 @@ import { Cookie } from 'tough-cookie';
 import fetch from 'cross-fetch';
 
 type FetchParameters = [input: RequestInfo | URL, init?: RequestInit];
+
 interface FetchTransformOptions {
     /**
      * Transforms the request options before a request is made. This executes after all of the default
@@ -20,6 +21,66 @@ interface FetchTransformOptions {
     response: (response: Response) => Response | Promise<Response>;
 }
 
+declare class ApiError extends Error {
+    readonly response: Response;
+    readonly data: any;
+    private constructor();
+    static fromResponse(response: Response): Promise<ApiError>;
+}
+
+/**
+ * Information about a rate-limiting event. Both the request and response
+ * information are provided.
+ */
+interface RateLimitEvent {
+    /** The complete arguments that were passed to the fetch function. */
+    fetchParameters: FetchParameters;
+    /** The failing HTTP response. */
+    response: Response;
+}
+/**
+ * The public interface for all rate-limiting strategies. Library consumers are
+ * welcome to provide their own implementations of this interface in the Scraper
+ * constructor options.
+ *
+ * The {@link RateLimitEvent} object contains both the request and response
+ * information associated with the event.
+ *
+ * @example
+ * import { Scraper, RateLimitStrategy } from "@the-convocation/twitter-scraper";
+ *
+ * // A custom rate-limiting implementation that just logs request/response information.
+ * class ConsoleLogRateLimitStrategy implements RateLimitStrategy {
+ *   async onRateLimit(event: RateLimitEvent): Promise<void> {
+ *     console.log(event.fetchParameters, event.response);
+ *   }
+ * }
+ *
+ * const scraper = new Scraper({
+ *   rateLimitStrategy: new ConsoleLogRateLimitStrategy(),
+ * });
+ */
+interface RateLimitStrategy {
+    /**
+     * Called when the scraper is rate limited.
+     * @param event The event information, including the request and response info.
+     */
+    onRateLimit(event: RateLimitEvent): Promise<void>;
+}
+/**
+ * A rate-limiting strategy that simply waits for the current rate limit period to expire.
+ * This has been known to take up to 13 minutes, in some cases.
+ */
+declare class WaitingRateLimitStrategy implements RateLimitStrategy {
+    onRateLimit({ response: res }: RateLimitEvent): Promise<void>;
+}
+/**
+ * A rate-limiting strategy that throws an {@link ApiError} when a rate limiting event occurs.
+ */
+declare class ErrorRateLimitStrategy implements RateLimitStrategy {
+    onRateLimit({ response: res }: RateLimitEvent): Promise<void>;
+}
+
 interface LegacyUserRaw {
     created_at?: string;
     description?: string;
@@ -291,6 +352,10 @@ interface ScraperOptions {
      * proxy requests through other hosts, for example.
      */
     transform: Partial<FetchTransformOptions>;
+    /**
+     * A handling strategy for rate limits (HTTP 429).
+     */
+    rateLimitStrategy: RateLimitStrategy;
 }
 /**
  * An interface to Twitter's undocumented API.
@@ -533,4 +598,4 @@ declare class Scraper {
     private handleResponse;
 }
 
-export { type Mention, type Photo, type PlaceRaw, type Profile, type QueryProfilesResponse, type QueryTweetsResponse, Scraper, SearchMode, type Tweet, type Video };
+export { ApiError, ErrorRateLimitStrategy, type FetchParameters, type FetchTransformOptions, type Mention, type Photo, type PlaceRaw, type Profile, type QueryProfilesResponse, type QueryTweetsResponse, type RateLimitEvent, type RateLimitStrategy, Scraper, type ScraperOptions, SearchMode, type Tweet, type TweetQuery, type Video, WaitingRateLimitStrategy };

package/package.json

@@ -7,7 +7,7 @@
     "scraper",
     "crawler"
   ],
-  "version": "0.14.1",
+  "version": "0.15.0",
   "main": "dist/default/cjs/index.js",
   "types": "./dist/types/index.d.ts",
   "exports": {
@@ -74,7 +74,7 @@
     "rollup-plugin-dts": "^6.1.1",
     "rollup-plugin-esbuild": "^6.1.1",
     "ts-jest": "^29.1.0",
-    "typedoc": "^0.24.7",
+    "typedoc": "^0.27.6",
     "typescript": "^5.0.4"
   },
   "config": {