@the-convocation/twitter-scraper 0.21.1 → 0.22.0

package/dist/types/index.d.ts

@@ -23,6 +23,62 @@ interface FetchTransformOptions {
     response: (response: Response) => Response | Promise<Response>;
 }
 
+/**
+ * castle.ts - Local Castle.io v11 token generation for Twitter/X login flow.
+ *
+ * Ported from yubie-re/castleio-gen (Python, MIT license, archived May 2025)
+ * to TypeScript. Generates device fingerprint tokens required by Twitter's
+ * login flow to avoid error 399 ("suspicious activity").
+ *
+ * This generates Castle.io SDK v2.6.0 compatible tokens (version 11).
+ *
+ * Token structure overview:
+ *   1. Collect device/browser fingerprint data (3 parts)
+ *   2. Collect behavioral event data (mouse/keyboard/touch metrics)
+ *   3. Apply layered XOR encryption with timestamp and UUID keys
+ *   4. Prepend header (timestamp, SDK version, publisher key, UUID)
+ *   5. XXTEA-encrypt the entire payload
+ *   6. Base64URL-encode with version prefix and random XOR byte
+ */
+/**
+ * Simulated browser environment values embedded in the fingerprint.
+ * These should match a realistic Chrome-on-Windows configuration.
+ *
+ * Users can provide a partial override via `ScraperOptions.experimental.browserProfile`
+ * to customize the fingerprint. Unspecified fields are randomized from realistic pools.
+ */
+interface BrowserProfile {
+    locale: string;
+    language: string;
+    timezone: string;
+    screenWidth: number;
+    screenHeight: number;
+    /** Available screen width (excludes OS chrome like taskbars) */
+    availableWidth: number;
+    /** Available screen height (excludes OS chrome like taskbars) */
+    availableHeight: number;
+    /** WebGL ANGLE renderer string */
+    gpuRenderer: string;
+    /** Device memory in GB (encoded as value * 10) */
+    deviceMemoryGB: number;
+    /** Logical CPU core count */
+    hardwareConcurrency: number;
+    /** Screen color depth in bits */
+    colorDepth: number;
+    /** CSS device pixel ratio (encoded as value * 10) */
+    devicePixelRatio: number;
+}
+/**
+ * Generate a randomized browser profile by picking values from realistic pools.
+ *
+ * **Caution:** GPU renderer is NOT randomized because the canvas fingerprint
+ * hashes in the token are static and correspond to the DEFAULT_PROFILE GPU.
+ * Mismatching GPU + canvas hashes triggers Twitter 399 bot-detection errors.
+ * Screen resolution, device memory, and CPU cores are safe to randomize since
+ * they don't affect canvas rendering.
+ */
+declare function randomizeBrowserProfile(): BrowserProfile;
+
 /**
  * Information about a rate-limiting event. Both the request and response
  * information are provided.
@@ -713,6 +769,18 @@ interface ScraperOptions {
          * Enables the generation of the `x-xp-forwarded-for` header on requests. This may resolve some errors.
          */
         xpff: boolean;
+        /**
+         * Delay in milliseconds between login flow steps, to mimic human-like timing.
+         * Without a delay, Twitter may flag rapid-fire requests as bot activity (error 399).
+         * Set to 0 to disable. Default is 1-3 seconds (average ~2s) with random jitter.
+         */
+        flowStepDelay?: number;
+        /**
+         * Override the browser profile used for Castle.io fingerprint token generation.
+         * Unspecified fields are randomized from realistic value pools.
+         * Set this if you want a consistent fingerprint or need to match specific hardware.
+         */
+        browserProfile?: Partial<BrowserProfile>;
     };
 }
 /**
@@ -724,6 +792,7 @@ declare class Scraper {
     private auth;
     private authTrends;
     private token;
+    private readonly subtaskHandlers;
     /**
      * Creates a new Scraper object.
      * - Scrapers maintain their own guest tokens for Twitter's internal API.
@@ -1002,4 +1071,4 @@ declare class Scraper {
     private handleResponse;
 }
 
-export { ApiError, AuthenticationError, type DmConversation, type DmConversationResponse, type DmConversationTimeline, type DmInbox, type DmInboxResponse, type DmInboxTimelines, type DmMessage, type DmMessageData, type DmMessageEntities, type DmMessageEntry, type DmMessageUrl, type DmParticipant, type DmReaction, type DmStatus, type DmTimelineState, type DmWelcomeMessage, ErrorRateLimitStrategy, type FetchParameters, type FetchTransformOptions, type FlowSubtaskHandler, type FlowSubtaskHandlerApi, type FlowTokenResult, type FlowTokenResultError, type FlowTokenResultSuccess, type Mention, type Photo, type PlaceRaw, type Profile, type QueryProfilesResponse, type QueryTweetsResponse, type RateLimitEvent, type RateLimitStrategy, Scraper, type ScraperOptions, SearchMode, type Tweet, type TweetQuery, type TwitterApiErrorExtensions, type TwitterApiErrorPosition, type TwitterApiErrorRaw, type TwitterApiErrorTraceInfo, type TwitterUserAuthCredentials, type TwitterUserAuthFlowInitRequest, type TwitterUserAuthFlowRequest, type TwitterUserAuthFlowResponse, type TwitterUserAuthFlowSubtaskRequest, type Video, WaitingRateLimitStrategy };
+export { ApiError, AuthenticationError, type BrowserProfile, type DmConversation, type DmConversationResponse, type DmConversationTimeline, type DmInbox, type DmInboxResponse, type DmInboxTimelines, type DmMessage, type DmMessageData, type DmMessageEntities, type DmMessageEntry, type DmMessageUrl, type DmParticipant, type DmReaction, type DmStatus, type DmTimelineState, type DmWelcomeMessage, ErrorRateLimitStrategy, type FetchParameters, type FetchTransformOptions, type FlowSubtaskHandler, type FlowSubtaskHandlerApi, type FlowTokenResult, type FlowTokenResultError, type FlowTokenResultSuccess, type Mention, type Photo, type PlaceRaw, type Profile, type QueryProfilesResponse, type QueryTweetsResponse, type RateLimitEvent, type RateLimitStrategy, Scraper, type ScraperOptions, SearchMode, type Tweet, type TweetQuery, type TwitterApiErrorExtensions, type TwitterApiErrorPosition, type TwitterApiErrorRaw, type TwitterApiErrorTraceInfo, type TwitterUserAuthCredentials, type TwitterUserAuthFlowInitRequest, type TwitterUserAuthFlowRequest, type TwitterUserAuthFlowResponse, type TwitterUserAuthFlowSubtaskRequest, type Video, WaitingRateLimitStrategy, randomizeBrowserProfile };

package/package.json

@@ -7,7 +7,7 @@
     "scraper",
     "crawler"
   ],
-  "version": "0.21.1",
+  "version": "0.22.0",
   "main": "dist/default/cjs/index.js",
   "types": "./dist/types/index.d.ts",
   "exports": {
@@ -41,6 +41,8 @@
     "docs:generate": "typedoc --options typedoc.json",
     "docs:deploy": "yarn docs:generate && gh-pages -d docs",
     "format": "prettier --write src/**/*.ts examples/**/*.{ts,js,tsx}",
+    "lint": "eslint src/",
+    "lint:fix": "eslint src/ --fix",
     "prepare": "husky install",
     "test": "node --experimental-vm-modules ./node_modules/jest/bin/jest.js --runInBand --forceExit"
   },
@@ -76,7 +78,7 @@
     "@types/tough-cookie": "^4.0.2",
     "@typescript-eslint/eslint-plugin": "^5.59.7",
     "@typescript-eslint/parser": "^5.59.7",
-    "cycletls": "^2.0.4",
+    "cycletls": "^2.0.5",
     "cz-conventional-changelog": "^3.3.0",
     "dotenv": "^16.3.1",
     "esbuild": "^0.21.5",