@the-convocation/twitter-scraper 0.19.0 → 0.20.0

package/dist/types/index.d.ts

@@ -701,6 +701,19 @@ interface ScraperOptions {
      * A handling strategy for rate limits (HTTP 429).
      */
     rateLimitStrategy: RateLimitStrategy;
+    /**
+     * Experimental features that may be added, changed, or removed at any time. Use with caution.
+     */
+    experimental: {
+        /**
+         * Enables the generation of the `x-client-transaction-id` header on requests. This may resolve some errors.
+         */
+        xClientTransactionId: boolean;
+        /**
+         * Enables the generation of the `x-xp-forwarded-for` header on requests. This may resolve some errors.
+         */
+        xpff: boolean;
+    };
 }
 /**
  * An interface to Twitter's undocumented API.

package/examples/cycletls/README.md

@@ -4,11 +4,11 @@ This example demonstrates how to use the `@the-convocation/twitter-scraper/cycle
 
 ## Problem
 
-Twitter's authentication endpoints may be protected by Cloudflare's advanced bot detection, which analyzes TLS fingerprints to detect non-browser clients. Standard Node.js TLS handshakes can trigger `403 Forbidden` errors during login.
+Twitter's authentication endpoints may be protected by Cloudflare's bot detection, which analyzes TLS fingerprints to detect non-browser clients. Standard Node.js TLS handshakes can trigger `403 Forbidden` errors during login.
 
 ## Solution
 
-This example uses [CycleTLS](https://github.com/Danny-Dasilva/CycleTLS), which leverages golang to mimic Chrome browser TLS fingerprints, allowing requests to pass through Cloudflare's protection.
+This example uses [CycleTLS](https://github.com/Danny-Dasilva/CycleTLS) to mimic Chrome browser TLS fingerprints, allowing requests to pass through Cloudflare's protection.
 
 ## Installation
 
@@ -18,7 +18,7 @@ yarn install
 
 ## Configuration
 
-Create a `.env` file in the root of the repository (two levels up) with your Twitter credentials:
+Create a `.env` file in this directory with your Twitter credentials:
 
 ```
 TWITTER_USERNAME=your_username
@@ -41,14 +41,8 @@ import { Scraper } from '@the-convocation/twitter-scraper';
 import { cycleTLSFetch, cycleTLSExit } from '@the-convocation/twitter-scraper/cycletls';
 
 const scraper = new Scraper({
-  fetch: cycleTLSFetch as any,
+  fetch: cycleTLSFetch,
 });
 ```
 
 This replaces the default fetch implementation with one that uses Chrome-like TLS fingerprints, bypassing Cloudflare's detection.
-
-## Important Notes
-
-- **Node.js only**: The `/cycletls` entrypoint requires Node.js and will not work in browsers
-- **Cleanup required**: Always call `cycleTLSExit()` when done to cleanup golang resources
-- **Optional dependency**: `cycletls` must be explicitly installed alongside the main package

package/examples/cycletls/package.json

@@ -7,6 +7,7 @@
   "dependencies": {
     "@the-convocation/twitter-scraper": "file:../../",
     "cycletls": "^2.0.4",
+    "dotenv": "^16.3.1",
     "tsx": "^4.15.5"
   }
 }

package/examples/node-integration/package.json

@@ -6,7 +6,8 @@
     "start:cjs": "node src/index.cjs"
   },
   "dependencies": {
-    "@the-convocation/twitter-scraper": "file:../../"
+    "@the-convocation/twitter-scraper": "file:../../",
+    "dotenv": "^17.2.3"
   },
   "devDependencies": {}
 }

package/package.json

@@ -7,7 +7,7 @@
     "scraper",
     "crawler"
   ],
-  "version": "0.19.0",
+  "version": "0.20.0",
   "main": "dist/default/cjs/index.js",
   "types": "./dist/types/index.d.ts",
   "exports": {
@@ -42,7 +42,7 @@
     "docs:deploy": "yarn docs:generate && gh-pages -d docs",
     "format": "prettier --write src/**/*.ts examples/**/*.{ts,js,tsx}",
     "prepare": "husky install",
-    "test": "jest"
+    "test": "jest --runInBand --forceExit"
   },
   "dependencies": {
     "@sinclair/typebox": "^0.32.20",
@@ -50,13 +50,15 @@
     "debug": "^4.4.1",
     "headers-polyfill": "^3.1.2",
     "json-stable-stringify": "^1.0.2",
+    "linkedom": "^0.18.12",
     "otpauth": "^9.2.2",
     "set-cookie-parser": "^2.6.0",
     "tough-cookie": "^4.1.2",
-    "tslib": "^2.5.2"
+    "tslib": "^2.5.2",
+    "x-client-transaction-id": "^0.1.9"
   },
   "peerDependencies": {
-    "cycletls": "^2.0.4"
+    "cycletls": "^2.0.5"
   },
   "peerDependenciesMeta": {
     "cycletls": {