@the-convocation/twitter-scraper 0.18.3 → 0.19.0

package/examples/cycletls/README.md

@@ -0,0 +1,54 @@
+# CycleTLS Cloudflare Bypass Example
+
+This example demonstrates how to use the `@the-convocation/twitter-scraper/cycletls` entrypoint to bypass Cloudflare bot detection when authenticating with Twitter.
+
+## Problem
+
+Twitter's authentication endpoints may be protected by Cloudflare's advanced bot detection, which analyzes TLS fingerprints to detect non-browser clients. Standard Node.js TLS handshakes can trigger `403 Forbidden` errors during login.
+
+## Solution
+
+This example uses [CycleTLS](https://github.com/Danny-Dasilva/CycleTLS), which leverages golang to mimic Chrome browser TLS fingerprints, allowing requests to pass through Cloudflare's protection.
+
+## Installation
+
+```sh
+yarn install
+```
+
+## Configuration
+
+Create a `.env` file in the root of the repository (two levels up) with your Twitter credentials:
+
+```
+TWITTER_USERNAME=your_username
+TWITTER_PASSWORD=your_password
+TWITTER_EMAIL=your_email
+```
+
+## Usage
+
+```sh
+yarn start
+```
+
+## How it works
+
+The example imports the `cycleTLSFetch` function from the `/cycletls` subpath:
+
+```ts
+import { Scraper } from '@the-convocation/twitter-scraper';
+import { cycleTLSFetch, cycleTLSExit } from '@the-convocation/twitter-scraper/cycletls';
+
+const scraper = new Scraper({
+  fetch: cycleTLSFetch as any,
+});
+```
+
+This replaces the default fetch implementation with one that uses Chrome-like TLS fingerprints, bypassing Cloudflare's detection.
+
+## Important Notes
+
+- **Node.js only**: The `/cycletls` entrypoint requires Node.js and will not work in browsers
+- **Cleanup required**: Always call `cycleTLSExit()` when done to cleanup golang resources
+- **Optional dependency**: `cycletls` must be explicitly installed alongside the main package

package/examples/cycletls/package.json

@@ -0,0 +1,12 @@
+{
+  "private": true,
+  "packageManager": "yarn@1.22.19",
+  "scripts": {
+    "start": "tsx src/index.ts"
+  },
+  "dependencies": {
+    "@the-convocation/twitter-scraper": "file:../../",
+    "cycletls": "^2.0.4",
+    "tsx": "^4.15.5"
+  }
+}

package/package.json

@@ -7,18 +7,25 @@
     "scraper",
     "crawler"
   ],
-  "version": "0.18.3",
+  "version": "0.19.0",
   "main": "dist/default/cjs/index.js",
   "types": "./dist/types/index.d.ts",
   "exports": {
-    "types": "./dist/types/index.d.ts",
-    "node": {
-      "import": "./dist/node/esm/index.mjs",
-      "require": "./dist/node/cjs/index.cjs"
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "node": {
+        "import": "./dist/node/esm/index.mjs",
+        "require": "./dist/node/cjs/index.cjs"
+      },
+      "default": {
+        "import": "./dist/default/esm/index.mjs",
+        "require": "./dist/default/cjs/index.js"
+      }
     },
-    "default": {
-      "import": "./dist/default/esm/index.mjs",
-      "require": "./dist/default/cjs/index.js"
+    "./cycletls": {
+      "types": "./dist/cycletls/index.d.ts",
+      "import": "./dist/cycletls/esm/index.mjs",
+      "require": "./dist/cycletls/cjs/index.cjs"
     }
   },
   "repository": "https://github.com/the-convocation/twitter-scraper.git",
@@ -33,7 +40,7 @@
     "commit": "cz",
     "docs:generate": "typedoc --options typedoc.json",
     "docs:deploy": "yarn docs:generate && gh-pages -d docs",
-    "format": "prettier --write src/**/*.ts",
+    "format": "prettier --write src/**/*.ts examples/**/*.{ts,js,tsx}",
     "prepare": "husky install",
     "test": "jest"
   },
@@ -48,6 +55,14 @@
     "tough-cookie": "^4.1.2",
     "tslib": "^2.5.2"
   },
+  "peerDependencies": {
+    "cycletls": "^2.0.4"
+  },
+  "peerDependenciesMeta": {
+    "cycletls": {
+      "optional": true
+    }
+  },
   "devDependencies": {
     "@commitlint/cli": "^17.6.3",
     "@commitlint/config-conventional": "^17.6.3",
@@ -59,6 +74,7 @@
     "@types/tough-cookie": "^4.0.2",
     "@typescript-eslint/eslint-plugin": "^5.59.7",
     "@typescript-eslint/parser": "^5.59.7",
+    "cycletls": "^2.0.4",
     "cz-conventional-changelog": "^3.3.0",
     "dotenv": "^16.3.1",
     "esbuild": "^0.21.5",

package/rollup.config.mjs

@@ -58,4 +58,38 @@ export default [
       format: 'es',
     },
   },
+  // CycleTLS build (Node-only)
+  {
+    input: 'src/_cycletls.ts',
+    external: ['cycletls'],
+    plugins: [
+      esbuild({
+        define: {
+          PLATFORM_NODE: 'true',
+          PLATFORM_NODE_JEST: 'false',
+        },
+      }),
+    ],
+    output: [
+      {
+        file: 'dist/cycletls/cjs/index.cjs',
+        format: 'cjs',
+        sourcemap: true,
+      },
+      {
+        file: 'dist/cycletls/esm/index.mjs',
+        format: 'es',
+        sourcemap: true,
+      },
+    ],
+  },
+  {
+    input: 'src/_cycletls.ts',
+    external: ['cycletls'],
+    plugins: [dts()],
+    output: {
+      file: 'dist/cycletls/index.d.ts',
+      format: 'es',
+    },
+  },
 ];