@the-convocation/twitter-scraper 0.18.3 → 0.19.0

package/README.md

@@ -168,6 +168,41 @@ const scraper = new Scraper({
 });
 ```
 
+### Bypassing Cloudflare bot detection
+
+In some cases, Twitter's authentication endpoints may be protected by Cloudflare's advanced bot detection, resulting in `403 Forbidden` errors during login. This typically happens because standard Node.js TLS fingerprints are detected as non-browser clients.
+
+To bypass this protection, you can use the optional CycleTLS integration, which uses golang to mimic Chrome browser TLS fingerprints:
+
+**Installation:**
+
+```sh
+npm install cycletls
+# or
+yarn add cycletls
+```
+
+**Usage:**
+
+```ts
+import { Scraper } from '@the-convocation/twitter-scraper';
+import { cycleTLSFetch, cycleTLSExit } from '@the-convocation/twitter-scraper/cycletls';
+
+const scraper = new Scraper({
+  fetch: cycleTLSFetch,
+});
+
+// Use the scraper normally
+await scraper.login(username, password, email);
+
+// Important: cleanup CycleTLS resources when done
+cycleTLSExit();
+```
+
+**Note:** The `/cycletls` entrypoint is Node.js only and will not work in browser environments. It's provided as a separate optional entrypoint to avoid bundling golang dependencies in environments where they cannot run.
+
+See the [cycletls-cloudflare example](./examples/cycletls-cloudflare/) for a complete working example.
+
 ### Rate limiting
 The Twitter API heavily rate-limits clients, requiring that the scraper has its own
 rate-limit handling to behave predictably when rate-limiting occurs. By default, the

package/dist/cycletls/cjs/index.cjs

@@ -0,0 +1,99 @@
+'use strict';
+
+var initCycleTLS = require('cycletls');
+var headersPolyfill = require('headers-polyfill');
+var debug = require('debug');
+
+const log = debug("twitter-scraper:cycletls");
+let cycleTLSInstance = null;
+async function initCycleTLSFetch() {
+  if (!cycleTLSInstance) {
+    log("Initializing CycleTLS...");
+    cycleTLSInstance = await initCycleTLS();
+    log("CycleTLS initialized successfully");
+  }
+  return cycleTLSInstance;
+}
+function cycleTLSExit() {
+  if (cycleTLSInstance) {
+    log("Exiting CycleTLS...");
+    cycleTLSInstance.exit();
+    cycleTLSInstance = null;
+  }
+}
+async function cycleTLSFetch(input, init) {
+  const instance = await initCycleTLSFetch();
+  const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
+  const method = (init?.method || "GET").toUpperCase();
+  log(`Making ${method} request to ${url}`);
+  const headers = {};
+  if (init?.headers) {
+    if (init.headers instanceof headersPolyfill.Headers) {
+      init.headers.forEach((value, key) => {
+        headers[key] = value;
+      });
+    } else if (Array.isArray(init.headers)) {
+      init.headers.forEach(([key, value]) => {
+        headers[key] = value;
+      });
+    } else {
+      Object.assign(headers, init.headers);
+    }
+  }
+  let body;
+  if (init?.body) {
+    if (typeof init.body === "string") {
+      body = init.body;
+    } else if (init.body instanceof URLSearchParams) {
+      body = init.body.toString();
+    } else {
+      body = init.body.toString();
+    }
+  }
+  const options = {
+    body,
+    headers,
+    // Chrome 120 on Windows 10
+    ja3: "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
+    userAgent: headers["user-agent"] || "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+  };
+  try {
+    const response = await instance(
+      url,
+      options,
+      method.toLowerCase()
+    );
+    const responseHeaders = new headersPolyfill.Headers();
+    if (response.headers) {
+      Object.entries(response.headers).forEach(([key, value]) => {
+        if (Array.isArray(value)) {
+          value.forEach((v) => {
+            responseHeaders.append(key, v);
+          });
+        } else if (typeof value === "string") {
+          responseHeaders.set(key, value);
+        }
+      });
+    }
+    let responseBody = "";
+    if (typeof response.text === "function") {
+      responseBody = await response.text();
+    } else if (response.body) {
+      responseBody = response.body;
+    }
+    const fetchResponse = new Response(responseBody, {
+      status: response.status,
+      statusText: "",
+      // CycleTLS doesn't provide status text
+      headers: responseHeaders
+    });
+    return fetchResponse;
+  } catch (error) {
+    log(`CycleTLS request failed: ${error}`);
+    throw error;
+  }
+}
+
+exports.cycleTLSExit = cycleTLSExit;
+exports.cycleTLSFetch = cycleTLSFetch;
+//# sourceMappingURL=index.cjs.map

package/dist/cycletls/cjs/index.cjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.cjs","sources":["../../../src/cycletls-fetch.ts"],"sourcesContent":["import initCycleTLS from 'cycletls';\nimport { Headers } from 'headers-polyfill';\nimport debug from 'debug';\n\nconst log = debug('twitter-scraper:cycletls');\n\nlet cycleTLSInstance: Awaited<ReturnType<typeof initCycleTLS>> | null = null;\n\n/**\n * Initialize the CycleTLS instance. This should be called once before using the fetch wrapper.\n */\nexport async function initCycleTLSFetch() {\n  if (!cycleTLSInstance) {\n    log('Initializing CycleTLS...');\n    cycleTLSInstance = await initCycleTLS();\n    log('CycleTLS initialized successfully');\n  }\n  return cycleTLSInstance;\n}\n\n/**\n * Cleanup the CycleTLS instance. Call this when you're done making requests.\n */\nexport function cycleTLSExit() {\n  if (cycleTLSInstance) {\n    log('Exiting CycleTLS...');\n    cycleTLSInstance.exit();\n    cycleTLSInstance = null;\n  }\n}\n\n/**\n * A fetch-compatible wrapper around CycleTLS that mimics Chrome's TLS fingerprint\n * to bypass Cloudflare and other bot detection systems.\n */\nexport async function cycleTLSFetch(\n  input: RequestInfo | URL,\n  init?: RequestInit,\n): Promise<Response> {\n  const instance = await initCycleTLSFetch();\n\n  const url =\n    typeof input === 'string'\n      ? input\n      : input instanceof URL\n      ? input.toString()\n      : input.url;\n  const method = (init?.method || 'GET').toUpperCase();\n\n  log(`Making ${method} request to ${url}`);\n\n  // Extract headers from RequestInit\n  const headers: Record<string, string> = {};\n  if (init?.headers) {\n    if (init.headers instanceof Headers) {\n      init.headers.forEach((value, key) => {\n        headers[key] = value;\n      });\n    } else if (Array.isArray(init.headers)) {\n      init.headers.forEach(([key, value]) => {\n        headers[key] = value;\n      });\n    } else {\n      Object.assign(headers, init.headers);\n    }\n  }\n\n  // Convert body to string if needed\n  let body: string | undefined;\n  if (init?.body) {\n    if (typeof init.body === 'string') {\n      body = init.body;\n    } else if (init.body instanceof URLSearchParams) {\n      body = init.body.toString();\n    } else {\n      body = init.body.toString();\n    }\n  }\n\n  // Use Chrome 120 JA3 fingerprint for maximum compatibility\n  const options = {\n    body,\n    headers,\n    // Chrome 120 on Windows 10\n    ja3: '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0',\n    userAgent:\n      headers['user-agent'] ||\n      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',\n  };\n\n  try {\n    const response = await instance(\n      url,\n      options,\n      method.toLowerCase() as\n        | 'get'\n        | 'post'\n        | 'put'\n        | 'delete'\n        | 'patch'\n        | 'head'\n        | 'options',\n    );\n\n    // Convert CycleTLS response to fetch Response\n    // CycleTLS returns headers as an object\n    const responseHeaders = new Headers();\n    if (response.headers) {\n      Object.entries(response.headers).forEach(([key, value]) => {\n        if (Array.isArray(value)) {\n          value.forEach((v) => {\n            responseHeaders.append(key, v);\n          });\n        } else if (typeof value === 'string') {\n          responseHeaders.set(key, value);\n        }\n      });\n    }\n\n    // Get response body - cycletls provides helper methods, but we need the raw text\n    // The response object has a text() method that returns the body as text\n    let responseBody = '';\n    if (typeof response.text === 'function') {\n      responseBody = await response.text();\n    } else if ((response as any).body) {\n      responseBody = (response as any).body;\n    }\n\n    // Create a proper Response object using standard Response constructor\n    const fetchResponse = new Response(responseBody, {\n      status: response.status,\n      statusText: '', // CycleTLS doesn't provide status text\n      headers: responseHeaders,\n    });\n\n    return fetchResponse;\n  } catch (error) {\n    log(`CycleTLS request failed: ${error}`);\n    throw error;\n  }\n}\n"],"names":["Headers"],"mappings":";;;;;;AAIA,MAAM,GAAA,GAAM,MAAM,0BAA0B,CAAA,CAAA;AAE5C,IAAI,gBAAoE,GAAA,IAAA,CAAA;AAKxE,eAAsB,iBAAoB,GAAA;AACxC,EAAA,IAAI,CAAC,gBAAkB,EAAA;AACrB,IAAA,GAAA,CAAI,0BAA0B,CAAA,CAAA;AAC9B,IAAA,gBAAA,GAAmB,MAAM,YAAa,EAAA,CAAA;AACtC,IAAA,GAAA,CAAI,mCAAmC,CAAA,CAAA;AAAA,GACzC;AACA,EAAO,OAAA,gBAAA,CAAA;AACT,CAAA;AAKO,SAAS,YAAe,GAAA;AAC7B,EAAA,IAAI,gBAAkB,EAAA;AACpB,IAAA,GAAA,CAAI,qBAAqB,CAAA,CAAA;AACzB,IAAA,gBAAA,CAAiB,IAAK,EAAA,CAAA;AACtB,IAAmB,gBAAA,GAAA,IAAA,CAAA;AAAA,GACrB;AACF,CAAA;AAMsB,eAAA,aAAA,CACpB,OACA,IACmB,EAAA;AACnB,EAAM,MAAA,QAAA,GAAW,MAAM,iBAAkB,EAAA,CAAA;AAEzC,EAAM,MAAA,GAAA,GACJ,OAAO,KAAA,KAAU,QACb,GAAA,KAAA,GACA,iBAAiB,GACjB,GAAA,KAAA,CAAM,QAAS,EAAA,GACf,KAAM,CAAA,GAAA,CAAA;AACZ,EAAA,MAAM,MAAU,GAAA,CAAA,IAAA,EAAM,MAAU,IAAA,KAAA,EAAO,WAAY,EAAA,CAAA;AAEnD,EAAA,GAAA,CAAI,CAAU,OAAA,EAAA,MAAM,CAAe,YAAA,EAAA,GAAG,CAAE,CAAA,CAAA,CAAA;AAGxC,EAAA,MAAM,UAAkC,EAAC,CAAA;AACzC,EAAA,IAAI,MAAM,OAAS,EAAA;AACjB,IAAI,IAAA,IAAA,CAAK,mBAAmBA,uBAAS,EAAA;AACnC,MAAA,IAAA,CAAK,OAAQ,CAAA,OAAA,CAAQ,CAAC,KAAA,EAAO,GAAQ,KAAA;AACnC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACQ,MAAA,IAAA,KAAA,CAAM,OAAQ,CAAA,IAAA,CAAK,OAAO,CAAG,EAAA;AACtC,MAAA,IAAA,CAAK,QAAQ,OAAQ,CAAA,CAAC,CAAC,GAAA,EAAK,KAAK,CAAM,KAAA;AACrC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACI,MAAA;AACL,MAAO,MAAA,CAAA,MAAA,CAAO,OAAS,EAAA,IAAA,CAAK,OAAO,CAAA,CAAA;AAAA,KACrC;AAAA,GACF;AAGA,EAAI,IAAA,IAAA,CAAA;AACJ,EAAA,IAAI,MAAM,IAAM,EAAA;AACd,IAAI,IAAA,OAAO,IAAK,CAAA,IAAA,KAAS,QAAU,EAAA;AACjC,MAAA,IAAA,GAAO,IAAK,CAAA,IAAA,CAAA;AAAA,KACd,MAAA,IAAW,IAAK,CAAA,IAAA,YAAgB,eAAiB,EAAA;AAC/C,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KACrB,MAAA;AACL,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KAC5B;AAAA,GACF;AAGA,EAAA,MAAM,OAAU,GAAA;AAAA,IACd,IAAA;AAAA,IACA,OAAA;AAAA;AAAA,IAEA,GAAK,EAAA,8IAAA;AAAA,IACL,SAAA,EACE,OAAQ,CAAA,YAAY,CACpB,IAAA,iHAAA;AAAA,GACJ,CAAA;AAEA,EAAI,IAAA;AACF,IAAA,MAAM,WAAW,MAAM,QAAA;AAAA,MACrB,GAAA;AAAA,MACA,OAAA;AAAA,MACA,OAAO,WAAY,EAAA;AAAA,KAQrB,CAAA;AAIA,IAAM,MAAA,eAAA,GAAkB,IAAIA,uBAAQ,EAAA,CAAA;AACpC,IAAA,IAAI,SAAS,OAAS,EAAA;AACpB,MAAO,MAAA,CAAA,OAAA,CAAQ,SAAS,OAAO,CAAA,CAAE,QAAQ,CAAC,CAAC,GAAK,EAAA,KAAK,CAAM,KAAA;AACzD,QAAI,IAAA,KAAA,CAAM,OAAQ,CAAA,KAAK,CAAG,EAAA;AACxB,UAAM,KAAA,CAAA,OAAA,CAAQ,CAAC,CAAM,KAAA;AACnB,YAAgB,eAAA,CAAA,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA;AAAA,WAC9B,CAAA,CAAA;AAAA,SACH,MAAA,IAAW,OAAO,KAAA,KAAU,QAAU,EAAA;AACpC,UAAgB,eAAA,CAAA,GAAA,CAAI,KAAK,KAAK,CAAA,CAAA;AAAA,SAChC;AAAA,OACD,CAAA,CAAA;AAAA,KACH;AAIA,IAAA,IAAI,YAAe,GAAA,EAAA,CAAA;AACnB,IAAI,IAAA,OAAO,QAAS,CAAA,IAAA,KAAS,UAAY,EAAA;AACvC,MAAe,YAAA,GAAA,MAAM,SAAS,IAAK,EAAA,CAAA;AAAA,KACrC,MAAA,IAAY,SAAiB,IAAM,EAAA;AACjC,MAAA,YAAA,GAAgB,QAAiB,CAAA,IAAA,CAAA;AAAA,KACnC;AAGA,IAAM,MAAA,aAAA,GAAgB,IAAI,QAAA,CAAS,YAAc,EAAA;AAAA,MAC/C,QAAQ,QAAS,CAAA,MAAA;AAAA,MACjB,UAAY,EAAA,EAAA;AAAA;AAAA,MACZ,OAAS,EAAA,eAAA;AAAA,KACV,CAAA,CAAA;AAED,IAAO,OAAA,aAAA,CAAA;AAAA,WACA,KAAO,EAAA;AACd,IAAI,GAAA,CAAA,CAAA,yBAAA,EAA4B,KAAK,CAAE,CAAA,CAAA,CAAA;AACvC,IAAM,MAAA,KAAA,CAAA;AAAA,GACR;AACF;;;;;"}

package/dist/cycletls/esm/index.mjs

@@ -0,0 +1,96 @@
+import initCycleTLS from 'cycletls';
+import { Headers } from 'headers-polyfill';
+import debug from 'debug';
+
+const log = debug("twitter-scraper:cycletls");
+let cycleTLSInstance = null;
+async function initCycleTLSFetch() {
+  if (!cycleTLSInstance) {
+    log("Initializing CycleTLS...");
+    cycleTLSInstance = await initCycleTLS();
+    log("CycleTLS initialized successfully");
+  }
+  return cycleTLSInstance;
+}
+function cycleTLSExit() {
+  if (cycleTLSInstance) {
+    log("Exiting CycleTLS...");
+    cycleTLSInstance.exit();
+    cycleTLSInstance = null;
+  }
+}
+async function cycleTLSFetch(input, init) {
+  const instance = await initCycleTLSFetch();
+  const url = typeof input === "string" ? input : input instanceof URL ? input.toString() : input.url;
+  const method = (init?.method || "GET").toUpperCase();
+  log(`Making ${method} request to ${url}`);
+  const headers = {};
+  if (init?.headers) {
+    if (init.headers instanceof Headers) {
+      init.headers.forEach((value, key) => {
+        headers[key] = value;
+      });
+    } else if (Array.isArray(init.headers)) {
+      init.headers.forEach(([key, value]) => {
+        headers[key] = value;
+      });
+    } else {
+      Object.assign(headers, init.headers);
+    }
+  }
+  let body;
+  if (init?.body) {
+    if (typeof init.body === "string") {
+      body = init.body;
+    } else if (init.body instanceof URLSearchParams) {
+      body = init.body.toString();
+    } else {
+      body = init.body.toString();
+    }
+  }
+  const options = {
+    body,
+    headers,
+    // Chrome 120 on Windows 10
+    ja3: "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0",
+    userAgent: headers["user-agent"] || "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+  };
+  try {
+    const response = await instance(
+      url,
+      options,
+      method.toLowerCase()
+    );
+    const responseHeaders = new Headers();
+    if (response.headers) {
+      Object.entries(response.headers).forEach(([key, value]) => {
+        if (Array.isArray(value)) {
+          value.forEach((v) => {
+            responseHeaders.append(key, v);
+          });
+        } else if (typeof value === "string") {
+          responseHeaders.set(key, value);
+        }
+      });
+    }
+    let responseBody = "";
+    if (typeof response.text === "function") {
+      responseBody = await response.text();
+    } else if (response.body) {
+      responseBody = response.body;
+    }
+    const fetchResponse = new Response(responseBody, {
+      status: response.status,
+      statusText: "",
+      // CycleTLS doesn't provide status text
+      headers: responseHeaders
+    });
+    return fetchResponse;
+  } catch (error) {
+    log(`CycleTLS request failed: ${error}`);
+    throw error;
+  }
+}
+
+export { cycleTLSExit, cycleTLSFetch };
+//# sourceMappingURL=index.mjs.map

package/dist/cycletls/esm/index.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.mjs","sources":["../../../src/cycletls-fetch.ts"],"sourcesContent":["import initCycleTLS from 'cycletls';\nimport { Headers } from 'headers-polyfill';\nimport debug from 'debug';\n\nconst log = debug('twitter-scraper:cycletls');\n\nlet cycleTLSInstance: Awaited<ReturnType<typeof initCycleTLS>> | null = null;\n\n/**\n * Initialize the CycleTLS instance. This should be called once before using the fetch wrapper.\n */\nexport async function initCycleTLSFetch() {\n  if (!cycleTLSInstance) {\n    log('Initializing CycleTLS...');\n    cycleTLSInstance = await initCycleTLS();\n    log('CycleTLS initialized successfully');\n  }\n  return cycleTLSInstance;\n}\n\n/**\n * Cleanup the CycleTLS instance. Call this when you're done making requests.\n */\nexport function cycleTLSExit() {\n  if (cycleTLSInstance) {\n    log('Exiting CycleTLS...');\n    cycleTLSInstance.exit();\n    cycleTLSInstance = null;\n  }\n}\n\n/**\n * A fetch-compatible wrapper around CycleTLS that mimics Chrome's TLS fingerprint\n * to bypass Cloudflare and other bot detection systems.\n */\nexport async function cycleTLSFetch(\n  input: RequestInfo | URL,\n  init?: RequestInit,\n): Promise<Response> {\n  const instance = await initCycleTLSFetch();\n\n  const url =\n    typeof input === 'string'\n      ? input\n      : input instanceof URL\n      ? input.toString()\n      : input.url;\n  const method = (init?.method || 'GET').toUpperCase();\n\n  log(`Making ${method} request to ${url}`);\n\n  // Extract headers from RequestInit\n  const headers: Record<string, string> = {};\n  if (init?.headers) {\n    if (init.headers instanceof Headers) {\n      init.headers.forEach((value, key) => {\n        headers[key] = value;\n      });\n    } else if (Array.isArray(init.headers)) {\n      init.headers.forEach(([key, value]) => {\n        headers[key] = value;\n      });\n    } else {\n      Object.assign(headers, init.headers);\n    }\n  }\n\n  // Convert body to string if needed\n  let body: string | undefined;\n  if (init?.body) {\n    if (typeof init.body === 'string') {\n      body = init.body;\n    } else if (init.body instanceof URLSearchParams) {\n      body = init.body.toString();\n    } else {\n      body = init.body.toString();\n    }\n  }\n\n  // Use Chrome 120 JA3 fingerprint for maximum compatibility\n  const options = {\n    body,\n    headers,\n    // Chrome 120 on Windows 10\n    ja3: '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0',\n    userAgent:\n      headers['user-agent'] ||\n      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',\n  };\n\n  try {\n    const response = await instance(\n      url,\n      options,\n      method.toLowerCase() as\n        | 'get'\n        | 'post'\n        | 'put'\n        | 'delete'\n        | 'patch'\n        | 'head'\n        | 'options',\n    );\n\n    // Convert CycleTLS response to fetch Response\n    // CycleTLS returns headers as an object\n    const responseHeaders = new Headers();\n    if (response.headers) {\n      Object.entries(response.headers).forEach(([key, value]) => {\n        if (Array.isArray(value)) {\n          value.forEach((v) => {\n            responseHeaders.append(key, v);\n          });\n        } else if (typeof value === 'string') {\n          responseHeaders.set(key, value);\n        }\n      });\n    }\n\n    // Get response body - cycletls provides helper methods, but we need the raw text\n    // The response object has a text() method that returns the body as text\n    let responseBody = '';\n    if (typeof response.text === 'function') {\n      responseBody = await response.text();\n    } else if ((response as any).body) {\n      responseBody = (response as any).body;\n    }\n\n    // Create a proper Response object using standard Response constructor\n    const fetchResponse = new Response(responseBody, {\n      status: response.status,\n      statusText: '', // CycleTLS doesn't provide status text\n      headers: responseHeaders,\n    });\n\n    return fetchResponse;\n  } catch (error) {\n    log(`CycleTLS request failed: ${error}`);\n    throw error;\n  }\n}\n"],"names":[],"mappings":";;;;AAIA,MAAM,GAAA,GAAM,MAAM,0BAA0B,CAAA,CAAA;AAE5C,IAAI,gBAAoE,GAAA,IAAA,CAAA;AAKxE,eAAsB,iBAAoB,GAAA;AACxC,EAAA,IAAI,CAAC,gBAAkB,EAAA;AACrB,IAAA,GAAA,CAAI,0BAA0B,CAAA,CAAA;AAC9B,IAAA,gBAAA,GAAmB,MAAM,YAAa,EAAA,CAAA;AACtC,IAAA,GAAA,CAAI,mCAAmC,CAAA,CAAA;AAAA,GACzC;AACA,EAAO,OAAA,gBAAA,CAAA;AACT,CAAA;AAKO,SAAS,YAAe,GAAA;AAC7B,EAAA,IAAI,gBAAkB,EAAA;AACpB,IAAA,GAAA,CAAI,qBAAqB,CAAA,CAAA;AACzB,IAAA,gBAAA,CAAiB,IAAK,EAAA,CAAA;AACtB,IAAmB,gBAAA,GAAA,IAAA,CAAA;AAAA,GACrB;AACF,CAAA;AAMsB,eAAA,aAAA,CACpB,OACA,IACmB,EAAA;AACnB,EAAM,MAAA,QAAA,GAAW,MAAM,iBAAkB,EAAA,CAAA;AAEzC,EAAM,MAAA,GAAA,GACJ,OAAO,KAAA,KAAU,QACb,GAAA,KAAA,GACA,iBAAiB,GACjB,GAAA,KAAA,CAAM,QAAS,EAAA,GACf,KAAM,CAAA,GAAA,CAAA;AACZ,EAAA,MAAM,MAAU,GAAA,CAAA,IAAA,EAAM,MAAU,IAAA,KAAA,EAAO,WAAY,EAAA,CAAA;AAEnD,EAAA,GAAA,CAAI,CAAU,OAAA,EAAA,MAAM,CAAe,YAAA,EAAA,GAAG,CAAE,CAAA,CAAA,CAAA;AAGxC,EAAA,MAAM,UAAkC,EAAC,CAAA;AACzC,EAAA,IAAI,MAAM,OAAS,EAAA;AACjB,IAAI,IAAA,IAAA,CAAK,mBAAmB,OAAS,EAAA;AACnC,MAAA,IAAA,CAAK,OAAQ,CAAA,OAAA,CAAQ,CAAC,KAAA,EAAO,GAAQ,KAAA;AACnC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACQ,MAAA,IAAA,KAAA,CAAM,OAAQ,CAAA,IAAA,CAAK,OAAO,CAAG,EAAA;AACtC,MAAA,IAAA,CAAK,QAAQ,OAAQ,CAAA,CAAC,CAAC,GAAA,EAAK,KAAK,CAAM,KAAA;AACrC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACI,MAAA;AACL,MAAO,MAAA,CAAA,MAAA,CAAO,OAAS,EAAA,IAAA,CAAK,OAAO,CAAA,CAAA;AAAA,KACrC;AAAA,GACF;AAGA,EAAI,IAAA,IAAA,CAAA;AACJ,EAAA,IAAI,MAAM,IAAM,EAAA;AACd,IAAI,IAAA,OAAO,IAAK,CAAA,IAAA,KAAS,QAAU,EAAA;AACjC,MAAA,IAAA,GAAO,IAAK,CAAA,IAAA,CAAA;AAAA,KACd,MAAA,IAAW,IAAK,CAAA,IAAA,YAAgB,eAAiB,EAAA;AAC/C,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KACrB,MAAA;AACL,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KAC5B;AAAA,GACF;AAGA,EAAA,MAAM,OAAU,GAAA;AAAA,IACd,IAAA;AAAA,IACA,OAAA;AAAA;AAAA,IAEA,GAAK,EAAA,8IAAA;AAAA,IACL,SAAA,EACE,OAAQ,CAAA,YAAY,CACpB,IAAA,iHAAA;AAAA,GACJ,CAAA;AAEA,EAAI,IAAA;AACF,IAAA,MAAM,WAAW,MAAM,QAAA;AAAA,MACrB,GAAA;AAAA,MACA,OAAA;AAAA,MACA,OAAO,WAAY,EAAA;AAAA,KAQrB,CAAA;AAIA,IAAM,MAAA,eAAA,GAAkB,IAAI,OAAQ,EAAA,CAAA;AACpC,IAAA,IAAI,SAAS,OAAS,EAAA;AACpB,MAAO,MAAA,CAAA,OAAA,CAAQ,SAAS,OAAO,CAAA,CAAE,QAAQ,CAAC,CAAC,GAAK,EAAA,KAAK,CAAM,KAAA;AACzD,QAAI,IAAA,KAAA,CAAM,OAAQ,CAAA,KAAK,CAAG,EAAA;AACxB,UAAM,KAAA,CAAA,OAAA,CAAQ,CAAC,CAAM,KAAA;AACnB,YAAgB,eAAA,CAAA,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA;AAAA,WAC9B,CAAA,CAAA;AAAA,SACH,MAAA,IAAW,OAAO,KAAA,KAAU,QAAU,EAAA;AACpC,UAAgB,eAAA,CAAA,GAAA,CAAI,KAAK,KAAK,CAAA,CAAA;AAAA,SAChC;AAAA,OACD,CAAA,CAAA;AAAA,KACH;AAIA,IAAA,IAAI,YAAe,GAAA,EAAA,CAAA;AACnB,IAAI,IAAA,OAAO,QAAS,CAAA,IAAA,KAAS,UAAY,EAAA;AACvC,MAAe,YAAA,GAAA,MAAM,SAAS,IAAK,EAAA,CAAA;AAAA,KACrC,MAAA,IAAY,SAAiB,IAAM,EAAA;AACjC,MAAA,YAAA,GAAgB,QAAiB,CAAA,IAAA,CAAA;AAAA,KACnC;AAGA,IAAM,MAAA,aAAA,GAAgB,IAAI,QAAA,CAAS,YAAc,EAAA;AAAA,MAC/C,QAAQ,QAAS,CAAA,MAAA;AAAA,MACjB,UAAY,EAAA,EAAA;AAAA;AAAA,MACZ,OAAS,EAAA,eAAA;AAAA,KACV,CAAA,CAAA;AAED,IAAO,OAAA,aAAA,CAAA;AAAA,WACA,KAAO,EAAA;AACd,IAAI,GAAA,CAAA,CAAA,yBAAA,EAA4B,KAAK,CAAE,CAAA,CAAA,CAAA;AACvC,IAAM,MAAA,KAAA,CAAA;AAAA,GACR;AACF;;;;"}

package/dist/cycletls/index.d.ts

@@ -0,0 +1,11 @@
+/**
+ * Cleanup the CycleTLS instance. Call this when you're done making requests.
+ */
+declare function cycleTLSExit(): void;
+/**
+ * A fetch-compatible wrapper around CycleTLS that mimics Chrome's TLS fingerprint
+ * to bypass Cloudflare and other bot detection systems.
+ */
+declare function cycleTLSFetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response>;
+
+export { cycleTLSExit, cycleTLSFetch };

package/dist/default/cjs/index.js

@@ -70,13 +70,13 @@ class AuthenticationError extends Error {
   }
 }
 
-const log$3 = debug("twitter-scraper:rate-limit");
+const log$4 = debug("twitter-scraper:rate-limit");
 class WaitingRateLimitStrategy {
   async onRateLimit({ response: res }) {
     const xRateLimitLimit = res.headers.get("x-rate-limit-limit");
     const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
     const xRateLimitReset = res.headers.get("x-rate-limit-reset");
-    log$3(
+    log$4(
       `Rate limit event: limit=${xRateLimitLimit}, remaining=${xRateLimitRemaining}, reset=${xRateLimitReset}`
     );
     if (xRateLimitRemaining == "0" && xRateLimitReset) {
@@ -108,16 +108,47 @@ class Platform {
   }
 }
 
+const log$3 = debug("twitter-scraper:requests");
 async function updateCookieJar(cookieJar, headers) {
-  const setCookieHeader = headers.get("set-cookie");
-  if (setCookieHeader) {
-    const cookies = setCookie.splitCookiesString(setCookieHeader);
-    for (const cookie of cookies.map((c) => toughCookie.Cookie.parse(c))) {
-      if (!cookie) continue;
-      await cookieJar.setCookie(
-        cookie,
-        `${cookie.secure ? "https" : "http"}://${cookie.domain}${cookie.path}`
-      );
+  let setCookieHeaders = [];
+  if (typeof headers.getSetCookie === "function") {
+    setCookieHeaders = headers.getSetCookie();
+  } else {
+    const setCookieHeader = headers.get("set-cookie");
+    if (setCookieHeader) {
+      setCookieHeaders = setCookie.splitCookiesString(setCookieHeader);
+    }
+  }
+  if (setCookieHeaders.length > 0) {
+    for (const cookieStr of setCookieHeaders) {
+      const cookie = toughCookie.Cookie.parse(cookieStr);
+      if (!cookie) {
+        log$3(`Failed to parse cookie: ${cookieStr.substring(0, 100)}`);
+        continue;
+      }
+      if (cookie.maxAge === 0 || cookie.expires && cookie.expires < /* @__PURE__ */ new Date()) {
+        if (cookie.key === "ct0") {
+          log$3(`Skipping deletion of ct0 cookie (Max-Age=0)`);
+        }
+        continue;
+      }
+      try {
+        const url = `${cookie.secure ? "https" : "http"}://${cookie.domain}${cookie.path}`;
+        await cookieJar.setCookie(cookie, url);
+        if (cookie.key === "ct0") {
+          log$3(
+            `Successfully set ct0 cookie with value: ${cookie.value.substring(
+              0,
+              20
+            )}...`
+          );
+        }
+      } catch (err) {
+        log$3(`Failed to set cookie ${cookie.key}: ${err}`);
+        if (cookie.key === "ct0") {
+          log$3(`FAILED to set ct0 cookie! Error: ${err}`);
+        }
+      }
     }
   } else if (typeof document !== "undefined") {
     for (const cookie of document.cookie.split(";")) {
@@ -135,9 +166,8 @@ async function jitter(maxMs) {
   const jitter2 = Math.random() * maxMs;
   await new Promise((resolve) => setTimeout(resolve, jitter2));
 }
-async function requestApi(url, auth, method = "GET", platform = new Platform()) {
+async function requestApi(url, auth, method = "GET", platform = new Platform(), headers = new headersPolyfill.Headers()) {
   log$2(`Making ${method} request to ${url}`);
-  const headers = new headersPolyfill.Headers();
   await auth.installTo(headers, url);
   await platform.randomizeCiphers();
   let res;
@@ -323,6 +353,16 @@ class TwitterGuestAuth {
     }
     headers.set("cookie", await this.getCookieString());
   }
+  async setCookie(key, value) {
+    const cookie = toughCookie.Cookie.parse(`${key}=${value}`);
+    if (!cookie) {
+      throw new Error("Failed to parse cookie.");
+    }
+    await this.jar.setCookie(cookie, this.getCookieJarUrl());
+    if (typeof document !== "undefined") {
+      document.cookie = cookie.toString();
+    }
+  }
   async getCookies() {
     return this.jar.getCookies(this.getCookieJarUrl());
   }
@@ -373,6 +413,8 @@ class TwitterGuestAuth {
     }
     this.guestToken = newGuestToken;
     this.guestCreatedAt = /* @__PURE__ */ new Date();
+    await this.setCookie("gt", newGuestToken);
+    log$1(`Updated guest token: ${newGuestToken}`);
   }
   /**
    * Returns if the authentication token needs to be updated or not.
@@ -499,7 +541,11 @@ class TwitterUserAuth extends TwitterGuestAuth {
   }
   async installTo(headers) {
     headers.set("authorization", `Bearer ${this.bearerToken}`);
-    headers.set("cookie", await this.getCookieString());
+    const cookie = await this.getCookieString();
+    headers.set("cookie", cookie);
+    if (this.guestToken) {
+      headers.set("x-guest-token", this.guestToken);
+    }
     await this.installCsrfToken(headers);
   }
   async initLogin() {
@@ -712,16 +758,27 @@ class TwitterUserAuth extends TwitterGuestAuth {
       );
     }
     const headers = new headersPolyfill.Headers({
-      authorization: `Bearer ${this.bearerToken}`,
-      cookie: await this.getCookieString(),
+      accept: "*/*",
+      "accept-language": "en-US,en;q=0.9",
       "content-type": "application/json",
-      "User-Agent": "Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36",
+      "cache-control": "no-cache",
+      origin: "https://x.com",
+      pragma: "no-cache",
+      priority: "u=1, i",
+      referer: "https://x.com/",
+      "sec-ch-ua": '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
+      "sec-ch-ua-mobile": "?0",
+      "sec-ch-ua-platform": '"Windows"',
+      "sec-fetch-dest": "empty",
+      "sec-fetch-mode": "cors",
+      "sec-fetch-site": "same-origin",
+      "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
       "x-guest-token": token,
       "x-twitter-auth-type": "OAuth2Client",
       "x-twitter-active-user": "yes",
       "x-twitter-client-language": "en"
     });
-    await this.installCsrfToken(headers);
+    await this.installTo(headers);
     let res;
     do {
       const fetchParameters = [