@the-convocation/twitter-scraper 0.19.1 → 0.20.0

package/dist/node/cjs/index.cjs

@@ -72,13 +72,13 @@ class AuthenticationError extends Error {
   }
 }
 
-const log$4 = debug("twitter-scraper:rate-limit");
+const log$6 = debug("twitter-scraper:rate-limit");
 class WaitingRateLimitStrategy {
   async onRateLimit({ response: res }) {
     const xRateLimitLimit = res.headers.get("x-rate-limit-limit");
     const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
     const xRateLimitReset = res.headers.get("x-rate-limit-reset");
-    log$4(
+    log$6(
       `Rate limit event: limit=${xRateLimitLimit}, remaining=${xRateLimitRemaining}, reset=${xRateLimitReset}`
     );
     if (xRateLimitRemaining == "0" && xRateLimitReset) {
@@ -94,20 +94,7 @@ class ErrorRateLimitStrategy {
   }
 }
 
-class Platform {
-  async randomizeCiphers() {
-    const platform = await Platform.importPlatform();
-    await platform?.randomizeCiphers();
-  }
-  static async importPlatform() {
-    {
-      const { platform } = await Promise.resolve().then(function () { return index; });
-      return platform;
-    }
-  }
-}
-
-const log$3 = debug("twitter-scraper:requests");
+const log$5 = debug("twitter-scraper:requests");
 async function updateCookieJar(cookieJar, headers) {
   let setCookieHeaders = [];
   if (typeof headers.getSetCookie === "function") {
@@ -122,12 +109,12 @@ async function updateCookieJar(cookieJar, headers) {
     for (const cookieStr of setCookieHeaders) {
       const cookie = toughCookie.Cookie.parse(cookieStr);
       if (!cookie) {
-        log$3(`Failed to parse cookie: ${cookieStr.substring(0, 100)}`);
+        log$5(`Failed to parse cookie: ${cookieStr.substring(0, 100)}`);
         continue;
       }
       if (cookie.maxAge === 0 || cookie.expires && cookie.expires < /* @__PURE__ */ new Date()) {
         if (cookie.key === "ct0") {
-          log$3(`Skipping deletion of ct0 cookie (Max-Age=0)`);
+          log$5(`Skipping deletion of ct0 cookie (Max-Age=0)`);
         }
         continue;
       }
@@ -135,7 +122,7 @@ async function updateCookieJar(cookieJar, headers) {
         const url = `${cookie.secure ? "https" : "http"}://${cookie.domain}${cookie.path}`;
         await cookieJar.setCookie(cookie, url);
         if (cookie.key === "ct0") {
-          log$3(
+          log$5(
             `Successfully set ct0 cookie with value: ${cookie.value.substring(
               0,
               20
@@ -143,9 +130,9 @@ async function updateCookieJar(cookieJar, headers) {
           );
         }
       } catch (err) {
-        log$3(`Failed to set cookie ${cookie.key}: ${err}`);
+        log$5(`Failed to set cookie ${cookie.key}: ${err}`);
         if (cookie.key === "ct0") {
-          log$3(`FAILED to set ct0 cookie! Error: ${err}`);
+          log$5(`FAILED to set ct0 cookie! Error: ${err}`);
         }
       }
     }
@@ -159,141 +146,84 @@ async function updateCookieJar(cookieJar, headers) {
   }
 }
 
-const log$2 = debug("twitter-scraper:api");
-const bearerToken = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF";
-async function jitter(maxMs) {
-  const jitter2 = Math.random() * maxMs;
-  await new Promise((resolve) => setTimeout(resolve, jitter2));
-}
-async function requestApi(url, auth, method = "GET", platform = new Platform(), headers = new headersPolyfill.Headers()) {
-  log$2(`Making ${method} request to ${url}`);
-  await auth.installTo(headers, url);
-  await platform.randomizeCiphers();
-  let res;
-  do {
-    const fetchParameters = [
-      url,
+const log$4 = debug("twitter-scraper:xpff");
+let isoCrypto = null;
+function getCrypto() {
+  if (isoCrypto != null) {
+    return isoCrypto;
+  }
+  if (typeof crypto === "undefined") {
+    log$4("Global crypto is undefined, importing from crypto module...");
+    const { webcrypto } = require("crypto");
+    isoCrypto = webcrypto;
+    return webcrypto;
+  }
+  isoCrypto = crypto;
+  return crypto;
+}
+async function sha256(message) {
+  const msgBuffer = new TextEncoder().encode(message);
+  const hashBuffer = await getCrypto().subtle.digest("SHA-256", msgBuffer);
+  return new Uint8Array(hashBuffer);
+}
+function buf2hex(buffer) {
+  return [...new Uint8Array(buffer)].map((x) => x.toString(16).padStart(2, "0")).join("");
+}
+class XPFFHeaderGenerator {
+  constructor(seed) {
+    this.seed = seed;
+  }
+  async deriveKey(guestId) {
+    const combined = `${this.seed}${guestId}`;
+    const result = await sha256(combined);
+    return result;
+  }
+  async generateHeader(plaintext, guestId) {
+    log$4(`Generating XPFF key for guest ID: ${guestId}`);
+    const key = await this.deriveKey(guestId);
+    const nonce = getCrypto().getRandomValues(new Uint8Array(12));
+    const cipher = await getCrypto().subtle.importKey(
+      "raw",
+      key,
+      { name: "AES-GCM" },
+      false,
+      ["encrypt"]
+    );
+    const encrypted = await getCrypto().subtle.encrypt(
       {
-        method,
-        headers,
-        credentials: "include"
-      }
-    ];
-    try {
-      res = await auth.fetch(...fetchParameters);
-    } catch (err) {
-      if (!(err instanceof Error)) {
-        throw err;
-      }
-      return {
-        success: false,
-        err: new Error("Failed to perform request.")
-      };
-    }
-    await updateCookieJar(auth.cookieJar(), res.headers);
-    if (res.status === 429) {
-      log$2("Rate limit hit, waiting for retry...");
-      await auth.onRateLimit({
-        fetchParameters,
-        response: res
-      });
-    }
-  } while (res.status === 429);
-  if (!res.ok) {
-    return {
-      success: false,
-      err: await ApiError.fromResponse(res)
-    };
-  }
-  const value = await flexParseJson(res);
-  if (res.headers.get("x-rate-limit-incoming") == "0") {
-    auth.deleteToken();
-    return { success: true, value };
-  } else {
-    return { success: true, value };
-  }
-}
-async function flexParseJson(res) {
-  try {
-    return await res.json();
-  } catch {
-    log$2("Failed to parse response as JSON, trying text parse...");
-    const text = await res.text();
-    log$2("Response text:", text);
-    return JSON.parse(text);
-  }
-}
-function addApiFeatures(o) {
-  return {
-    ...o,
-    rweb_lists_timeline_redesign_enabled: true,
-    responsive_web_graphql_exclude_directive_enabled: true,
-    verified_phone_label_enabled: false,
-    creator_subscriptions_tweet_preview_api_enabled: true,
-    responsive_web_graphql_timeline_navigation_enabled: true,
-    responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
-    tweetypie_unmention_optimization_enabled: true,
-    responsive_web_edit_tweet_api_enabled: true,
-    graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
-    view_counts_everywhere_api_enabled: true,
-    longform_notetweets_consumption_enabled: true,
-    tweet_awards_web_tipping_enabled: false,
-    freedom_of_speech_not_reach_fetch_enabled: true,
-    standardized_nudges_misinfo: true,
-    longform_notetweets_rich_text_read_enabled: true,
-    responsive_web_enhance_cards_enabled: false,
-    subscriptions_verification_info_enabled: true,
-    subscriptions_verification_info_reason_enabled: true,
-    subscriptions_verification_info_verified_since_enabled: true,
-    super_follow_badge_privacy_enabled: false,
-    super_follow_exclusive_tweet_notifications_enabled: false,
-    super_follow_tweet_api_enabled: false,
-    super_follow_user_api_enabled: false,
-    android_graphql_skip_api_media_color_palette: false,
-    creator_subscriptions_subscription_count_enabled: false,
-    blue_business_profile_image_shape_enabled: false,
-    unified_cards_ad_metadata_container_dynamic_card_content_query_enabled: false
-  };
+        name: "AES-GCM",
+        iv: nonce
+      },
+      cipher,
+      new TextEncoder().encode(plaintext)
+    );
+    const combined = new Uint8Array(nonce.length + encrypted.byteLength);
+    combined.set(nonce);
+    combined.set(new Uint8Array(encrypted), nonce.length);
+    const result = buf2hex(combined);
+    log$4(`XPFF header generated for guest ID ${guestId}: ${result}`);
+    return result;
+  }
+}
+const xpffBaseKey = "0e6be1f1e21ffc33590b888fd4dc81b19713e570e805d4e5df80a493c9571a05";
+function xpffPlain() {
+  const timestamp = Date.now();
+  return JSON.stringify({
+    navigator_properties: {
+      hasBeenActive: "true",
+      userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/137.0.0.0 Safari/537.36",
+      webdriver: "false"
+    },
+    created_at: timestamp
+  });
 }
-function addApiParams(params, includeTweetReplies) {
-  params.set("include_profile_interstitial_type", "1");
-  params.set("include_blocking", "1");
-  params.set("include_blocked_by", "1");
-  params.set("include_followed_by", "1");
-  params.set("include_want_retweets", "1");
-  params.set("include_mute_edge", "1");
-  params.set("include_can_dm", "1");
-  params.set("include_can_media_tag", "1");
-  params.set("include_ext_has_nft_avatar", "1");
-  params.set("include_ext_is_blue_verified", "1");
-  params.set("include_ext_verified_type", "1");
-  params.set("skip_status", "1");
-  params.set("cards_platform", "Web-12");
-  params.set("include_cards", "1");
-  params.set("include_ext_alt_text", "true");
-  params.set("include_ext_limited_action_results", "false");
-  params.set("include_quote_count", "true");
-  params.set("include_reply_count", "1");
-  params.set("tweet_mode", "extended");
-  params.set("include_ext_collab_control", "true");
-  params.set("include_ext_views", "true");
-  params.set("include_entities", "true");
-  params.set("include_user_entities", "true");
-  params.set("include_ext_media_color", "true");
-  params.set("include_ext_media_availability", "true");
-  params.set("include_ext_sensitive_media_warning", "true");
-  params.set("include_ext_trusted_friends_metadata", "true");
-  params.set("send_error_codes", "true");
-  params.set("simple_quoted_tweet", "true");
-  params.set("include_tweet_replies", `${includeTweetReplies}`);
-  params.set(
-    "ext",
-    "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,collab_control,vibe"
-  );
-  return params;
+async function generateXPFFHeader(guestId) {
+  const generator = new XPFFHeaderGenerator(xpffBaseKey);
+  const plaintext = xpffPlain();
+  return generator.generateHeader(plaintext, guestId);
 }
 
-const log$1 = debug("twitter-scraper:auth");
+const log$3 = debug("twitter-scraper:auth");
 function withTransform(fetchFn, transform) {
   return async (input, init) => {
     const fetchArgs = await transform?.request?.(input, init) ?? [
@@ -347,24 +277,30 @@ class TwitterGuestAuth {
     if (this.shouldUpdate()) {
       await this.updateGuestToken();
     }
-    const token = this.guestToken;
-    if (token == null) {
-      throw new AuthenticationError(
-        "Authentication token is null or undefined."
-      );
+    if (this.guestToken) {
+      headers.set("x-guest-token", this.guestToken);
     }
     headers.set("authorization", `Bearer ${this.bearerToken}`);
-    headers.set("x-guest-token", token);
     headers.set(
       "user-agent",
       "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
     );
+    await this.installCsrfToken(headers);
+    if (this.options?.experimental?.xpff) {
+      const guestId = await this.guestId();
+      if (guestId != null) {
+        const xpffHeader = await generateXPFFHeader(guestId);
+        headers.set("x-xp-forwarded-for", xpffHeader);
+      }
+    }
+    headers.set("cookie", await this.getCookieString());
+  }
+  async installCsrfToken(headers) {
     const cookies = await this.getCookies();
     const xCsrfToken = cookies.find((cookie) => cookie.key === "ct0");
     if (xCsrfToken) {
       headers.set("x-csrf-token", xCsrfToken.value);
     }
-    headers.set("cookie", await this.getCookieString());
   }
   async setCookie(key, value) {
     const cookie = toughCookie.Cookie.parse(`${key}=${value}`);
@@ -397,16 +333,28 @@ class TwitterGuestAuth {
   getCookieJarUrl() {
     return typeof document !== "undefined" ? document.location.toString() : "https://x.com";
   }
+  async guestId() {
+    const cookies = await this.getCookies();
+    const guestIdCookie = cookies.find((cookie) => cookie.key === "guest_id");
+    return guestIdCookie ? guestIdCookie.value : null;
+  }
   /**
    * Updates the authentication state with a new guest token from the Twitter API.
    */
   async updateGuestToken() {
+    try {
+      await this.updateGuestTokenCore();
+    } catch (err) {
+      log$3("Failed to update guest token; this may cause issues:", err);
+    }
+  }
+  async updateGuestTokenCore() {
     const guestActivateUrl = "https://api.x.com/1.1/guest/activate.json";
     const headers = new headersPolyfill.Headers({
       Authorization: `Bearer ${this.bearerToken}`,
       Cookie: await this.getCookieString()
     });
-    log$1(`Making POST request to ${guestActivateUrl}`);
+    log$3(`Making POST request to ${guestActivateUrl}`);
     const res = await this.fetch(guestActivateUrl, {
       method: "POST",
       headers,
@@ -427,7 +375,7 @@ class TwitterGuestAuth {
     this.guestToken = newGuestToken;
     this.guestCreatedAt = /* @__PURE__ */ new Date();
     await this.setCookie("gt", newGuestToken);
-    log$1(`Updated guest token: ${newGuestToken}`);
+    log$3(`Updated guest token: ${newGuestToken}`);
   }
   /**
    * Returns if the authentication token needs to be updated or not.
@@ -438,6 +386,277 @@ class TwitterGuestAuth {
   }
 }
 
+class Platform {
+  async randomizeCiphers() {
+    const platform = await Platform.importPlatform();
+    await platform?.randomizeCiphers();
+  }
+  static async importPlatform() {
+    {
+      const { platform } = await Promise.resolve().then(function () { return index; });
+      return platform;
+    }
+  }
+}
+
+const log$2 = debug("twitter-scraper:xctxid");
+let linkedom = null;
+function linkedomImport() {
+  if (!linkedom) {
+    const mod = require("linkedom");
+    linkedom = mod;
+    return mod;
+  }
+  return linkedom;
+}
+async function parseHTML(html) {
+  if (typeof window !== "undefined") {
+    const { defaultView } = new DOMParser().parseFromString(html, "text/html");
+    if (!defaultView) {
+      throw new Error("Failed to get defaultView from parsed HTML.");
+    }
+    return defaultView;
+  } else {
+    const { DOMParser: DOMParser2 } = linkedomImport();
+    return new DOMParser2().parseFromString(html, "text/html").defaultView;
+  }
+}
+async function handleXMigration(fetchFn) {
+  const headers = {
+    accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
+    "accept-language": "ja",
+    "cache-control": "no-cache",
+    pragma: "no-cache",
+    priority: "u=0, i",
+    "sec-ch-ua": '"Google Chrome";v="135", "Not-A.Brand";v="8", "Chromium";v="135"',
+    "sec-ch-ua-mobile": "?0",
+    "sec-ch-ua-platform": '"Windows"',
+    "sec-fetch-dest": "document",
+    "sec-fetch-mode": "navigate",
+    "sec-fetch-site": "none",
+    "sec-fetch-user": "?1",
+    "upgrade-insecure-requests": "1",
+    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+  };
+  const response = await fetchFn("https://x.com", {
+    headers
+  });
+  if (!response.ok) {
+    throw new Error(`Failed to fetch X homepage: ${response.statusText}`);
+  }
+  const htmlText = await response.text();
+  let dom = await parseHTML(htmlText);
+  let document = dom.window.document;
+  const migrationRedirectionRegex = new RegExp(
+    "(http(?:s)?://(?:www\\.)?(twitter|x){1}\\.com(/x)?/migrate([/?])?tok=[a-zA-Z0-9%\\-_]+)+",
+    "i"
+  );
+  const metaRefresh = document.querySelector("meta[http-equiv='refresh']");
+  const metaContent = metaRefresh ? metaRefresh.getAttribute("content") || "" : "";
+  const migrationRedirectionUrl = migrationRedirectionRegex.exec(metaContent) || migrationRedirectionRegex.exec(htmlText);
+  if (migrationRedirectionUrl) {
+    const redirectResponse = await fetch(migrationRedirectionUrl[0]);
+    if (!redirectResponse.ok) {
+      throw new Error(
+        `Failed to follow migration redirection: ${redirectResponse.statusText}`
+      );
+    }
+    const redirectHtml = await redirectResponse.text();
+    dom = await parseHTML(redirectHtml);
+    document = dom.window.document;
+  }
+  const migrationForm = document.querySelector("form[name='f']") || document.querySelector("form[action='https://x.com/x/migrate']");
+  if (migrationForm) {
+    const url = migrationForm.getAttribute("action") || "https://x.com/x/migrate";
+    const method = migrationForm.getAttribute("method") || "POST";
+    const requestPayload = new FormData();
+    const inputFields = migrationForm.querySelectorAll("input");
+    for (const element of Array.from(inputFields)) {
+      const name = element.getAttribute("name");
+      const value = element.getAttribute("value");
+      if (name && value) {
+        requestPayload.append(name, value);
+      }
+    }
+    const formResponse = await fetch(url, {
+      method,
+      body: requestPayload,
+      headers
+    });
+    if (!formResponse.ok) {
+      throw new Error(
+        `Failed to submit migration form: ${formResponse.statusText}`
+      );
+    }
+    const formHtml = await formResponse.text();
+    dom = await parseHTML(formHtml);
+    document = dom.window.document;
+  }
+  return document;
+}
+let ClientTransaction = null;
+function clientTransaction() {
+  if (!ClientTransaction) {
+    const mod = require("x-client-transaction-id");
+    const ctx = mod.ClientTransaction;
+    ClientTransaction = ctx;
+    return ctx;
+  }
+  return ClientTransaction;
+}
+async function generateTransactionId(url, fetchFn, method) {
+  const parsedUrl = new URL(url);
+  const path = parsedUrl.pathname;
+  log$2(`Generating transaction ID for ${method} ${path}`);
+  const document = await handleXMigration(fetchFn);
+  const transaction = await clientTransaction().create(document);
+  const transactionId = await transaction.generateTransactionId(method, path);
+  log$2(`Transaction ID: ${transactionId}`);
+  return transactionId;
+}
+
+const log$1 = debug("twitter-scraper:api");
+const bearerToken = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF";
+async function jitter(maxMs) {
+  const jitter2 = Math.random() * maxMs;
+  await new Promise((resolve) => setTimeout(resolve, jitter2));
+}
+async function requestApi(url, auth, method = "GET", platform = new Platform(), headers = new headersPolyfill.Headers()) {
+  log$1(`Making ${method} request to ${url}`);
+  await auth.installTo(headers, url);
+  await platform.randomizeCiphers();
+  if (auth instanceof TwitterGuestAuth && auth.options?.experimental?.xClientTransactionId) {
+    const transactionId = await generateTransactionId(
+      url,
+      auth.fetch.bind(auth),
+      method
+    );
+    headers.set("x-client-transaction-id", transactionId);
+  }
+  let res;
+  do {
+    const fetchParameters = [
+      url,
+      {
+        method,
+        headers,
+        credentials: "include"
+      }
+    ];
+    try {
+      res = await auth.fetch(...fetchParameters);
+    } catch (err) {
+      if (!(err instanceof Error)) {
+        throw err;
+      }
+      return {
+        success: false,
+        err: new Error("Failed to perform request.")
+      };
+    }
+    await updateCookieJar(auth.cookieJar(), res.headers);
+    if (res.status === 429) {
+      log$1("Rate limit hit, waiting for retry...");
+      await auth.onRateLimit({
+        fetchParameters,
+        response: res
+      });
+    }
+  } while (res.status === 429);
+  if (!res.ok) {
+    return {
+      success: false,
+      err: await ApiError.fromResponse(res)
+    };
+  }
+  const value = await flexParseJson(res);
+  if (res.headers.get("x-rate-limit-incoming") == "0") {
+    auth.deleteToken();
+    return { success: true, value };
+  } else {
+    return { success: true, value };
+  }
+}
+async function flexParseJson(res) {
+  try {
+    return await res.json();
+  } catch {
+    log$1("Failed to parse response as JSON, trying text parse...");
+    const text = await res.text();
+    log$1("Response text:", text);
+    return JSON.parse(text);
+  }
+}
+function addApiFeatures(o) {
+  return {
+    ...o,
+    rweb_lists_timeline_redesign_enabled: true,
+    responsive_web_graphql_exclude_directive_enabled: true,
+    verified_phone_label_enabled: false,
+    creator_subscriptions_tweet_preview_api_enabled: true,
+    responsive_web_graphql_timeline_navigation_enabled: true,
+    responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
+    tweetypie_unmention_optimization_enabled: true,
+    responsive_web_edit_tweet_api_enabled: true,
+    graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
+    view_counts_everywhere_api_enabled: true,
+    longform_notetweets_consumption_enabled: true,
+    tweet_awards_web_tipping_enabled: false,
+    freedom_of_speech_not_reach_fetch_enabled: true,
+    standardized_nudges_misinfo: true,
+    longform_notetweets_rich_text_read_enabled: true,
+    responsive_web_enhance_cards_enabled: false,
+    subscriptions_verification_info_enabled: true,
+    subscriptions_verification_info_reason_enabled: true,
+    subscriptions_verification_info_verified_since_enabled: true,
+    super_follow_badge_privacy_enabled: false,
+    super_follow_exclusive_tweet_notifications_enabled: false,
+    super_follow_tweet_api_enabled: false,
+    super_follow_user_api_enabled: false,
+    android_graphql_skip_api_media_color_palette: false,
+    creator_subscriptions_subscription_count_enabled: false,
+    blue_business_profile_image_shape_enabled: false,
+    unified_cards_ad_metadata_container_dynamic_card_content_query_enabled: false
+  };
+}
+function addApiParams(params, includeTweetReplies) {
+  params.set("include_profile_interstitial_type", "1");
+  params.set("include_blocking", "1");
+  params.set("include_blocked_by", "1");
+  params.set("include_followed_by", "1");
+  params.set("include_want_retweets", "1");
+  params.set("include_mute_edge", "1");
+  params.set("include_can_dm", "1");
+  params.set("include_can_media_tag", "1");
+  params.set("include_ext_has_nft_avatar", "1");
+  params.set("include_ext_is_blue_verified", "1");
+  params.set("include_ext_verified_type", "1");
+  params.set("skip_status", "1");
+  params.set("cards_platform", "Web-12");
+  params.set("include_cards", "1");
+  params.set("include_ext_alt_text", "true");
+  params.set("include_ext_limited_action_results", "false");
+  params.set("include_quote_count", "true");
+  params.set("include_reply_count", "1");
+  params.set("tweet_mode", "extended");
+  params.set("include_ext_collab_control", "true");
+  params.set("include_ext_views", "true");
+  params.set("include_entities", "true");
+  params.set("include_user_entities", "true");
+  params.set("include_ext_media_color", "true");
+  params.set("include_ext_media_availability", "true");
+  params.set("include_ext_sensitive_media_warning", "true");
+  params.set("include_ext_trusted_friends_metadata", "true");
+  params.set("send_error_codes", "true");
+  params.set("simple_quoted_tweet", "true");
+  params.set("include_tweet_replies", `${includeTweetReplies}`);
+  params.set(
+    "ext",
+    "mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,collab_control,vibe"
+  );
+  return params;
+}
+
 const log = debug("twitter-scraper:auth-user");
 const TwitterUserAuthSubtask = typebox.Type.Object({
   subtask_id: typebox.Type.String(),
@@ -545,25 +764,25 @@ class TwitterUserAuth extends TwitterGuestAuth {
       this.jar = new toughCookie.CookieJar();
     }
   }
-  async installCsrfToken(headers) {
-    const cookies = await this.getCookies();
-    const xCsrfToken = cookies.find((cookie) => cookie.key === "ct0");
-    if (xCsrfToken) {
-      headers.set("x-csrf-token", xCsrfToken.value);
-    }
-  }
   async installTo(headers) {
     headers.set("authorization", `Bearer ${this.bearerToken}`);
-    const cookie = await this.getCookieString();
-    headers.set("cookie", cookie);
-    if (this.guestToken) {
-      headers.set("x-guest-token", this.guestToken);
-    }
     headers.set(
       "user-agent",
       "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
     );
+    if (this.guestToken) {
+      headers.set("x-guest-token", this.guestToken);
+    }
     await this.installCsrfToken(headers);
+    if (this.options?.experimental?.xpff) {
+      const guestId = await this.guestId();
+      if (guestId != null) {
+        const xpffHeader = await generateXPFFHeader(guestId);
+        headers.set("x-xp-forwarded-for", xpffHeader);
+      }
+    }
+    const cookie = await this.getCookieString();
+    headers.set("cookie", cookie);
   }
   async initLogin() {
     this.removeCookie("twitter_ads_id=");
@@ -768,12 +987,6 @@ class TwitterUserAuth extends TwitterGuestAuth {
       onboardingTaskUrl = `https://api.x.com/1.1/onboarding/task.json?flow_name=${data.flow_name}`;
     }
     log(`Making POST request to ${onboardingTaskUrl}`);
-    const token = this.guestToken;
-    if (token == null) {
-      throw new AuthenticationError(
-        "Authentication token is null or undefined."
-      );
-    }
     const headers = new headersPolyfill.Headers({
       accept: "*/*",
       "accept-language": "en-US,en;q=0.9",
@@ -790,12 +1003,19 @@ class TwitterUserAuth extends TwitterGuestAuth {
       "sec-fetch-mode": "cors",
       "sec-fetch-site": "same-origin",
       "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36",
-      "x-guest-token": token,
       "x-twitter-auth-type": "OAuth2Client",
       "x-twitter-active-user": "yes",
       "x-twitter-client-language": "en"
     });
     await this.installTo(headers);
+    if (this.options?.experimental?.xClientTransactionId) {
+      const transactionId = await generateTransactionId(
+        onboardingTaskUrl,
+        this.fetch.bind(this),
+        "POST"
+      );
+      headers.set("x-client-transaction-id", transactionId);
+    }
     let res;
     do {
       const fetchParameters = [
@@ -2620,7 +2840,11 @@ class Scraper {
     return {
       fetch: this.options?.fetch,
       transform: this.options?.transform,
-      rateLimitStrategy: this.options?.rateLimitStrategy
+      rateLimitStrategy: this.options?.rateLimitStrategy,
+      experimental: {
+        xClientTransactionId: this.options?.experimental?.xClientTransactionId,
+        xpff: this.options?.experimental?.xpff
+      }
     };
   }
   handleResponse(res) {