@the-convocation/twitter-scraper 0.17.2 → 0.18.0

package/dist/default/esm/index.mjs

@@ -791,13 +791,16 @@ class ApiRequest {
   toRequestUrl() {
     const params = new URLSearchParams();
     if (this.variables) {
-      params.set("variables", stringify(this.variables));
+      const variablesStr = stringify(this.variables);
+      if (variablesStr) params.set("variables", variablesStr);
     }
     if (this.features) {
-      params.set("features", stringify(this.features));
+      const featuresStr = stringify(this.features);
+      if (featuresStr) params.set("features", featuresStr);
     }
     if (this.fieldToggles) {
-      params.set("fieldToggles", stringify(this.fieldToggles));
+      const fieldTogglesStr = stringify(this.fieldToggles);
+      if (fieldTogglesStr) params.set("fieldToggles", fieldTogglesStr);
     }
     return `${this.url}?${params.toString()}`;
   }
@@ -1523,9 +1526,12 @@ async function getSearchTimeline(query, maxItems, searchMode, auth, cursor) {
       break;
   }
   const params = new URLSearchParams();
-  params.set("features", stringify(features));
-  params.set("fieldToggles", stringify(fieldToggles));
-  params.set("variables", stringify(variables));
+  const featuresStr = stringify(features);
+  const fieldTogglesStr = stringify(fieldToggles);
+  const variablesStr = stringify(variables);
+  if (featuresStr) params.set("features", featuresStr);
+  if (fieldTogglesStr) params.set("fieldToggles", fieldTogglesStr);
+  if (variablesStr) params.set("variables", variablesStr);
   const res = await requestApi(
     `https://api.x.com/graphql/gkjsKepM6gl_HmFWoWKfgg/SearchTimeline?${params.toString()}`,
     auth
@@ -1639,8 +1645,10 @@ async function getFollowingTimeline(userId, maxItems, auth, cursor) {
     variables["cursor"] = cursor;
   }
   const params = new URLSearchParams();
-  params.set("features", stringify(features));
-  params.set("variables", stringify(variables));
+  const featuresStr = stringify(features);
+  const variablesStr = stringify(variables);
+  if (featuresStr) params.set("features", featuresStr);
+  if (variablesStr) params.set("variables", variablesStr);
   const res = await requestApi(
     `https://x.com/i/api/graphql/iSicc7LrzWGBgDPL0tM_TQ/Following?${params.toString()}`,
     auth
@@ -1674,8 +1682,10 @@ async function getFollowersTimeline(userId, maxItems, auth, cursor) {
     variables["cursor"] = cursor;
   }
   const params = new URLSearchParams();
-  params.set("features", stringify(features));
-  params.set("variables", stringify(variables));
+  const featuresStr = stringify(features);
+  const variablesStr = stringify(variables);
+  if (featuresStr) params.set("features", featuresStr);
+  if (variablesStr) params.set("variables", variablesStr);
   const res = await requestApi(
     `https://x.com/i/api/graphql/rRXFSG5vR6drKr5M37YOTw/Followers?${params.toString()}`,
     auth
@@ -1951,6 +1961,155 @@ async function getTweetAnonymous(id, auth) {
   return parseTimelineEntryItemContentRaw(res.value.data, id);
 }
 
+async function* getDmConversationMessagesGenerator(conversationId, maxMessages, initialCursor, fetchFunc) {
+  let nMessages = 0;
+  let cursor = initialCursor;
+  while (nMessages < maxMessages) {
+    const batch = await fetchFunc(
+      conversationId,
+      maxMessages,
+      cursor
+    );
+    const { conversation, next } = batch;
+    if (!conversation?.entries || conversation?.entries?.length === 0) {
+      break;
+    }
+    for (const entry of conversation.entries) {
+      if (nMessages < maxMessages) {
+        yield entry;
+        nMessages++;
+      } else {
+        break;
+      }
+    }
+    cursor = next;
+    if (conversation.status === "AT_END" || !next) {
+      break;
+    }
+    await jitter(1e3);
+  }
+}
+
+async function fetchDmInbox(auth) {
+  if (!await auth.isLoggedIn()) {
+    throw new AuthenticationError(
+      "Scraper is not logged-in for fetching direct messages."
+    );
+  }
+  const params = new URLSearchParams();
+  addApiParams(params, false);
+  params.set("nsfw_filtering_enabled", "false");
+  params.set("filter_low_quality", "true");
+  params.set("include_quality", "all");
+  params.set("include_ext_profile_image_shape", "1");
+  params.set("dm_secret_conversations_enabled", "false");
+  params.set("krs_registration_enabled", "false");
+  params.set("include_ext_limited_action_results", "true");
+  params.set("dm_users", "true");
+  params.set("include_groups", "true");
+  params.set("include_inbox_timelines", "true");
+  params.set("supports_reactions", "true");
+  params.set("supports_edit", "true");
+  params.set("include_ext_edit_control", "true");
+  params.set("include_ext_business_affiliations_label", "true");
+  params.set("include_ext_parody_commentary_fan_label", "true");
+  params.set(
+    "ext",
+    "mediaColor,altText,mediaStats,highlightedLabel,parodyCommentaryFanLabel,voiceInfo,birdwatchPivot,superFollowMetadata,unmentionInfo,editControl,article"
+  );
+  const res = await requestApi(
+    `https://x.com/i/api/1.1/dm/inbox_initial_state.json?${params.toString()}`,
+    auth
+  );
+  if (!res.success) {
+    throw res.err;
+  }
+  return parseDmInbox(res.value);
+}
+async function parseDmInbox(inbox) {
+  return inbox.inbox_initial_state;
+}
+async function getDmInbox(auth) {
+  return await fetchDmInbox(auth);
+}
+async function fetchDmConversation(conversationId, cursor, auth) {
+  if (!await auth.isLoggedIn()) {
+    throw new AuthenticationError(
+      "Scraper is not logged-in for fetching direct messages."
+    );
+  }
+  const params = new URLSearchParams();
+  addApiParams(params, false);
+  params.set("context", "FETCH_DM_CONVERSATION_HISTORY");
+  params.set("include_ext_profile_image_shape", "1");
+  params.set("dm_secret_conversations_enabled", "false");
+  params.set("krs_registration_enabled", "false");
+  params.set("include_ext_limited_action_results", "true");
+  params.set("dm_users", "true");
+  params.set("include_groups", "true");
+  params.set("include_inbox_timelines", "true");
+  params.set("supports_reactions", "true");
+  params.set("supports_edit", "true");
+  params.set("include_conversation_info", "true");
+  params.set(
+    "ext",
+    "mediaColor,altText,mediaStats,highlightedLabel,parodyCommentaryFanLabel,voiceInfo,birdwatchPivot,superFollowMetadata,unmentionInfo,editControl,article"
+  );
+  if (cursor) {
+    if (cursor.maxId) {
+      params.set("max_id", cursor.maxId);
+    }
+    if (cursor.minId) {
+      params.set("min_id", cursor.minId);
+    }
+  }
+  const url = `https://x.com/i/api/1.1/dm/conversation/${conversationId}.json?${params.toString()}`;
+  const res = await requestApi(url, auth);
+  if (!res.success) {
+    throw res.err;
+  }
+  return parseDmConversation(res.value);
+}
+async function parseDmConversation(conversation) {
+  return conversation.conversation_timeline;
+}
+async function getDmConversation(conversationId, cursor, auth) {
+  return await fetchDmConversation(conversationId, cursor, auth);
+}
+function getDmMessages(conversationId, maxMessages, cursor, auth) {
+  return getDmConversationMessagesGenerator(
+    conversationId,
+    maxMessages,
+    cursor,
+    async (id, _max, cursor2) => {
+      const conversation = await fetchDmConversation(id, cursor2, auth);
+      let next = void 0;
+      if (cursor2?.minId && conversation.max_entry_id) {
+        next = { minId: conversation.max_entry_id };
+      } else if (conversation.min_entry_id) {
+        next = { maxId: conversation.min_entry_id };
+      }
+      return {
+        conversation,
+        next
+      };
+    }
+  );
+}
+function findDmConversationsByUserId(inbox, userId) {
+  const conversations = [];
+  for (const conversationId in inbox.conversations) {
+    const conversation = inbox.conversations[conversationId];
+    const hasUser = conversation.participants.some(
+      (participant) => participant.user_id === userId
+    );
+    if (hasUser) {
+      conversations.push(conversation);
+    }
+  }
+  return conversations;
+}
+
 const twUrl = "https://x.com";
 class Scraper {
   /**
@@ -2215,6 +2374,45 @@ class Scraper {
       return getTweetAnonymous(id, this.auth);
     }
   }
+  /**
+   * Retrieves the direct message inbox for the authenticated user.
+   *
+   * @return A promise that resolves to an object representing the direct message inbox.
+   */
+  getDmInbox() {
+    return getDmInbox(this.auth);
+  }
+  /**
+   * Retrieves the direct message conversation for the specified conversation ID.
+   *
+   * @param conversationId - The unique identifier of the DM conversation to retrieve.
+   * @param cursor - Use `maxId` to get messages before a message ID (older messages), or `minId` to get messages after a message ID (newer messages).
+   * @return A promise that resolves to the timeline of the DM conversation.
+   */
+  getDmConversation(conversationId, cursor) {
+    return getDmConversation(conversationId, cursor, this.auth);
+  }
+  /**
+   * Retrieves direct messages from a specific conversation.
+   *
+   * @param conversationId - The unique identifier of the conversation to fetch messages from.
+   * @param [maxMessages=20] - The maximum number of messages to retrieve per request.
+   * @param cursor - Use `maxId` to get messages before a message ID (older messages), or `minId` to get messages after a message ID (newer messages).
+   * @returns An {@link AsyncGenerator} of messages from the provided conversation.
+   */
+  getDmMessages(conversationId, maxMessages = 20, cursor) {
+    return getDmMessages(conversationId, maxMessages, cursor, this.auth);
+  }
+  /**
+   * Retrieves a list of direct message conversations for a specific user based on their user ID.
+   *
+   * @param inbox - The DM inbox containing all available conversations.
+   * @param userId - The unique identifier of the user whose DM conversations are to be retrieved.
+   * @return An array of DM conversations associated with the specified user ID.
+   */
+  findDmConversationsByUserId(inbox, userId) {
+    return findDmConversationsByUserId(inbox, userId);
+  }
   /**
    * Returns if the scraper has a guest token. The token may not be valid.
    * @returns `true` if the scraper has a guest token; otherwise `false`.