@the-convocation/twitter-scraper 0.13.0 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +71 -38
- package/dist/default/cjs/index.js +2126 -0
- package/dist/default/cjs/index.js.map +1 -0
- package/dist/default/esm/index.mjs +2104 -0
- package/dist/default/esm/index.mjs.map +1 -0
- package/dist/node/cjs/index.cjs +2156 -0
- package/dist/node/cjs/index.cjs.map +1 -0
- package/dist/node/esm/index.mjs +2134 -0
- package/dist/node/esm/index.mjs.map +1 -0
- package/dist/{scraper.d.ts → types/index.d.ts} +284 -8
- package/examples/cors-proxy/package.json +18 -0
- package/examples/node-integration/package.json +12 -0
- package/examples/react-integration/README.md +30 -0
- package/examples/react-integration/index.html +13 -0
- package/examples/react-integration/package.json +29 -0
- package/examples/react-integration/public/vite.svg +1 -0
- package/examples/react-integration/tsconfig.node.json +11 -0
- package/examples/react-integration/vite.config.ts +7 -0
- package/package.json +20 -3
- package/rollup.config.mjs +61 -0
- package/test-setup.js +2 -0
- package/dist/_module.d.ts +0 -6
- package/dist/_module.d.ts.map +0 -1
- package/dist/_module.js +0 -8
- package/dist/_module.js.map +0 -1
- package/dist/api-data.d.ts +0 -47
- package/dist/api-data.d.ts.map +0 -1
- package/dist/api-data.js +0 -84
- package/dist/api-data.js.map +0 -1
- package/dist/api.d.ts +0 -32
- package/dist/api.d.ts.map +0 -1
- package/dist/api.js +0 -138
- package/dist/api.js.map +0 -1
- package/dist/auth-user.d.ts +0 -23
- package/dist/auth-user.d.ts.map +0 -1
- package/dist/auth-user.js +0 -290
- package/dist/auth-user.js.map +0 -1
- package/dist/auth.d.ts +0 -82
- package/dist/auth.d.ts.map +0 -1
- package/dist/auth.js +0 -122
- package/dist/auth.js.map +0 -1
- package/dist/errors.d.ts +0 -28
- package/dist/errors.d.ts.map +0 -1
- package/dist/errors.js +0 -26
- package/dist/errors.js.map +0 -1
- package/dist/profile.d.ts +0 -80
- package/dist/profile.d.ts.map +0 -1
- package/dist/profile.js +0 -127
- package/dist/profile.js.map +0 -1
- package/dist/relationships.d.ts +0 -8
- package/dist/relationships.d.ts.map +0 -1
- package/dist/relationships.js +0 -93
- package/dist/relationships.js.map +0 -1
- package/dist/requests.d.ts +0 -9
- package/dist/requests.d.ts.map +0 -1
- package/dist/requests.js +0 -26
- package/dist/requests.js.map +0 -1
- package/dist/scraper.d.ts.map +0 -1
- package/dist/scraper.js +0 -357
- package/dist/scraper.js.map +0 -1
- package/dist/search.d.ts +0 -19
- package/dist/search.d.ts.map +0 -1
- package/dist/search.js +0 -99
- package/dist/search.js.map +0 -1
- package/dist/timeline-async.d.ts +0 -15
- package/dist/timeline-async.d.ts.map +0 -1
- package/dist/timeline-async.js +0 -53
- package/dist/timeline-async.js.map +0 -1
- package/dist/timeline-list.d.ts +0 -19
- package/dist/timeline-list.d.ts.map +0 -1
- package/dist/timeline-list.js +0 -46
- package/dist/timeline-list.js.map +0 -1
- package/dist/timeline-relationship.d.ts +0 -39
- package/dist/timeline-relationship.d.ts.map +0 -1
- package/dist/timeline-relationship.js +0 -46
- package/dist/timeline-relationship.js.map +0 -1
- package/dist/timeline-search.d.ts +0 -20
- package/dist/timeline-search.d.ts.map +0 -1
- package/dist/timeline-search.js +0 -93
- package/dist/timeline-search.js.map +0 -1
- package/dist/timeline-tweet-util.d.ts +0 -9
- package/dist/timeline-tweet-util.d.ts.map +0 -1
- package/dist/timeline-tweet-util.js +0 -108
- package/dist/timeline-tweet-util.js.map +0 -1
- package/dist/timeline-v1.d.ts +0 -233
- package/dist/timeline-v1.d.ts.map +0 -1
- package/dist/timeline-v1.js +0 -197
- package/dist/timeline-v1.js.map +0 -1
- package/dist/timeline-v2.d.ts +0 -94
- package/dist/timeline-v2.d.ts.map +0 -1
- package/dist/timeline-v2.js +0 -253
- package/dist/timeline-v2.js.map +0 -1
- package/dist/trends.d.ts +0 -3
- package/dist/trends.d.ts.map +0 -1
- package/dist/trends.js +0 -39
- package/dist/trends.js.map +0 -1
- package/dist/tweets.d.ts +0 -117
- package/dist/tweets.d.ts.map +0 -1
- package/dist/tweets.js +0 -202
- package/dist/tweets.js.map +0 -1
- package/dist/type-util.d.ts +0 -6
- package/dist/type-util.d.ts.map +0 -1
- package/dist/type-util.js +0 -14
- package/dist/type-util.js.map +0 -1
|
@@ -0,0 +1,2126 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
var toughCookie = require('tough-cookie');
|
|
4
|
+
var setCookie = require('set-cookie-parser');
|
|
5
|
+
var headersPolyfill = require('headers-polyfill');
|
|
6
|
+
var fetch = require('cross-fetch');
|
|
7
|
+
var typebox = require('@sinclair/typebox');
|
|
8
|
+
var value = require('@sinclair/typebox/value');
|
|
9
|
+
var OTPAuth = require('otpauth');
|
|
10
|
+
var stringify = require('json-stable-stringify');
|
|
11
|
+
|
|
12
|
+
function _interopNamespaceDefault(e) {
|
|
13
|
+
var n = Object.create(null);
|
|
14
|
+
if (e) {
|
|
15
|
+
Object.keys(e).forEach(function (k) {
|
|
16
|
+
if (k !== 'default') {
|
|
17
|
+
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
18
|
+
Object.defineProperty(n, k, d.get ? d : {
|
|
19
|
+
enumerable: true,
|
|
20
|
+
get: function () { return e[k]; }
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
n.default = e;
|
|
26
|
+
return Object.freeze(n);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
var OTPAuth__namespace = /*#__PURE__*/_interopNamespaceDefault(OTPAuth);
|
|
30
|
+
|
|
31
|
+
class ApiError extends Error {
|
|
32
|
+
constructor(response, data, message) {
|
|
33
|
+
super(message);
|
|
34
|
+
this.response = response;
|
|
35
|
+
this.data = data;
|
|
36
|
+
}
|
|
37
|
+
static async fromResponse(response) {
|
|
38
|
+
let data = void 0;
|
|
39
|
+
try {
|
|
40
|
+
data = await response.json();
|
|
41
|
+
} catch {
|
|
42
|
+
try {
|
|
43
|
+
data = await response.text();
|
|
44
|
+
} catch {
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return new ApiError(response, data, `Response status: ${response.status}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
const genericPlatform = new class {
|
|
52
|
+
randomizeCiphers() {
|
|
53
|
+
return Promise.resolve();
|
|
54
|
+
}
|
|
55
|
+
}();
|
|
56
|
+
|
|
57
|
+
class Platform {
|
|
58
|
+
async randomizeCiphers() {
|
|
59
|
+
const platform = await Platform.importPlatform();
|
|
60
|
+
await platform?.randomizeCiphers();
|
|
61
|
+
}
|
|
62
|
+
static async importPlatform() {
|
|
63
|
+
return genericPlatform;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
async function updateCookieJar(cookieJar, headers) {
|
|
68
|
+
const setCookieHeader = headers.get("set-cookie");
|
|
69
|
+
if (setCookieHeader) {
|
|
70
|
+
const cookies = setCookie.splitCookiesString(setCookieHeader);
|
|
71
|
+
for (const cookie of cookies.map((c) => toughCookie.Cookie.parse(c))) {
|
|
72
|
+
if (!cookie) continue;
|
|
73
|
+
await cookieJar.setCookie(
|
|
74
|
+
cookie,
|
|
75
|
+
`${cookie.secure ? "https" : "http"}://${cookie.domain}${cookie.path}`
|
|
76
|
+
);
|
|
77
|
+
}
|
|
78
|
+
} else if (typeof document !== "undefined") {
|
|
79
|
+
for (const cookie of document.cookie.split(";")) {
|
|
80
|
+
const hardCookie = toughCookie.Cookie.parse(cookie);
|
|
81
|
+
if (hardCookie) {
|
|
82
|
+
await cookieJar.setCookie(hardCookie, document.location.toString());
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
const bearerToken = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF";
|
|
89
|
+
async function requestApi(url, auth, method = "GET", platform = new Platform()) {
|
|
90
|
+
const headers = new headersPolyfill.Headers();
|
|
91
|
+
await auth.installTo(headers, url);
|
|
92
|
+
await platform.randomizeCiphers();
|
|
93
|
+
let res;
|
|
94
|
+
do {
|
|
95
|
+
try {
|
|
96
|
+
res = await auth.fetch(url, {
|
|
97
|
+
method,
|
|
98
|
+
headers,
|
|
99
|
+
credentials: "include"
|
|
100
|
+
});
|
|
101
|
+
} catch (err) {
|
|
102
|
+
if (!(err instanceof Error)) {
|
|
103
|
+
throw err;
|
|
104
|
+
}
|
|
105
|
+
return {
|
|
106
|
+
success: false,
|
|
107
|
+
err: new Error("Failed to perform request.")
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
await updateCookieJar(auth.cookieJar(), res.headers);
|
|
111
|
+
if (res.status === 429) {
|
|
112
|
+
const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
|
|
113
|
+
const xRateLimitReset = res.headers.get("x-rate-limit-reset");
|
|
114
|
+
if (xRateLimitRemaining == "0" && xRateLimitReset) {
|
|
115
|
+
const currentTime = (/* @__PURE__ */ new Date()).valueOf() / 1e3;
|
|
116
|
+
const timeDeltaMs = 1e3 * (parseInt(xRateLimitReset) - currentTime);
|
|
117
|
+
await new Promise((resolve) => setTimeout(resolve, timeDeltaMs));
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
} while (res.status === 429);
|
|
121
|
+
if (!res.ok) {
|
|
122
|
+
return {
|
|
123
|
+
success: false,
|
|
124
|
+
err: await ApiError.fromResponse(res)
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
const value = await res.json();
|
|
128
|
+
if (res.headers.get("x-rate-limit-incoming") == "0") {
|
|
129
|
+
auth.deleteToken();
|
|
130
|
+
return { success: true, value };
|
|
131
|
+
} else {
|
|
132
|
+
return { success: true, value };
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
function addApiFeatures(o) {
|
|
136
|
+
return {
|
|
137
|
+
...o,
|
|
138
|
+
rweb_lists_timeline_redesign_enabled: true,
|
|
139
|
+
responsive_web_graphql_exclude_directive_enabled: true,
|
|
140
|
+
verified_phone_label_enabled: false,
|
|
141
|
+
creator_subscriptions_tweet_preview_api_enabled: true,
|
|
142
|
+
responsive_web_graphql_timeline_navigation_enabled: true,
|
|
143
|
+
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
144
|
+
tweetypie_unmention_optimization_enabled: true,
|
|
145
|
+
responsive_web_edit_tweet_api_enabled: true,
|
|
146
|
+
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
|
147
|
+
view_counts_everywhere_api_enabled: true,
|
|
148
|
+
longform_notetweets_consumption_enabled: true,
|
|
149
|
+
tweet_awards_web_tipping_enabled: false,
|
|
150
|
+
freedom_of_speech_not_reach_fetch_enabled: true,
|
|
151
|
+
standardized_nudges_misinfo: true,
|
|
152
|
+
longform_notetweets_rich_text_read_enabled: true,
|
|
153
|
+
responsive_web_enhance_cards_enabled: false,
|
|
154
|
+
subscriptions_verification_info_enabled: true,
|
|
155
|
+
subscriptions_verification_info_reason_enabled: true,
|
|
156
|
+
subscriptions_verification_info_verified_since_enabled: true,
|
|
157
|
+
super_follow_badge_privacy_enabled: false,
|
|
158
|
+
super_follow_exclusive_tweet_notifications_enabled: false,
|
|
159
|
+
super_follow_tweet_api_enabled: false,
|
|
160
|
+
super_follow_user_api_enabled: false,
|
|
161
|
+
android_graphql_skip_api_media_color_palette: false,
|
|
162
|
+
creator_subscriptions_subscription_count_enabled: false,
|
|
163
|
+
blue_business_profile_image_shape_enabled: false,
|
|
164
|
+
unified_cards_ad_metadata_container_dynamic_card_content_query_enabled: false
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
function addApiParams(params, includeTweetReplies) {
|
|
168
|
+
params.set("include_profile_interstitial_type", "1");
|
|
169
|
+
params.set("include_blocking", "1");
|
|
170
|
+
params.set("include_blocked_by", "1");
|
|
171
|
+
params.set("include_followed_by", "1");
|
|
172
|
+
params.set("include_want_retweets", "1");
|
|
173
|
+
params.set("include_mute_edge", "1");
|
|
174
|
+
params.set("include_can_dm", "1");
|
|
175
|
+
params.set("include_can_media_tag", "1");
|
|
176
|
+
params.set("include_ext_has_nft_avatar", "1");
|
|
177
|
+
params.set("include_ext_is_blue_verified", "1");
|
|
178
|
+
params.set("include_ext_verified_type", "1");
|
|
179
|
+
params.set("skip_status", "1");
|
|
180
|
+
params.set("cards_platform", "Web-12");
|
|
181
|
+
params.set("include_cards", "1");
|
|
182
|
+
params.set("include_ext_alt_text", "true");
|
|
183
|
+
params.set("include_ext_limited_action_results", "false");
|
|
184
|
+
params.set("include_quote_count", "true");
|
|
185
|
+
params.set("include_reply_count", "1");
|
|
186
|
+
params.set("tweet_mode", "extended");
|
|
187
|
+
params.set("include_ext_collab_control", "true");
|
|
188
|
+
params.set("include_ext_views", "true");
|
|
189
|
+
params.set("include_entities", "true");
|
|
190
|
+
params.set("include_user_entities", "true");
|
|
191
|
+
params.set("include_ext_media_color", "true");
|
|
192
|
+
params.set("include_ext_media_availability", "true");
|
|
193
|
+
params.set("include_ext_sensitive_media_warning", "true");
|
|
194
|
+
params.set("include_ext_trusted_friends_metadata", "true");
|
|
195
|
+
params.set("send_error_codes", "true");
|
|
196
|
+
params.set("simple_quoted_tweet", "true");
|
|
197
|
+
params.set("include_tweet_replies", `${includeTweetReplies}`);
|
|
198
|
+
params.set(
|
|
199
|
+
"ext",
|
|
200
|
+
"mediaStats,highlightedLabel,hasNftAvatar,voiceInfo,birdwatchPivot,enrichments,superFollowMetadata,unmentionInfo,editControl,collab_control,vibe"
|
|
201
|
+
);
|
|
202
|
+
return params;
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
function withTransform(fetchFn, transform) {
|
|
206
|
+
return async (input, init) => {
|
|
207
|
+
const fetchArgs = await transform?.request?.(input, init) ?? [
|
|
208
|
+
input,
|
|
209
|
+
init
|
|
210
|
+
];
|
|
211
|
+
const res = await fetchFn(...fetchArgs);
|
|
212
|
+
return await transform?.response?.(res) ?? res;
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
class TwitterGuestAuth {
|
|
216
|
+
constructor(bearerToken, options) {
|
|
217
|
+
this.options = options;
|
|
218
|
+
this.fetch = withTransform(options?.fetch ?? fetch, options?.transform);
|
|
219
|
+
this.bearerToken = bearerToken;
|
|
220
|
+
this.jar = new toughCookie.CookieJar();
|
|
221
|
+
}
|
|
222
|
+
cookieJar() {
|
|
223
|
+
return this.jar;
|
|
224
|
+
}
|
|
225
|
+
isLoggedIn() {
|
|
226
|
+
return Promise.resolve(false);
|
|
227
|
+
}
|
|
228
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
229
|
+
login(_username, _password, _email) {
|
|
230
|
+
return this.updateGuestToken();
|
|
231
|
+
}
|
|
232
|
+
logout() {
|
|
233
|
+
this.deleteToken();
|
|
234
|
+
this.jar = new toughCookie.CookieJar();
|
|
235
|
+
return Promise.resolve();
|
|
236
|
+
}
|
|
237
|
+
deleteToken() {
|
|
238
|
+
delete this.guestToken;
|
|
239
|
+
delete this.guestCreatedAt;
|
|
240
|
+
}
|
|
241
|
+
hasToken() {
|
|
242
|
+
return this.guestToken != null;
|
|
243
|
+
}
|
|
244
|
+
authenticatedAt() {
|
|
245
|
+
if (this.guestCreatedAt == null) {
|
|
246
|
+
return null;
|
|
247
|
+
}
|
|
248
|
+
return new Date(this.guestCreatedAt);
|
|
249
|
+
}
|
|
250
|
+
async installTo(headers) {
|
|
251
|
+
if (this.shouldUpdate()) {
|
|
252
|
+
await this.updateGuestToken();
|
|
253
|
+
}
|
|
254
|
+
const token = this.guestToken;
|
|
255
|
+
if (token == null) {
|
|
256
|
+
throw new Error("Authentication token is null or undefined.");
|
|
257
|
+
}
|
|
258
|
+
headers.set("authorization", `Bearer ${this.bearerToken}`);
|
|
259
|
+
headers.set("x-guest-token", token);
|
|
260
|
+
const cookies = await this.getCookies();
|
|
261
|
+
const xCsrfToken = cookies.find((cookie) => cookie.key === "ct0");
|
|
262
|
+
if (xCsrfToken) {
|
|
263
|
+
headers.set("x-csrf-token", xCsrfToken.value);
|
|
264
|
+
}
|
|
265
|
+
headers.set("cookie", await this.getCookieString());
|
|
266
|
+
}
|
|
267
|
+
getCookies() {
|
|
268
|
+
return this.jar.getCookies(this.getCookieJarUrl());
|
|
269
|
+
}
|
|
270
|
+
getCookieString() {
|
|
271
|
+
return this.jar.getCookieString(this.getCookieJarUrl());
|
|
272
|
+
}
|
|
273
|
+
async removeCookie(key) {
|
|
274
|
+
const store = this.jar.store;
|
|
275
|
+
const cookies = await this.jar.getCookies(this.getCookieJarUrl());
|
|
276
|
+
for (const cookie of cookies) {
|
|
277
|
+
if (!cookie.domain || !cookie.path) continue;
|
|
278
|
+
store.removeCookie(cookie.domain, cookie.path, key);
|
|
279
|
+
if (typeof document !== "undefined") {
|
|
280
|
+
document.cookie = `${cookie.key}=; Max-Age=0; path=${cookie.path}; domain=${cookie.domain}`;
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
getCookieJarUrl() {
|
|
285
|
+
return typeof document !== "undefined" ? document.location.toString() : "https://twitter.com";
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Updates the authentication state with a new guest token from the Twitter API.
|
|
289
|
+
*/
|
|
290
|
+
async updateGuestToken() {
|
|
291
|
+
const guestActivateUrl = "https://api.twitter.com/1.1/guest/activate.json";
|
|
292
|
+
const headers = new headersPolyfill.Headers({
|
|
293
|
+
Authorization: `Bearer ${this.bearerToken}`,
|
|
294
|
+
Cookie: await this.getCookieString()
|
|
295
|
+
});
|
|
296
|
+
const res = await this.fetch(guestActivateUrl, {
|
|
297
|
+
method: "POST",
|
|
298
|
+
headers,
|
|
299
|
+
referrerPolicy: "no-referrer"
|
|
300
|
+
});
|
|
301
|
+
await updateCookieJar(this.jar, res.headers);
|
|
302
|
+
if (!res.ok) {
|
|
303
|
+
throw new Error(await res.text());
|
|
304
|
+
}
|
|
305
|
+
const o = await res.json();
|
|
306
|
+
if (o == null || o["guest_token"] == null) {
|
|
307
|
+
throw new Error("guest_token not found.");
|
|
308
|
+
}
|
|
309
|
+
const newGuestToken = o["guest_token"];
|
|
310
|
+
if (typeof newGuestToken !== "string") {
|
|
311
|
+
throw new Error("guest_token was not a string.");
|
|
312
|
+
}
|
|
313
|
+
this.guestToken = newGuestToken;
|
|
314
|
+
this.guestCreatedAt = /* @__PURE__ */ new Date();
|
|
315
|
+
}
|
|
316
|
+
/**
|
|
317
|
+
* Returns if the authentication token needs to be updated or not.
|
|
318
|
+
* @returns `true` if the token needs to be updated; `false` otherwise.
|
|
319
|
+
*/
|
|
320
|
+
shouldUpdate() {
|
|
321
|
+
return !this.hasToken() || this.guestCreatedAt != null && this.guestCreatedAt < new Date((/* @__PURE__ */ new Date()).valueOf() - 3 * 60 * 60 * 1e3);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
const TwitterUserAuthSubtask = typebox.Type.Object({
|
|
326
|
+
subtask_id: typebox.Type.String(),
|
|
327
|
+
enter_text: typebox.Type.Optional(typebox.Type.Object({}))
|
|
328
|
+
});
|
|
329
|
+
class TwitterUserAuth extends TwitterGuestAuth {
|
|
330
|
+
constructor(bearerToken, options) {
|
|
331
|
+
super(bearerToken, options);
|
|
332
|
+
}
|
|
333
|
+
async isLoggedIn() {
|
|
334
|
+
const res = await requestApi(
|
|
335
|
+
"https://api.twitter.com/1.1/account/verify_credentials.json",
|
|
336
|
+
this
|
|
337
|
+
);
|
|
338
|
+
if (!res.success) {
|
|
339
|
+
return false;
|
|
340
|
+
}
|
|
341
|
+
const { value: verify } = res;
|
|
342
|
+
return verify && !verify.errors?.length;
|
|
343
|
+
}
|
|
344
|
+
async login(username, password, email, twoFactorSecret) {
|
|
345
|
+
await this.updateGuestToken();
|
|
346
|
+
let next = await this.initLogin();
|
|
347
|
+
while ("subtask" in next && next.subtask) {
|
|
348
|
+
if (next.subtask.subtask_id === "LoginJsInstrumentationSubtask") {
|
|
349
|
+
next = await this.handleJsInstrumentationSubtask(next);
|
|
350
|
+
} else if (next.subtask.subtask_id === "LoginEnterUserIdentifierSSO") {
|
|
351
|
+
next = await this.handleEnterUserIdentifierSSO(next, username);
|
|
352
|
+
} else if (next.subtask.subtask_id === "LoginEnterAlternateIdentifierSubtask") {
|
|
353
|
+
next = await this.handleEnterAlternateIdentifierSubtask(
|
|
354
|
+
next,
|
|
355
|
+
email
|
|
356
|
+
);
|
|
357
|
+
} else if (next.subtask.subtask_id === "LoginEnterPassword") {
|
|
358
|
+
next = await this.handleEnterPassword(next, password);
|
|
359
|
+
} else if (next.subtask.subtask_id === "AccountDuplicationCheck") {
|
|
360
|
+
next = await this.handleAccountDuplicationCheck(next);
|
|
361
|
+
} else if (next.subtask.subtask_id === "LoginTwoFactorAuthChallenge") {
|
|
362
|
+
if (twoFactorSecret) {
|
|
363
|
+
next = await this.handleTwoFactorAuthChallenge(next, twoFactorSecret);
|
|
364
|
+
} else {
|
|
365
|
+
throw new Error(
|
|
366
|
+
"Requested two factor authentication code but no secret provided"
|
|
367
|
+
);
|
|
368
|
+
}
|
|
369
|
+
} else if (next.subtask.subtask_id === "LoginAcid") {
|
|
370
|
+
next = await this.handleAcid(next, email);
|
|
371
|
+
} else if (next.subtask.subtask_id === "LoginSuccessSubtask") {
|
|
372
|
+
next = await this.handleSuccessSubtask(next);
|
|
373
|
+
} else {
|
|
374
|
+
throw new Error(`Unknown subtask ${next.subtask.subtask_id}`);
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
if ("err" in next) {
|
|
378
|
+
throw next.err;
|
|
379
|
+
}
|
|
380
|
+
}
|
|
381
|
+
async logout() {
|
|
382
|
+
if (!this.isLoggedIn()) {
|
|
383
|
+
return;
|
|
384
|
+
}
|
|
385
|
+
await requestApi(
|
|
386
|
+
"https://api.twitter.com/1.1/account/logout.json",
|
|
387
|
+
this,
|
|
388
|
+
"POST"
|
|
389
|
+
);
|
|
390
|
+
this.deleteToken();
|
|
391
|
+
this.jar = new toughCookie.CookieJar();
|
|
392
|
+
}
|
|
393
|
+
async installCsrfToken(headers) {
|
|
394
|
+
const cookies = await this.getCookies();
|
|
395
|
+
const xCsrfToken = cookies.find((cookie) => cookie.key === "ct0");
|
|
396
|
+
if (xCsrfToken) {
|
|
397
|
+
headers.set("x-csrf-token", xCsrfToken.value);
|
|
398
|
+
}
|
|
399
|
+
}
|
|
400
|
+
async installTo(headers) {
|
|
401
|
+
headers.set("authorization", `Bearer ${this.bearerToken}`);
|
|
402
|
+
headers.set("cookie", await this.getCookieString());
|
|
403
|
+
await this.installCsrfToken(headers);
|
|
404
|
+
}
|
|
405
|
+
async initLogin() {
|
|
406
|
+
this.removeCookie("twitter_ads_id=");
|
|
407
|
+
this.removeCookie("ads_prefs=");
|
|
408
|
+
this.removeCookie("_twitter_sess=");
|
|
409
|
+
this.removeCookie("zipbox_forms_auth_token=");
|
|
410
|
+
this.removeCookie("lang=");
|
|
411
|
+
this.removeCookie("bouncer_reset_cookie=");
|
|
412
|
+
this.removeCookie("twid=");
|
|
413
|
+
this.removeCookie("twitter_ads_idb=");
|
|
414
|
+
this.removeCookie("email_uid=");
|
|
415
|
+
this.removeCookie("external_referer=");
|
|
416
|
+
this.removeCookie("ct0=");
|
|
417
|
+
this.removeCookie("aa_u=");
|
|
418
|
+
return await this.executeFlowTask({
|
|
419
|
+
flow_name: "login",
|
|
420
|
+
input_flow_data: {
|
|
421
|
+
flow_context: {
|
|
422
|
+
debug_overrides: {},
|
|
423
|
+
start_location: {
|
|
424
|
+
location: "splash_screen"
|
|
425
|
+
}
|
|
426
|
+
}
|
|
427
|
+
}
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
async handleJsInstrumentationSubtask(prev) {
|
|
431
|
+
return await this.executeFlowTask({
|
|
432
|
+
flow_token: prev.flowToken,
|
|
433
|
+
subtask_inputs: [
|
|
434
|
+
{
|
|
435
|
+
subtask_id: "LoginJsInstrumentationSubtask",
|
|
436
|
+
js_instrumentation: {
|
|
437
|
+
response: "{}",
|
|
438
|
+
link: "next_link"
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
]
|
|
442
|
+
});
|
|
443
|
+
}
|
|
444
|
+
async handleEnterAlternateIdentifierSubtask(prev, email) {
|
|
445
|
+
return await this.executeFlowTask({
|
|
446
|
+
flow_token: prev.flowToken,
|
|
447
|
+
subtask_inputs: [
|
|
448
|
+
{
|
|
449
|
+
subtask_id: "LoginEnterAlternateIdentifierSubtask",
|
|
450
|
+
enter_text: {
|
|
451
|
+
text: email,
|
|
452
|
+
link: "next_link"
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
]
|
|
456
|
+
});
|
|
457
|
+
}
|
|
458
|
+
async handleEnterUserIdentifierSSO(prev, username) {
|
|
459
|
+
return await this.executeFlowTask({
|
|
460
|
+
flow_token: prev.flowToken,
|
|
461
|
+
subtask_inputs: [
|
|
462
|
+
{
|
|
463
|
+
subtask_id: "LoginEnterUserIdentifierSSO",
|
|
464
|
+
settings_list: {
|
|
465
|
+
setting_responses: [
|
|
466
|
+
{
|
|
467
|
+
key: "user_identifier",
|
|
468
|
+
response_data: {
|
|
469
|
+
text_data: { result: username }
|
|
470
|
+
}
|
|
471
|
+
}
|
|
472
|
+
],
|
|
473
|
+
link: "next_link"
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
]
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
async handleEnterPassword(prev, password) {
|
|
480
|
+
return await this.executeFlowTask({
|
|
481
|
+
flow_token: prev.flowToken,
|
|
482
|
+
subtask_inputs: [
|
|
483
|
+
{
|
|
484
|
+
subtask_id: "LoginEnterPassword",
|
|
485
|
+
enter_password: {
|
|
486
|
+
password,
|
|
487
|
+
link: "next_link"
|
|
488
|
+
}
|
|
489
|
+
}
|
|
490
|
+
]
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
async handleAccountDuplicationCheck(prev) {
|
|
494
|
+
return await this.executeFlowTask({
|
|
495
|
+
flow_token: prev.flowToken,
|
|
496
|
+
subtask_inputs: [
|
|
497
|
+
{
|
|
498
|
+
subtask_id: "AccountDuplicationCheck",
|
|
499
|
+
check_logged_in_account: {
|
|
500
|
+
link: "AccountDuplicationCheck_false"
|
|
501
|
+
}
|
|
502
|
+
}
|
|
503
|
+
]
|
|
504
|
+
});
|
|
505
|
+
}
|
|
506
|
+
async handleTwoFactorAuthChallenge(prev, secret) {
|
|
507
|
+
const totp = new OTPAuth__namespace.TOTP({ secret });
|
|
508
|
+
let error;
|
|
509
|
+
for (let attempts = 1; attempts < 4; attempts += 1) {
|
|
510
|
+
try {
|
|
511
|
+
return await this.executeFlowTask({
|
|
512
|
+
flow_token: prev.flowToken,
|
|
513
|
+
subtask_inputs: [
|
|
514
|
+
{
|
|
515
|
+
subtask_id: "LoginTwoFactorAuthChallenge",
|
|
516
|
+
enter_text: {
|
|
517
|
+
link: "next_link",
|
|
518
|
+
text: totp.generate()
|
|
519
|
+
}
|
|
520
|
+
}
|
|
521
|
+
]
|
|
522
|
+
});
|
|
523
|
+
} catch (err) {
|
|
524
|
+
error = err;
|
|
525
|
+
await new Promise((resolve) => setTimeout(resolve, 2e3 * attempts));
|
|
526
|
+
}
|
|
527
|
+
}
|
|
528
|
+
throw error;
|
|
529
|
+
}
|
|
530
|
+
async handleAcid(prev, email) {
|
|
531
|
+
return await this.executeFlowTask({
|
|
532
|
+
flow_token: prev.flowToken,
|
|
533
|
+
subtask_inputs: [
|
|
534
|
+
{
|
|
535
|
+
subtask_id: "LoginAcid",
|
|
536
|
+
enter_text: {
|
|
537
|
+
text: email,
|
|
538
|
+
link: "next_link"
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
]
|
|
542
|
+
});
|
|
543
|
+
}
|
|
544
|
+
async handleSuccessSubtask(prev) {
|
|
545
|
+
return await this.executeFlowTask({
|
|
546
|
+
flow_token: prev.flowToken,
|
|
547
|
+
subtask_inputs: []
|
|
548
|
+
});
|
|
549
|
+
}
|
|
550
|
+
async executeFlowTask(data) {
|
|
551
|
+
const onboardingTaskUrl = "https://api.twitter.com/1.1/onboarding/task.json";
|
|
552
|
+
const token = this.guestToken;
|
|
553
|
+
if (token == null) {
|
|
554
|
+
throw new Error("Authentication token is null or undefined.");
|
|
555
|
+
}
|
|
556
|
+
const headers = new headersPolyfill.Headers({
|
|
557
|
+
authorization: `Bearer ${this.bearerToken}`,
|
|
558
|
+
cookie: await this.getCookieString(),
|
|
559
|
+
"content-type": "application/json",
|
|
560
|
+
"User-Agent": "Mozilla/5.0 (Linux; Android 11; Nokia G20) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.88 Mobile Safari/537.36",
|
|
561
|
+
"x-guest-token": token,
|
|
562
|
+
"x-twitter-auth-type": "OAuth2Client",
|
|
563
|
+
"x-twitter-active-user": "yes",
|
|
564
|
+
"x-twitter-client-language": "en"
|
|
565
|
+
});
|
|
566
|
+
await this.installCsrfToken(headers);
|
|
567
|
+
const res = await this.fetch(onboardingTaskUrl, {
|
|
568
|
+
credentials: "include",
|
|
569
|
+
method: "POST",
|
|
570
|
+
headers,
|
|
571
|
+
body: JSON.stringify(data)
|
|
572
|
+
});
|
|
573
|
+
await updateCookieJar(this.jar, res.headers);
|
|
574
|
+
if (!res.ok) {
|
|
575
|
+
return { status: "error", err: new Error(await res.text()) };
|
|
576
|
+
}
|
|
577
|
+
const flow = await res.json();
|
|
578
|
+
if (flow?.flow_token == null) {
|
|
579
|
+
return { status: "error", err: new Error("flow_token not found.") };
|
|
580
|
+
}
|
|
581
|
+
if (flow.errors?.length) {
|
|
582
|
+
return {
|
|
583
|
+
status: "error",
|
|
584
|
+
err: new Error(
|
|
585
|
+
`Authentication error (${flow.errors[0].code}): ${flow.errors[0].message}`
|
|
586
|
+
)
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
if (typeof flow.flow_token !== "string") {
|
|
590
|
+
return {
|
|
591
|
+
status: "error",
|
|
592
|
+
err: new Error("flow_token was not a string.")
|
|
593
|
+
};
|
|
594
|
+
}
|
|
595
|
+
const subtask = flow.subtasks?.length ? flow.subtasks[0] : void 0;
|
|
596
|
+
value.Check(TwitterUserAuthSubtask, subtask);
|
|
597
|
+
if (subtask && subtask.subtask_id === "DenyLoginSubtask") {
|
|
598
|
+
return {
|
|
599
|
+
status: "error",
|
|
600
|
+
err: new Error("Authentication error: DenyLoginSubtask")
|
|
601
|
+
};
|
|
602
|
+
}
|
|
603
|
+
return {
|
|
604
|
+
status: "success",
|
|
605
|
+
subtask,
|
|
606
|
+
flowToken: flow.flow_token
|
|
607
|
+
};
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
function getAvatarOriginalSizeUrl(avatarUrl) {
|
|
612
|
+
return avatarUrl ? avatarUrl.replace("_normal", "") : void 0;
|
|
613
|
+
}
|
|
614
|
+
function parseProfile(user, isBlueVerified) {
|
|
615
|
+
const profile = {
|
|
616
|
+
avatar: getAvatarOriginalSizeUrl(user.profile_image_url_https),
|
|
617
|
+
banner: user.profile_banner_url,
|
|
618
|
+
biography: user.description,
|
|
619
|
+
followersCount: user.followers_count,
|
|
620
|
+
followingCount: user.friends_count,
|
|
621
|
+
friendsCount: user.friends_count,
|
|
622
|
+
mediaCount: user.media_count,
|
|
623
|
+
isPrivate: user.protected ?? false,
|
|
624
|
+
isVerified: user.verified,
|
|
625
|
+
likesCount: user.favourites_count,
|
|
626
|
+
listedCount: user.listed_count,
|
|
627
|
+
location: user.location,
|
|
628
|
+
name: user.name,
|
|
629
|
+
pinnedTweetIds: user.pinned_tweet_ids_str,
|
|
630
|
+
tweetsCount: user.statuses_count,
|
|
631
|
+
url: `https://twitter.com/${user.screen_name}`,
|
|
632
|
+
userId: user.id_str,
|
|
633
|
+
username: user.screen_name,
|
|
634
|
+
isBlueVerified: isBlueVerified ?? false,
|
|
635
|
+
canDm: user.can_dm
|
|
636
|
+
};
|
|
637
|
+
if (user.created_at != null) {
|
|
638
|
+
profile.joined = new Date(Date.parse(user.created_at));
|
|
639
|
+
}
|
|
640
|
+
const urls = user.entities?.url?.urls;
|
|
641
|
+
if (urls?.length != null && urls?.length > 0) {
|
|
642
|
+
profile.website = urls[0].expanded_url;
|
|
643
|
+
}
|
|
644
|
+
return profile;
|
|
645
|
+
}
|
|
646
|
+
async function getProfile(username, auth) {
|
|
647
|
+
const params = new URLSearchParams();
|
|
648
|
+
params.set(
|
|
649
|
+
"variables",
|
|
650
|
+
stringify({
|
|
651
|
+
screen_name: username,
|
|
652
|
+
withSafetyModeUserFields: true
|
|
653
|
+
})
|
|
654
|
+
);
|
|
655
|
+
params.set(
|
|
656
|
+
"features",
|
|
657
|
+
stringify({
|
|
658
|
+
hidden_profile_likes_enabled: false,
|
|
659
|
+
hidden_profile_subscriptions_enabled: false,
|
|
660
|
+
// Auth-restricted
|
|
661
|
+
responsive_web_graphql_exclude_directive_enabled: true,
|
|
662
|
+
verified_phone_label_enabled: false,
|
|
663
|
+
subscriptions_verification_info_is_identity_verified_enabled: false,
|
|
664
|
+
subscriptions_verification_info_verified_since_enabled: true,
|
|
665
|
+
highlights_tweets_tab_ui_enabled: true,
|
|
666
|
+
creator_subscriptions_tweet_preview_api_enabled: true,
|
|
667
|
+
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
668
|
+
responsive_web_graphql_timeline_navigation_enabled: true
|
|
669
|
+
})
|
|
670
|
+
);
|
|
671
|
+
params.set("fieldToggles", stringify({ withAuxiliaryUserLabels: false }));
|
|
672
|
+
const res = await requestApi(
|
|
673
|
+
`https://twitter.com/i/api/graphql/G3KGOASz96M-Qu0nwmGXNg/UserByScreenName?${params.toString()}`,
|
|
674
|
+
auth
|
|
675
|
+
);
|
|
676
|
+
if (!res.success) {
|
|
677
|
+
return res;
|
|
678
|
+
}
|
|
679
|
+
const { value } = res;
|
|
680
|
+
const { errors } = value;
|
|
681
|
+
if (errors != null && errors.length > 0) {
|
|
682
|
+
return {
|
|
683
|
+
success: false,
|
|
684
|
+
err: new Error(errors[0].message)
|
|
685
|
+
};
|
|
686
|
+
}
|
|
687
|
+
if (!value.data || !value.data.user || !value.data.user.result) {
|
|
688
|
+
return {
|
|
689
|
+
success: false,
|
|
690
|
+
err: new Error("User not found.")
|
|
691
|
+
};
|
|
692
|
+
}
|
|
693
|
+
const { result: user } = value.data.user;
|
|
694
|
+
const { legacy } = user;
|
|
695
|
+
if (user.rest_id == null || user.rest_id.length === 0) {
|
|
696
|
+
return {
|
|
697
|
+
success: false,
|
|
698
|
+
err: new Error("rest_id not found.")
|
|
699
|
+
};
|
|
700
|
+
}
|
|
701
|
+
legacy.id_str = user.rest_id;
|
|
702
|
+
if (legacy.screen_name == null || legacy.screen_name.length === 0) {
|
|
703
|
+
return {
|
|
704
|
+
success: false,
|
|
705
|
+
err: new Error(`Either ${username} does not exist or is private.`)
|
|
706
|
+
};
|
|
707
|
+
}
|
|
708
|
+
return {
|
|
709
|
+
success: true,
|
|
710
|
+
value: parseProfile(user.legacy, user.is_blue_verified)
|
|
711
|
+
};
|
|
712
|
+
}
|
|
713
|
+
const idCache = /* @__PURE__ */ new Map();
|
|
714
|
+
async function getUserIdByScreenName(screenName, auth) {
|
|
715
|
+
const cached = idCache.get(screenName);
|
|
716
|
+
if (cached != null) {
|
|
717
|
+
return { success: true, value: cached };
|
|
718
|
+
}
|
|
719
|
+
const profileRes = await getProfile(screenName, auth);
|
|
720
|
+
if (!profileRes.success) {
|
|
721
|
+
return profileRes;
|
|
722
|
+
}
|
|
723
|
+
const profile = profileRes.value;
|
|
724
|
+
if (profile.userId != null) {
|
|
725
|
+
idCache.set(screenName, profile.userId);
|
|
726
|
+
return {
|
|
727
|
+
success: true,
|
|
728
|
+
value: profile.userId
|
|
729
|
+
};
|
|
730
|
+
}
|
|
731
|
+
return {
|
|
732
|
+
success: false,
|
|
733
|
+
err: new Error("User ID is undefined.")
|
|
734
|
+
};
|
|
735
|
+
}
|
|
736
|
+
|
|
737
|
+
async function* getUserTimeline(query, maxProfiles, fetchFunc) {
|
|
738
|
+
let nProfiles = 0;
|
|
739
|
+
let cursor = void 0;
|
|
740
|
+
let consecutiveEmptyBatches = 0;
|
|
741
|
+
while (nProfiles < maxProfiles) {
|
|
742
|
+
const batch = await fetchFunc(
|
|
743
|
+
query,
|
|
744
|
+
maxProfiles,
|
|
745
|
+
cursor
|
|
746
|
+
);
|
|
747
|
+
const { profiles, next } = batch;
|
|
748
|
+
cursor = next;
|
|
749
|
+
if (profiles.length === 0) {
|
|
750
|
+
consecutiveEmptyBatches++;
|
|
751
|
+
if (consecutiveEmptyBatches > 5) break;
|
|
752
|
+
} else consecutiveEmptyBatches = 0;
|
|
753
|
+
for (const profile of profiles) {
|
|
754
|
+
if (nProfiles < maxProfiles) yield profile;
|
|
755
|
+
else break;
|
|
756
|
+
nProfiles++;
|
|
757
|
+
}
|
|
758
|
+
if (!next) break;
|
|
759
|
+
}
|
|
760
|
+
}
|
|
761
|
+
async function* getTweetTimeline(query, maxTweets, fetchFunc) {
|
|
762
|
+
let nTweets = 0;
|
|
763
|
+
let cursor = void 0;
|
|
764
|
+
while (nTweets < maxTweets) {
|
|
765
|
+
const batch = await fetchFunc(
|
|
766
|
+
query,
|
|
767
|
+
maxTweets,
|
|
768
|
+
cursor
|
|
769
|
+
);
|
|
770
|
+
const { tweets, next } = batch;
|
|
771
|
+
if (tweets.length === 0) {
|
|
772
|
+
break;
|
|
773
|
+
}
|
|
774
|
+
for (const tweet of tweets) {
|
|
775
|
+
if (nTweets < maxTweets) {
|
|
776
|
+
cursor = next;
|
|
777
|
+
yield tweet;
|
|
778
|
+
} else {
|
|
779
|
+
break;
|
|
780
|
+
}
|
|
781
|
+
nTweets++;
|
|
782
|
+
}
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
function isFieldDefined(key) {
|
|
787
|
+
return function(value) {
|
|
788
|
+
return isDefined(value[key]);
|
|
789
|
+
};
|
|
790
|
+
}
|
|
791
|
+
function isDefined(value) {
|
|
792
|
+
return value != null;
|
|
793
|
+
}
|
|
794
|
+
|
|
795
|
+
const reHashtag = /\B(\#\S+\b)/g;
|
|
796
|
+
const reCashtag = /\B(\$\S+\b)/g;
|
|
797
|
+
const reTwitterUrl = /https:(\/\/t\.co\/([A-Za-z0-9]|[A-Za-z]){10})/g;
|
|
798
|
+
const reUsername = /\B(\@\S{1,15}\b)/g;
|
|
799
|
+
function parseMediaGroups(media) {
|
|
800
|
+
const photos = [];
|
|
801
|
+
const videos = [];
|
|
802
|
+
let sensitiveContent = void 0;
|
|
803
|
+
for (const m of media.filter(isFieldDefined("id_str")).filter(isFieldDefined("media_url_https"))) {
|
|
804
|
+
if (m.type === "photo") {
|
|
805
|
+
photos.push({
|
|
806
|
+
id: m.id_str,
|
|
807
|
+
url: m.media_url_https,
|
|
808
|
+
alt_text: m.ext_alt_text
|
|
809
|
+
});
|
|
810
|
+
} else if (m.type === "video") {
|
|
811
|
+
videos.push(parseVideo(m));
|
|
812
|
+
}
|
|
813
|
+
const sensitive = m.ext_sensitive_media_warning;
|
|
814
|
+
if (sensitive != null) {
|
|
815
|
+
sensitiveContent = sensitive.adult_content || sensitive.graphic_violence || sensitive.other;
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
return { sensitiveContent, photos, videos };
|
|
819
|
+
}
|
|
820
|
+
function parseVideo(m) {
|
|
821
|
+
const video = {
|
|
822
|
+
id: m.id_str,
|
|
823
|
+
preview: m.media_url_https
|
|
824
|
+
};
|
|
825
|
+
let maxBitrate = 0;
|
|
826
|
+
const variants = m.video_info?.variants ?? [];
|
|
827
|
+
for (const variant of variants) {
|
|
828
|
+
const bitrate = variant.bitrate;
|
|
829
|
+
if (bitrate != null && bitrate > maxBitrate && variant.url != null) {
|
|
830
|
+
let variantUrl = variant.url;
|
|
831
|
+
const stringStart = 0;
|
|
832
|
+
const tagSuffixIdx = variantUrl.indexOf("?tag=10");
|
|
833
|
+
if (tagSuffixIdx !== -1) {
|
|
834
|
+
variantUrl = variantUrl.substring(stringStart, tagSuffixIdx + 1);
|
|
835
|
+
}
|
|
836
|
+
video.url = variantUrl;
|
|
837
|
+
maxBitrate = bitrate;
|
|
838
|
+
}
|
|
839
|
+
}
|
|
840
|
+
return video;
|
|
841
|
+
}
|
|
842
|
+
function reconstructTweetHtml(tweet, photos, videos) {
|
|
843
|
+
const media = [];
|
|
844
|
+
let html = tweet.full_text ?? "";
|
|
845
|
+
html = html.replace(reHashtag, linkHashtagHtml);
|
|
846
|
+
html = html.replace(reCashtag, linkCashtagHtml);
|
|
847
|
+
html = html.replace(reUsername, linkUsernameHtml);
|
|
848
|
+
html = html.replace(reTwitterUrl, unwrapTcoUrlHtml(tweet, media));
|
|
849
|
+
for (const { url } of photos) {
|
|
850
|
+
if (media.indexOf(url) !== -1) {
|
|
851
|
+
continue;
|
|
852
|
+
}
|
|
853
|
+
html += `<br><img src="${url}"/>`;
|
|
854
|
+
}
|
|
855
|
+
for (const { preview: url } of videos) {
|
|
856
|
+
if (media.indexOf(url) !== -1) {
|
|
857
|
+
continue;
|
|
858
|
+
}
|
|
859
|
+
html += `<br><img src="${url}"/>`;
|
|
860
|
+
}
|
|
861
|
+
html = html.replace(/\n/g, "<br>");
|
|
862
|
+
return html;
|
|
863
|
+
}
|
|
864
|
+
function linkHashtagHtml(hashtag) {
|
|
865
|
+
return `<a href="https://twitter.com/hashtag/${hashtag.replace(
|
|
866
|
+
"#",
|
|
867
|
+
""
|
|
868
|
+
)}">${hashtag}</a>`;
|
|
869
|
+
}
|
|
870
|
+
function linkCashtagHtml(cashtag) {
|
|
871
|
+
return `<a href="https://twitter.com/search?q=%24${cashtag.replace(
|
|
872
|
+
"$",
|
|
873
|
+
""
|
|
874
|
+
)}">${cashtag}</a>`;
|
|
875
|
+
}
|
|
876
|
+
function linkUsernameHtml(username) {
|
|
877
|
+
return `<a href="https://twitter.com/${username.replace(
|
|
878
|
+
"@",
|
|
879
|
+
""
|
|
880
|
+
)}">${username}</a>`;
|
|
881
|
+
}
|
|
882
|
+
function unwrapTcoUrlHtml(tweet, foundedMedia) {
|
|
883
|
+
return function(tco) {
|
|
884
|
+
for (const entity of tweet.entities?.urls ?? []) {
|
|
885
|
+
if (tco === entity.url && entity.expanded_url != null) {
|
|
886
|
+
return `<a href="${entity.expanded_url}">${tco}</a>`;
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
for (const entity of tweet.extended_entities?.media ?? []) {
|
|
890
|
+
if (tco === entity.url && entity.media_url_https != null) {
|
|
891
|
+
foundedMedia.push(entity.media_url_https);
|
|
892
|
+
return `<br><a href="${tco}"><img src="${entity.media_url_https}"/></a>`;
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
return tco;
|
|
896
|
+
};
|
|
897
|
+
}
|
|
898
|
+
|
|
899
|
+
function getLegacyTweetId(tweet) {
|
|
900
|
+
if (tweet.id_str) {
|
|
901
|
+
return tweet.id_str;
|
|
902
|
+
}
|
|
903
|
+
return tweet.conversation_id_str;
|
|
904
|
+
}
|
|
905
|
+
function parseLegacyTweet(user, tweet, editControl) {
|
|
906
|
+
if (tweet == null) {
|
|
907
|
+
return {
|
|
908
|
+
success: false,
|
|
909
|
+
err: new Error("Tweet was not found in the timeline object.")
|
|
910
|
+
};
|
|
911
|
+
}
|
|
912
|
+
if (user == null) {
|
|
913
|
+
return {
|
|
914
|
+
success: false,
|
|
915
|
+
err: new Error("User was not found in the timeline object.")
|
|
916
|
+
};
|
|
917
|
+
}
|
|
918
|
+
const tweetId = getLegacyTweetId(tweet);
|
|
919
|
+
if (!tweetId) {
|
|
920
|
+
return {
|
|
921
|
+
success: false,
|
|
922
|
+
err: new Error("Tweet ID was not found in object.")
|
|
923
|
+
};
|
|
924
|
+
}
|
|
925
|
+
const hashtags = tweet.entities?.hashtags ?? [];
|
|
926
|
+
const mentions = tweet.entities?.user_mentions ?? [];
|
|
927
|
+
const media = tweet.extended_entities?.media ?? [];
|
|
928
|
+
const pinnedTweets = new Set(
|
|
929
|
+
user.pinned_tweet_ids_str ?? []
|
|
930
|
+
);
|
|
931
|
+
const urls = tweet.entities?.urls ?? [];
|
|
932
|
+
const { photos, videos, sensitiveContent } = parseMediaGroups(media);
|
|
933
|
+
const tweetVersions = editControl?.edit_tweet_ids ?? [tweetId];
|
|
934
|
+
const editIds = tweetVersions.filter((id) => id !== tweetId);
|
|
935
|
+
const tw = {
|
|
936
|
+
__raw_UNSTABLE: tweet,
|
|
937
|
+
bookmarkCount: tweet.bookmark_count,
|
|
938
|
+
conversationId: tweet.conversation_id_str,
|
|
939
|
+
id: tweetId,
|
|
940
|
+
hashtags: hashtags.filter(isFieldDefined("text")).map((hashtag) => hashtag.text),
|
|
941
|
+
likes: tweet.favorite_count,
|
|
942
|
+
mentions: mentions.filter(isFieldDefined("id_str")).map((mention) => ({
|
|
943
|
+
id: mention.id_str,
|
|
944
|
+
username: mention.screen_name,
|
|
945
|
+
name: mention.name
|
|
946
|
+
})),
|
|
947
|
+
name: user.name,
|
|
948
|
+
permanentUrl: `https://twitter.com/${user.screen_name}/status/${tweetId}`,
|
|
949
|
+
photos,
|
|
950
|
+
replies: tweet.reply_count,
|
|
951
|
+
retweets: tweet.retweet_count,
|
|
952
|
+
text: tweet.full_text,
|
|
953
|
+
thread: [],
|
|
954
|
+
urls: urls.filter(isFieldDefined("expanded_url")).map((url) => url.expanded_url),
|
|
955
|
+
userId: tweet.user_id_str,
|
|
956
|
+
username: user.screen_name,
|
|
957
|
+
videos,
|
|
958
|
+
isQuoted: false,
|
|
959
|
+
isReply: false,
|
|
960
|
+
isEdited: editIds.length > 1,
|
|
961
|
+
versions: tweetVersions,
|
|
962
|
+
isRetweet: false,
|
|
963
|
+
isPin: false,
|
|
964
|
+
sensitiveContent: false
|
|
965
|
+
};
|
|
966
|
+
if (tweet.created_at) {
|
|
967
|
+
tw.timeParsed = new Date(Date.parse(tweet.created_at));
|
|
968
|
+
tw.timestamp = Math.floor(tw.timeParsed.valueOf() / 1e3);
|
|
969
|
+
}
|
|
970
|
+
if (tweet.place?.id) {
|
|
971
|
+
tw.place = tweet.place;
|
|
972
|
+
}
|
|
973
|
+
const quotedStatusIdStr = tweet.quoted_status_id_str;
|
|
974
|
+
const inReplyToStatusIdStr = tweet.in_reply_to_status_id_str;
|
|
975
|
+
const retweetedStatusIdStr = tweet.retweeted_status_id_str;
|
|
976
|
+
const retweetedStatusResult = tweet.retweeted_status_result?.result;
|
|
977
|
+
if (quotedStatusIdStr) {
|
|
978
|
+
tw.isQuoted = true;
|
|
979
|
+
tw.quotedStatusId = quotedStatusIdStr;
|
|
980
|
+
}
|
|
981
|
+
if (inReplyToStatusIdStr) {
|
|
982
|
+
tw.isReply = true;
|
|
983
|
+
tw.inReplyToStatusId = inReplyToStatusIdStr;
|
|
984
|
+
}
|
|
985
|
+
if (retweetedStatusIdStr || retweetedStatusResult) {
|
|
986
|
+
tw.isRetweet = true;
|
|
987
|
+
tw.retweetedStatusId = retweetedStatusIdStr;
|
|
988
|
+
if (retweetedStatusResult) {
|
|
989
|
+
const parsedResult = parseLegacyTweet(
|
|
990
|
+
retweetedStatusResult?.core?.user_results?.result?.legacy,
|
|
991
|
+
retweetedStatusResult?.legacy
|
|
992
|
+
);
|
|
993
|
+
if (parsedResult.success) {
|
|
994
|
+
tw.retweetedStatus = parsedResult.tweet;
|
|
995
|
+
}
|
|
996
|
+
}
|
|
997
|
+
}
|
|
998
|
+
const views = parseInt(tweet.ext_views?.count ?? "");
|
|
999
|
+
if (!isNaN(views)) {
|
|
1000
|
+
tw.views = views;
|
|
1001
|
+
}
|
|
1002
|
+
if (pinnedTweets.has(tweetId)) {
|
|
1003
|
+
tw.isPin = true;
|
|
1004
|
+
}
|
|
1005
|
+
if (sensitiveContent) {
|
|
1006
|
+
tw.sensitiveContent = true;
|
|
1007
|
+
}
|
|
1008
|
+
tw.html = reconstructTweetHtml(tweet, tw.photos, tw.videos);
|
|
1009
|
+
return { success: true, tweet: tw };
|
|
1010
|
+
}
|
|
1011
|
+
function parseResult(result) {
|
|
1012
|
+
const noteTweetResultText = result?.note_tweet?.note_tweet_results?.result?.text;
|
|
1013
|
+
if (result?.legacy && noteTweetResultText) {
|
|
1014
|
+
result.legacy.full_text = noteTweetResultText;
|
|
1015
|
+
}
|
|
1016
|
+
const tweetResult = parseLegacyTweet(
|
|
1017
|
+
result?.core?.user_results?.result?.legacy,
|
|
1018
|
+
result?.legacy
|
|
1019
|
+
);
|
|
1020
|
+
if (!tweetResult.success) {
|
|
1021
|
+
return tweetResult;
|
|
1022
|
+
}
|
|
1023
|
+
if (!tweetResult.tweet.views && result?.views?.count) {
|
|
1024
|
+
const views = parseInt(result.views.count);
|
|
1025
|
+
if (!isNaN(views)) {
|
|
1026
|
+
tweetResult.tweet.views = views;
|
|
1027
|
+
}
|
|
1028
|
+
}
|
|
1029
|
+
const quotedResult = result?.quoted_status_result?.result;
|
|
1030
|
+
if (quotedResult) {
|
|
1031
|
+
if (quotedResult.legacy && quotedResult.rest_id) {
|
|
1032
|
+
quotedResult.legacy.id_str = quotedResult.rest_id;
|
|
1033
|
+
}
|
|
1034
|
+
const quotedTweetResult = parseResult(quotedResult);
|
|
1035
|
+
if (quotedTweetResult.success) {
|
|
1036
|
+
tweetResult.tweet.quotedStatus = quotedTweetResult.tweet;
|
|
1037
|
+
}
|
|
1038
|
+
}
|
|
1039
|
+
return tweetResult;
|
|
1040
|
+
}
|
|
1041
|
+
const expectedEntryTypes = ["tweet", "profile-conversation"];
|
|
1042
|
+
function parseTimelineTweetsV2(timeline) {
|
|
1043
|
+
let bottomCursor;
|
|
1044
|
+
let topCursor;
|
|
1045
|
+
const tweets = [];
|
|
1046
|
+
const instructions = timeline.data?.user?.result?.timeline_v2?.timeline?.instructions ?? [];
|
|
1047
|
+
for (const instruction of instructions) {
|
|
1048
|
+
const entries = instruction.entries ?? [];
|
|
1049
|
+
for (const entry of entries) {
|
|
1050
|
+
const entryContent = entry.content;
|
|
1051
|
+
if (!entryContent) continue;
|
|
1052
|
+
if (entryContent.cursorType === "Bottom") {
|
|
1053
|
+
bottomCursor = entryContent.value;
|
|
1054
|
+
continue;
|
|
1055
|
+
} else if (entryContent.cursorType === "Top") {
|
|
1056
|
+
topCursor = entryContent.value;
|
|
1057
|
+
continue;
|
|
1058
|
+
}
|
|
1059
|
+
const idStr = entry.entryId;
|
|
1060
|
+
if (!expectedEntryTypes.some((entryType) => idStr.startsWith(entryType))) {
|
|
1061
|
+
continue;
|
|
1062
|
+
}
|
|
1063
|
+
if (entryContent.itemContent) {
|
|
1064
|
+
parseAndPush(tweets, entryContent.itemContent, idStr);
|
|
1065
|
+
} else if (entryContent.items) {
|
|
1066
|
+
for (const item of entryContent.items) {
|
|
1067
|
+
if (item.item?.itemContent) {
|
|
1068
|
+
parseAndPush(tweets, item.item.itemContent, idStr);
|
|
1069
|
+
}
|
|
1070
|
+
}
|
|
1071
|
+
}
|
|
1072
|
+
}
|
|
1073
|
+
}
|
|
1074
|
+
return { tweets, next: bottomCursor, previous: topCursor };
|
|
1075
|
+
}
|
|
1076
|
+
function parseTimelineEntryItemContentRaw(content, entryId, isConversation = false) {
|
|
1077
|
+
let result = content.tweet_results?.result ?? content.tweetResult?.result;
|
|
1078
|
+
if (result?.__typename === "Tweet" || result?.__typename === "TweetWithVisibilityResults" && result?.tweet) {
|
|
1079
|
+
if (result?.__typename === "TweetWithVisibilityResults")
|
|
1080
|
+
result = result.tweet;
|
|
1081
|
+
if (result?.legacy) {
|
|
1082
|
+
result.legacy.id_str = result.rest_id ?? entryId.replace("conversation-", "").replace("tweet-", "");
|
|
1083
|
+
}
|
|
1084
|
+
const tweetResult = parseResult(result);
|
|
1085
|
+
if (tweetResult.success) {
|
|
1086
|
+
if (isConversation) {
|
|
1087
|
+
if (content?.tweetDisplayType === "SelfThread") {
|
|
1088
|
+
tweetResult.tweet.isSelfThread = true;
|
|
1089
|
+
}
|
|
1090
|
+
}
|
|
1091
|
+
return tweetResult.tweet;
|
|
1092
|
+
}
|
|
1093
|
+
}
|
|
1094
|
+
return null;
|
|
1095
|
+
}
|
|
1096
|
+
function parseAndPush(tweets, content, entryId, isConversation = false) {
|
|
1097
|
+
const tweet = parseTimelineEntryItemContentRaw(
|
|
1098
|
+
content,
|
|
1099
|
+
entryId,
|
|
1100
|
+
isConversation
|
|
1101
|
+
);
|
|
1102
|
+
if (tweet) {
|
|
1103
|
+
tweets.push(tweet);
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
function parseThreadedConversation(conversation) {
|
|
1107
|
+
const tweets = [];
|
|
1108
|
+
const instructions = conversation.data?.threaded_conversation_with_injections_v2?.instructions ?? [];
|
|
1109
|
+
for (const instruction of instructions) {
|
|
1110
|
+
const entries = instruction.entries ?? [];
|
|
1111
|
+
for (const entry of entries) {
|
|
1112
|
+
const entryContent = entry.content?.itemContent;
|
|
1113
|
+
if (entryContent) {
|
|
1114
|
+
parseAndPush(tweets, entryContent, entry.entryId, true);
|
|
1115
|
+
}
|
|
1116
|
+
for (const item of entry.content?.items ?? []) {
|
|
1117
|
+
const itemContent = item.item?.itemContent;
|
|
1118
|
+
if (itemContent) {
|
|
1119
|
+
parseAndPush(tweets, itemContent, entry.entryId, true);
|
|
1120
|
+
}
|
|
1121
|
+
}
|
|
1122
|
+
}
|
|
1123
|
+
}
|
|
1124
|
+
for (const tweet of tweets) {
|
|
1125
|
+
if (tweet.inReplyToStatusId) {
|
|
1126
|
+
for (const parentTweet of tweets) {
|
|
1127
|
+
if (parentTweet.id === tweet.inReplyToStatusId) {
|
|
1128
|
+
tweet.inReplyToStatus = parentTweet;
|
|
1129
|
+
break;
|
|
1130
|
+
}
|
|
1131
|
+
}
|
|
1132
|
+
}
|
|
1133
|
+
if (tweet.isSelfThread && tweet.conversationId === tweet.id) {
|
|
1134
|
+
for (const childTweet of tweets) {
|
|
1135
|
+
if (childTweet.isSelfThread && childTweet.id !== tweet.id) {
|
|
1136
|
+
tweet.thread.push(childTweet);
|
|
1137
|
+
}
|
|
1138
|
+
}
|
|
1139
|
+
if (tweet.thread.length === 0) {
|
|
1140
|
+
tweet.isSelfThread = false;
|
|
1141
|
+
}
|
|
1142
|
+
}
|
|
1143
|
+
}
|
|
1144
|
+
return tweets;
|
|
1145
|
+
}
|
|
1146
|
+
|
|
1147
|
+
function parseSearchTimelineTweets(timeline) {
|
|
1148
|
+
let bottomCursor;
|
|
1149
|
+
let topCursor;
|
|
1150
|
+
const tweets = [];
|
|
1151
|
+
const instructions = timeline.data?.search_by_raw_query?.search_timeline?.timeline?.instructions ?? [];
|
|
1152
|
+
for (const instruction of instructions) {
|
|
1153
|
+
if (instruction.type === "TimelineAddEntries" || instruction.type === "TimelineReplaceEntry") {
|
|
1154
|
+
if (instruction.entry?.content?.cursorType === "Bottom") {
|
|
1155
|
+
bottomCursor = instruction.entry.content.value;
|
|
1156
|
+
continue;
|
|
1157
|
+
} else if (instruction.entry?.content?.cursorType === "Top") {
|
|
1158
|
+
topCursor = instruction.entry.content.value;
|
|
1159
|
+
continue;
|
|
1160
|
+
}
|
|
1161
|
+
const entries = instruction.entries ?? [];
|
|
1162
|
+
for (const entry of entries) {
|
|
1163
|
+
const itemContent = entry.content?.itemContent;
|
|
1164
|
+
if (itemContent?.tweetDisplayType === "Tweet") {
|
|
1165
|
+
const tweetResultRaw = itemContent.tweet_results?.result;
|
|
1166
|
+
const tweetResult = parseLegacyTweet(
|
|
1167
|
+
tweetResultRaw?.core?.user_results?.result?.legacy,
|
|
1168
|
+
tweetResultRaw?.legacy,
|
|
1169
|
+
tweetResultRaw?.edit_control?.edit_control_initial
|
|
1170
|
+
);
|
|
1171
|
+
if (tweetResult.success) {
|
|
1172
|
+
if (!tweetResult.tweet.views && tweetResultRaw?.views?.count) {
|
|
1173
|
+
const views = parseInt(tweetResultRaw.views.count);
|
|
1174
|
+
if (!isNaN(views)) {
|
|
1175
|
+
tweetResult.tweet.views = views;
|
|
1176
|
+
}
|
|
1177
|
+
}
|
|
1178
|
+
tweets.push(tweetResult.tweet);
|
|
1179
|
+
}
|
|
1180
|
+
} else if (entry.content?.cursorType === "Bottom") {
|
|
1181
|
+
bottomCursor = entry.content.value;
|
|
1182
|
+
} else if (entry.content?.cursorType === "Top") {
|
|
1183
|
+
topCursor = entry.content.value;
|
|
1184
|
+
}
|
|
1185
|
+
}
|
|
1186
|
+
}
|
|
1187
|
+
}
|
|
1188
|
+
return { tweets, next: bottomCursor, previous: topCursor };
|
|
1189
|
+
}
|
|
1190
|
+
function parseSearchTimelineUsers(timeline) {
|
|
1191
|
+
let bottomCursor;
|
|
1192
|
+
let topCursor;
|
|
1193
|
+
const profiles = [];
|
|
1194
|
+
const instructions = timeline.data?.search_by_raw_query?.search_timeline?.timeline?.instructions ?? [];
|
|
1195
|
+
for (const instruction of instructions) {
|
|
1196
|
+
if (instruction.type === "TimelineAddEntries" || instruction.type === "TimelineReplaceEntry") {
|
|
1197
|
+
if (instruction.entry?.content?.cursorType === "Bottom") {
|
|
1198
|
+
bottomCursor = instruction.entry.content.value;
|
|
1199
|
+
continue;
|
|
1200
|
+
} else if (instruction.entry?.content?.cursorType === "Top") {
|
|
1201
|
+
topCursor = instruction.entry.content.value;
|
|
1202
|
+
continue;
|
|
1203
|
+
}
|
|
1204
|
+
const entries = instruction.entries ?? [];
|
|
1205
|
+
for (const entry of entries) {
|
|
1206
|
+
const itemContent = entry.content?.itemContent;
|
|
1207
|
+
if (itemContent?.userDisplayType === "User") {
|
|
1208
|
+
const userResultRaw = itemContent.user_results?.result;
|
|
1209
|
+
if (userResultRaw?.legacy) {
|
|
1210
|
+
const profile = parseProfile(
|
|
1211
|
+
userResultRaw.legacy,
|
|
1212
|
+
userResultRaw.is_blue_verified
|
|
1213
|
+
);
|
|
1214
|
+
if (!profile.userId) {
|
|
1215
|
+
profile.userId = userResultRaw.rest_id;
|
|
1216
|
+
}
|
|
1217
|
+
profiles.push(profile);
|
|
1218
|
+
}
|
|
1219
|
+
} else if (entry.content?.cursorType === "Bottom") {
|
|
1220
|
+
bottomCursor = entry.content.value;
|
|
1221
|
+
} else if (entry.content?.cursorType === "Top") {
|
|
1222
|
+
topCursor = entry.content.value;
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1225
|
+
}
|
|
1226
|
+
}
|
|
1227
|
+
return { profiles, next: bottomCursor, previous: topCursor };
|
|
1228
|
+
}
|
|
1229
|
+
|
|
1230
|
+
var SearchMode = /* @__PURE__ */ ((SearchMode2) => {
|
|
1231
|
+
SearchMode2[SearchMode2["Top"] = 0] = "Top";
|
|
1232
|
+
SearchMode2[SearchMode2["Latest"] = 1] = "Latest";
|
|
1233
|
+
SearchMode2[SearchMode2["Photos"] = 2] = "Photos";
|
|
1234
|
+
SearchMode2[SearchMode2["Videos"] = 3] = "Videos";
|
|
1235
|
+
SearchMode2[SearchMode2["Users"] = 4] = "Users";
|
|
1236
|
+
return SearchMode2;
|
|
1237
|
+
})(SearchMode || {});
|
|
1238
|
+
function searchTweets(query, maxTweets, searchMode, auth) {
|
|
1239
|
+
return getTweetTimeline(query, maxTweets, (q, mt, c) => {
|
|
1240
|
+
return fetchSearchTweets(q, mt, searchMode, auth, c);
|
|
1241
|
+
});
|
|
1242
|
+
}
|
|
1243
|
+
function searchProfiles(query, maxProfiles, auth) {
|
|
1244
|
+
return getUserTimeline(query, maxProfiles, (q, mt, c) => {
|
|
1245
|
+
return fetchSearchProfiles(q, mt, auth, c);
|
|
1246
|
+
});
|
|
1247
|
+
}
|
|
1248
|
+
async function fetchSearchTweets(query, maxTweets, searchMode, auth, cursor) {
|
|
1249
|
+
const timeline = await getSearchTimeline(
|
|
1250
|
+
query,
|
|
1251
|
+
maxTweets,
|
|
1252
|
+
searchMode,
|
|
1253
|
+
auth,
|
|
1254
|
+
cursor
|
|
1255
|
+
);
|
|
1256
|
+
return parseSearchTimelineTweets(timeline);
|
|
1257
|
+
}
|
|
1258
|
+
async function fetchSearchProfiles(query, maxProfiles, auth, cursor) {
|
|
1259
|
+
const timeline = await getSearchTimeline(
|
|
1260
|
+
query,
|
|
1261
|
+
maxProfiles,
|
|
1262
|
+
4 /* Users */,
|
|
1263
|
+
auth,
|
|
1264
|
+
cursor
|
|
1265
|
+
);
|
|
1266
|
+
return parseSearchTimelineUsers(timeline);
|
|
1267
|
+
}
|
|
1268
|
+
async function getSearchTimeline(query, maxItems, searchMode, auth, cursor) {
|
|
1269
|
+
if (!auth.isLoggedIn()) {
|
|
1270
|
+
throw new Error("Scraper is not logged-in for search.");
|
|
1271
|
+
}
|
|
1272
|
+
if (maxItems > 50) {
|
|
1273
|
+
maxItems = 50;
|
|
1274
|
+
}
|
|
1275
|
+
const variables = {
|
|
1276
|
+
rawQuery: query,
|
|
1277
|
+
count: maxItems,
|
|
1278
|
+
querySource: "typed_query",
|
|
1279
|
+
product: "Top"
|
|
1280
|
+
};
|
|
1281
|
+
const features = addApiFeatures({
|
|
1282
|
+
longform_notetweets_inline_media_enabled: true,
|
|
1283
|
+
responsive_web_enhance_cards_enabled: false,
|
|
1284
|
+
responsive_web_media_download_video_enabled: false,
|
|
1285
|
+
responsive_web_twitter_article_tweet_consumption_enabled: false,
|
|
1286
|
+
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
1287
|
+
interactive_text_enabled: false,
|
|
1288
|
+
responsive_web_text_conversations_enabled: false,
|
|
1289
|
+
vibe_api_enabled: false
|
|
1290
|
+
});
|
|
1291
|
+
const fieldToggles = {
|
|
1292
|
+
withArticleRichContentState: false
|
|
1293
|
+
};
|
|
1294
|
+
if (cursor != null && cursor != "") {
|
|
1295
|
+
variables["cursor"] = cursor;
|
|
1296
|
+
}
|
|
1297
|
+
switch (searchMode) {
|
|
1298
|
+
case 1 /* Latest */:
|
|
1299
|
+
variables.product = "Latest";
|
|
1300
|
+
break;
|
|
1301
|
+
case 2 /* Photos */:
|
|
1302
|
+
variables.product = "Photos";
|
|
1303
|
+
break;
|
|
1304
|
+
case 3 /* Videos */:
|
|
1305
|
+
variables.product = "Videos";
|
|
1306
|
+
break;
|
|
1307
|
+
case 4 /* Users */:
|
|
1308
|
+
variables.product = "People";
|
|
1309
|
+
break;
|
|
1310
|
+
}
|
|
1311
|
+
const params = new URLSearchParams();
|
|
1312
|
+
params.set("features", stringify(features));
|
|
1313
|
+
params.set("fieldToggles", stringify(fieldToggles));
|
|
1314
|
+
params.set("variables", stringify(variables));
|
|
1315
|
+
const res = await requestApi(
|
|
1316
|
+
`https://api.twitter.com/graphql/gkjsKepM6gl_HmFWoWKfgg/SearchTimeline?${params.toString()}`,
|
|
1317
|
+
auth
|
|
1318
|
+
);
|
|
1319
|
+
if (!res.success) {
|
|
1320
|
+
throw res.err;
|
|
1321
|
+
}
|
|
1322
|
+
return res.value;
|
|
1323
|
+
}
|
|
1324
|
+
|
|
1325
|
+
function parseRelationshipTimeline(timeline) {
|
|
1326
|
+
let bottomCursor;
|
|
1327
|
+
let topCursor;
|
|
1328
|
+
const profiles = [];
|
|
1329
|
+
const instructions = timeline.data?.user?.result?.timeline?.timeline?.instructions ?? [];
|
|
1330
|
+
for (const instruction of instructions) {
|
|
1331
|
+
if (instruction.type === "TimelineAddEntries" || instruction.type === "TimelineReplaceEntry") {
|
|
1332
|
+
if (instruction.entry?.content?.cursorType === "Bottom") {
|
|
1333
|
+
bottomCursor = instruction.entry.content.value;
|
|
1334
|
+
continue;
|
|
1335
|
+
}
|
|
1336
|
+
if (instruction.entry?.content?.cursorType === "Top") {
|
|
1337
|
+
topCursor = instruction.entry.content.value;
|
|
1338
|
+
continue;
|
|
1339
|
+
}
|
|
1340
|
+
const entries = instruction.entries ?? [];
|
|
1341
|
+
for (const entry of entries) {
|
|
1342
|
+
const itemContent = entry.content?.itemContent;
|
|
1343
|
+
if (itemContent?.userDisplayType === "User") {
|
|
1344
|
+
const userResultRaw = itemContent.user_results?.result;
|
|
1345
|
+
if (userResultRaw?.legacy) {
|
|
1346
|
+
const profile = parseProfile(
|
|
1347
|
+
userResultRaw.legacy,
|
|
1348
|
+
userResultRaw.is_blue_verified
|
|
1349
|
+
);
|
|
1350
|
+
if (!profile.userId) {
|
|
1351
|
+
profile.userId = userResultRaw.rest_id;
|
|
1352
|
+
}
|
|
1353
|
+
profiles.push(profile);
|
|
1354
|
+
}
|
|
1355
|
+
} else if (entry.content?.cursorType === "Bottom") {
|
|
1356
|
+
bottomCursor = entry.content.value;
|
|
1357
|
+
} else if (entry.content?.cursorType === "Top") {
|
|
1358
|
+
topCursor = entry.content.value;
|
|
1359
|
+
}
|
|
1360
|
+
}
|
|
1361
|
+
}
|
|
1362
|
+
}
|
|
1363
|
+
return { profiles, next: bottomCursor, previous: topCursor };
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
function getFollowing(userId, maxProfiles, auth) {
|
|
1367
|
+
return getUserTimeline(userId, maxProfiles, (q, mt, c) => {
|
|
1368
|
+
return fetchProfileFollowing(q, mt, auth, c);
|
|
1369
|
+
});
|
|
1370
|
+
}
|
|
1371
|
+
function getFollowers(userId, maxProfiles, auth) {
|
|
1372
|
+
return getUserTimeline(userId, maxProfiles, (q, mt, c) => {
|
|
1373
|
+
return fetchProfileFollowers(q, mt, auth, c);
|
|
1374
|
+
});
|
|
1375
|
+
}
|
|
1376
|
+
async function fetchProfileFollowing(userId, maxProfiles, auth, cursor) {
|
|
1377
|
+
const timeline = await getFollowingTimeline(
|
|
1378
|
+
userId,
|
|
1379
|
+
maxProfiles,
|
|
1380
|
+
auth,
|
|
1381
|
+
cursor
|
|
1382
|
+
);
|
|
1383
|
+
return parseRelationshipTimeline(timeline);
|
|
1384
|
+
}
|
|
1385
|
+
async function fetchProfileFollowers(userId, maxProfiles, auth, cursor) {
|
|
1386
|
+
const timeline = await getFollowersTimeline(
|
|
1387
|
+
userId,
|
|
1388
|
+
maxProfiles,
|
|
1389
|
+
auth,
|
|
1390
|
+
cursor
|
|
1391
|
+
);
|
|
1392
|
+
return parseRelationshipTimeline(timeline);
|
|
1393
|
+
}
|
|
1394
|
+
async function getFollowingTimeline(userId, maxItems, auth, cursor) {
|
|
1395
|
+
if (!auth.isLoggedIn()) {
|
|
1396
|
+
throw new Error("Scraper is not logged-in for profile following.");
|
|
1397
|
+
}
|
|
1398
|
+
if (maxItems > 50) {
|
|
1399
|
+
maxItems = 50;
|
|
1400
|
+
}
|
|
1401
|
+
const variables = {
|
|
1402
|
+
userId,
|
|
1403
|
+
count: maxItems,
|
|
1404
|
+
includePromotedContent: false
|
|
1405
|
+
};
|
|
1406
|
+
const features = addApiFeatures({
|
|
1407
|
+
responsive_web_twitter_article_tweet_consumption_enabled: false,
|
|
1408
|
+
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
1409
|
+
longform_notetweets_inline_media_enabled: true,
|
|
1410
|
+
responsive_web_media_download_video_enabled: false
|
|
1411
|
+
});
|
|
1412
|
+
if (cursor != null && cursor != "") {
|
|
1413
|
+
variables["cursor"] = cursor;
|
|
1414
|
+
}
|
|
1415
|
+
const params = new URLSearchParams();
|
|
1416
|
+
params.set("features", stringify(features));
|
|
1417
|
+
params.set("variables", stringify(variables));
|
|
1418
|
+
const res = await requestApi(
|
|
1419
|
+
`https://twitter.com/i/api/graphql/iSicc7LrzWGBgDPL0tM_TQ/Following?${params.toString()}`,
|
|
1420
|
+
auth
|
|
1421
|
+
);
|
|
1422
|
+
if (!res.success) {
|
|
1423
|
+
throw res.err;
|
|
1424
|
+
}
|
|
1425
|
+
return res.value;
|
|
1426
|
+
}
|
|
1427
|
+
async function getFollowersTimeline(userId, maxItems, auth, cursor) {
|
|
1428
|
+
if (!auth.isLoggedIn()) {
|
|
1429
|
+
throw new Error("Scraper is not logged-in for profile followers.");
|
|
1430
|
+
}
|
|
1431
|
+
if (maxItems > 50) {
|
|
1432
|
+
maxItems = 50;
|
|
1433
|
+
}
|
|
1434
|
+
const variables = {
|
|
1435
|
+
userId,
|
|
1436
|
+
count: maxItems,
|
|
1437
|
+
includePromotedContent: false
|
|
1438
|
+
};
|
|
1439
|
+
const features = addApiFeatures({
|
|
1440
|
+
responsive_web_twitter_article_tweet_consumption_enabled: false,
|
|
1441
|
+
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
|
1442
|
+
longform_notetweets_inline_media_enabled: true,
|
|
1443
|
+
responsive_web_media_download_video_enabled: false
|
|
1444
|
+
});
|
|
1445
|
+
if (cursor != null && cursor != "") {
|
|
1446
|
+
variables["cursor"] = cursor;
|
|
1447
|
+
}
|
|
1448
|
+
const params = new URLSearchParams();
|
|
1449
|
+
params.set("features", stringify(features));
|
|
1450
|
+
params.set("variables", stringify(variables));
|
|
1451
|
+
const res = await requestApi(
|
|
1452
|
+
`https://twitter.com/i/api/graphql/rRXFSG5vR6drKr5M37YOTw/Followers?${params.toString()}`,
|
|
1453
|
+
auth
|
|
1454
|
+
);
|
|
1455
|
+
if (!res.success) {
|
|
1456
|
+
throw res.err;
|
|
1457
|
+
}
|
|
1458
|
+
return res.value;
|
|
1459
|
+
}
|
|
1460
|
+
|
|
1461
|
+
async function getTrends(auth) {
|
|
1462
|
+
const params = new URLSearchParams();
|
|
1463
|
+
addApiParams(params, false);
|
|
1464
|
+
params.set("count", "20");
|
|
1465
|
+
params.set("candidate_source", "trends");
|
|
1466
|
+
params.set("include_page_configuration", "false");
|
|
1467
|
+
params.set("entity_tokens", "false");
|
|
1468
|
+
const res = await requestApi(
|
|
1469
|
+
`https://api.twitter.com/2/guide.json?${params.toString()}`,
|
|
1470
|
+
auth
|
|
1471
|
+
);
|
|
1472
|
+
if (!res.success) {
|
|
1473
|
+
throw res.err;
|
|
1474
|
+
}
|
|
1475
|
+
const instructions = res.value.timeline?.instructions ?? [];
|
|
1476
|
+
if (instructions.length < 2) {
|
|
1477
|
+
throw new Error("No trend entries found.");
|
|
1478
|
+
}
|
|
1479
|
+
const entries = instructions[1].addEntries?.entries ?? [];
|
|
1480
|
+
if (entries.length < 2) {
|
|
1481
|
+
throw new Error("No trend entries found.");
|
|
1482
|
+
}
|
|
1483
|
+
const items = entries[1].content?.timelineModule?.items ?? [];
|
|
1484
|
+
const trends = [];
|
|
1485
|
+
for (const item of items) {
|
|
1486
|
+
const trend = item.item?.clientEventInfo?.details?.guideDetails?.transparentGuideDetails?.trendMetadata?.trendName;
|
|
1487
|
+
if (trend != null) {
|
|
1488
|
+
trends.push(trend);
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
return trends;
|
|
1492
|
+
}
|
|
1493
|
+
|
|
1494
|
+
const endpoints = {
|
|
1495
|
+
// TODO: Migrate other endpoint URLs here
|
|
1496
|
+
UserTweets: "https://twitter.com/i/api/graphql/V7H0Ap3_Hh2FyS75OCDO3Q/UserTweets?variables=%7B%22userId%22%3A%224020276615%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
1497
|
+
UserTweetsAndReplies: "https://twitter.com/i/api/graphql/E4wA5vo2sjVyvpliUffSCw/UserTweetsAndReplies?variables=%7B%22userId%22%3A%224020276615%22%2C%22count%22%3A40%2C%22cursor%22%3A%22DAABCgABGPWl-F-ATiIKAAIY9YfiF1rRAggAAwAAAAEAAA%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
1498
|
+
UserLikedTweets: "https://twitter.com/i/api/graphql/eSSNbhECHHWWALkkQq-YTA/Likes?variables=%7B%22userId%22%3A%222244196397%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withClientEventToken%22%3Afalse%2C%22withBirdwatchNotes%22%3Afalse%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D",
|
|
1499
|
+
TweetDetail: "https://twitter.com/i/api/graphql/xOhkmRac04YFZmOzU9PJHg/TweetDetail?variables=%7B%22focalTweetId%22%3A%221237110546383724547%22%2C%22with_rux_injections%22%3Afalse%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Afalse%7D",
|
|
1500
|
+
TweetResultByRestId: "https://twitter.com/i/api/graphql/DJS3BdhUhcaEpZ7B7irJDg/TweetResultByRestId?variables=%7B%22tweetId%22%3A%221237110546383724547%22%2C%22withCommunity%22%3Afalse%2C%22includePromotedContent%22%3Afalse%2C%22withVoice%22%3Afalse%7D&features=%7B%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D",
|
|
1501
|
+
ListTweets: "https://twitter.com/i/api/graphql/whF0_KH1fCkdLLoyNPMoEw/ListLatestTweetsTimeline?variables=%7B%22listId%22%3A%221736495155002106192%22%2C%22count%22%3A20%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D"
|
|
1502
|
+
};
|
|
1503
|
+
class ApiRequest {
|
|
1504
|
+
constructor(info) {
|
|
1505
|
+
this.url = info.url;
|
|
1506
|
+
this.variables = info.variables;
|
|
1507
|
+
this.features = info.features;
|
|
1508
|
+
this.fieldToggles = info.fieldToggles;
|
|
1509
|
+
}
|
|
1510
|
+
toRequestUrl() {
|
|
1511
|
+
const params = new URLSearchParams();
|
|
1512
|
+
if (this.variables) {
|
|
1513
|
+
params.set("variables", stringify(this.variables));
|
|
1514
|
+
}
|
|
1515
|
+
if (this.features) {
|
|
1516
|
+
params.set("features", stringify(this.features));
|
|
1517
|
+
}
|
|
1518
|
+
if (this.fieldToggles) {
|
|
1519
|
+
params.set("fieldToggles", stringify(this.fieldToggles));
|
|
1520
|
+
}
|
|
1521
|
+
return `${this.url}?${params.toString()}`;
|
|
1522
|
+
}
|
|
1523
|
+
}
|
|
1524
|
+
function parseEndpointExample(example) {
|
|
1525
|
+
const { protocol, host, pathname, searchParams: query } = new URL(example);
|
|
1526
|
+
const base = `${protocol}//${host}${pathname}`;
|
|
1527
|
+
const variables = query.get("variables");
|
|
1528
|
+
const features = query.get("features");
|
|
1529
|
+
const fieldToggles = query.get("fieldToggles");
|
|
1530
|
+
return new ApiRequest({
|
|
1531
|
+
url: base,
|
|
1532
|
+
variables: variables ? JSON.parse(variables) : void 0,
|
|
1533
|
+
features: features ? JSON.parse(features) : void 0,
|
|
1534
|
+
fieldToggles: fieldToggles ? JSON.parse(fieldToggles) : void 0
|
|
1535
|
+
});
|
|
1536
|
+
}
|
|
1537
|
+
function createApiRequestFactory(endpoints2) {
|
|
1538
|
+
return Object.entries(endpoints2).map(([endpointName, endpointExample]) => {
|
|
1539
|
+
return {
|
|
1540
|
+
[`create${endpointName}Request`]: () => {
|
|
1541
|
+
return parseEndpointExample(endpointExample);
|
|
1542
|
+
}
|
|
1543
|
+
};
|
|
1544
|
+
}).reduce((agg, next) => {
|
|
1545
|
+
return Object.assign(agg, next);
|
|
1546
|
+
});
|
|
1547
|
+
}
|
|
1548
|
+
const apiRequestFactory = createApiRequestFactory(endpoints);
|
|
1549
|
+
|
|
1550
|
+
function parseListTimelineTweets(timeline) {
|
|
1551
|
+
let bottomCursor;
|
|
1552
|
+
let topCursor;
|
|
1553
|
+
const tweets = [];
|
|
1554
|
+
const instructions = timeline.data?.list?.tweets_timeline?.timeline?.instructions ?? [];
|
|
1555
|
+
for (const instruction of instructions) {
|
|
1556
|
+
const entries = instruction.entries ?? [];
|
|
1557
|
+
for (const entry of entries) {
|
|
1558
|
+
const entryContent = entry.content;
|
|
1559
|
+
if (!entryContent) continue;
|
|
1560
|
+
if (entryContent.cursorType === "Bottom") {
|
|
1561
|
+
bottomCursor = entryContent.value;
|
|
1562
|
+
continue;
|
|
1563
|
+
} else if (entryContent.cursorType === "Top") {
|
|
1564
|
+
topCursor = entryContent.value;
|
|
1565
|
+
continue;
|
|
1566
|
+
}
|
|
1567
|
+
const idStr = entry.entryId;
|
|
1568
|
+
if (!idStr.startsWith("tweet") && !idStr.startsWith("list-conversation")) {
|
|
1569
|
+
continue;
|
|
1570
|
+
}
|
|
1571
|
+
if (entryContent.itemContent) {
|
|
1572
|
+
parseAndPush(tweets, entryContent.itemContent, idStr);
|
|
1573
|
+
} else if (entryContent.items) {
|
|
1574
|
+
for (const contentItem of entryContent.items) {
|
|
1575
|
+
if (contentItem.item && contentItem.item.itemContent && contentItem.entryId) {
|
|
1576
|
+
parseAndPush(
|
|
1577
|
+
tweets,
|
|
1578
|
+
contentItem.item.itemContent,
|
|
1579
|
+
contentItem.entryId.split("tweet-")[1]
|
|
1580
|
+
);
|
|
1581
|
+
}
|
|
1582
|
+
}
|
|
1583
|
+
}
|
|
1584
|
+
}
|
|
1585
|
+
}
|
|
1586
|
+
return { tweets, next: bottomCursor, previous: topCursor };
|
|
1587
|
+
}
|
|
1588
|
+
|
|
1589
|
+
addApiFeatures({
|
|
1590
|
+
interactive_text_enabled: true,
|
|
1591
|
+
longform_notetweets_inline_media_enabled: false,
|
|
1592
|
+
responsive_web_text_conversations_enabled: false,
|
|
1593
|
+
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: false,
|
|
1594
|
+
vibe_api_enabled: false
|
|
1595
|
+
});
|
|
1596
|
+
async function fetchTweets(userId, maxTweets, cursor, auth) {
|
|
1597
|
+
if (maxTweets > 200) {
|
|
1598
|
+
maxTweets = 200;
|
|
1599
|
+
}
|
|
1600
|
+
const userTweetsRequest = apiRequestFactory.createUserTweetsRequest();
|
|
1601
|
+
userTweetsRequest.variables.userId = userId;
|
|
1602
|
+
userTweetsRequest.variables.count = maxTweets;
|
|
1603
|
+
userTweetsRequest.variables.includePromotedContent = false;
|
|
1604
|
+
if (cursor != null && cursor != "") {
|
|
1605
|
+
userTweetsRequest.variables["cursor"] = cursor;
|
|
1606
|
+
}
|
|
1607
|
+
const res = await requestApi(
|
|
1608
|
+
userTweetsRequest.toRequestUrl(),
|
|
1609
|
+
auth
|
|
1610
|
+
);
|
|
1611
|
+
if (!res.success) {
|
|
1612
|
+
throw res.err;
|
|
1613
|
+
}
|
|
1614
|
+
return parseTimelineTweetsV2(res.value);
|
|
1615
|
+
}
|
|
1616
|
+
async function fetchTweetsAndReplies(userId, maxTweets, cursor, auth) {
|
|
1617
|
+
if (maxTweets > 40) {
|
|
1618
|
+
maxTweets = 40;
|
|
1619
|
+
}
|
|
1620
|
+
const userTweetsRequest = apiRequestFactory.createUserTweetsAndRepliesRequest();
|
|
1621
|
+
userTweetsRequest.variables.userId = userId;
|
|
1622
|
+
userTweetsRequest.variables.count = maxTweets;
|
|
1623
|
+
userTweetsRequest.variables.includePromotedContent = false;
|
|
1624
|
+
if (cursor != null && cursor != "") {
|
|
1625
|
+
userTweetsRequest.variables["cursor"] = cursor;
|
|
1626
|
+
}
|
|
1627
|
+
const res = await requestApi(
|
|
1628
|
+
userTweetsRequest.toRequestUrl(),
|
|
1629
|
+
auth
|
|
1630
|
+
);
|
|
1631
|
+
if (!res.success) {
|
|
1632
|
+
throw res.err;
|
|
1633
|
+
}
|
|
1634
|
+
return parseTimelineTweetsV2(res.value);
|
|
1635
|
+
}
|
|
1636
|
+
async function fetchListTweets(listId, maxTweets, cursor, auth) {
|
|
1637
|
+
if (maxTweets > 200) {
|
|
1638
|
+
maxTweets = 200;
|
|
1639
|
+
}
|
|
1640
|
+
const listTweetsRequest = apiRequestFactory.createListTweetsRequest();
|
|
1641
|
+
listTweetsRequest.variables.listId = listId;
|
|
1642
|
+
listTweetsRequest.variables.count = maxTweets;
|
|
1643
|
+
if (cursor != null && cursor != "") {
|
|
1644
|
+
listTweetsRequest.variables["cursor"] = cursor;
|
|
1645
|
+
}
|
|
1646
|
+
const res = await requestApi(
|
|
1647
|
+
listTweetsRequest.toRequestUrl(),
|
|
1648
|
+
auth
|
|
1649
|
+
);
|
|
1650
|
+
if (!res.success) {
|
|
1651
|
+
throw res.err;
|
|
1652
|
+
}
|
|
1653
|
+
return parseListTimelineTweets(res.value);
|
|
1654
|
+
}
|
|
1655
|
+
function getTweets(user, maxTweets, auth) {
|
|
1656
|
+
return getTweetTimeline(user, maxTweets, async (q, mt, c) => {
|
|
1657
|
+
const userIdRes = await getUserIdByScreenName(q, auth);
|
|
1658
|
+
if (!userIdRes.success) {
|
|
1659
|
+
throw userIdRes.err;
|
|
1660
|
+
}
|
|
1661
|
+
const { value: userId } = userIdRes;
|
|
1662
|
+
return fetchTweets(userId, mt, c, auth);
|
|
1663
|
+
});
|
|
1664
|
+
}
|
|
1665
|
+
function getTweetsByUserId(userId, maxTweets, auth) {
|
|
1666
|
+
return getTweetTimeline(userId, maxTweets, (q, mt, c) => {
|
|
1667
|
+
return fetchTweets(q, mt, c, auth);
|
|
1668
|
+
});
|
|
1669
|
+
}
|
|
1670
|
+
function getTweetsAndReplies(user, maxTweets, auth) {
|
|
1671
|
+
return getTweetTimeline(user, maxTweets, async (q, mt, c) => {
|
|
1672
|
+
const userIdRes = await getUserIdByScreenName(q, auth);
|
|
1673
|
+
if (!userIdRes.success) {
|
|
1674
|
+
throw userIdRes.err;
|
|
1675
|
+
}
|
|
1676
|
+
const { value: userId } = userIdRes;
|
|
1677
|
+
return fetchTweetsAndReplies(userId, mt, c, auth);
|
|
1678
|
+
});
|
|
1679
|
+
}
|
|
1680
|
+
function getTweetsAndRepliesByUserId(userId, maxTweets, auth) {
|
|
1681
|
+
return getTweetTimeline(userId, maxTweets, (q, mt, c) => {
|
|
1682
|
+
return fetchTweetsAndReplies(q, mt, c, auth);
|
|
1683
|
+
});
|
|
1684
|
+
}
|
|
1685
|
+
async function fetchLikedTweets(userId, maxTweets, cursor, auth) {
|
|
1686
|
+
if (!auth.isLoggedIn()) {
|
|
1687
|
+
throw new Error("Scraper is not logged-in for fetching liked tweets.");
|
|
1688
|
+
}
|
|
1689
|
+
if (maxTweets > 200) {
|
|
1690
|
+
maxTweets = 200;
|
|
1691
|
+
}
|
|
1692
|
+
const userTweetsRequest = apiRequestFactory.createUserLikedTweetsRequest();
|
|
1693
|
+
userTweetsRequest.variables.userId = userId;
|
|
1694
|
+
userTweetsRequest.variables.count = maxTweets;
|
|
1695
|
+
userTweetsRequest.variables.includePromotedContent = false;
|
|
1696
|
+
if (cursor != null && cursor != "") {
|
|
1697
|
+
userTweetsRequest.variables["cursor"] = cursor;
|
|
1698
|
+
}
|
|
1699
|
+
const res = await requestApi(
|
|
1700
|
+
userTweetsRequest.toRequestUrl(),
|
|
1701
|
+
auth
|
|
1702
|
+
);
|
|
1703
|
+
if (!res.success) {
|
|
1704
|
+
throw res.err;
|
|
1705
|
+
}
|
|
1706
|
+
return parseTimelineTweetsV2(res.value);
|
|
1707
|
+
}
|
|
1708
|
+
function getLikedTweets(user, maxTweets, auth) {
|
|
1709
|
+
return getTweetTimeline(user, maxTweets, async (q, mt, c) => {
|
|
1710
|
+
const userIdRes = await getUserIdByScreenName(q, auth);
|
|
1711
|
+
if (!userIdRes.success) {
|
|
1712
|
+
throw userIdRes.err;
|
|
1713
|
+
}
|
|
1714
|
+
const { value: userId } = userIdRes;
|
|
1715
|
+
return fetchLikedTweets(userId, mt, c, auth);
|
|
1716
|
+
});
|
|
1717
|
+
}
|
|
1718
|
+
async function getTweetWhere(tweets, query) {
|
|
1719
|
+
const isCallback = typeof query === "function";
|
|
1720
|
+
for await (const tweet of tweets) {
|
|
1721
|
+
const matches = isCallback ? await query(tweet) : checkTweetMatches(tweet, query);
|
|
1722
|
+
if (matches) {
|
|
1723
|
+
return tweet;
|
|
1724
|
+
}
|
|
1725
|
+
}
|
|
1726
|
+
return null;
|
|
1727
|
+
}
|
|
1728
|
+
async function getTweetsWhere(tweets, query) {
|
|
1729
|
+
const isCallback = typeof query === "function";
|
|
1730
|
+
const filtered = [];
|
|
1731
|
+
for await (const tweet of tweets) {
|
|
1732
|
+
const matches = isCallback ? query(tweet) : checkTweetMatches(tweet, query);
|
|
1733
|
+
if (!matches) continue;
|
|
1734
|
+
filtered.push(tweet);
|
|
1735
|
+
}
|
|
1736
|
+
return filtered;
|
|
1737
|
+
}
|
|
1738
|
+
function checkTweetMatches(tweet, options) {
|
|
1739
|
+
return Object.keys(options).every((k) => {
|
|
1740
|
+
const key = k;
|
|
1741
|
+
return tweet[key] === options[key];
|
|
1742
|
+
});
|
|
1743
|
+
}
|
|
1744
|
+
async function getLatestTweet(user, includeRetweets, max, auth) {
|
|
1745
|
+
const timeline = getTweets(user, max, auth);
|
|
1746
|
+
return max === 1 ? (await timeline.next()).value : await getTweetWhere(timeline, { isRetweet: includeRetweets });
|
|
1747
|
+
}
|
|
1748
|
+
async function getTweet(id, auth) {
|
|
1749
|
+
const tweetDetailRequest = apiRequestFactory.createTweetDetailRequest();
|
|
1750
|
+
tweetDetailRequest.variables.focalTweetId = id;
|
|
1751
|
+
const res = await requestApi(
|
|
1752
|
+
tweetDetailRequest.toRequestUrl(),
|
|
1753
|
+
auth
|
|
1754
|
+
);
|
|
1755
|
+
if (!res.success) {
|
|
1756
|
+
throw res.err;
|
|
1757
|
+
}
|
|
1758
|
+
if (!res.value) {
|
|
1759
|
+
return null;
|
|
1760
|
+
}
|
|
1761
|
+
const tweets = parseThreadedConversation(res.value);
|
|
1762
|
+
return tweets.find((tweet) => tweet.id === id) ?? null;
|
|
1763
|
+
}
|
|
1764
|
+
async function getTweetAnonymous(id, auth) {
|
|
1765
|
+
const tweetResultByRestIdRequest = apiRequestFactory.createTweetResultByRestIdRequest();
|
|
1766
|
+
tweetResultByRestIdRequest.variables.tweetId = id;
|
|
1767
|
+
const res = await requestApi(
|
|
1768
|
+
tweetResultByRestIdRequest.toRequestUrl(),
|
|
1769
|
+
auth
|
|
1770
|
+
);
|
|
1771
|
+
if (!res.success) {
|
|
1772
|
+
throw res.err;
|
|
1773
|
+
}
|
|
1774
|
+
if (!res.value.data) {
|
|
1775
|
+
return null;
|
|
1776
|
+
}
|
|
1777
|
+
return parseTimelineEntryItemContentRaw(res.value.data, id);
|
|
1778
|
+
}
|
|
1779
|
+
|
|
1780
|
+
const twUrl = "https://twitter.com";
|
|
1781
|
+
class Scraper {
|
|
1782
|
+
/**
|
|
1783
|
+
* Creates a new Scraper object.
|
|
1784
|
+
* - Scrapers maintain their own guest tokens for Twitter's internal API.
|
|
1785
|
+
* - Reusing Scraper objects is recommended to minimize the time spent authenticating unnecessarily.
|
|
1786
|
+
*/
|
|
1787
|
+
constructor(options) {
|
|
1788
|
+
this.options = options;
|
|
1789
|
+
this.token = bearerToken;
|
|
1790
|
+
this.useGuestAuth();
|
|
1791
|
+
}
|
|
1792
|
+
/**
|
|
1793
|
+
* Initializes auth properties using a guest token.
|
|
1794
|
+
* Used when creating a new instance of this class, and when logging out.
|
|
1795
|
+
* @internal
|
|
1796
|
+
*/
|
|
1797
|
+
useGuestAuth() {
|
|
1798
|
+
this.auth = new TwitterGuestAuth(this.token, this.getAuthOptions());
|
|
1799
|
+
this.authTrends = new TwitterGuestAuth(this.token, this.getAuthOptions());
|
|
1800
|
+
}
|
|
1801
|
+
/**
|
|
1802
|
+
* Fetches a Twitter profile.
|
|
1803
|
+
* @param username The Twitter username of the profile to fetch, without an `@` at the beginning.
|
|
1804
|
+
* @returns The requested {@link Profile}.
|
|
1805
|
+
*/
|
|
1806
|
+
async getProfile(username) {
|
|
1807
|
+
const res = await getProfile(username, this.auth);
|
|
1808
|
+
return this.handleResponse(res);
|
|
1809
|
+
}
|
|
1810
|
+
/**
|
|
1811
|
+
* Fetches the user ID corresponding to the provided screen name.
|
|
1812
|
+
* @param screenName The Twitter screen name of the profile to fetch.
|
|
1813
|
+
* @returns The ID of the corresponding account.
|
|
1814
|
+
*/
|
|
1815
|
+
async getUserIdByScreenName(screenName) {
|
|
1816
|
+
const res = await getUserIdByScreenName(screenName, this.auth);
|
|
1817
|
+
return this.handleResponse(res);
|
|
1818
|
+
}
|
|
1819
|
+
/**
|
|
1820
|
+
* Fetches tweets from Twitter.
|
|
1821
|
+
* @param query The search query. Any Twitter-compatible query format can be used.
|
|
1822
|
+
* @param maxTweets The maximum number of tweets to return.
|
|
1823
|
+
* @param includeReplies Whether or not replies should be included in the response.
|
|
1824
|
+
* @param searchMode The category filter to apply to the search. Defaults to `Top`.
|
|
1825
|
+
* @returns An {@link AsyncGenerator} of tweets matching the provided filters.
|
|
1826
|
+
*/
|
|
1827
|
+
searchTweets(query, maxTweets, searchMode = SearchMode.Top) {
|
|
1828
|
+
return searchTweets(query, maxTweets, searchMode, this.auth);
|
|
1829
|
+
}
|
|
1830
|
+
/**
|
|
1831
|
+
* Fetches profiles from Twitter.
|
|
1832
|
+
* @param query The search query. Any Twitter-compatible query format can be used.
|
|
1833
|
+
* @param maxProfiles The maximum number of profiles to return.
|
|
1834
|
+
* @returns An {@link AsyncGenerator} of tweets matching the provided filter(s).
|
|
1835
|
+
*/
|
|
1836
|
+
searchProfiles(query, maxProfiles) {
|
|
1837
|
+
return searchProfiles(query, maxProfiles, this.auth);
|
|
1838
|
+
}
|
|
1839
|
+
/**
|
|
1840
|
+
* Fetches tweets from Twitter.
|
|
1841
|
+
* @param query The search query. Any Twitter-compatible query format can be used.
|
|
1842
|
+
* @param maxTweets The maximum number of tweets to return.
|
|
1843
|
+
* @param includeReplies Whether or not replies should be included in the response.
|
|
1844
|
+
* @param searchMode The category filter to apply to the search. Defaults to `Top`.
|
|
1845
|
+
* @param cursor The search cursor, which can be passed into further requests for more results.
|
|
1846
|
+
* @returns A page of results, containing a cursor that can be used in further requests.
|
|
1847
|
+
*/
|
|
1848
|
+
fetchSearchTweets(query, maxTweets, searchMode, cursor) {
|
|
1849
|
+
return fetchSearchTweets(query, maxTweets, searchMode, this.auth, cursor);
|
|
1850
|
+
}
|
|
1851
|
+
/**
|
|
1852
|
+
* Fetches profiles from Twitter.
|
|
1853
|
+
* @param query The search query. Any Twitter-compatible query format can be used.
|
|
1854
|
+
* @param maxProfiles The maximum number of profiles to return.
|
|
1855
|
+
* @param cursor The search cursor, which can be passed into further requests for more results.
|
|
1856
|
+
* @returns A page of results, containing a cursor that can be used in further requests.
|
|
1857
|
+
*/
|
|
1858
|
+
fetchSearchProfiles(query, maxProfiles, cursor) {
|
|
1859
|
+
return fetchSearchProfiles(query, maxProfiles, this.auth, cursor);
|
|
1860
|
+
}
|
|
1861
|
+
/**
|
|
1862
|
+
* Fetches list tweets from Twitter.
|
|
1863
|
+
* @param listId The list id
|
|
1864
|
+
* @param maxTweets The maximum number of tweets to return.
|
|
1865
|
+
* @param cursor The search cursor, which can be passed into further requests for more results.
|
|
1866
|
+
* @returns A page of results, containing a cursor that can be used in further requests.
|
|
1867
|
+
*/
|
|
1868
|
+
fetchListTweets(listId, maxTweets, cursor) {
|
|
1869
|
+
return fetchListTweets(listId, maxTweets, cursor, this.auth);
|
|
1870
|
+
}
|
|
1871
|
+
/**
|
|
1872
|
+
* Fetch the profiles a user is following
|
|
1873
|
+
* @param userId The user whose following should be returned
|
|
1874
|
+
* @param maxProfiles The maximum number of profiles to return.
|
|
1875
|
+
* @returns An {@link AsyncGenerator} of following profiles for the provided user.
|
|
1876
|
+
*/
|
|
1877
|
+
getFollowing(userId, maxProfiles) {
|
|
1878
|
+
return getFollowing(userId, maxProfiles, this.auth);
|
|
1879
|
+
}
|
|
1880
|
+
/**
|
|
1881
|
+
* Fetch the profiles that follow a user
|
|
1882
|
+
* @param userId The user whose followers should be returned
|
|
1883
|
+
* @param maxProfiles The maximum number of profiles to return.
|
|
1884
|
+
* @returns An {@link AsyncGenerator} of profiles following the provided user.
|
|
1885
|
+
*/
|
|
1886
|
+
getFollowers(userId, maxProfiles) {
|
|
1887
|
+
return getFollowers(userId, maxProfiles, this.auth);
|
|
1888
|
+
}
|
|
1889
|
+
/**
|
|
1890
|
+
* Fetches following profiles from Twitter.
|
|
1891
|
+
* @param userId The user whose following should be returned
|
|
1892
|
+
* @param maxProfiles The maximum number of profiles to return.
|
|
1893
|
+
* @param cursor The search cursor, which can be passed into further requests for more results.
|
|
1894
|
+
* @returns A page of results, containing a cursor that can be used in further requests.
|
|
1895
|
+
*/
|
|
1896
|
+
fetchProfileFollowing(userId, maxProfiles, cursor) {
|
|
1897
|
+
return fetchProfileFollowing(userId, maxProfiles, this.auth, cursor);
|
|
1898
|
+
}
|
|
1899
|
+
/**
|
|
1900
|
+
* Fetches profile followers from Twitter.
|
|
1901
|
+
* @param userId The user whose following should be returned
|
|
1902
|
+
* @param maxProfiles The maximum number of profiles to return.
|
|
1903
|
+
* @param cursor The search cursor, which can be passed into further requests for more results.
|
|
1904
|
+
* @returns A page of results, containing a cursor that can be used in further requests.
|
|
1905
|
+
*/
|
|
1906
|
+
fetchProfileFollowers(userId, maxProfiles, cursor) {
|
|
1907
|
+
return fetchProfileFollowers(userId, maxProfiles, this.auth, cursor);
|
|
1908
|
+
}
|
|
1909
|
+
/**
|
|
1910
|
+
* Fetches the current trends from Twitter.
|
|
1911
|
+
* @returns The current list of trends.
|
|
1912
|
+
*/
|
|
1913
|
+
getTrends() {
|
|
1914
|
+
return getTrends(this.authTrends);
|
|
1915
|
+
}
|
|
1916
|
+
/**
|
|
1917
|
+
* Fetches tweets from a Twitter user.
|
|
1918
|
+
* @param user The user whose tweets should be returned.
|
|
1919
|
+
* @param maxTweets The maximum number of tweets to return. Defaults to `200`.
|
|
1920
|
+
* @returns An {@link AsyncGenerator} of tweets from the provided user.
|
|
1921
|
+
*/
|
|
1922
|
+
getTweets(user, maxTweets = 200) {
|
|
1923
|
+
return getTweets(user, maxTweets, this.auth);
|
|
1924
|
+
}
|
|
1925
|
+
/**
|
|
1926
|
+
* Fetches liked tweets from a Twitter user. Requires authentication.
|
|
1927
|
+
* @param user The user whose likes should be returned.
|
|
1928
|
+
* @param maxTweets The maximum number of tweets to return. Defaults to `200`.
|
|
1929
|
+
* @returns An {@link AsyncGenerator} of liked tweets from the provided user.
|
|
1930
|
+
*/
|
|
1931
|
+
getLikedTweets(user, maxTweets = 200) {
|
|
1932
|
+
return getLikedTweets(user, maxTweets, this.auth);
|
|
1933
|
+
}
|
|
1934
|
+
/**
|
|
1935
|
+
* Fetches tweets from a Twitter user using their ID.
|
|
1936
|
+
* @param userId The user whose tweets should be returned.
|
|
1937
|
+
* @param maxTweets The maximum number of tweets to return. Defaults to `200`.
|
|
1938
|
+
* @returns An {@link AsyncGenerator} of tweets from the provided user.
|
|
1939
|
+
*/
|
|
1940
|
+
getTweetsByUserId(userId, maxTweets = 200) {
|
|
1941
|
+
return getTweetsByUserId(userId, maxTweets, this.auth);
|
|
1942
|
+
}
|
|
1943
|
+
/**
|
|
1944
|
+
* Fetches tweets and replies from a Twitter user.
|
|
1945
|
+
* @param user The user whose tweets should be returned.
|
|
1946
|
+
* @param maxTweets The maximum number of tweets to return. Defaults to `200`.
|
|
1947
|
+
* @returns An {@link AsyncGenerator} of tweets from the provided user.
|
|
1948
|
+
*/
|
|
1949
|
+
getTweetsAndReplies(user, maxTweets = 200) {
|
|
1950
|
+
return getTweetsAndReplies(user, maxTweets, this.auth);
|
|
1951
|
+
}
|
|
1952
|
+
/**
|
|
1953
|
+
* Fetches tweets and replies from a Twitter user using their ID.
|
|
1954
|
+
* @param userId The user whose tweets should be returned.
|
|
1955
|
+
* @param maxTweets The maximum number of tweets to return. Defaults to `200`.
|
|
1956
|
+
* @returns An {@link AsyncGenerator} of tweets from the provided user.
|
|
1957
|
+
*/
|
|
1958
|
+
getTweetsAndRepliesByUserId(userId, maxTweets = 200) {
|
|
1959
|
+
return getTweetsAndRepliesByUserId(userId, maxTweets, this.auth);
|
|
1960
|
+
}
|
|
1961
|
+
/**
|
|
1962
|
+
* Fetches the first tweet matching the given query.
|
|
1963
|
+
*
|
|
1964
|
+
* Example:
|
|
1965
|
+
* ```js
|
|
1966
|
+
* const timeline = scraper.getTweets('user', 200);
|
|
1967
|
+
* const retweet = await scraper.getTweetWhere(timeline, { isRetweet: true });
|
|
1968
|
+
* ```
|
|
1969
|
+
* @param tweets The {@link AsyncIterable} of tweets to search through.
|
|
1970
|
+
* @param query A query to test **all** tweets against. This may be either an
|
|
1971
|
+
* object of key/value pairs or a predicate. If this query is an object, all
|
|
1972
|
+
* key/value pairs must match a {@link Tweet} for it to be returned. If this query
|
|
1973
|
+
* is a predicate, it must resolve to `true` for a {@link Tweet} to be returned.
|
|
1974
|
+
* - All keys are optional.
|
|
1975
|
+
* - If specified, the key must be implemented by that of {@link Tweet}.
|
|
1976
|
+
*/
|
|
1977
|
+
getTweetWhere(tweets, query) {
|
|
1978
|
+
return getTweetWhere(tweets, query);
|
|
1979
|
+
}
|
|
1980
|
+
/**
|
|
1981
|
+
* Fetches all tweets matching the given query.
|
|
1982
|
+
*
|
|
1983
|
+
* Example:
|
|
1984
|
+
* ```js
|
|
1985
|
+
* const timeline = scraper.getTweets('user', 200);
|
|
1986
|
+
* const retweets = await scraper.getTweetsWhere(timeline, { isRetweet: true });
|
|
1987
|
+
* ```
|
|
1988
|
+
* @param tweets The {@link AsyncIterable} of tweets to search through.
|
|
1989
|
+
* @param query A query to test **all** tweets against. This may be either an
|
|
1990
|
+
* object of key/value pairs or a predicate. If this query is an object, all
|
|
1991
|
+
* key/value pairs must match a {@link Tweet} for it to be returned. If this query
|
|
1992
|
+
* is a predicate, it must resolve to `true` for a {@link Tweet} to be returned.
|
|
1993
|
+
* - All keys are optional.
|
|
1994
|
+
* - If specified, the key must be implemented by that of {@link Tweet}.
|
|
1995
|
+
*/
|
|
1996
|
+
getTweetsWhere(tweets, query) {
|
|
1997
|
+
return getTweetsWhere(tweets, query);
|
|
1998
|
+
}
|
|
1999
|
+
/**
|
|
2000
|
+
* Fetches the most recent tweet from a Twitter user.
|
|
2001
|
+
* @param user The user whose latest tweet should be returned.
|
|
2002
|
+
* @param includeRetweets Whether or not to include retweets. Defaults to `false`.
|
|
2003
|
+
* @returns The {@link Tweet} object or `null`/`undefined` if it couldn't be fetched.
|
|
2004
|
+
*/
|
|
2005
|
+
getLatestTweet(user, includeRetweets = false, max = 200) {
|
|
2006
|
+
return getLatestTweet(user, includeRetweets, max, this.auth);
|
|
2007
|
+
}
|
|
2008
|
+
/**
|
|
2009
|
+
* Fetches a single tweet.
|
|
2010
|
+
* @param id The ID of the tweet to fetch.
|
|
2011
|
+
* @returns The {@link Tweet} object, or `null` if it couldn't be fetched.
|
|
2012
|
+
*/
|
|
2013
|
+
getTweet(id) {
|
|
2014
|
+
if (this.auth instanceof TwitterUserAuth) {
|
|
2015
|
+
return getTweet(id, this.auth);
|
|
2016
|
+
} else {
|
|
2017
|
+
return getTweetAnonymous(id, this.auth);
|
|
2018
|
+
}
|
|
2019
|
+
}
|
|
2020
|
+
/**
|
|
2021
|
+
* Returns if the scraper has a guest token. The token may not be valid.
|
|
2022
|
+
* @returns `true` if the scraper has a guest token; otherwise `false`.
|
|
2023
|
+
*/
|
|
2024
|
+
hasGuestToken() {
|
|
2025
|
+
return this.auth.hasToken() || this.authTrends.hasToken();
|
|
2026
|
+
}
|
|
2027
|
+
/**
|
|
2028
|
+
* Returns if the scraper is logged in as a real user.
|
|
2029
|
+
* @returns `true` if the scraper is logged in with a real user account; otherwise `false`.
|
|
2030
|
+
*/
|
|
2031
|
+
async isLoggedIn() {
|
|
2032
|
+
return await this.auth.isLoggedIn() && await this.authTrends.isLoggedIn();
|
|
2033
|
+
}
|
|
2034
|
+
/**
|
|
2035
|
+
* Login to Twitter as a real Twitter account. This enables running
|
|
2036
|
+
* searches.
|
|
2037
|
+
* @param username The username of the Twitter account to login with.
|
|
2038
|
+
* @param password The password of the Twitter account to login with.
|
|
2039
|
+
* @param email The email to log in with, if you have email confirmation enabled.
|
|
2040
|
+
* @param twoFactorSecret The secret to generate two factor authentication tokens with, if you have two factor authentication enabled.
|
|
2041
|
+
*/
|
|
2042
|
+
async login(username, password, email, twoFactorSecret) {
|
|
2043
|
+
const userAuth = new TwitterUserAuth(this.token, this.getAuthOptions());
|
|
2044
|
+
await userAuth.login(username, password, email, twoFactorSecret);
|
|
2045
|
+
this.auth = userAuth;
|
|
2046
|
+
this.authTrends = userAuth;
|
|
2047
|
+
}
|
|
2048
|
+
/**
|
|
2049
|
+
* Log out of Twitter.
|
|
2050
|
+
*/
|
|
2051
|
+
async logout() {
|
|
2052
|
+
await this.auth.logout();
|
|
2053
|
+
await this.authTrends.logout();
|
|
2054
|
+
this.useGuestAuth();
|
|
2055
|
+
}
|
|
2056
|
+
/**
|
|
2057
|
+
* Retrieves all cookies for the current session.
|
|
2058
|
+
* @returns All cookies for the current session.
|
|
2059
|
+
*/
|
|
2060
|
+
async getCookies() {
|
|
2061
|
+
return await this.authTrends.cookieJar().getCookies(
|
|
2062
|
+
typeof document !== "undefined" ? document.location.toString() : twUrl
|
|
2063
|
+
);
|
|
2064
|
+
}
|
|
2065
|
+
/**
|
|
2066
|
+
* Set cookies for the current session.
|
|
2067
|
+
* @param cookies The cookies to set for the current session.
|
|
2068
|
+
*/
|
|
2069
|
+
async setCookies(cookies) {
|
|
2070
|
+
const userAuth = new TwitterUserAuth(this.token, this.getAuthOptions());
|
|
2071
|
+
for (const cookie of cookies) {
|
|
2072
|
+
await userAuth.cookieJar().setCookie(cookie, twUrl);
|
|
2073
|
+
}
|
|
2074
|
+
this.auth = userAuth;
|
|
2075
|
+
this.authTrends = userAuth;
|
|
2076
|
+
}
|
|
2077
|
+
/**
|
|
2078
|
+
* Clear all cookies for the current session.
|
|
2079
|
+
*/
|
|
2080
|
+
async clearCookies() {
|
|
2081
|
+
await this.auth.cookieJar().removeAllCookies();
|
|
2082
|
+
await this.authTrends.cookieJar().removeAllCookies();
|
|
2083
|
+
}
|
|
2084
|
+
/**
|
|
2085
|
+
* Sets the optional cookie to be used in requests.
|
|
2086
|
+
* @param _cookie The cookie to be used in requests.
|
|
2087
|
+
* @deprecated This function no longer represents any part of Twitter's auth flow.
|
|
2088
|
+
* @returns This scraper instance.
|
|
2089
|
+
*/
|
|
2090
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
2091
|
+
withCookie(_cookie) {
|
|
2092
|
+
console.warn(
|
|
2093
|
+
"Warning: Scraper#withCookie is deprecated and will be removed in a later version. Use Scraper#login or Scraper#setCookies instead."
|
|
2094
|
+
);
|
|
2095
|
+
return this;
|
|
2096
|
+
}
|
|
2097
|
+
/**
|
|
2098
|
+
* Sets the optional CSRF token to be used in requests.
|
|
2099
|
+
* @param _token The CSRF token to be used in requests.
|
|
2100
|
+
* @deprecated This function no longer represents any part of Twitter's auth flow.
|
|
2101
|
+
* @returns This scraper instance.
|
|
2102
|
+
*/
|
|
2103
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
2104
|
+
withXCsrfToken(_token) {
|
|
2105
|
+
console.warn(
|
|
2106
|
+
"Warning: Scraper#withXCsrfToken is deprecated and will be removed in a later version."
|
|
2107
|
+
);
|
|
2108
|
+
return this;
|
|
2109
|
+
}
|
|
2110
|
+
getAuthOptions() {
|
|
2111
|
+
return {
|
|
2112
|
+
fetch: this.options?.fetch,
|
|
2113
|
+
transform: this.options?.transform
|
|
2114
|
+
};
|
|
2115
|
+
}
|
|
2116
|
+
handleResponse(res) {
|
|
2117
|
+
if (!res.success) {
|
|
2118
|
+
throw res.err;
|
|
2119
|
+
}
|
|
2120
|
+
return res.value;
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2124
|
+
exports.Scraper = Scraper;
|
|
2125
|
+
exports.SearchMode = SearchMode;
|
|
2126
|
+
//# sourceMappingURL=index.js.map
|