@the-convocation/twitter-scraper 0.15.1 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/default/cjs/index.js +270 -177
- package/dist/default/cjs/index.js.map +1 -1
- package/dist/default/esm/index.mjs +270 -178
- package/dist/default/esm/index.mjs.map +1 -1
- package/dist/node/cjs/index.cjs +270 -177
- package/dist/node/cjs/index.cjs.map +1 -1
- package/dist/node/esm/index.mjs +270 -178
- package/dist/node/esm/index.mjs.map +1 -1
- package/dist/types/index.d.ts +195 -8
- package/package.json +1 -1
|
@@ -29,22 +29,43 @@ function _interopNamespaceDefault(e) {
|
|
|
29
29
|
var OTPAuth__namespace = /*#__PURE__*/_interopNamespaceDefault(OTPAuth);
|
|
30
30
|
|
|
31
31
|
class ApiError extends Error {
|
|
32
|
-
constructor(response, data
|
|
33
|
-
super(
|
|
32
|
+
constructor(response, data) {
|
|
33
|
+
super(
|
|
34
|
+
`Response status: ${response.status} | headers: ${JSON.stringify(
|
|
35
|
+
headersToString(response.headers)
|
|
36
|
+
)} | data: ${data}`
|
|
37
|
+
);
|
|
34
38
|
this.response = response;
|
|
35
39
|
this.data = data;
|
|
36
40
|
}
|
|
37
41
|
static async fromResponse(response) {
|
|
38
42
|
let data = void 0;
|
|
39
43
|
try {
|
|
40
|
-
|
|
44
|
+
if (response.headers.get("content-type")?.includes("application/json")) {
|
|
45
|
+
data = await response.json();
|
|
46
|
+
} else {
|
|
47
|
+
data = await response.text();
|
|
48
|
+
}
|
|
41
49
|
} catch {
|
|
42
50
|
try {
|
|
43
51
|
data = await response.text();
|
|
44
52
|
} catch {
|
|
45
53
|
}
|
|
46
54
|
}
|
|
47
|
-
return new ApiError(response, data
|
|
55
|
+
return new ApiError(response, data);
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
function headersToString(headers) {
|
|
59
|
+
const result = [];
|
|
60
|
+
headers.forEach((value, key) => {
|
|
61
|
+
result.push(`${key}: ${value}`);
|
|
62
|
+
});
|
|
63
|
+
return result.join("\n");
|
|
64
|
+
}
|
|
65
|
+
class AuthenticationError extends Error {
|
|
66
|
+
constructor(message) {
|
|
67
|
+
super(message || "Authentication failed");
|
|
68
|
+
this.name = "AuthenticationError";
|
|
48
69
|
}
|
|
49
70
|
}
|
|
50
71
|
|
|
@@ -103,6 +124,10 @@ async function updateCookieJar(cookieJar, headers) {
|
|
|
103
124
|
}
|
|
104
125
|
|
|
105
126
|
const bearerToken = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF";
|
|
127
|
+
async function jitter(maxMs) {
|
|
128
|
+
const jitter2 = Math.random() * maxMs;
|
|
129
|
+
await new Promise((resolve) => setTimeout(resolve, jitter2));
|
|
130
|
+
}
|
|
106
131
|
async function requestApi(url, auth, method = "GET", platform = new Platform()) {
|
|
107
132
|
const headers = new headersPolyfill.Headers();
|
|
108
133
|
await auth.installTo(headers, url);
|
|
@@ -275,7 +300,9 @@ class TwitterGuestAuth {
|
|
|
275
300
|
}
|
|
276
301
|
const token = this.guestToken;
|
|
277
302
|
if (token == null) {
|
|
278
|
-
throw new
|
|
303
|
+
throw new AuthenticationError(
|
|
304
|
+
"Authentication token is null or undefined."
|
|
305
|
+
);
|
|
279
306
|
}
|
|
280
307
|
headers.set("authorization", `Bearer ${this.bearerToken}`);
|
|
281
308
|
headers.set("x-guest-token", token);
|
|
@@ -322,15 +349,15 @@ class TwitterGuestAuth {
|
|
|
322
349
|
});
|
|
323
350
|
await updateCookieJar(this.jar, res.headers);
|
|
324
351
|
if (!res.ok) {
|
|
325
|
-
throw new
|
|
352
|
+
throw new AuthenticationError(await res.text());
|
|
326
353
|
}
|
|
327
354
|
const o = await res.json();
|
|
328
355
|
if (o == null || o["guest_token"] == null) {
|
|
329
|
-
throw new
|
|
356
|
+
throw new AuthenticationError("guest_token not found.");
|
|
330
357
|
}
|
|
331
358
|
const newGuestToken = o["guest_token"];
|
|
332
359
|
if (typeof newGuestToken !== "string") {
|
|
333
|
-
throw new
|
|
360
|
+
throw new AuthenticationError("guest_token was not a string.");
|
|
334
361
|
}
|
|
335
362
|
this.guestToken = newGuestToken;
|
|
336
363
|
this.guestCreatedAt = /* @__PURE__ */ new Date();
|
|
@@ -351,6 +378,47 @@ const TwitterUserAuthSubtask = typebox.Type.Object({
|
|
|
351
378
|
class TwitterUserAuth extends TwitterGuestAuth {
|
|
352
379
|
constructor(bearerToken, options) {
|
|
353
380
|
super(bearerToken, options);
|
|
381
|
+
this.subtaskHandlers = /* @__PURE__ */ new Map();
|
|
382
|
+
this.initializeDefaultHandlers();
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* Register a custom subtask handler or override an existing one
|
|
386
|
+
* @param subtaskId The ID of the subtask to handle
|
|
387
|
+
* @param handler The handler function that processes the subtask
|
|
388
|
+
*/
|
|
389
|
+
registerSubtaskHandler(subtaskId, handler) {
|
|
390
|
+
this.subtaskHandlers.set(subtaskId, handler);
|
|
391
|
+
}
|
|
392
|
+
initializeDefaultHandlers() {
|
|
393
|
+
this.subtaskHandlers.set(
|
|
394
|
+
"LoginJsInstrumentationSubtask",
|
|
395
|
+
this.handleJsInstrumentationSubtask.bind(this)
|
|
396
|
+
);
|
|
397
|
+
this.subtaskHandlers.set(
|
|
398
|
+
"LoginEnterUserIdentifierSSO",
|
|
399
|
+
this.handleEnterUserIdentifierSSO.bind(this)
|
|
400
|
+
);
|
|
401
|
+
this.subtaskHandlers.set(
|
|
402
|
+
"LoginEnterAlternateIdentifierSubtask",
|
|
403
|
+
this.handleEnterAlternateIdentifierSubtask.bind(this)
|
|
404
|
+
);
|
|
405
|
+
this.subtaskHandlers.set(
|
|
406
|
+
"LoginEnterPassword",
|
|
407
|
+
this.handleEnterPassword.bind(this)
|
|
408
|
+
);
|
|
409
|
+
this.subtaskHandlers.set(
|
|
410
|
+
"AccountDuplicationCheck",
|
|
411
|
+
this.handleAccountDuplicationCheck.bind(this)
|
|
412
|
+
);
|
|
413
|
+
this.subtaskHandlers.set(
|
|
414
|
+
"LoginTwoFactorAuthChallenge",
|
|
415
|
+
this.handleTwoFactorAuthChallenge.bind(this)
|
|
416
|
+
);
|
|
417
|
+
this.subtaskHandlers.set("LoginAcid", this.handleAcid.bind(this));
|
|
418
|
+
this.subtaskHandlers.set(
|
|
419
|
+
"LoginSuccessSubtask",
|
|
420
|
+
this.handleSuccessSubtask.bind(this)
|
|
421
|
+
);
|
|
354
422
|
}
|
|
355
423
|
async isLoggedIn() {
|
|
356
424
|
const res = await requestApi(
|
|
@@ -365,52 +433,49 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
365
433
|
}
|
|
366
434
|
async login(username, password, email, twoFactorSecret) {
|
|
367
435
|
await this.updateGuestToken();
|
|
436
|
+
const credentials = {
|
|
437
|
+
username,
|
|
438
|
+
password,
|
|
439
|
+
email,
|
|
440
|
+
twoFactorSecret
|
|
441
|
+
};
|
|
368
442
|
let next = await this.initLogin();
|
|
369
|
-
while ("
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
} else if (next.subtask.subtask_id === "AccountDuplicationCheck") {
|
|
382
|
-
next = await this.handleAccountDuplicationCheck(next);
|
|
383
|
-
} else if (next.subtask.subtask_id === "LoginTwoFactorAuthChallenge") {
|
|
384
|
-
if (twoFactorSecret) {
|
|
385
|
-
next = await this.handleTwoFactorAuthChallenge(next, twoFactorSecret);
|
|
386
|
-
} else {
|
|
387
|
-
throw new Error(
|
|
388
|
-
"Requested two factor authentication code but no secret provided"
|
|
389
|
-
);
|
|
390
|
-
}
|
|
391
|
-
} else if (next.subtask.subtask_id === "LoginAcid") {
|
|
392
|
-
next = await this.handleAcid(next, email);
|
|
393
|
-
} else if (next.subtask.subtask_id === "LoginSuccessSubtask") {
|
|
394
|
-
next = await this.handleSuccessSubtask(next);
|
|
443
|
+
while (next.status === "success" && next.response.subtasks?.length) {
|
|
444
|
+
const flowToken = next.response.flow_token;
|
|
445
|
+
if (flowToken == null) {
|
|
446
|
+
throw new Error("flow_token not found.");
|
|
447
|
+
}
|
|
448
|
+
const subtaskId = next.response.subtasks[0].subtask_id;
|
|
449
|
+
const handler = this.subtaskHandlers.get(subtaskId);
|
|
450
|
+
if (handler) {
|
|
451
|
+
next = await handler(subtaskId, next.response, credentials, {
|
|
452
|
+
sendFlowRequest: this.executeFlowTask.bind(this),
|
|
453
|
+
getFlowToken: () => flowToken
|
|
454
|
+
});
|
|
395
455
|
} else {
|
|
396
|
-
throw new Error(`Unknown subtask ${
|
|
456
|
+
throw new Error(`Unknown subtask ${subtaskId}`);
|
|
397
457
|
}
|
|
398
458
|
}
|
|
399
|
-
if ("
|
|
459
|
+
if (next.status === "error") {
|
|
400
460
|
throw next.err;
|
|
401
461
|
}
|
|
402
462
|
}
|
|
403
463
|
async logout() {
|
|
404
|
-
if (!this.
|
|
464
|
+
if (!this.hasToken()) {
|
|
405
465
|
return;
|
|
406
466
|
}
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
467
|
+
try {
|
|
468
|
+
await requestApi(
|
|
469
|
+
"https://api.twitter.com/1.1/account/logout.json",
|
|
470
|
+
this,
|
|
471
|
+
"POST"
|
|
472
|
+
);
|
|
473
|
+
} catch (error) {
|
|
474
|
+
console.warn("Error during logout:", error);
|
|
475
|
+
} finally {
|
|
476
|
+
this.deleteToken();
|
|
477
|
+
this.jar = new toughCookie.CookieJar();
|
|
478
|
+
}
|
|
414
479
|
}
|
|
415
480
|
async installCsrfToken(headers) {
|
|
416
481
|
const cookies = await this.getCookies();
|
|
@@ -449,12 +514,12 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
449
514
|
}
|
|
450
515
|
});
|
|
451
516
|
}
|
|
452
|
-
async handleJsInstrumentationSubtask(
|
|
453
|
-
return await
|
|
454
|
-
flow_token:
|
|
517
|
+
async handleJsInstrumentationSubtask(subtaskId, _prev, _credentials, api) {
|
|
518
|
+
return await api.sendFlowRequest({
|
|
519
|
+
flow_token: api.getFlowToken(),
|
|
455
520
|
subtask_inputs: [
|
|
456
521
|
{
|
|
457
|
-
subtask_id:
|
|
522
|
+
subtask_id: subtaskId,
|
|
458
523
|
js_instrumentation: {
|
|
459
524
|
response: "{}",
|
|
460
525
|
link: "next_link"
|
|
@@ -463,32 +528,32 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
463
528
|
]
|
|
464
529
|
});
|
|
465
530
|
}
|
|
466
|
-
async handleEnterAlternateIdentifierSubtask(
|
|
531
|
+
async handleEnterAlternateIdentifierSubtask(subtaskId, _prev, credentials, api) {
|
|
467
532
|
return await this.executeFlowTask({
|
|
468
|
-
flow_token:
|
|
533
|
+
flow_token: api.getFlowToken(),
|
|
469
534
|
subtask_inputs: [
|
|
470
535
|
{
|
|
471
|
-
subtask_id:
|
|
536
|
+
subtask_id: subtaskId,
|
|
472
537
|
enter_text: {
|
|
473
|
-
text: email,
|
|
538
|
+
text: credentials.email,
|
|
474
539
|
link: "next_link"
|
|
475
540
|
}
|
|
476
541
|
}
|
|
477
542
|
]
|
|
478
543
|
});
|
|
479
544
|
}
|
|
480
|
-
async handleEnterUserIdentifierSSO(
|
|
545
|
+
async handleEnterUserIdentifierSSO(subtaskId, _prev, credentials, api) {
|
|
481
546
|
return await this.executeFlowTask({
|
|
482
|
-
flow_token:
|
|
547
|
+
flow_token: api.getFlowToken(),
|
|
483
548
|
subtask_inputs: [
|
|
484
549
|
{
|
|
485
|
-
subtask_id:
|
|
550
|
+
subtask_id: subtaskId,
|
|
486
551
|
settings_list: {
|
|
487
552
|
setting_responses: [
|
|
488
553
|
{
|
|
489
554
|
key: "user_identifier",
|
|
490
555
|
response_data: {
|
|
491
|
-
text_data: { result: username }
|
|
556
|
+
text_data: { result: credentials.username }
|
|
492
557
|
}
|
|
493
558
|
}
|
|
494
559
|
],
|
|
@@ -498,26 +563,26 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
498
563
|
]
|
|
499
564
|
});
|
|
500
565
|
}
|
|
501
|
-
async handleEnterPassword(
|
|
566
|
+
async handleEnterPassword(subtaskId, _prev, credentials, api) {
|
|
502
567
|
return await this.executeFlowTask({
|
|
503
|
-
flow_token:
|
|
568
|
+
flow_token: api.getFlowToken(),
|
|
504
569
|
subtask_inputs: [
|
|
505
570
|
{
|
|
506
|
-
subtask_id:
|
|
571
|
+
subtask_id: subtaskId,
|
|
507
572
|
enter_password: {
|
|
508
|
-
password,
|
|
573
|
+
password: credentials.password,
|
|
509
574
|
link: "next_link"
|
|
510
575
|
}
|
|
511
576
|
}
|
|
512
577
|
]
|
|
513
578
|
});
|
|
514
579
|
}
|
|
515
|
-
async handleAccountDuplicationCheck(
|
|
580
|
+
async handleAccountDuplicationCheck(subtaskId, _prev, _credentials, api) {
|
|
516
581
|
return await this.executeFlowTask({
|
|
517
|
-
flow_token:
|
|
582
|
+
flow_token: api.getFlowToken(),
|
|
518
583
|
subtask_inputs: [
|
|
519
584
|
{
|
|
520
|
-
subtask_id:
|
|
585
|
+
subtask_id: subtaskId,
|
|
521
586
|
check_logged_in_account: {
|
|
522
587
|
link: "AccountDuplicationCheck_false"
|
|
523
588
|
}
|
|
@@ -525,16 +590,24 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
525
590
|
]
|
|
526
591
|
});
|
|
527
592
|
}
|
|
528
|
-
async handleTwoFactorAuthChallenge(
|
|
529
|
-
|
|
593
|
+
async handleTwoFactorAuthChallenge(subtaskId, _prev, credentials, api) {
|
|
594
|
+
if (!credentials.twoFactorSecret) {
|
|
595
|
+
return {
|
|
596
|
+
status: "error",
|
|
597
|
+
err: new AuthenticationError(
|
|
598
|
+
"Two-factor authentication is required but no secret was provided"
|
|
599
|
+
)
|
|
600
|
+
};
|
|
601
|
+
}
|
|
602
|
+
const totp = new OTPAuth__namespace.TOTP({ secret: credentials.twoFactorSecret });
|
|
530
603
|
let error;
|
|
531
604
|
for (let attempts = 1; attempts < 4; attempts += 1) {
|
|
532
605
|
try {
|
|
533
|
-
return await
|
|
534
|
-
flow_token:
|
|
606
|
+
return await api.sendFlowRequest({
|
|
607
|
+
flow_token: api.getFlowToken(),
|
|
535
608
|
subtask_inputs: [
|
|
536
609
|
{
|
|
537
|
-
subtask_id:
|
|
610
|
+
subtask_id: subtaskId,
|
|
538
611
|
enter_text: {
|
|
539
612
|
link: "next_link",
|
|
540
613
|
text: totp.generate()
|
|
@@ -549,23 +622,23 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
549
622
|
}
|
|
550
623
|
throw error;
|
|
551
624
|
}
|
|
552
|
-
async handleAcid(
|
|
625
|
+
async handleAcid(subtaskId, _prev, credentials, api) {
|
|
553
626
|
return await this.executeFlowTask({
|
|
554
|
-
flow_token:
|
|
627
|
+
flow_token: api.getFlowToken(),
|
|
555
628
|
subtask_inputs: [
|
|
556
629
|
{
|
|
557
|
-
subtask_id:
|
|
630
|
+
subtask_id: subtaskId,
|
|
558
631
|
enter_text: {
|
|
559
|
-
text: email,
|
|
632
|
+
text: credentials.email,
|
|
560
633
|
link: "next_link"
|
|
561
634
|
}
|
|
562
635
|
}
|
|
563
636
|
]
|
|
564
637
|
});
|
|
565
638
|
}
|
|
566
|
-
async handleSuccessSubtask(
|
|
639
|
+
async handleSuccessSubtask(_subtaskId, _prev, _credentials, api) {
|
|
567
640
|
return await this.executeFlowTask({
|
|
568
|
-
flow_token:
|
|
641
|
+
flow_token: api.getFlowToken(),
|
|
569
642
|
subtask_inputs: []
|
|
570
643
|
});
|
|
571
644
|
}
|
|
@@ -573,7 +646,9 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
573
646
|
const onboardingTaskUrl = "https://api.twitter.com/1.1/onboarding/task.json";
|
|
574
647
|
const token = this.guestToken;
|
|
575
648
|
if (token == null) {
|
|
576
|
-
throw new
|
|
649
|
+
throw new AuthenticationError(
|
|
650
|
+
"Authentication token is null or undefined."
|
|
651
|
+
);
|
|
577
652
|
}
|
|
578
653
|
const headers = new headersPolyfill.Headers({
|
|
579
654
|
authorization: `Bearer ${this.bearerToken}`,
|
|
@@ -594,16 +669,19 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
594
669
|
});
|
|
595
670
|
await updateCookieJar(this.jar, res.headers);
|
|
596
671
|
if (!res.ok) {
|
|
597
|
-
return { status: "error", err:
|
|
672
|
+
return { status: "error", err: await ApiError.fromResponse(res) };
|
|
598
673
|
}
|
|
599
674
|
const flow = await res.json();
|
|
600
675
|
if (flow?.flow_token == null) {
|
|
601
|
-
return {
|
|
676
|
+
return {
|
|
677
|
+
status: "error",
|
|
678
|
+
err: new AuthenticationError("flow_token not found.")
|
|
679
|
+
};
|
|
602
680
|
}
|
|
603
681
|
if (flow.errors?.length) {
|
|
604
682
|
return {
|
|
605
683
|
status: "error",
|
|
606
|
-
err: new
|
|
684
|
+
err: new AuthenticationError(
|
|
607
685
|
`Authentication error (${flow.errors[0].code}): ${flow.errors[0].message}`
|
|
608
686
|
)
|
|
609
687
|
};
|
|
@@ -611,7 +689,7 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
611
689
|
if (typeof flow.flow_token !== "string") {
|
|
612
690
|
return {
|
|
613
691
|
status: "error",
|
|
614
|
-
err: new
|
|
692
|
+
err: new AuthenticationError("flow_token was not a string.")
|
|
615
693
|
};
|
|
616
694
|
}
|
|
617
695
|
const subtask = flow.subtasks?.length ? flow.subtasks[0] : void 0;
|
|
@@ -619,17 +697,73 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
619
697
|
if (subtask && subtask.subtask_id === "DenyLoginSubtask") {
|
|
620
698
|
return {
|
|
621
699
|
status: "error",
|
|
622
|
-
err: new
|
|
700
|
+
err: new AuthenticationError("Authentication error: DenyLoginSubtask")
|
|
623
701
|
};
|
|
624
702
|
}
|
|
625
703
|
return {
|
|
626
704
|
status: "success",
|
|
627
|
-
|
|
628
|
-
flowToken: flow.flow_token
|
|
705
|
+
response: flow
|
|
629
706
|
};
|
|
630
707
|
}
|
|
631
708
|
}
|
|
632
709
|
|
|
710
|
+
const endpoints = {
|
|
711
|
+
// TODO: Migrate other endpoint URLs here
|
|
712
|
+
UserTweets: "https://x.com/i/api/graphql/Li2XXGESVev94TzFtntrgA/UserTweets?variables=%7B%22userId%22%3A%221806359170830172162%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
713
|
+
UserTweetsAndReplies: "https://x.com/i/api/graphql/Hk4KlJ-ONjlJsucqR55P7g/UserTweetsAndReplies?variables=%7B%22userId%22%3A%221806359170830172162%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
714
|
+
UserLikedTweets: "https://x.com/i/api/graphql/XHTMjDbiTGLQ9cP1em-aqQ/Likes?variables=%7B%22userId%22%3A%222244196397%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withClientEventToken%22%3Afalse%2C%22withBirdwatchNotes%22%3Afalse%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
715
|
+
UserByScreenName: "https://x.com/i/api/graphql/xWw45l6nX7DP2FKRyePXSw/UserByScreenName?variables=%7B%22screen_name%22%3A%22geminiapp%22%7D&features=%7B%22hidden_profile_subscriptions_enabled%22%3Atrue%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_is_identity_verified_enabled%22%3Atrue%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22responsive_web_twitter_article_notes_tab_enabled%22%3Atrue%2C%22subscriptions_feature_can_gift_premium%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withAuxiliaryUserLabels%22%3Atrue%7D",
|
|
716
|
+
TweetDetail: "https://x.com/i/api/graphql/u5Tij6ERlSH2LZvCUqallw/TweetDetail?variables=%7B%22focalTweetId%22%3A%221924893675529900467%22%2C%22referrer%22%3A%22profile%22%2C%22with_rux_injections%22%3Afalse%2C%22rankingMode%22%3A%22Relevance%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Atrue%2C%22withArticlePlainText%22%3Afalse%2C%22withGrokAnalyze%22%3Afalse%2C%22withDisallowedReplyControls%22%3Afalse%7D",
|
|
717
|
+
TweetResultByRestId: "https://api.x.com/graphql/Opujkru5iJSDWj4DuJISOw/TweetResultByRestId?variables=%7B%22tweetId%22%3A%221924893675529900467%22%2C%22withCommunity%22%3Afalse%2C%22includePromotedContent%22%3Afalse%2C%22withVoice%22%3Afalse%7D&features=%7B%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Afalse%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Atrue%2C%22withArticlePlainText%22%3Afalse%2C%22withGrokAnalyze%22%3Afalse%2C%22withDisallowedReplyControls%22%3Afalse%7D",
|
|
718
|
+
ListTweets: "https://x.com/i/api/graphql/S1Sm3_mNJwa-fnY9htcaAQ/ListLatestTweetsTimeline?variables=%7B%22listId%22%3A%221736495155002106192%22%2C%22count%22%3A20%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D"
|
|
719
|
+
};
|
|
720
|
+
class ApiRequest {
|
|
721
|
+
constructor(info) {
|
|
722
|
+
this.url = info.url;
|
|
723
|
+
this.variables = info.variables;
|
|
724
|
+
this.features = info.features;
|
|
725
|
+
this.fieldToggles = info.fieldToggles;
|
|
726
|
+
}
|
|
727
|
+
toRequestUrl() {
|
|
728
|
+
const params = new URLSearchParams();
|
|
729
|
+
if (this.variables) {
|
|
730
|
+
params.set("variables", stringify(this.variables));
|
|
731
|
+
}
|
|
732
|
+
if (this.features) {
|
|
733
|
+
params.set("features", stringify(this.features));
|
|
734
|
+
}
|
|
735
|
+
if (this.fieldToggles) {
|
|
736
|
+
params.set("fieldToggles", stringify(this.fieldToggles));
|
|
737
|
+
}
|
|
738
|
+
return `${this.url}?${params.toString()}`;
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
function parseEndpointExample(example) {
|
|
742
|
+
const { protocol, host, pathname, searchParams: query } = new URL(example);
|
|
743
|
+
const base = `${protocol}//${host}${pathname}`;
|
|
744
|
+
const variables = query.get("variables");
|
|
745
|
+
const features = query.get("features");
|
|
746
|
+
const fieldToggles = query.get("fieldToggles");
|
|
747
|
+
return new ApiRequest({
|
|
748
|
+
url: base,
|
|
749
|
+
variables: variables ? JSON.parse(variables) : void 0,
|
|
750
|
+
features: features ? JSON.parse(features) : void 0,
|
|
751
|
+
fieldToggles: fieldToggles ? JSON.parse(fieldToggles) : void 0
|
|
752
|
+
});
|
|
753
|
+
}
|
|
754
|
+
function createApiRequestFactory(endpoints2) {
|
|
755
|
+
return Object.entries(endpoints2).map(([endpointName, endpointExample]) => {
|
|
756
|
+
return {
|
|
757
|
+
[`create${endpointName}Request`]: () => {
|
|
758
|
+
return parseEndpointExample(endpointExample);
|
|
759
|
+
}
|
|
760
|
+
};
|
|
761
|
+
}).reduce((agg, next) => {
|
|
762
|
+
return Object.assign(agg, next);
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
const apiRequestFactory = createApiRequestFactory(endpoints);
|
|
766
|
+
|
|
633
767
|
function getAvatarOriginalSizeUrl(avatarUrl) {
|
|
634
768
|
return avatarUrl ? avatarUrl.replace("_normal", "") : void 0;
|
|
635
769
|
}
|
|
@@ -666,35 +800,12 @@ function parseProfile(user, isBlueVerified) {
|
|
|
666
800
|
return profile;
|
|
667
801
|
}
|
|
668
802
|
async function getProfile(username, auth) {
|
|
669
|
-
const
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
})
|
|
676
|
-
);
|
|
677
|
-
params.set(
|
|
678
|
-
"features",
|
|
679
|
-
stringify({
|
|
680
|
-
hidden_profile_likes_enabled: false,
|
|
681
|
-
hidden_profile_subscriptions_enabled: false,
|
|
682
|
-
// Auth-restricted
|
|
683
|
-
responsive_web_graphql_exclude_directive_enabled: true,
|
|
684
|
-
verified_phone_label_enabled: false,
|
|
685
|
-
subscriptions_verification_info_is_identity_verified_enabled: false,
|
|
686
|
-
subscriptions_verification_info_verified_since_enabled: true,
|
|
687
|
-
highlights_tweets_tab_ui_enabled: true,
|
|
688
|
-
creator_subscriptions_tweet_preview_api_enabled: true,
|
|
689
|
-
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
|
690
|
-
responsive_web_graphql_timeline_navigation_enabled: true
|
|
691
|
-
})
|
|
692
|
-
);
|
|
693
|
-
params.set("fieldToggles", stringify({ withAuxiliaryUserLabels: false }));
|
|
694
|
-
const res = await requestApi(
|
|
695
|
-
`https://twitter.com/i/api/graphql/G3KGOASz96M-Qu0nwmGXNg/UserByScreenName?${params.toString()}`,
|
|
696
|
-
auth
|
|
697
|
-
);
|
|
803
|
+
const request = apiRequestFactory.createUserByScreenNameRequest();
|
|
804
|
+
request.variables.screen_name = username;
|
|
805
|
+
request.variables.withSafetyModeUserFields = true;
|
|
806
|
+
request.features.hidden_profile_subscriptions_enabled = false;
|
|
807
|
+
request.fieldToggles.withAuxiliaryUserLabels = false;
|
|
808
|
+
const res = await requestApi(request.toRequestUrl(), auth);
|
|
698
809
|
if (!res.success) {
|
|
699
810
|
return res;
|
|
700
811
|
}
|
|
@@ -778,6 +889,7 @@ async function* getUserTimeline(query, maxProfiles, fetchFunc) {
|
|
|
778
889
|
nProfiles++;
|
|
779
890
|
}
|
|
780
891
|
if (!next) break;
|
|
892
|
+
await jitter(1e3);
|
|
781
893
|
}
|
|
782
894
|
}
|
|
783
895
|
async function* getTweetTimeline(query, maxTweets, fetchFunc) {
|
|
@@ -802,6 +914,7 @@ async function* getTweetTimeline(query, maxTweets, fetchFunc) {
|
|
|
802
914
|
}
|
|
803
915
|
nTweets++;
|
|
804
916
|
}
|
|
917
|
+
await jitter(1e3);
|
|
805
918
|
}
|
|
806
919
|
}
|
|
807
920
|
|
|
@@ -924,7 +1037,7 @@ function getLegacyTweetId(tweet) {
|
|
|
924
1037
|
}
|
|
925
1038
|
return tweet.conversation_id_str;
|
|
926
1039
|
}
|
|
927
|
-
function parseLegacyTweet(user, tweet, editControl) {
|
|
1040
|
+
function parseLegacyTweet(coreUser, user, tweet, editControl) {
|
|
928
1041
|
if (tweet == null) {
|
|
929
1042
|
return {
|
|
930
1043
|
success: false,
|
|
@@ -954,6 +1067,8 @@ function parseLegacyTweet(user, tweet, editControl) {
|
|
|
954
1067
|
const { photos, videos, sensitiveContent } = parseMediaGroups(media);
|
|
955
1068
|
const tweetVersions = editControl?.edit_tweet_ids ?? [tweetId];
|
|
956
1069
|
const editIds = tweetVersions.filter((id) => id !== tweetId);
|
|
1070
|
+
const name = user.name ?? coreUser?.name;
|
|
1071
|
+
const username = user.screen_name ?? coreUser?.screen_name;
|
|
957
1072
|
const tw = {
|
|
958
1073
|
__raw_UNSTABLE: tweet,
|
|
959
1074
|
bookmarkCount: tweet.bookmark_count,
|
|
@@ -966,8 +1081,8 @@ function parseLegacyTweet(user, tweet, editControl) {
|
|
|
966
1081
|
username: mention.screen_name,
|
|
967
1082
|
name: mention.name
|
|
968
1083
|
})),
|
|
969
|
-
name
|
|
970
|
-
permanentUrl: `https://twitter.com/${
|
|
1084
|
+
name,
|
|
1085
|
+
permanentUrl: `https://twitter.com/${username}/status/${tweetId}`,
|
|
971
1086
|
photos,
|
|
972
1087
|
replies: tweet.reply_count,
|
|
973
1088
|
retweets: tweet.retweet_count,
|
|
@@ -975,7 +1090,7 @@ function parseLegacyTweet(user, tweet, editControl) {
|
|
|
975
1090
|
thread: [],
|
|
976
1091
|
urls: urls.filter(isFieldDefined("expanded_url")).map((url) => url.expanded_url),
|
|
977
1092
|
userId: tweet.user_id_str,
|
|
978
|
-
username
|
|
1093
|
+
username,
|
|
979
1094
|
videos,
|
|
980
1095
|
isQuoted: false,
|
|
981
1096
|
isReply: false,
|
|
@@ -1009,6 +1124,7 @@ function parseLegacyTweet(user, tweet, editControl) {
|
|
|
1009
1124
|
tw.retweetedStatusId = retweetedStatusIdStr;
|
|
1010
1125
|
if (retweetedStatusResult) {
|
|
1011
1126
|
const parsedResult = parseLegacyTweet(
|
|
1127
|
+
retweetedStatusResult?.core?.user_results?.result?.core,
|
|
1012
1128
|
retweetedStatusResult?.core?.user_results?.result?.legacy,
|
|
1013
1129
|
retweetedStatusResult?.legacy
|
|
1014
1130
|
);
|
|
@@ -1036,6 +1152,7 @@ function parseResult(result) {
|
|
|
1036
1152
|
result.legacy.full_text = noteTweetResultText;
|
|
1037
1153
|
}
|
|
1038
1154
|
const tweetResult = parseLegacyTweet(
|
|
1155
|
+
result?.core?.user_results?.result?.core,
|
|
1039
1156
|
result?.core?.user_results?.result?.legacy,
|
|
1040
1157
|
result?.legacy
|
|
1041
1158
|
);
|
|
@@ -1186,6 +1303,7 @@ function parseSearchTimelineTweets(timeline) {
|
|
|
1186
1303
|
if (itemContent?.tweetDisplayType === "Tweet") {
|
|
1187
1304
|
const tweetResultRaw = itemContent.tweet_results?.result;
|
|
1188
1305
|
const tweetResult = parseLegacyTweet(
|
|
1306
|
+
tweetResultRaw?.core?.user_results?.result?.core,
|
|
1189
1307
|
tweetResultRaw?.core?.user_results?.result?.legacy,
|
|
1190
1308
|
tweetResultRaw?.legacy,
|
|
1191
1309
|
tweetResultRaw?.edit_control?.edit_control_initial
|
|
@@ -1288,8 +1406,8 @@ async function fetchSearchProfiles(query, maxProfiles, auth, cursor) {
|
|
|
1288
1406
|
return parseSearchTimelineUsers(timeline);
|
|
1289
1407
|
}
|
|
1290
1408
|
async function getSearchTimeline(query, maxItems, searchMode, auth, cursor) {
|
|
1291
|
-
if (!auth.isLoggedIn()) {
|
|
1292
|
-
throw new
|
|
1409
|
+
if (!await auth.isLoggedIn()) {
|
|
1410
|
+
throw new AuthenticationError("Scraper is not logged-in for search.");
|
|
1293
1411
|
}
|
|
1294
1412
|
if (maxItems > 50) {
|
|
1295
1413
|
maxItems = 50;
|
|
@@ -1396,6 +1514,11 @@ function getFollowers(userId, maxProfiles, auth) {
|
|
|
1396
1514
|
});
|
|
1397
1515
|
}
|
|
1398
1516
|
async function fetchProfileFollowing(userId, maxProfiles, auth, cursor) {
|
|
1517
|
+
if (!await auth.isLoggedIn()) {
|
|
1518
|
+
throw new AuthenticationError(
|
|
1519
|
+
"Scraper is not logged-in for profile following."
|
|
1520
|
+
);
|
|
1521
|
+
}
|
|
1399
1522
|
const timeline = await getFollowingTimeline(
|
|
1400
1523
|
userId,
|
|
1401
1524
|
maxProfiles,
|
|
@@ -1405,6 +1528,11 @@ async function fetchProfileFollowing(userId, maxProfiles, auth, cursor) {
|
|
|
1405
1528
|
return parseRelationshipTimeline(timeline);
|
|
1406
1529
|
}
|
|
1407
1530
|
async function fetchProfileFollowers(userId, maxProfiles, auth, cursor) {
|
|
1531
|
+
if (!await auth.isLoggedIn()) {
|
|
1532
|
+
throw new AuthenticationError(
|
|
1533
|
+
"Scraper is not logged-in for profile followers."
|
|
1534
|
+
);
|
|
1535
|
+
}
|
|
1408
1536
|
const timeline = await getFollowersTimeline(
|
|
1409
1537
|
userId,
|
|
1410
1538
|
maxProfiles,
|
|
@@ -1415,7 +1543,9 @@ async function fetchProfileFollowers(userId, maxProfiles, auth, cursor) {
|
|
|
1415
1543
|
}
|
|
1416
1544
|
async function getFollowingTimeline(userId, maxItems, auth, cursor) {
|
|
1417
1545
|
if (!auth.isLoggedIn()) {
|
|
1418
|
-
throw new
|
|
1546
|
+
throw new AuthenticationError(
|
|
1547
|
+
"Scraper is not logged-in for profile following."
|
|
1548
|
+
);
|
|
1419
1549
|
}
|
|
1420
1550
|
if (maxItems > 50) {
|
|
1421
1551
|
maxItems = 50;
|
|
@@ -1448,7 +1578,9 @@ async function getFollowingTimeline(userId, maxItems, auth, cursor) {
|
|
|
1448
1578
|
}
|
|
1449
1579
|
async function getFollowersTimeline(userId, maxItems, auth, cursor) {
|
|
1450
1580
|
if (!auth.isLoggedIn()) {
|
|
1451
|
-
throw new
|
|
1581
|
+
throw new AuthenticationError(
|
|
1582
|
+
"Scraper is not logged-in for profile followers."
|
|
1583
|
+
);
|
|
1452
1584
|
}
|
|
1453
1585
|
if (maxItems > 50) {
|
|
1454
1586
|
maxItems = 50;
|
|
@@ -1513,62 +1645,6 @@ async function getTrends(auth) {
|
|
|
1513
1645
|
return trends;
|
|
1514
1646
|
}
|
|
1515
1647
|
|
|
1516
|
-
const endpoints = {
|
|
1517
|
-
// TODO: Migrate other endpoint URLs here
|
|
1518
|
-
UserTweets: "https://twitter.com/i/api/graphql/V7H0Ap3_Hh2FyS75OCDO3Q/UserTweets?variables=%7B%22userId%22%3A%224020276615%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
1519
|
-
UserTweetsAndReplies: "https://twitter.com/i/api/graphql/E4wA5vo2sjVyvpliUffSCw/UserTweetsAndReplies?variables=%7B%22userId%22%3A%224020276615%22%2C%22count%22%3A40%2C%22cursor%22%3A%22DAABCgABGPWl-F-ATiIKAAIY9YfiF1rRAggAAwAAAAEAAA%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
1520
|
-
UserLikedTweets: "https://twitter.com/i/api/graphql/eSSNbhECHHWWALkkQq-YTA/Likes?variables=%7B%22userId%22%3A%222244196397%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withClientEventToken%22%3Afalse%2C%22withBirdwatchNotes%22%3Afalse%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D",
|
|
1521
|
-
TweetDetail: "https://twitter.com/i/api/graphql/xOhkmRac04YFZmOzU9PJHg/TweetDetail?variables=%7B%22focalTweetId%22%3A%221237110546383724547%22%2C%22with_rux_injections%22%3Afalse%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withV2Timeline%22%3Atrue%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Afalse%7D",
|
|
1522
|
-
TweetResultByRestId: "https://twitter.com/i/api/graphql/DJS3BdhUhcaEpZ7B7irJDg/TweetResultByRestId?variables=%7B%22tweetId%22%3A%221237110546383724547%22%2C%22withCommunity%22%3Afalse%2C%22includePromotedContent%22%3Afalse%2C%22withVoice%22%3Afalse%7D&features=%7B%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D",
|
|
1523
|
-
ListTweets: "https://twitter.com/i/api/graphql/whF0_KH1fCkdLLoyNPMoEw/ListLatestTweetsTimeline?variables=%7B%22listId%22%3A%221736495155002106192%22%2C%22count%22%3A20%7D&features=%7B%22responsive_web_graphql_exclude_directive_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22tweetypie_unmention_optimization_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Afalse%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22rweb_video_timestamps_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_media_download_video_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D"
|
|
1524
|
-
};
|
|
1525
|
-
class ApiRequest {
|
|
1526
|
-
constructor(info) {
|
|
1527
|
-
this.url = info.url;
|
|
1528
|
-
this.variables = info.variables;
|
|
1529
|
-
this.features = info.features;
|
|
1530
|
-
this.fieldToggles = info.fieldToggles;
|
|
1531
|
-
}
|
|
1532
|
-
toRequestUrl() {
|
|
1533
|
-
const params = new URLSearchParams();
|
|
1534
|
-
if (this.variables) {
|
|
1535
|
-
params.set("variables", stringify(this.variables));
|
|
1536
|
-
}
|
|
1537
|
-
if (this.features) {
|
|
1538
|
-
params.set("features", stringify(this.features));
|
|
1539
|
-
}
|
|
1540
|
-
if (this.fieldToggles) {
|
|
1541
|
-
params.set("fieldToggles", stringify(this.fieldToggles));
|
|
1542
|
-
}
|
|
1543
|
-
return `${this.url}?${params.toString()}`;
|
|
1544
|
-
}
|
|
1545
|
-
}
|
|
1546
|
-
function parseEndpointExample(example) {
|
|
1547
|
-
const { protocol, host, pathname, searchParams: query } = new URL(example);
|
|
1548
|
-
const base = `${protocol}//${host}${pathname}`;
|
|
1549
|
-
const variables = query.get("variables");
|
|
1550
|
-
const features = query.get("features");
|
|
1551
|
-
const fieldToggles = query.get("fieldToggles");
|
|
1552
|
-
return new ApiRequest({
|
|
1553
|
-
url: base,
|
|
1554
|
-
variables: variables ? JSON.parse(variables) : void 0,
|
|
1555
|
-
features: features ? JSON.parse(features) : void 0,
|
|
1556
|
-
fieldToggles: fieldToggles ? JSON.parse(fieldToggles) : void 0
|
|
1557
|
-
});
|
|
1558
|
-
}
|
|
1559
|
-
function createApiRequestFactory(endpoints2) {
|
|
1560
|
-
return Object.entries(endpoints2).map(([endpointName, endpointExample]) => {
|
|
1561
|
-
return {
|
|
1562
|
-
[`create${endpointName}Request`]: () => {
|
|
1563
|
-
return parseEndpointExample(endpointExample);
|
|
1564
|
-
}
|
|
1565
|
-
};
|
|
1566
|
-
}).reduce((agg, next) => {
|
|
1567
|
-
return Object.assign(agg, next);
|
|
1568
|
-
});
|
|
1569
|
-
}
|
|
1570
|
-
const apiRequestFactory = createApiRequestFactory(endpoints);
|
|
1571
|
-
|
|
1572
1648
|
function parseListTimelineTweets(timeline) {
|
|
1573
1649
|
let bottomCursor;
|
|
1574
1650
|
let topCursor;
|
|
@@ -1705,8 +1781,10 @@ function getTweetsAndRepliesByUserId(userId, maxTweets, auth) {
|
|
|
1705
1781
|
});
|
|
1706
1782
|
}
|
|
1707
1783
|
async function fetchLikedTweets(userId, maxTweets, cursor, auth) {
|
|
1708
|
-
if (!auth.isLoggedIn()) {
|
|
1709
|
-
throw new
|
|
1784
|
+
if (!await auth.isLoggedIn()) {
|
|
1785
|
+
throw new AuthenticationError(
|
|
1786
|
+
"Scraper is not logged-in for fetching liked tweets."
|
|
1787
|
+
);
|
|
1710
1788
|
}
|
|
1711
1789
|
if (maxTweets > 200) {
|
|
1712
1790
|
maxTweets = 200;
|
|
@@ -1811,6 +1889,20 @@ class Scraper {
|
|
|
1811
1889
|
this.token = bearerToken;
|
|
1812
1890
|
this.useGuestAuth();
|
|
1813
1891
|
}
|
|
1892
|
+
/**
|
|
1893
|
+
* Registers a subtask handler for the given subtask ID. This
|
|
1894
|
+
* will override any existing handler for the same subtask.
|
|
1895
|
+
* @param subtaskId The ID of the subtask to register the handler for.
|
|
1896
|
+
* @param subtaskHandler The handler function to register.
|
|
1897
|
+
*/
|
|
1898
|
+
registerAuthSubtaskHandler(subtaskId, subtaskHandler) {
|
|
1899
|
+
if (this.auth instanceof TwitterUserAuth) {
|
|
1900
|
+
this.auth.registerSubtaskHandler(subtaskId, subtaskHandler);
|
|
1901
|
+
}
|
|
1902
|
+
if (this.authTrends instanceof TwitterUserAuth) {
|
|
1903
|
+
this.authTrends.registerSubtaskHandler(subtaskId, subtaskHandler);
|
|
1904
|
+
}
|
|
1905
|
+
}
|
|
1814
1906
|
/**
|
|
1815
1907
|
* Initializes auth properties using a guest token.
|
|
1816
1908
|
* Used when creating a new instance of this class, and when logging out.
|
|
@@ -2145,6 +2237,7 @@ class Scraper {
|
|
|
2145
2237
|
}
|
|
2146
2238
|
|
|
2147
2239
|
exports.ApiError = ApiError;
|
|
2240
|
+
exports.AuthenticationError = AuthenticationError;
|
|
2148
2241
|
exports.ErrorRateLimitStrategy = ErrorRateLimitStrategy;
|
|
2149
2242
|
exports.Scraper = Scraper;
|
|
2150
2243
|
exports.SearchMode = SearchMode;
|