@the-convocation/twitter-scraper 0.21.1 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -0
- package/dist/cycletls/cjs/index.cjs +48 -3
- package/dist/cycletls/cjs/index.cjs.map +1 -1
- package/dist/cycletls/esm/index.mjs +48 -3
- package/dist/cycletls/esm/index.mjs.map +1 -1
- package/dist/default/cjs/index.js +1320 -154
- package/dist/default/cjs/index.js.map +1 -1
- package/dist/default/esm/index.mjs +1320 -155
- package/dist/default/esm/index.mjs.map +1 -1
- package/dist/node/cjs/index.cjs +1320 -154
- package/dist/node/cjs/index.cjs.map +1 -1
- package/dist/node/esm/index.mjs +1320 -155
- package/dist/node/esm/index.mjs.map +1 -1
- package/dist/types/index.d.ts +70 -1
- package/package.json +4 -2
|
@@ -70,13 +70,13 @@ class AuthenticationError extends Error {
|
|
|
70
70
|
}
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
const log$
|
|
73
|
+
const log$8 = debug("twitter-scraper:rate-limit");
|
|
74
74
|
class WaitingRateLimitStrategy {
|
|
75
75
|
async onRateLimit({ response: res }) {
|
|
76
76
|
const xRateLimitLimit = res.headers.get("x-rate-limit-limit");
|
|
77
77
|
const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
|
|
78
78
|
const xRateLimitReset = res.headers.get("x-rate-limit-reset");
|
|
79
|
-
log$
|
|
79
|
+
log$8(
|
|
80
80
|
`Rate limit event: limit=${xRateLimitLimit}, remaining=${xRateLimitRemaining}, reset=${xRateLimitReset}`
|
|
81
81
|
);
|
|
82
82
|
if (xRateLimitRemaining == "0" && xRateLimitReset) {
|
|
@@ -92,7 +92,794 @@ class ErrorRateLimitStrategy {
|
|
|
92
92
|
}
|
|
93
93
|
}
|
|
94
94
|
|
|
95
|
-
const log$
|
|
95
|
+
const log$7 = debug("twitter-scraper:castle");
|
|
96
|
+
var FieldEncoding = /* @__PURE__ */ ((FieldEncoding2) => {
|
|
97
|
+
FieldEncoding2[FieldEncoding2["Empty"] = -1] = "Empty";
|
|
98
|
+
FieldEncoding2[FieldEncoding2["Marker"] = 1] = "Marker";
|
|
99
|
+
FieldEncoding2[FieldEncoding2["Byte"] = 3] = "Byte";
|
|
100
|
+
FieldEncoding2[FieldEncoding2["EncryptedBytes"] = 4] = "EncryptedBytes";
|
|
101
|
+
FieldEncoding2[FieldEncoding2["CompactInt"] = 5] = "CompactInt";
|
|
102
|
+
FieldEncoding2[FieldEncoding2["RoundedByte"] = 6] = "RoundedByte";
|
|
103
|
+
FieldEncoding2[FieldEncoding2["RawAppend"] = 7] = "RawAppend";
|
|
104
|
+
return FieldEncoding2;
|
|
105
|
+
})(FieldEncoding || {});
|
|
106
|
+
const TWITTER_CASTLE_PK = "AvRa79bHyJSYSQHnRpcVtzyxetSvFerx";
|
|
107
|
+
const XXTEA_KEY = [1164413191, 3891440048, 185273099, 2746598870];
|
|
108
|
+
const PER_FIELD_KEY_TAIL = [
|
|
109
|
+
16373134,
|
|
110
|
+
643144773,
|
|
111
|
+
1762804430,
|
|
112
|
+
1186572681,
|
|
113
|
+
1164413191
|
|
114
|
+
];
|
|
115
|
+
const TS_EPOCH = 1535e6;
|
|
116
|
+
const SDK_VERSION = 27008;
|
|
117
|
+
const TOKEN_VERSION = 11;
|
|
118
|
+
const FP_PART = {
|
|
119
|
+
DEVICE: 0,
|
|
120
|
+
// Part 1: hardware/OS/rendering fingerprint
|
|
121
|
+
BROWSER: 4,
|
|
122
|
+
// Part 2: browser environment fingerprint
|
|
123
|
+
TIMING: 7
|
|
124
|
+
// Part 3: timing-based fingerprint
|
|
125
|
+
};
|
|
126
|
+
const DEFAULT_PROFILE = {
|
|
127
|
+
locale: "en-US",
|
|
128
|
+
language: "en",
|
|
129
|
+
timezone: "America/New_York",
|
|
130
|
+
screenWidth: 1920,
|
|
131
|
+
screenHeight: 1080,
|
|
132
|
+
availableWidth: 1920,
|
|
133
|
+
availableHeight: 1032,
|
|
134
|
+
// 1080 minus Windows taskbar (~48px)
|
|
135
|
+
gpuRenderer: "ANGLE (NVIDIA, NVIDIA GeForce GTX 1080 Ti Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
136
|
+
deviceMemoryGB: 8,
|
|
137
|
+
hardwareConcurrency: 24,
|
|
138
|
+
colorDepth: 24,
|
|
139
|
+
devicePixelRatio: 1
|
|
140
|
+
};
|
|
141
|
+
const SCREEN_RESOLUTIONS = [
|
|
142
|
+
{ w: 1920, h: 1080, ah: 1032 },
|
|
143
|
+
{ w: 2560, h: 1440, ah: 1392 },
|
|
144
|
+
{ w: 1366, h: 768, ah: 720 },
|
|
145
|
+
{ w: 1536, h: 864, ah: 816 },
|
|
146
|
+
{ w: 1440, h: 900, ah: 852 },
|
|
147
|
+
{ w: 1680, h: 1050, ah: 1002 },
|
|
148
|
+
{ w: 3840, h: 2160, ah: 2112 }
|
|
149
|
+
];
|
|
150
|
+
const DEVICE_MEMORY_VALUES = [4, 8, 8, 16];
|
|
151
|
+
const HARDWARE_CONCURRENCY_VALUES = [4, 8, 8, 12, 16, 24];
|
|
152
|
+
function randomizeBrowserProfile() {
|
|
153
|
+
const screen = SCREEN_RESOLUTIONS[randInt(0, SCREEN_RESOLUTIONS.length - 1)];
|
|
154
|
+
return {
|
|
155
|
+
...DEFAULT_PROFILE,
|
|
156
|
+
screenWidth: screen.w,
|
|
157
|
+
screenHeight: screen.h,
|
|
158
|
+
availableWidth: screen.w,
|
|
159
|
+
availableHeight: screen.ah,
|
|
160
|
+
// gpuRenderer intentionally NOT randomized — see JSDoc above
|
|
161
|
+
deviceMemoryGB: DEVICE_MEMORY_VALUES[randInt(0, DEVICE_MEMORY_VALUES.length - 1)],
|
|
162
|
+
hardwareConcurrency: HARDWARE_CONCURRENCY_VALUES[randInt(0, HARDWARE_CONCURRENCY_VALUES.length - 1)]
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
function getRandomBytes(n) {
|
|
166
|
+
const buf = new Uint8Array(n);
|
|
167
|
+
if (typeof globalThis.crypto !== "undefined" && globalThis.crypto.getRandomValues) {
|
|
168
|
+
globalThis.crypto.getRandomValues(buf);
|
|
169
|
+
} else {
|
|
170
|
+
for (let i = 0; i < n; i++) buf[i] = Math.floor(Math.random() * 256);
|
|
171
|
+
}
|
|
172
|
+
return buf;
|
|
173
|
+
}
|
|
174
|
+
function randInt(min, max) {
|
|
175
|
+
return min + Math.floor(Math.random() * (max - min + 1));
|
|
176
|
+
}
|
|
177
|
+
function randFloat(min, max) {
|
|
178
|
+
return min + Math.random() * (max - min);
|
|
179
|
+
}
|
|
180
|
+
function concat(...arrays) {
|
|
181
|
+
const len = arrays.reduce((s, a) => s + a.length, 0);
|
|
182
|
+
const out = new Uint8Array(len);
|
|
183
|
+
let off = 0;
|
|
184
|
+
for (const a of arrays) {
|
|
185
|
+
out.set(a, off);
|
|
186
|
+
off += a.length;
|
|
187
|
+
}
|
|
188
|
+
return out;
|
|
189
|
+
}
|
|
190
|
+
function toHex(input) {
|
|
191
|
+
return Array.from(input).map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
192
|
+
}
|
|
193
|
+
function fromHex(hex) {
|
|
194
|
+
const out = new Uint8Array(hex.length / 2);
|
|
195
|
+
for (let i = 0; i < hex.length; i += 2)
|
|
196
|
+
out[i / 2] = parseInt(hex.substring(i, i + 2), 16);
|
|
197
|
+
return out;
|
|
198
|
+
}
|
|
199
|
+
function textEnc(s) {
|
|
200
|
+
return new TextEncoder().encode(s);
|
|
201
|
+
}
|
|
202
|
+
function u8(...vals) {
|
|
203
|
+
return new Uint8Array(vals);
|
|
204
|
+
}
|
|
205
|
+
function be16(v) {
|
|
206
|
+
return u8(v >>> 8 & 255, v & 255);
|
|
207
|
+
}
|
|
208
|
+
function be32(v) {
|
|
209
|
+
return u8(v >>> 24 & 255, v >>> 16 & 255, v >>> 8 & 255, v & 255);
|
|
210
|
+
}
|
|
211
|
+
function xorBytes(data, key) {
|
|
212
|
+
const out = new Uint8Array(data.length);
|
|
213
|
+
for (let i = 0; i < data.length; i++) out[i] = data[i] ^ key[i % key.length];
|
|
214
|
+
return out;
|
|
215
|
+
}
|
|
216
|
+
function xorNibbles(nibbles, keyNibble) {
|
|
217
|
+
const k = parseInt(keyNibble, 16);
|
|
218
|
+
return nibbles.split("").map((n) => (parseInt(n, 16) ^ k).toString(16)).join("");
|
|
219
|
+
}
|
|
220
|
+
function base64url(data) {
|
|
221
|
+
if (typeof Buffer !== "undefined") {
|
|
222
|
+
return Buffer.from(data).toString("base64url");
|
|
223
|
+
}
|
|
224
|
+
let bin = "";
|
|
225
|
+
for (let i = 0; i < data.length; i++) bin += String.fromCharCode(data[i]);
|
|
226
|
+
return btoa(bin).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
|
227
|
+
}
|
|
228
|
+
function xxteaEncrypt(data, key) {
|
|
229
|
+
const padLen = Math.ceil(data.length / 4) * 4;
|
|
230
|
+
const padded = new Uint8Array(padLen);
|
|
231
|
+
padded.set(data);
|
|
232
|
+
const n = padLen / 4;
|
|
233
|
+
const v = new Uint32Array(n);
|
|
234
|
+
for (let i = 0; i < n; i++) {
|
|
235
|
+
v[i] = (padded[i * 4] | padded[i * 4 + 1] << 8 | padded[i * 4 + 2] << 16 | padded[i * 4 + 3] << 24) >>> 0;
|
|
236
|
+
}
|
|
237
|
+
if (n <= 1) return padded;
|
|
238
|
+
const k = new Uint32Array(key.map((x) => x >>> 0));
|
|
239
|
+
const DELTA = 2654435769;
|
|
240
|
+
const u = n - 1;
|
|
241
|
+
let sum = 0;
|
|
242
|
+
let z = v[u];
|
|
243
|
+
let y;
|
|
244
|
+
let rounds = 6 + Math.floor(52 / (u + 1));
|
|
245
|
+
while (rounds-- > 0) {
|
|
246
|
+
sum = sum + DELTA >>> 0;
|
|
247
|
+
const e = sum >>> 2 & 3;
|
|
248
|
+
for (let p = 0; p < u; p++) {
|
|
249
|
+
y = v[p + 1];
|
|
250
|
+
const mx2 = ((z >>> 5 ^ y << 2) >>> 0) + ((y >>> 3 ^ z << 4) >>> 0) ^ ((sum ^ y) >>> 0) + ((k[p & 3 ^ e] ^ z) >>> 0);
|
|
251
|
+
v[p] = v[p] + mx2 >>> 0;
|
|
252
|
+
z = v[p];
|
|
253
|
+
}
|
|
254
|
+
y = v[0];
|
|
255
|
+
const mx = ((z >>> 5 ^ y << 2) >>> 0) + ((y >>> 3 ^ z << 4) >>> 0) ^ ((sum ^ y) >>> 0) + ((k[u & 3 ^ e] ^ z) >>> 0);
|
|
256
|
+
v[u] = v[u] + mx >>> 0;
|
|
257
|
+
z = v[u];
|
|
258
|
+
}
|
|
259
|
+
const out = new Uint8Array(n * 4);
|
|
260
|
+
for (let i = 0; i < n; i++) {
|
|
261
|
+
out[i * 4] = v[i] & 255;
|
|
262
|
+
out[i * 4 + 1] = v[i] >>> 8 & 255;
|
|
263
|
+
out[i * 4 + 2] = v[i] >>> 16 & 255;
|
|
264
|
+
out[i * 4 + 3] = v[i] >>> 24 & 255;
|
|
265
|
+
}
|
|
266
|
+
return out;
|
|
267
|
+
}
|
|
268
|
+
function fieldEncrypt(data, fieldIndex, initTime) {
|
|
269
|
+
return xxteaEncrypt(data, [
|
|
270
|
+
fieldIndex,
|
|
271
|
+
Math.floor(initTime),
|
|
272
|
+
...PER_FIELD_KEY_TAIL
|
|
273
|
+
]);
|
|
274
|
+
}
|
|
275
|
+
function encodeTimestampBytes(ms) {
|
|
276
|
+
let t = Math.floor(ms / 1e3 - TS_EPOCH);
|
|
277
|
+
t = Math.max(Math.min(t, 268435455), 0);
|
|
278
|
+
return be32(t);
|
|
279
|
+
}
|
|
280
|
+
function xorAndAppendKey(buf, key) {
|
|
281
|
+
const hex = toHex(buf);
|
|
282
|
+
const keyNib = (key & 15).toString(16);
|
|
283
|
+
return xorNibbles(hex.substring(1), keyNib) + keyNib;
|
|
284
|
+
}
|
|
285
|
+
function encodeTimestampEncrypted(ms) {
|
|
286
|
+
const tsBytes = encodeTimestampBytes(ms);
|
|
287
|
+
const slice = Math.floor(ms) % 1e3;
|
|
288
|
+
const sliceBytes = be16(slice);
|
|
289
|
+
const k = randInt(0, 15);
|
|
290
|
+
return xorAndAppendKey(tsBytes, k) + xorAndAppendKey(sliceBytes, k);
|
|
291
|
+
}
|
|
292
|
+
function deriveAndXor(keyHex, sliceLen, rotChar, data) {
|
|
293
|
+
const sub = keyHex.substring(0, sliceLen).split("");
|
|
294
|
+
if (sub.length === 0) return data;
|
|
295
|
+
const rot = parseInt(rotChar, 16) % sub.length;
|
|
296
|
+
const rotated = sub.slice(rot).concat(sub.slice(0, rot)).join("");
|
|
297
|
+
return xorBytes(data, fromHex(rotated));
|
|
298
|
+
}
|
|
299
|
+
function customFloatEncode(expBits, manBits, value) {
|
|
300
|
+
if (value === 0) return 0;
|
|
301
|
+
let n = Math.abs(value);
|
|
302
|
+
let exp = 0;
|
|
303
|
+
while (2 <= n) {
|
|
304
|
+
n /= 2;
|
|
305
|
+
exp++;
|
|
306
|
+
}
|
|
307
|
+
while (n < 1 && n > 0) {
|
|
308
|
+
n *= 2;
|
|
309
|
+
exp--;
|
|
310
|
+
}
|
|
311
|
+
exp = Math.min(exp, (1 << expBits) - 1);
|
|
312
|
+
const frac = n - Math.floor(n);
|
|
313
|
+
let mantissa = 0;
|
|
314
|
+
if (frac > 0) {
|
|
315
|
+
let pos = 1;
|
|
316
|
+
let tmp = frac;
|
|
317
|
+
while (tmp !== 0 && pos <= manBits) {
|
|
318
|
+
tmp *= 2;
|
|
319
|
+
const bit = Math.floor(tmp);
|
|
320
|
+
mantissa |= bit << manBits - pos;
|
|
321
|
+
tmp -= bit;
|
|
322
|
+
pos++;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
return exp << manBits | mantissa;
|
|
326
|
+
}
|
|
327
|
+
function encodeFloatVal(v) {
|
|
328
|
+
const n = Math.max(v, 0);
|
|
329
|
+
if (n <= 15) return 64 | customFloatEncode(2, 4, n + 1);
|
|
330
|
+
return 128 | customFloatEncode(4, 3, n - 14);
|
|
331
|
+
}
|
|
332
|
+
function encodeField(index, encoding, val, initTime) {
|
|
333
|
+
const hdr = u8((31 & index) << 3 | 7 & encoding);
|
|
334
|
+
if (encoding === -1 /* Empty */ || encoding === 1 /* Marker */)
|
|
335
|
+
return hdr;
|
|
336
|
+
let body;
|
|
337
|
+
switch (encoding) {
|
|
338
|
+
case 3 /* Byte */:
|
|
339
|
+
body = u8(val);
|
|
340
|
+
break;
|
|
341
|
+
case 6 /* RoundedByte */:
|
|
342
|
+
body = u8(Math.round(val));
|
|
343
|
+
break;
|
|
344
|
+
case 5 /* CompactInt */: {
|
|
345
|
+
const v = val;
|
|
346
|
+
body = v <= 127 ? u8(v) : be16(1 << 15 | 32767 & v);
|
|
347
|
+
break;
|
|
348
|
+
}
|
|
349
|
+
case 4 /* EncryptedBytes */: {
|
|
350
|
+
if (initTime == null) {
|
|
351
|
+
throw new Error("initTime is required for EncryptedBytes encoding");
|
|
352
|
+
}
|
|
353
|
+
const enc = fieldEncrypt(val, index, initTime);
|
|
354
|
+
body = concat(u8(enc.length), enc);
|
|
355
|
+
break;
|
|
356
|
+
}
|
|
357
|
+
case 7 /* RawAppend */:
|
|
358
|
+
body = val instanceof Uint8Array ? val : u8(val);
|
|
359
|
+
break;
|
|
360
|
+
default:
|
|
361
|
+
body = new Uint8Array(0);
|
|
362
|
+
}
|
|
363
|
+
return concat(hdr, body);
|
|
364
|
+
}
|
|
365
|
+
function encodeBits(bits, byteSize) {
|
|
366
|
+
const numBytes = byteSize / 8;
|
|
367
|
+
const arr = new Uint8Array(numBytes);
|
|
368
|
+
for (const bit of bits) {
|
|
369
|
+
const bi = numBytes - 1 - Math.floor(bit / 8);
|
|
370
|
+
if (bi >= 0 && bi < numBytes) arr[bi] |= 1 << bit % 8;
|
|
371
|
+
}
|
|
372
|
+
return arr;
|
|
373
|
+
}
|
|
374
|
+
function screenDimBytes(screen, avail) {
|
|
375
|
+
const r = 32767 & screen;
|
|
376
|
+
const e = 65535 & avail;
|
|
377
|
+
return r === e ? be16(32768 | r) : concat(be16(r), be16(e));
|
|
378
|
+
}
|
|
379
|
+
function boolsToBin(arr, totalBits) {
|
|
380
|
+
const e = arr.length > totalBits ? arr.slice(0, totalBits) : arr;
|
|
381
|
+
const c = e.length;
|
|
382
|
+
let r = 0;
|
|
383
|
+
for (let i = c - 1; i >= 0; i--) {
|
|
384
|
+
if (e[i]) r |= 1 << c - i - 1;
|
|
385
|
+
}
|
|
386
|
+
if (c < totalBits) r <<= totalBits - c;
|
|
387
|
+
return r;
|
|
388
|
+
}
|
|
389
|
+
function encodeCodecPlayability() {
|
|
390
|
+
const codecs = {
|
|
391
|
+
webm: 2,
|
|
392
|
+
// VP8/VP9
|
|
393
|
+
mp4: 2,
|
|
394
|
+
// H.264
|
|
395
|
+
ogg: 0,
|
|
396
|
+
// Theora (Chrome dropped support)
|
|
397
|
+
aac: 2,
|
|
398
|
+
// AAC audio
|
|
399
|
+
xm4a: 1,
|
|
400
|
+
// M4A container
|
|
401
|
+
wav: 2,
|
|
402
|
+
// PCM audio
|
|
403
|
+
mpeg: 2,
|
|
404
|
+
// MP3 audio
|
|
405
|
+
ogg2: 2
|
|
406
|
+
// Vorbis audio
|
|
407
|
+
};
|
|
408
|
+
const bits = Object.values(codecs).map((c) => c.toString(2).padStart(2, "0")).join("");
|
|
409
|
+
return be16(parseInt(bits, 2));
|
|
410
|
+
}
|
|
411
|
+
const TIMEZONE_ENUM = {
|
|
412
|
+
"America/New_York": 0,
|
|
413
|
+
"America/Sao_Paulo": 1,
|
|
414
|
+
"America/Chicago": 2,
|
|
415
|
+
"America/Los_Angeles": 3,
|
|
416
|
+
"America/Mexico_City": 4,
|
|
417
|
+
"Asia/Shanghai": 5
|
|
418
|
+
};
|
|
419
|
+
function getTimezoneInfo(tz) {
|
|
420
|
+
const knownOffsets = {
|
|
421
|
+
"America/New_York": { offset: 20, dstDiff: 4 },
|
|
422
|
+
"America/Chicago": { offset: 24, dstDiff: 4 },
|
|
423
|
+
"America/Los_Angeles": { offset: 32, dstDiff: 4 },
|
|
424
|
+
"America/Denver": { offset: 28, dstDiff: 4 },
|
|
425
|
+
"America/Sao_Paulo": { offset: 12, dstDiff: 4 },
|
|
426
|
+
"America/Mexico_City": { offset: 24, dstDiff: 4 },
|
|
427
|
+
"Asia/Shanghai": { offset: 246, dstDiff: 0 },
|
|
428
|
+
"Asia/Tokyo": { offset: 220, dstDiff: 0 },
|
|
429
|
+
"Europe/London": { offset: 0, dstDiff: 4 },
|
|
430
|
+
"Europe/Berlin": { offset: 252, dstDiff: 4 },
|
|
431
|
+
UTC: { offset: 0, dstDiff: 0 }
|
|
432
|
+
};
|
|
433
|
+
try {
|
|
434
|
+
const now = /* @__PURE__ */ new Date();
|
|
435
|
+
const jan = new Date(now.getFullYear(), 0, 1);
|
|
436
|
+
const jul = new Date(now.getFullYear(), 6, 1);
|
|
437
|
+
const getOffset = (date, zone) => {
|
|
438
|
+
const utc = new Date(date.toLocaleString("en-US", { timeZone: "UTC" }));
|
|
439
|
+
const local = new Date(date.toLocaleString("en-US", { timeZone: zone }));
|
|
440
|
+
return (utc.getTime() - local.getTime()) / 6e4;
|
|
441
|
+
};
|
|
442
|
+
const currentOffset = getOffset(now, tz);
|
|
443
|
+
const janOffset = getOffset(jan, tz);
|
|
444
|
+
const julOffset = getOffset(jul, tz);
|
|
445
|
+
const dstDifference = Math.abs(janOffset - julOffset);
|
|
446
|
+
return {
|
|
447
|
+
offset: Math.floor(currentOffset / 15) & 255,
|
|
448
|
+
dstDiff: Math.floor(dstDifference / 15) & 255
|
|
449
|
+
};
|
|
450
|
+
} catch {
|
|
451
|
+
return knownOffsets[tz] || { offset: 20, dstDiff: 4 };
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
function buildDeviceFingerprint(initTime, profile, userAgent) {
|
|
455
|
+
const tz = getTimezoneInfo(profile.timezone);
|
|
456
|
+
const { Byte, EncryptedBytes, CompactInt, RoundedByte, RawAppend } = FieldEncoding;
|
|
457
|
+
const encryptedUA = fieldEncrypt(textEnc(userAgent), 12, initTime);
|
|
458
|
+
const uaPayload = concat(u8(1), u8(encryptedUA.length), encryptedUA);
|
|
459
|
+
const fields = [
|
|
460
|
+
encodeField(0, Byte, 1),
|
|
461
|
+
// Platform: Win32
|
|
462
|
+
encodeField(1, Byte, 0),
|
|
463
|
+
// Vendor: Google Inc.
|
|
464
|
+
encodeField(2, EncryptedBytes, textEnc(profile.locale), initTime),
|
|
465
|
+
// Locale
|
|
466
|
+
encodeField(3, RoundedByte, profile.deviceMemoryGB * 10),
|
|
467
|
+
// Device memory (GB * 10)
|
|
468
|
+
encodeField(
|
|
469
|
+
4,
|
|
470
|
+
RawAppend,
|
|
471
|
+
concat(
|
|
472
|
+
// Screen dimensions (width + height)
|
|
473
|
+
screenDimBytes(profile.screenWidth, profile.availableWidth),
|
|
474
|
+
screenDimBytes(profile.screenHeight, profile.availableHeight)
|
|
475
|
+
)
|
|
476
|
+
),
|
|
477
|
+
encodeField(5, CompactInt, profile.colorDepth),
|
|
478
|
+
// Screen color depth
|
|
479
|
+
encodeField(6, CompactInt, profile.hardwareConcurrency),
|
|
480
|
+
// CPU logical cores
|
|
481
|
+
encodeField(7, RoundedByte, profile.devicePixelRatio * 10),
|
|
482
|
+
// Pixel ratio (* 10)
|
|
483
|
+
encodeField(8, RawAppend, u8(tz.offset, tz.dstDiff)),
|
|
484
|
+
// Timezone offset info
|
|
485
|
+
// MIME type hash — captured from Chrome 144 on Windows 10.
|
|
486
|
+
// Source: yubie-re/castleio-gen (Python SDK, MIT license).
|
|
487
|
+
encodeField(9, RawAppend, u8(2, 125, 95, 201, 167)),
|
|
488
|
+
// Browser plugins hash — Chrome no longer exposes plugins to navigator.plugins,
|
|
489
|
+
// so this is a fixed hash. Source: yubie-re/castleio-gen (Python SDK, MIT license).
|
|
490
|
+
encodeField(10, RawAppend, u8(5, 114, 147, 2, 8)),
|
|
491
|
+
encodeField(
|
|
492
|
+
11,
|
|
493
|
+
RawAppend,
|
|
494
|
+
// Browser feature flags
|
|
495
|
+
concat(u8(12), encodeBits([0, 1, 2, 3, 4, 5, 6], 16))
|
|
496
|
+
),
|
|
497
|
+
encodeField(12, RawAppend, uaPayload),
|
|
498
|
+
// User agent (encrypted)
|
|
499
|
+
// Canvas font rendering hash — generated by Castle.io SDK's canvas fingerprinting (text rendering).
|
|
500
|
+
// Captured from Chrome 144 on Windows 10. Source: yubie-re/castleio-gen (Python SDK, MIT license).
|
|
501
|
+
encodeField(13, EncryptedBytes, textEnc("54b4b5cf"), initTime),
|
|
502
|
+
encodeField(
|
|
503
|
+
14,
|
|
504
|
+
RawAppend,
|
|
505
|
+
// Media input devices
|
|
506
|
+
concat(u8(3), encodeBits([0, 1, 2], 8))
|
|
507
|
+
),
|
|
508
|
+
// Fields 15 (DoNotTrack) and 16 (JavaEnabled) intentionally omitted
|
|
509
|
+
encodeField(17, Byte, 0),
|
|
510
|
+
// productSub type
|
|
511
|
+
// Canvas circle rendering hash — generated by Castle.io SDK's canvas fingerprinting (arc drawing).
|
|
512
|
+
// Captured from Chrome 144 on Windows 10. Source: yubie-re/castleio-gen (Python SDK, MIT license).
|
|
513
|
+
encodeField(18, EncryptedBytes, textEnc("c6749e76"), initTime),
|
|
514
|
+
encodeField(19, EncryptedBytes, textEnc(profile.gpuRenderer), initTime),
|
|
515
|
+
// WebGL renderer
|
|
516
|
+
encodeField(
|
|
517
|
+
20,
|
|
518
|
+
EncryptedBytes,
|
|
519
|
+
// Epoch locale string
|
|
520
|
+
textEnc("12/31/1969, 7:00:00 PM"),
|
|
521
|
+
initTime
|
|
522
|
+
),
|
|
523
|
+
encodeField(
|
|
524
|
+
21,
|
|
525
|
+
RawAppend,
|
|
526
|
+
// WebDriver flags (none set)
|
|
527
|
+
concat(u8(8), encodeBits([], 8))
|
|
528
|
+
),
|
|
529
|
+
encodeField(22, CompactInt, 33),
|
|
530
|
+
// eval.toString() length
|
|
531
|
+
// Field 23 (navigator.buildID) intentionally omitted (Chrome doesn't have it)
|
|
532
|
+
encodeField(24, CompactInt, 12549),
|
|
533
|
+
// Max recursion depth
|
|
534
|
+
encodeField(25, Byte, 0),
|
|
535
|
+
// Recursion error message type
|
|
536
|
+
encodeField(26, Byte, 1),
|
|
537
|
+
// Recursion error name type
|
|
538
|
+
encodeField(27, CompactInt, 4644),
|
|
539
|
+
// Stack trace string length
|
|
540
|
+
encodeField(28, RawAppend, u8(0)),
|
|
541
|
+
// Touch support metric
|
|
542
|
+
encodeField(29, Byte, 3),
|
|
543
|
+
// Undefined call error type
|
|
544
|
+
// Navigator properties hash — hash of enumerable navigator property names.
|
|
545
|
+
// Captured from Chrome 144 on Windows 10. Source: yubie-re/castleio-gen (Python SDK, MIT license).
|
|
546
|
+
encodeField(30, RawAppend, u8(93, 197, 171, 181, 136)),
|
|
547
|
+
encodeField(31, RawAppend, encodeCodecPlayability())
|
|
548
|
+
// Codec playability
|
|
549
|
+
];
|
|
550
|
+
const data = concat(...fields);
|
|
551
|
+
const sizeIdx = (7 & FP_PART.DEVICE) << 5 | 31 & fields.length;
|
|
552
|
+
return concat(u8(sizeIdx), data);
|
|
553
|
+
}
|
|
554
|
+
function buildBrowserFingerprint(profile, initTime) {
|
|
555
|
+
const { Byte, EncryptedBytes, CompactInt, Marker, RawAppend } = FieldEncoding;
|
|
556
|
+
const timezoneField = profile.timezone in TIMEZONE_ENUM ? encodeField(1, Byte, TIMEZONE_ENUM[profile.timezone]) : encodeField(1, EncryptedBytes, textEnc(profile.timezone), initTime);
|
|
557
|
+
const fields = [
|
|
558
|
+
encodeField(0, Byte, 0),
|
|
559
|
+
// Constant marker
|
|
560
|
+
timezoneField,
|
|
561
|
+
// Timezone
|
|
562
|
+
encodeField(
|
|
563
|
+
2,
|
|
564
|
+
EncryptedBytes,
|
|
565
|
+
// Language list
|
|
566
|
+
textEnc(`${profile.locale},${profile.language}`),
|
|
567
|
+
initTime
|
|
568
|
+
),
|
|
569
|
+
encodeField(6, CompactInt, 0),
|
|
570
|
+
// Expected property count
|
|
571
|
+
encodeField(
|
|
572
|
+
10,
|
|
573
|
+
RawAppend,
|
|
574
|
+
// Castle data bitfield
|
|
575
|
+
concat(u8(4), encodeBits([1, 2, 3], 8))
|
|
576
|
+
),
|
|
577
|
+
encodeField(12, CompactInt, 80),
|
|
578
|
+
// Negative error string length
|
|
579
|
+
encodeField(13, RawAppend, u8(9, 0, 0)),
|
|
580
|
+
// Driver check values
|
|
581
|
+
encodeField(
|
|
582
|
+
17,
|
|
583
|
+
RawAppend,
|
|
584
|
+
// Chrome feature flags
|
|
585
|
+
concat(u8(13), encodeBits([1, 5, 8, 9, 10], 16))
|
|
586
|
+
),
|
|
587
|
+
encodeField(18, Marker, 0),
|
|
588
|
+
// Device logic expected
|
|
589
|
+
encodeField(21, RawAppend, u8(0, 0, 0, 0)),
|
|
590
|
+
// Class properties count
|
|
591
|
+
encodeField(22, EncryptedBytes, textEnc(profile.locale), initTime),
|
|
592
|
+
// User locale (secondary)
|
|
593
|
+
encodeField(
|
|
594
|
+
23,
|
|
595
|
+
RawAppend,
|
|
596
|
+
// Worker capabilities
|
|
597
|
+
concat(u8(2), encodeBits([0], 8))
|
|
598
|
+
),
|
|
599
|
+
encodeField(
|
|
600
|
+
24,
|
|
601
|
+
RawAppend,
|
|
602
|
+
// Inner/outer dimension diff
|
|
603
|
+
concat(be16(0), be16(randInt(10, 30)))
|
|
604
|
+
)
|
|
605
|
+
];
|
|
606
|
+
const data = concat(...fields);
|
|
607
|
+
const sizeIdx = (7 & FP_PART.BROWSER) << 5 | 31 & fields.length;
|
|
608
|
+
return concat(u8(sizeIdx), data);
|
|
609
|
+
}
|
|
610
|
+
function buildTimingFingerprint(initTime) {
|
|
611
|
+
const minute = new Date(initTime).getUTCMinutes();
|
|
612
|
+
const fields = [
|
|
613
|
+
encodeField(3, 5 /* CompactInt */, 1),
|
|
614
|
+
// Time since window.open (ms)
|
|
615
|
+
encodeField(4, 5 /* CompactInt */, minute)
|
|
616
|
+
// Castle init time (minutes)
|
|
617
|
+
];
|
|
618
|
+
const data = concat(...fields);
|
|
619
|
+
const sizeIdx = (7 & FP_PART.TIMING) << 5 | 31 & fields.length;
|
|
620
|
+
return concat(u8(sizeIdx), data);
|
|
621
|
+
}
|
|
622
|
+
const EventType = {
|
|
623
|
+
CLICK: 0,
|
|
624
|
+
FOCUS: 5,
|
|
625
|
+
BLUR: 6,
|
|
626
|
+
ANIMATIONSTART: 18,
|
|
627
|
+
MOUSEMOVE: 21,
|
|
628
|
+
MOUSELEAVE: 25,
|
|
629
|
+
MOUSEENTER: 26,
|
|
630
|
+
RESIZE: 27
|
|
631
|
+
};
|
|
632
|
+
const HAS_TARGET_FLAG = 128;
|
|
633
|
+
const TARGET_UNKNOWN = 63;
|
|
634
|
+
function generateEventLog() {
|
|
635
|
+
const simpleEvents = [
|
|
636
|
+
EventType.MOUSEMOVE,
|
|
637
|
+
EventType.ANIMATIONSTART,
|
|
638
|
+
EventType.MOUSELEAVE,
|
|
639
|
+
EventType.MOUSEENTER,
|
|
640
|
+
EventType.RESIZE
|
|
641
|
+
];
|
|
642
|
+
const targetedEvents = [
|
|
643
|
+
EventType.CLICK,
|
|
644
|
+
EventType.BLUR,
|
|
645
|
+
EventType.FOCUS
|
|
646
|
+
];
|
|
647
|
+
const allEvents = [...simpleEvents, ...targetedEvents];
|
|
648
|
+
const count = randInt(30, 70);
|
|
649
|
+
const eventBytes = [];
|
|
650
|
+
for (let i = 0; i < count; i++) {
|
|
651
|
+
const eventId = allEvents[randInt(0, allEvents.length - 1)];
|
|
652
|
+
if (targetedEvents.includes(eventId)) {
|
|
653
|
+
eventBytes.push(eventId | HAS_TARGET_FLAG);
|
|
654
|
+
eventBytes.push(TARGET_UNKNOWN);
|
|
655
|
+
} else {
|
|
656
|
+
eventBytes.push(eventId);
|
|
657
|
+
}
|
|
658
|
+
}
|
|
659
|
+
const inner = concat(u8(0), be16(count), new Uint8Array(eventBytes));
|
|
660
|
+
return concat(be16(inner.length), inner);
|
|
661
|
+
}
|
|
662
|
+
function buildBehavioralBitfield() {
|
|
663
|
+
const flags = new Array(15).fill(false);
|
|
664
|
+
flags[2] = true;
|
|
665
|
+
flags[3] = true;
|
|
666
|
+
flags[5] = true;
|
|
667
|
+
flags[6] = true;
|
|
668
|
+
flags[9] = true;
|
|
669
|
+
flags[11] = true;
|
|
670
|
+
flags[12] = true;
|
|
671
|
+
const packedBits = boolsToBin(flags, 16);
|
|
672
|
+
const encoded = 6 << 20 | 2 << 16 | 65535 & packedBits;
|
|
673
|
+
return u8(encoded >>> 16 & 255, encoded >>> 8 & 255, encoded & 255);
|
|
674
|
+
}
|
|
675
|
+
const NO_DATA = -1;
|
|
676
|
+
function buildFloatMetrics() {
|
|
677
|
+
const metrics = [
|
|
678
|
+
// ── Mouse & key timing ──
|
|
679
|
+
randFloat(40, 50),
|
|
680
|
+
// 0: Mouse angle vector mean
|
|
681
|
+
NO_DATA,
|
|
682
|
+
// 1: Touch angle vector (no touch device)
|
|
683
|
+
randFloat(70, 80),
|
|
684
|
+
// 2: Key same-time difference
|
|
685
|
+
NO_DATA,
|
|
686
|
+
// 3: (unused)
|
|
687
|
+
randFloat(60, 70),
|
|
688
|
+
// 4: Mouse down-to-up time mean
|
|
689
|
+
NO_DATA,
|
|
690
|
+
// 5: (unused)
|
|
691
|
+
0,
|
|
692
|
+
// 6: (zero placeholder)
|
|
693
|
+
0,
|
|
694
|
+
// 7: Mouse click time difference
|
|
695
|
+
// ── Duration distributions ──
|
|
696
|
+
randFloat(60, 80),
|
|
697
|
+
// 8: Mouse down-up duration median
|
|
698
|
+
randFloat(5, 10),
|
|
699
|
+
// 9: Mouse down-up duration std deviation
|
|
700
|
+
randFloat(30, 40),
|
|
701
|
+
// 10: Key press duration median
|
|
702
|
+
randFloat(2, 5),
|
|
703
|
+
// 11: Key press duration std deviation
|
|
704
|
+
// ── Touch metrics (all disabled for desktop) ──
|
|
705
|
+
NO_DATA,
|
|
706
|
+
NO_DATA,
|
|
707
|
+
NO_DATA,
|
|
708
|
+
NO_DATA,
|
|
709
|
+
// 12-15
|
|
710
|
+
NO_DATA,
|
|
711
|
+
NO_DATA,
|
|
712
|
+
NO_DATA,
|
|
713
|
+
NO_DATA,
|
|
714
|
+
// 16-19
|
|
715
|
+
// ── Mouse trajectory analysis ──
|
|
716
|
+
randFloat(150, 180),
|
|
717
|
+
// 20: Mouse movement angle mean
|
|
718
|
+
randFloat(3, 6),
|
|
719
|
+
// 21: Mouse movement angle std deviation
|
|
720
|
+
randFloat(150, 180),
|
|
721
|
+
// 22: Mouse movement angle mean (500ms window)
|
|
722
|
+
randFloat(3, 6),
|
|
723
|
+
// 23: Mouse movement angle std (500ms window)
|
|
724
|
+
randFloat(0, 2),
|
|
725
|
+
// 24: Mouse position deviation X
|
|
726
|
+
randFloat(0, 2),
|
|
727
|
+
// 25: Mouse position deviation Y
|
|
728
|
+
0,
|
|
729
|
+
0,
|
|
730
|
+
// 26-27: (zero placeholders)
|
|
731
|
+
// ── Touch sequential/gesture metrics (disabled) ──
|
|
732
|
+
NO_DATA,
|
|
733
|
+
NO_DATA,
|
|
734
|
+
// 28-29
|
|
735
|
+
NO_DATA,
|
|
736
|
+
NO_DATA,
|
|
737
|
+
// 30-31
|
|
738
|
+
// ── Key pattern analysis ──
|
|
739
|
+
0,
|
|
740
|
+
0,
|
|
741
|
+
// 32-33: Letter-digit transition ratio
|
|
742
|
+
0,
|
|
743
|
+
0,
|
|
744
|
+
// 34-35: Digit-invalid transition ratio
|
|
745
|
+
0,
|
|
746
|
+
0,
|
|
747
|
+
// 36-37: Double-invalid transition ratio
|
|
748
|
+
// ── Mouse vector differences ──
|
|
749
|
+
1,
|
|
750
|
+
0,
|
|
751
|
+
// 38-39: Mouse vector diff (mean, std)
|
|
752
|
+
1,
|
|
753
|
+
0,
|
|
754
|
+
// 40-41: Mouse vector diff 2 (mean, std)
|
|
755
|
+
randFloat(0, 4),
|
|
756
|
+
// 42: Mouse vector diff (500ms mean)
|
|
757
|
+
randFloat(0, 3),
|
|
758
|
+
// 43: Mouse vector diff (500ms std)
|
|
759
|
+
// ── Rounded movement metrics ──
|
|
760
|
+
randFloat(25, 50),
|
|
761
|
+
// 44: Mouse time diff (rounded mean)
|
|
762
|
+
randFloat(25, 50),
|
|
763
|
+
// 45: Mouse time diff (rounded std)
|
|
764
|
+
randFloat(25, 50),
|
|
765
|
+
// 46: Mouse vector diff (rounded mean)
|
|
766
|
+
randFloat(25, 30),
|
|
767
|
+
// 47: Mouse vector diff (rounded std)
|
|
768
|
+
// ── Speed change analysis ──
|
|
769
|
+
randFloat(0, 2),
|
|
770
|
+
// 48: Mouse speed change mean
|
|
771
|
+
randFloat(0, 1),
|
|
772
|
+
// 49: Mouse speed change std
|
|
773
|
+
randFloat(0, 1),
|
|
774
|
+
// 50: Mouse vector 500ms aggregate
|
|
775
|
+
// ── Trailing ──
|
|
776
|
+
1,
|
|
777
|
+
// 51: Universal flag
|
|
778
|
+
0
|
|
779
|
+
// 52: Terminator
|
|
780
|
+
];
|
|
781
|
+
const out = new Uint8Array(metrics.length);
|
|
782
|
+
for (let i = 0; i < metrics.length; i++) {
|
|
783
|
+
out[i] = metrics[i] === NO_DATA ? 0 : encodeFloatVal(metrics[i]);
|
|
784
|
+
}
|
|
785
|
+
return out;
|
|
786
|
+
}
|
|
787
|
+
function buildEventCounts() {
|
|
788
|
+
const counts = [
|
|
789
|
+
randInt(100, 200),
|
|
790
|
+
// 0: mousemove events
|
|
791
|
+
randInt(1, 5),
|
|
792
|
+
// 1: keyup events
|
|
793
|
+
randInt(1, 5),
|
|
794
|
+
// 2: click events
|
|
795
|
+
0,
|
|
796
|
+
// 3: touchstart events (none on desktop)
|
|
797
|
+
randInt(0, 5),
|
|
798
|
+
// 4: keydown events
|
|
799
|
+
0,
|
|
800
|
+
// 5: touchmove events (none)
|
|
801
|
+
0,
|
|
802
|
+
// 6: mousedown-mouseup pairs
|
|
803
|
+
0,
|
|
804
|
+
// 7: vector diff samples
|
|
805
|
+
randInt(0, 5),
|
|
806
|
+
// 8: wheel events
|
|
807
|
+
randInt(0, 11),
|
|
808
|
+
// 9: (internal counter)
|
|
809
|
+
randInt(0, 1)
|
|
810
|
+
// 10: (internal counter)
|
|
811
|
+
];
|
|
812
|
+
return concat(new Uint8Array(counts), u8(counts.length));
|
|
813
|
+
}
|
|
814
|
+
function buildBehavioralData() {
|
|
815
|
+
return concat(
|
|
816
|
+
buildBehavioralBitfield(),
|
|
817
|
+
buildFloatMetrics(),
|
|
818
|
+
buildEventCounts()
|
|
819
|
+
);
|
|
820
|
+
}
|
|
821
|
+
function buildTokenHeader(uuid, publisherKey, initTime) {
|
|
822
|
+
const timestamp = fromHex(encodeTimestampEncrypted(initTime));
|
|
823
|
+
const version = be16(SDK_VERSION);
|
|
824
|
+
const pkBytes = textEnc(publisherKey);
|
|
825
|
+
const uuidBytes = fromHex(uuid);
|
|
826
|
+
return concat(timestamp, version, pkBytes, uuidBytes);
|
|
827
|
+
}
|
|
828
|
+
function generateLocalCastleToken(userAgent, profileOverride) {
|
|
829
|
+
const now = Date.now();
|
|
830
|
+
const profile = { ...DEFAULT_PROFILE, ...profileOverride };
|
|
831
|
+
const initTime = now - randFloat(2 * 60 * 1e3, 30 * 60 * 1e3);
|
|
832
|
+
log$7("Generating local Castle.io v11 token");
|
|
833
|
+
const deviceFp = buildDeviceFingerprint(initTime, profile, userAgent);
|
|
834
|
+
const browserFp = buildBrowserFingerprint(profile, initTime);
|
|
835
|
+
const timingFp = buildTimingFingerprint(initTime);
|
|
836
|
+
const eventLog = generateEventLog();
|
|
837
|
+
const behavioral = buildBehavioralData();
|
|
838
|
+
const fingerprintData = concat(
|
|
839
|
+
deviceFp,
|
|
840
|
+
browserFp,
|
|
841
|
+
timingFp,
|
|
842
|
+
eventLog,
|
|
843
|
+
behavioral,
|
|
844
|
+
u8(255)
|
|
845
|
+
);
|
|
846
|
+
const sendTime = Date.now();
|
|
847
|
+
const timestampKey = encodeTimestampEncrypted(sendTime);
|
|
848
|
+
const xorPass1 = deriveAndXor(
|
|
849
|
+
timestampKey,
|
|
850
|
+
4,
|
|
851
|
+
timestampKey[3],
|
|
852
|
+
fingerprintData
|
|
853
|
+
);
|
|
854
|
+
const tokenUuid = toHex(getRandomBytes(16));
|
|
855
|
+
const withTimestampPrefix = concat(fromHex(timestampKey), xorPass1);
|
|
856
|
+
const xorPass2 = deriveAndXor(
|
|
857
|
+
tokenUuid,
|
|
858
|
+
8,
|
|
859
|
+
tokenUuid[9],
|
|
860
|
+
withTimestampPrefix
|
|
861
|
+
);
|
|
862
|
+
const header = buildTokenHeader(tokenUuid, TWITTER_CASTLE_PK, initTime);
|
|
863
|
+
const plaintext = concat(header, xorPass2);
|
|
864
|
+
const encrypted = xxteaEncrypt(plaintext, XXTEA_KEY);
|
|
865
|
+
const paddingBytes = encrypted.length - plaintext.length;
|
|
866
|
+
const versioned = concat(u8(TOKEN_VERSION, paddingBytes), encrypted);
|
|
867
|
+
const randomByte = getRandomBytes(1)[0];
|
|
868
|
+
const checksum = versioned.length * 2 & 255;
|
|
869
|
+
const withChecksum = concat(versioned, u8(checksum));
|
|
870
|
+
const xored = xorBytes(withChecksum, u8(randomByte));
|
|
871
|
+
const finalPayload = concat(u8(randomByte), xored);
|
|
872
|
+
const token = base64url(finalPayload);
|
|
873
|
+
log$7(
|
|
874
|
+
`Generated castle token: ${token.length} chars, cuid: ${tokenUuid.substring(
|
|
875
|
+
0,
|
|
876
|
+
6
|
|
877
|
+
)}...`
|
|
878
|
+
);
|
|
879
|
+
return { token, cuid: tokenUuid };
|
|
880
|
+
}
|
|
881
|
+
|
|
882
|
+
const log$6 = debug("twitter-scraper:requests");
|
|
96
883
|
async function updateCookieJar(cookieJar, headers) {
|
|
97
884
|
let setCookieHeaders = [];
|
|
98
885
|
if (typeof headers.getSetCookie === "function") {
|
|
@@ -107,12 +894,12 @@ async function updateCookieJar(cookieJar, headers) {
|
|
|
107
894
|
for (const cookieStr of setCookieHeaders) {
|
|
108
895
|
const cookie = toughCookie.Cookie.parse(cookieStr);
|
|
109
896
|
if (!cookie) {
|
|
110
|
-
log$
|
|
897
|
+
log$6(`Failed to parse cookie: ${cookieStr.substring(0, 100)}`);
|
|
111
898
|
continue;
|
|
112
899
|
}
|
|
113
900
|
if (cookie.maxAge === 0 || cookie.expires && cookie.expires < /* @__PURE__ */ new Date()) {
|
|
114
901
|
if (cookie.key === "ct0") {
|
|
115
|
-
log$
|
|
902
|
+
log$6(`Skipping deletion of ct0 cookie (Max-Age=0)`);
|
|
116
903
|
}
|
|
117
904
|
continue;
|
|
118
905
|
}
|
|
@@ -120,7 +907,7 @@ async function updateCookieJar(cookieJar, headers) {
|
|
|
120
907
|
const url = `${cookie.secure ? "https" : "http"}://${cookie.domain}${cookie.path}`;
|
|
121
908
|
await cookieJar.setCookie(cookie, url);
|
|
122
909
|
if (cookie.key === "ct0") {
|
|
123
|
-
log$
|
|
910
|
+
log$6(
|
|
124
911
|
`Successfully set ct0 cookie with value: ${cookie.value.substring(
|
|
125
912
|
0,
|
|
126
913
|
20
|
|
@@ -128,9 +915,9 @@ async function updateCookieJar(cookieJar, headers) {
|
|
|
128
915
|
);
|
|
129
916
|
}
|
|
130
917
|
} catch (err) {
|
|
131
|
-
log$
|
|
918
|
+
log$6(`Failed to set cookie ${cookie.key}: ${err}`);
|
|
132
919
|
if (cookie.key === "ct0") {
|
|
133
|
-
log$
|
|
920
|
+
log$6(`FAILED to set ct0 cookie! Error: ${err}`);
|
|
134
921
|
}
|
|
135
922
|
}
|
|
136
923
|
}
|
|
@@ -144,14 +931,14 @@ async function updateCookieJar(cookieJar, headers) {
|
|
|
144
931
|
}
|
|
145
932
|
}
|
|
146
933
|
|
|
147
|
-
const log$
|
|
934
|
+
const log$5 = debug("twitter-scraper:xpff");
|
|
148
935
|
let isoCrypto = null;
|
|
149
936
|
async function getCrypto() {
|
|
150
937
|
if (isoCrypto != null) {
|
|
151
938
|
return isoCrypto;
|
|
152
939
|
}
|
|
153
940
|
if (typeof crypto === "undefined") {
|
|
154
|
-
log$
|
|
941
|
+
log$5("Global crypto is undefined, importing from crypto module...");
|
|
155
942
|
const { webcrypto } = await import('crypto');
|
|
156
943
|
isoCrypto = webcrypto;
|
|
157
944
|
return webcrypto;
|
|
@@ -178,7 +965,7 @@ class XPFFHeaderGenerator {
|
|
|
178
965
|
return result;
|
|
179
966
|
}
|
|
180
967
|
async generateHeader(plaintext, guestId) {
|
|
181
|
-
log$
|
|
968
|
+
log$5(`Generating XPFF key for guest ID: ${guestId}`);
|
|
182
969
|
const key = await this.deriveKey(guestId);
|
|
183
970
|
const crypto2 = await getCrypto();
|
|
184
971
|
const nonce = crypto2.getRandomValues(new Uint8Array(12));
|
|
@@ -201,7 +988,7 @@ class XPFFHeaderGenerator {
|
|
|
201
988
|
combined.set(nonce);
|
|
202
989
|
combined.set(new Uint8Array(encrypted), nonce.length);
|
|
203
990
|
const result = buf2hex(combined.buffer);
|
|
204
|
-
log$
|
|
991
|
+
log$5(`XPFF header generated for guest ID ${guestId}: ${result}`);
|
|
205
992
|
return result;
|
|
206
993
|
}
|
|
207
994
|
}
|
|
@@ -223,7 +1010,7 @@ async function generateXPFFHeader(guestId) {
|
|
|
223
1010
|
return generator.generateHeader(plaintext, guestId);
|
|
224
1011
|
}
|
|
225
1012
|
|
|
226
|
-
const log$
|
|
1013
|
+
const log$4 = debug("twitter-scraper:auth");
|
|
227
1014
|
function withTransform(fetchFn, transform) {
|
|
228
1015
|
return async (input, init) => {
|
|
229
1016
|
const fetchArgs = await transform?.request?.(input, init) ?? [
|
|
@@ -273,8 +1060,14 @@ class TwitterGuestAuth {
|
|
|
273
1060
|
}
|
|
274
1061
|
return new Date(this.guestCreatedAt);
|
|
275
1062
|
}
|
|
276
|
-
|
|
1063
|
+
/**
|
|
1064
|
+
* Install only authentication credentials (bearer token, guest token, cookies)
|
|
1065
|
+
* without browser fingerprint or platform headers. Useful for callers that
|
|
1066
|
+
* build their own header set (e.g. the login flow).
|
|
1067
|
+
*/
|
|
1068
|
+
async installAuthCredentials(headers, bearerTokenOverride) {
|
|
277
1069
|
const tokenToUse = bearerTokenOverride ?? this.bearerToken;
|
|
1070
|
+
headers.set("authorization", `Bearer ${tokenToUse}`);
|
|
278
1071
|
if (!bearerTokenOverride) {
|
|
279
1072
|
if (this.shouldUpdate()) {
|
|
280
1073
|
await this.updateGuestToken();
|
|
@@ -283,11 +1076,27 @@ class TwitterGuestAuth {
|
|
|
283
1076
|
headers.set("x-guest-token", this.guestToken);
|
|
284
1077
|
}
|
|
285
1078
|
}
|
|
286
|
-
headers.set("
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
);
|
|
1079
|
+
headers.set("cookie", await this.getCookieString());
|
|
1080
|
+
}
|
|
1081
|
+
async installTo(headers, url, bearerTokenOverride) {
|
|
1082
|
+
await this.installAuthCredentials(headers, bearerTokenOverride);
|
|
1083
|
+
headers.set("user-agent", CHROME_USER_AGENT);
|
|
1084
|
+
if (!headers.has("accept")) {
|
|
1085
|
+
headers.set("accept", "*/*");
|
|
1086
|
+
}
|
|
1087
|
+
headers.set("accept-language", "en-US,en;q=0.9");
|
|
1088
|
+
headers.set("sec-ch-ua", CHROME_SEC_CH_UA);
|
|
1089
|
+
headers.set("sec-ch-ua-mobile", "?0");
|
|
1090
|
+
headers.set("sec-ch-ua-platform", '"Windows"');
|
|
1091
|
+
headers.set("referer", "https://x.com/");
|
|
1092
|
+
headers.set("origin", "https://x.com");
|
|
1093
|
+
headers.set("sec-fetch-site", "same-site");
|
|
1094
|
+
headers.set("sec-fetch-mode", "cors");
|
|
1095
|
+
headers.set("sec-fetch-dest", "empty");
|
|
1096
|
+
headers.set("priority", "u=1, i");
|
|
1097
|
+
if (!headers.has("content-type") && (url.includes("api.x.com/graphql/") || url.includes("x.com/i/api/graphql/"))) {
|
|
1098
|
+
headers.set("content-type", "application/json");
|
|
1099
|
+
}
|
|
291
1100
|
await this.installCsrfToken(headers);
|
|
292
1101
|
if (this.options?.experimental?.xpff) {
|
|
293
1102
|
const guestId = await this.guestId();
|
|
@@ -296,7 +1105,6 @@ class TwitterGuestAuth {
|
|
|
296
1105
|
headers.set("x-xp-forwarded-for", xpffHeader);
|
|
297
1106
|
}
|
|
298
1107
|
}
|
|
299
|
-
headers.set("cookie", await this.getCookieString());
|
|
300
1108
|
}
|
|
301
1109
|
async installCsrfToken(headers) {
|
|
302
1110
|
const cookies = await this.getCookies();
|
|
@@ -327,7 +1135,7 @@ class TwitterGuestAuth {
|
|
|
327
1135
|
const cookies = await this.jar.getCookies(this.getCookieJarUrl());
|
|
328
1136
|
for (const cookie of cookies) {
|
|
329
1137
|
if (!cookie.domain || !cookie.path) continue;
|
|
330
|
-
store.removeCookie(cookie.domain, cookie.path, key);
|
|
1138
|
+
await store.removeCookie(cookie.domain, cookie.path, key);
|
|
331
1139
|
if (typeof document !== "undefined") {
|
|
332
1140
|
document.cookie = `${cookie.key}=; Max-Age=0; path=${cookie.path}; domain=${cookie.domain}`;
|
|
333
1141
|
}
|
|
@@ -348,20 +1156,31 @@ class TwitterGuestAuth {
|
|
|
348
1156
|
try {
|
|
349
1157
|
await this.updateGuestTokenCore();
|
|
350
1158
|
} catch (err) {
|
|
351
|
-
log$
|
|
1159
|
+
log$4("Failed to update guest token; this may cause issues:", err);
|
|
352
1160
|
}
|
|
353
1161
|
}
|
|
354
1162
|
async updateGuestTokenCore() {
|
|
355
1163
|
const guestActivateUrl = "https://api.x.com/1.1/guest/activate.json";
|
|
356
1164
|
const headers = new headersPolyfill.Headers({
|
|
357
|
-
|
|
358
|
-
|
|
1165
|
+
authorization: `Bearer ${this.bearerToken}`,
|
|
1166
|
+
"user-agent": CHROME_USER_AGENT,
|
|
1167
|
+
accept: "*/*",
|
|
1168
|
+
"accept-language": "en-US,en;q=0.9",
|
|
1169
|
+
"content-type": "application/x-www-form-urlencoded",
|
|
1170
|
+
"sec-ch-ua": CHROME_SEC_CH_UA,
|
|
1171
|
+
"sec-ch-ua-mobile": "?0",
|
|
1172
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
1173
|
+
origin: "https://x.com",
|
|
1174
|
+
referer: "https://x.com/",
|
|
1175
|
+
"sec-fetch-site": "same-site",
|
|
1176
|
+
"sec-fetch-mode": "cors",
|
|
1177
|
+
"sec-fetch-dest": "empty",
|
|
1178
|
+
cookie: await this.getCookieString()
|
|
359
1179
|
});
|
|
360
|
-
log$
|
|
1180
|
+
log$4(`Making POST request to ${guestActivateUrl}`);
|
|
361
1181
|
const res = await this.fetch(guestActivateUrl, {
|
|
362
1182
|
method: "POST",
|
|
363
|
-
headers
|
|
364
|
-
referrerPolicy: "no-referrer"
|
|
1183
|
+
headers
|
|
365
1184
|
});
|
|
366
1185
|
await updateCookieJar(this.jar, res.headers);
|
|
367
1186
|
if (!res.ok) {
|
|
@@ -378,7 +1197,7 @@ class TwitterGuestAuth {
|
|
|
378
1197
|
this.guestToken = newGuestToken;
|
|
379
1198
|
this.guestCreatedAt = /* @__PURE__ */ new Date();
|
|
380
1199
|
await this.setCookie("gt", newGuestToken);
|
|
381
|
-
log$
|
|
1200
|
+
log$4(`Updated guest token (length: ${newGuestToken.length})`);
|
|
382
1201
|
}
|
|
383
1202
|
/**
|
|
384
1203
|
* Returns if the authentication token needs to be updated or not.
|
|
@@ -405,7 +1224,7 @@ class Platform {
|
|
|
405
1224
|
}
|
|
406
1225
|
}
|
|
407
1226
|
|
|
408
|
-
const log$
|
|
1227
|
+
const log$3 = debug("twitter-scraper:xctxid");
|
|
409
1228
|
let linkedom = null;
|
|
410
1229
|
async function linkedomImport() {
|
|
411
1230
|
if (!linkedom) {
|
|
@@ -434,7 +1253,7 @@ async function handleXMigration(fetchFn) {
|
|
|
434
1253
|
"cache-control": "no-cache",
|
|
435
1254
|
pragma: "no-cache",
|
|
436
1255
|
priority: "u=0, i",
|
|
437
|
-
"sec-ch-ua":
|
|
1256
|
+
"sec-ch-ua": CHROME_SEC_CH_UA,
|
|
438
1257
|
"sec-ch-ua-mobile": "?0",
|
|
439
1258
|
"sec-ch-ua-platform": '"Windows"',
|
|
440
1259
|
"sec-fetch-dest": "document",
|
|
@@ -442,7 +1261,7 @@ async function handleXMigration(fetchFn) {
|
|
|
442
1261
|
"sec-fetch-site": "none",
|
|
443
1262
|
"sec-fetch-user": "?1",
|
|
444
1263
|
"upgrade-insecure-requests": "1",
|
|
445
|
-
"user-agent":
|
|
1264
|
+
"user-agent": CHROME_USER_AGENT
|
|
446
1265
|
};
|
|
447
1266
|
const response = await fetchFn("https://x.com", {
|
|
448
1267
|
headers
|
|
@@ -461,7 +1280,7 @@ async function handleXMigration(fetchFn) {
|
|
|
461
1280
|
const metaContent = metaRefresh ? metaRefresh.getAttribute("content") || "" : "";
|
|
462
1281
|
const migrationRedirectionUrl = migrationRedirectionRegex.exec(metaContent) || migrationRedirectionRegex.exec(htmlText);
|
|
463
1282
|
if (migrationRedirectionUrl) {
|
|
464
|
-
const redirectResponse = await
|
|
1283
|
+
const redirectResponse = await fetchFn(migrationRedirectionUrl[0]);
|
|
465
1284
|
if (!redirectResponse.ok) {
|
|
466
1285
|
throw new Error(
|
|
467
1286
|
`Failed to follow migration redirection: ${redirectResponse.statusText}`
|
|
@@ -484,7 +1303,7 @@ async function handleXMigration(fetchFn) {
|
|
|
484
1303
|
requestPayload.append(name, value);
|
|
485
1304
|
}
|
|
486
1305
|
}
|
|
487
|
-
const formResponse = await
|
|
1306
|
+
const formResponse = await fetchFn(url, {
|
|
488
1307
|
method,
|
|
489
1308
|
body: requestPayload,
|
|
490
1309
|
headers
|
|
@@ -500,6 +1319,23 @@ async function handleXMigration(fetchFn) {
|
|
|
500
1319
|
}
|
|
501
1320
|
return document;
|
|
502
1321
|
}
|
|
1322
|
+
let cachedDocumentPromise = null;
|
|
1323
|
+
let cachedDocumentTimestamp = 0;
|
|
1324
|
+
const DOCUMENT_CACHE_TTL = 5 * 60 * 1e3;
|
|
1325
|
+
async function getCachedDocument(fetchFn) {
|
|
1326
|
+
const now = Date.now();
|
|
1327
|
+
if (!cachedDocumentPromise || now - cachedDocumentTimestamp > DOCUMENT_CACHE_TTL) {
|
|
1328
|
+
log$3("Fetching fresh x.com document for transaction ID generation");
|
|
1329
|
+
cachedDocumentTimestamp = now;
|
|
1330
|
+
cachedDocumentPromise = handleXMigration(fetchFn).catch((err) => {
|
|
1331
|
+
cachedDocumentPromise = null;
|
|
1332
|
+
throw err;
|
|
1333
|
+
});
|
|
1334
|
+
} else {
|
|
1335
|
+
log$3("Using cached x.com document for transaction ID generation");
|
|
1336
|
+
}
|
|
1337
|
+
return cachedDocumentPromise;
|
|
1338
|
+
}
|
|
503
1339
|
let ClientTransaction = null;
|
|
504
1340
|
async function clientTransaction() {
|
|
505
1341
|
if (!ClientTransaction) {
|
|
@@ -512,16 +1348,19 @@ async function clientTransaction() {
|
|
|
512
1348
|
async function generateTransactionId(url, fetchFn, method) {
|
|
513
1349
|
const parsedUrl = new URL(url);
|
|
514
1350
|
const path = parsedUrl.pathname;
|
|
515
|
-
log$
|
|
516
|
-
const document = await
|
|
1351
|
+
log$3(`Generating transaction ID for ${method} ${path}`);
|
|
1352
|
+
const document = await getCachedDocument(fetchFn);
|
|
517
1353
|
const ClientTransactionClass = await clientTransaction();
|
|
518
1354
|
const transaction = await ClientTransactionClass.create(document);
|
|
519
1355
|
const transactionId = await transaction.generateTransactionId(method, path);
|
|
520
|
-
log$
|
|
1356
|
+
log$3(`Transaction ID: ${transactionId}`);
|
|
521
1357
|
return transactionId;
|
|
522
1358
|
}
|
|
523
1359
|
|
|
524
|
-
const
|
|
1360
|
+
const CHROME_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36";
|
|
1361
|
+
const CHROME_SEC_CH_UA = '"Not(A:Brand";v="8", "Chromium";v="144", "Google Chrome";v="144"';
|
|
1362
|
+
|
|
1363
|
+
const log$2 = debug("twitter-scraper:api");
|
|
525
1364
|
const bearerToken = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF";
|
|
526
1365
|
const bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
|
|
527
1366
|
async function jitter(maxMs) {
|
|
@@ -529,7 +1368,7 @@ async function jitter(maxMs) {
|
|
|
529
1368
|
await new Promise((resolve) => setTimeout(resolve, jitter2));
|
|
530
1369
|
}
|
|
531
1370
|
async function requestApi(url, auth, method = "GET", platform = new Platform(), headers = new headersPolyfill.Headers(), bearerTokenOverride) {
|
|
532
|
-
log$
|
|
1371
|
+
log$2(`Making ${method} request to ${url}`);
|
|
533
1372
|
await auth.installTo(headers, url, bearerTokenOverride);
|
|
534
1373
|
await platform.randomizeCiphers();
|
|
535
1374
|
if (auth instanceof TwitterGuestAuth && auth.options?.experimental?.xClientTransactionId) {
|
|
@@ -558,12 +1397,12 @@ async function requestApi(url, auth, method = "GET", platform = new Platform(),
|
|
|
558
1397
|
}
|
|
559
1398
|
return {
|
|
560
1399
|
success: false,
|
|
561
|
-
err
|
|
1400
|
+
err
|
|
562
1401
|
};
|
|
563
1402
|
}
|
|
564
1403
|
await updateCookieJar(auth.cookieJar(), res.headers);
|
|
565
1404
|
if (res.status === 429) {
|
|
566
|
-
log$
|
|
1405
|
+
log$2("Rate limit hit, waiting for retry...");
|
|
567
1406
|
await auth.onRateLimit({
|
|
568
1407
|
fetchParameters,
|
|
569
1408
|
response: res
|
|
@@ -588,9 +1427,9 @@ async function flexParseJson(res) {
|
|
|
588
1427
|
try {
|
|
589
1428
|
return await res.json();
|
|
590
1429
|
} catch {
|
|
591
|
-
log$
|
|
1430
|
+
log$2("Failed to parse response as JSON, trying text parse...");
|
|
592
1431
|
const text = await res.text();
|
|
593
|
-
log$
|
|
1432
|
+
log$2("Response text:", text);
|
|
594
1433
|
return JSON.parse(text);
|
|
595
1434
|
}
|
|
596
1435
|
}
|
|
@@ -664,12 +1503,12 @@ function addApiParams(params, includeTweetReplies) {
|
|
|
664
1503
|
return params;
|
|
665
1504
|
}
|
|
666
1505
|
|
|
667
|
-
const log = debug("twitter-scraper:auth-user");
|
|
1506
|
+
const log$1 = debug("twitter-scraper:auth-user");
|
|
668
1507
|
const TwitterUserAuthSubtask = typebox.Type.Object({
|
|
669
1508
|
subtask_id: typebox.Type.String(),
|
|
670
1509
|
enter_text: typebox.Type.Optional(typebox.Type.Object({}))
|
|
671
1510
|
});
|
|
672
|
-
class
|
|
1511
|
+
const _TwitterUserAuth = class _TwitterUserAuth extends TwitterGuestAuth {
|
|
673
1512
|
constructor(bearerToken, options) {
|
|
674
1513
|
super(bearerToken, options);
|
|
675
1514
|
this.subtaskHandlers = /* @__PURE__ */ new Map();
|
|
@@ -715,11 +1554,14 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
715
1554
|
);
|
|
716
1555
|
}
|
|
717
1556
|
async isLoggedIn() {
|
|
718
|
-
const
|
|
719
|
-
return
|
|
1557
|
+
const cookies = await this.getCookies();
|
|
1558
|
+
return cookies.some((c) => c.key === "ct0") && cookies.some((c) => c.key === "auth_token");
|
|
720
1559
|
}
|
|
721
1560
|
async login(username, password, email, twoFactorSecret) {
|
|
722
|
-
await this.
|
|
1561
|
+
await this.preflight();
|
|
1562
|
+
if (!this.guestToken) {
|
|
1563
|
+
await this.updateGuestToken();
|
|
1564
|
+
}
|
|
723
1565
|
const credentials = {
|
|
724
1566
|
username,
|
|
725
1567
|
password,
|
|
@@ -733,6 +1575,12 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
733
1575
|
throw new Error("flow_token not found.");
|
|
734
1576
|
}
|
|
735
1577
|
const subtaskId = next.response.subtasks[0].subtask_id;
|
|
1578
|
+
const configuredDelay = this.options?.experimental?.flowStepDelay;
|
|
1579
|
+
const delay = configuredDelay !== void 0 ? configuredDelay : 1e3 + Math.floor(Math.random() * 2e3);
|
|
1580
|
+
if (delay > 0) {
|
|
1581
|
+
log$1(`Waiting ${delay}ms before handling subtask: ${subtaskId}`);
|
|
1582
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1583
|
+
}
|
|
736
1584
|
const handler = this.subtaskHandlers.get(subtaskId);
|
|
737
1585
|
if (handler) {
|
|
738
1586
|
next = await handler(subtaskId, next.response, credentials, {
|
|
@@ -747,65 +1595,92 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
747
1595
|
throw next.err;
|
|
748
1596
|
}
|
|
749
1597
|
}
|
|
1598
|
+
/**
|
|
1599
|
+
* Pre-flight request to establish Cloudflare cookies and session context.
|
|
1600
|
+
* Mimics a real browser visiting x.com before starting the login API flow.
|
|
1601
|
+
*/
|
|
1602
|
+
async preflight() {
|
|
1603
|
+
try {
|
|
1604
|
+
const headers = new headersPolyfill.Headers({
|
|
1605
|
+
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
1606
|
+
"accept-language": "en-US,en;q=0.9",
|
|
1607
|
+
"sec-ch-ua": CHROME_SEC_CH_UA,
|
|
1608
|
+
"sec-ch-ua-mobile": "?0",
|
|
1609
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
1610
|
+
"sec-fetch-dest": "document",
|
|
1611
|
+
"sec-fetch-mode": "navigate",
|
|
1612
|
+
"sec-fetch-site": "none",
|
|
1613
|
+
"sec-fetch-user": "?1",
|
|
1614
|
+
"upgrade-insecure-requests": "1",
|
|
1615
|
+
"user-agent": CHROME_USER_AGENT
|
|
1616
|
+
});
|
|
1617
|
+
log$1("Pre-flight: fetching https://x.com/i/flow/login");
|
|
1618
|
+
const res = await this.fetch("https://x.com/i/flow/login", {
|
|
1619
|
+
redirect: "follow",
|
|
1620
|
+
headers
|
|
1621
|
+
});
|
|
1622
|
+
await updateCookieJar(this.jar, res.headers);
|
|
1623
|
+
log$1(`Pre-flight response: ${res.status}`);
|
|
1624
|
+
try {
|
|
1625
|
+
const html = await res.text();
|
|
1626
|
+
const gtMatch = html.match(/document\.cookie="gt=(\d+)/);
|
|
1627
|
+
if (gtMatch) {
|
|
1628
|
+
this.guestToken = gtMatch[1];
|
|
1629
|
+
this.guestCreatedAt = /* @__PURE__ */ new Date();
|
|
1630
|
+
await this.setCookie("gt", gtMatch[1]);
|
|
1631
|
+
log$1(`Extracted guest token from HTML (length: ${gtMatch[1].length})`);
|
|
1632
|
+
}
|
|
1633
|
+
} catch (err) {
|
|
1634
|
+
log$1("Failed to extract guest token from HTML (non-fatal):", err);
|
|
1635
|
+
}
|
|
1636
|
+
} catch (err) {
|
|
1637
|
+
log$1("Pre-flight request failed (non-fatal):", err);
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
750
1640
|
async logout() {
|
|
751
1641
|
if (!this.hasToken()) {
|
|
752
1642
|
return;
|
|
753
1643
|
}
|
|
754
1644
|
try {
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
1645
|
+
const logoutUrl = "https://api.x.com/1.1/account/logout.json";
|
|
1646
|
+
const headers = new headersPolyfill.Headers();
|
|
1647
|
+
await this.installTo(headers, logoutUrl);
|
|
1648
|
+
await this.fetch(logoutUrl, {
|
|
1649
|
+
method: "POST",
|
|
1650
|
+
headers
|
|
1651
|
+
});
|
|
760
1652
|
} catch (error) {
|
|
761
|
-
|
|
1653
|
+
log$1("Error during logout:", error);
|
|
762
1654
|
} finally {
|
|
763
1655
|
this.deleteToken();
|
|
764
1656
|
this.jar = new toughCookie.CookieJar();
|
|
765
1657
|
}
|
|
766
1658
|
}
|
|
767
|
-
async installTo(headers,
|
|
768
|
-
|
|
769
|
-
headers.set("
|
|
770
|
-
headers.set(
|
|
771
|
-
|
|
772
|
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
|
|
773
|
-
);
|
|
774
|
-
if (this.guestToken) {
|
|
775
|
-
headers.set("x-guest-token", this.guestToken);
|
|
776
|
-
}
|
|
777
|
-
await this.installCsrfToken(headers);
|
|
778
|
-
if (this.options?.experimental?.xpff) {
|
|
779
|
-
const guestId = await this.guestId();
|
|
780
|
-
if (guestId != null) {
|
|
781
|
-
const xpffHeader = await generateXPFFHeader(guestId);
|
|
782
|
-
headers.set("x-xp-forwarded-for", xpffHeader);
|
|
783
|
-
}
|
|
784
|
-
}
|
|
785
|
-
const cookie = await this.getCookieString();
|
|
786
|
-
headers.set("cookie", cookie);
|
|
1659
|
+
async installTo(headers, url, bearerTokenOverride) {
|
|
1660
|
+
await super.installTo(headers, url, bearerTokenOverride);
|
|
1661
|
+
headers.set("x-twitter-auth-type", "OAuth2Session");
|
|
1662
|
+
headers.set("x-twitter-active-user", "yes");
|
|
1663
|
+
headers.set("x-twitter-client-language", "en");
|
|
787
1664
|
}
|
|
788
1665
|
async initLogin() {
|
|
789
|
-
this.removeCookie("twitter_ads_id
|
|
790
|
-
this.removeCookie("ads_prefs
|
|
791
|
-
this.removeCookie("_twitter_sess
|
|
792
|
-
this.removeCookie("zipbox_forms_auth_token
|
|
793
|
-
this.removeCookie("lang
|
|
794
|
-
this.removeCookie("bouncer_reset_cookie
|
|
795
|
-
this.removeCookie("twid
|
|
796
|
-
this.removeCookie("twitter_ads_idb
|
|
797
|
-
this.removeCookie("email_uid
|
|
798
|
-
this.removeCookie("external_referer
|
|
799
|
-
this.removeCookie("
|
|
800
|
-
this.removeCookie("aa_u=");
|
|
801
|
-
this.removeCookie("__cf_bm=");
|
|
1666
|
+
await this.removeCookie("twitter_ads_id");
|
|
1667
|
+
await this.removeCookie("ads_prefs");
|
|
1668
|
+
await this.removeCookie("_twitter_sess");
|
|
1669
|
+
await this.removeCookie("zipbox_forms_auth_token");
|
|
1670
|
+
await this.removeCookie("lang");
|
|
1671
|
+
await this.removeCookie("bouncer_reset_cookie");
|
|
1672
|
+
await this.removeCookie("twid");
|
|
1673
|
+
await this.removeCookie("twitter_ads_idb");
|
|
1674
|
+
await this.removeCookie("email_uid");
|
|
1675
|
+
await this.removeCookie("external_referer");
|
|
1676
|
+
await this.removeCookie("aa_u");
|
|
802
1677
|
return await this.executeFlowTask({
|
|
803
1678
|
flow_name: "login",
|
|
804
1679
|
input_flow_data: {
|
|
805
1680
|
flow_context: {
|
|
806
1681
|
debug_overrides: {},
|
|
807
1682
|
start_location: {
|
|
808
|
-
location: "
|
|
1683
|
+
location: "manual_link"
|
|
809
1684
|
}
|
|
810
1685
|
}
|
|
811
1686
|
},
|
|
@@ -854,20 +1729,157 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
854
1729
|
}
|
|
855
1730
|
});
|
|
856
1731
|
}
|
|
857
|
-
async handleJsInstrumentationSubtask(subtaskId,
|
|
1732
|
+
async handleJsInstrumentationSubtask(subtaskId, prev, _credentials, api) {
|
|
1733
|
+
const subtasks = prev.subtasks;
|
|
1734
|
+
const jsSubtask = subtasks?.find((s) => s.subtask_id === subtaskId);
|
|
1735
|
+
const jsUrl = jsSubtask?.js_instrumentation?.url;
|
|
1736
|
+
let metricsResponse = "{}";
|
|
1737
|
+
if (jsUrl) {
|
|
1738
|
+
try {
|
|
1739
|
+
metricsResponse = await this.executeJsInstrumentation(jsUrl);
|
|
1740
|
+
log$1(
|
|
1741
|
+
`JS instrumentation executed successfully, response length: ${metricsResponse.length}`
|
|
1742
|
+
);
|
|
1743
|
+
} catch (err) {
|
|
1744
|
+
log$1("Failed to execute JS instrumentation (falling back to {})", err);
|
|
1745
|
+
}
|
|
1746
|
+
}
|
|
858
1747
|
return await api.sendFlowRequest({
|
|
859
1748
|
flow_token: api.getFlowToken(),
|
|
860
1749
|
subtask_inputs: [
|
|
861
1750
|
{
|
|
862
1751
|
subtask_id: subtaskId,
|
|
863
1752
|
js_instrumentation: {
|
|
864
|
-
response:
|
|
1753
|
+
response: metricsResponse,
|
|
865
1754
|
link: "next_link"
|
|
866
1755
|
}
|
|
867
1756
|
}
|
|
868
1757
|
]
|
|
869
1758
|
});
|
|
870
1759
|
}
|
|
1760
|
+
// 512KB
|
|
1761
|
+
/**
|
|
1762
|
+
* Fetches and executes the JS instrumentation script to generate browser
|
|
1763
|
+
* fingerprinting data. The result is written to an input element named
|
|
1764
|
+
* 'ui_metrics'.
|
|
1765
|
+
*
|
|
1766
|
+
* In browser environments, uses a hidden iframe with native DOM APIs.
|
|
1767
|
+
* In Node.js, uses linkedom (for DOM) and the vm module for execution.
|
|
1768
|
+
*
|
|
1769
|
+
* @security This method executes **remote JavaScript** fetched from Twitter's servers.
|
|
1770
|
+
* - In browsers, execution is isolated in a disposable iframe.
|
|
1771
|
+
* - In Node.js, `vm.runInContext` is used for convenience, NOT for security.
|
|
1772
|
+
* Node's `vm` module provides NO security sandbox — a malicious script can
|
|
1773
|
+
* trivially escape the context (e.g., via `this.constructor.constructor('return process')()`).
|
|
1774
|
+
* The only real trust boundary is that scripts are fetched from Twitter's known CDN URLs.
|
|
1775
|
+
* Setting `process: undefined` etc. in the sandbox context is cosmetic and does not
|
|
1776
|
+
* prevent escape.
|
|
1777
|
+
* - A maximum script size limit (512KB) and a 5-second timeout provide basic sanity checks.
|
|
1778
|
+
*/
|
|
1779
|
+
async executeJsInstrumentation(url) {
|
|
1780
|
+
log$1(`Fetching JS instrumentation from: ${url}`);
|
|
1781
|
+
const response = await this.fetch(url);
|
|
1782
|
+
const scriptContent = await response.text();
|
|
1783
|
+
log$1(`JS instrumentation script fetched, length: ${scriptContent.length}`);
|
|
1784
|
+
if (scriptContent.length > _TwitterUserAuth.JS_INSTRUMENTATION_MAX_SIZE) {
|
|
1785
|
+
log$1(
|
|
1786
|
+
`WARNING: JS instrumentation script exceeds size limit (${scriptContent.length} > ${_TwitterUserAuth.JS_INSTRUMENTATION_MAX_SIZE}), skipping execution`
|
|
1787
|
+
);
|
|
1788
|
+
return "{}";
|
|
1789
|
+
}
|
|
1790
|
+
if (typeof window !== "undefined" && typeof document !== "undefined") {
|
|
1791
|
+
return this.executeJsInstrumentationBrowser(scriptContent);
|
|
1792
|
+
}
|
|
1793
|
+
return this.executeJsInstrumentationNode(scriptContent);
|
|
1794
|
+
}
|
|
1795
|
+
/**
|
|
1796
|
+
* Execute JS instrumentation in a browser environment using a hidden iframe.
|
|
1797
|
+
* The iframe provides natural isolation — the script gets its own document
|
|
1798
|
+
* and window, and we can override setTimeout without affecting the host page.
|
|
1799
|
+
*/
|
|
1800
|
+
async executeJsInstrumentationBrowser(scriptContent) {
|
|
1801
|
+
const iframe = document.createElement("iframe");
|
|
1802
|
+
iframe.style.display = "none";
|
|
1803
|
+
document.body.appendChild(iframe);
|
|
1804
|
+
try {
|
|
1805
|
+
const iframeWin = iframe.contentWindow;
|
|
1806
|
+
const iframeDoc = iframe.contentDocument;
|
|
1807
|
+
if (!iframeWin || !iframeDoc) {
|
|
1808
|
+
log$1("WARNING: Could not access iframe document/window");
|
|
1809
|
+
return "{}";
|
|
1810
|
+
}
|
|
1811
|
+
const input = iframeDoc.createElement("input");
|
|
1812
|
+
input.name = "ui_metrics";
|
|
1813
|
+
input.type = "hidden";
|
|
1814
|
+
iframeDoc.body.appendChild(input);
|
|
1815
|
+
iframeWin.setTimeout = (fn) => fn();
|
|
1816
|
+
const script = iframeDoc.createElement("script");
|
|
1817
|
+
script.textContent = scriptContent;
|
|
1818
|
+
iframeDoc.body.appendChild(script);
|
|
1819
|
+
const value = input.value;
|
|
1820
|
+
if (value) {
|
|
1821
|
+
log$1(`JS instrumentation result extracted, length: ${value.length}`);
|
|
1822
|
+
return value;
|
|
1823
|
+
}
|
|
1824
|
+
log$1("WARNING: No ui_metrics value found after script execution");
|
|
1825
|
+
return "{}";
|
|
1826
|
+
} finally {
|
|
1827
|
+
document.body.removeChild(iframe);
|
|
1828
|
+
}
|
|
1829
|
+
}
|
|
1830
|
+
/**
|
|
1831
|
+
* Execute JS instrumentation in Node.js using linkedom for DOM emulation
|
|
1832
|
+
* and the vm module for sandboxed script execution.
|
|
1833
|
+
*
|
|
1834
|
+
* @security Node's `vm` module does NOT provide a security sandbox. A
|
|
1835
|
+
* malicious script can trivially escape the context. The only real trust
|
|
1836
|
+
* boundary is that scripts come from Twitter's CDN. The undefined globals
|
|
1837
|
+
* (process, require, etc.) are cosmetic — they do not prevent sandbox escape.
|
|
1838
|
+
*/
|
|
1839
|
+
async executeJsInstrumentationNode(scriptContent) {
|
|
1840
|
+
const { parseHTML } = await import('linkedom');
|
|
1841
|
+
const { document: doc, window: win } = parseHTML(
|
|
1842
|
+
'<html><head></head><body><input name="ui_metrics" type="hidden" value="" /></body></html>'
|
|
1843
|
+
);
|
|
1844
|
+
if (typeof doc.getElementsByName !== "function") {
|
|
1845
|
+
doc.getElementsByName = (name) => doc.querySelectorAll(`[name="${name}"]`);
|
|
1846
|
+
}
|
|
1847
|
+
const vm = await import('vm');
|
|
1848
|
+
const origSetTimeout = win.setTimeout;
|
|
1849
|
+
win.setTimeout = (fn) => fn();
|
|
1850
|
+
try {
|
|
1851
|
+
Object.defineProperty(doc, "readyState", {
|
|
1852
|
+
value: "complete",
|
|
1853
|
+
writable: true,
|
|
1854
|
+
configurable: true
|
|
1855
|
+
});
|
|
1856
|
+
} catch {
|
|
1857
|
+
}
|
|
1858
|
+
const sandbox = {
|
|
1859
|
+
document: doc,
|
|
1860
|
+
window: win,
|
|
1861
|
+
Date,
|
|
1862
|
+
JSON,
|
|
1863
|
+
parseInt,
|
|
1864
|
+
// Deny access to Node.js internals to limit sandbox escape surface
|
|
1865
|
+
process: void 0,
|
|
1866
|
+
require: void 0,
|
|
1867
|
+
global: void 0,
|
|
1868
|
+
globalThis: void 0
|
|
1869
|
+
};
|
|
1870
|
+
vm.runInNewContext(scriptContent, sandbox, { timeout: 5e3 });
|
|
1871
|
+
win.setTimeout = origSetTimeout;
|
|
1872
|
+
const inputs = doc.getElementsByName("ui_metrics");
|
|
1873
|
+
if (inputs && inputs.length > 0) {
|
|
1874
|
+
const value = inputs[0].value || inputs[0].getAttribute("value");
|
|
1875
|
+
if (value) {
|
|
1876
|
+
log$1(`JS instrumentation result extracted, length: ${value.length}`);
|
|
1877
|
+
return value;
|
|
1878
|
+
}
|
|
1879
|
+
}
|
|
1880
|
+
log$1("WARNING: No ui_metrics value found after script execution");
|
|
1881
|
+
return "{}";
|
|
1882
|
+
}
|
|
871
1883
|
async handleEnterAlternateIdentifierSubtask(subtaskId, _prev, credentials, api) {
|
|
872
1884
|
return await this.executeFlowTask({
|
|
873
1885
|
flow_token: api.getFlowToken(),
|
|
@@ -883,36 +1895,76 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
883
1895
|
});
|
|
884
1896
|
}
|
|
885
1897
|
async handleEnterUserIdentifierSSO(subtaskId, _prev, credentials, api) {
|
|
1898
|
+
let castleToken;
|
|
1899
|
+
try {
|
|
1900
|
+
castleToken = await this.generateCastleToken();
|
|
1901
|
+
log$1(`Castle token generated, length: ${castleToken.length}`);
|
|
1902
|
+
} catch (err) {
|
|
1903
|
+
log$1("Failed to generate castle token (continuing without it):", err);
|
|
1904
|
+
}
|
|
1905
|
+
const settingsList = {
|
|
1906
|
+
setting_responses: [
|
|
1907
|
+
{
|
|
1908
|
+
key: "user_identifier",
|
|
1909
|
+
response_data: {
|
|
1910
|
+
text_data: { result: credentials.username }
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
],
|
|
1914
|
+
link: "next_link"
|
|
1915
|
+
};
|
|
1916
|
+
if (castleToken) {
|
|
1917
|
+
settingsList.castle_token = castleToken;
|
|
1918
|
+
}
|
|
886
1919
|
return await this.executeFlowTask({
|
|
887
1920
|
flow_token: api.getFlowToken(),
|
|
888
1921
|
subtask_inputs: [
|
|
889
1922
|
{
|
|
890
1923
|
subtask_id: subtaskId,
|
|
891
|
-
settings_list:
|
|
892
|
-
setting_responses: [
|
|
893
|
-
{
|
|
894
|
-
key: "user_identifier",
|
|
895
|
-
response_data: {
|
|
896
|
-
text_data: { result: credentials.username }
|
|
897
|
-
}
|
|
898
|
-
}
|
|
899
|
-
],
|
|
900
|
-
link: "next_link"
|
|
901
|
-
}
|
|
1924
|
+
settings_list: settingsList
|
|
902
1925
|
}
|
|
903
1926
|
]
|
|
904
1927
|
});
|
|
905
1928
|
}
|
|
1929
|
+
/**
|
|
1930
|
+
* Generates a Castle.io device fingerprint token for the login flow.
|
|
1931
|
+
* Uses local token generation (Castle.io v11 format) to avoid external
|
|
1932
|
+
* API dependencies and rate limits.
|
|
1933
|
+
*/
|
|
1934
|
+
async generateCastleToken() {
|
|
1935
|
+
const userAgent = CHROME_USER_AGENT;
|
|
1936
|
+
const browserProfile = this.options?.experimental?.browserProfile;
|
|
1937
|
+
const { token, cuid } = generateLocalCastleToken(userAgent, browserProfile);
|
|
1938
|
+
await this.setCookie("__cuid", cuid);
|
|
1939
|
+
log$1(
|
|
1940
|
+
`Castle token generated locally, length: ${token.length}, cuid: ${cuid.substring(0, 6)}...`
|
|
1941
|
+
);
|
|
1942
|
+
return token;
|
|
1943
|
+
}
|
|
906
1944
|
async handleEnterPassword(subtaskId, _prev, credentials, api) {
|
|
1945
|
+
let castleToken;
|
|
1946
|
+
try {
|
|
1947
|
+
castleToken = await this.generateCastleToken();
|
|
1948
|
+
log$1(`Castle token for password step, length: ${castleToken.length}`);
|
|
1949
|
+
} catch (err) {
|
|
1950
|
+
log$1(
|
|
1951
|
+
"Failed to generate castle token for password (continuing without):",
|
|
1952
|
+
err
|
|
1953
|
+
);
|
|
1954
|
+
}
|
|
1955
|
+
const enterPassword = {
|
|
1956
|
+
password: credentials.password,
|
|
1957
|
+
link: "next_link"
|
|
1958
|
+
};
|
|
1959
|
+
if (castleToken) {
|
|
1960
|
+
enterPassword.castle_token = castleToken;
|
|
1961
|
+
}
|
|
907
1962
|
return await this.executeFlowTask({
|
|
908
1963
|
flow_token: api.getFlowToken(),
|
|
909
1964
|
subtask_inputs: [
|
|
910
1965
|
{
|
|
911
1966
|
subtask_id: subtaskId,
|
|
912
|
-
enter_password:
|
|
913
|
-
password: credentials.password,
|
|
914
|
-
link: "next_link"
|
|
915
|
-
}
|
|
1967
|
+
enter_password: enterPassword
|
|
916
1968
|
}
|
|
917
1969
|
]
|
|
918
1970
|
});
|
|
@@ -940,27 +1992,28 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
940
1992
|
};
|
|
941
1993
|
}
|
|
942
1994
|
const totp = new OTPAuth__namespace.TOTP({ secret: credentials.twoFactorSecret });
|
|
943
|
-
let
|
|
1995
|
+
let lastResult;
|
|
944
1996
|
for (let attempts = 1; attempts < 4; attempts += 1) {
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
text: totp.generate()
|
|
954
|
-
}
|
|
1997
|
+
const result = await api.sendFlowRequest({
|
|
1998
|
+
flow_token: api.getFlowToken(),
|
|
1999
|
+
subtask_inputs: [
|
|
2000
|
+
{
|
|
2001
|
+
subtask_id: subtaskId,
|
|
2002
|
+
enter_text: {
|
|
2003
|
+
link: "next_link",
|
|
2004
|
+
text: totp.generate()
|
|
955
2005
|
}
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
}
|
|
959
|
-
|
|
960
|
-
|
|
2006
|
+
}
|
|
2007
|
+
]
|
|
2008
|
+
});
|
|
2009
|
+
if (result.status === "success") {
|
|
2010
|
+
return result;
|
|
961
2011
|
}
|
|
2012
|
+
lastResult = result;
|
|
2013
|
+
log$1(`2FA attempt ${attempts} failed: ${result.err.message}`);
|
|
2014
|
+
await new Promise((resolve) => setTimeout(resolve, 2e3 * attempts));
|
|
962
2015
|
}
|
|
963
|
-
|
|
2016
|
+
return lastResult;
|
|
964
2017
|
}
|
|
965
2018
|
async handleAcid(subtaskId, _prev, credentials, api) {
|
|
966
2019
|
return await this.executeFlowTask({
|
|
@@ -977,7 +2030,7 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
977
2030
|
});
|
|
978
2031
|
}
|
|
979
2032
|
async handleSuccessSubtask() {
|
|
980
|
-
log("Successfully logged in with user credentials.");
|
|
2033
|
+
log$1("Successfully logged in with user credentials.");
|
|
981
2034
|
return {
|
|
982
2035
|
status: "success",
|
|
983
2036
|
response: {}
|
|
@@ -988,28 +2041,33 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
988
2041
|
if ("flow_name" in data) {
|
|
989
2042
|
onboardingTaskUrl = `https://api.x.com/1.1/onboarding/task.json?flow_name=${data.flow_name}`;
|
|
990
2043
|
}
|
|
991
|
-
log(`Making POST request to ${onboardingTaskUrl}`);
|
|
2044
|
+
log$1(`Making POST request to ${onboardingTaskUrl}`);
|
|
2045
|
+
log$1(
|
|
2046
|
+
"Request data:",
|
|
2047
|
+
JSON.stringify(
|
|
2048
|
+
data,
|
|
2049
|
+
(key, value) => key === "password" ? "[REDACTED]" : value,
|
|
2050
|
+
2
|
|
2051
|
+
)
|
|
2052
|
+
);
|
|
992
2053
|
const headers = new headersPolyfill.Headers({
|
|
993
2054
|
accept: "*/*",
|
|
994
2055
|
"accept-language": "en-US,en;q=0.9",
|
|
995
2056
|
"content-type": "application/json",
|
|
996
|
-
"cache-control": "no-cache",
|
|
997
2057
|
origin: "https://x.com",
|
|
998
|
-
pragma: "no-cache",
|
|
999
2058
|
priority: "u=1, i",
|
|
1000
2059
|
referer: "https://x.com/",
|
|
1001
|
-
"sec-ch-ua":
|
|
2060
|
+
"sec-ch-ua": CHROME_SEC_CH_UA,
|
|
1002
2061
|
"sec-ch-ua-mobile": "?0",
|
|
1003
2062
|
"sec-ch-ua-platform": '"Windows"',
|
|
1004
2063
|
"sec-fetch-dest": "empty",
|
|
1005
2064
|
"sec-fetch-mode": "cors",
|
|
1006
|
-
"sec-fetch-site": "same-
|
|
1007
|
-
"user-agent":
|
|
1008
|
-
"x-twitter-auth-type": "OAuth2Client",
|
|
2065
|
+
"sec-fetch-site": "same-site",
|
|
2066
|
+
"user-agent": CHROME_USER_AGENT,
|
|
1009
2067
|
"x-twitter-active-user": "yes",
|
|
1010
2068
|
"x-twitter-client-language": "en"
|
|
1011
2069
|
});
|
|
1012
|
-
await this.
|
|
2070
|
+
await this.installAuthCredentials(headers);
|
|
1013
2071
|
if (this.options?.experimental?.xClientTransactionId) {
|
|
1014
2072
|
const transactionId = await generateTransactionId(
|
|
1015
2073
|
onboardingTaskUrl,
|
|
@@ -1018,6 +2076,10 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1018
2076
|
);
|
|
1019
2077
|
headers.set("x-client-transaction-id", transactionId);
|
|
1020
2078
|
}
|
|
2079
|
+
const bodyData = { ...data };
|
|
2080
|
+
if ("flow_name" in bodyData) {
|
|
2081
|
+
delete bodyData.flow_name;
|
|
2082
|
+
}
|
|
1021
2083
|
let res;
|
|
1022
2084
|
do {
|
|
1023
2085
|
const fetchParameters = [
|
|
@@ -1026,7 +2088,7 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1026
2088
|
credentials: "include",
|
|
1027
2089
|
method: "POST",
|
|
1028
2090
|
headers,
|
|
1029
|
-
body: JSON.stringify(
|
|
2091
|
+
body: JSON.stringify(bodyData)
|
|
1030
2092
|
}
|
|
1031
2093
|
];
|
|
1032
2094
|
try {
|
|
@@ -1041,25 +2103,58 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1041
2103
|
};
|
|
1042
2104
|
}
|
|
1043
2105
|
await updateCookieJar(this.jar, res.headers);
|
|
2106
|
+
log$1(`Response status: ${res.status}`);
|
|
1044
2107
|
if (res.status === 429) {
|
|
1045
|
-
log("Rate limit hit, waiting before retrying...");
|
|
2108
|
+
log$1("Rate limit hit, waiting before retrying...");
|
|
1046
2109
|
await this.onRateLimit({
|
|
1047
2110
|
fetchParameters,
|
|
1048
2111
|
response: res
|
|
1049
2112
|
});
|
|
1050
2113
|
}
|
|
1051
2114
|
} while (res.status === 429);
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
2115
|
+
let flow;
|
|
2116
|
+
try {
|
|
2117
|
+
flow = await flexParseJson(res);
|
|
2118
|
+
} catch {
|
|
2119
|
+
if (!res.ok) {
|
|
2120
|
+
return {
|
|
2121
|
+
status: "error",
|
|
2122
|
+
err: new ApiError(res, "Failed to parse response body")
|
|
2123
|
+
};
|
|
2124
|
+
}
|
|
1057
2125
|
return {
|
|
1058
2126
|
status: "error",
|
|
1059
|
-
err: new AuthenticationError("
|
|
2127
|
+
err: new AuthenticationError("Failed to parse flow response.")
|
|
1060
2128
|
};
|
|
1061
2129
|
}
|
|
2130
|
+
log$1(
|
|
2131
|
+
"Flow response: status=%s subtasks=%s",
|
|
2132
|
+
flow.status,
|
|
2133
|
+
flow.subtasks?.map((s) => s.subtask_id).join(", ")
|
|
2134
|
+
);
|
|
1062
2135
|
if (flow.errors?.length) {
|
|
2136
|
+
log$1("Twitter auth flow errors:", JSON.stringify(flow.errors, null, 2));
|
|
2137
|
+
if (flow.errors[0].code === 399) {
|
|
2138
|
+
const message = flow.errors[0].message || "";
|
|
2139
|
+
const challengeMatch = message.match(/g;[^:]+:[^:]+:[0-9]+/);
|
|
2140
|
+
if (challengeMatch) {
|
|
2141
|
+
log$1("Twitter challenge token detected:", challengeMatch[0]);
|
|
2142
|
+
}
|
|
2143
|
+
return {
|
|
2144
|
+
status: "error",
|
|
2145
|
+
err: new AuthenticationError(
|
|
2146
|
+
`Twitter blocked this login attempt due to suspicious activity (error 399). This is not an issue with your credentials - Twitter requires additional authentication.
|
|
2147
|
+
|
|
2148
|
+
Solutions:
|
|
2149
|
+
1. Use cookie-based authentication (RECOMMENDED): Export cookies from your browser and use scraper.setCookies() - see README for details
|
|
2150
|
+
2. Enable Two-Factor Authentication (2FA) on your account and provide totp_secret
|
|
2151
|
+
3. Wait 15 minutes before retrying (Twitter rate limit for suspicious logins)
|
|
2152
|
+
4. Login via browser first to establish device trust
|
|
2153
|
+
|
|
2154
|
+
Original error: ${message}`
|
|
2155
|
+
)
|
|
2156
|
+
};
|
|
2157
|
+
}
|
|
1063
2158
|
return {
|
|
1064
2159
|
status: "error",
|
|
1065
2160
|
err: new AuthenticationError(
|
|
@@ -1067,6 +2162,15 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1067
2162
|
)
|
|
1068
2163
|
};
|
|
1069
2164
|
}
|
|
2165
|
+
if (!res.ok) {
|
|
2166
|
+
return { status: "error", err: new ApiError(res, flow) };
|
|
2167
|
+
}
|
|
2168
|
+
if (flow?.flow_token == null) {
|
|
2169
|
+
return {
|
|
2170
|
+
status: "error",
|
|
2171
|
+
err: new AuthenticationError("flow_token not found.")
|
|
2172
|
+
};
|
|
2173
|
+
}
|
|
1070
2174
|
if (typeof flow.flow_token !== "string") {
|
|
1071
2175
|
return {
|
|
1072
2176
|
status: "error",
|
|
@@ -1074,7 +2178,12 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1074
2178
|
};
|
|
1075
2179
|
}
|
|
1076
2180
|
const subtask = flow.subtasks?.length ? flow.subtasks[0] : void 0;
|
|
1077
|
-
value.Check(TwitterUserAuthSubtask, subtask)
|
|
2181
|
+
if (subtask && !value.Check(TwitterUserAuthSubtask, subtask)) {
|
|
2182
|
+
log$1(
|
|
2183
|
+
"WARNING: Subtask failed schema validation: %s",
|
|
2184
|
+
subtask.subtask_id ?? "unknown"
|
|
2185
|
+
);
|
|
2186
|
+
}
|
|
1078
2187
|
if (subtask && subtask.subtask_id === "DenyLoginSubtask") {
|
|
1079
2188
|
return {
|
|
1080
2189
|
status: "error",
|
|
@@ -1086,19 +2195,26 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1086
2195
|
response: flow
|
|
1087
2196
|
};
|
|
1088
2197
|
}
|
|
1089
|
-
}
|
|
2198
|
+
};
|
|
2199
|
+
/**
|
|
2200
|
+
* Maximum allowed size (in bytes) for the JS instrumentation script.
|
|
2201
|
+
* Twitter's scripts are typically ~50-100KB. Anything significantly larger
|
|
2202
|
+
* may indicate tampering or an unexpected response.
|
|
2203
|
+
*/
|
|
2204
|
+
_TwitterUserAuth.JS_INSTRUMENTATION_MAX_SIZE = 512 * 1024;
|
|
2205
|
+
let TwitterUserAuth = _TwitterUserAuth;
|
|
1090
2206
|
|
|
1091
2207
|
const endpoints = {
|
|
1092
|
-
UserTweets: "https://x.com/
|
|
1093
|
-
UserTweetsAndReplies: "https://x.com/
|
|
1094
|
-
UserLikedTweets: "https://x.com/
|
|
1095
|
-
UserByScreenName: "https://api.x.com/graphql
|
|
1096
|
-
TweetDetail: "https://x.com/
|
|
1097
|
-
TweetResultByRestId: "https://api.x.com/graphql/
|
|
1098
|
-
ListTweets: "https://x.com/
|
|
1099
|
-
SearchTimeline: "https://x.com/
|
|
1100
|
-
Followers: "https://x.com/
|
|
1101
|
-
Following: "https://x.com/
|
|
2208
|
+
UserTweets: "https://api.x.com/graphql/N2tFDY-MlrLxXJ9F_ZxJGA/UserTweets?variables=%7B%22userId%22%3A%2244196397%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
2209
|
+
UserTweetsAndReplies: "https://api.x.com/graphql/2NDLUdBmT_IB5uGwZ3tHRg/UserTweetsAndReplies?variables=%7B%22userId%22%3A%221806359170830172162%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
2210
|
+
UserLikedTweets: "https://api.x.com/graphql/Pcw-j9lrSeDMmkgnIejJiQ/Likes?variables=%7B%22userId%22%3A%222244196397%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withClientEventToken%22%3Afalse%2C%22withBirdwatchNotes%22%3Afalse%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
2211
|
+
UserByScreenName: "https://api.x.com/graphql/AWbeRIdkLtqTRN7yL_H8yw/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withGrokTranslatedBio%22%3Atrue%7D&features=%7B%22hidden_profile_subscriptions_enabled%22%3Atrue%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_is_identity_verified_enabled%22%3Atrue%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22responsive_web_twitter_article_notes_tab_enabled%22%3Atrue%2C%22subscriptions_feature_can_gift_premium%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withPayments%22%3Afalse%2C%22withAuxiliaryUserLabels%22%3Atrue%7D",
|
|
2212
|
+
TweetDetail: "https://api.x.com/graphql/YCNdW_ZytXfV9YR3cJK9kw/TweetDetail?variables=%7B%22focalTweetId%22%3A%221985465713096794294%22%2C%22with_rux_injections%22%3Afalse%2C%22rankingMode%22%3A%22Relevance%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Atrue%2C%22withArticlePlainText%22%3Afalse%2C%22withGrokAnalyze%22%3Afalse%2C%22withDisallowedReplyControls%22%3Afalse%7D",
|
|
2213
|
+
TweetResultByRestId: "https://api.x.com/graphql/4PdbzTmQ5PTjz9RiureISQ/TweetResultByRestId?variables=%7B%22tweetId%22%3A%221985465713096794294%22%2C%22includePromotedContent%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withCommunity%22%3Atrue%7D&features=%7B%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Atrue%2C%22withArticlePlainText%22%3Afalse%7D",
|
|
2214
|
+
ListTweets: "https://api.x.com/graphql/Uv3buKIUElzL3Iuc0L0O5g/ListLatestTweetsTimeline?variables=%7B%22listId%22%3A%221736495155002106192%22%2C%22count%22%3A20%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D",
|
|
2215
|
+
SearchTimeline: "https://api.x.com/graphql/ML-n2SfAxx5S_9QMqNejbg/SearchTimeline?variables=%7B%22rawQuery%22%3A%22twitter%22%2C%22count%22%3A20%2C%22querySource%22%3A%22typed_query%22%2C%22product%22%3A%22Top%22%2C%22withGrokTranslatedBio%22%3Afalse%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D",
|
|
2216
|
+
Followers: "https://api.x.com/graphql/P7m4Qr-rJEB8KUluOenU6A/Followers?variables=%7B%22userId%22%3A%221806359170830172162%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withGrokTranslatedBio%22%3Afalse%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D",
|
|
2217
|
+
Following: "https://api.x.com/graphql/T5wihsMTYHncY7BB4YxHSg/Following?variables=%7B%22userId%22%3A%221806359170830172162%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withGrokTranslatedBio%22%3Afalse%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D"
|
|
1102
2218
|
};
|
|
1103
2219
|
class ApiRequest {
|
|
1104
2220
|
constructor(info) {
|
|
@@ -1897,6 +3013,15 @@ function parseRelationshipTimeline(timeline) {
|
|
|
1897
3013
|
if (!profile.userId) {
|
|
1898
3014
|
profile.userId = userResultRaw.rest_id;
|
|
1899
3015
|
}
|
|
3016
|
+
if (!profile.username && userResultRaw.core?.screen_name) {
|
|
3017
|
+
profile.username = userResultRaw.core.screen_name;
|
|
3018
|
+
profile.url = `https://x.com/${profile.username}`;
|
|
3019
|
+
}
|
|
3020
|
+
if (!profile.joined && userResultRaw.core?.created_at) {
|
|
3021
|
+
profile.joined = new Date(
|
|
3022
|
+
Date.parse(userResultRaw.core.created_at)
|
|
3023
|
+
);
|
|
3024
|
+
}
|
|
1900
3025
|
profiles.push(profile);
|
|
1901
3026
|
}
|
|
1902
3027
|
} else if (entry.content?.cursorType === "Bottom") {
|
|
@@ -2449,6 +3574,7 @@ function findDmConversationsByUserId(inbox, userId) {
|
|
|
2449
3574
|
return conversations;
|
|
2450
3575
|
}
|
|
2451
3576
|
|
|
3577
|
+
const log = debug("twitter-scraper:scraper");
|
|
2452
3578
|
const twUrl = "https://x.com";
|
|
2453
3579
|
class Scraper {
|
|
2454
3580
|
/**
|
|
@@ -2458,6 +3584,7 @@ class Scraper {
|
|
|
2458
3584
|
*/
|
|
2459
3585
|
constructor(options) {
|
|
2460
3586
|
this.options = options;
|
|
3587
|
+
this.subtaskHandlers = /* @__PURE__ */ new Map();
|
|
2461
3588
|
this.token = bearerToken;
|
|
2462
3589
|
this.useGuestAuth();
|
|
2463
3590
|
}
|
|
@@ -2468,6 +3595,7 @@ class Scraper {
|
|
|
2468
3595
|
* @param subtaskHandler The handler function to register.
|
|
2469
3596
|
*/
|
|
2470
3597
|
registerAuthSubtaskHandler(subtaskId, subtaskHandler) {
|
|
3598
|
+
this.subtaskHandlers.set(subtaskId, subtaskHandler);
|
|
2471
3599
|
if (this.auth instanceof TwitterUserAuth) {
|
|
2472
3600
|
this.auth.registerSubtaskHandler(subtaskId, subtaskHandler);
|
|
2473
3601
|
}
|
|
@@ -2475,6 +3603,15 @@ class Scraper {
|
|
|
2475
3603
|
this.authTrends.registerSubtaskHandler(subtaskId, subtaskHandler);
|
|
2476
3604
|
}
|
|
2477
3605
|
}
|
|
3606
|
+
/**
|
|
3607
|
+
* Applies all stored subtask handlers to the given auth instance.
|
|
3608
|
+
* @internal
|
|
3609
|
+
*/
|
|
3610
|
+
applySubtaskHandlers(auth) {
|
|
3611
|
+
for (const [subtaskId, handler] of this.subtaskHandlers) {
|
|
3612
|
+
auth.registerSubtaskHandler(subtaskId, handler);
|
|
3613
|
+
}
|
|
3614
|
+
}
|
|
2478
3615
|
/**
|
|
2479
3616
|
* Initializes auth properties using a guest token.
|
|
2480
3617
|
* Used when creating a new instance of this class, and when logging out.
|
|
@@ -2775,7 +3912,8 @@ class Scraper {
|
|
|
2775
3912
|
* @param twoFactorSecret The secret to generate two factor authentication tokens with, if you have two factor authentication enabled.
|
|
2776
3913
|
*/
|
|
2777
3914
|
async login(username, password, email, twoFactorSecret) {
|
|
2778
|
-
const userAuth = new TwitterUserAuth(
|
|
3915
|
+
const userAuth = new TwitterUserAuth(bearerToken2, this.getAuthOptions());
|
|
3916
|
+
this.applySubtaskHandlers(userAuth);
|
|
2779
3917
|
await userAuth.login(username, password, email, twoFactorSecret);
|
|
2780
3918
|
this.auth = userAuth;
|
|
2781
3919
|
this.authTrends = userAuth;
|
|
@@ -2802,12 +3940,37 @@ class Scraper {
|
|
|
2802
3940
|
* @param cookies The cookies to set for the current session.
|
|
2803
3941
|
*/
|
|
2804
3942
|
async setCookies(cookies) {
|
|
2805
|
-
const userAuth = new TwitterUserAuth(
|
|
3943
|
+
const userAuth = new TwitterUserAuth(bearerToken2, this.getAuthOptions());
|
|
3944
|
+
this.applySubtaskHandlers(userAuth);
|
|
2806
3945
|
for (const cookie of cookies) {
|
|
2807
|
-
|
|
3946
|
+
if (cookie == null) continue;
|
|
3947
|
+
if (typeof cookie === "string") {
|
|
3948
|
+
try {
|
|
3949
|
+
await userAuth.cookieJar().setCookie(cookie, "https://x.com");
|
|
3950
|
+
} catch (err) {
|
|
3951
|
+
log(`Failed to parse cookie string: ${err.message}`);
|
|
3952
|
+
}
|
|
3953
|
+
} else {
|
|
3954
|
+
if (cookie.domain && cookie.domain.startsWith(".")) {
|
|
3955
|
+
cookie.domain = cookie.domain.substring(1);
|
|
3956
|
+
cookie.hostOnly = false;
|
|
3957
|
+
}
|
|
3958
|
+
const cookieDomain = cookie.domain || "x.com";
|
|
3959
|
+
const cookieUrl = `https://${cookieDomain}`;
|
|
3960
|
+
await userAuth.cookieJar().setCookie(cookie, cookieUrl);
|
|
3961
|
+
}
|
|
2808
3962
|
}
|
|
2809
3963
|
this.auth = userAuth;
|
|
2810
3964
|
this.authTrends = userAuth;
|
|
3965
|
+
const isLoggedIn = await userAuth.isLoggedIn();
|
|
3966
|
+
if (!isLoggedIn) {
|
|
3967
|
+
const cookieString = await userAuth.cookieJar().getCookies(twUrl).then((c) => c.map((cookie) => cookie.key));
|
|
3968
|
+
if (cookieString.includes("ct0") && !cookieString.includes("auth_token")) {
|
|
3969
|
+
log(
|
|
3970
|
+
"auth_token cookie is missing. This is required for authenticated API access. The auth_token is an HttpOnly cookie that cannot be accessed via document.cookie. Export it using a browser extension (e.g., EditThisCookie) or DevTools Application tab."
|
|
3971
|
+
);
|
|
3972
|
+
}
|
|
3973
|
+
}
|
|
2811
3974
|
}
|
|
2812
3975
|
/**
|
|
2813
3976
|
* Clear all cookies for the current session.
|
|
@@ -2849,7 +4012,9 @@ class Scraper {
|
|
|
2849
4012
|
rateLimitStrategy: this.options?.rateLimitStrategy,
|
|
2850
4013
|
experimental: {
|
|
2851
4014
|
xClientTransactionId: this.options?.experimental?.xClientTransactionId,
|
|
2852
|
-
xpff: this.options?.experimental?.xpff
|
|
4015
|
+
xpff: this.options?.experimental?.xpff,
|
|
4016
|
+
flowStepDelay: this.options?.experimental?.flowStepDelay,
|
|
4017
|
+
browserProfile: this.options?.experimental?.browserProfile
|
|
2853
4018
|
}
|
|
2854
4019
|
};
|
|
2855
4020
|
}
|
|
@@ -2867,4 +4032,5 @@ exports.ErrorRateLimitStrategy = ErrorRateLimitStrategy;
|
|
|
2867
4032
|
exports.Scraper = Scraper;
|
|
2868
4033
|
exports.SearchMode = SearchMode;
|
|
2869
4034
|
exports.WaitingRateLimitStrategy = WaitingRateLimitStrategy;
|
|
4035
|
+
exports.randomizeBrowserProfile = randomizeBrowserProfile;
|
|
2870
4036
|
//# sourceMappingURL=index.js.map
|