@the-convocation/twitter-scraper 0.21.1 → 0.22.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +58 -0
- package/dist/cycletls/cjs/index.cjs +48 -3
- package/dist/cycletls/cjs/index.cjs.map +1 -1
- package/dist/cycletls/esm/index.mjs +48 -3
- package/dist/cycletls/esm/index.mjs.map +1 -1
- package/dist/default/cjs/index.js +1320 -154
- package/dist/default/cjs/index.js.map +1 -1
- package/dist/default/esm/index.mjs +1320 -155
- package/dist/default/esm/index.mjs.map +1 -1
- package/dist/node/cjs/index.cjs +1320 -154
- package/dist/node/cjs/index.cjs.map +1 -1
- package/dist/node/esm/index.mjs +1320 -155
- package/dist/node/esm/index.mjs.map +1 -1
- package/dist/types/index.d.ts +70 -1
- package/package.json +4 -2
package/dist/node/cjs/index.cjs
CHANGED
|
@@ -72,13 +72,13 @@ class AuthenticationError extends Error {
|
|
|
72
72
|
}
|
|
73
73
|
}
|
|
74
74
|
|
|
75
|
-
const log$
|
|
75
|
+
const log$8 = debug("twitter-scraper:rate-limit");
|
|
76
76
|
class WaitingRateLimitStrategy {
|
|
77
77
|
async onRateLimit({ response: res }) {
|
|
78
78
|
const xRateLimitLimit = res.headers.get("x-rate-limit-limit");
|
|
79
79
|
const xRateLimitRemaining = res.headers.get("x-rate-limit-remaining");
|
|
80
80
|
const xRateLimitReset = res.headers.get("x-rate-limit-reset");
|
|
81
|
-
log$
|
|
81
|
+
log$8(
|
|
82
82
|
`Rate limit event: limit=${xRateLimitLimit}, remaining=${xRateLimitRemaining}, reset=${xRateLimitReset}`
|
|
83
83
|
);
|
|
84
84
|
if (xRateLimitRemaining == "0" && xRateLimitReset) {
|
|
@@ -94,7 +94,794 @@ class ErrorRateLimitStrategy {
|
|
|
94
94
|
}
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
-
const log$
|
|
97
|
+
const log$7 = debug("twitter-scraper:castle");
|
|
98
|
+
var FieldEncoding = /* @__PURE__ */ ((FieldEncoding2) => {
|
|
99
|
+
FieldEncoding2[FieldEncoding2["Empty"] = -1] = "Empty";
|
|
100
|
+
FieldEncoding2[FieldEncoding2["Marker"] = 1] = "Marker";
|
|
101
|
+
FieldEncoding2[FieldEncoding2["Byte"] = 3] = "Byte";
|
|
102
|
+
FieldEncoding2[FieldEncoding2["EncryptedBytes"] = 4] = "EncryptedBytes";
|
|
103
|
+
FieldEncoding2[FieldEncoding2["CompactInt"] = 5] = "CompactInt";
|
|
104
|
+
FieldEncoding2[FieldEncoding2["RoundedByte"] = 6] = "RoundedByte";
|
|
105
|
+
FieldEncoding2[FieldEncoding2["RawAppend"] = 7] = "RawAppend";
|
|
106
|
+
return FieldEncoding2;
|
|
107
|
+
})(FieldEncoding || {});
|
|
108
|
+
const TWITTER_CASTLE_PK = "AvRa79bHyJSYSQHnRpcVtzyxetSvFerx";
|
|
109
|
+
const XXTEA_KEY = [1164413191, 3891440048, 185273099, 2746598870];
|
|
110
|
+
const PER_FIELD_KEY_TAIL = [
|
|
111
|
+
16373134,
|
|
112
|
+
643144773,
|
|
113
|
+
1762804430,
|
|
114
|
+
1186572681,
|
|
115
|
+
1164413191
|
|
116
|
+
];
|
|
117
|
+
const TS_EPOCH = 1535e6;
|
|
118
|
+
const SDK_VERSION = 27008;
|
|
119
|
+
const TOKEN_VERSION = 11;
|
|
120
|
+
const FP_PART = {
|
|
121
|
+
DEVICE: 0,
|
|
122
|
+
// Part 1: hardware/OS/rendering fingerprint
|
|
123
|
+
BROWSER: 4,
|
|
124
|
+
// Part 2: browser environment fingerprint
|
|
125
|
+
TIMING: 7
|
|
126
|
+
// Part 3: timing-based fingerprint
|
|
127
|
+
};
|
|
128
|
+
const DEFAULT_PROFILE = {
|
|
129
|
+
locale: "en-US",
|
|
130
|
+
language: "en",
|
|
131
|
+
timezone: "America/New_York",
|
|
132
|
+
screenWidth: 1920,
|
|
133
|
+
screenHeight: 1080,
|
|
134
|
+
availableWidth: 1920,
|
|
135
|
+
availableHeight: 1032,
|
|
136
|
+
// 1080 minus Windows taskbar (~48px)
|
|
137
|
+
gpuRenderer: "ANGLE (NVIDIA, NVIDIA GeForce GTX 1080 Ti Direct3D11 vs_5_0 ps_5_0, D3D11)",
|
|
138
|
+
deviceMemoryGB: 8,
|
|
139
|
+
hardwareConcurrency: 24,
|
|
140
|
+
colorDepth: 24,
|
|
141
|
+
devicePixelRatio: 1
|
|
142
|
+
};
|
|
143
|
+
const SCREEN_RESOLUTIONS = [
|
|
144
|
+
{ w: 1920, h: 1080, ah: 1032 },
|
|
145
|
+
{ w: 2560, h: 1440, ah: 1392 },
|
|
146
|
+
{ w: 1366, h: 768, ah: 720 },
|
|
147
|
+
{ w: 1536, h: 864, ah: 816 },
|
|
148
|
+
{ w: 1440, h: 900, ah: 852 },
|
|
149
|
+
{ w: 1680, h: 1050, ah: 1002 },
|
|
150
|
+
{ w: 3840, h: 2160, ah: 2112 }
|
|
151
|
+
];
|
|
152
|
+
const DEVICE_MEMORY_VALUES = [4, 8, 8, 16];
|
|
153
|
+
const HARDWARE_CONCURRENCY_VALUES = [4, 8, 8, 12, 16, 24];
|
|
154
|
+
function randomizeBrowserProfile() {
|
|
155
|
+
const screen = SCREEN_RESOLUTIONS[randInt(0, SCREEN_RESOLUTIONS.length - 1)];
|
|
156
|
+
return {
|
|
157
|
+
...DEFAULT_PROFILE,
|
|
158
|
+
screenWidth: screen.w,
|
|
159
|
+
screenHeight: screen.h,
|
|
160
|
+
availableWidth: screen.w,
|
|
161
|
+
availableHeight: screen.ah,
|
|
162
|
+
// gpuRenderer intentionally NOT randomized — see JSDoc above
|
|
163
|
+
deviceMemoryGB: DEVICE_MEMORY_VALUES[randInt(0, DEVICE_MEMORY_VALUES.length - 1)],
|
|
164
|
+
hardwareConcurrency: HARDWARE_CONCURRENCY_VALUES[randInt(0, HARDWARE_CONCURRENCY_VALUES.length - 1)]
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
function getRandomBytes(n) {
|
|
168
|
+
const buf = new Uint8Array(n);
|
|
169
|
+
if (typeof globalThis.crypto !== "undefined" && globalThis.crypto.getRandomValues) {
|
|
170
|
+
globalThis.crypto.getRandomValues(buf);
|
|
171
|
+
} else {
|
|
172
|
+
for (let i = 0; i < n; i++) buf[i] = Math.floor(Math.random() * 256);
|
|
173
|
+
}
|
|
174
|
+
return buf;
|
|
175
|
+
}
|
|
176
|
+
function randInt(min, max) {
|
|
177
|
+
return min + Math.floor(Math.random() * (max - min + 1));
|
|
178
|
+
}
|
|
179
|
+
function randFloat(min, max) {
|
|
180
|
+
return min + Math.random() * (max - min);
|
|
181
|
+
}
|
|
182
|
+
function concat(...arrays) {
|
|
183
|
+
const len = arrays.reduce((s, a) => s + a.length, 0);
|
|
184
|
+
const out = new Uint8Array(len);
|
|
185
|
+
let off = 0;
|
|
186
|
+
for (const a of arrays) {
|
|
187
|
+
out.set(a, off);
|
|
188
|
+
off += a.length;
|
|
189
|
+
}
|
|
190
|
+
return out;
|
|
191
|
+
}
|
|
192
|
+
function toHex(input) {
|
|
193
|
+
return Array.from(input).map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
194
|
+
}
|
|
195
|
+
function fromHex(hex) {
|
|
196
|
+
const out = new Uint8Array(hex.length / 2);
|
|
197
|
+
for (let i = 0; i < hex.length; i += 2)
|
|
198
|
+
out[i / 2] = parseInt(hex.substring(i, i + 2), 16);
|
|
199
|
+
return out;
|
|
200
|
+
}
|
|
201
|
+
function textEnc(s) {
|
|
202
|
+
return new TextEncoder().encode(s);
|
|
203
|
+
}
|
|
204
|
+
function u8(...vals) {
|
|
205
|
+
return new Uint8Array(vals);
|
|
206
|
+
}
|
|
207
|
+
function be16(v) {
|
|
208
|
+
return u8(v >>> 8 & 255, v & 255);
|
|
209
|
+
}
|
|
210
|
+
function be32(v) {
|
|
211
|
+
return u8(v >>> 24 & 255, v >>> 16 & 255, v >>> 8 & 255, v & 255);
|
|
212
|
+
}
|
|
213
|
+
function xorBytes(data, key) {
|
|
214
|
+
const out = new Uint8Array(data.length);
|
|
215
|
+
for (let i = 0; i < data.length; i++) out[i] = data[i] ^ key[i % key.length];
|
|
216
|
+
return out;
|
|
217
|
+
}
|
|
218
|
+
function xorNibbles(nibbles, keyNibble) {
|
|
219
|
+
const k = parseInt(keyNibble, 16);
|
|
220
|
+
return nibbles.split("").map((n) => (parseInt(n, 16) ^ k).toString(16)).join("");
|
|
221
|
+
}
|
|
222
|
+
function base64url(data) {
|
|
223
|
+
if (typeof Buffer !== "undefined") {
|
|
224
|
+
return Buffer.from(data).toString("base64url");
|
|
225
|
+
}
|
|
226
|
+
let bin = "";
|
|
227
|
+
for (let i = 0; i < data.length; i++) bin += String.fromCharCode(data[i]);
|
|
228
|
+
return btoa(bin).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "");
|
|
229
|
+
}
|
|
230
|
+
function xxteaEncrypt(data, key) {
|
|
231
|
+
const padLen = Math.ceil(data.length / 4) * 4;
|
|
232
|
+
const padded = new Uint8Array(padLen);
|
|
233
|
+
padded.set(data);
|
|
234
|
+
const n = padLen / 4;
|
|
235
|
+
const v = new Uint32Array(n);
|
|
236
|
+
for (let i = 0; i < n; i++) {
|
|
237
|
+
v[i] = (padded[i * 4] | padded[i * 4 + 1] << 8 | padded[i * 4 + 2] << 16 | padded[i * 4 + 3] << 24) >>> 0;
|
|
238
|
+
}
|
|
239
|
+
if (n <= 1) return padded;
|
|
240
|
+
const k = new Uint32Array(key.map((x) => x >>> 0));
|
|
241
|
+
const DELTA = 2654435769;
|
|
242
|
+
const u = n - 1;
|
|
243
|
+
let sum = 0;
|
|
244
|
+
let z = v[u];
|
|
245
|
+
let y;
|
|
246
|
+
let rounds = 6 + Math.floor(52 / (u + 1));
|
|
247
|
+
while (rounds-- > 0) {
|
|
248
|
+
sum = sum + DELTA >>> 0;
|
|
249
|
+
const e = sum >>> 2 & 3;
|
|
250
|
+
for (let p = 0; p < u; p++) {
|
|
251
|
+
y = v[p + 1];
|
|
252
|
+
const mx2 = ((z >>> 5 ^ y << 2) >>> 0) + ((y >>> 3 ^ z << 4) >>> 0) ^ ((sum ^ y) >>> 0) + ((k[p & 3 ^ e] ^ z) >>> 0);
|
|
253
|
+
v[p] = v[p] + mx2 >>> 0;
|
|
254
|
+
z = v[p];
|
|
255
|
+
}
|
|
256
|
+
y = v[0];
|
|
257
|
+
const mx = ((z >>> 5 ^ y << 2) >>> 0) + ((y >>> 3 ^ z << 4) >>> 0) ^ ((sum ^ y) >>> 0) + ((k[u & 3 ^ e] ^ z) >>> 0);
|
|
258
|
+
v[u] = v[u] + mx >>> 0;
|
|
259
|
+
z = v[u];
|
|
260
|
+
}
|
|
261
|
+
const out = new Uint8Array(n * 4);
|
|
262
|
+
for (let i = 0; i < n; i++) {
|
|
263
|
+
out[i * 4] = v[i] & 255;
|
|
264
|
+
out[i * 4 + 1] = v[i] >>> 8 & 255;
|
|
265
|
+
out[i * 4 + 2] = v[i] >>> 16 & 255;
|
|
266
|
+
out[i * 4 + 3] = v[i] >>> 24 & 255;
|
|
267
|
+
}
|
|
268
|
+
return out;
|
|
269
|
+
}
|
|
270
|
+
function fieldEncrypt(data, fieldIndex, initTime) {
|
|
271
|
+
return xxteaEncrypt(data, [
|
|
272
|
+
fieldIndex,
|
|
273
|
+
Math.floor(initTime),
|
|
274
|
+
...PER_FIELD_KEY_TAIL
|
|
275
|
+
]);
|
|
276
|
+
}
|
|
277
|
+
function encodeTimestampBytes(ms) {
|
|
278
|
+
let t = Math.floor(ms / 1e3 - TS_EPOCH);
|
|
279
|
+
t = Math.max(Math.min(t, 268435455), 0);
|
|
280
|
+
return be32(t);
|
|
281
|
+
}
|
|
282
|
+
function xorAndAppendKey(buf, key) {
|
|
283
|
+
const hex = toHex(buf);
|
|
284
|
+
const keyNib = (key & 15).toString(16);
|
|
285
|
+
return xorNibbles(hex.substring(1), keyNib) + keyNib;
|
|
286
|
+
}
|
|
287
|
+
function encodeTimestampEncrypted(ms) {
|
|
288
|
+
const tsBytes = encodeTimestampBytes(ms);
|
|
289
|
+
const slice = Math.floor(ms) % 1e3;
|
|
290
|
+
const sliceBytes = be16(slice);
|
|
291
|
+
const k = randInt(0, 15);
|
|
292
|
+
return xorAndAppendKey(tsBytes, k) + xorAndAppendKey(sliceBytes, k);
|
|
293
|
+
}
|
|
294
|
+
function deriveAndXor(keyHex, sliceLen, rotChar, data) {
|
|
295
|
+
const sub = keyHex.substring(0, sliceLen).split("");
|
|
296
|
+
if (sub.length === 0) return data;
|
|
297
|
+
const rot = parseInt(rotChar, 16) % sub.length;
|
|
298
|
+
const rotated = sub.slice(rot).concat(sub.slice(0, rot)).join("");
|
|
299
|
+
return xorBytes(data, fromHex(rotated));
|
|
300
|
+
}
|
|
301
|
+
function customFloatEncode(expBits, manBits, value) {
|
|
302
|
+
if (value === 0) return 0;
|
|
303
|
+
let n = Math.abs(value);
|
|
304
|
+
let exp = 0;
|
|
305
|
+
while (2 <= n) {
|
|
306
|
+
n /= 2;
|
|
307
|
+
exp++;
|
|
308
|
+
}
|
|
309
|
+
while (n < 1 && n > 0) {
|
|
310
|
+
n *= 2;
|
|
311
|
+
exp--;
|
|
312
|
+
}
|
|
313
|
+
exp = Math.min(exp, (1 << expBits) - 1);
|
|
314
|
+
const frac = n - Math.floor(n);
|
|
315
|
+
let mantissa = 0;
|
|
316
|
+
if (frac > 0) {
|
|
317
|
+
let pos = 1;
|
|
318
|
+
let tmp = frac;
|
|
319
|
+
while (tmp !== 0 && pos <= manBits) {
|
|
320
|
+
tmp *= 2;
|
|
321
|
+
const bit = Math.floor(tmp);
|
|
322
|
+
mantissa |= bit << manBits - pos;
|
|
323
|
+
tmp -= bit;
|
|
324
|
+
pos++;
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
return exp << manBits | mantissa;
|
|
328
|
+
}
|
|
329
|
+
function encodeFloatVal(v) {
|
|
330
|
+
const n = Math.max(v, 0);
|
|
331
|
+
if (n <= 15) return 64 | customFloatEncode(2, 4, n + 1);
|
|
332
|
+
return 128 | customFloatEncode(4, 3, n - 14);
|
|
333
|
+
}
|
|
334
|
+
function encodeField(index, encoding, val, initTime) {
|
|
335
|
+
const hdr = u8((31 & index) << 3 | 7 & encoding);
|
|
336
|
+
if (encoding === -1 /* Empty */ || encoding === 1 /* Marker */)
|
|
337
|
+
return hdr;
|
|
338
|
+
let body;
|
|
339
|
+
switch (encoding) {
|
|
340
|
+
case 3 /* Byte */:
|
|
341
|
+
body = u8(val);
|
|
342
|
+
break;
|
|
343
|
+
case 6 /* RoundedByte */:
|
|
344
|
+
body = u8(Math.round(val));
|
|
345
|
+
break;
|
|
346
|
+
case 5 /* CompactInt */: {
|
|
347
|
+
const v = val;
|
|
348
|
+
body = v <= 127 ? u8(v) : be16(1 << 15 | 32767 & v);
|
|
349
|
+
break;
|
|
350
|
+
}
|
|
351
|
+
case 4 /* EncryptedBytes */: {
|
|
352
|
+
if (initTime == null) {
|
|
353
|
+
throw new Error("initTime is required for EncryptedBytes encoding");
|
|
354
|
+
}
|
|
355
|
+
const enc = fieldEncrypt(val, index, initTime);
|
|
356
|
+
body = concat(u8(enc.length), enc);
|
|
357
|
+
break;
|
|
358
|
+
}
|
|
359
|
+
case 7 /* RawAppend */:
|
|
360
|
+
body = val instanceof Uint8Array ? val : u8(val);
|
|
361
|
+
break;
|
|
362
|
+
default:
|
|
363
|
+
body = new Uint8Array(0);
|
|
364
|
+
}
|
|
365
|
+
return concat(hdr, body);
|
|
366
|
+
}
|
|
367
|
+
function encodeBits(bits, byteSize) {
|
|
368
|
+
const numBytes = byteSize / 8;
|
|
369
|
+
const arr = new Uint8Array(numBytes);
|
|
370
|
+
for (const bit of bits) {
|
|
371
|
+
const bi = numBytes - 1 - Math.floor(bit / 8);
|
|
372
|
+
if (bi >= 0 && bi < numBytes) arr[bi] |= 1 << bit % 8;
|
|
373
|
+
}
|
|
374
|
+
return arr;
|
|
375
|
+
}
|
|
376
|
+
function screenDimBytes(screen, avail) {
|
|
377
|
+
const r = 32767 & screen;
|
|
378
|
+
const e = 65535 & avail;
|
|
379
|
+
return r === e ? be16(32768 | r) : concat(be16(r), be16(e));
|
|
380
|
+
}
|
|
381
|
+
function boolsToBin(arr, totalBits) {
|
|
382
|
+
const e = arr.length > totalBits ? arr.slice(0, totalBits) : arr;
|
|
383
|
+
const c = e.length;
|
|
384
|
+
let r = 0;
|
|
385
|
+
for (let i = c - 1; i >= 0; i--) {
|
|
386
|
+
if (e[i]) r |= 1 << c - i - 1;
|
|
387
|
+
}
|
|
388
|
+
if (c < totalBits) r <<= totalBits - c;
|
|
389
|
+
return r;
|
|
390
|
+
}
|
|
391
|
+
function encodeCodecPlayability() {
|
|
392
|
+
const codecs = {
|
|
393
|
+
webm: 2,
|
|
394
|
+
// VP8/VP9
|
|
395
|
+
mp4: 2,
|
|
396
|
+
// H.264
|
|
397
|
+
ogg: 0,
|
|
398
|
+
// Theora (Chrome dropped support)
|
|
399
|
+
aac: 2,
|
|
400
|
+
// AAC audio
|
|
401
|
+
xm4a: 1,
|
|
402
|
+
// M4A container
|
|
403
|
+
wav: 2,
|
|
404
|
+
// PCM audio
|
|
405
|
+
mpeg: 2,
|
|
406
|
+
// MP3 audio
|
|
407
|
+
ogg2: 2
|
|
408
|
+
// Vorbis audio
|
|
409
|
+
};
|
|
410
|
+
const bits = Object.values(codecs).map((c) => c.toString(2).padStart(2, "0")).join("");
|
|
411
|
+
return be16(parseInt(bits, 2));
|
|
412
|
+
}
|
|
413
|
+
const TIMEZONE_ENUM = {
|
|
414
|
+
"America/New_York": 0,
|
|
415
|
+
"America/Sao_Paulo": 1,
|
|
416
|
+
"America/Chicago": 2,
|
|
417
|
+
"America/Los_Angeles": 3,
|
|
418
|
+
"America/Mexico_City": 4,
|
|
419
|
+
"Asia/Shanghai": 5
|
|
420
|
+
};
|
|
421
|
+
function getTimezoneInfo(tz) {
|
|
422
|
+
const knownOffsets = {
|
|
423
|
+
"America/New_York": { offset: 20, dstDiff: 4 },
|
|
424
|
+
"America/Chicago": { offset: 24, dstDiff: 4 },
|
|
425
|
+
"America/Los_Angeles": { offset: 32, dstDiff: 4 },
|
|
426
|
+
"America/Denver": { offset: 28, dstDiff: 4 },
|
|
427
|
+
"America/Sao_Paulo": { offset: 12, dstDiff: 4 },
|
|
428
|
+
"America/Mexico_City": { offset: 24, dstDiff: 4 },
|
|
429
|
+
"Asia/Shanghai": { offset: 246, dstDiff: 0 },
|
|
430
|
+
"Asia/Tokyo": { offset: 220, dstDiff: 0 },
|
|
431
|
+
"Europe/London": { offset: 0, dstDiff: 4 },
|
|
432
|
+
"Europe/Berlin": { offset: 252, dstDiff: 4 },
|
|
433
|
+
UTC: { offset: 0, dstDiff: 0 }
|
|
434
|
+
};
|
|
435
|
+
try {
|
|
436
|
+
const now = /* @__PURE__ */ new Date();
|
|
437
|
+
const jan = new Date(now.getFullYear(), 0, 1);
|
|
438
|
+
const jul = new Date(now.getFullYear(), 6, 1);
|
|
439
|
+
const getOffset = (date, zone) => {
|
|
440
|
+
const utc = new Date(date.toLocaleString("en-US", { timeZone: "UTC" }));
|
|
441
|
+
const local = new Date(date.toLocaleString("en-US", { timeZone: zone }));
|
|
442
|
+
return (utc.getTime() - local.getTime()) / 6e4;
|
|
443
|
+
};
|
|
444
|
+
const currentOffset = getOffset(now, tz);
|
|
445
|
+
const janOffset = getOffset(jan, tz);
|
|
446
|
+
const julOffset = getOffset(jul, tz);
|
|
447
|
+
const dstDifference = Math.abs(janOffset - julOffset);
|
|
448
|
+
return {
|
|
449
|
+
offset: Math.floor(currentOffset / 15) & 255,
|
|
450
|
+
dstDiff: Math.floor(dstDifference / 15) & 255
|
|
451
|
+
};
|
|
452
|
+
} catch {
|
|
453
|
+
return knownOffsets[tz] || { offset: 20, dstDiff: 4 };
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
function buildDeviceFingerprint(initTime, profile, userAgent) {
|
|
457
|
+
const tz = getTimezoneInfo(profile.timezone);
|
|
458
|
+
const { Byte, EncryptedBytes, CompactInt, RoundedByte, RawAppend } = FieldEncoding;
|
|
459
|
+
const encryptedUA = fieldEncrypt(textEnc(userAgent), 12, initTime);
|
|
460
|
+
const uaPayload = concat(u8(1), u8(encryptedUA.length), encryptedUA);
|
|
461
|
+
const fields = [
|
|
462
|
+
encodeField(0, Byte, 1),
|
|
463
|
+
// Platform: Win32
|
|
464
|
+
encodeField(1, Byte, 0),
|
|
465
|
+
// Vendor: Google Inc.
|
|
466
|
+
encodeField(2, EncryptedBytes, textEnc(profile.locale), initTime),
|
|
467
|
+
// Locale
|
|
468
|
+
encodeField(3, RoundedByte, profile.deviceMemoryGB * 10),
|
|
469
|
+
// Device memory (GB * 10)
|
|
470
|
+
encodeField(
|
|
471
|
+
4,
|
|
472
|
+
RawAppend,
|
|
473
|
+
concat(
|
|
474
|
+
// Screen dimensions (width + height)
|
|
475
|
+
screenDimBytes(profile.screenWidth, profile.availableWidth),
|
|
476
|
+
screenDimBytes(profile.screenHeight, profile.availableHeight)
|
|
477
|
+
)
|
|
478
|
+
),
|
|
479
|
+
encodeField(5, CompactInt, profile.colorDepth),
|
|
480
|
+
// Screen color depth
|
|
481
|
+
encodeField(6, CompactInt, profile.hardwareConcurrency),
|
|
482
|
+
// CPU logical cores
|
|
483
|
+
encodeField(7, RoundedByte, profile.devicePixelRatio * 10),
|
|
484
|
+
// Pixel ratio (* 10)
|
|
485
|
+
encodeField(8, RawAppend, u8(tz.offset, tz.dstDiff)),
|
|
486
|
+
// Timezone offset info
|
|
487
|
+
// MIME type hash — captured from Chrome 144 on Windows 10.
|
|
488
|
+
// Source: yubie-re/castleio-gen (Python SDK, MIT license).
|
|
489
|
+
encodeField(9, RawAppend, u8(2, 125, 95, 201, 167)),
|
|
490
|
+
// Browser plugins hash — Chrome no longer exposes plugins to navigator.plugins,
|
|
491
|
+
// so this is a fixed hash. Source: yubie-re/castleio-gen (Python SDK, MIT license).
|
|
492
|
+
encodeField(10, RawAppend, u8(5, 114, 147, 2, 8)),
|
|
493
|
+
encodeField(
|
|
494
|
+
11,
|
|
495
|
+
RawAppend,
|
|
496
|
+
// Browser feature flags
|
|
497
|
+
concat(u8(12), encodeBits([0, 1, 2, 3, 4, 5, 6], 16))
|
|
498
|
+
),
|
|
499
|
+
encodeField(12, RawAppend, uaPayload),
|
|
500
|
+
// User agent (encrypted)
|
|
501
|
+
// Canvas font rendering hash — generated by Castle.io SDK's canvas fingerprinting (text rendering).
|
|
502
|
+
// Captured from Chrome 144 on Windows 10. Source: yubie-re/castleio-gen (Python SDK, MIT license).
|
|
503
|
+
encodeField(13, EncryptedBytes, textEnc("54b4b5cf"), initTime),
|
|
504
|
+
encodeField(
|
|
505
|
+
14,
|
|
506
|
+
RawAppend,
|
|
507
|
+
// Media input devices
|
|
508
|
+
concat(u8(3), encodeBits([0, 1, 2], 8))
|
|
509
|
+
),
|
|
510
|
+
// Fields 15 (DoNotTrack) and 16 (JavaEnabled) intentionally omitted
|
|
511
|
+
encodeField(17, Byte, 0),
|
|
512
|
+
// productSub type
|
|
513
|
+
// Canvas circle rendering hash — generated by Castle.io SDK's canvas fingerprinting (arc drawing).
|
|
514
|
+
// Captured from Chrome 144 on Windows 10. Source: yubie-re/castleio-gen (Python SDK, MIT license).
|
|
515
|
+
encodeField(18, EncryptedBytes, textEnc("c6749e76"), initTime),
|
|
516
|
+
encodeField(19, EncryptedBytes, textEnc(profile.gpuRenderer), initTime),
|
|
517
|
+
// WebGL renderer
|
|
518
|
+
encodeField(
|
|
519
|
+
20,
|
|
520
|
+
EncryptedBytes,
|
|
521
|
+
// Epoch locale string
|
|
522
|
+
textEnc("12/31/1969, 7:00:00 PM"),
|
|
523
|
+
initTime
|
|
524
|
+
),
|
|
525
|
+
encodeField(
|
|
526
|
+
21,
|
|
527
|
+
RawAppend,
|
|
528
|
+
// WebDriver flags (none set)
|
|
529
|
+
concat(u8(8), encodeBits([], 8))
|
|
530
|
+
),
|
|
531
|
+
encodeField(22, CompactInt, 33),
|
|
532
|
+
// eval.toString() length
|
|
533
|
+
// Field 23 (navigator.buildID) intentionally omitted (Chrome doesn't have it)
|
|
534
|
+
encodeField(24, CompactInt, 12549),
|
|
535
|
+
// Max recursion depth
|
|
536
|
+
encodeField(25, Byte, 0),
|
|
537
|
+
// Recursion error message type
|
|
538
|
+
encodeField(26, Byte, 1),
|
|
539
|
+
// Recursion error name type
|
|
540
|
+
encodeField(27, CompactInt, 4644),
|
|
541
|
+
// Stack trace string length
|
|
542
|
+
encodeField(28, RawAppend, u8(0)),
|
|
543
|
+
// Touch support metric
|
|
544
|
+
encodeField(29, Byte, 3),
|
|
545
|
+
// Undefined call error type
|
|
546
|
+
// Navigator properties hash — hash of enumerable navigator property names.
|
|
547
|
+
// Captured from Chrome 144 on Windows 10. Source: yubie-re/castleio-gen (Python SDK, MIT license).
|
|
548
|
+
encodeField(30, RawAppend, u8(93, 197, 171, 181, 136)),
|
|
549
|
+
encodeField(31, RawAppend, encodeCodecPlayability())
|
|
550
|
+
// Codec playability
|
|
551
|
+
];
|
|
552
|
+
const data = concat(...fields);
|
|
553
|
+
const sizeIdx = (7 & FP_PART.DEVICE) << 5 | 31 & fields.length;
|
|
554
|
+
return concat(u8(sizeIdx), data);
|
|
555
|
+
}
|
|
556
|
+
function buildBrowserFingerprint(profile, initTime) {
|
|
557
|
+
const { Byte, EncryptedBytes, CompactInt, Marker, RawAppend } = FieldEncoding;
|
|
558
|
+
const timezoneField = profile.timezone in TIMEZONE_ENUM ? encodeField(1, Byte, TIMEZONE_ENUM[profile.timezone]) : encodeField(1, EncryptedBytes, textEnc(profile.timezone), initTime);
|
|
559
|
+
const fields = [
|
|
560
|
+
encodeField(0, Byte, 0),
|
|
561
|
+
// Constant marker
|
|
562
|
+
timezoneField,
|
|
563
|
+
// Timezone
|
|
564
|
+
encodeField(
|
|
565
|
+
2,
|
|
566
|
+
EncryptedBytes,
|
|
567
|
+
// Language list
|
|
568
|
+
textEnc(`${profile.locale},${profile.language}`),
|
|
569
|
+
initTime
|
|
570
|
+
),
|
|
571
|
+
encodeField(6, CompactInt, 0),
|
|
572
|
+
// Expected property count
|
|
573
|
+
encodeField(
|
|
574
|
+
10,
|
|
575
|
+
RawAppend,
|
|
576
|
+
// Castle data bitfield
|
|
577
|
+
concat(u8(4), encodeBits([1, 2, 3], 8))
|
|
578
|
+
),
|
|
579
|
+
encodeField(12, CompactInt, 80),
|
|
580
|
+
// Negative error string length
|
|
581
|
+
encodeField(13, RawAppend, u8(9, 0, 0)),
|
|
582
|
+
// Driver check values
|
|
583
|
+
encodeField(
|
|
584
|
+
17,
|
|
585
|
+
RawAppend,
|
|
586
|
+
// Chrome feature flags
|
|
587
|
+
concat(u8(13), encodeBits([1, 5, 8, 9, 10], 16))
|
|
588
|
+
),
|
|
589
|
+
encodeField(18, Marker, 0),
|
|
590
|
+
// Device logic expected
|
|
591
|
+
encodeField(21, RawAppend, u8(0, 0, 0, 0)),
|
|
592
|
+
// Class properties count
|
|
593
|
+
encodeField(22, EncryptedBytes, textEnc(profile.locale), initTime),
|
|
594
|
+
// User locale (secondary)
|
|
595
|
+
encodeField(
|
|
596
|
+
23,
|
|
597
|
+
RawAppend,
|
|
598
|
+
// Worker capabilities
|
|
599
|
+
concat(u8(2), encodeBits([0], 8))
|
|
600
|
+
),
|
|
601
|
+
encodeField(
|
|
602
|
+
24,
|
|
603
|
+
RawAppend,
|
|
604
|
+
// Inner/outer dimension diff
|
|
605
|
+
concat(be16(0), be16(randInt(10, 30)))
|
|
606
|
+
)
|
|
607
|
+
];
|
|
608
|
+
const data = concat(...fields);
|
|
609
|
+
const sizeIdx = (7 & FP_PART.BROWSER) << 5 | 31 & fields.length;
|
|
610
|
+
return concat(u8(sizeIdx), data);
|
|
611
|
+
}
|
|
612
|
+
function buildTimingFingerprint(initTime) {
|
|
613
|
+
const minute = new Date(initTime).getUTCMinutes();
|
|
614
|
+
const fields = [
|
|
615
|
+
encodeField(3, 5 /* CompactInt */, 1),
|
|
616
|
+
// Time since window.open (ms)
|
|
617
|
+
encodeField(4, 5 /* CompactInt */, minute)
|
|
618
|
+
// Castle init time (minutes)
|
|
619
|
+
];
|
|
620
|
+
const data = concat(...fields);
|
|
621
|
+
const sizeIdx = (7 & FP_PART.TIMING) << 5 | 31 & fields.length;
|
|
622
|
+
return concat(u8(sizeIdx), data);
|
|
623
|
+
}
|
|
624
|
+
const EventType = {
|
|
625
|
+
CLICK: 0,
|
|
626
|
+
FOCUS: 5,
|
|
627
|
+
BLUR: 6,
|
|
628
|
+
ANIMATIONSTART: 18,
|
|
629
|
+
MOUSEMOVE: 21,
|
|
630
|
+
MOUSELEAVE: 25,
|
|
631
|
+
MOUSEENTER: 26,
|
|
632
|
+
RESIZE: 27
|
|
633
|
+
};
|
|
634
|
+
const HAS_TARGET_FLAG = 128;
|
|
635
|
+
const TARGET_UNKNOWN = 63;
|
|
636
|
+
function generateEventLog() {
|
|
637
|
+
const simpleEvents = [
|
|
638
|
+
EventType.MOUSEMOVE,
|
|
639
|
+
EventType.ANIMATIONSTART,
|
|
640
|
+
EventType.MOUSELEAVE,
|
|
641
|
+
EventType.MOUSEENTER,
|
|
642
|
+
EventType.RESIZE
|
|
643
|
+
];
|
|
644
|
+
const targetedEvents = [
|
|
645
|
+
EventType.CLICK,
|
|
646
|
+
EventType.BLUR,
|
|
647
|
+
EventType.FOCUS
|
|
648
|
+
];
|
|
649
|
+
const allEvents = [...simpleEvents, ...targetedEvents];
|
|
650
|
+
const count = randInt(30, 70);
|
|
651
|
+
const eventBytes = [];
|
|
652
|
+
for (let i = 0; i < count; i++) {
|
|
653
|
+
const eventId = allEvents[randInt(0, allEvents.length - 1)];
|
|
654
|
+
if (targetedEvents.includes(eventId)) {
|
|
655
|
+
eventBytes.push(eventId | HAS_TARGET_FLAG);
|
|
656
|
+
eventBytes.push(TARGET_UNKNOWN);
|
|
657
|
+
} else {
|
|
658
|
+
eventBytes.push(eventId);
|
|
659
|
+
}
|
|
660
|
+
}
|
|
661
|
+
const inner = concat(u8(0), be16(count), new Uint8Array(eventBytes));
|
|
662
|
+
return concat(be16(inner.length), inner);
|
|
663
|
+
}
|
|
664
|
+
function buildBehavioralBitfield() {
|
|
665
|
+
const flags = new Array(15).fill(false);
|
|
666
|
+
flags[2] = true;
|
|
667
|
+
flags[3] = true;
|
|
668
|
+
flags[5] = true;
|
|
669
|
+
flags[6] = true;
|
|
670
|
+
flags[9] = true;
|
|
671
|
+
flags[11] = true;
|
|
672
|
+
flags[12] = true;
|
|
673
|
+
const packedBits = boolsToBin(flags, 16);
|
|
674
|
+
const encoded = 6 << 20 | 2 << 16 | 65535 & packedBits;
|
|
675
|
+
return u8(encoded >>> 16 & 255, encoded >>> 8 & 255, encoded & 255);
|
|
676
|
+
}
|
|
677
|
+
const NO_DATA = -1;
|
|
678
|
+
function buildFloatMetrics() {
|
|
679
|
+
const metrics = [
|
|
680
|
+
// ── Mouse & key timing ──
|
|
681
|
+
randFloat(40, 50),
|
|
682
|
+
// 0: Mouse angle vector mean
|
|
683
|
+
NO_DATA,
|
|
684
|
+
// 1: Touch angle vector (no touch device)
|
|
685
|
+
randFloat(70, 80),
|
|
686
|
+
// 2: Key same-time difference
|
|
687
|
+
NO_DATA,
|
|
688
|
+
// 3: (unused)
|
|
689
|
+
randFloat(60, 70),
|
|
690
|
+
// 4: Mouse down-to-up time mean
|
|
691
|
+
NO_DATA,
|
|
692
|
+
// 5: (unused)
|
|
693
|
+
0,
|
|
694
|
+
// 6: (zero placeholder)
|
|
695
|
+
0,
|
|
696
|
+
// 7: Mouse click time difference
|
|
697
|
+
// ── Duration distributions ──
|
|
698
|
+
randFloat(60, 80),
|
|
699
|
+
// 8: Mouse down-up duration median
|
|
700
|
+
randFloat(5, 10),
|
|
701
|
+
// 9: Mouse down-up duration std deviation
|
|
702
|
+
randFloat(30, 40),
|
|
703
|
+
// 10: Key press duration median
|
|
704
|
+
randFloat(2, 5),
|
|
705
|
+
// 11: Key press duration std deviation
|
|
706
|
+
// ── Touch metrics (all disabled for desktop) ──
|
|
707
|
+
NO_DATA,
|
|
708
|
+
NO_DATA,
|
|
709
|
+
NO_DATA,
|
|
710
|
+
NO_DATA,
|
|
711
|
+
// 12-15
|
|
712
|
+
NO_DATA,
|
|
713
|
+
NO_DATA,
|
|
714
|
+
NO_DATA,
|
|
715
|
+
NO_DATA,
|
|
716
|
+
// 16-19
|
|
717
|
+
// ── Mouse trajectory analysis ──
|
|
718
|
+
randFloat(150, 180),
|
|
719
|
+
// 20: Mouse movement angle mean
|
|
720
|
+
randFloat(3, 6),
|
|
721
|
+
// 21: Mouse movement angle std deviation
|
|
722
|
+
randFloat(150, 180),
|
|
723
|
+
// 22: Mouse movement angle mean (500ms window)
|
|
724
|
+
randFloat(3, 6),
|
|
725
|
+
// 23: Mouse movement angle std (500ms window)
|
|
726
|
+
randFloat(0, 2),
|
|
727
|
+
// 24: Mouse position deviation X
|
|
728
|
+
randFloat(0, 2),
|
|
729
|
+
// 25: Mouse position deviation Y
|
|
730
|
+
0,
|
|
731
|
+
0,
|
|
732
|
+
// 26-27: (zero placeholders)
|
|
733
|
+
// ── Touch sequential/gesture metrics (disabled) ──
|
|
734
|
+
NO_DATA,
|
|
735
|
+
NO_DATA,
|
|
736
|
+
// 28-29
|
|
737
|
+
NO_DATA,
|
|
738
|
+
NO_DATA,
|
|
739
|
+
// 30-31
|
|
740
|
+
// ── Key pattern analysis ──
|
|
741
|
+
0,
|
|
742
|
+
0,
|
|
743
|
+
// 32-33: Letter-digit transition ratio
|
|
744
|
+
0,
|
|
745
|
+
0,
|
|
746
|
+
// 34-35: Digit-invalid transition ratio
|
|
747
|
+
0,
|
|
748
|
+
0,
|
|
749
|
+
// 36-37: Double-invalid transition ratio
|
|
750
|
+
// ── Mouse vector differences ──
|
|
751
|
+
1,
|
|
752
|
+
0,
|
|
753
|
+
// 38-39: Mouse vector diff (mean, std)
|
|
754
|
+
1,
|
|
755
|
+
0,
|
|
756
|
+
// 40-41: Mouse vector diff 2 (mean, std)
|
|
757
|
+
randFloat(0, 4),
|
|
758
|
+
// 42: Mouse vector diff (500ms mean)
|
|
759
|
+
randFloat(0, 3),
|
|
760
|
+
// 43: Mouse vector diff (500ms std)
|
|
761
|
+
// ── Rounded movement metrics ──
|
|
762
|
+
randFloat(25, 50),
|
|
763
|
+
// 44: Mouse time diff (rounded mean)
|
|
764
|
+
randFloat(25, 50),
|
|
765
|
+
// 45: Mouse time diff (rounded std)
|
|
766
|
+
randFloat(25, 50),
|
|
767
|
+
// 46: Mouse vector diff (rounded mean)
|
|
768
|
+
randFloat(25, 30),
|
|
769
|
+
// 47: Mouse vector diff (rounded std)
|
|
770
|
+
// ── Speed change analysis ──
|
|
771
|
+
randFloat(0, 2),
|
|
772
|
+
// 48: Mouse speed change mean
|
|
773
|
+
randFloat(0, 1),
|
|
774
|
+
// 49: Mouse speed change std
|
|
775
|
+
randFloat(0, 1),
|
|
776
|
+
// 50: Mouse vector 500ms aggregate
|
|
777
|
+
// ── Trailing ──
|
|
778
|
+
1,
|
|
779
|
+
// 51: Universal flag
|
|
780
|
+
0
|
|
781
|
+
// 52: Terminator
|
|
782
|
+
];
|
|
783
|
+
const out = new Uint8Array(metrics.length);
|
|
784
|
+
for (let i = 0; i < metrics.length; i++) {
|
|
785
|
+
out[i] = metrics[i] === NO_DATA ? 0 : encodeFloatVal(metrics[i]);
|
|
786
|
+
}
|
|
787
|
+
return out;
|
|
788
|
+
}
|
|
789
|
+
function buildEventCounts() {
|
|
790
|
+
const counts = [
|
|
791
|
+
randInt(100, 200),
|
|
792
|
+
// 0: mousemove events
|
|
793
|
+
randInt(1, 5),
|
|
794
|
+
// 1: keyup events
|
|
795
|
+
randInt(1, 5),
|
|
796
|
+
// 2: click events
|
|
797
|
+
0,
|
|
798
|
+
// 3: touchstart events (none on desktop)
|
|
799
|
+
randInt(0, 5),
|
|
800
|
+
// 4: keydown events
|
|
801
|
+
0,
|
|
802
|
+
// 5: touchmove events (none)
|
|
803
|
+
0,
|
|
804
|
+
// 6: mousedown-mouseup pairs
|
|
805
|
+
0,
|
|
806
|
+
// 7: vector diff samples
|
|
807
|
+
randInt(0, 5),
|
|
808
|
+
// 8: wheel events
|
|
809
|
+
randInt(0, 11),
|
|
810
|
+
// 9: (internal counter)
|
|
811
|
+
randInt(0, 1)
|
|
812
|
+
// 10: (internal counter)
|
|
813
|
+
];
|
|
814
|
+
return concat(new Uint8Array(counts), u8(counts.length));
|
|
815
|
+
}
|
|
816
|
+
function buildBehavioralData() {
|
|
817
|
+
return concat(
|
|
818
|
+
buildBehavioralBitfield(),
|
|
819
|
+
buildFloatMetrics(),
|
|
820
|
+
buildEventCounts()
|
|
821
|
+
);
|
|
822
|
+
}
|
|
823
|
+
function buildTokenHeader(uuid, publisherKey, initTime) {
|
|
824
|
+
const timestamp = fromHex(encodeTimestampEncrypted(initTime));
|
|
825
|
+
const version = be16(SDK_VERSION);
|
|
826
|
+
const pkBytes = textEnc(publisherKey);
|
|
827
|
+
const uuidBytes = fromHex(uuid);
|
|
828
|
+
return concat(timestamp, version, pkBytes, uuidBytes);
|
|
829
|
+
}
|
|
830
|
+
function generateLocalCastleToken(userAgent, profileOverride) {
|
|
831
|
+
const now = Date.now();
|
|
832
|
+
const profile = { ...DEFAULT_PROFILE, ...profileOverride };
|
|
833
|
+
const initTime = now - randFloat(2 * 60 * 1e3, 30 * 60 * 1e3);
|
|
834
|
+
log$7("Generating local Castle.io v11 token");
|
|
835
|
+
const deviceFp = buildDeviceFingerprint(initTime, profile, userAgent);
|
|
836
|
+
const browserFp = buildBrowserFingerprint(profile, initTime);
|
|
837
|
+
const timingFp = buildTimingFingerprint(initTime);
|
|
838
|
+
const eventLog = generateEventLog();
|
|
839
|
+
const behavioral = buildBehavioralData();
|
|
840
|
+
const fingerprintData = concat(
|
|
841
|
+
deviceFp,
|
|
842
|
+
browserFp,
|
|
843
|
+
timingFp,
|
|
844
|
+
eventLog,
|
|
845
|
+
behavioral,
|
|
846
|
+
u8(255)
|
|
847
|
+
);
|
|
848
|
+
const sendTime = Date.now();
|
|
849
|
+
const timestampKey = encodeTimestampEncrypted(sendTime);
|
|
850
|
+
const xorPass1 = deriveAndXor(
|
|
851
|
+
timestampKey,
|
|
852
|
+
4,
|
|
853
|
+
timestampKey[3],
|
|
854
|
+
fingerprintData
|
|
855
|
+
);
|
|
856
|
+
const tokenUuid = toHex(getRandomBytes(16));
|
|
857
|
+
const withTimestampPrefix = concat(fromHex(timestampKey), xorPass1);
|
|
858
|
+
const xorPass2 = deriveAndXor(
|
|
859
|
+
tokenUuid,
|
|
860
|
+
8,
|
|
861
|
+
tokenUuid[9],
|
|
862
|
+
withTimestampPrefix
|
|
863
|
+
);
|
|
864
|
+
const header = buildTokenHeader(tokenUuid, TWITTER_CASTLE_PK, initTime);
|
|
865
|
+
const plaintext = concat(header, xorPass2);
|
|
866
|
+
const encrypted = xxteaEncrypt(plaintext, XXTEA_KEY);
|
|
867
|
+
const paddingBytes = encrypted.length - plaintext.length;
|
|
868
|
+
const versioned = concat(u8(TOKEN_VERSION, paddingBytes), encrypted);
|
|
869
|
+
const randomByte = getRandomBytes(1)[0];
|
|
870
|
+
const checksum = versioned.length * 2 & 255;
|
|
871
|
+
const withChecksum = concat(versioned, u8(checksum));
|
|
872
|
+
const xored = xorBytes(withChecksum, u8(randomByte));
|
|
873
|
+
const finalPayload = concat(u8(randomByte), xored);
|
|
874
|
+
const token = base64url(finalPayload);
|
|
875
|
+
log$7(
|
|
876
|
+
`Generated castle token: ${token.length} chars, cuid: ${tokenUuid.substring(
|
|
877
|
+
0,
|
|
878
|
+
6
|
|
879
|
+
)}...`
|
|
880
|
+
);
|
|
881
|
+
return { token, cuid: tokenUuid };
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
const log$6 = debug("twitter-scraper:requests");
|
|
98
885
|
async function updateCookieJar(cookieJar, headers) {
|
|
99
886
|
let setCookieHeaders = [];
|
|
100
887
|
if (typeof headers.getSetCookie === "function") {
|
|
@@ -109,12 +896,12 @@ async function updateCookieJar(cookieJar, headers) {
|
|
|
109
896
|
for (const cookieStr of setCookieHeaders) {
|
|
110
897
|
const cookie = toughCookie.Cookie.parse(cookieStr);
|
|
111
898
|
if (!cookie) {
|
|
112
|
-
log$
|
|
899
|
+
log$6(`Failed to parse cookie: ${cookieStr.substring(0, 100)}`);
|
|
113
900
|
continue;
|
|
114
901
|
}
|
|
115
902
|
if (cookie.maxAge === 0 || cookie.expires && cookie.expires < /* @__PURE__ */ new Date()) {
|
|
116
903
|
if (cookie.key === "ct0") {
|
|
117
|
-
log$
|
|
904
|
+
log$6(`Skipping deletion of ct0 cookie (Max-Age=0)`);
|
|
118
905
|
}
|
|
119
906
|
continue;
|
|
120
907
|
}
|
|
@@ -122,7 +909,7 @@ async function updateCookieJar(cookieJar, headers) {
|
|
|
122
909
|
const url = `${cookie.secure ? "https" : "http"}://${cookie.domain}${cookie.path}`;
|
|
123
910
|
await cookieJar.setCookie(cookie, url);
|
|
124
911
|
if (cookie.key === "ct0") {
|
|
125
|
-
log$
|
|
912
|
+
log$6(
|
|
126
913
|
`Successfully set ct0 cookie with value: ${cookie.value.substring(
|
|
127
914
|
0,
|
|
128
915
|
20
|
|
@@ -130,9 +917,9 @@ async function updateCookieJar(cookieJar, headers) {
|
|
|
130
917
|
);
|
|
131
918
|
}
|
|
132
919
|
} catch (err) {
|
|
133
|
-
log$
|
|
920
|
+
log$6(`Failed to set cookie ${cookie.key}: ${err}`);
|
|
134
921
|
if (cookie.key === "ct0") {
|
|
135
|
-
log$
|
|
922
|
+
log$6(`FAILED to set ct0 cookie! Error: ${err}`);
|
|
136
923
|
}
|
|
137
924
|
}
|
|
138
925
|
}
|
|
@@ -146,14 +933,14 @@ async function updateCookieJar(cookieJar, headers) {
|
|
|
146
933
|
}
|
|
147
934
|
}
|
|
148
935
|
|
|
149
|
-
const log$
|
|
936
|
+
const log$5 = debug("twitter-scraper:xpff");
|
|
150
937
|
let isoCrypto = null;
|
|
151
938
|
async function getCrypto() {
|
|
152
939
|
if (isoCrypto != null) {
|
|
153
940
|
return isoCrypto;
|
|
154
941
|
}
|
|
155
942
|
if (typeof crypto === "undefined") {
|
|
156
|
-
log$
|
|
943
|
+
log$5("Global crypto is undefined, importing from crypto module...");
|
|
157
944
|
const { webcrypto } = await import('crypto');
|
|
158
945
|
isoCrypto = webcrypto;
|
|
159
946
|
return webcrypto;
|
|
@@ -180,7 +967,7 @@ class XPFFHeaderGenerator {
|
|
|
180
967
|
return result;
|
|
181
968
|
}
|
|
182
969
|
async generateHeader(plaintext, guestId) {
|
|
183
|
-
log$
|
|
970
|
+
log$5(`Generating XPFF key for guest ID: ${guestId}`);
|
|
184
971
|
const key = await this.deriveKey(guestId);
|
|
185
972
|
const crypto2 = await getCrypto();
|
|
186
973
|
const nonce = crypto2.getRandomValues(new Uint8Array(12));
|
|
@@ -203,7 +990,7 @@ class XPFFHeaderGenerator {
|
|
|
203
990
|
combined.set(nonce);
|
|
204
991
|
combined.set(new Uint8Array(encrypted), nonce.length);
|
|
205
992
|
const result = buf2hex(combined.buffer);
|
|
206
|
-
log$
|
|
993
|
+
log$5(`XPFF header generated for guest ID ${guestId}: ${result}`);
|
|
207
994
|
return result;
|
|
208
995
|
}
|
|
209
996
|
}
|
|
@@ -225,7 +1012,7 @@ async function generateXPFFHeader(guestId) {
|
|
|
225
1012
|
return generator.generateHeader(plaintext, guestId);
|
|
226
1013
|
}
|
|
227
1014
|
|
|
228
|
-
const log$
|
|
1015
|
+
const log$4 = debug("twitter-scraper:auth");
|
|
229
1016
|
function withTransform(fetchFn, transform) {
|
|
230
1017
|
return async (input, init) => {
|
|
231
1018
|
const fetchArgs = await transform?.request?.(input, init) ?? [
|
|
@@ -275,8 +1062,14 @@ class TwitterGuestAuth {
|
|
|
275
1062
|
}
|
|
276
1063
|
return new Date(this.guestCreatedAt);
|
|
277
1064
|
}
|
|
278
|
-
|
|
1065
|
+
/**
|
|
1066
|
+
* Install only authentication credentials (bearer token, guest token, cookies)
|
|
1067
|
+
* without browser fingerprint or platform headers. Useful for callers that
|
|
1068
|
+
* build their own header set (e.g. the login flow).
|
|
1069
|
+
*/
|
|
1070
|
+
async installAuthCredentials(headers, bearerTokenOverride) {
|
|
279
1071
|
const tokenToUse = bearerTokenOverride ?? this.bearerToken;
|
|
1072
|
+
headers.set("authorization", `Bearer ${tokenToUse}`);
|
|
280
1073
|
if (!bearerTokenOverride) {
|
|
281
1074
|
if (this.shouldUpdate()) {
|
|
282
1075
|
await this.updateGuestToken();
|
|
@@ -285,11 +1078,27 @@ class TwitterGuestAuth {
|
|
|
285
1078
|
headers.set("x-guest-token", this.guestToken);
|
|
286
1079
|
}
|
|
287
1080
|
}
|
|
288
|
-
headers.set("
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
);
|
|
1081
|
+
headers.set("cookie", await this.getCookieString());
|
|
1082
|
+
}
|
|
1083
|
+
async installTo(headers, url, bearerTokenOverride) {
|
|
1084
|
+
await this.installAuthCredentials(headers, bearerTokenOverride);
|
|
1085
|
+
headers.set("user-agent", CHROME_USER_AGENT);
|
|
1086
|
+
if (!headers.has("accept")) {
|
|
1087
|
+
headers.set("accept", "*/*");
|
|
1088
|
+
}
|
|
1089
|
+
headers.set("accept-language", "en-US,en;q=0.9");
|
|
1090
|
+
headers.set("sec-ch-ua", CHROME_SEC_CH_UA);
|
|
1091
|
+
headers.set("sec-ch-ua-mobile", "?0");
|
|
1092
|
+
headers.set("sec-ch-ua-platform", '"Windows"');
|
|
1093
|
+
headers.set("referer", "https://x.com/");
|
|
1094
|
+
headers.set("origin", "https://x.com");
|
|
1095
|
+
headers.set("sec-fetch-site", "same-site");
|
|
1096
|
+
headers.set("sec-fetch-mode", "cors");
|
|
1097
|
+
headers.set("sec-fetch-dest", "empty");
|
|
1098
|
+
headers.set("priority", "u=1, i");
|
|
1099
|
+
if (!headers.has("content-type") && (url.includes("api.x.com/graphql/") || url.includes("x.com/i/api/graphql/"))) {
|
|
1100
|
+
headers.set("content-type", "application/json");
|
|
1101
|
+
}
|
|
293
1102
|
await this.installCsrfToken(headers);
|
|
294
1103
|
if (this.options?.experimental?.xpff) {
|
|
295
1104
|
const guestId = await this.guestId();
|
|
@@ -298,7 +1107,6 @@ class TwitterGuestAuth {
|
|
|
298
1107
|
headers.set("x-xp-forwarded-for", xpffHeader);
|
|
299
1108
|
}
|
|
300
1109
|
}
|
|
301
|
-
headers.set("cookie", await this.getCookieString());
|
|
302
1110
|
}
|
|
303
1111
|
async installCsrfToken(headers) {
|
|
304
1112
|
const cookies = await this.getCookies();
|
|
@@ -329,7 +1137,7 @@ class TwitterGuestAuth {
|
|
|
329
1137
|
const cookies = await this.jar.getCookies(this.getCookieJarUrl());
|
|
330
1138
|
for (const cookie of cookies) {
|
|
331
1139
|
if (!cookie.domain || !cookie.path) continue;
|
|
332
|
-
store.removeCookie(cookie.domain, cookie.path, key);
|
|
1140
|
+
await store.removeCookie(cookie.domain, cookie.path, key);
|
|
333
1141
|
if (typeof document !== "undefined") {
|
|
334
1142
|
document.cookie = `${cookie.key}=; Max-Age=0; path=${cookie.path}; domain=${cookie.domain}`;
|
|
335
1143
|
}
|
|
@@ -350,20 +1158,31 @@ class TwitterGuestAuth {
|
|
|
350
1158
|
try {
|
|
351
1159
|
await this.updateGuestTokenCore();
|
|
352
1160
|
} catch (err) {
|
|
353
|
-
log$
|
|
1161
|
+
log$4("Failed to update guest token; this may cause issues:", err);
|
|
354
1162
|
}
|
|
355
1163
|
}
|
|
356
1164
|
async updateGuestTokenCore() {
|
|
357
1165
|
const guestActivateUrl = "https://api.x.com/1.1/guest/activate.json";
|
|
358
1166
|
const headers = new headersPolyfill.Headers({
|
|
359
|
-
|
|
360
|
-
|
|
1167
|
+
authorization: `Bearer ${this.bearerToken}`,
|
|
1168
|
+
"user-agent": CHROME_USER_AGENT,
|
|
1169
|
+
accept: "*/*",
|
|
1170
|
+
"accept-language": "en-US,en;q=0.9",
|
|
1171
|
+
"content-type": "application/x-www-form-urlencoded",
|
|
1172
|
+
"sec-ch-ua": CHROME_SEC_CH_UA,
|
|
1173
|
+
"sec-ch-ua-mobile": "?0",
|
|
1174
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
1175
|
+
origin: "https://x.com",
|
|
1176
|
+
referer: "https://x.com/",
|
|
1177
|
+
"sec-fetch-site": "same-site",
|
|
1178
|
+
"sec-fetch-mode": "cors",
|
|
1179
|
+
"sec-fetch-dest": "empty",
|
|
1180
|
+
cookie: await this.getCookieString()
|
|
361
1181
|
});
|
|
362
|
-
log$
|
|
1182
|
+
log$4(`Making POST request to ${guestActivateUrl}`);
|
|
363
1183
|
const res = await this.fetch(guestActivateUrl, {
|
|
364
1184
|
method: "POST",
|
|
365
|
-
headers
|
|
366
|
-
referrerPolicy: "no-referrer"
|
|
1185
|
+
headers
|
|
367
1186
|
});
|
|
368
1187
|
await updateCookieJar(this.jar, res.headers);
|
|
369
1188
|
if (!res.ok) {
|
|
@@ -380,7 +1199,7 @@ class TwitterGuestAuth {
|
|
|
380
1199
|
this.guestToken = newGuestToken;
|
|
381
1200
|
this.guestCreatedAt = /* @__PURE__ */ new Date();
|
|
382
1201
|
await this.setCookie("gt", newGuestToken);
|
|
383
|
-
log$
|
|
1202
|
+
log$4(`Updated guest token (length: ${newGuestToken.length})`);
|
|
384
1203
|
}
|
|
385
1204
|
/**
|
|
386
1205
|
* Returns if the authentication token needs to be updated or not.
|
|
@@ -404,7 +1223,7 @@ class Platform {
|
|
|
404
1223
|
}
|
|
405
1224
|
}
|
|
406
1225
|
|
|
407
|
-
const log$
|
|
1226
|
+
const log$3 = debug("twitter-scraper:xctxid");
|
|
408
1227
|
let linkedom = null;
|
|
409
1228
|
async function linkedomImport() {
|
|
410
1229
|
if (!linkedom) {
|
|
@@ -433,7 +1252,7 @@ async function handleXMigration(fetchFn) {
|
|
|
433
1252
|
"cache-control": "no-cache",
|
|
434
1253
|
pragma: "no-cache",
|
|
435
1254
|
priority: "u=0, i",
|
|
436
|
-
"sec-ch-ua":
|
|
1255
|
+
"sec-ch-ua": CHROME_SEC_CH_UA,
|
|
437
1256
|
"sec-ch-ua-mobile": "?0",
|
|
438
1257
|
"sec-ch-ua-platform": '"Windows"',
|
|
439
1258
|
"sec-fetch-dest": "document",
|
|
@@ -441,7 +1260,7 @@ async function handleXMigration(fetchFn) {
|
|
|
441
1260
|
"sec-fetch-site": "none",
|
|
442
1261
|
"sec-fetch-user": "?1",
|
|
443
1262
|
"upgrade-insecure-requests": "1",
|
|
444
|
-
"user-agent":
|
|
1263
|
+
"user-agent": CHROME_USER_AGENT
|
|
445
1264
|
};
|
|
446
1265
|
const response = await fetchFn("https://x.com", {
|
|
447
1266
|
headers
|
|
@@ -460,7 +1279,7 @@ async function handleXMigration(fetchFn) {
|
|
|
460
1279
|
const metaContent = metaRefresh ? metaRefresh.getAttribute("content") || "" : "";
|
|
461
1280
|
const migrationRedirectionUrl = migrationRedirectionRegex.exec(metaContent) || migrationRedirectionRegex.exec(htmlText);
|
|
462
1281
|
if (migrationRedirectionUrl) {
|
|
463
|
-
const redirectResponse = await
|
|
1282
|
+
const redirectResponse = await fetchFn(migrationRedirectionUrl[0]);
|
|
464
1283
|
if (!redirectResponse.ok) {
|
|
465
1284
|
throw new Error(
|
|
466
1285
|
`Failed to follow migration redirection: ${redirectResponse.statusText}`
|
|
@@ -483,7 +1302,7 @@ async function handleXMigration(fetchFn) {
|
|
|
483
1302
|
requestPayload.append(name, value);
|
|
484
1303
|
}
|
|
485
1304
|
}
|
|
486
|
-
const formResponse = await
|
|
1305
|
+
const formResponse = await fetchFn(url, {
|
|
487
1306
|
method,
|
|
488
1307
|
body: requestPayload,
|
|
489
1308
|
headers
|
|
@@ -499,6 +1318,23 @@ async function handleXMigration(fetchFn) {
|
|
|
499
1318
|
}
|
|
500
1319
|
return document;
|
|
501
1320
|
}
|
|
1321
|
+
let cachedDocumentPromise = null;
|
|
1322
|
+
let cachedDocumentTimestamp = 0;
|
|
1323
|
+
const DOCUMENT_CACHE_TTL = 5 * 60 * 1e3;
|
|
1324
|
+
async function getCachedDocument(fetchFn) {
|
|
1325
|
+
const now = Date.now();
|
|
1326
|
+
if (!cachedDocumentPromise || now - cachedDocumentTimestamp > DOCUMENT_CACHE_TTL) {
|
|
1327
|
+
log$3("Fetching fresh x.com document for transaction ID generation");
|
|
1328
|
+
cachedDocumentTimestamp = now;
|
|
1329
|
+
cachedDocumentPromise = handleXMigration(fetchFn).catch((err) => {
|
|
1330
|
+
cachedDocumentPromise = null;
|
|
1331
|
+
throw err;
|
|
1332
|
+
});
|
|
1333
|
+
} else {
|
|
1334
|
+
log$3("Using cached x.com document for transaction ID generation");
|
|
1335
|
+
}
|
|
1336
|
+
return cachedDocumentPromise;
|
|
1337
|
+
}
|
|
502
1338
|
let ClientTransaction = null;
|
|
503
1339
|
async function clientTransaction() {
|
|
504
1340
|
if (!ClientTransaction) {
|
|
@@ -511,16 +1347,19 @@ async function clientTransaction() {
|
|
|
511
1347
|
async function generateTransactionId(url, fetchFn, method) {
|
|
512
1348
|
const parsedUrl = new URL(url);
|
|
513
1349
|
const path = parsedUrl.pathname;
|
|
514
|
-
log$
|
|
515
|
-
const document = await
|
|
1350
|
+
log$3(`Generating transaction ID for ${method} ${path}`);
|
|
1351
|
+
const document = await getCachedDocument(fetchFn);
|
|
516
1352
|
const ClientTransactionClass = await clientTransaction();
|
|
517
1353
|
const transaction = await ClientTransactionClass.create(document);
|
|
518
1354
|
const transactionId = await transaction.generateTransactionId(method, path);
|
|
519
|
-
log$
|
|
1355
|
+
log$3(`Transaction ID: ${transactionId}`);
|
|
520
1356
|
return transactionId;
|
|
521
1357
|
}
|
|
522
1358
|
|
|
523
|
-
const
|
|
1359
|
+
const CHROME_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36";
|
|
1360
|
+
const CHROME_SEC_CH_UA = '"Not(A:Brand";v="8", "Chromium";v="144", "Google Chrome";v="144"';
|
|
1361
|
+
|
|
1362
|
+
const log$2 = debug("twitter-scraper:api");
|
|
524
1363
|
const bearerToken = "AAAAAAAAAAAAAAAAAAAAAFQODgEAAAAAVHTp76lzh3rFzcHbmHVvQxYYpTw%3DckAlMINMjmCwxUcaXbAN4XqJVdgMJaHqNOFgPMK0zN1qLqLQCF";
|
|
525
1364
|
const bearerToken2 = "AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA";
|
|
526
1365
|
async function jitter(maxMs) {
|
|
@@ -528,7 +1367,7 @@ async function jitter(maxMs) {
|
|
|
528
1367
|
await new Promise((resolve) => setTimeout(resolve, jitter2));
|
|
529
1368
|
}
|
|
530
1369
|
async function requestApi(url, auth, method = "GET", platform = new Platform(), headers = new headersPolyfill.Headers(), bearerTokenOverride) {
|
|
531
|
-
log$
|
|
1370
|
+
log$2(`Making ${method} request to ${url}`);
|
|
532
1371
|
await auth.installTo(headers, url, bearerTokenOverride);
|
|
533
1372
|
await platform.randomizeCiphers();
|
|
534
1373
|
if (auth instanceof TwitterGuestAuth && auth.options?.experimental?.xClientTransactionId) {
|
|
@@ -557,12 +1396,12 @@ async function requestApi(url, auth, method = "GET", platform = new Platform(),
|
|
|
557
1396
|
}
|
|
558
1397
|
return {
|
|
559
1398
|
success: false,
|
|
560
|
-
err
|
|
1399
|
+
err
|
|
561
1400
|
};
|
|
562
1401
|
}
|
|
563
1402
|
await updateCookieJar(auth.cookieJar(), res.headers);
|
|
564
1403
|
if (res.status === 429) {
|
|
565
|
-
log$
|
|
1404
|
+
log$2("Rate limit hit, waiting for retry...");
|
|
566
1405
|
await auth.onRateLimit({
|
|
567
1406
|
fetchParameters,
|
|
568
1407
|
response: res
|
|
@@ -587,9 +1426,9 @@ async function flexParseJson(res) {
|
|
|
587
1426
|
try {
|
|
588
1427
|
return await res.json();
|
|
589
1428
|
} catch {
|
|
590
|
-
log$
|
|
1429
|
+
log$2("Failed to parse response as JSON, trying text parse...");
|
|
591
1430
|
const text = await res.text();
|
|
592
|
-
log$
|
|
1431
|
+
log$2("Response text:", text);
|
|
593
1432
|
return JSON.parse(text);
|
|
594
1433
|
}
|
|
595
1434
|
}
|
|
@@ -663,12 +1502,12 @@ function addApiParams(params, includeTweetReplies) {
|
|
|
663
1502
|
return params;
|
|
664
1503
|
}
|
|
665
1504
|
|
|
666
|
-
const log = debug("twitter-scraper:auth-user");
|
|
1505
|
+
const log$1 = debug("twitter-scraper:auth-user");
|
|
667
1506
|
const TwitterUserAuthSubtask = typebox.Type.Object({
|
|
668
1507
|
subtask_id: typebox.Type.String(),
|
|
669
1508
|
enter_text: typebox.Type.Optional(typebox.Type.Object({}))
|
|
670
1509
|
});
|
|
671
|
-
class
|
|
1510
|
+
const _TwitterUserAuth = class _TwitterUserAuth extends TwitterGuestAuth {
|
|
672
1511
|
constructor(bearerToken, options) {
|
|
673
1512
|
super(bearerToken, options);
|
|
674
1513
|
this.subtaskHandlers = /* @__PURE__ */ new Map();
|
|
@@ -714,11 +1553,14 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
714
1553
|
);
|
|
715
1554
|
}
|
|
716
1555
|
async isLoggedIn() {
|
|
717
|
-
const
|
|
718
|
-
return
|
|
1556
|
+
const cookies = await this.getCookies();
|
|
1557
|
+
return cookies.some((c) => c.key === "ct0") && cookies.some((c) => c.key === "auth_token");
|
|
719
1558
|
}
|
|
720
1559
|
async login(username, password, email, twoFactorSecret) {
|
|
721
|
-
await this.
|
|
1560
|
+
await this.preflight();
|
|
1561
|
+
if (!this.guestToken) {
|
|
1562
|
+
await this.updateGuestToken();
|
|
1563
|
+
}
|
|
722
1564
|
const credentials = {
|
|
723
1565
|
username,
|
|
724
1566
|
password,
|
|
@@ -732,6 +1574,12 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
732
1574
|
throw new Error("flow_token not found.");
|
|
733
1575
|
}
|
|
734
1576
|
const subtaskId = next.response.subtasks[0].subtask_id;
|
|
1577
|
+
const configuredDelay = this.options?.experimental?.flowStepDelay;
|
|
1578
|
+
const delay = configuredDelay !== void 0 ? configuredDelay : 1e3 + Math.floor(Math.random() * 2e3);
|
|
1579
|
+
if (delay > 0) {
|
|
1580
|
+
log$1(`Waiting ${delay}ms before handling subtask: ${subtaskId}`);
|
|
1581
|
+
await new Promise((resolve) => setTimeout(resolve, delay));
|
|
1582
|
+
}
|
|
735
1583
|
const handler = this.subtaskHandlers.get(subtaskId);
|
|
736
1584
|
if (handler) {
|
|
737
1585
|
next = await handler(subtaskId, next.response, credentials, {
|
|
@@ -746,65 +1594,92 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
746
1594
|
throw next.err;
|
|
747
1595
|
}
|
|
748
1596
|
}
|
|
1597
|
+
/**
|
|
1598
|
+
* Pre-flight request to establish Cloudflare cookies and session context.
|
|
1599
|
+
* Mimics a real browser visiting x.com before starting the login API flow.
|
|
1600
|
+
*/
|
|
1601
|
+
async preflight() {
|
|
1602
|
+
try {
|
|
1603
|
+
const headers = new headersPolyfill.Headers({
|
|
1604
|
+
accept: "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
1605
|
+
"accept-language": "en-US,en;q=0.9",
|
|
1606
|
+
"sec-ch-ua": CHROME_SEC_CH_UA,
|
|
1607
|
+
"sec-ch-ua-mobile": "?0",
|
|
1608
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
1609
|
+
"sec-fetch-dest": "document",
|
|
1610
|
+
"sec-fetch-mode": "navigate",
|
|
1611
|
+
"sec-fetch-site": "none",
|
|
1612
|
+
"sec-fetch-user": "?1",
|
|
1613
|
+
"upgrade-insecure-requests": "1",
|
|
1614
|
+
"user-agent": CHROME_USER_AGENT
|
|
1615
|
+
});
|
|
1616
|
+
log$1("Pre-flight: fetching https://x.com/i/flow/login");
|
|
1617
|
+
const res = await this.fetch("https://x.com/i/flow/login", {
|
|
1618
|
+
redirect: "follow",
|
|
1619
|
+
headers
|
|
1620
|
+
});
|
|
1621
|
+
await updateCookieJar(this.jar, res.headers);
|
|
1622
|
+
log$1(`Pre-flight response: ${res.status}`);
|
|
1623
|
+
try {
|
|
1624
|
+
const html = await res.text();
|
|
1625
|
+
const gtMatch = html.match(/document\.cookie="gt=(\d+)/);
|
|
1626
|
+
if (gtMatch) {
|
|
1627
|
+
this.guestToken = gtMatch[1];
|
|
1628
|
+
this.guestCreatedAt = /* @__PURE__ */ new Date();
|
|
1629
|
+
await this.setCookie("gt", gtMatch[1]);
|
|
1630
|
+
log$1(`Extracted guest token from HTML (length: ${gtMatch[1].length})`);
|
|
1631
|
+
}
|
|
1632
|
+
} catch (err) {
|
|
1633
|
+
log$1("Failed to extract guest token from HTML (non-fatal):", err);
|
|
1634
|
+
}
|
|
1635
|
+
} catch (err) {
|
|
1636
|
+
log$1("Pre-flight request failed (non-fatal):", err);
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
749
1639
|
async logout() {
|
|
750
1640
|
if (!this.hasToken()) {
|
|
751
1641
|
return;
|
|
752
1642
|
}
|
|
753
1643
|
try {
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
1644
|
+
const logoutUrl = "https://api.x.com/1.1/account/logout.json";
|
|
1645
|
+
const headers = new headersPolyfill.Headers();
|
|
1646
|
+
await this.installTo(headers, logoutUrl);
|
|
1647
|
+
await this.fetch(logoutUrl, {
|
|
1648
|
+
method: "POST",
|
|
1649
|
+
headers
|
|
1650
|
+
});
|
|
759
1651
|
} catch (error) {
|
|
760
|
-
|
|
1652
|
+
log$1("Error during logout:", error);
|
|
761
1653
|
} finally {
|
|
762
1654
|
this.deleteToken();
|
|
763
1655
|
this.jar = new toughCookie.CookieJar();
|
|
764
1656
|
}
|
|
765
1657
|
}
|
|
766
|
-
async installTo(headers,
|
|
767
|
-
|
|
768
|
-
headers.set("
|
|
769
|
-
headers.set(
|
|
770
|
-
|
|
771
|
-
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
|
|
772
|
-
);
|
|
773
|
-
if (this.guestToken) {
|
|
774
|
-
headers.set("x-guest-token", this.guestToken);
|
|
775
|
-
}
|
|
776
|
-
await this.installCsrfToken(headers);
|
|
777
|
-
if (this.options?.experimental?.xpff) {
|
|
778
|
-
const guestId = await this.guestId();
|
|
779
|
-
if (guestId != null) {
|
|
780
|
-
const xpffHeader = await generateXPFFHeader(guestId);
|
|
781
|
-
headers.set("x-xp-forwarded-for", xpffHeader);
|
|
782
|
-
}
|
|
783
|
-
}
|
|
784
|
-
const cookie = await this.getCookieString();
|
|
785
|
-
headers.set("cookie", cookie);
|
|
1658
|
+
async installTo(headers, url, bearerTokenOverride) {
|
|
1659
|
+
await super.installTo(headers, url, bearerTokenOverride);
|
|
1660
|
+
headers.set("x-twitter-auth-type", "OAuth2Session");
|
|
1661
|
+
headers.set("x-twitter-active-user", "yes");
|
|
1662
|
+
headers.set("x-twitter-client-language", "en");
|
|
786
1663
|
}
|
|
787
1664
|
async initLogin() {
|
|
788
|
-
this.removeCookie("twitter_ads_id
|
|
789
|
-
this.removeCookie("ads_prefs
|
|
790
|
-
this.removeCookie("_twitter_sess
|
|
791
|
-
this.removeCookie("zipbox_forms_auth_token
|
|
792
|
-
this.removeCookie("lang
|
|
793
|
-
this.removeCookie("bouncer_reset_cookie
|
|
794
|
-
this.removeCookie("twid
|
|
795
|
-
this.removeCookie("twitter_ads_idb
|
|
796
|
-
this.removeCookie("email_uid
|
|
797
|
-
this.removeCookie("external_referer
|
|
798
|
-
this.removeCookie("
|
|
799
|
-
this.removeCookie("aa_u=");
|
|
800
|
-
this.removeCookie("__cf_bm=");
|
|
1665
|
+
await this.removeCookie("twitter_ads_id");
|
|
1666
|
+
await this.removeCookie("ads_prefs");
|
|
1667
|
+
await this.removeCookie("_twitter_sess");
|
|
1668
|
+
await this.removeCookie("zipbox_forms_auth_token");
|
|
1669
|
+
await this.removeCookie("lang");
|
|
1670
|
+
await this.removeCookie("bouncer_reset_cookie");
|
|
1671
|
+
await this.removeCookie("twid");
|
|
1672
|
+
await this.removeCookie("twitter_ads_idb");
|
|
1673
|
+
await this.removeCookie("email_uid");
|
|
1674
|
+
await this.removeCookie("external_referer");
|
|
1675
|
+
await this.removeCookie("aa_u");
|
|
801
1676
|
return await this.executeFlowTask({
|
|
802
1677
|
flow_name: "login",
|
|
803
1678
|
input_flow_data: {
|
|
804
1679
|
flow_context: {
|
|
805
1680
|
debug_overrides: {},
|
|
806
1681
|
start_location: {
|
|
807
|
-
location: "
|
|
1682
|
+
location: "manual_link"
|
|
808
1683
|
}
|
|
809
1684
|
}
|
|
810
1685
|
},
|
|
@@ -853,20 +1728,157 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
853
1728
|
}
|
|
854
1729
|
});
|
|
855
1730
|
}
|
|
856
|
-
async handleJsInstrumentationSubtask(subtaskId,
|
|
1731
|
+
async handleJsInstrumentationSubtask(subtaskId, prev, _credentials, api) {
|
|
1732
|
+
const subtasks = prev.subtasks;
|
|
1733
|
+
const jsSubtask = subtasks?.find((s) => s.subtask_id === subtaskId);
|
|
1734
|
+
const jsUrl = jsSubtask?.js_instrumentation?.url;
|
|
1735
|
+
let metricsResponse = "{}";
|
|
1736
|
+
if (jsUrl) {
|
|
1737
|
+
try {
|
|
1738
|
+
metricsResponse = await this.executeJsInstrumentation(jsUrl);
|
|
1739
|
+
log$1(
|
|
1740
|
+
`JS instrumentation executed successfully, response length: ${metricsResponse.length}`
|
|
1741
|
+
);
|
|
1742
|
+
} catch (err) {
|
|
1743
|
+
log$1("Failed to execute JS instrumentation (falling back to {})", err);
|
|
1744
|
+
}
|
|
1745
|
+
}
|
|
857
1746
|
return await api.sendFlowRequest({
|
|
858
1747
|
flow_token: api.getFlowToken(),
|
|
859
1748
|
subtask_inputs: [
|
|
860
1749
|
{
|
|
861
1750
|
subtask_id: subtaskId,
|
|
862
1751
|
js_instrumentation: {
|
|
863
|
-
response:
|
|
1752
|
+
response: metricsResponse,
|
|
864
1753
|
link: "next_link"
|
|
865
1754
|
}
|
|
866
1755
|
}
|
|
867
1756
|
]
|
|
868
1757
|
});
|
|
869
1758
|
}
|
|
1759
|
+
// 512KB
|
|
1760
|
+
/**
|
|
1761
|
+
* Fetches and executes the JS instrumentation script to generate browser
|
|
1762
|
+
* fingerprinting data. The result is written to an input element named
|
|
1763
|
+
* 'ui_metrics'.
|
|
1764
|
+
*
|
|
1765
|
+
* In browser environments, uses a hidden iframe with native DOM APIs.
|
|
1766
|
+
* In Node.js, uses linkedom (for DOM) and the vm module for execution.
|
|
1767
|
+
*
|
|
1768
|
+
* @security This method executes **remote JavaScript** fetched from Twitter's servers.
|
|
1769
|
+
* - In browsers, execution is isolated in a disposable iframe.
|
|
1770
|
+
* - In Node.js, `vm.runInContext` is used for convenience, NOT for security.
|
|
1771
|
+
* Node's `vm` module provides NO security sandbox — a malicious script can
|
|
1772
|
+
* trivially escape the context (e.g., via `this.constructor.constructor('return process')()`).
|
|
1773
|
+
* The only real trust boundary is that scripts are fetched from Twitter's known CDN URLs.
|
|
1774
|
+
* Setting `process: undefined` etc. in the sandbox context is cosmetic and does not
|
|
1775
|
+
* prevent escape.
|
|
1776
|
+
* - A maximum script size limit (512KB) and a 5-second timeout provide basic sanity checks.
|
|
1777
|
+
*/
|
|
1778
|
+
async executeJsInstrumentation(url) {
|
|
1779
|
+
log$1(`Fetching JS instrumentation from: ${url}`);
|
|
1780
|
+
const response = await this.fetch(url);
|
|
1781
|
+
const scriptContent = await response.text();
|
|
1782
|
+
log$1(`JS instrumentation script fetched, length: ${scriptContent.length}`);
|
|
1783
|
+
if (scriptContent.length > _TwitterUserAuth.JS_INSTRUMENTATION_MAX_SIZE) {
|
|
1784
|
+
log$1(
|
|
1785
|
+
`WARNING: JS instrumentation script exceeds size limit (${scriptContent.length} > ${_TwitterUserAuth.JS_INSTRUMENTATION_MAX_SIZE}), skipping execution`
|
|
1786
|
+
);
|
|
1787
|
+
return "{}";
|
|
1788
|
+
}
|
|
1789
|
+
if (typeof window !== "undefined" && typeof document !== "undefined") {
|
|
1790
|
+
return this.executeJsInstrumentationBrowser(scriptContent);
|
|
1791
|
+
}
|
|
1792
|
+
return this.executeJsInstrumentationNode(scriptContent);
|
|
1793
|
+
}
|
|
1794
|
+
/**
|
|
1795
|
+
* Execute JS instrumentation in a browser environment using a hidden iframe.
|
|
1796
|
+
* The iframe provides natural isolation — the script gets its own document
|
|
1797
|
+
* and window, and we can override setTimeout without affecting the host page.
|
|
1798
|
+
*/
|
|
1799
|
+
async executeJsInstrumentationBrowser(scriptContent) {
|
|
1800
|
+
const iframe = document.createElement("iframe");
|
|
1801
|
+
iframe.style.display = "none";
|
|
1802
|
+
document.body.appendChild(iframe);
|
|
1803
|
+
try {
|
|
1804
|
+
const iframeWin = iframe.contentWindow;
|
|
1805
|
+
const iframeDoc = iframe.contentDocument;
|
|
1806
|
+
if (!iframeWin || !iframeDoc) {
|
|
1807
|
+
log$1("WARNING: Could not access iframe document/window");
|
|
1808
|
+
return "{}";
|
|
1809
|
+
}
|
|
1810
|
+
const input = iframeDoc.createElement("input");
|
|
1811
|
+
input.name = "ui_metrics";
|
|
1812
|
+
input.type = "hidden";
|
|
1813
|
+
iframeDoc.body.appendChild(input);
|
|
1814
|
+
iframeWin.setTimeout = (fn) => fn();
|
|
1815
|
+
const script = iframeDoc.createElement("script");
|
|
1816
|
+
script.textContent = scriptContent;
|
|
1817
|
+
iframeDoc.body.appendChild(script);
|
|
1818
|
+
const value = input.value;
|
|
1819
|
+
if (value) {
|
|
1820
|
+
log$1(`JS instrumentation result extracted, length: ${value.length}`);
|
|
1821
|
+
return value;
|
|
1822
|
+
}
|
|
1823
|
+
log$1("WARNING: No ui_metrics value found after script execution");
|
|
1824
|
+
return "{}";
|
|
1825
|
+
} finally {
|
|
1826
|
+
document.body.removeChild(iframe);
|
|
1827
|
+
}
|
|
1828
|
+
}
|
|
1829
|
+
/**
|
|
1830
|
+
* Execute JS instrumentation in Node.js using linkedom for DOM emulation
|
|
1831
|
+
* and the vm module for sandboxed script execution.
|
|
1832
|
+
*
|
|
1833
|
+
* @security Node's `vm` module does NOT provide a security sandbox. A
|
|
1834
|
+
* malicious script can trivially escape the context. The only real trust
|
|
1835
|
+
* boundary is that scripts come from Twitter's CDN. The undefined globals
|
|
1836
|
+
* (process, require, etc.) are cosmetic — they do not prevent sandbox escape.
|
|
1837
|
+
*/
|
|
1838
|
+
async executeJsInstrumentationNode(scriptContent) {
|
|
1839
|
+
const { parseHTML } = await import('linkedom');
|
|
1840
|
+
const { document: doc, window: win } = parseHTML(
|
|
1841
|
+
'<html><head></head><body><input name="ui_metrics" type="hidden" value="" /></body></html>'
|
|
1842
|
+
);
|
|
1843
|
+
if (typeof doc.getElementsByName !== "function") {
|
|
1844
|
+
doc.getElementsByName = (name) => doc.querySelectorAll(`[name="${name}"]`);
|
|
1845
|
+
}
|
|
1846
|
+
const vm = await import('vm');
|
|
1847
|
+
const origSetTimeout = win.setTimeout;
|
|
1848
|
+
win.setTimeout = (fn) => fn();
|
|
1849
|
+
try {
|
|
1850
|
+
Object.defineProperty(doc, "readyState", {
|
|
1851
|
+
value: "complete",
|
|
1852
|
+
writable: true,
|
|
1853
|
+
configurable: true
|
|
1854
|
+
});
|
|
1855
|
+
} catch {
|
|
1856
|
+
}
|
|
1857
|
+
const sandbox = {
|
|
1858
|
+
document: doc,
|
|
1859
|
+
window: win,
|
|
1860
|
+
Date,
|
|
1861
|
+
JSON,
|
|
1862
|
+
parseInt,
|
|
1863
|
+
// Deny access to Node.js internals to limit sandbox escape surface
|
|
1864
|
+
process: void 0,
|
|
1865
|
+
require: void 0,
|
|
1866
|
+
global: void 0,
|
|
1867
|
+
globalThis: void 0
|
|
1868
|
+
};
|
|
1869
|
+
vm.runInNewContext(scriptContent, sandbox, { timeout: 5e3 });
|
|
1870
|
+
win.setTimeout = origSetTimeout;
|
|
1871
|
+
const inputs = doc.getElementsByName("ui_metrics");
|
|
1872
|
+
if (inputs && inputs.length > 0) {
|
|
1873
|
+
const value = inputs[0].value || inputs[0].getAttribute("value");
|
|
1874
|
+
if (value) {
|
|
1875
|
+
log$1(`JS instrumentation result extracted, length: ${value.length}`);
|
|
1876
|
+
return value;
|
|
1877
|
+
}
|
|
1878
|
+
}
|
|
1879
|
+
log$1("WARNING: No ui_metrics value found after script execution");
|
|
1880
|
+
return "{}";
|
|
1881
|
+
}
|
|
870
1882
|
async handleEnterAlternateIdentifierSubtask(subtaskId, _prev, credentials, api) {
|
|
871
1883
|
return await this.executeFlowTask({
|
|
872
1884
|
flow_token: api.getFlowToken(),
|
|
@@ -882,36 +1894,76 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
882
1894
|
});
|
|
883
1895
|
}
|
|
884
1896
|
async handleEnterUserIdentifierSSO(subtaskId, _prev, credentials, api) {
|
|
1897
|
+
let castleToken;
|
|
1898
|
+
try {
|
|
1899
|
+
castleToken = await this.generateCastleToken();
|
|
1900
|
+
log$1(`Castle token generated, length: ${castleToken.length}`);
|
|
1901
|
+
} catch (err) {
|
|
1902
|
+
log$1("Failed to generate castle token (continuing without it):", err);
|
|
1903
|
+
}
|
|
1904
|
+
const settingsList = {
|
|
1905
|
+
setting_responses: [
|
|
1906
|
+
{
|
|
1907
|
+
key: "user_identifier",
|
|
1908
|
+
response_data: {
|
|
1909
|
+
text_data: { result: credentials.username }
|
|
1910
|
+
}
|
|
1911
|
+
}
|
|
1912
|
+
],
|
|
1913
|
+
link: "next_link"
|
|
1914
|
+
};
|
|
1915
|
+
if (castleToken) {
|
|
1916
|
+
settingsList.castle_token = castleToken;
|
|
1917
|
+
}
|
|
885
1918
|
return await this.executeFlowTask({
|
|
886
1919
|
flow_token: api.getFlowToken(),
|
|
887
1920
|
subtask_inputs: [
|
|
888
1921
|
{
|
|
889
1922
|
subtask_id: subtaskId,
|
|
890
|
-
settings_list:
|
|
891
|
-
setting_responses: [
|
|
892
|
-
{
|
|
893
|
-
key: "user_identifier",
|
|
894
|
-
response_data: {
|
|
895
|
-
text_data: { result: credentials.username }
|
|
896
|
-
}
|
|
897
|
-
}
|
|
898
|
-
],
|
|
899
|
-
link: "next_link"
|
|
900
|
-
}
|
|
1923
|
+
settings_list: settingsList
|
|
901
1924
|
}
|
|
902
1925
|
]
|
|
903
1926
|
});
|
|
904
1927
|
}
|
|
1928
|
+
/**
|
|
1929
|
+
* Generates a Castle.io device fingerprint token for the login flow.
|
|
1930
|
+
* Uses local token generation (Castle.io v11 format) to avoid external
|
|
1931
|
+
* API dependencies and rate limits.
|
|
1932
|
+
*/
|
|
1933
|
+
async generateCastleToken() {
|
|
1934
|
+
const userAgent = CHROME_USER_AGENT;
|
|
1935
|
+
const browserProfile = this.options?.experimental?.browserProfile;
|
|
1936
|
+
const { token, cuid } = generateLocalCastleToken(userAgent, browserProfile);
|
|
1937
|
+
await this.setCookie("__cuid", cuid);
|
|
1938
|
+
log$1(
|
|
1939
|
+
`Castle token generated locally, length: ${token.length}, cuid: ${cuid.substring(0, 6)}...`
|
|
1940
|
+
);
|
|
1941
|
+
return token;
|
|
1942
|
+
}
|
|
905
1943
|
async handleEnterPassword(subtaskId, _prev, credentials, api) {
|
|
1944
|
+
let castleToken;
|
|
1945
|
+
try {
|
|
1946
|
+
castleToken = await this.generateCastleToken();
|
|
1947
|
+
log$1(`Castle token for password step, length: ${castleToken.length}`);
|
|
1948
|
+
} catch (err) {
|
|
1949
|
+
log$1(
|
|
1950
|
+
"Failed to generate castle token for password (continuing without):",
|
|
1951
|
+
err
|
|
1952
|
+
);
|
|
1953
|
+
}
|
|
1954
|
+
const enterPassword = {
|
|
1955
|
+
password: credentials.password,
|
|
1956
|
+
link: "next_link"
|
|
1957
|
+
};
|
|
1958
|
+
if (castleToken) {
|
|
1959
|
+
enterPassword.castle_token = castleToken;
|
|
1960
|
+
}
|
|
906
1961
|
return await this.executeFlowTask({
|
|
907
1962
|
flow_token: api.getFlowToken(),
|
|
908
1963
|
subtask_inputs: [
|
|
909
1964
|
{
|
|
910
1965
|
subtask_id: subtaskId,
|
|
911
|
-
enter_password:
|
|
912
|
-
password: credentials.password,
|
|
913
|
-
link: "next_link"
|
|
914
|
-
}
|
|
1966
|
+
enter_password: enterPassword
|
|
915
1967
|
}
|
|
916
1968
|
]
|
|
917
1969
|
});
|
|
@@ -939,27 +1991,28 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
939
1991
|
};
|
|
940
1992
|
}
|
|
941
1993
|
const totp = new OTPAuth__namespace.TOTP({ secret: credentials.twoFactorSecret });
|
|
942
|
-
let
|
|
1994
|
+
let lastResult;
|
|
943
1995
|
for (let attempts = 1; attempts < 4; attempts += 1) {
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
text: totp.generate()
|
|
953
|
-
}
|
|
1996
|
+
const result = await api.sendFlowRequest({
|
|
1997
|
+
flow_token: api.getFlowToken(),
|
|
1998
|
+
subtask_inputs: [
|
|
1999
|
+
{
|
|
2000
|
+
subtask_id: subtaskId,
|
|
2001
|
+
enter_text: {
|
|
2002
|
+
link: "next_link",
|
|
2003
|
+
text: totp.generate()
|
|
954
2004
|
}
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
}
|
|
958
|
-
|
|
959
|
-
|
|
2005
|
+
}
|
|
2006
|
+
]
|
|
2007
|
+
});
|
|
2008
|
+
if (result.status === "success") {
|
|
2009
|
+
return result;
|
|
960
2010
|
}
|
|
2011
|
+
lastResult = result;
|
|
2012
|
+
log$1(`2FA attempt ${attempts} failed: ${result.err.message}`);
|
|
2013
|
+
await new Promise((resolve) => setTimeout(resolve, 2e3 * attempts));
|
|
961
2014
|
}
|
|
962
|
-
|
|
2015
|
+
return lastResult;
|
|
963
2016
|
}
|
|
964
2017
|
async handleAcid(subtaskId, _prev, credentials, api) {
|
|
965
2018
|
return await this.executeFlowTask({
|
|
@@ -976,7 +2029,7 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
976
2029
|
});
|
|
977
2030
|
}
|
|
978
2031
|
async handleSuccessSubtask() {
|
|
979
|
-
log("Successfully logged in with user credentials.");
|
|
2032
|
+
log$1("Successfully logged in with user credentials.");
|
|
980
2033
|
return {
|
|
981
2034
|
status: "success",
|
|
982
2035
|
response: {}
|
|
@@ -987,28 +2040,33 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
987
2040
|
if ("flow_name" in data) {
|
|
988
2041
|
onboardingTaskUrl = `https://api.x.com/1.1/onboarding/task.json?flow_name=${data.flow_name}`;
|
|
989
2042
|
}
|
|
990
|
-
log(`Making POST request to ${onboardingTaskUrl}`);
|
|
2043
|
+
log$1(`Making POST request to ${onboardingTaskUrl}`);
|
|
2044
|
+
log$1(
|
|
2045
|
+
"Request data:",
|
|
2046
|
+
JSON.stringify(
|
|
2047
|
+
data,
|
|
2048
|
+
(key, value) => key === "password" ? "[REDACTED]" : value,
|
|
2049
|
+
2
|
|
2050
|
+
)
|
|
2051
|
+
);
|
|
991
2052
|
const headers = new headersPolyfill.Headers({
|
|
992
2053
|
accept: "*/*",
|
|
993
2054
|
"accept-language": "en-US,en;q=0.9",
|
|
994
2055
|
"content-type": "application/json",
|
|
995
|
-
"cache-control": "no-cache",
|
|
996
2056
|
origin: "https://x.com",
|
|
997
|
-
pragma: "no-cache",
|
|
998
2057
|
priority: "u=1, i",
|
|
999
2058
|
referer: "https://x.com/",
|
|
1000
|
-
"sec-ch-ua":
|
|
2059
|
+
"sec-ch-ua": CHROME_SEC_CH_UA,
|
|
1001
2060
|
"sec-ch-ua-mobile": "?0",
|
|
1002
2061
|
"sec-ch-ua-platform": '"Windows"',
|
|
1003
2062
|
"sec-fetch-dest": "empty",
|
|
1004
2063
|
"sec-fetch-mode": "cors",
|
|
1005
|
-
"sec-fetch-site": "same-
|
|
1006
|
-
"user-agent":
|
|
1007
|
-
"x-twitter-auth-type": "OAuth2Client",
|
|
2064
|
+
"sec-fetch-site": "same-site",
|
|
2065
|
+
"user-agent": CHROME_USER_AGENT,
|
|
1008
2066
|
"x-twitter-active-user": "yes",
|
|
1009
2067
|
"x-twitter-client-language": "en"
|
|
1010
2068
|
});
|
|
1011
|
-
await this.
|
|
2069
|
+
await this.installAuthCredentials(headers);
|
|
1012
2070
|
if (this.options?.experimental?.xClientTransactionId) {
|
|
1013
2071
|
const transactionId = await generateTransactionId(
|
|
1014
2072
|
onboardingTaskUrl,
|
|
@@ -1017,6 +2075,10 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1017
2075
|
);
|
|
1018
2076
|
headers.set("x-client-transaction-id", transactionId);
|
|
1019
2077
|
}
|
|
2078
|
+
const bodyData = { ...data };
|
|
2079
|
+
if ("flow_name" in bodyData) {
|
|
2080
|
+
delete bodyData.flow_name;
|
|
2081
|
+
}
|
|
1020
2082
|
let res;
|
|
1021
2083
|
do {
|
|
1022
2084
|
const fetchParameters = [
|
|
@@ -1025,7 +2087,7 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1025
2087
|
credentials: "include",
|
|
1026
2088
|
method: "POST",
|
|
1027
2089
|
headers,
|
|
1028
|
-
body: JSON.stringify(
|
|
2090
|
+
body: JSON.stringify(bodyData)
|
|
1029
2091
|
}
|
|
1030
2092
|
];
|
|
1031
2093
|
try {
|
|
@@ -1040,25 +2102,58 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1040
2102
|
};
|
|
1041
2103
|
}
|
|
1042
2104
|
await updateCookieJar(this.jar, res.headers);
|
|
2105
|
+
log$1(`Response status: ${res.status}`);
|
|
1043
2106
|
if (res.status === 429) {
|
|
1044
|
-
log("Rate limit hit, waiting before retrying...");
|
|
2107
|
+
log$1("Rate limit hit, waiting before retrying...");
|
|
1045
2108
|
await this.onRateLimit({
|
|
1046
2109
|
fetchParameters,
|
|
1047
2110
|
response: res
|
|
1048
2111
|
});
|
|
1049
2112
|
}
|
|
1050
2113
|
} while (res.status === 429);
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
2114
|
+
let flow;
|
|
2115
|
+
try {
|
|
2116
|
+
flow = await flexParseJson(res);
|
|
2117
|
+
} catch {
|
|
2118
|
+
if (!res.ok) {
|
|
2119
|
+
return {
|
|
2120
|
+
status: "error",
|
|
2121
|
+
err: new ApiError(res, "Failed to parse response body")
|
|
2122
|
+
};
|
|
2123
|
+
}
|
|
1056
2124
|
return {
|
|
1057
2125
|
status: "error",
|
|
1058
|
-
err: new AuthenticationError("
|
|
2126
|
+
err: new AuthenticationError("Failed to parse flow response.")
|
|
1059
2127
|
};
|
|
1060
2128
|
}
|
|
2129
|
+
log$1(
|
|
2130
|
+
"Flow response: status=%s subtasks=%s",
|
|
2131
|
+
flow.status,
|
|
2132
|
+
flow.subtasks?.map((s) => s.subtask_id).join(", ")
|
|
2133
|
+
);
|
|
1061
2134
|
if (flow.errors?.length) {
|
|
2135
|
+
log$1("Twitter auth flow errors:", JSON.stringify(flow.errors, null, 2));
|
|
2136
|
+
if (flow.errors[0].code === 399) {
|
|
2137
|
+
const message = flow.errors[0].message || "";
|
|
2138
|
+
const challengeMatch = message.match(/g;[^:]+:[^:]+:[0-9]+/);
|
|
2139
|
+
if (challengeMatch) {
|
|
2140
|
+
log$1("Twitter challenge token detected:", challengeMatch[0]);
|
|
2141
|
+
}
|
|
2142
|
+
return {
|
|
2143
|
+
status: "error",
|
|
2144
|
+
err: new AuthenticationError(
|
|
2145
|
+
`Twitter blocked this login attempt due to suspicious activity (error 399). This is not an issue with your credentials - Twitter requires additional authentication.
|
|
2146
|
+
|
|
2147
|
+
Solutions:
|
|
2148
|
+
1. Use cookie-based authentication (RECOMMENDED): Export cookies from your browser and use scraper.setCookies() - see README for details
|
|
2149
|
+
2. Enable Two-Factor Authentication (2FA) on your account and provide totp_secret
|
|
2150
|
+
3. Wait 15 minutes before retrying (Twitter rate limit for suspicious logins)
|
|
2151
|
+
4. Login via browser first to establish device trust
|
|
2152
|
+
|
|
2153
|
+
Original error: ${message}`
|
|
2154
|
+
)
|
|
2155
|
+
};
|
|
2156
|
+
}
|
|
1062
2157
|
return {
|
|
1063
2158
|
status: "error",
|
|
1064
2159
|
err: new AuthenticationError(
|
|
@@ -1066,6 +2161,15 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1066
2161
|
)
|
|
1067
2162
|
};
|
|
1068
2163
|
}
|
|
2164
|
+
if (!res.ok) {
|
|
2165
|
+
return { status: "error", err: new ApiError(res, flow) };
|
|
2166
|
+
}
|
|
2167
|
+
if (flow?.flow_token == null) {
|
|
2168
|
+
return {
|
|
2169
|
+
status: "error",
|
|
2170
|
+
err: new AuthenticationError("flow_token not found.")
|
|
2171
|
+
};
|
|
2172
|
+
}
|
|
1069
2173
|
if (typeof flow.flow_token !== "string") {
|
|
1070
2174
|
return {
|
|
1071
2175
|
status: "error",
|
|
@@ -1073,7 +2177,12 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1073
2177
|
};
|
|
1074
2178
|
}
|
|
1075
2179
|
const subtask = flow.subtasks?.length ? flow.subtasks[0] : void 0;
|
|
1076
|
-
value.Check(TwitterUserAuthSubtask, subtask)
|
|
2180
|
+
if (subtask && !value.Check(TwitterUserAuthSubtask, subtask)) {
|
|
2181
|
+
log$1(
|
|
2182
|
+
"WARNING: Subtask failed schema validation: %s",
|
|
2183
|
+
subtask.subtask_id ?? "unknown"
|
|
2184
|
+
);
|
|
2185
|
+
}
|
|
1077
2186
|
if (subtask && subtask.subtask_id === "DenyLoginSubtask") {
|
|
1078
2187
|
return {
|
|
1079
2188
|
status: "error",
|
|
@@ -1085,19 +2194,26 @@ class TwitterUserAuth extends TwitterGuestAuth {
|
|
|
1085
2194
|
response: flow
|
|
1086
2195
|
};
|
|
1087
2196
|
}
|
|
1088
|
-
}
|
|
2197
|
+
};
|
|
2198
|
+
/**
|
|
2199
|
+
* Maximum allowed size (in bytes) for the JS instrumentation script.
|
|
2200
|
+
* Twitter's scripts are typically ~50-100KB. Anything significantly larger
|
|
2201
|
+
* may indicate tampering or an unexpected response.
|
|
2202
|
+
*/
|
|
2203
|
+
_TwitterUserAuth.JS_INSTRUMENTATION_MAX_SIZE = 512 * 1024;
|
|
2204
|
+
let TwitterUserAuth = _TwitterUserAuth;
|
|
1089
2205
|
|
|
1090
2206
|
const endpoints = {
|
|
1091
|
-
UserTweets: "https://x.com/
|
|
1092
|
-
UserTweetsAndReplies: "https://x.com/
|
|
1093
|
-
UserLikedTweets: "https://x.com/
|
|
1094
|
-
UserByScreenName: "https://api.x.com/graphql
|
|
1095
|
-
TweetDetail: "https://x.com/
|
|
1096
|
-
TweetResultByRestId: "https://api.x.com/graphql/
|
|
1097
|
-
ListTweets: "https://x.com/
|
|
1098
|
-
SearchTimeline: "https://x.com/
|
|
1099
|
-
Followers: "https://x.com/
|
|
1100
|
-
Following: "https://x.com/
|
|
2207
|
+
UserTweets: "https://api.x.com/graphql/N2tFDY-MlrLxXJ9F_ZxJGA/UserTweets?variables=%7B%22userId%22%3A%2244196397%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
2208
|
+
UserTweetsAndReplies: "https://api.x.com/graphql/2NDLUdBmT_IB5uGwZ3tHRg/UserTweetsAndReplies?variables=%7B%22userId%22%3A%221806359170830172162%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
2209
|
+
UserLikedTweets: "https://api.x.com/graphql/Pcw-j9lrSeDMmkgnIejJiQ/Likes?variables=%7B%22userId%22%3A%222244196397%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withClientEventToken%22%3Afalse%2C%22withBirdwatchNotes%22%3Afalse%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D",
|
|
2210
|
+
UserByScreenName: "https://api.x.com/graphql/AWbeRIdkLtqTRN7yL_H8yw/UserByScreenName?variables=%7B%22screen_name%22%3A%22elonmusk%22%2C%22withGrokTranslatedBio%22%3Atrue%7D&features=%7B%22hidden_profile_subscriptions_enabled%22%3Atrue%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22subscriptions_verification_info_is_identity_verified_enabled%22%3Atrue%2C%22subscriptions_verification_info_verified_since_enabled%22%3Atrue%2C%22highlights_tweets_tab_ui_enabled%22%3Atrue%2C%22responsive_web_twitter_article_notes_tab_enabled%22%3Atrue%2C%22subscriptions_feature_can_gift_premium%22%3Atrue%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%7D&fieldToggles=%7B%22withPayments%22%3Afalse%2C%22withAuxiliaryUserLabels%22%3Atrue%7D",
|
|
2211
|
+
TweetDetail: "https://api.x.com/graphql/YCNdW_ZytXfV9YR3cJK9kw/TweetDetail?variables=%7B%22focalTweetId%22%3A%221985465713096794294%22%2C%22with_rux_injections%22%3Afalse%2C%22rankingMode%22%3A%22Relevance%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withQuickPromoteEligibilityTweetFields%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Atrue%2C%22withArticlePlainText%22%3Afalse%2C%22withGrokAnalyze%22%3Afalse%2C%22withDisallowedReplyControls%22%3Afalse%7D",
|
|
2212
|
+
TweetResultByRestId: "https://api.x.com/graphql/4PdbzTmQ5PTjz9RiureISQ/TweetResultByRestId?variables=%7B%22tweetId%22%3A%221985465713096794294%22%2C%22includePromotedContent%22%3Atrue%2C%22withBirdwatchNotes%22%3Atrue%2C%22withVoice%22%3Atrue%2C%22withCommunity%22%3Atrue%7D&features=%7B%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticleRichContentState%22%3Atrue%2C%22withArticlePlainText%22%3Afalse%7D",
|
|
2213
|
+
ListTweets: "https://api.x.com/graphql/Uv3buKIUElzL3Iuc0L0O5g/ListLatestTweetsTimeline?variables=%7B%22listId%22%3A%221736495155002106192%22%2C%22count%22%3A20%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D",
|
|
2214
|
+
SearchTimeline: "https://api.x.com/graphql/ML-n2SfAxx5S_9QMqNejbg/SearchTimeline?variables=%7B%22rawQuery%22%3A%22twitter%22%2C%22count%22%3A20%2C%22querySource%22%3A%22typed_query%22%2C%22product%22%3A%22Top%22%2C%22withGrokTranslatedBio%22%3Afalse%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D",
|
|
2215
|
+
Followers: "https://api.x.com/graphql/P7m4Qr-rJEB8KUluOenU6A/Followers?variables=%7B%22userId%22%3A%221806359170830172162%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withGrokTranslatedBio%22%3Afalse%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D",
|
|
2216
|
+
Following: "https://api.x.com/graphql/T5wihsMTYHncY7BB4YxHSg/Following?variables=%7B%22userId%22%3A%221806359170830172162%22%2C%22count%22%3A20%2C%22includePromotedContent%22%3Afalse%2C%22withGrokTranslatedBio%22%3Afalse%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22responsive_web_profile_redirect_enabled%22%3Afalse%2C%22rweb_tipjar_consumption_enabled%22%3Afalse%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Atrue%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22responsive_web_grok_annotations_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Atrue%2C%22responsive_web_grok_analysis_button_from_backend%22%3Atrue%2C%22post_ctas_fetch_enabled%22%3Atrue%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_imagine_annotation_enabled%22%3Atrue%2C%22responsive_web_grok_community_note_auto_translation_is_enabled%22%3Afalse%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D"
|
|
1101
2217
|
};
|
|
1102
2218
|
class ApiRequest {
|
|
1103
2219
|
constructor(info) {
|
|
@@ -1896,6 +3012,15 @@ function parseRelationshipTimeline(timeline) {
|
|
|
1896
3012
|
if (!profile.userId) {
|
|
1897
3013
|
profile.userId = userResultRaw.rest_id;
|
|
1898
3014
|
}
|
|
3015
|
+
if (!profile.username && userResultRaw.core?.screen_name) {
|
|
3016
|
+
profile.username = userResultRaw.core.screen_name;
|
|
3017
|
+
profile.url = `https://x.com/${profile.username}`;
|
|
3018
|
+
}
|
|
3019
|
+
if (!profile.joined && userResultRaw.core?.created_at) {
|
|
3020
|
+
profile.joined = new Date(
|
|
3021
|
+
Date.parse(userResultRaw.core.created_at)
|
|
3022
|
+
);
|
|
3023
|
+
}
|
|
1899
3024
|
profiles.push(profile);
|
|
1900
3025
|
}
|
|
1901
3026
|
} else if (entry.content?.cursorType === "Bottom") {
|
|
@@ -2448,6 +3573,7 @@ function findDmConversationsByUserId(inbox, userId) {
|
|
|
2448
3573
|
return conversations;
|
|
2449
3574
|
}
|
|
2450
3575
|
|
|
3576
|
+
const log = debug("twitter-scraper:scraper");
|
|
2451
3577
|
const twUrl = "https://x.com";
|
|
2452
3578
|
class Scraper {
|
|
2453
3579
|
/**
|
|
@@ -2457,6 +3583,7 @@ class Scraper {
|
|
|
2457
3583
|
*/
|
|
2458
3584
|
constructor(options) {
|
|
2459
3585
|
this.options = options;
|
|
3586
|
+
this.subtaskHandlers = /* @__PURE__ */ new Map();
|
|
2460
3587
|
this.token = bearerToken;
|
|
2461
3588
|
this.useGuestAuth();
|
|
2462
3589
|
}
|
|
@@ -2467,6 +3594,7 @@ class Scraper {
|
|
|
2467
3594
|
* @param subtaskHandler The handler function to register.
|
|
2468
3595
|
*/
|
|
2469
3596
|
registerAuthSubtaskHandler(subtaskId, subtaskHandler) {
|
|
3597
|
+
this.subtaskHandlers.set(subtaskId, subtaskHandler);
|
|
2470
3598
|
if (this.auth instanceof TwitterUserAuth) {
|
|
2471
3599
|
this.auth.registerSubtaskHandler(subtaskId, subtaskHandler);
|
|
2472
3600
|
}
|
|
@@ -2474,6 +3602,15 @@ class Scraper {
|
|
|
2474
3602
|
this.authTrends.registerSubtaskHandler(subtaskId, subtaskHandler);
|
|
2475
3603
|
}
|
|
2476
3604
|
}
|
|
3605
|
+
/**
|
|
3606
|
+
* Applies all stored subtask handlers to the given auth instance.
|
|
3607
|
+
* @internal
|
|
3608
|
+
*/
|
|
3609
|
+
applySubtaskHandlers(auth) {
|
|
3610
|
+
for (const [subtaskId, handler] of this.subtaskHandlers) {
|
|
3611
|
+
auth.registerSubtaskHandler(subtaskId, handler);
|
|
3612
|
+
}
|
|
3613
|
+
}
|
|
2477
3614
|
/**
|
|
2478
3615
|
* Initializes auth properties using a guest token.
|
|
2479
3616
|
* Used when creating a new instance of this class, and when logging out.
|
|
@@ -2774,7 +3911,8 @@ class Scraper {
|
|
|
2774
3911
|
* @param twoFactorSecret The secret to generate two factor authentication tokens with, if you have two factor authentication enabled.
|
|
2775
3912
|
*/
|
|
2776
3913
|
async login(username, password, email, twoFactorSecret) {
|
|
2777
|
-
const userAuth = new TwitterUserAuth(
|
|
3914
|
+
const userAuth = new TwitterUserAuth(bearerToken2, this.getAuthOptions());
|
|
3915
|
+
this.applySubtaskHandlers(userAuth);
|
|
2778
3916
|
await userAuth.login(username, password, email, twoFactorSecret);
|
|
2779
3917
|
this.auth = userAuth;
|
|
2780
3918
|
this.authTrends = userAuth;
|
|
@@ -2801,12 +3939,37 @@ class Scraper {
|
|
|
2801
3939
|
* @param cookies The cookies to set for the current session.
|
|
2802
3940
|
*/
|
|
2803
3941
|
async setCookies(cookies) {
|
|
2804
|
-
const userAuth = new TwitterUserAuth(
|
|
3942
|
+
const userAuth = new TwitterUserAuth(bearerToken2, this.getAuthOptions());
|
|
3943
|
+
this.applySubtaskHandlers(userAuth);
|
|
2805
3944
|
for (const cookie of cookies) {
|
|
2806
|
-
|
|
3945
|
+
if (cookie == null) continue;
|
|
3946
|
+
if (typeof cookie === "string") {
|
|
3947
|
+
try {
|
|
3948
|
+
await userAuth.cookieJar().setCookie(cookie, "https://x.com");
|
|
3949
|
+
} catch (err) {
|
|
3950
|
+
log(`Failed to parse cookie string: ${err.message}`);
|
|
3951
|
+
}
|
|
3952
|
+
} else {
|
|
3953
|
+
if (cookie.domain && cookie.domain.startsWith(".")) {
|
|
3954
|
+
cookie.domain = cookie.domain.substring(1);
|
|
3955
|
+
cookie.hostOnly = false;
|
|
3956
|
+
}
|
|
3957
|
+
const cookieDomain = cookie.domain || "x.com";
|
|
3958
|
+
const cookieUrl = `https://${cookieDomain}`;
|
|
3959
|
+
await userAuth.cookieJar().setCookie(cookie, cookieUrl);
|
|
3960
|
+
}
|
|
2807
3961
|
}
|
|
2808
3962
|
this.auth = userAuth;
|
|
2809
3963
|
this.authTrends = userAuth;
|
|
3964
|
+
const isLoggedIn = await userAuth.isLoggedIn();
|
|
3965
|
+
if (!isLoggedIn) {
|
|
3966
|
+
const cookieString = await userAuth.cookieJar().getCookies(twUrl).then((c) => c.map((cookie) => cookie.key));
|
|
3967
|
+
if (cookieString.includes("ct0") && !cookieString.includes("auth_token")) {
|
|
3968
|
+
log(
|
|
3969
|
+
"auth_token cookie is missing. This is required for authenticated API access. The auth_token is an HttpOnly cookie that cannot be accessed via document.cookie. Export it using a browser extension (e.g., EditThisCookie) or DevTools Application tab."
|
|
3970
|
+
);
|
|
3971
|
+
}
|
|
3972
|
+
}
|
|
2810
3973
|
}
|
|
2811
3974
|
/**
|
|
2812
3975
|
* Clear all cookies for the current session.
|
|
@@ -2848,7 +4011,9 @@ class Scraper {
|
|
|
2848
4011
|
rateLimitStrategy: this.options?.rateLimitStrategy,
|
|
2849
4012
|
experimental: {
|
|
2850
4013
|
xClientTransactionId: this.options?.experimental?.xClientTransactionId,
|
|
2851
|
-
xpff: this.options?.experimental?.xpff
|
|
4014
|
+
xpff: this.options?.experimental?.xpff,
|
|
4015
|
+
flowStepDelay: this.options?.experimental?.flowStepDelay,
|
|
4016
|
+
browserProfile: this.options?.experimental?.browserProfile
|
|
2852
4017
|
}
|
|
2853
4018
|
};
|
|
2854
4019
|
}
|
|
@@ -2897,4 +4062,5 @@ exports.ErrorRateLimitStrategy = ErrorRateLimitStrategy;
|
|
|
2897
4062
|
exports.Scraper = Scraper;
|
|
2898
4063
|
exports.SearchMode = SearchMode;
|
|
2899
4064
|
exports.WaitingRateLimitStrategy = WaitingRateLimitStrategy;
|
|
4065
|
+
exports.randomizeBrowserProfile = randomizeBrowserProfile;
|
|
2900
4066
|
//# sourceMappingURL=index.cjs.map
|