advanced-tls-client 5.0.0 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +5 -0
- package/dist/index.js +247 -356
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -61,6 +61,7 @@ interface FullResponse {
|
|
|
61
61
|
end: number;
|
|
62
62
|
total: number;
|
|
63
63
|
};
|
|
64
|
+
redirectCount?: number;
|
|
64
65
|
}
|
|
65
66
|
interface ChromeProfile {
|
|
66
67
|
name: string;
|
|
@@ -144,6 +145,7 @@ interface Response extends EventEmitter {
|
|
|
144
145
|
raw?: Buffer;
|
|
145
146
|
cookies?: Record<string, string>;
|
|
146
147
|
parser?: string;
|
|
148
|
+
redirectCount?: number;
|
|
147
149
|
}
|
|
148
150
|
declare class ProfileRegistry {
|
|
149
151
|
private static getGrease;
|
|
@@ -163,6 +165,9 @@ declare class AdvancedTLSClient {
|
|
|
163
165
|
constructor(profileName?: string);
|
|
164
166
|
private setup;
|
|
165
167
|
private isRequestOptions;
|
|
168
|
+
private shouldFollowRedirect;
|
|
169
|
+
private getRedirectLocation;
|
|
170
|
+
private adjustHeadersForRedirect;
|
|
166
171
|
request(uri: string, dataOrOptions?: any, optionsOrCallback?: RequestOptions | ((err: any, res?: Response) => void), callback?: (err: any, res?: Response) => void): Promise<FullResponse | PassThrough>;
|
|
167
172
|
private generateJA3;
|
|
168
173
|
private buildMultipart;
|
package/dist/index.js
CHANGED
|
@@ -36,6 +36,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
36
36
|
exports.ProfileRegistry = exports.AdvancedTLSClient = void 0;
|
|
37
37
|
const tls = __importStar(require("tls"));
|
|
38
38
|
const net = __importStar(require("net"));
|
|
39
|
+
const https = __importStar(require("https"));
|
|
39
40
|
const http2 = __importStar(require("http2"));
|
|
40
41
|
const crypto = __importStar(require("crypto"));
|
|
41
42
|
const events_1 = require("events");
|
|
@@ -62,66 +63,11 @@ class ProfileRegistry {
|
|
|
62
63
|
name: 'chrome_mobile_143_android',
|
|
63
64
|
version: '143.0.0.0',
|
|
64
65
|
tls: {
|
|
65
|
-
cipherSuites: [
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
49195,
|
|
71
|
-
49199,
|
|
72
|
-
49196,
|
|
73
|
-
49200,
|
|
74
|
-
52393,
|
|
75
|
-
52392,
|
|
76
|
-
49171,
|
|
77
|
-
49172,
|
|
78
|
-
156,
|
|
79
|
-
157,
|
|
80
|
-
47,
|
|
81
|
-
53
|
|
82
|
-
],
|
|
83
|
-
extensions: [
|
|
84
|
-
grease2,
|
|
85
|
-
65037,
|
|
86
|
-
27,
|
|
87
|
-
43,
|
|
88
|
-
16,
|
|
89
|
-
35,
|
|
90
|
-
45,
|
|
91
|
-
11,
|
|
92
|
-
65281,
|
|
93
|
-
5,
|
|
94
|
-
10,
|
|
95
|
-
23,
|
|
96
|
-
0,
|
|
97
|
-
17513,
|
|
98
|
-
51,
|
|
99
|
-
18,
|
|
100
|
-
13,
|
|
101
|
-
grease3
|
|
102
|
-
],
|
|
103
|
-
supportedGroups: [
|
|
104
|
-
grease4,
|
|
105
|
-
25497,
|
|
106
|
-
29,
|
|
107
|
-
23,
|
|
108
|
-
24
|
|
109
|
-
],
|
|
110
|
-
signatureAlgorithms: [
|
|
111
|
-
1027,
|
|
112
|
-
2052,
|
|
113
|
-
1025,
|
|
114
|
-
1283,
|
|
115
|
-
2053,
|
|
116
|
-
1281,
|
|
117
|
-
2054,
|
|
118
|
-
1537
|
|
119
|
-
],
|
|
120
|
-
supportedVersions: [
|
|
121
|
-
grease2,
|
|
122
|
-
772,
|
|
123
|
-
771
|
|
124
|
-
],
|
|
66
|
+
cipherSuites: [grease1, 4865, 4866, 4867, 49195, 49199, 49196, 49200, 52393, 52392, 49171, 49172, 156, 157, 47, 53],
|
|
67
|
+
extensions: [grease2, 65037, 27, 43, 16, 35, 45, 11, 65281, 5, 10, 23, 0, 17513, 51, 18, 13, grease3],
|
|
68
|
+
supportedGroups: [grease4, 25497, 29, 23, 24],
|
|
69
|
+
signatureAlgorithms: [1027, 2052, 1025, 1283, 2053, 1281, 2054, 1537],
|
|
70
|
+
supportedVersions: [grease2, 772, 771],
|
|
125
71
|
ecPointFormats: [0],
|
|
126
72
|
alpnProtocols: ['h2', 'http/1.1'],
|
|
127
73
|
pskKeyExchangeModes: [1],
|
|
@@ -141,29 +87,9 @@ class ProfileRegistry {
|
|
|
141
87
|
maxHeaderListSize: 262144,
|
|
142
88
|
settingsOrder: [1, 2, 4, 6],
|
|
143
89
|
connectionPreface: Buffer.from('PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n'),
|
|
144
|
-
priorityFrames: [
|
|
145
|
-
{
|
|
146
|
-
streamId: 0,
|
|
147
|
-
weight: 256,
|
|
148
|
-
dependency: 0,
|
|
149
|
-
exclusive: true
|
|
150
|
-
}
|
|
151
|
-
],
|
|
90
|
+
priorityFrames: [{ streamId: 0, weight: 256, dependency: 0, exclusive: true }],
|
|
152
91
|
pseudoHeaderOrder: [':method', ':authority', ':scheme', ':path'],
|
|
153
|
-
headerOrder: [
|
|
154
|
-
'sec-ch-ua',
|
|
155
|
-
'sec-ch-ua-mobile',
|
|
156
|
-
'sec-ch-ua-platform',
|
|
157
|
-
'upgrade-insecure-requests',
|
|
158
|
-
'user-agent',
|
|
159
|
-
'accept',
|
|
160
|
-
'sec-fetch-site',
|
|
161
|
-
'sec-fetch-mode',
|
|
162
|
-
'sec-fetch-dest',
|
|
163
|
-
'accept-encoding',
|
|
164
|
-
'accept-language',
|
|
165
|
-
'priority'
|
|
166
|
-
]
|
|
92
|
+
headerOrder: ['sec-ch-ua', 'sec-ch-ua-mobile', 'sec-ch-ua-platform', 'upgrade-insecure-requests', 'user-agent', 'accept', 'sec-fetch-site', 'sec-fetch-mode', 'sec-fetch-dest', 'accept-encoding', 'accept-language', 'priority']
|
|
167
93
|
},
|
|
168
94
|
userAgent: 'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Mobile Safari/537.36',
|
|
169
95
|
secChUa: '"Google Chrome";v="143", "Chromium";v="143", "Not A(Brand";v="24"',
|
|
@@ -177,15 +103,9 @@ class ProfileRegistry {
|
|
|
177
103
|
priority: 'u=0, i'
|
|
178
104
|
};
|
|
179
105
|
}
|
|
180
|
-
static get(name) {
|
|
181
|
-
|
|
182
|
-
}
|
|
183
|
-
static get latest() {
|
|
184
|
-
return this.chromeMobile143Android();
|
|
185
|
-
}
|
|
186
|
-
static get latestMobile() {
|
|
187
|
-
return this.chromeMobile143Android();
|
|
188
|
-
}
|
|
106
|
+
static get(name) { return this.chromeMobile143Android(); }
|
|
107
|
+
static get latest() { return this.chromeMobile143Android(); }
|
|
108
|
+
static get latestMobile() { return this.chromeMobile143Android(); }
|
|
189
109
|
}
|
|
190
110
|
exports.ProfileRegistry = ProfileRegistry;
|
|
191
111
|
class CookieJar {
|
|
@@ -247,10 +167,6 @@ class CookieJar {
|
|
|
247
167
|
}
|
|
248
168
|
}
|
|
249
169
|
}
|
|
250
|
-
const Protocol = {
|
|
251
|
-
HTTP1: 'http1',
|
|
252
|
-
HTTP2: 'http2'
|
|
253
|
-
};
|
|
254
170
|
class UnifiedClientManager extends events_1.EventEmitter {
|
|
255
171
|
constructor(tlsSocket, profile, hostname, cookieJar) {
|
|
256
172
|
super();
|
|
@@ -291,10 +207,7 @@ class UnifiedClientManager extends events_1.EventEmitter {
|
|
|
291
207
|
break;
|
|
292
208
|
}
|
|
293
209
|
});
|
|
294
|
-
this.http2Client = http2.connect(`https://${this.hostname}`, {
|
|
295
|
-
createConnection: () => this.tlsSocket,
|
|
296
|
-
settings
|
|
297
|
-
});
|
|
210
|
+
this.http2Client = http2.connect(`https://${this.hostname}`, { createConnection: () => this.tlsSocket, settings });
|
|
298
211
|
this.http2Client.once('connect', resolve);
|
|
299
212
|
this.http2Client.once('error', reject);
|
|
300
213
|
this.http2Client.once('timeout', () => reject(new Error('HTTP/2 session timeout')));
|
|
@@ -321,21 +234,11 @@ class UnifiedClientManager extends events_1.EventEmitter {
|
|
|
321
234
|
}
|
|
322
235
|
async requestHttp2(path, options, timingStart, headers, post_data, out, callback) {
|
|
323
236
|
const normalizedPath = path.startsWith('/') ? path : `/${path}`;
|
|
324
|
-
const reqHeaders = {
|
|
325
|
-
':method': options.method?.toUpperCase() || 'GET',
|
|
326
|
-
':authority': this.hostname,
|
|
327
|
-
':scheme': 'https',
|
|
328
|
-
':path': normalizedPath,
|
|
329
|
-
...headers
|
|
330
|
-
};
|
|
237
|
+
const reqHeaders = { ':method': options.method?.toUpperCase() || 'GET', ':authority': this.hostname, ':scheme': 'https', ':path': normalizedPath, ...headers };
|
|
331
238
|
const stream = this.http2Client.request(reqHeaders);
|
|
332
239
|
const response = new events_1.EventEmitter();
|
|
333
240
|
response.timing = { socket: timingStart, lookup: 0, connect: 0, secureConnect: 0, response: 0, end: 0, total: 0 };
|
|
334
|
-
response.fingerprints = {
|
|
335
|
-
ja3: this.generateJA3(),
|
|
336
|
-
ja3Hash: crypto.createHash('md5').update(this.generateJA3()).digest('hex'),
|
|
337
|
-
akamai: `1:${this.profile.http2.headerTableSize};2:${this.profile.http2.enablePush};4:${this.profile.http2.initialWindowSize};6:${this.profile.http2.maxHeaderListSize ?? ''}|00|0|m,a,s,p`
|
|
338
|
-
};
|
|
241
|
+
response.fingerprints = { ja3: this.generateJA3(), ja3Hash: crypto.createHash('md5').update(this.generateJA3()).digest('hex'), akamai: `1:${this.profile.http2.headerTableSize};2:${this.profile.http2.enablePush};4:${this.profile.http2.initialWindowSize};6:${this.profile.http2.maxHeaderListSize ?? ''}|00|0|m,a,s,p` };
|
|
339
242
|
const chunks = [];
|
|
340
243
|
const rawChunks = [];
|
|
341
244
|
let finished = false;
|
|
@@ -370,9 +273,8 @@ class UnifiedClientManager extends events_1.EventEmitter {
|
|
|
370
273
|
response.body = body;
|
|
371
274
|
try {
|
|
372
275
|
response.text = body.toString('utf-8');
|
|
373
|
-
if (response.headers['content-type']?.includes('application/json'))
|
|
276
|
+
if (response.headers['content-type']?.includes('application/json'))
|
|
374
277
|
response.json = JSON.parse(response.text);
|
|
375
|
-
}
|
|
376
278
|
}
|
|
377
279
|
catch { }
|
|
378
280
|
response.timing.end = Date.now() - timingStart;
|
|
@@ -411,45 +313,29 @@ class UnifiedClientManager extends events_1.EventEmitter {
|
|
|
411
313
|
}
|
|
412
314
|
catch { }
|
|
413
315
|
}
|
|
414
|
-
setImmediate(() => {
|
|
415
|
-
|
|
416
|
-
stream.end();
|
|
417
|
-
}
|
|
418
|
-
});
|
|
316
|
+
setImmediate(() => { if (!stream.closed && !stream.destroyed && stream.writable)
|
|
317
|
+
stream.end(); });
|
|
419
318
|
}
|
|
420
319
|
else if (post_data instanceof stream_1.Readable) {
|
|
421
320
|
post_data.pipe(stream, { end: true });
|
|
422
321
|
post_data.on('error', err => stream.destroy(err));
|
|
423
322
|
}
|
|
424
323
|
else {
|
|
425
|
-
setImmediate(() => {
|
|
426
|
-
|
|
427
|
-
stream.end();
|
|
428
|
-
});
|
|
324
|
+
setImmediate(() => { if (stream.writable)
|
|
325
|
+
stream.end(); });
|
|
429
326
|
}
|
|
430
327
|
}
|
|
431
328
|
else {
|
|
432
|
-
setImmediate(() => {
|
|
433
|
-
|
|
434
|
-
stream.end();
|
|
435
|
-
});
|
|
329
|
+
setImmediate(() => { if (stream.writable)
|
|
330
|
+
stream.end(); });
|
|
436
331
|
}
|
|
437
332
|
}
|
|
438
333
|
async requestHttp1(path, options, timingStart, headers, post_data, out, callback) {
|
|
439
|
-
const reqOptions = {
|
|
440
|
-
|
|
441
|
-
path,
|
|
442
|
-
headers: { host: this.hostname, ...headers },
|
|
443
|
-
createConnection: () => this.tlsSocket
|
|
444
|
-
};
|
|
445
|
-
const req = require('https').request(reqOptions);
|
|
334
|
+
const reqOptions = { method: options.method?.toUpperCase() || 'GET', path, headers: { host: this.hostname, ...headers }, createConnection: () => this.tlsSocket };
|
|
335
|
+
const req = https.request(reqOptions);
|
|
446
336
|
const response = new events_1.EventEmitter();
|
|
447
337
|
response.timing = { socket: timingStart, lookup: 0, connect: 0, secureConnect: 0, response: 0, end: 0, total: 0 };
|
|
448
|
-
response.fingerprints = {
|
|
449
|
-
ja3: this.generateJA3(),
|
|
450
|
-
ja3Hash: crypto.createHash('md5').update(this.generateJA3()).digest('hex'),
|
|
451
|
-
akamai: ''
|
|
452
|
-
};
|
|
338
|
+
response.fingerprints = { ja3: this.generateJA3(), ja3Hash: crypto.createHash('md5').update(this.generateJA3()).digest('hex'), akamai: '' };
|
|
453
339
|
const chunks = [];
|
|
454
340
|
const rawChunks = [];
|
|
455
341
|
let finished = false;
|
|
@@ -486,9 +372,8 @@ class UnifiedClientManager extends events_1.EventEmitter {
|
|
|
486
372
|
response.body = body;
|
|
487
373
|
try {
|
|
488
374
|
response.text = body.toString('utf-8');
|
|
489
|
-
if (res.headers['content-type']?.includes('application/json'))
|
|
375
|
+
if (res.headers['content-type']?.includes('application/json'))
|
|
490
376
|
response.json = JSON.parse(response.text);
|
|
491
|
-
}
|
|
492
377
|
}
|
|
493
378
|
catch { }
|
|
494
379
|
response.timing.end = Date.now() - timingStart;
|
|
@@ -565,15 +450,8 @@ class TLSSocketManager extends events_1.EventEmitter {
|
|
|
565
450
|
this.port = port;
|
|
566
451
|
this.startTime = Date.now();
|
|
567
452
|
return new Promise((resolve, reject) => {
|
|
568
|
-
const timeout = setTimeout(() => {
|
|
569
|
-
|
|
570
|
-
reject(new Error('Connection timeout'));
|
|
571
|
-
}, 30000);
|
|
572
|
-
const handleError = (err) => {
|
|
573
|
-
clearTimeout(timeout);
|
|
574
|
-
this.destroy();
|
|
575
|
-
reject(err);
|
|
576
|
-
};
|
|
453
|
+
const timeout = setTimeout(() => { this.destroy(); reject(new Error('Connection timeout')); }, 30000);
|
|
454
|
+
const handleError = (err) => { clearTimeout(timeout); this.destroy(); reject(err); };
|
|
577
455
|
if (this.proxy) {
|
|
578
456
|
const proxyUrl = new url.URL(this.proxy);
|
|
579
457
|
this.socket = net.connect(+proxyUrl.port || 80, proxyUrl.hostname, () => {
|
|
@@ -605,9 +483,7 @@ class TLSSocketManager extends events_1.EventEmitter {
|
|
|
605
483
|
this.socket = new net.Socket();
|
|
606
484
|
this.socket.setNoDelay(true);
|
|
607
485
|
this.socket.setKeepAlive(true, 60000);
|
|
608
|
-
this.socket.once('lookup', () => {
|
|
609
|
-
this.timing.lookup = Date.now() - this.startTime;
|
|
610
|
-
});
|
|
486
|
+
this.socket.once('lookup', () => { this.timing.lookup = Date.now() - this.startTime; });
|
|
611
487
|
this.socket.connect(port, hostname, () => {
|
|
612
488
|
this.timing.connect = Date.now() - this.startTime;
|
|
613
489
|
this.proceedWithTLS(resolve, reject, timeout);
|
|
@@ -618,37 +494,8 @@ class TLSSocketManager extends events_1.EventEmitter {
|
|
|
618
494
|
});
|
|
619
495
|
}
|
|
620
496
|
proceedWithTLS(resolve, reject, timeout) {
|
|
621
|
-
const validCiphers = [
|
|
622
|
-
|
|
623
|
-
'TLS_CHACHA20_POLY1305_SHA256',
|
|
624
|
-
'TLS_AES_128_GCM_SHA256',
|
|
625
|
-
'ECDHE-ECDSA-AES256-GCM-SHA384',
|
|
626
|
-
'ECDHE-RSA-AES256-GCM-SHA384',
|
|
627
|
-
'ECDHE-ECDSA-CHACHA20-POLY1305',
|
|
628
|
-
'ECDHE-RSA-CHACHA20-POLY1305',
|
|
629
|
-
'ECDHE-ECDSA-AES128-GCM-SHA256',
|
|
630
|
-
'ECDHE-RSA-AES128-GCM-SHA256',
|
|
631
|
-
'ECDHE-ECDSA-AES256-SHA',
|
|
632
|
-
'ECDHE-RSA-AES256-SHA',
|
|
633
|
-
'ECDHE-ECDSA-AES128-SHA',
|
|
634
|
-
'ECDHE-RSA-AES128-SHA',
|
|
635
|
-
'AES256-GCM-SHA384',
|
|
636
|
-
'AES128-GCM-SHA256',
|
|
637
|
-
'AES256-SHA',
|
|
638
|
-
'AES128-SHA'
|
|
639
|
-
].join(':');
|
|
640
|
-
const tlsOptions = {
|
|
641
|
-
socket: this.socket,
|
|
642
|
-
servername: this.hostname,
|
|
643
|
-
ALPNProtocols: this.profile.tls.alpnProtocols,
|
|
644
|
-
ciphers: validCiphers,
|
|
645
|
-
minVersion: 'TLSv1.2',
|
|
646
|
-
maxVersion: 'TLSv1.3',
|
|
647
|
-
rejectUnauthorized: false,
|
|
648
|
-
requestCert: false,
|
|
649
|
-
honorCipherOrder: true,
|
|
650
|
-
sessionTimeout: 300
|
|
651
|
-
};
|
|
497
|
+
const validCiphers = ['TLS_AES_256_GCM_SHA384', 'TLS_CHACHA20_POLY1305_SHA256', 'TLS_AES_128_GCM_SHA256', 'ECDHE-ECDSA-AES256-GCM-SHA384', 'ECDHE-RSA-AES256-GCM-SHA384', 'ECDHE-ECDSA-CHACHA20-POLY1305', 'ECDHE-RSA-CHACHA20-POLY1305', 'ECDHE-ECDSA-AES128-GCM-SHA256', 'ECDHE-RSA-AES128-GCM-SHA256', 'ECDHE-ECDSA-AES256-SHA', 'ECDHE-RSA-AES256-SHA', 'ECDHE-ECDSA-AES128-SHA', 'ECDHE-RSA-AES128-SHA', 'AES256-GCM-SHA384', 'AES128-GCM-SHA256', 'AES256-SHA', 'AES128-SHA'].join(':');
|
|
498
|
+
const tlsOptions = { socket: this.socket, servername: this.hostname, ALPNProtocols: this.profile.tls.alpnProtocols, ciphers: validCiphers, minVersion: 'TLSv1.2', maxVersion: 'TLSv1.3', rejectUnauthorized: false, requestCert: false, honorCipherOrder: true, sessionTimeout: 300 };
|
|
652
499
|
this.tlsSocket = tls.connect(tlsOptions);
|
|
653
500
|
this.tlsSocket.once('secureConnect', () => {
|
|
654
501
|
clearTimeout(timeout);
|
|
@@ -656,24 +503,11 @@ class TLSSocketManager extends events_1.EventEmitter {
|
|
|
656
503
|
this.timing.socket = this.startTime;
|
|
657
504
|
resolve(this.tlsSocket);
|
|
658
505
|
});
|
|
659
|
-
this.tlsSocket.on('error', (err) => {
|
|
660
|
-
clearTimeout(timeout);
|
|
661
|
-
this.destroy();
|
|
662
|
-
reject(err);
|
|
663
|
-
});
|
|
664
|
-
}
|
|
665
|
-
getTiming() {
|
|
666
|
-
return { ...this.timing, total: Date.now() - this.startTime };
|
|
667
|
-
}
|
|
668
|
-
destroy() {
|
|
669
|
-
this.tlsSocket?.destroy();
|
|
670
|
-
this.socket?.destroy();
|
|
671
|
-
this.tlsSocket = null;
|
|
672
|
-
this.socket = null;
|
|
673
|
-
}
|
|
674
|
-
getSocket() {
|
|
675
|
-
return this.tlsSocket;
|
|
506
|
+
this.tlsSocket.on('error', (err) => { clearTimeout(timeout); this.destroy(); reject(err); });
|
|
676
507
|
}
|
|
508
|
+
getTiming() { return { ...this.timing, total: Date.now() - this.startTime }; }
|
|
509
|
+
destroy() { this.tlsSocket?.destroy(); this.socket?.destroy(); this.tlsSocket = null; this.socket = null; }
|
|
510
|
+
getSocket() { return this.tlsSocket; }
|
|
677
511
|
}
|
|
678
512
|
class AdvancedTLSClient {
|
|
679
513
|
constructor(profileName) {
|
|
@@ -720,11 +554,7 @@ class AdvancedTLSClient {
|
|
|
720
554
|
}, 60000);
|
|
721
555
|
}
|
|
722
556
|
setup(uri, options) {
|
|
723
|
-
const config = {
|
|
724
|
-
headers: {},
|
|
725
|
-
proxy: options.proxy || this.defaults.proxy,
|
|
726
|
-
decompress: options.decompress !== undefined ? options.decompress : true
|
|
727
|
-
};
|
|
557
|
+
const config = { headers: {}, proxy: options.proxy || this.defaults.proxy, decompress: options.decompress !== undefined ? options.decompress : true };
|
|
728
558
|
config.headers['user-agent'] = this.profile.userAgent;
|
|
729
559
|
config.headers['accept-language'] = this.profile.acceptLanguage || 'en-US,en;q=0.9';
|
|
730
560
|
config.headers['accept-encoding'] = 'gzip, deflate, br';
|
|
@@ -739,9 +569,8 @@ class AdvancedTLSClient {
|
|
|
739
569
|
config.headers['upgrade-insecure-requests'] = this.profile.upgradeInsecureRequests;
|
|
740
570
|
if (this.profile.priority)
|
|
741
571
|
config.headers['priority'] = this.profile.priority;
|
|
742
|
-
if (options.headers)
|
|
572
|
+
if (options.headers)
|
|
743
573
|
Object.assign(config.headers, options.headers);
|
|
744
|
-
}
|
|
745
574
|
return config;
|
|
746
575
|
}
|
|
747
576
|
isRequestOptions(obj) {
|
|
@@ -753,174 +582,238 @@ class AdvancedTLSClient {
|
|
|
753
582
|
const keys = Object.keys(obj);
|
|
754
583
|
return keys.some(key => optionKeys.includes(key));
|
|
755
584
|
}
|
|
585
|
+
shouldFollowRedirect(status, options) {
|
|
586
|
+
if (options.followRedirects === false)
|
|
587
|
+
return false;
|
|
588
|
+
return [301, 302, 303, 307, 308].includes(status);
|
|
589
|
+
}
|
|
590
|
+
getRedirectLocation(headers, baseUrl) {
|
|
591
|
+
const location = headers['location'];
|
|
592
|
+
if (!location)
|
|
593
|
+
return null;
|
|
594
|
+
const loc = Array.isArray(location) ? location[0] : location;
|
|
595
|
+
if (!loc)
|
|
596
|
+
return null;
|
|
597
|
+
try {
|
|
598
|
+
return new url.URL(loc, baseUrl).href;
|
|
599
|
+
}
|
|
600
|
+
catch {
|
|
601
|
+
return null;
|
|
602
|
+
}
|
|
603
|
+
}
|
|
604
|
+
adjustHeadersForRedirect(headers, status) {
|
|
605
|
+
const newHeaders = { ...headers };
|
|
606
|
+
if (status === 303 || status === 301 || status === 302) {
|
|
607
|
+
delete newHeaders['content-length'];
|
|
608
|
+
delete newHeaders['Content-Length'];
|
|
609
|
+
}
|
|
610
|
+
if (status === 303) {
|
|
611
|
+
delete newHeaders['content-type'];
|
|
612
|
+
delete newHeaders['Content-Type'];
|
|
613
|
+
}
|
|
614
|
+
return newHeaders;
|
|
615
|
+
}
|
|
756
616
|
async request(uri, dataOrOptions, optionsOrCallback, callback) {
|
|
757
617
|
let data = null;
|
|
758
618
|
let options = {};
|
|
759
619
|
let cb = undefined;
|
|
760
|
-
if (typeof dataOrOptions === 'function')
|
|
620
|
+
if (typeof dataOrOptions === 'function')
|
|
761
621
|
cb = dataOrOptions;
|
|
762
|
-
}
|
|
763
622
|
else if (dataOrOptions !== undefined && dataOrOptions !== null) {
|
|
764
|
-
if (this.isRequestOptions(dataOrOptions))
|
|
623
|
+
if (this.isRequestOptions(dataOrOptions))
|
|
765
624
|
options = dataOrOptions;
|
|
766
|
-
|
|
767
|
-
else {
|
|
625
|
+
else
|
|
768
626
|
data = dataOrOptions;
|
|
769
|
-
}
|
|
770
627
|
}
|
|
771
|
-
if (typeof optionsOrCallback === 'function')
|
|
628
|
+
if (typeof optionsOrCallback === 'function')
|
|
772
629
|
cb = optionsOrCallback;
|
|
773
|
-
|
|
774
|
-
else if (optionsOrCallback !== undefined && optionsOrCallback !== null) {
|
|
630
|
+
else if (optionsOrCallback !== undefined && optionsOrCallback !== null)
|
|
775
631
|
options = optionsOrCallback;
|
|
776
|
-
|
|
777
|
-
if (callback !== undefined) {
|
|
632
|
+
if (callback !== undefined)
|
|
778
633
|
cb = callback;
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
const
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
const
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
let post_data = null;
|
|
808
|
-
let json = options.json !== undefined ? options.json : false;
|
|
809
|
-
if (json === undefined && config.headers['content-type']?.includes('application/json')) {
|
|
810
|
-
json = true;
|
|
811
|
-
}
|
|
812
|
-
if (data) {
|
|
813
|
-
if (options.multipart) {
|
|
814
|
-
const boundary = options.boundary || this.defaults.boundary;
|
|
815
|
-
post_data = await this.buildMultipart(data, boundary);
|
|
816
|
-
config.headers['content-type'] = `multipart/form-data; boundary=${boundary}`;
|
|
817
|
-
}
|
|
818
|
-
else if (data instanceof stream_1.Readable) {
|
|
819
|
-
post_data = data;
|
|
820
|
-
}
|
|
821
|
-
else if (Buffer.isBuffer(data)) {
|
|
822
|
-
post_data = data;
|
|
823
|
-
}
|
|
824
|
-
else if (typeof data === 'string') {
|
|
825
|
-
post_data = data;
|
|
634
|
+
const maxRedirects = options.maxRedirects ?? options.follow_max ?? 20;
|
|
635
|
+
let redirectCount = 0;
|
|
636
|
+
let currentUri = uri;
|
|
637
|
+
let currentMethod = options.method?.toUpperCase() || 'GET';
|
|
638
|
+
let currentBody = data;
|
|
639
|
+
let currentHeaders = options.headers || {};
|
|
640
|
+
const execute = async (targetUri) => {
|
|
641
|
+
const parsed = new url.URL(targetUri);
|
|
642
|
+
const hostname = parsed.hostname;
|
|
643
|
+
const port = parsed.port ? +parsed.port : 443;
|
|
644
|
+
const path = parsed.pathname + parsed.search;
|
|
645
|
+
const cacheKey = `${hostname}:${port}`;
|
|
646
|
+
const startTime = Date.now();
|
|
647
|
+
const out = new stream_1.PassThrough();
|
|
648
|
+
let session = this.sessionCache.get(cacheKey);
|
|
649
|
+
if (!session || session.tlsManager.getSocket()?.destroyed) {
|
|
650
|
+
const tlsManager = new TLSSocketManager(this.profile, options.proxy);
|
|
651
|
+
const tlsSocket = await tlsManager.connect(hostname, port);
|
|
652
|
+
const clientManager = new UnifiedClientManager(tlsSocket, this.profile, hostname, this.cookieJar);
|
|
653
|
+
await clientManager.initialize();
|
|
654
|
+
session = { tlsManager, clientManager, lastUsed: Date.now() };
|
|
655
|
+
this.sessionCache.set(cacheKey, session);
|
|
656
|
+
if (this.sessionCache.size > this.maxCachedSessions) {
|
|
657
|
+
const oldest = [...this.sessionCache.entries()].sort((a, b) => a[1].lastUsed - b[1].lastUsed)[0];
|
|
658
|
+
oldest[1].clientManager.destroy();
|
|
659
|
+
oldest[1].tlsManager.destroy();
|
|
660
|
+
this.sessionCache.delete(oldest[0]);
|
|
661
|
+
}
|
|
826
662
|
}
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
663
|
+
session.lastUsed = Date.now();
|
|
664
|
+
const config = this.setup(targetUri, options);
|
|
665
|
+
config.headers = { ...config.headers, ...currentHeaders };
|
|
666
|
+
let post_data = null;
|
|
667
|
+
let json = options.json !== undefined ? options.json : false;
|
|
668
|
+
if (currentBody) {
|
|
669
|
+
if (options.multipart) {
|
|
670
|
+
const boundary = options.boundary || this.defaults.boundary;
|
|
671
|
+
post_data = await this.buildMultipart(currentBody, boundary);
|
|
672
|
+
config.headers['content-type'] = `multipart/form-data; boundary=${boundary}`;
|
|
673
|
+
}
|
|
674
|
+
else if (currentBody instanceof stream_1.Readable) {
|
|
675
|
+
post_data = currentBody;
|
|
676
|
+
}
|
|
677
|
+
else if (Buffer.isBuffer(currentBody)) {
|
|
678
|
+
post_data = currentBody;
|
|
679
|
+
}
|
|
680
|
+
else if (typeof currentBody === 'string') {
|
|
681
|
+
post_data = currentBody;
|
|
682
|
+
}
|
|
683
|
+
else if (json) {
|
|
684
|
+
post_data = JSON.stringify(currentBody);
|
|
685
|
+
config.headers['content-type'] = 'application/json; charset=utf-8';
|
|
686
|
+
}
|
|
687
|
+
else {
|
|
688
|
+
post_data = new URLSearchParams(currentBody).toString();
|
|
689
|
+
config.headers['content-type'] = 'application/x-www-form-urlencoded';
|
|
690
|
+
}
|
|
691
|
+
if (post_data && (typeof post_data === 'string' || Buffer.isBuffer(post_data))) {
|
|
692
|
+
config.headers['content-length'] = Buffer.byteLength(post_data).toString();
|
|
693
|
+
}
|
|
830
694
|
}
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
config.headers['
|
|
695
|
+
const cookies = this.cookieJar.getCookies(hostname, parsed.pathname, parsed.protocol === 'https:');
|
|
696
|
+
if (cookies)
|
|
697
|
+
config.headers['cookie'] = cookies;
|
|
698
|
+
if (options.stream === true) {
|
|
699
|
+
session.clientManager.request(path, options, startTime, config.headers, post_data, out);
|
|
700
|
+
return out;
|
|
834
701
|
}
|
|
835
|
-
if (
|
|
836
|
-
config.headers
|
|
702
|
+
if (cb) {
|
|
703
|
+
session.clientManager.request(path, options, startTime, config.headers, post_data, out, cb);
|
|
704
|
+
return out;
|
|
837
705
|
}
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
const
|
|
867
|
-
if (
|
|
706
|
+
return new Promise((resolve, reject) => {
|
|
707
|
+
let statusCode = 0;
|
|
708
|
+
let headers = {};
|
|
709
|
+
const chunks = [];
|
|
710
|
+
let done = false;
|
|
711
|
+
const complete = async (err) => {
|
|
712
|
+
if (done)
|
|
713
|
+
return;
|
|
714
|
+
done = true;
|
|
715
|
+
if (err)
|
|
716
|
+
return reject(err);
|
|
717
|
+
let body = Buffer.concat(chunks);
|
|
718
|
+
if (options.decompress !== false && headers) {
|
|
719
|
+
const encoding = headers['content-encoding']?.toLowerCase();
|
|
720
|
+
if (encoding) {
|
|
721
|
+
try {
|
|
722
|
+
if (encoding.includes('gzip'))
|
|
723
|
+
body = await gunzipAsync(body);
|
|
724
|
+
else if (encoding.includes('br'))
|
|
725
|
+
body = await brotliDecompressAsync(body);
|
|
726
|
+
else if (encoding.includes('deflate'))
|
|
727
|
+
body = await inflateAsync(body);
|
|
728
|
+
}
|
|
729
|
+
catch { }
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
const text = body.toString('utf-8');
|
|
733
|
+
let parsedJson = undefined;
|
|
734
|
+
const ct = headers['content-type']?.toLowerCase();
|
|
735
|
+
if (ct?.includes('application/json')) {
|
|
868
736
|
try {
|
|
869
|
-
|
|
870
|
-
body = await gunzipAsync(body);
|
|
871
|
-
else if (encoding.includes('br'))
|
|
872
|
-
body = await brotliDecompressAsync(body);
|
|
873
|
-
else if (encoding.includes('deflate'))
|
|
874
|
-
body = await inflateAsync(body);
|
|
737
|
+
parsedJson = JSON.parse(text);
|
|
875
738
|
}
|
|
876
739
|
catch { }
|
|
877
740
|
}
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
741
|
+
const responseObj = {
|
|
742
|
+
statusCode,
|
|
743
|
+
headers,
|
|
744
|
+
body,
|
|
745
|
+
text,
|
|
746
|
+
json: parsedJson,
|
|
747
|
+
fingerprints: {
|
|
748
|
+
ja3: this.generateJA3(),
|
|
749
|
+
ja3Hash: crypto.createHash('md5').update(this.generateJA3()).digest('hex'),
|
|
750
|
+
akamai: `1:${this.profile.http2.headerTableSize};2:${this.profile.http2.enablePush};4:${this.profile.http2.initialWindowSize};6:${this.profile.http2.maxHeaderListSize ?? ''}|00|0|m,a,s,p`
|
|
751
|
+
},
|
|
752
|
+
timing: {
|
|
753
|
+
socket: startTime,
|
|
754
|
+
lookup: 0,
|
|
755
|
+
connect: 0,
|
|
756
|
+
secureConnect: 0,
|
|
757
|
+
response: Date.now() - startTime,
|
|
758
|
+
end: Date.now() - startTime,
|
|
759
|
+
total: Date.now() - startTime
|
|
760
|
+
},
|
|
761
|
+
redirectCount
|
|
762
|
+
};
|
|
763
|
+
resolve(responseObj);
|
|
764
|
+
};
|
|
765
|
+
out.on('header', (status, hdrs) => {
|
|
766
|
+
statusCode = status;
|
|
767
|
+
headers = hdrs;
|
|
768
|
+
if (this.shouldFollowRedirect(status, options)) {
|
|
769
|
+
const newLocation = this.getRedirectLocation(hdrs, targetUri);
|
|
770
|
+
if (newLocation) {
|
|
771
|
+
redirectCount++;
|
|
772
|
+
if (redirectCount > maxRedirects)
|
|
773
|
+
return complete(new Error(`Too many redirects (${maxRedirects} max)`));
|
|
774
|
+
currentUri = newLocation;
|
|
775
|
+
currentHeaders = this.adjustHeadersForRedirect(config.headers, status);
|
|
776
|
+
if (status === 303) {
|
|
777
|
+
currentMethod = 'GET';
|
|
778
|
+
currentBody = null;
|
|
779
|
+
}
|
|
780
|
+
else if (status === 301 || status === 302) {
|
|
781
|
+
if (currentMethod === 'POST') {
|
|
782
|
+
currentMethod = 'GET';
|
|
783
|
+
currentBody = null;
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
out.removeAllListeners();
|
|
787
|
+
execute(currentUri).then(resolve).catch(reject);
|
|
788
|
+
return;
|
|
789
|
+
}
|
|
909
790
|
}
|
|
791
|
+
out.on('data', (chunk) => { if (!done)
|
|
792
|
+
chunks.push(chunk); });
|
|
793
|
+
out.once('end', () => complete());
|
|
794
|
+
out.once('error', complete);
|
|
910
795
|
});
|
|
911
|
-
|
|
912
|
-
out.on('header', (status, hdrs) => {
|
|
913
|
-
statusCode = status;
|
|
914
|
-
headers = hdrs;
|
|
915
|
-
});
|
|
916
|
-
out.on('data', (chunk) => {
|
|
917
|
-
if (!done)
|
|
918
|
-
chunks.push(chunk);
|
|
796
|
+
session.clientManager.request(path, options, startTime, config.headers, post_data, out).catch(complete);
|
|
919
797
|
});
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
798
|
+
};
|
|
799
|
+
try {
|
|
800
|
+
const result = await execute(currentUri);
|
|
801
|
+
if (cb) {
|
|
802
|
+
if (result instanceof stream_1.PassThrough) {
|
|
803
|
+
result.on('error', err => cb(err));
|
|
804
|
+
result.on('end', () => cb(null, result));
|
|
805
|
+
}
|
|
806
|
+
else {
|
|
807
|
+
cb(null, result);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
return result;
|
|
811
|
+
}
|
|
812
|
+
catch (err) {
|
|
813
|
+
if (cb)
|
|
814
|
+
cb(err);
|
|
815
|
+
throw err;
|
|
816
|
+
}
|
|
924
817
|
}
|
|
925
818
|
generateJA3() {
|
|
926
819
|
const version = '771';
|
|
@@ -938,10 +831,8 @@ class AdvancedTLSClient {
|
|
|
938
831
|
if (count === 0)
|
|
939
832
|
return reject(new Error('Empty multipart body'));
|
|
940
833
|
let doneCount = count;
|
|
941
|
-
const done = () => {
|
|
942
|
-
|
|
943
|
-
resolve(Buffer.from(body + '--' + boundary + '--\r\n'));
|
|
944
|
-
};
|
|
834
|
+
const done = () => { if (--doneCount === 0)
|
|
835
|
+
resolve(Buffer.from(body + '--' + boundary + '--\r\n')); };
|
|
945
836
|
for (const key in object) {
|
|
946
837
|
const value = object[key];
|
|
947
838
|
if (value === null || typeof value === 'undefined') {
|