curl-wrap 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +210 -0
  3. package/index.js +1217 -0
  4. package/package.json +28 -0
package/index.js ADDED
@@ -0,0 +1,1217 @@
1
+ const {spawn, spawnSync} = require('child_process');
2
+ const fs = require('fs/promises');
3
+ const {CookieJar} = require('tough-cookie');
4
+
5
+ function getDefaultProxyPort(proxyType) {
6
+ if (proxyType === 'http') return 80;
7
+ if (proxyType === 'https') return 443;
8
+ return 1080;
9
+ }
10
+
11
+ /**
12
+ * Returns proxy url based on the proxy options.
13
+ *
14
+ * @param {object} proxy proxy options
15
+ * @return {string} proxy url based on the options
16
+ * @private
17
+ */
18
+ // eslint-disable-next-line complexity
19
+ function makeProxyUrl(proxy, options) {
20
+ if (!proxy) return '';
21
+ let address = typeof proxy === 'string' ? proxy : (proxy.host || proxy.address || proxy.url);
22
+ if (!address) return '';
23
+ if (!address.includes('://')) {
24
+ let type = proxy.type || options.type || 'http';
25
+ if (type === 'socks') type = 'socks5';
26
+ address = `${type}://${address}`;
27
+ }
28
+
29
+ const auth = proxy.auth || options.auth || {};
30
+ const uri = new URL(proxy);
31
+ if (!uri.port) {
32
+ uri.port = proxy.port || options.port || getDefaultProxyPort(uri.protocol.replace(':', ''));
33
+ }
34
+ if (!uri.username && options.username) {
35
+ uri.username = proxy.username || options.username || auth.username || '';
36
+ }
37
+ if (!uri.password && options.password) {
38
+ uri.password = proxy.password || options.password || auth.password || '';
39
+ }
40
+ return uri.toString();
41
+ }
42
+
43
+ class CurlResponse {
44
+ constructor() {
45
+ this.url = '';
46
+ this.body = '';
47
+ this.headers = {};
48
+ this.statusCode = 0;
49
+ this.ip = '';
50
+ this.errorMsg = '';
51
+ }
52
+
53
+ get status() {
54
+ return this.statusCode;
55
+ }
56
+
57
+ setCurlJson(json, options = {}) {
58
+ if (!json) return;
59
+ const data = json.json || {};
60
+ this.statusCode = data.response_code || 0;
61
+ this.ip = data.remote_ip || '';
62
+ this.url = data.url_effective || data.url || '';
63
+ this.errorMsg = data.errormsg || '';
64
+
65
+ const headers = json.headers || {};
66
+ for (const [key, value] of Object.entries(headers)) {
67
+ this.headers[key.toLowerCase()] = value[0];
68
+ }
69
+ if (options.cookieJar) {
70
+ const setCookies = headers['set-cookie'];
71
+ if (setCookies) {
72
+ setCookies.forEach((cookie) => {
73
+ options.cookieJar.setCookie(cookie, this.url);
74
+ });
75
+ }
76
+ }
77
+ }
78
+ }
79
+
80
+ class Curl {
81
+ /**
82
+ * Creates a new Curl object.
83
+ * @constructor
84
+ */
85
+ constructor() {
86
+ /**
87
+ * Various options (or parameters) defining the Connection
88
+ * @private
89
+ */
90
+ this.options = {
91
+ // url to send request at
92
+ url: null,
93
+ // method of the request (GET / POST / OPTIONS / DELETE / PATCH / PUT)
94
+ method: 'GET',
95
+ // headers to set
96
+ headers: {
97
+ accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
98
+ // 'accept-encoding': 'gzip, deflate, br, zstd',
99
+ 'accept-language': 'en-US,en-IN;q=0.9,en;q=0.8',
100
+ // set empty cookie header if no cookies exist
101
+ // this is because some sites expect cookie header to be there
102
+ cookie: '',
103
+ // 'sec-ch-ua': '"Google Chrome";v="129", "Not=A?Brand";v="8", "Chromium";v="129"',
104
+ // 'sec-ch-ua-arch': '"x86"',
105
+ // 'sec-ch-ua-bitness': '"64"',
106
+ // 'sec-ch-ua-full-version': '"129.0.6668.58"',
107
+ // 'sec-ch-ua-full-version-list': '"Google Chrome";v="129.0.6668.58", "Not=A?Brand";v="8.0.0.0", "Chromium";v="129.0.6668.58"',
108
+ // 'sec-ch-ua-mobile': '?0',
109
+ // 'sec-ch-ua-model': '""',
110
+ // 'sec-ch-ua-platform': '"Linux"',
111
+ // 'sec-ch-ua-platform-version': '"6.2.0"',
112
+ // 'sec-fetch-dest': 'empty',
113
+ // 'sec-fetch-mode': 'navigate',
114
+ // 'sec-fetch-site': 'same-origin',
115
+ // 'upgrade-insecure-requests': 1,
116
+ 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36',
117
+ },
118
+ // whether to follow redirects
119
+ followRedirect: true,
120
+ // maximum number of redirects to follow
121
+ maxRedirects: 6,
122
+ // whether to ask for compressed response (automatically handles accept-encoding)
123
+ compress: true,
124
+ // timeout of the request
125
+ timeout: 120 * 1000,
126
+ // whether to verify ssl certificate
127
+ ignoreSSLError: true,
128
+ // proxy details (should be {address, port, type, auth: {username, password}})
129
+ // type can be http, https, socks
130
+ proxy: {},
131
+ // body of the request (valid in case of POST / PUT / PATCH / DELETE)
132
+ body: '',
133
+ };
134
+ }
135
+
136
+ get _cookies() {
137
+ let cookies = this.options.cookies;
138
+ if (!cookies) {
139
+ cookies = {};
140
+ this.options.cookies = cookies;
141
+ }
142
+ return cookies;
143
+ }
144
+
145
+ get _fields() {
146
+ // post fields
147
+ let fields = this.options.fields;
148
+ if (!fields) {
149
+ fields = {};
150
+ this.options.fields = fields;
151
+ }
152
+ return fields;
153
+ }
154
+
155
+ get _query() {
156
+ // query params
157
+ let query = this.options.query;
158
+ if (!query) {
159
+ query = {};
160
+ this.options.query = query;
161
+ }
162
+ return query;
163
+ }
164
+
165
+ /**
166
+ * Set the url for the connection.
167
+ *
168
+ * @param {string} url
169
+ * @return {Curl} self
170
+ */
171
+ url(url) {
172
+ this.options.url = url;
173
+ return this;
174
+ }
175
+
176
+ /**
177
+ * @static
178
+ * Creates and returns a new Curl object with the given url.
179
+ *
180
+ * @param {string} url
181
+ * @return {Curl} A new Curl object with url set to the given url
182
+ */
183
+ static url(url) {
184
+ const curl = new this();
185
+ curl.url(url);
186
+ return curl;
187
+ }
188
+
189
+ /**
190
+ * @static
191
+ * Creates and returns a new Curl object (with get method) with the given url.
192
+ *
193
+ * @param {string} url
194
+ * @return {Curl} A new Curl object with url set to the given url
195
+ */
196
+ static get(url) {
197
+ const curl = new this();
198
+ curl.url(url);
199
+ curl.get();
200
+ return curl;
201
+ }
202
+
203
+ /**
204
+ * @static
205
+ * Creates and returns a new Curl object (with post method) with the given url.
206
+ *
207
+ * @param {string} url
208
+ * @return {Curl} A new Curl object with url set to the given url
209
+ */
210
+ static post(url) {
211
+ const curl = new this();
212
+ curl.url(url);
213
+ curl.post();
214
+ return curl;
215
+ }
216
+
217
+ /**
218
+ * @static
219
+ * Creates and returns a new Curl object (with put method) with the given url.
220
+ *
221
+ * @param {string} url
222
+ * @return {Curl} A new Curl object with url set to the given url
223
+ */
224
+ static put(url) {
225
+ const curl = new this();
226
+ curl.url(url);
227
+ curl.put();
228
+ return curl;
229
+ }
230
+
231
+ /**
232
+ * @static
233
+ * Returns a new cookie jar.
234
+ * @param {Array<any>} args
235
+ * @return {CookieJar} A cookie jar
236
+ */
237
+ static getNewCookieJar(...args) {
238
+ return new CookieJar(...args);
239
+ }
240
+
241
+ /**
242
+ * @static
243
+ * Returns the global cookie jar.
244
+ * @returns {CookieJar} global cookie jar
245
+ */
246
+ static getGlobalCookieJar() {
247
+ if (!this._globalCookieJar) {
248
+ this._globalCookieJar = this.getNewCookieJar();
249
+ }
250
+ return this._globalCookieJar;
251
+ }
252
+
253
+ /**
254
+ * @static
255
+ * whether curl-impersonate-chrome is available or not
256
+ * @see https://github.com/lwthiker/curl-impersonate
257
+ */
258
+ static hasCurlImpersonateChrome() {
259
+ if (this._hasCurlImpersonateChrome === undefined) {
260
+ this._hasCurlImpersonateChrome = spawnSync('curl-impersonate-chrome', ['--version']).status === 0;
261
+ }
262
+ return this._hasCurlImpersonateChrome;
263
+ }
264
+
265
+ /**
266
+ * @static
267
+ * whether curl-impersonate-ff is available or not
268
+ * @see https://github.com/lwthiker/curl-impersonate
269
+ */
270
+ static hasCurlImpersonateFirefox() {
271
+ if (this._hasCurlImpersonateFirefox === undefined) {
272
+ this._hasCurlImpersonateFirefox = spawnSync('curl-impersonate-ff', ['--version']).status === 0;
273
+ }
274
+ return this._hasCurlImpersonateFirefox;
275
+ }
276
+
277
+ /**
278
+ * set cli command (default: curl)
279
+ *
280
+ * @param {string} command name of curl binary
281
+ * @return {Curl} self
282
+ */
283
+ cliCommand(command) {
284
+ this.options.cliCommand = command;
285
+ return this;
286
+ }
287
+
288
+ /**
289
+ * add curl cli options
290
+ *
291
+ * @param {string|Array<string>} options curl cli options
292
+ * @return {Curl} self
293
+ */
294
+ cliOptions(options) {
295
+ const cliOptions = this.options.cliOptions || (this.options.cliOptions = []);
296
+ if (typeof options === 'string') {
297
+ cliOptions.push(options);
298
+ }
299
+ else {
300
+ cliOptions.push(...options);
301
+ }
302
+ return this;
303
+ }
304
+
305
+ /**
306
+ * impersonate a browser
307
+ *
308
+ * @param {string} browser browser to impersonate (chrome / chromeMobile / edge / safari / firefox)
309
+ * @return {Curl} self
310
+ */
311
+ impersonate(browser = 'chrome') {
312
+ if (browser === 'chrome') {
313
+ if (this.constructor.hasCurlImpersonateChrome()) {
314
+ this.cliCommand('curl-impersonate-chrome');
315
+ this.cliOptions([
316
+ '--ciphers',
317
+ 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA',
318
+ '--http2',
319
+ '--http2-no-server-push',
320
+ '--compressed',
321
+ '--tlsv1.2',
322
+ '--alps',
323
+ '--tls-permute-extensions',
324
+ '--cert-compression',
325
+ 'brotli',
326
+ ]);
327
+ }
328
+ this.headers({
329
+ 'sec-ch-ua': '"Chromium";v="116", "Not)A;Brand";v="24", "Google Chrome";v="116"',
330
+ 'sec-ch-ua-mobile': '?0',
331
+ 'sec-ch-ua-platform': '"Windows"',
332
+ 'Upgrade-Insecure-Requests': '1',
333
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/116.0.0.0 Safari/537.36',
334
+ Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
335
+ 'Sec-Fetch-Site': 'none',
336
+ 'Sec-Fetch-Mode': 'navigate',
337
+ 'Sec-Fetch-User': '?1',
338
+ 'Sec-Fetch-Dest': 'document',
339
+ 'Accept-Encoding': 'gzip, deflate, br',
340
+ 'Accept-Language': 'en-US,en;q=0.9',
341
+ });
342
+ }
343
+ else if (browser === 'chromeMobile') {
344
+ if (this.constructor.hasCurlImpersonateChrome()) {
345
+ this.cliCommand('curl-impersonate-chrome');
346
+ this.cliOptions([
347
+ '--ciphers',
348
+ 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA',
349
+ '--http2',
350
+ '--compressed',
351
+ '--tlsv1.2',
352
+ '--alps',
353
+ '--cert-compression',
354
+ 'brotli',
355
+ ]);
356
+ }
357
+ this.headers({
358
+ 'sec-ch-ua': ' Not A;Brand";v="99", "Chromium";v="99", "Google Chrome";v="99"',
359
+ 'sec-ch-ua-mobile': '?1',
360
+ 'sec-ch-ua-platform': '"Android"',
361
+ 'Upgrade-Insecure-Requests': '1',
362
+ 'User-Agent': 'Mozilla/5.0 (Linux; Android 12; Pixel 6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/99.0.4844.58 Mobile Safari/537.36',
363
+ Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
364
+ 'Sec-Fetch-Site': 'none',
365
+ 'Sec-Fetch-Mode': 'navigate',
366
+ 'Sec-Fetch-User': '?1',
367
+ 'Sec-Fetch-Dest': 'document',
368
+ 'Accept-Encoding': 'gzip, deflate, br',
369
+ 'Accept-Language': 'en-US,en;q=0.9',
370
+ });
371
+ }
372
+ else if (browser === 'edge') {
373
+ if (this.constructor.hasCurlImpersonateChrome()) {
374
+ this.cliCommand('curl-impersonate-chrome');
375
+ this.cliOptions([
376
+ '--ciphers',
377
+ 'TLS_AES_128_GCM_SHA256,TLS_AES_256_GCM_SHA384,TLS_CHACHA20_POLY1305_SHA256,ECDHE-ECDSA-AES128-GCM-SHA256,ECDHE-RSA-AES128-GCM-SHA256,ECDHE-ECDSA-AES256-GCM-SHA384,ECDHE-RSA-AES256-GCM-SHA384,ECDHE-ECDSA-CHACHA20-POLY1305,ECDHE-RSA-CHACHA20-POLY1305,ECDHE-RSA-AES128-SHA,ECDHE-RSA-AES256-SHA,AES128-GCM-SHA256,AES256-GCM-SHA384,AES128-SHA,AES256-SHA',
378
+ '--http2',
379
+ '--compressed',
380
+ '--tlsv1.2',
381
+ '--alps',
382
+ '--cert-compression',
383
+ 'brotli',
384
+ ]);
385
+ }
386
+ this.headers({
387
+ 'sec-ch-ua': ' Not A;Brand";v="99", "Chromium";v="101", "Microsoft Edge";v="101"',
388
+ 'sec-ch-ua-mobile': '?0',
389
+ 'sec-ch-ua-platform': '"Windows"',
390
+ 'Upgrade-Insecure-Requests': '1',
391
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/101.0.4951.64 Safari/537.36 Edg/101.0.1210.47',
392
+ Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
393
+ 'Sec-Fetch-Site': 'none',
394
+ 'Sec-Fetch-Mode': 'navigate',
395
+ 'Sec-Fetch-User': '?1',
396
+ 'Sec-Fetch-Dest': 'document',
397
+ 'Accept-Encoding': 'gzip, deflate, br',
398
+ 'Accept-Language': 'en-US,en;q=0.9',
399
+ });
400
+ }
401
+ else if (browser === 'safari') {
402
+ if (this.constructor.hasCurlImpersonateChrome()) {
403
+ this.cliCommand('curl-impersonate-chrome');
404
+ this.cliOptions([
405
+ '--ciphers',
406
+ 'TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA:TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA:TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA:TLS_RSA_WITH_AES_256_GCM_SHA384:TLS_RSA_WITH_AES_128_GCM_SHA256:TLS_RSA_WITH_AES_256_CBC_SHA:TLS_RSA_WITH_AES_128_CBC_SHA:TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA:TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA:TLS_RSA_WITH_3DES_EDE_CBC_SHA',
407
+ '--curves',
408
+ 'X25519:P-256:P-384:P-521',
409
+ '--signature-hashes',
410
+ 'ecdsa_secp256r1_sha256,rsa_pss_rsae_sha256,rsa_pkcs1_sha256,ecdsa_secp384r1_sha384,ecdsa_sha1,rsa_pss_rsae_sha384,rsa_pss_rsae_sha384,rsa_pkcs1_sha384,rsa_pss_rsae_sha512,rsa_pkcs1_sha512,rsa_pkcs1_sha1',
411
+ '--http2',
412
+ '--compressed',
413
+ '--tlsv1.0',
414
+ '--no-tls-session-ticket',
415
+ '--cert-compression',
416
+ 'zlib',
417
+ '--http2-pseudo-headers-order',
418
+ 'mspa',
419
+ ]);
420
+ }
421
+ this.headers({
422
+ 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15',
423
+ Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
424
+ 'Accept-Language': 'en-GB,en-US;q=0.9,en;q=0.8',
425
+ 'Accept-Encoding': 'gzip, deflate, br',
426
+ });
427
+ }
428
+ else if (browser === 'firefox') {
429
+ if (this.constructor.hasCurlImpersonateFirefox()) {
430
+ this.cliCommand('curl-impersonate-ff');
431
+ this.cliOptions([
432
+ '--ciphers',
433
+ 'aes_128_gcm_sha_256,chacha20_poly1305_sha_256,aes_256_gcm_sha_384,ecdhe_ecdsa_aes_128_gcm_sha_256,ecdhe_rsa_aes_128_gcm_sha_256,ecdhe_ecdsa_chacha20_poly1305_sha_256,ecdhe_rsa_chacha20_poly1305_sha_256,ecdhe_ecdsa_aes_256_gcm_sha_384,ecdhe_rsa_aes_256_gcm_sha_384,ecdhe_ecdsa_aes_256_sha,ecdhe_ecdsa_aes_128_sha,ecdhe_rsa_aes_128_sha,ecdhe_rsa_aes_256_sha,rsa_aes_128_gcm_sha_256,rsa_aes_256_gcm_sha_384,rsa_aes_128_sha,rsa_aes_256_sha',
434
+ '--http2',
435
+ '--compressed',
436
+ ]);
437
+ }
438
+ this.headers({
439
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/117.0',
440
+ Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
441
+ 'Accept-Language': 'en-US,en;q=0.5',
442
+ 'Accept-Encoding': 'gzip, deflate, br',
443
+ 'Upgrade-Insecure-Requests': '1',
444
+ 'Sec-Fetch-Dest': 'document',
445
+ 'Sec-Fetch-Mode': 'navigate',
446
+ 'Sec-Fetch-Site': 'none',
447
+ 'Sec-Fetch-User': '?1',
448
+ TE: 'Trailers',
449
+ });
450
+ }
451
+ return this;
452
+ }
453
+
454
+ /**
455
+ * Set or unset the followRedirect option for the connection.
456
+ *
457
+ * @param {boolean} shouldFollowRedirect boolean representing whether to follow redirect or not
458
+ * @return {Curl} self
459
+ */
460
+ followRedirect(shouldFollowRedirect = true) {
461
+ this.options.followRedirect = shouldFollowRedirect;
462
+ return this;
463
+ }
464
+
465
+ /**
466
+ * Set the number of maximum redirects to follow
467
+ * @param {number} numRedirects max number of redirects
468
+ */
469
+ maxRedirects(numRedirects) {
470
+ this.options.maxRedirects = numRedirects;
471
+ return this;
472
+ }
473
+
474
+ /**
475
+ * Set value of a header parameter for the connection.
476
+ *
477
+ * @param {string|object} headerName name of the header parameter whose value is to be set
478
+ * @param {string|undefined} headerValue value to be set
479
+ * @return {Curl} self
480
+ */
481
+ header(headerName, headerValue) {
482
+ if (typeof headerName === 'string') {
483
+ this.options.headers[headerName.toLowerCase()] = headerValue;
484
+ }
485
+ else {
486
+ Object.assign(
487
+ this.options.headers,
488
+ Object.fromEntries(
489
+ Object.entries(headerName).map(([key, val]) => [key.toLowerCase(), val])
490
+ ),
491
+ );
492
+ }
493
+
494
+ return this;
495
+ }
496
+
497
+ /**
498
+ * Set value of the headers for the connection.
499
+ *
500
+ * @param {object} headers object representing the headers for the connection
501
+ * @return {Curl} self
502
+ */
503
+ headers(headers) {
504
+ this.header(headers);
505
+ return this;
506
+ }
507
+
508
+ /**
509
+ * Set the content type as json
510
+ * Optionally also set the body of the request.
511
+ *
512
+ * @param {object|undefined} body value for body
513
+ * @return {Curl} self
514
+ */
515
+ json(body) {
516
+ this.header('content-type', 'application/json');
517
+ if (body !== undefined) {
518
+ this.options.body = body;
519
+ }
520
+ return this;
521
+ }
522
+
523
+ /**
524
+ * Set the body of the connection object.
525
+ *
526
+ * @param {any} body value for body
527
+ * if body is an object, contentType will be set to application/json and body will be stringified
528
+ * @param {string} [contentType=null] string representing the content type of the body
529
+ * contentType can be null or json
530
+ * @return {Curl} self
531
+ */
532
+ body(body, contentType = null) {
533
+ if (contentType) {
534
+ this.contentType = contentType;
535
+ }
536
+
537
+ this.options.body = body;
538
+ return this;
539
+ }
540
+
541
+ /**
542
+ * Set the 'Referer' field in the headers.
543
+ *
544
+ * @param {string} referer referer value
545
+ * @return {Curl}
546
+ */
547
+ referer(referer) {
548
+ this.header('referer', referer);
549
+ return this;
550
+ }
551
+
552
+ /**
553
+ * Set the 'Referer' field in the headers.
554
+ *
555
+ * @param {string} referer referer value
556
+ * @return {Curl}
557
+ */
558
+ referrer(referrer) {
559
+ this.header('referer', referrer);
560
+ return this;
561
+ }
562
+
563
+ /**
564
+ * Set the 'User-Agent' field in the headers.
565
+ *
566
+ * @param {string} userAgent name of the user-agent or its value
567
+ * @return {Curl} self
568
+ */
569
+ userAgent(userAgent) {
570
+ this.header('user-agent', userAgent);
571
+ return this;
572
+ }
573
+
574
+ /**
575
+ * Set the 'Content-Type' field in the headers.
576
+ *
577
+ * @param {string} contentType value for content-type
578
+ * @return {Curl}
579
+ */
580
+ contentType(contentType) {
581
+ if (contentType === 'json') {
582
+ this.header('content-type', 'application/json');
583
+ }
584
+ else if (contentType === 'form') {
585
+ this.header('content-type', 'application/x-www-form-urlencoded');
586
+ }
587
+ else {
588
+ this.header('content-type', contentType);
589
+ }
590
+
591
+ return this;
592
+ }
593
+
594
+ /**
595
+ * Returns whether the content-type is JSON or not
596
+ *
597
+ * @return {boolean} true, if content-type is JSON; false, otherwise
598
+ */
599
+ isJSON() {
600
+ const contentType = this.options.headers['content-type'] || '';
601
+ return contentType.startsWith('application/json');
602
+ }
603
+
604
+ /**
605
+ * Returns whether the content-type is Form or not
606
+ *
607
+ * @return {boolean} true, if content-type is JSON; false, otherwise
608
+ */
609
+ isForm() {
610
+ return this.options.headers['content-type'] === 'application/x-www-form-urlencoded';
611
+ }
612
+
613
+ /**
614
+ * Sets the value of a cookie.
615
+ * Can be used to enable global cookies, if cookieName is set to true
616
+ * and cookieValue is undefined (or is not passed as an argument).
617
+ * Can also be used to set multiple cookies by passing in an object
618
+ * representing the cookies and their values as key:value pairs.
619
+ *
620
+ * @param {string|boolean|object} cookieName represents the name of the
621
+ * cookie to be set, or the cookies object
622
+ * @param {string|undefined} [cookieValue] cookie value to be set
623
+ * @return {Curl} self
624
+ */
625
+ cookie(cookieName, cookieValue) {
626
+ if (cookieValue === undefined) {
627
+ this.globalCookies(cookieName);
628
+ }
629
+ else if (typeof cookieName === 'string') {
630
+ if (cookieValue === null || cookieValue === false) {
631
+ delete this._cookies[cookieName];
632
+ }
633
+ else {
634
+ this._cookies[cookieName] = cookieValue;
635
+ }
636
+ }
637
+ else if (cookieName && typeof cookieName === 'object') {
638
+ Object.assign(this._cookies, cookieName);
639
+ }
640
+
641
+ return this;
642
+ }
643
+
644
+ /**
645
+ * Sets multiple cookies.
646
+ * Can be used to enable global cookies, if cookies is set to true.
647
+ *
648
+ * @param {object|boolean} cookies object representing the cookies
649
+ * and their values as key:value pairs.
650
+ * @return {Curl} self
651
+ */
652
+ cookies(cookies) {
653
+ if (cookies === true || cookies === false || cookies == null) {
654
+ this.globalCookies(cookies);
655
+ }
656
+ else if (cookies && typeof cookies === 'object') {
657
+ Object.assign(this._cookies, cookies);
658
+ }
659
+
660
+ return this;
661
+ }
662
+
663
+ /**
664
+ * Enable global cookies.
665
+ *
666
+ * @param {boolean|object} [options=true]
667
+ * @return {Curl} self
668
+ */
669
+ globalCookies(options = true) {
670
+ if (options === false || options === null) {
671
+ delete this.options.cookieJar;
672
+ delete this.options.readCookieJar;
673
+ return this;
674
+ }
675
+
676
+ const jar = this.constructor.getGlobalCookieJar();
677
+ this.cookieJar(jar, options);
678
+ return this;
679
+ }
680
+
681
+ /**
682
+ * Set the value of cookie jar.
683
+ *
684
+ * @param {CookieJar} cookieJar value to be set
685
+ * @return {Curl} self
686
+ */
687
+ cookieJar(cookieJar, options = {}) {
688
+ delete this._cookieFileFn;
689
+ delete this._cookieFileFnRes;
690
+
691
+ if (options.readOnly) {
692
+ this.options.readCookieJar = cookieJar;
693
+ }
694
+ else {
695
+ this.options.cookieJar = cookieJar;
696
+ }
697
+
698
+ return this;
699
+ }
700
+
701
+ /**
702
+ * Set the value of cookie jar based on a file (cookie store).
703
+ *
704
+ * @param {string} fileName name of (or path to) the file
705
+ * @return {Curl} self
706
+ */
707
+ cookieFile(fileName, options = {}) {
708
+ const setCookieJar = (cookieJar) => {
709
+ if (options.readOnly) {
710
+ this.options.readCookieJar = cookieJar;
711
+ }
712
+ else {
713
+ this.options.cookieJar = cookieJar;
714
+ }
715
+ };
716
+
717
+ this._cookieFileFn = async () => {
718
+ try {
719
+ const contents = await fs.readFile(fileName, {encoding: 'utf8'});
720
+ const obj = JSON.parse(contents);
721
+ const cookieJar = CookieJar.fromJSON(obj);
722
+ setCookieJar(cookieJar);
723
+ }
724
+ catch (e) {
725
+ setCookieJar(this.constructor.getNewCookieJar());
726
+ }
727
+ };
728
+
729
+ if (!options.readOnly) {
730
+ this._cookieFileFnRes = async () => {
731
+ const cookieJar = this.options.cookieJar;
732
+ if (!cookieJar) return;
733
+ try {
734
+ const contents = JSON.stringify(cookieJar.toJSON());
735
+ await fs.writeFile(fileName, contents, {encoding: 'utf8'});
736
+ }
737
+ catch (e) {
738
+ // ignore errors
739
+ }
740
+ };
741
+ }
742
+
743
+ return this;
744
+ }
745
+
746
+ /**
747
+ * Set request timeout.
748
+ *
749
+ * @param {number} timeout timeout value in seconds
750
+ * @return {Curl} self
751
+ */
752
+ timeout(timeout) {
753
+ this.options.timeout = timeout * 1000;
754
+ return this;
755
+ }
756
+
757
+ /**
758
+ * Set request timeout.
759
+ *
760
+ * @param {number} timeoutInMs timeout value in milliseconds
761
+ * @return {Curl} self
762
+ */
763
+ timeoutMs(timeoutInMs) {
764
+ this.options.timeout = timeoutInMs;
765
+ return this;
766
+ }
767
+
768
+ /**
769
+ * Set value of a field in the options.
770
+ * Can also be used to set multiple fields by passing in an object
771
+ * representing the field-names and their values as key:value pairs.
772
+ *
773
+ * @param {string|object} fieldName name of the field to be set, or the fields object
774
+ * @param {string|undefined} [fieldValue] value to be set
775
+ * @return {Curl} self
776
+ */
777
+ field(fieldName, fieldValue) {
778
+ if (typeof fieldName === 'string') {
779
+ this._fields[fieldName] = fieldValue;
780
+ }
781
+ else if (fieldName && typeof fieldName === 'object') {
782
+ Object.assign(this._fields, fieldName);
783
+ }
784
+
785
+ return this;
786
+ }
787
+
788
+ /**
789
+ * Set multiple fields.
790
+ *
791
+ * @param {object} fields object representing the field-names and their
792
+ * values as key:value pairs
793
+ * @return {Curl} self
794
+ */
795
+ fields(fields) {
796
+ if (fields && typeof fields === 'object') {
797
+ Object.assign(this._fields, fields);
798
+ }
799
+ return this;
800
+ }
801
+
802
+ /**
803
+ * Set value of a query parameter
804
+ * Can also be used to set multiple query params by passing in an object
805
+ * representing the param-names and their values as key:value pairs.
806
+ *
807
+ * @param {string|object} fieldName name of the field to be set, or the fields object
808
+ * @param {string|undefined} [fieldValue] value to be set
809
+ * @return {Curl} self
810
+ */
811
+ query(name, value) {
812
+ if (typeof name === 'string') {
813
+ this._query[name] = value;
814
+ }
815
+ else if (name && typeof name === 'object') {
816
+ Object.assign(this._query, name);
817
+ }
818
+
819
+ return this;
820
+ }
821
+
822
+ /**
823
+ * set whether to ask for compressed response (handles decompression automatically)
824
+ * @param {boolean} [askForCompression=true] whether to ask for compressed response
825
+ * @return {Curl} self
826
+ */
827
+ compress(askForCompression = true) {
828
+ this.options.compress = askForCompression;
829
+ return this;
830
+ }
831
+
832
+ /**
833
+ * Set the request method for the connection.
834
+ *
835
+ * @param {string} method one of the HTTP request methods ('GET', 'PUT', 'POST', etc.)
836
+ * @return {Curl} self
837
+ */
838
+ method(method) {
839
+ this.options.method = (method || 'GET').toUpperCase();
840
+ return this;
841
+ }
842
+
843
+ /**
844
+ * @typedef {object} auth
845
+ * @property {string} username
846
+ * @property {string} password
847
+ */
848
+
849
+ /**
850
+ * Set username and password for authentication.
851
+ *
852
+ * @param {string | auth} username
853
+ * @param {string|undefined} password
854
+ * @return {Curl} self
855
+ */
856
+ httpAuth(username, password) {
857
+ let auth = '';
858
+ if (typeof username === 'string') {
859
+ if (password === undefined) {
860
+ // username is of the format username:password
861
+ auth = username;
862
+ }
863
+ else {
864
+ // username & password are strings
865
+ auth = `${username}:${password}`;
866
+ }
867
+ }
868
+ else if (username.username) {
869
+ // username argument is an object of {username, password}
870
+ auth = `${username.username}:${username.password}`;
871
+ }
872
+
873
+ this.header('authorization', 'Basic ' + Buffer.from(auth).toString('base64'));
874
+ return this;
875
+ }
876
+
877
+ /**
878
+ * Set bearer token for authorization
879
+ * @param {string} token
880
+ * @return {Curl} self
881
+ */
882
+ bearerToken(token) {
883
+ this.header('authorization', `Bearer ${token}`);
884
+ return this;
885
+ }
886
+
887
+ /**
888
+ * Set api token using x-api-token header
889
+ * @param {string} token
890
+ * @return {Curl} self
891
+ */
892
+ apiToken(token) {
893
+ this.header('x-api-token', token);
894
+ return this;
895
+ }
896
+
897
+ /**
898
+ * Set proxy address (or options).
899
+ * Proxy type can be http, https, or socks5.
900
+ *
901
+ * @param {string|object} proxy proxy address, or object representing proxy options
902
+ * @param {object} [options={}] options for proxy ({username, password, type})
903
+ * @return {Curl} self
904
+ */
905
+ proxy(proxy, options = {}) {
906
+ if (proxy === false || proxy === null) {
907
+ delete this.options.proxy;
908
+ }
909
+ this.options.proxy = makeProxyUrl(proxy, options);
910
+ return this;
911
+ }
912
+
913
+ /**
914
+ * Set keepalive connection option
915
+ *
916
+ * @param {boolean} [isKeepAlive=true] whether to keepalive or not
917
+ * @returns {Curl} self
918
+ */
919
+ keepalive(isKeepAlive = true) {
920
+ this.options.keepalive = isKeepAlive;
921
+ return this;
922
+ }
923
+
924
+ /**
925
+ * Set request method to 'GET'.
926
+ *
927
+ * @return {Curl} self
928
+ */
929
+ get() {
930
+ this.method('GET');
931
+ return this;
932
+ }
933
+
934
+ /**
935
+ * Set request method to 'POST'.
936
+ *
937
+ * @return {Curl} self
938
+ */
939
+ post() {
940
+ this.method('POST');
941
+ return this;
942
+ }
943
+
944
+ /**
945
+ * Set request method to 'PUT'.
946
+ *
947
+ * @return {Curl} self
948
+ */
949
+ put() {
950
+ this.method('PUT');
951
+ return this;
952
+ }
953
+
954
+ /**
955
+ * Use curl --verbose option to get verbose output.
956
+ *
957
+ * @return {Curl} self
958
+ */
959
+ verbose(isVerbose = true) {
960
+ this.options.verbose = isVerbose;
961
+ return this;
962
+ }
963
+
964
+ /**
965
+ * Add the options 'fields' to the options body, form or qs
966
+ * on the basis of the request method.
967
+ *
968
+ * NOTE: This function is for internal use
969
+ * @private
970
+ */
971
+ getUrlAndBody() {
972
+ const options = this.options;
973
+ let url = options.url;
974
+ let query = options.query;
975
+ let body = options.body;
976
+ const fields = options.fields;
977
+ const hasBody = ['POST', 'PUT', 'PATCH', 'DELETE'].includes(options.method);
978
+
979
+ if (!body) {
980
+ if (fields) {
981
+ body = fields;
982
+ if (hasBody && !this.options.headers['content-type']) {
983
+ this.contentType('form');
984
+ }
985
+ }
986
+ }
987
+ else if (typeof body === 'object') {
988
+ if (fields) {
989
+ Object.assign(body, fields);
990
+ }
991
+ if (hasBody && !options.headers['content-type']) {
992
+ this.contentType('json');
993
+ }
994
+ }
995
+
996
+ if (hasBody) {
997
+ if (this.isJSON()) {
998
+ if (typeof body === 'object') {
999
+ body = JSON.stringify(body);
1000
+ }
1001
+ }
1002
+ else if (this.isForm()) {
1003
+ if (typeof body === 'object') {
1004
+ body = (new URLSearchParams(body)).toString();
1005
+ }
1006
+ }
1007
+ }
1008
+ else if (body && (typeof body === 'object')) {
1009
+ if (query) {
1010
+ Object.assign(body, query);
1011
+ }
1012
+ query = body;
1013
+ body = '';
1014
+ }
1015
+
1016
+ if (query) {
1017
+ const qs = (new URLSearchParams(query)).toString();
1018
+ const joiner = url.includes('?') ? '&' : '?';
1019
+ url += (joiner + qs);
1020
+ }
1021
+
1022
+ return {
1023
+ url,
1024
+ body,
1025
+ };
1026
+ }
1027
+
1028
+ async getCookieHeader() {
1029
+ const cookies = [];
1030
+ const options = this.options;
1031
+
1032
+ const cookieMap = options.cookies;
1033
+ if (cookieMap) {
1034
+ for (const [key, value] of Object.entries(cookieMap)) {
1035
+ cookies.push(`${key}=${value}`);
1036
+ }
1037
+ }
1038
+
1039
+ if (this._cookieFileFn) {
1040
+ await this._cookieFileFn();
1041
+ }
1042
+
1043
+ const cookieJar = options.readCookieJar || options.cookieJar;
1044
+ if (cookieJar) {
1045
+ const jarCookies = await cookieJar.getCookies(options.url);
1046
+ if (jarCookies) {
1047
+ if (cookieMap) {
1048
+ jarCookies.forEach((cookie) => {
1049
+ if (!cookieMap[cookie.key]) {
1050
+ cookies.push(`${cookie.key}=${cookie.value}`);
1051
+ }
1052
+ });
1053
+ }
1054
+ else {
1055
+ jarCookies.forEach((cookie) => {
1056
+ cookies.push(`${cookie.key}=${cookie.value}`);
1057
+ });
1058
+ }
1059
+ }
1060
+ }
1061
+
1062
+ return cookies.join('; ');
1063
+ }
1064
+
1065
+ async getCurlArgs() {
1066
+ const {url, body} = this.getUrlAndBody();
1067
+
1068
+ const options = this.options;
1069
+ const args = [
1070
+ url,
1071
+ '--request',
1072
+ options.method,
1073
+ options.keepalive ? '--keepalive' : '--no-keepalive',
1074
+ '--silent',
1075
+ '--write-out',
1076
+ '%{stderr}===<json>==={"json":%{json},"headers":%{header_json}}===</json>===',
1077
+ ];
1078
+
1079
+ if (body) {
1080
+ args.push('--data-raw', body);
1081
+ }
1082
+
1083
+ const cookieHeader = await this.getCookieHeader();
1084
+ if (cookieHeader) {
1085
+ this.header('cookie', cookieHeader);
1086
+ }
1087
+
1088
+ for (const [key, value] of Object.entries(options.headers)) {
1089
+ args.push('--header', `${key}: ${value}`);
1090
+ }
1091
+ if (options.compress) {
1092
+ args.push('--compressed');
1093
+ }
1094
+ if (options.proxy) {
1095
+ args.push('--proxy', options.proxy);
1096
+ }
1097
+ if (options.timeout) {
1098
+ args.push('--max-time', options.timeout / 1000);
1099
+ }
1100
+ if (options.followRedirect) {
1101
+ args.push('--location');
1102
+ }
1103
+ if (options.maxRedirects) {
1104
+ args.push('--max-redirs', options.maxRedirects);
1105
+ }
1106
+ if (options.ignoreSSLError) {
1107
+ args.push('--insecure');
1108
+ }
1109
+ if (options.verbose) {
1110
+ args.push('--verbose');
1111
+ }
1112
+
1113
+ const cliOptions = options.cliOptions;
1114
+ if (cliOptions) {
1115
+ args.push(...cliOptions);
1116
+ }
1117
+
1118
+ return args;
1119
+ }
1120
+
1121
+ async getPromise() {
1122
+ const cmd = this.options.cliCommand || 'curl';
1123
+ const args = await this.getCurlArgs();
1124
+ const curl = spawn(cmd, args);
1125
+ const cookieJar = this.options.cookieJar;
1126
+
1127
+ let stdout = '';
1128
+ let stderr = '';
1129
+ const response = new CurlResponse();
1130
+ return new Promise((resolve, reject) => {
1131
+ curl.stdout.on('data', (data) => {
1132
+ stdout += data;
1133
+ });
1134
+
1135
+ curl.stderr.on('data', (data) => {
1136
+ stderr += data;
1137
+ });
1138
+
1139
+ curl.on('error', (error) => {
1140
+ reject(error);
1141
+ });
1142
+
1143
+ curl.on('close', async (code) => {
1144
+ response.exitCode = code;
1145
+ response.url = this.options.url;
1146
+ response.body = stdout;
1147
+ stderr = stderr.replace(/===<json>===(.*)===<\/json>===/s, (match, p1) => {
1148
+ try {
1149
+ response.setCurlJson(JSON.parse(p1), {cookieJar});
1150
+ }
1151
+ catch (e) {
1152
+ // ignore error
1153
+ }
1154
+ return '';
1155
+ });
1156
+ response.stderr = stderr;
1157
+
1158
+ if (this._cookieFileFnRes) {
1159
+ try {
1160
+ await this._cookieFileFnRes();
1161
+ }
1162
+ catch (e) {
1163
+ // ignore errors
1164
+ }
1165
+ }
1166
+
1167
+ if (code === 0) {
1168
+ resolve(response);
1169
+ }
1170
+ else {
1171
+ const error = new Error(response.errorMsg || stderr);
1172
+ error.response = response;
1173
+ reject(error);
1174
+ }
1175
+ });
1176
+ });
1177
+ }
1178
+
1179
+ /**
1180
+ * It is used for method chaining.
1181
+ *
1182
+ * @template T
1183
+ * @param {function(response):T} successCallback To be called if the Promise is fulfilled
1184
+ * @param {function(Error):T} [errorCallback] function to be called if the Promise is rejected
1185
+ * @return {Promise<T>} a Promise in pending state
1186
+ */
1187
+ then(successCallback, errorCallback) {
1188
+ return this.getPromise().then(successCallback, errorCallback);
1189
+ }
1190
+
1191
+ /**
1192
+ * It is also used for method chaining, but handles rejected cases only.
1193
+ *
1194
+ * @template T
1195
+ * @param {function(Error):T} errorCallback function to be called if the Promise is rejected
1196
+ * @return {Promise<T>} a Promise in pending state
1197
+ */
1198
+ catch(errorCallback) {
1199
+ return this.getPromise().catch(errorCallback);
1200
+ }
1201
+
1202
+ /**
1203
+ * finally method of promise returned
1204
+ *
1205
+ * @template T
1206
+ * @param {function():T} callback function to be called if the promise is fullfilled or rejected
1207
+ * @return {Promise<T>} a Promise in pending state
1208
+ */
1209
+ finally(callback) {
1210
+ return this.getPromise().finally(callback);
1211
+ }
1212
+ }
1213
+
1214
+ module.exports = {
1215
+ Curl,
1216
+ CurlResponse,
1217
+ };