@the-convocation/twitter-scraper 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.commitlintrc +5 -0
- package/.eslintrc.js +24 -0
- package/.gitattributes +78 -0
- package/.prettierignore +3 -0
- package/.prettierrc +5 -0
- package/.yarnrc.yml +1 -0
- package/README.md +97 -28
- package/dist/cycletls/cjs/index.cjs +48 -3
- package/dist/cycletls/cjs/index.cjs.map +1 -1
- package/dist/cycletls/esm/index.mjs +48 -3
- package/dist/cycletls/esm/index.mjs.map +1 -1
- package/dist/default/cjs/index.js +1320 -154
- package/dist/default/cjs/index.js.map +1 -1
- package/dist/default/esm/index.mjs +1320 -155
- package/dist/default/esm/index.mjs.map +1 -1
- package/dist/node/cjs/index.cjs +1320 -154
- package/dist/node/cjs/index.cjs.map +1 -1
- package/dist/node/esm/index.mjs +1320 -155
- package/dist/node/esm/index.mjs.map +1 -1
- package/dist/types/index.d.ts +70 -1
- package/examples/react-integration/.env.example +4 -0
- package/examples/react-integration/.eslintrc.cjs +18 -0
- package/package.json +4 -2
package/.commitlintrc
ADDED
package/.eslintrc.js
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
module.exports = {
|
|
2
|
+
parser: '@typescript-eslint/parser',
|
|
3
|
+
parserOptions: {
|
|
4
|
+
tsconfigRootDir: __dirname,
|
|
5
|
+
sourceType: 'module',
|
|
6
|
+
},
|
|
7
|
+
plugins: ['@typescript-eslint/eslint-plugin'],
|
|
8
|
+
extends: [
|
|
9
|
+
'plugin:@typescript-eslint/recommended',
|
|
10
|
+
'plugin:prettier/recommended',
|
|
11
|
+
],
|
|
12
|
+
root: true,
|
|
13
|
+
env: {
|
|
14
|
+
node: true,
|
|
15
|
+
jest: true,
|
|
16
|
+
},
|
|
17
|
+
ignorePatterns: ['**/*.js'],
|
|
18
|
+
rules: {
|
|
19
|
+
'@typescript-eslint/interface-name-prefix': 'off',
|
|
20
|
+
'@typescript-eslint/explicit-function-return-type': 'off',
|
|
21
|
+
'@typescript-eslint/explicit-module-boundary-types': 'off',
|
|
22
|
+
'@typescript-eslint/no-explicit-any': 'off',
|
|
23
|
+
},
|
|
24
|
+
};
|
package/.gitattributes
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# Auto detect text files and perform LF normalization
|
|
2
|
+
* text=auto
|
|
3
|
+
|
|
4
|
+
#
|
|
5
|
+
# The above will handle all files NOT found below
|
|
6
|
+
#
|
|
7
|
+
|
|
8
|
+
# Code
|
|
9
|
+
*.js text eol=lf
|
|
10
|
+
*.ts text eol=lf
|
|
11
|
+
|
|
12
|
+
# Documents
|
|
13
|
+
*.bibtex text diff=bibtex
|
|
14
|
+
*.doc diff=astextplain
|
|
15
|
+
*.DOC diff=astextplain
|
|
16
|
+
*.docx diff=astextplain
|
|
17
|
+
*.DOCX diff=astextplain
|
|
18
|
+
*.dot diff=astextplain
|
|
19
|
+
*.DOT diff=astextplain
|
|
20
|
+
*.pdf diff=astextplain
|
|
21
|
+
*.PDF diff=astextplain
|
|
22
|
+
*.rtf diff=astextplain
|
|
23
|
+
*.RTF diff=astextplain
|
|
24
|
+
*.md text diff=markdown
|
|
25
|
+
*.mdx text diff=markdown
|
|
26
|
+
*.tex text diff=tex
|
|
27
|
+
*.adoc text
|
|
28
|
+
*.textile text
|
|
29
|
+
*.mustache text
|
|
30
|
+
*.csv text
|
|
31
|
+
*.tab text
|
|
32
|
+
*.tsv text
|
|
33
|
+
*.txt text
|
|
34
|
+
*.sql text
|
|
35
|
+
|
|
36
|
+
# Graphics
|
|
37
|
+
*.png binary
|
|
38
|
+
*.jpg binary
|
|
39
|
+
*.jpeg binary
|
|
40
|
+
*.gif binary
|
|
41
|
+
*.tif binary
|
|
42
|
+
*.tiff binary
|
|
43
|
+
*.ico binary
|
|
44
|
+
*.svg text
|
|
45
|
+
*.eps binary
|
|
46
|
+
|
|
47
|
+
# Scripts
|
|
48
|
+
*.bash text eol=lf
|
|
49
|
+
*.fish text eol=lf
|
|
50
|
+
*.sh text eol=lf
|
|
51
|
+
*.zsh text eol=lf
|
|
52
|
+
*.bat text eol=crlf
|
|
53
|
+
*.cmd text eol=crlf
|
|
54
|
+
*.ps1 text eol=crlf
|
|
55
|
+
|
|
56
|
+
# Serialisation
|
|
57
|
+
*.json text
|
|
58
|
+
*.toml text
|
|
59
|
+
*.xml text
|
|
60
|
+
*.yaml text
|
|
61
|
+
*.yml text
|
|
62
|
+
|
|
63
|
+
# Archives
|
|
64
|
+
*.7z binary
|
|
65
|
+
*.gz binary
|
|
66
|
+
*.tar binary
|
|
67
|
+
*.tgz binary
|
|
68
|
+
*.zip binary
|
|
69
|
+
|
|
70
|
+
# Text files where line endings should be preserved
|
|
71
|
+
*.patch -text
|
|
72
|
+
|
|
73
|
+
#
|
|
74
|
+
# Exclude files from exporting
|
|
75
|
+
#
|
|
76
|
+
|
|
77
|
+
.gitattributes export-ignore
|
|
78
|
+
.gitignore export-ignore
|
package/.prettierignore
ADDED
package/.prettierrc
ADDED
package/.yarnrc.yml
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
nodeLinker: node-modules
|
package/README.md
CHANGED
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
[](https://the-convocation.github.io/twitter-scraper/)
|
|
4
4
|
|
|
5
|
-
A port of [n0madic/twitter-scraper](https://github.com/n0madic/twitter-scraper)
|
|
6
|
-
to Node.js.
|
|
5
|
+
A port of the now-archived [n0madic/twitter-scraper](https://github.com/n0madic/twitter-scraper) to Node.js.
|
|
7
6
|
|
|
8
7
|
> Twitter's API is annoying to work with, and has lots of limitations — luckily
|
|
9
8
|
> their frontend (JavaScript) has it's own API, which I reverse-engineered. No
|
|
@@ -11,12 +10,20 @@ to Node.js.
|
|
|
11
10
|
>
|
|
12
11
|
> You can use this library to get the text of any user's Tweets trivially.
|
|
13
12
|
|
|
14
|
-
|
|
13
|
+
Many things have changed since X (the company formerly known as Twitter) was acquired in 2022:
|
|
15
14
|
|
|
16
|
-
-
|
|
17
|
-
`scraper.login()`.
|
|
15
|
+
- Several operations require logging in with a real user account via
|
|
16
|
+
`scraper.login()`. **While we are not aware of confirmed cases caused
|
|
17
|
+
by this library, any account you log into with this library is subject
|
|
18
|
+
to being banned at any time. You have been warned.**
|
|
18
19
|
- Twitter's frontend API does in fact have rate limits
|
|
19
|
-
([#11](https://github.com/the-convocation/twitter-scraper/issues/11))
|
|
20
|
+
([#11](https://github.com/the-convocation/twitter-scraper/issues/11)).
|
|
21
|
+
The rate limits are dynamic and sometimes change, so we don't know
|
|
22
|
+
exactly what they are at all times. Refer to [rate limiting](#rate-limiting)
|
|
23
|
+
for more information.
|
|
24
|
+
- Twitter's authentication requirements and frontend API endpoints
|
|
25
|
+
change frequently, breaking this library. Fixes for these issues
|
|
26
|
+
typically take at least a few days to go out.
|
|
20
27
|
|
|
21
28
|
## Installation
|
|
22
29
|
|
|
@@ -62,15 +69,15 @@ const scraper = new Scraper({
|
|
|
62
69
|
// The arguments here are the same as the parameters to fetch(), and
|
|
63
70
|
// are kept as-is for flexibility of both the library and applications.
|
|
64
71
|
if (input instanceof URL) {
|
|
65
|
-
const proxy =
|
|
66
|
-
encodeURIComponent(input.toString());
|
|
72
|
+
const proxy =
|
|
73
|
+
'https://corsproxy.io/?' + encodeURIComponent(input.toString());
|
|
67
74
|
return [proxy, init];
|
|
68
|
-
} else if (typeof input ===
|
|
69
|
-
const proxy =
|
|
75
|
+
} else if (typeof input === 'string') {
|
|
76
|
+
const proxy = 'https://corsproxy.io/?' + encodeURIComponent(input);
|
|
70
77
|
return [proxy, init];
|
|
71
78
|
} else {
|
|
72
79
|
// Omitting handling for example
|
|
73
|
-
throw new Error(
|
|
80
|
+
throw new Error('Unexpected request input type');
|
|
74
81
|
}
|
|
75
82
|
},
|
|
76
83
|
},
|
|
@@ -87,10 +94,10 @@ front page).
|
|
|
87
94
|
#### Next.js 13.x example:
|
|
88
95
|
|
|
89
96
|
```tsx
|
|
90
|
-
|
|
97
|
+
'use client';
|
|
91
98
|
|
|
92
|
-
import { Scraper, Tweet } from
|
|
93
|
-
import { useEffect, useMemo, useState } from
|
|
99
|
+
import { Scraper, Tweet } from '@the-convocation/twitter-scraper';
|
|
100
|
+
import { useEffect, useMemo, useState } from 'react';
|
|
94
101
|
|
|
95
102
|
export default function Home() {
|
|
96
103
|
const scraper = useMemo(
|
|
@@ -99,15 +106,15 @@ export default function Home() {
|
|
|
99
106
|
transform: {
|
|
100
107
|
request(input: RequestInfo | URL, init?: RequestInit) {
|
|
101
108
|
if (input instanceof URL) {
|
|
102
|
-
const proxy =
|
|
103
|
-
encodeURIComponent(input.toString());
|
|
109
|
+
const proxy =
|
|
110
|
+
'https://corsproxy.io/?' + encodeURIComponent(input.toString());
|
|
104
111
|
return [proxy, init];
|
|
105
|
-
} else if (typeof input ===
|
|
106
|
-
const proxy =
|
|
107
|
-
encodeURIComponent(input);
|
|
112
|
+
} else if (typeof input === 'string') {
|
|
113
|
+
const proxy =
|
|
114
|
+
'https://corsproxy.io/?' + encodeURIComponent(input);
|
|
108
115
|
return [proxy, init];
|
|
109
116
|
} else {
|
|
110
|
-
throw new Error(
|
|
117
|
+
throw new Error('Unexpected request input type');
|
|
111
118
|
}
|
|
112
119
|
},
|
|
113
120
|
},
|
|
@@ -118,7 +125,7 @@ export default function Home() {
|
|
|
118
125
|
|
|
119
126
|
useEffect(() => {
|
|
120
127
|
async function getTweet() {
|
|
121
|
-
const latestTweet = await scraper.getLatestTweet(
|
|
128
|
+
const latestTweet = await scraper.getLatestTweet('twitter');
|
|
122
129
|
if (latestTweet) {
|
|
123
130
|
setTweet(latestTweet);
|
|
124
131
|
}
|
|
@@ -159,11 +166,10 @@ supported directly by interceptors):
|
|
|
159
166
|
const scraper = new Scraper({
|
|
160
167
|
fetch: (input, init) => {
|
|
161
168
|
// Transform input and init into your function's expected types...
|
|
162
|
-
return fetch(input, init)
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
});
|
|
169
|
+
return fetch(input, init).then((res) => {
|
|
170
|
+
// Transform res into a web-compliant response...
|
|
171
|
+
return res;
|
|
172
|
+
});
|
|
167
173
|
},
|
|
168
174
|
});
|
|
169
175
|
```
|
|
@@ -186,7 +192,10 @@ yarn add cycletls
|
|
|
186
192
|
|
|
187
193
|
```ts
|
|
188
194
|
import { Scraper } from '@the-convocation/twitter-scraper';
|
|
189
|
-
import {
|
|
195
|
+
import {
|
|
196
|
+
cycleTLSFetch,
|
|
197
|
+
cycleTLSExit,
|
|
198
|
+
} from '@the-convocation/twitter-scraper/cycletls';
|
|
190
199
|
|
|
191
200
|
const scraper = new Scraper({
|
|
192
201
|
fetch: cycleTLSFetch,
|
|
@@ -203,7 +212,66 @@ cycleTLSExit();
|
|
|
203
212
|
|
|
204
213
|
See the [cycletls example](./examples/cycletls/) for a complete working example.
|
|
205
214
|
|
|
215
|
+
### Cookie-based authentication
|
|
216
|
+
|
|
217
|
+
If you're encountering `error 399` ("Incorrect. Please try again") or Twitter's suspicious activity detection during login, you can use cookies exported from an already-authenticated browser session instead. This approach:
|
|
218
|
+
|
|
219
|
+
- Avoids Twitter's anti-bot protection that blocks automated logins
|
|
220
|
+
- No need to store or handle passwords in code
|
|
221
|
+
- Uses your established browser session
|
|
222
|
+
- Bypasses rate limiting on authentication endpoints
|
|
223
|
+
|
|
224
|
+
**Step 1: Export cookies from your browser**
|
|
225
|
+
|
|
226
|
+
Using Chrome/Edge:
|
|
227
|
+
|
|
228
|
+
1. Log in to X.com in your browser
|
|
229
|
+
2. Open DevTools (F12) → Application tab → Cookies
|
|
230
|
+
3. Click the URL bar that says "Filter cookies" and press Ctrl+A to select all cookies
|
|
231
|
+
4. Copy all cookies (they'll be in format: `name1=value1; name2=value2; ...`)
|
|
232
|
+
|
|
233
|
+
Using Firefox:
|
|
234
|
+
|
|
235
|
+
1. Log in to X.com in your browser
|
|
236
|
+
2. Open DevTools (F12) → Storage tab → Cookies → `https://x.com`
|
|
237
|
+
3. Find the `ct0` cookie and copy its value
|
|
238
|
+
4. Find the `auth_token` cookie and copy its value
|
|
239
|
+
5. Construct the cookie string: `ct0=<value>; auth_token=<value>`
|
|
240
|
+
|
|
241
|
+
> **Tip:** You can use the [Cookie-Editor](https://addons.mozilla.org/en-US/firefox/addon/cookie-editor/) extension to export cookies in a convenient format.
|
|
242
|
+
|
|
243
|
+
**Step 2: Use cookies in your code**
|
|
244
|
+
|
|
245
|
+
```ts
|
|
246
|
+
import { Cookie } from 'tough-cookie';
|
|
247
|
+
import { Scraper } from '@the-convocation/twitter-scraper';
|
|
248
|
+
|
|
249
|
+
// Your cookie string from browser (name=value; name2=value2; ...)
|
|
250
|
+
const cookieString = 'ct0=abc123; auth_token=xyz789; lang=en; ...';
|
|
251
|
+
|
|
252
|
+
// Parse the cookie string
|
|
253
|
+
const cookies = cookieString
|
|
254
|
+
.split(';')
|
|
255
|
+
.map((c) => Cookie.parse(c))
|
|
256
|
+
.filter(Boolean);
|
|
257
|
+
|
|
258
|
+
// Create scraper and set cookies
|
|
259
|
+
const scraper = new Scraper();
|
|
260
|
+
await scraper.setCookies(cookies);
|
|
261
|
+
|
|
262
|
+
// Verify authentication works
|
|
263
|
+
const isLoggedIn = await scraper.isLoggedIn();
|
|
264
|
+
if (isLoggedIn) {
|
|
265
|
+
console.log('✓ Successfully authenticated with cookies!');
|
|
266
|
+
// Now you can use authenticated features
|
|
267
|
+
const profile = await scraper.getProfile('username');
|
|
268
|
+
}
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Cookies expire over time. If authentication fails, you may need to export fresh cookies from your browser.
|
|
272
|
+
|
|
206
273
|
### Rate limiting
|
|
274
|
+
|
|
207
275
|
The Twitter API heavily rate-limits clients, requiring that the scraper has its own
|
|
208
276
|
rate-limit handling to behave predictably when rate-limiting occurs. By default, the
|
|
209
277
|
scraper uses a rate-limiting strategy that waits for the current rate-limiting period
|
|
@@ -216,7 +284,7 @@ scrapers logged-in to different accounts (refer to [#116](https://github.com/the
|
|
|
216
284
|
implementation to the `rateLimitStrategy` option in the scraper constructor:
|
|
217
285
|
|
|
218
286
|
```ts
|
|
219
|
-
import { Scraper, RateLimitStrategy } from
|
|
287
|
+
import { Scraper, RateLimitStrategy } from '@the-convocation/twitter-scraper';
|
|
220
288
|
|
|
221
289
|
class CustomRateLimitStrategy implements RateLimitStrategy {
|
|
222
290
|
async onRateLimit(event: RateLimitEvent): Promise<void> {
|
|
@@ -231,6 +299,7 @@ const scraper = new Scraper({
|
|
|
231
299
|
|
|
232
300
|
More information on this interface can be found on the [`RateLimitStrategy`](https://the-convocation.github.io/twitter-scraper/interfaces/RateLimitStrategy.html)
|
|
233
301
|
page in the documentation. The library provides two pre-written implementations to choose from:
|
|
302
|
+
|
|
234
303
|
- `WaitingRateLimitStrategy`: The default, which waits for the limit to expire.
|
|
235
304
|
- `ErrorRateLimitStrategy`: A strategy that throws if any rate-limit event occurs.
|
|
236
305
|
|
|
@@ -4,6 +4,47 @@ var initCycleTLS = require('cycletls');
|
|
|
4
4
|
var headersPolyfill = require('headers-polyfill');
|
|
5
5
|
var debug = require('debug');
|
|
6
6
|
|
|
7
|
+
const CHROME_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36";
|
|
8
|
+
const CHROME_JA3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-10-35-16-11-51-27-65037-43-45-18-23-5-65281-13-17613,4588-29-23-24,0";
|
|
9
|
+
const CHROME_JA4R = "t13d1516h2_002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0012,0017,001b,0023,002b,002d,0033,44cd,fe0d,ff01_0403,0804,0401,0503,0805,0501,0806,0601";
|
|
10
|
+
const CHROME_HTTP2_FINGERPRINT = "1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p";
|
|
11
|
+
const CHROME_HEADER_ORDER = [
|
|
12
|
+
// HTTP/2 pseudo-headers (Chrome 144 order: method, authority, scheme, path)
|
|
13
|
+
":method",
|
|
14
|
+
":authority",
|
|
15
|
+
":scheme",
|
|
16
|
+
":path",
|
|
17
|
+
// Chrome Client Hints (mandatory for modern detection bypass)
|
|
18
|
+
"sec-ch-ua",
|
|
19
|
+
"sec-ch-ua-mobile",
|
|
20
|
+
"sec-ch-ua-platform",
|
|
21
|
+
// Standard browser headers
|
|
22
|
+
"upgrade-insecure-requests",
|
|
23
|
+
"user-agent",
|
|
24
|
+
"accept",
|
|
25
|
+
"origin",
|
|
26
|
+
"sec-fetch-site",
|
|
27
|
+
"sec-fetch-mode",
|
|
28
|
+
"sec-fetch-user",
|
|
29
|
+
"sec-fetch-dest",
|
|
30
|
+
"referer",
|
|
31
|
+
"accept-encoding",
|
|
32
|
+
"accept-language",
|
|
33
|
+
"priority",
|
|
34
|
+
// Authentication headers
|
|
35
|
+
"authorization",
|
|
36
|
+
"x-csrf-token",
|
|
37
|
+
"x-guest-token",
|
|
38
|
+
"x-twitter-auth-type",
|
|
39
|
+
"x-twitter-active-user",
|
|
40
|
+
"x-twitter-client-language",
|
|
41
|
+
"x-client-transaction-id",
|
|
42
|
+
"x-xp-forwarded-for",
|
|
43
|
+
// POST-specific
|
|
44
|
+
"content-type",
|
|
45
|
+
"cookie"
|
|
46
|
+
];
|
|
47
|
+
|
|
7
48
|
const log = debug("twitter-scraper:cycletls");
|
|
8
49
|
let cycleTLSInstance = null;
|
|
9
50
|
async function initCycleTLSFetch() {
|
|
@@ -53,9 +94,13 @@ async function cycleTLSFetch(input, init) {
|
|
|
53
94
|
const options = {
|
|
54
95
|
body,
|
|
55
96
|
headers,
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
97
|
+
ja3: CHROME_JA3,
|
|
98
|
+
ja4r: CHROME_JA4R,
|
|
99
|
+
http2Fingerprint: CHROME_HTTP2_FINGERPRINT,
|
|
100
|
+
headerOrder: CHROME_HEADER_ORDER,
|
|
101
|
+
orderAsProvided: true,
|
|
102
|
+
disableGrease: false,
|
|
103
|
+
userAgent: headers["user-agent"] || CHROME_USER_AGENT
|
|
59
104
|
};
|
|
60
105
|
try {
|
|
61
106
|
const response = await instance(
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.cjs","sources":["../../../src/cycletls-fetch.ts"],"sourcesContent":["import initCycleTLS from 'cycletls';\nimport { Headers } from 'headers-polyfill';\nimport debug from 'debug';\n\nconst log = debug('twitter-scraper:cycletls');\n\nlet cycleTLSInstance: Awaited<ReturnType<typeof initCycleTLS>> | null = null;\n\n/**\n * Initialize the CycleTLS instance. This should be called once before using the fetch wrapper.\n */\nexport async function initCycleTLSFetch() {\n if (!cycleTLSInstance) {\n log('Initializing CycleTLS...');\n cycleTLSInstance = await initCycleTLS();\n log('CycleTLS initialized successfully');\n }\n return cycleTLSInstance;\n}\n\n/**\n * Cleanup the CycleTLS instance. Call this when you're done making requests.\n */\nexport function cycleTLSExit() {\n if (cycleTLSInstance) {\n log('Exiting CycleTLS...');\n cycleTLSInstance.exit();\n cycleTLSInstance = null;\n }\n}\n\n/**\n * A fetch-compatible wrapper around CycleTLS that mimics Chrome's TLS fingerprint\n * to bypass Cloudflare and other bot detection systems.\n */\nexport async function cycleTLSFetch(\n input: RequestInfo | URL,\n init?: RequestInit,\n): Promise<Response> {\n const instance = await initCycleTLSFetch();\n\n const url =\n typeof input === 'string'\n ? input\n : input instanceof URL\n ? input.toString()\n : input.url;\n const method = (init?.method || 'GET').toUpperCase();\n\n log(`Making ${method} request to ${url}`);\n\n // Extract headers from RequestInit\n const headers: Record<string, string> = {};\n if (init?.headers) {\n if (init.headers instanceof Headers) {\n init.headers.forEach((value, key) => {\n headers[key] = value;\n });\n } else if (Array.isArray(init.headers)) {\n init.headers.forEach(([key, value]) => {\n headers[key] = value;\n });\n } else {\n Object.assign(headers, init.headers);\n }\n }\n\n // Convert body to string if needed\n let body: string | undefined;\n if (init?.body) {\n if (typeof init.body === 'string') {\n body = init.body;\n } else if (init.body instanceof URLSearchParams) {\n body = init.body.toString();\n } else {\n body = init.body.toString();\n }\n }\n\n // Use Chrome 120 JA3 fingerprint for maximum compatibility\n const options = {\n body,\n headers,\n // Chrome 120 on Windows 10\n ja3: '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0',\n userAgent:\n headers['user-agent'] ||\n 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',\n };\n\n try {\n const response = await instance(\n url,\n options,\n method.toLowerCase() as\n | 'get'\n | 'post'\n | 'put'\n | 'delete'\n | 'patch'\n | 'head'\n | 'options',\n );\n\n // Convert CycleTLS response to fetch Response\n // CycleTLS returns headers as an object\n const responseHeaders = new Headers();\n if (response.headers) {\n Object.entries(response.headers).forEach(([key, value]) => {\n if (Array.isArray(value)) {\n value.forEach((v) => {\n responseHeaders.append(key, v);\n });\n } else if (typeof value === 'string') {\n responseHeaders.set(key, value);\n }\n });\n }\n\n // Get response body - cycletls provides helper methods, but we need the raw text\n // The response object has a text() method that returns the body as text\n let responseBody = '';\n if (typeof response.text === 'function') {\n responseBody = await response.text();\n } else if ((response as any).body) {\n responseBody = (response as any).body;\n }\n\n // Create a proper Response object using standard Response constructor\n const fetchResponse = new Response(responseBody, {\n status: response.status,\n statusText: '', // CycleTLS doesn't provide status text\n headers: responseHeaders,\n });\n\n return fetchResponse;\n } catch (error) {\n log(`CycleTLS request failed: ${error}`);\n throw error;\n }\n}\n"],"names":["Headers"],"mappings":";;;;;;AAIA,MAAM,GAAA,GAAM,MAAM,0BAA0B,CAAA,CAAA;AAE5C,IAAI,gBAAoE,GAAA,IAAA,CAAA;AAKxE,eAAsB,iBAAoB,GAAA;AACxC,EAAA,IAAI,CAAC,gBAAkB,EAAA;AACrB,IAAA,GAAA,CAAI,0BAA0B,CAAA,CAAA;AAC9B,IAAA,gBAAA,GAAmB,MAAM,YAAa,EAAA,CAAA;AACtC,IAAA,GAAA,CAAI,mCAAmC,CAAA,CAAA;AAAA,GACzC;AACA,EAAO,OAAA,gBAAA,CAAA;AACT,CAAA;AAKO,SAAS,YAAe,GAAA;AAC7B,EAAA,IAAI,gBAAkB,EAAA;AACpB,IAAA,GAAA,CAAI,qBAAqB,CAAA,CAAA;AACzB,IAAA,gBAAA,CAAiB,IAAK,EAAA,CAAA;AACtB,IAAmB,gBAAA,GAAA,IAAA,CAAA;AAAA,GACrB;AACF,CAAA;AAMsB,eAAA,aAAA,CACpB,OACA,IACmB,EAAA;AACnB,EAAM,MAAA,QAAA,GAAW,MAAM,iBAAkB,EAAA,CAAA;AAEzC,EAAM,MAAA,GAAA,GACJ,OAAO,KAAA,KAAU,QACb,GAAA,KAAA,GACA,iBAAiB,GACjB,GAAA,KAAA,CAAM,QAAS,EAAA,GACf,KAAM,CAAA,GAAA,CAAA;AACZ,EAAA,MAAM,MAAU,GAAA,CAAA,IAAA,EAAM,MAAU,IAAA,KAAA,EAAO,WAAY,EAAA,CAAA;AAEnD,EAAA,GAAA,CAAI,CAAU,OAAA,EAAA,MAAM,CAAe,YAAA,EAAA,GAAG,CAAE,CAAA,CAAA,CAAA;AAGxC,EAAA,MAAM,UAAkC,EAAC,CAAA;AACzC,EAAA,IAAI,MAAM,OAAS,EAAA;AACjB,IAAI,IAAA,IAAA,CAAK,mBAAmBA,uBAAS,EAAA;AACnC,MAAA,IAAA,CAAK,OAAQ,CAAA,OAAA,CAAQ,CAAC,KAAA,EAAO,GAAQ,KAAA;AACnC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACQ,MAAA,IAAA,KAAA,CAAM,OAAQ,CAAA,IAAA,CAAK,OAAO,CAAG,EAAA;AACtC,MAAA,IAAA,CAAK,QAAQ,OAAQ,CAAA,CAAC,CAAC,GAAA,EAAK,KAAK,CAAM,KAAA;AACrC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACI,MAAA;AACL,MAAO,MAAA,CAAA,MAAA,CAAO,OAAS,EAAA,IAAA,CAAK,OAAO,CAAA,CAAA;AAAA,KACrC;AAAA,GACF;AAGA,EAAI,IAAA,IAAA,CAAA;AACJ,EAAA,IAAI,MAAM,IAAM,EAAA;AACd,IAAI,IAAA,OAAO,IAAK,CAAA,IAAA,KAAS,QAAU,EAAA;AACjC,MAAA,IAAA,GAAO,IAAK,CAAA,IAAA,CAAA;AAAA,KACd,MAAA,IAAW,IAAK,CAAA,IAAA,YAAgB,eAAiB,EAAA;AAC/C,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KACrB,MAAA;AACL,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KAC5B;AAAA,GACF;AAGA,EAAA,MAAM,OAAU,GAAA;AAAA,IACd,IAAA;AAAA,IACA,OAAA;AAAA;AAAA,IAEA,GAAK,EAAA,8IAAA;AAAA,IACL,SAAA,EACE,OAAQ,CAAA,YAAY,CACpB,IAAA,iHAAA;AAAA,GACJ,CAAA;AAEA,EAAI,IAAA;AACF,IAAA,MAAM,WAAW,MAAM,QAAA;AAAA,MACrB,GAAA;AAAA,MACA,OAAA;AAAA,MACA,OAAO,WAAY,EAAA;AAAA,KAQrB,CAAA;AAIA,IAAM,MAAA,eAAA,GAAkB,IAAIA,uBAAQ,EAAA,CAAA;AACpC,IAAA,IAAI,SAAS,OAAS,EAAA;AACpB,MAAO,MAAA,CAAA,OAAA,CAAQ,SAAS,OAAO,CAAA,CAAE,QAAQ,CAAC,CAAC,GAAK,EAAA,KAAK,CAAM,KAAA;AACzD,QAAI,IAAA,KAAA,CAAM,OAAQ,CAAA,KAAK,CAAG,EAAA;AACxB,UAAM,KAAA,CAAA,OAAA,CAAQ,CAAC,CAAM,KAAA;AACnB,YAAgB,eAAA,CAAA,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA;AAAA,WAC9B,CAAA,CAAA;AAAA,SACH,MAAA,IAAW,OAAO,KAAA,KAAU,QAAU,EAAA;AACpC,UAAgB,eAAA,CAAA,GAAA,CAAI,KAAK,KAAK,CAAA,CAAA;AAAA,SAChC;AAAA,OACD,CAAA,CAAA;AAAA,KACH;AAIA,IAAA,IAAI,YAAe,GAAA,EAAA,CAAA;AACnB,IAAI,IAAA,OAAO,QAAS,CAAA,IAAA,KAAS,UAAY,EAAA;AACvC,MAAe,YAAA,GAAA,MAAM,SAAS,IAAK,EAAA,CAAA;AAAA,KACrC,MAAA,IAAY,SAAiB,IAAM,EAAA;AACjC,MAAA,YAAA,GAAgB,QAAiB,CAAA,IAAA,CAAA;AAAA,KACnC;AAGA,IAAM,MAAA,aAAA,GAAgB,IAAI,QAAA,CAAS,YAAc,EAAA;AAAA,MAC/C,QAAQ,QAAS,CAAA,MAAA;AAAA,MACjB,UAAY,EAAA,EAAA;AAAA;AAAA,MACZ,OAAS,EAAA,eAAA;AAAA,KACV,CAAA,CAAA;AAED,IAAO,OAAA,aAAA,CAAA;AAAA,WACA,KAAO,EAAA;AACd,IAAI,GAAA,CAAA,CAAA,yBAAA,EAA4B,KAAK,CAAE,CAAA,CAAA,CAAA;AACvC,IAAM,MAAA,KAAA,CAAA;AAAA,GACR;AACF;;;;;"}
|
|
1
|
+
{"version":3,"file":"index.cjs","sources":["../../../src/chrome-fingerprint.ts","../../../src/cycletls-fetch.ts"],"sourcesContent":["/**\n * Chrome version-dependent fingerprint constants.\n *\n * IMPORTANT: All constants in this file are tied to a specific Chrome version\n * (currently Chrome 144 on Windows 10). When bumping the Chrome version:\n * 1. Update CHROME_USER_AGENT with the new version string\n * 2. Update CHROME_SEC_CH_UA with the matching Client Hints\n * 3. Update CHROME_JA3 fingerprint (capture via tls.peet.ws)\n * 4. Update CHROME_JA4R fingerprint\n * 5. Update CHROME_HTTP2_FINGERPRINT settings frame\n * 6. Review CHROME_HEADER_ORDER if Chrome changes header ordering\n * 7. Update castle.ts DEFAULT_PROFILE if Chrome version affects fingerprint fields\n *\n * All values must be consistent with each other and match a real Chrome release.\n */\n\n/**\n * User-Agent string matching Chrome 144 on Windows 10.\n * Must be consistent across all requests and match the TLS fingerprint.\n */\nexport const CHROME_USER_AGENT =\n 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36';\n\n/**\n * Chrome Client Hints header matching the Chrome 144 user-agent.\n */\nexport const CHROME_SEC_CH_UA =\n '\"Not(A:Brand\";v=\"8\", \"Chromium\";v=\"144\", \"Google Chrome\";v=\"144\"';\n\n/**\n * JA3 TLS fingerprint for Chrome 144.\n * Captured from a real Chrome 144 browser session via tls.peet.ws.\n */\nexport const CHROME_JA3 =\n '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-10-35-16-11-51-27-65037-43-45-18-23-5-65281-13-17613,4588-29-23-24,0';\n\n/**\n * JA4r fingerprint for Chrome 144.\n * Format: t13d1516h2_CIPHERS_EXTENSIONS_SIG_ALGS\n */\nexport const CHROME_JA4R =\n 't13d1516h2_002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0012,0017,001b,0023,002b,002d,0033,44cd,fe0d,ff01_0403,0804,0401,0503,0805,0501,0806,0601';\n\n/**\n * Chrome 144 HTTP/2 fingerprint - mimics exact HTTP/2 SETTINGS frame.\n * Format: SETTINGS|window_size|unknown|priority_order\n */\nexport const CHROME_HTTP2_FINGERPRINT =\n '1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p';\n\n/**\n * Exact header order that Chrome 144 uses.\n * Header ordering is critical for HTTP/2 fingerprint evasion — servers can detect\n * non-browser clients by checking if headers arrive in a non-standard order.\n */\nexport const CHROME_HEADER_ORDER = [\n // HTTP/2 pseudo-headers (Chrome 144 order: method, authority, scheme, path)\n ':method',\n ':authority',\n ':scheme',\n ':path',\n // Chrome Client Hints (mandatory for modern detection bypass)\n 'sec-ch-ua',\n 'sec-ch-ua-mobile',\n 'sec-ch-ua-platform',\n // Standard browser headers\n 'upgrade-insecure-requests',\n 'user-agent',\n 'accept',\n 'origin',\n 'sec-fetch-site',\n 'sec-fetch-mode',\n 'sec-fetch-user',\n 'sec-fetch-dest',\n 'referer',\n 'accept-encoding',\n 'accept-language',\n 'priority',\n // Authentication headers\n 'authorization',\n 'x-csrf-token',\n 'x-guest-token',\n 'x-twitter-auth-type',\n 'x-twitter-active-user',\n 'x-twitter-client-language',\n 'x-client-transaction-id',\n 'x-xp-forwarded-for',\n // POST-specific\n 'content-type',\n 'cookie',\n];\n","import initCycleTLS from 'cycletls';\nimport { Headers } from 'headers-polyfill';\nimport debug from 'debug';\nimport {\n CHROME_USER_AGENT,\n CHROME_JA3,\n CHROME_JA4R,\n CHROME_HTTP2_FINGERPRINT,\n CHROME_HEADER_ORDER,\n} from './chrome-fingerprint';\n\nconst log = debug('twitter-scraper:cycletls');\n\nlet cycleTLSInstance: Awaited<ReturnType<typeof initCycleTLS>> | null = null;\n\n/**\n * Initialize the CycleTLS instance. This should be called once before using the fetch wrapper.\n */\nexport async function initCycleTLSFetch() {\n if (!cycleTLSInstance) {\n log('Initializing CycleTLS...');\n cycleTLSInstance = await initCycleTLS();\n log('CycleTLS initialized successfully');\n }\n return cycleTLSInstance;\n}\n\n/**\n * Cleanup the CycleTLS instance. Call this when you're done making requests.\n */\nexport function cycleTLSExit() {\n if (cycleTLSInstance) {\n log('Exiting CycleTLS...');\n cycleTLSInstance.exit();\n cycleTLSInstance = null;\n }\n}\n\n/**\n * A fetch-compatible wrapper around CycleTLS that mimics Chrome's TLS fingerprint\n * to bypass Cloudflare and other bot detection systems.\n */\nexport async function cycleTLSFetch(\n input: RequestInfo | URL,\n init?: RequestInit,\n): Promise<Response> {\n const instance = await initCycleTLSFetch();\n\n const url =\n typeof input === 'string'\n ? input\n : input instanceof URL\n ? input.toString()\n : input.url;\n const method = (init?.method || 'GET').toUpperCase();\n\n log(`Making ${method} request to ${url}`);\n\n // Extract headers from RequestInit\n const headers: Record<string, string> = {};\n if (init?.headers) {\n if (init.headers instanceof Headers) {\n init.headers.forEach((value, key) => {\n headers[key] = value;\n });\n } else if (Array.isArray(init.headers)) {\n init.headers.forEach(([key, value]) => {\n headers[key] = value;\n });\n } else {\n Object.assign(headers, init.headers);\n }\n }\n\n // Convert body to string if needed\n let body: string | undefined;\n if (init?.body) {\n if (typeof init.body === 'string') {\n body = init.body;\n } else if (init.body instanceof URLSearchParams) {\n body = init.body.toString();\n } else {\n body = init.body.toString();\n }\n }\n\n // All Chrome fingerprint constants imported from chrome-fingerprint.ts\n const options = {\n body,\n headers,\n ja3: CHROME_JA3,\n ja4r: CHROME_JA4R,\n http2Fingerprint: CHROME_HTTP2_FINGERPRINT,\n headerOrder: CHROME_HEADER_ORDER,\n orderAsProvided: true,\n disableGrease: false,\n userAgent: headers['user-agent'] || CHROME_USER_AGENT,\n };\n\n try {\n const response = await instance(\n url,\n options,\n method.toLowerCase() as\n | 'get'\n | 'post'\n | 'put'\n | 'delete'\n | 'patch'\n | 'head'\n | 'options',\n );\n\n // Convert CycleTLS response to fetch Response\n // CycleTLS returns headers as an object\n const responseHeaders = new Headers();\n if (response.headers) {\n Object.entries(response.headers).forEach(([key, value]) => {\n if (Array.isArray(value)) {\n value.forEach((v) => {\n responseHeaders.append(key, v);\n });\n } else if (typeof value === 'string') {\n responseHeaders.set(key, value);\n }\n });\n }\n\n // Get response body - cycletls provides helper methods, but we need the raw text\n // The response object has a text() method that returns the body as text\n let responseBody = '';\n if (typeof response.text === 'function') {\n responseBody = await response.text();\n } else if ((response as any).body) {\n responseBody = (response as any).body;\n }\n\n // Create a proper Response object using standard Response constructor\n const fetchResponse = new Response(responseBody, {\n status: response.status,\n statusText: '', // CycleTLS doesn't provide status text\n headers: responseHeaders,\n });\n\n return fetchResponse;\n } catch (error) {\n log(`CycleTLS request failed: ${error}`);\n throw error;\n }\n}\n"],"names":["Headers"],"mappings":";;;;;;AAoBO,MAAM,iBACX,GAAA,iHAAA,CAAA;AAYK,MAAM,UACX,GAAA,yJAAA,CAAA;AAMK,MAAM,WACX,GAAA,qMAAA,CAAA;AAMK,MAAM,wBACX,GAAA,mDAAA,CAAA;AAOK,MAAM,mBAAsB,GAAA;AAAA;AAAA,EAEjC,SAAA;AAAA,EACA,YAAA;AAAA,EACA,SAAA;AAAA,EACA,OAAA;AAAA;AAAA,EAEA,WAAA;AAAA,EACA,kBAAA;AAAA,EACA,oBAAA;AAAA;AAAA,EAEA,2BAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,QAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,SAAA;AAAA,EACA,iBAAA;AAAA,EACA,iBAAA;AAAA,EACA,UAAA;AAAA;AAAA,EAEA,eAAA;AAAA,EACA,cAAA;AAAA,EACA,eAAA;AAAA,EACA,qBAAA;AAAA,EACA,uBAAA;AAAA,EACA,2BAAA;AAAA,EACA,yBAAA;AAAA,EACA,oBAAA;AAAA;AAAA,EAEA,cAAA;AAAA,EACA,QAAA;AACF,CAAA;;AC/EA,MAAM,GAAA,GAAM,MAAM,0BAA0B,CAAA,CAAA;AAE5C,IAAI,gBAAoE,GAAA,IAAA,CAAA;AAKxE,eAAsB,iBAAoB,GAAA;AACxC,EAAA,IAAI,CAAC,gBAAkB,EAAA;AACrB,IAAA,GAAA,CAAI,0BAA0B,CAAA,CAAA;AAC9B,IAAA,gBAAA,GAAmB,MAAM,YAAa,EAAA,CAAA;AACtC,IAAA,GAAA,CAAI,mCAAmC,CAAA,CAAA;AAAA,GACzC;AACA,EAAO,OAAA,gBAAA,CAAA;AACT,CAAA;AAKO,SAAS,YAAe,GAAA;AAC7B,EAAA,IAAI,gBAAkB,EAAA;AACpB,IAAA,GAAA,CAAI,qBAAqB,CAAA,CAAA;AACzB,IAAA,gBAAA,CAAiB,IAAK,EAAA,CAAA;AACtB,IAAmB,gBAAA,GAAA,IAAA,CAAA;AAAA,GACrB;AACF,CAAA;AAMsB,eAAA,aAAA,CACpB,OACA,IACmB,EAAA;AACnB,EAAM,MAAA,QAAA,GAAW,MAAM,iBAAkB,EAAA,CAAA;AAEzC,EAAM,MAAA,GAAA,GACJ,OAAO,KAAA,KAAU,QACb,GAAA,KAAA,GACA,iBAAiB,GACjB,GAAA,KAAA,CAAM,QAAS,EAAA,GACf,KAAM,CAAA,GAAA,CAAA;AACZ,EAAA,MAAM,MAAU,GAAA,CAAA,IAAA,EAAM,MAAU,IAAA,KAAA,EAAO,WAAY,EAAA,CAAA;AAEnD,EAAA,GAAA,CAAI,CAAU,OAAA,EAAA,MAAM,CAAe,YAAA,EAAA,GAAG,CAAE,CAAA,CAAA,CAAA;AAGxC,EAAA,MAAM,UAAkC,EAAC,CAAA;AACzC,EAAA,IAAI,MAAM,OAAS,EAAA;AACjB,IAAI,IAAA,IAAA,CAAK,mBAAmBA,uBAAS,EAAA;AACnC,MAAA,IAAA,CAAK,OAAQ,CAAA,OAAA,CAAQ,CAAC,KAAA,EAAO,GAAQ,KAAA;AACnC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACQ,MAAA,IAAA,KAAA,CAAM,OAAQ,CAAA,IAAA,CAAK,OAAO,CAAG,EAAA;AACtC,MAAA,IAAA,CAAK,QAAQ,OAAQ,CAAA,CAAC,CAAC,GAAA,EAAK,KAAK,CAAM,KAAA;AACrC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACI,MAAA;AACL,MAAO,MAAA,CAAA,MAAA,CAAO,OAAS,EAAA,IAAA,CAAK,OAAO,CAAA,CAAA;AAAA,KACrC;AAAA,GACF;AAGA,EAAI,IAAA,IAAA,CAAA;AACJ,EAAA,IAAI,MAAM,IAAM,EAAA;AACd,IAAI,IAAA,OAAO,IAAK,CAAA,IAAA,KAAS,QAAU,EAAA;AACjC,MAAA,IAAA,GAAO,IAAK,CAAA,IAAA,CAAA;AAAA,KACd,MAAA,IAAW,IAAK,CAAA,IAAA,YAAgB,eAAiB,EAAA;AAC/C,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KACrB,MAAA;AACL,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KAC5B;AAAA,GACF;AAGA,EAAA,MAAM,OAAU,GAAA;AAAA,IACd,IAAA;AAAA,IACA,OAAA;AAAA,IACA,GAAK,EAAA,UAAA;AAAA,IACL,IAAM,EAAA,WAAA;AAAA,IACN,gBAAkB,EAAA,wBAAA;AAAA,IAClB,WAAa,EAAA,mBAAA;AAAA,IACb,eAAiB,EAAA,IAAA;AAAA,IACjB,aAAe,EAAA,KAAA;AAAA,IACf,SAAA,EAAW,OAAQ,CAAA,YAAY,CAAK,IAAA,iBAAA;AAAA,GACtC,CAAA;AAEA,EAAI,IAAA;AACF,IAAA,MAAM,WAAW,MAAM,QAAA;AAAA,MACrB,GAAA;AAAA,MACA,OAAA;AAAA,MACA,OAAO,WAAY,EAAA;AAAA,KAQrB,CAAA;AAIA,IAAM,MAAA,eAAA,GAAkB,IAAIA,uBAAQ,EAAA,CAAA;AACpC,IAAA,IAAI,SAAS,OAAS,EAAA;AACpB,MAAO,MAAA,CAAA,OAAA,CAAQ,SAAS,OAAO,CAAA,CAAE,QAAQ,CAAC,CAAC,GAAK,EAAA,KAAK,CAAM,KAAA;AACzD,QAAI,IAAA,KAAA,CAAM,OAAQ,CAAA,KAAK,CAAG,EAAA;AACxB,UAAM,KAAA,CAAA,OAAA,CAAQ,CAAC,CAAM,KAAA;AACnB,YAAgB,eAAA,CAAA,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA;AAAA,WAC9B,CAAA,CAAA;AAAA,SACH,MAAA,IAAW,OAAO,KAAA,KAAU,QAAU,EAAA;AACpC,UAAgB,eAAA,CAAA,GAAA,CAAI,KAAK,KAAK,CAAA,CAAA;AAAA,SAChC;AAAA,OACD,CAAA,CAAA;AAAA,KACH;AAIA,IAAA,IAAI,YAAe,GAAA,EAAA,CAAA;AACnB,IAAI,IAAA,OAAO,QAAS,CAAA,IAAA,KAAS,UAAY,EAAA;AACvC,MAAe,YAAA,GAAA,MAAM,SAAS,IAAK,EAAA,CAAA;AAAA,KACrC,MAAA,IAAY,SAAiB,IAAM,EAAA;AACjC,MAAA,YAAA,GAAgB,QAAiB,CAAA,IAAA,CAAA;AAAA,KACnC;AAGA,IAAM,MAAA,aAAA,GAAgB,IAAI,QAAA,CAAS,YAAc,EAAA;AAAA,MAC/C,QAAQ,QAAS,CAAA,MAAA;AAAA,MACjB,UAAY,EAAA,EAAA;AAAA;AAAA,MACZ,OAAS,EAAA,eAAA;AAAA,KACV,CAAA,CAAA;AAED,IAAO,OAAA,aAAA,CAAA;AAAA,WACA,KAAO,EAAA;AACd,IAAI,GAAA,CAAA,CAAA,yBAAA,EAA4B,KAAK,CAAE,CAAA,CAAA,CAAA;AACvC,IAAM,MAAA,KAAA,CAAA;AAAA,GACR;AACF;;;;;"}
|
|
@@ -2,6 +2,47 @@ import initCycleTLS from 'cycletls';
|
|
|
2
2
|
import { Headers } from 'headers-polyfill';
|
|
3
3
|
import debug from 'debug';
|
|
4
4
|
|
|
5
|
+
const CHROME_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36";
|
|
6
|
+
const CHROME_JA3 = "771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-10-35-16-11-51-27-65037-43-45-18-23-5-65281-13-17613,4588-29-23-24,0";
|
|
7
|
+
const CHROME_JA4R = "t13d1516h2_002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0012,0017,001b,0023,002b,002d,0033,44cd,fe0d,ff01_0403,0804,0401,0503,0805,0501,0806,0601";
|
|
8
|
+
const CHROME_HTTP2_FINGERPRINT = "1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p";
|
|
9
|
+
const CHROME_HEADER_ORDER = [
|
|
10
|
+
// HTTP/2 pseudo-headers (Chrome 144 order: method, authority, scheme, path)
|
|
11
|
+
":method",
|
|
12
|
+
":authority",
|
|
13
|
+
":scheme",
|
|
14
|
+
":path",
|
|
15
|
+
// Chrome Client Hints (mandatory for modern detection bypass)
|
|
16
|
+
"sec-ch-ua",
|
|
17
|
+
"sec-ch-ua-mobile",
|
|
18
|
+
"sec-ch-ua-platform",
|
|
19
|
+
// Standard browser headers
|
|
20
|
+
"upgrade-insecure-requests",
|
|
21
|
+
"user-agent",
|
|
22
|
+
"accept",
|
|
23
|
+
"origin",
|
|
24
|
+
"sec-fetch-site",
|
|
25
|
+
"sec-fetch-mode",
|
|
26
|
+
"sec-fetch-user",
|
|
27
|
+
"sec-fetch-dest",
|
|
28
|
+
"referer",
|
|
29
|
+
"accept-encoding",
|
|
30
|
+
"accept-language",
|
|
31
|
+
"priority",
|
|
32
|
+
// Authentication headers
|
|
33
|
+
"authorization",
|
|
34
|
+
"x-csrf-token",
|
|
35
|
+
"x-guest-token",
|
|
36
|
+
"x-twitter-auth-type",
|
|
37
|
+
"x-twitter-active-user",
|
|
38
|
+
"x-twitter-client-language",
|
|
39
|
+
"x-client-transaction-id",
|
|
40
|
+
"x-xp-forwarded-for",
|
|
41
|
+
// POST-specific
|
|
42
|
+
"content-type",
|
|
43
|
+
"cookie"
|
|
44
|
+
];
|
|
45
|
+
|
|
5
46
|
const log = debug("twitter-scraper:cycletls");
|
|
6
47
|
let cycleTLSInstance = null;
|
|
7
48
|
async function initCycleTLSFetch() {
|
|
@@ -51,9 +92,13 @@ async function cycleTLSFetch(input, init) {
|
|
|
51
92
|
const options = {
|
|
52
93
|
body,
|
|
53
94
|
headers,
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
95
|
+
ja3: CHROME_JA3,
|
|
96
|
+
ja4r: CHROME_JA4R,
|
|
97
|
+
http2Fingerprint: CHROME_HTTP2_FINGERPRINT,
|
|
98
|
+
headerOrder: CHROME_HEADER_ORDER,
|
|
99
|
+
orderAsProvided: true,
|
|
100
|
+
disableGrease: false,
|
|
101
|
+
userAgent: headers["user-agent"] || CHROME_USER_AGENT
|
|
57
102
|
};
|
|
58
103
|
try {
|
|
59
104
|
const response = await instance(
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.mjs","sources":["../../../src/cycletls-fetch.ts"],"sourcesContent":["import initCycleTLS from 'cycletls';\nimport { Headers } from 'headers-polyfill';\nimport debug from 'debug';\n\nconst log = debug('twitter-scraper:cycletls');\n\nlet cycleTLSInstance: Awaited<ReturnType<typeof initCycleTLS>> | null = null;\n\n/**\n * Initialize the CycleTLS instance. This should be called once before using the fetch wrapper.\n */\nexport async function initCycleTLSFetch() {\n if (!cycleTLSInstance) {\n log('Initializing CycleTLS...');\n cycleTLSInstance = await initCycleTLS();\n log('CycleTLS initialized successfully');\n }\n return cycleTLSInstance;\n}\n\n/**\n * Cleanup the CycleTLS instance. Call this when you're done making requests.\n */\nexport function cycleTLSExit() {\n if (cycleTLSInstance) {\n log('Exiting CycleTLS...');\n cycleTLSInstance.exit();\n cycleTLSInstance = null;\n }\n}\n\n/**\n * A fetch-compatible wrapper around CycleTLS that mimics Chrome's TLS fingerprint\n * to bypass Cloudflare and other bot detection systems.\n */\nexport async function cycleTLSFetch(\n input: RequestInfo | URL,\n init?: RequestInit,\n): Promise<Response> {\n const instance = await initCycleTLSFetch();\n\n const url =\n typeof input === 'string'\n ? input\n : input instanceof URL\n ? input.toString()\n : input.url;\n const method = (init?.method || 'GET').toUpperCase();\n\n log(`Making ${method} request to ${url}`);\n\n // Extract headers from RequestInit\n const headers: Record<string, string> = {};\n if (init?.headers) {\n if (init.headers instanceof Headers) {\n init.headers.forEach((value, key) => {\n headers[key] = value;\n });\n } else if (Array.isArray(init.headers)) {\n init.headers.forEach(([key, value]) => {\n headers[key] = value;\n });\n } else {\n Object.assign(headers, init.headers);\n }\n }\n\n // Convert body to string if needed\n let body: string | undefined;\n if (init?.body) {\n if (typeof init.body === 'string') {\n body = init.body;\n } else if (init.body instanceof URLSearchParams) {\n body = init.body.toString();\n } else {\n body = init.body.toString();\n }\n }\n\n // Use Chrome 120 JA3 fingerprint for maximum compatibility\n const options = {\n body,\n headers,\n // Chrome 120 on Windows 10\n ja3: '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0',\n userAgent:\n headers['user-agent'] ||\n 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36',\n };\n\n try {\n const response = await instance(\n url,\n options,\n method.toLowerCase() as\n | 'get'\n | 'post'\n | 'put'\n | 'delete'\n | 'patch'\n | 'head'\n | 'options',\n );\n\n // Convert CycleTLS response to fetch Response\n // CycleTLS returns headers as an object\n const responseHeaders = new Headers();\n if (response.headers) {\n Object.entries(response.headers).forEach(([key, value]) => {\n if (Array.isArray(value)) {\n value.forEach((v) => {\n responseHeaders.append(key, v);\n });\n } else if (typeof value === 'string') {\n responseHeaders.set(key, value);\n }\n });\n }\n\n // Get response body - cycletls provides helper methods, but we need the raw text\n // The response object has a text() method that returns the body as text\n let responseBody = '';\n if (typeof response.text === 'function') {\n responseBody = await response.text();\n } else if ((response as any).body) {\n responseBody = (response as any).body;\n }\n\n // Create a proper Response object using standard Response constructor\n const fetchResponse = new Response(responseBody, {\n status: response.status,\n statusText: '', // CycleTLS doesn't provide status text\n headers: responseHeaders,\n });\n\n return fetchResponse;\n } catch (error) {\n log(`CycleTLS request failed: ${error}`);\n throw error;\n }\n}\n"],"names":[],"mappings":";;;;AAIA,MAAM,GAAA,GAAM,MAAM,0BAA0B,CAAA,CAAA;AAE5C,IAAI,gBAAoE,GAAA,IAAA,CAAA;AAKxE,eAAsB,iBAAoB,GAAA;AACxC,EAAA,IAAI,CAAC,gBAAkB,EAAA;AACrB,IAAA,GAAA,CAAI,0BAA0B,CAAA,CAAA;AAC9B,IAAA,gBAAA,GAAmB,MAAM,YAAa,EAAA,CAAA;AACtC,IAAA,GAAA,CAAI,mCAAmC,CAAA,CAAA;AAAA,GACzC;AACA,EAAO,OAAA,gBAAA,CAAA;AACT,CAAA;AAKO,SAAS,YAAe,GAAA;AAC7B,EAAA,IAAI,gBAAkB,EAAA;AACpB,IAAA,GAAA,CAAI,qBAAqB,CAAA,CAAA;AACzB,IAAA,gBAAA,CAAiB,IAAK,EAAA,CAAA;AACtB,IAAmB,gBAAA,GAAA,IAAA,CAAA;AAAA,GACrB;AACF,CAAA;AAMsB,eAAA,aAAA,CACpB,OACA,IACmB,EAAA;AACnB,EAAM,MAAA,QAAA,GAAW,MAAM,iBAAkB,EAAA,CAAA;AAEzC,EAAM,MAAA,GAAA,GACJ,OAAO,KAAA,KAAU,QACb,GAAA,KAAA,GACA,iBAAiB,GACjB,GAAA,KAAA,CAAM,QAAS,EAAA,GACf,KAAM,CAAA,GAAA,CAAA;AACZ,EAAA,MAAM,MAAU,GAAA,CAAA,IAAA,EAAM,MAAU,IAAA,KAAA,EAAO,WAAY,EAAA,CAAA;AAEnD,EAAA,GAAA,CAAI,CAAU,OAAA,EAAA,MAAM,CAAe,YAAA,EAAA,GAAG,CAAE,CAAA,CAAA,CAAA;AAGxC,EAAA,MAAM,UAAkC,EAAC,CAAA;AACzC,EAAA,IAAI,MAAM,OAAS,EAAA;AACjB,IAAI,IAAA,IAAA,CAAK,mBAAmB,OAAS,EAAA;AACnC,MAAA,IAAA,CAAK,OAAQ,CAAA,OAAA,CAAQ,CAAC,KAAA,EAAO,GAAQ,KAAA;AACnC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACQ,MAAA,IAAA,KAAA,CAAM,OAAQ,CAAA,IAAA,CAAK,OAAO,CAAG,EAAA;AACtC,MAAA,IAAA,CAAK,QAAQ,OAAQ,CAAA,CAAC,CAAC,GAAA,EAAK,KAAK,CAAM,KAAA;AACrC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACI,MAAA;AACL,MAAO,MAAA,CAAA,MAAA,CAAO,OAAS,EAAA,IAAA,CAAK,OAAO,CAAA,CAAA;AAAA,KACrC;AAAA,GACF;AAGA,EAAI,IAAA,IAAA,CAAA;AACJ,EAAA,IAAI,MAAM,IAAM,EAAA;AACd,IAAI,IAAA,OAAO,IAAK,CAAA,IAAA,KAAS,QAAU,EAAA;AACjC,MAAA,IAAA,GAAO,IAAK,CAAA,IAAA,CAAA;AAAA,KACd,MAAA,IAAW,IAAK,CAAA,IAAA,YAAgB,eAAiB,EAAA;AAC/C,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KACrB,MAAA;AACL,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KAC5B;AAAA,GACF;AAGA,EAAA,MAAM,OAAU,GAAA;AAAA,IACd,IAAA;AAAA,IACA,OAAA;AAAA;AAAA,IAEA,GAAK,EAAA,8IAAA;AAAA,IACL,SAAA,EACE,OAAQ,CAAA,YAAY,CACpB,IAAA,iHAAA;AAAA,GACJ,CAAA;AAEA,EAAI,IAAA;AACF,IAAA,MAAM,WAAW,MAAM,QAAA;AAAA,MACrB,GAAA;AAAA,MACA,OAAA;AAAA,MACA,OAAO,WAAY,EAAA;AAAA,KAQrB,CAAA;AAIA,IAAM,MAAA,eAAA,GAAkB,IAAI,OAAQ,EAAA,CAAA;AACpC,IAAA,IAAI,SAAS,OAAS,EAAA;AACpB,MAAO,MAAA,CAAA,OAAA,CAAQ,SAAS,OAAO,CAAA,CAAE,QAAQ,CAAC,CAAC,GAAK,EAAA,KAAK,CAAM,KAAA;AACzD,QAAI,IAAA,KAAA,CAAM,OAAQ,CAAA,KAAK,CAAG,EAAA;AACxB,UAAM,KAAA,CAAA,OAAA,CAAQ,CAAC,CAAM,KAAA;AACnB,YAAgB,eAAA,CAAA,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA;AAAA,WAC9B,CAAA,CAAA;AAAA,SACH,MAAA,IAAW,OAAO,KAAA,KAAU,QAAU,EAAA;AACpC,UAAgB,eAAA,CAAA,GAAA,CAAI,KAAK,KAAK,CAAA,CAAA;AAAA,SAChC;AAAA,OACD,CAAA,CAAA;AAAA,KACH;AAIA,IAAA,IAAI,YAAe,GAAA,EAAA,CAAA;AACnB,IAAI,IAAA,OAAO,QAAS,CAAA,IAAA,KAAS,UAAY,EAAA;AACvC,MAAe,YAAA,GAAA,MAAM,SAAS,IAAK,EAAA,CAAA;AAAA,KACrC,MAAA,IAAY,SAAiB,IAAM,EAAA;AACjC,MAAA,YAAA,GAAgB,QAAiB,CAAA,IAAA,CAAA;AAAA,KACnC;AAGA,IAAM,MAAA,aAAA,GAAgB,IAAI,QAAA,CAAS,YAAc,EAAA;AAAA,MAC/C,QAAQ,QAAS,CAAA,MAAA;AAAA,MACjB,UAAY,EAAA,EAAA;AAAA;AAAA,MACZ,OAAS,EAAA,eAAA;AAAA,KACV,CAAA,CAAA;AAED,IAAO,OAAA,aAAA,CAAA;AAAA,WACA,KAAO,EAAA;AACd,IAAI,GAAA,CAAA,CAAA,yBAAA,EAA4B,KAAK,CAAE,CAAA,CAAA,CAAA;AACvC,IAAM,MAAA,KAAA,CAAA;AAAA,GACR;AACF;;;;"}
|
|
1
|
+
{"version":3,"file":"index.mjs","sources":["../../../src/chrome-fingerprint.ts","../../../src/cycletls-fetch.ts"],"sourcesContent":["/**\n * Chrome version-dependent fingerprint constants.\n *\n * IMPORTANT: All constants in this file are tied to a specific Chrome version\n * (currently Chrome 144 on Windows 10). When bumping the Chrome version:\n * 1. Update CHROME_USER_AGENT with the new version string\n * 2. Update CHROME_SEC_CH_UA with the matching Client Hints\n * 3. Update CHROME_JA3 fingerprint (capture via tls.peet.ws)\n * 4. Update CHROME_JA4R fingerprint\n * 5. Update CHROME_HTTP2_FINGERPRINT settings frame\n * 6. Review CHROME_HEADER_ORDER if Chrome changes header ordering\n * 7. Update castle.ts DEFAULT_PROFILE if Chrome version affects fingerprint fields\n *\n * All values must be consistent with each other and match a real Chrome release.\n */\n\n/**\n * User-Agent string matching Chrome 144 on Windows 10.\n * Must be consistent across all requests and match the TLS fingerprint.\n */\nexport const CHROME_USER_AGENT =\n 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36';\n\n/**\n * Chrome Client Hints header matching the Chrome 144 user-agent.\n */\nexport const CHROME_SEC_CH_UA =\n '\"Not(A:Brand\";v=\"8\", \"Chromium\";v=\"144\", \"Google Chrome\";v=\"144\"';\n\n/**\n * JA3 TLS fingerprint for Chrome 144.\n * Captured from a real Chrome 144 browser session via tls.peet.ws.\n */\nexport const CHROME_JA3 =\n '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,0-10-35-16-11-51-27-65037-43-45-18-23-5-65281-13-17613,4588-29-23-24,0';\n\n/**\n * JA4r fingerprint for Chrome 144.\n * Format: t13d1516h2_CIPHERS_EXTENSIONS_SIG_ALGS\n */\nexport const CHROME_JA4R =\n 't13d1516h2_002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0012,0017,001b,0023,002b,002d,0033,44cd,fe0d,ff01_0403,0804,0401,0503,0805,0501,0806,0601';\n\n/**\n * Chrome 144 HTTP/2 fingerprint - mimics exact HTTP/2 SETTINGS frame.\n * Format: SETTINGS|window_size|unknown|priority_order\n */\nexport const CHROME_HTTP2_FINGERPRINT =\n '1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p';\n\n/**\n * Exact header order that Chrome 144 uses.\n * Header ordering is critical for HTTP/2 fingerprint evasion — servers can detect\n * non-browser clients by checking if headers arrive in a non-standard order.\n */\nexport const CHROME_HEADER_ORDER = [\n // HTTP/2 pseudo-headers (Chrome 144 order: method, authority, scheme, path)\n ':method',\n ':authority',\n ':scheme',\n ':path',\n // Chrome Client Hints (mandatory for modern detection bypass)\n 'sec-ch-ua',\n 'sec-ch-ua-mobile',\n 'sec-ch-ua-platform',\n // Standard browser headers\n 'upgrade-insecure-requests',\n 'user-agent',\n 'accept',\n 'origin',\n 'sec-fetch-site',\n 'sec-fetch-mode',\n 'sec-fetch-user',\n 'sec-fetch-dest',\n 'referer',\n 'accept-encoding',\n 'accept-language',\n 'priority',\n // Authentication headers\n 'authorization',\n 'x-csrf-token',\n 'x-guest-token',\n 'x-twitter-auth-type',\n 'x-twitter-active-user',\n 'x-twitter-client-language',\n 'x-client-transaction-id',\n 'x-xp-forwarded-for',\n // POST-specific\n 'content-type',\n 'cookie',\n];\n","import initCycleTLS from 'cycletls';\nimport { Headers } from 'headers-polyfill';\nimport debug from 'debug';\nimport {\n CHROME_USER_AGENT,\n CHROME_JA3,\n CHROME_JA4R,\n CHROME_HTTP2_FINGERPRINT,\n CHROME_HEADER_ORDER,\n} from './chrome-fingerprint';\n\nconst log = debug('twitter-scraper:cycletls');\n\nlet cycleTLSInstance: Awaited<ReturnType<typeof initCycleTLS>> | null = null;\n\n/**\n * Initialize the CycleTLS instance. This should be called once before using the fetch wrapper.\n */\nexport async function initCycleTLSFetch() {\n if (!cycleTLSInstance) {\n log('Initializing CycleTLS...');\n cycleTLSInstance = await initCycleTLS();\n log('CycleTLS initialized successfully');\n }\n return cycleTLSInstance;\n}\n\n/**\n * Cleanup the CycleTLS instance. Call this when you're done making requests.\n */\nexport function cycleTLSExit() {\n if (cycleTLSInstance) {\n log('Exiting CycleTLS...');\n cycleTLSInstance.exit();\n cycleTLSInstance = null;\n }\n}\n\n/**\n * A fetch-compatible wrapper around CycleTLS that mimics Chrome's TLS fingerprint\n * to bypass Cloudflare and other bot detection systems.\n */\nexport async function cycleTLSFetch(\n input: RequestInfo | URL,\n init?: RequestInit,\n): Promise<Response> {\n const instance = await initCycleTLSFetch();\n\n const url =\n typeof input === 'string'\n ? input\n : input instanceof URL\n ? input.toString()\n : input.url;\n const method = (init?.method || 'GET').toUpperCase();\n\n log(`Making ${method} request to ${url}`);\n\n // Extract headers from RequestInit\n const headers: Record<string, string> = {};\n if (init?.headers) {\n if (init.headers instanceof Headers) {\n init.headers.forEach((value, key) => {\n headers[key] = value;\n });\n } else if (Array.isArray(init.headers)) {\n init.headers.forEach(([key, value]) => {\n headers[key] = value;\n });\n } else {\n Object.assign(headers, init.headers);\n }\n }\n\n // Convert body to string if needed\n let body: string | undefined;\n if (init?.body) {\n if (typeof init.body === 'string') {\n body = init.body;\n } else if (init.body instanceof URLSearchParams) {\n body = init.body.toString();\n } else {\n body = init.body.toString();\n }\n }\n\n // All Chrome fingerprint constants imported from chrome-fingerprint.ts\n const options = {\n body,\n headers,\n ja3: CHROME_JA3,\n ja4r: CHROME_JA4R,\n http2Fingerprint: CHROME_HTTP2_FINGERPRINT,\n headerOrder: CHROME_HEADER_ORDER,\n orderAsProvided: true,\n disableGrease: false,\n userAgent: headers['user-agent'] || CHROME_USER_AGENT,\n };\n\n try {\n const response = await instance(\n url,\n options,\n method.toLowerCase() as\n | 'get'\n | 'post'\n | 'put'\n | 'delete'\n | 'patch'\n | 'head'\n | 'options',\n );\n\n // Convert CycleTLS response to fetch Response\n // CycleTLS returns headers as an object\n const responseHeaders = new Headers();\n if (response.headers) {\n Object.entries(response.headers).forEach(([key, value]) => {\n if (Array.isArray(value)) {\n value.forEach((v) => {\n responseHeaders.append(key, v);\n });\n } else if (typeof value === 'string') {\n responseHeaders.set(key, value);\n }\n });\n }\n\n // Get response body - cycletls provides helper methods, but we need the raw text\n // The response object has a text() method that returns the body as text\n let responseBody = '';\n if (typeof response.text === 'function') {\n responseBody = await response.text();\n } else if ((response as any).body) {\n responseBody = (response as any).body;\n }\n\n // Create a proper Response object using standard Response constructor\n const fetchResponse = new Response(responseBody, {\n status: response.status,\n statusText: '', // CycleTLS doesn't provide status text\n headers: responseHeaders,\n });\n\n return fetchResponse;\n } catch (error) {\n log(`CycleTLS request failed: ${error}`);\n throw error;\n }\n}\n"],"names":[],"mappings":";;;;AAoBO,MAAM,iBACX,GAAA,iHAAA,CAAA;AAYK,MAAM,UACX,GAAA,yJAAA,CAAA;AAMK,MAAM,WACX,GAAA,qMAAA,CAAA;AAMK,MAAM,wBACX,GAAA,mDAAA,CAAA;AAOK,MAAM,mBAAsB,GAAA;AAAA;AAAA,EAEjC,SAAA;AAAA,EACA,YAAA;AAAA,EACA,SAAA;AAAA,EACA,OAAA;AAAA;AAAA,EAEA,WAAA;AAAA,EACA,kBAAA;AAAA,EACA,oBAAA;AAAA;AAAA,EAEA,2BAAA;AAAA,EACA,YAAA;AAAA,EACA,QAAA;AAAA,EACA,QAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,gBAAA;AAAA,EACA,SAAA;AAAA,EACA,iBAAA;AAAA,EACA,iBAAA;AAAA,EACA,UAAA;AAAA;AAAA,EAEA,eAAA;AAAA,EACA,cAAA;AAAA,EACA,eAAA;AAAA,EACA,qBAAA;AAAA,EACA,uBAAA;AAAA,EACA,2BAAA;AAAA,EACA,yBAAA;AAAA,EACA,oBAAA;AAAA;AAAA,EAEA,cAAA;AAAA,EACA,QAAA;AACF,CAAA;;AC/EA,MAAM,GAAA,GAAM,MAAM,0BAA0B,CAAA,CAAA;AAE5C,IAAI,gBAAoE,GAAA,IAAA,CAAA;AAKxE,eAAsB,iBAAoB,GAAA;AACxC,EAAA,IAAI,CAAC,gBAAkB,EAAA;AACrB,IAAA,GAAA,CAAI,0BAA0B,CAAA,CAAA;AAC9B,IAAA,gBAAA,GAAmB,MAAM,YAAa,EAAA,CAAA;AACtC,IAAA,GAAA,CAAI,mCAAmC,CAAA,CAAA;AAAA,GACzC;AACA,EAAO,OAAA,gBAAA,CAAA;AACT,CAAA;AAKO,SAAS,YAAe,GAAA;AAC7B,EAAA,IAAI,gBAAkB,EAAA;AACpB,IAAA,GAAA,CAAI,qBAAqB,CAAA,CAAA;AACzB,IAAA,gBAAA,CAAiB,IAAK,EAAA,CAAA;AACtB,IAAmB,gBAAA,GAAA,IAAA,CAAA;AAAA,GACrB;AACF,CAAA;AAMsB,eAAA,aAAA,CACpB,OACA,IACmB,EAAA;AACnB,EAAM,MAAA,QAAA,GAAW,MAAM,iBAAkB,EAAA,CAAA;AAEzC,EAAM,MAAA,GAAA,GACJ,OAAO,KAAA,KAAU,QACb,GAAA,KAAA,GACA,iBAAiB,GACjB,GAAA,KAAA,CAAM,QAAS,EAAA,GACf,KAAM,CAAA,GAAA,CAAA;AACZ,EAAA,MAAM,MAAU,GAAA,CAAA,IAAA,EAAM,MAAU,IAAA,KAAA,EAAO,WAAY,EAAA,CAAA;AAEnD,EAAA,GAAA,CAAI,CAAU,OAAA,EAAA,MAAM,CAAe,YAAA,EAAA,GAAG,CAAE,CAAA,CAAA,CAAA;AAGxC,EAAA,MAAM,UAAkC,EAAC,CAAA;AACzC,EAAA,IAAI,MAAM,OAAS,EAAA;AACjB,IAAI,IAAA,IAAA,CAAK,mBAAmB,OAAS,EAAA;AACnC,MAAA,IAAA,CAAK,OAAQ,CAAA,OAAA,CAAQ,CAAC,KAAA,EAAO,GAAQ,KAAA;AACnC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACQ,MAAA,IAAA,KAAA,CAAM,OAAQ,CAAA,IAAA,CAAK,OAAO,CAAG,EAAA;AACtC,MAAA,IAAA,CAAK,QAAQ,OAAQ,CAAA,CAAC,CAAC,GAAA,EAAK,KAAK,CAAM,KAAA;AACrC,QAAA,OAAA,CAAQ,GAAG,CAAI,GAAA,KAAA,CAAA;AAAA,OAChB,CAAA,CAAA;AAAA,KACI,MAAA;AACL,MAAO,MAAA,CAAA,MAAA,CAAO,OAAS,EAAA,IAAA,CAAK,OAAO,CAAA,CAAA;AAAA,KACrC;AAAA,GACF;AAGA,EAAI,IAAA,IAAA,CAAA;AACJ,EAAA,IAAI,MAAM,IAAM,EAAA;AACd,IAAI,IAAA,OAAO,IAAK,CAAA,IAAA,KAAS,QAAU,EAAA;AACjC,MAAA,IAAA,GAAO,IAAK,CAAA,IAAA,CAAA;AAAA,KACd,MAAA,IAAW,IAAK,CAAA,IAAA,YAAgB,eAAiB,EAAA;AAC/C,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KACrB,MAAA;AACL,MAAO,IAAA,GAAA,IAAA,CAAK,KAAK,QAAS,EAAA,CAAA;AAAA,KAC5B;AAAA,GACF;AAGA,EAAA,MAAM,OAAU,GAAA;AAAA,IACd,IAAA;AAAA,IACA,OAAA;AAAA,IACA,GAAK,EAAA,UAAA;AAAA,IACL,IAAM,EAAA,WAAA;AAAA,IACN,gBAAkB,EAAA,wBAAA;AAAA,IAClB,WAAa,EAAA,mBAAA;AAAA,IACb,eAAiB,EAAA,IAAA;AAAA,IACjB,aAAe,EAAA,KAAA;AAAA,IACf,SAAA,EAAW,OAAQ,CAAA,YAAY,CAAK,IAAA,iBAAA;AAAA,GACtC,CAAA;AAEA,EAAI,IAAA;AACF,IAAA,MAAM,WAAW,MAAM,QAAA;AAAA,MACrB,GAAA;AAAA,MACA,OAAA;AAAA,MACA,OAAO,WAAY,EAAA;AAAA,KAQrB,CAAA;AAIA,IAAM,MAAA,eAAA,GAAkB,IAAI,OAAQ,EAAA,CAAA;AACpC,IAAA,IAAI,SAAS,OAAS,EAAA;AACpB,MAAO,MAAA,CAAA,OAAA,CAAQ,SAAS,OAAO,CAAA,CAAE,QAAQ,CAAC,CAAC,GAAK,EAAA,KAAK,CAAM,KAAA;AACzD,QAAI,IAAA,KAAA,CAAM,OAAQ,CAAA,KAAK,CAAG,EAAA;AACxB,UAAM,KAAA,CAAA,OAAA,CAAQ,CAAC,CAAM,KAAA;AACnB,YAAgB,eAAA,CAAA,MAAA,CAAO,KAAK,CAAC,CAAA,CAAA;AAAA,WAC9B,CAAA,CAAA;AAAA,SACH,MAAA,IAAW,OAAO,KAAA,KAAU,QAAU,EAAA;AACpC,UAAgB,eAAA,CAAA,GAAA,CAAI,KAAK,KAAK,CAAA,CAAA;AAAA,SAChC;AAAA,OACD,CAAA,CAAA;AAAA,KACH;AAIA,IAAA,IAAI,YAAe,GAAA,EAAA,CAAA;AACnB,IAAI,IAAA,OAAO,QAAS,CAAA,IAAA,KAAS,UAAY,EAAA;AACvC,MAAe,YAAA,GAAA,MAAM,SAAS,IAAK,EAAA,CAAA;AAAA,KACrC,MAAA,IAAY,SAAiB,IAAM,EAAA;AACjC,MAAA,YAAA,GAAgB,QAAiB,CAAA,IAAA,CAAA;AAAA,KACnC;AAGA,IAAM,MAAA,aAAA,GAAgB,IAAI,QAAA,CAAS,YAAc,EAAA;AAAA,MAC/C,QAAQ,QAAS,CAAA,MAAA;AAAA,MACjB,UAAY,EAAA,EAAA;AAAA;AAAA,MACZ,OAAS,EAAA,eAAA;AAAA,KACV,CAAA,CAAA;AAED,IAAO,OAAA,aAAA,CAAA;AAAA,WACA,KAAO,EAAA;AACd,IAAI,GAAA,CAAA,CAAA,yBAAA,EAA4B,KAAK,CAAE,CAAA,CAAA,CAAA;AACvC,IAAM,MAAA,KAAA,CAAA;AAAA,GACR;AACF;;;;"}
|