@zeldrisho/pi-web-fetch 0.14.1 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +28 -1
- package/README.md +3 -5
- package/package.json +5 -2
- package/src/acquisition/fetch.ts +167 -12
- package/src/acquisition/llms.ts +42 -11
- package/src/challenges/aws-waf-resolver.ts +806 -0
- package/src/challenges/aws-waf-solver.ts +65 -0
- package/src/challenges/aws-waf-worker.ts +14 -0
- package/src/challenges/aws-waf.ts +34 -0
- package/src/document/content.ts +4 -0
- package/src/document/focus.ts +117 -11
- package/src/document/openapi.ts +80 -0
- package/src/document/source-format.ts +180 -0
- package/src/index.ts +7 -0
- package/src/network/cookies.ts +41 -0
- package/src/network/network-redirects.ts +15 -2
- package/src/network/network-transport.ts +21 -3
- package/src/service.ts +23 -5
- package/workers/aws-waf-solver.js +56 -0
- package/workers/aws-waf-worker.js +14 -0
|
@@ -0,0 +1,806 @@
|
|
|
1
|
+
import { createCipheriv, randomBytes, randomInt, randomUUID } from "node:crypto";
|
|
2
|
+
import { Worker } from "node:worker_threads";
|
|
3
|
+
import type { AwsWafWork } from "./aws-waf-solver.ts";
|
|
4
|
+
import type { IncomingMessage } from "node:http";
|
|
5
|
+
import { crc32 } from "node:zlib";
|
|
6
|
+
import type { AwsWafResolveContext, AwsWafResolver, AwsWafSolution } from "./aws-waf.ts";
|
|
7
|
+
import { CookieJar } from "../network/cookies.ts";
|
|
8
|
+
import {
|
|
9
|
+
requestFollowingRedirects,
|
|
10
|
+
type RedirectDependencies,
|
|
11
|
+
} from "../network/network-redirects.ts";
|
|
12
|
+
import { validateRemoteUrl } from "../network/network-policy.ts";
|
|
13
|
+
import {
|
|
14
|
+
FETCH_MAX_BYTES,
|
|
15
|
+
decodeResponse,
|
|
16
|
+
decompressResponse,
|
|
17
|
+
readResponseBytes,
|
|
18
|
+
requestPinned,
|
|
19
|
+
responseHeader,
|
|
20
|
+
} from "../network/network-transport.ts";
|
|
21
|
+
|
|
22
|
+
/** AWS WAF browser-challenge signal encryption key used by the current challenge family. */
|
|
23
|
+
const SIGNAL_KEY = Buffer.from(
|
|
24
|
+
"6f71a512b1e035eaab53d8be73120d3fb68a0ca346b9560aab3e5cdf753d5e98",
|
|
25
|
+
"hex",
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
/** Browser identity used for the challenge handshake and the retry. */
|
|
29
|
+
const BROWSER_USER_AGENT =
|
|
30
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36";
|
|
31
|
+
|
|
32
|
+
const BROWSER_VERSION = "144";
|
|
33
|
+
|
|
34
|
+
const SCREEN = "1920-1080-1080-24-*-*-*";
|
|
35
|
+
|
|
36
|
+
const PLUGINS = [
|
|
37
|
+
{ name: "PDF Viewer", str: "PDF Viewer " },
|
|
38
|
+
{ name: "Chrome PDF Viewer", str: "Chrome PDF Viewer " },
|
|
39
|
+
{ name: "Chromium PDF Viewer", str: "Chromium PDF Viewer " },
|
|
40
|
+
{ name: "Microsoft Edge PDF Viewer", str: "Microsoft Edge PDF Viewer " },
|
|
41
|
+
{ name: "WebKit built-in PDF", str: "WebKit built-in PDF " },
|
|
42
|
+
];
|
|
43
|
+
|
|
44
|
+
const PLUGIN_STRING = PLUGINS.map((plugin) => plugin.str).join("");
|
|
45
|
+
|
|
46
|
+
const MATH_VALUES = {
|
|
47
|
+
tan: "-1.4214488238747245",
|
|
48
|
+
sin: "0.8178819121159085",
|
|
49
|
+
cos: "-0.5753861119575491",
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const GPU_EXTENSIONS = [
|
|
53
|
+
"ANGLE_instanced_arrays",
|
|
54
|
+
"EXT_blend_minmax",
|
|
55
|
+
"EXT_clip_control",
|
|
56
|
+
"EXT_color_buffer_half_float",
|
|
57
|
+
"EXT_depth_clamp",
|
|
58
|
+
"EXT_disjoint_timer_query",
|
|
59
|
+
"EXT_float_blend",
|
|
60
|
+
"EXT_frag_depth",
|
|
61
|
+
"EXT_polygon_offset_clamp",
|
|
62
|
+
"EXT_shader_texture_lod",
|
|
63
|
+
"EXT_texture_compression_bptc",
|
|
64
|
+
"EXT_texture_compression_rgtc",
|
|
65
|
+
"EXT_texture_filter_anisotropic",
|
|
66
|
+
"EXT_texture_mirror_clamp_to_edge",
|
|
67
|
+
"EXT_sRGB",
|
|
68
|
+
"KHR_parallel_shader_compile",
|
|
69
|
+
"OES_element_index_uint",
|
|
70
|
+
"OES_fbo_render_mipmap",
|
|
71
|
+
"OES_standard_derivatives",
|
|
72
|
+
"OES_texture_float",
|
|
73
|
+
"OES_texture_float_linear",
|
|
74
|
+
"OES_texture_half_float",
|
|
75
|
+
"OES_texture_half_float_linear",
|
|
76
|
+
"OES_vertex_array_object",
|
|
77
|
+
"WEBGL_blend_func_extended",
|
|
78
|
+
"WEBGL_color_buffer_float",
|
|
79
|
+
"WEBGL_compressed_texture_astc",
|
|
80
|
+
"WEBGL_compressed_texture_etc",
|
|
81
|
+
"WEBGL_compressed_texture_etc1",
|
|
82
|
+
"WEBGL_compressed_texture_s3tc",
|
|
83
|
+
"WEBGL_compressed_texture_s3tc_srgb",
|
|
84
|
+
"WEBGL_debug_renderer_info",
|
|
85
|
+
"WEBGL_debug_shaders",
|
|
86
|
+
"WEBGL_depth_texture",
|
|
87
|
+
"WEBGL_lose_context",
|
|
88
|
+
"WEBGL_multi_draw",
|
|
89
|
+
"WEBGL_polygon_mode",
|
|
90
|
+
];
|
|
91
|
+
|
|
92
|
+
/** Maps challenge type hashes to their proof-of-work family. */
|
|
93
|
+
function normalizeChallengeType(value: string): string {
|
|
94
|
+
if (value.startsWith("ha9faaffd")) return "NetworkBandwidth";
|
|
95
|
+
|
|
96
|
+
if (value.startsWith("h72f957df")) return "HashcashScrypt";
|
|
97
|
+
|
|
98
|
+
if (value.startsWith("h7b0c470f")) return "SHA256";
|
|
99
|
+
|
|
100
|
+
return value;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Returns the submission endpoint for a challenge type. */
|
|
104
|
+
function challengeEndpoint(challengeType: string): string {
|
|
105
|
+
return challengeType === "NetworkBandwidth" ? "mp_verify" : "verify";
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const CHALLENGE_SAME_ORIGIN = /(\/__challenge_[A-Za-z0-9]+\/[a-f0-9]+\/[a-f0-9]+)/;
|
|
109
|
+
|
|
110
|
+
const CHALLENGE_EXTERNAL =
|
|
111
|
+
/(https:\/\/[a-z0-9]+\.[a-z0-9]+\.[a-z0-9-]+\.token\.awswaf\.com\/[^/\s"]+\/[^/\s"]+\/[^/\s"]+)/;
|
|
112
|
+
|
|
113
|
+
const CHALLENGE_SDK =
|
|
114
|
+
/(https:\/\/[a-z0-9]+\.edge\.sdk\.awswaf\.com\/[a-z0-9]+\/[a-z0-9]+)\/challenge\.js/;
|
|
115
|
+
|
|
116
|
+
const GOKU_PROPS = /window\.gokuProps\s*=\s*(\{[^}]+\})/;
|
|
117
|
+
|
|
118
|
+
type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
|
|
119
|
+
|
|
120
|
+
type JsonObject = { [key: string]: JsonValue };
|
|
121
|
+
|
|
122
|
+
interface TextResponse {
|
|
123
|
+
url: URL;
|
|
124
|
+
status: number;
|
|
125
|
+
text: string;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
interface FingerprintMetrics {
|
|
129
|
+
fp2: number;
|
|
130
|
+
browser: number;
|
|
131
|
+
capabilities: number;
|
|
132
|
+
gpu: number;
|
|
133
|
+
dnt: number;
|
|
134
|
+
math: number;
|
|
135
|
+
screen: number;
|
|
136
|
+
navigator: number;
|
|
137
|
+
auto: number;
|
|
138
|
+
stealth: number;
|
|
139
|
+
subtle: number;
|
|
140
|
+
canvas: number;
|
|
141
|
+
formdetector: number;
|
|
142
|
+
be: number;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
interface Collector {
|
|
146
|
+
name: keyof FingerprintMetrics;
|
|
147
|
+
metric: string;
|
|
148
|
+
low: number;
|
|
149
|
+
high: number;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const COLLECTORS: readonly Collector[] = [
|
|
153
|
+
{ name: "fp2", metric: "100", low: 0.5, high: 3 },
|
|
154
|
+
{ name: "browser", metric: "101", low: 0, high: 1 },
|
|
155
|
+
{ name: "capabilities", metric: "102", low: 2, high: 8 },
|
|
156
|
+
{ name: "gpu", metric: "103", low: 3, high: 12 },
|
|
157
|
+
{ name: "dnt", metric: "104", low: 0, high: 1 },
|
|
158
|
+
{ name: "math", metric: "105", low: 0, high: 1 },
|
|
159
|
+
{ name: "screen", metric: "106", low: 0, high: 1 },
|
|
160
|
+
{ name: "navigator", metric: "107", low: 0, high: 1 },
|
|
161
|
+
{ name: "auto", metric: "108", low: 0, high: 1 },
|
|
162
|
+
{ name: "stealth", metric: "undefined", low: 1, high: 4 },
|
|
163
|
+
{ name: "subtle", metric: "110", low: 0, high: 1 },
|
|
164
|
+
{ name: "canvas", metric: "111", low: 80, high: 200 },
|
|
165
|
+
{ name: "formdetector", metric: "112", low: 0, high: 3 },
|
|
166
|
+
{ name: "be", metric: "undefined", low: 0, high: 1 },
|
|
167
|
+
];
|
|
168
|
+
|
|
169
|
+
interface ChallengeData {
|
|
170
|
+
input: string;
|
|
171
|
+
hmac?: string;
|
|
172
|
+
region?: string;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
interface ChallengeInputs {
|
|
176
|
+
challenge: ChallengeData;
|
|
177
|
+
challengeType: string;
|
|
178
|
+
difficulty: number;
|
|
179
|
+
memory: number;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
interface BuiltMetrics {
|
|
183
|
+
metrics: JsonValue[];
|
|
184
|
+
fingerprint: FingerprintMetrics;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Returns whether a value is a plain JSON object. */
|
|
188
|
+
// oxlint-disable-next-line anti-slop/no-unknown-parameters -- parses untrusted challenge JSON at the I/O boundary
|
|
189
|
+
function isJsonObject(value: unknown): value is JsonObject {
|
|
190
|
+
// oxlint-disable-next-line anti-slop/no-runtime-typeof -- validates untrusted challenge JSON before use
|
|
191
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Parses untrusted JSON text into a plain object. */
|
|
195
|
+
function parseJsonObject(text: string): JsonObject | undefined {
|
|
196
|
+
try {
|
|
197
|
+
const parsed: unknown = JSON.parse(text);
|
|
198
|
+
|
|
199
|
+
return isJsonObject(parsed) ? parsed : undefined;
|
|
200
|
+
} catch {
|
|
201
|
+
return undefined;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Reads a string field, returning undefined for missing or non-string values. */
|
|
206
|
+
function stringField(value: JsonObject, key: string): string | undefined {
|
|
207
|
+
const field = value[key];
|
|
208
|
+
|
|
209
|
+
// oxlint-disable-next-line anti-slop/no-runtime-typeof -- validates an untrusted JSON field
|
|
210
|
+
return typeof field === "string" ? field : undefined;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Reads a numeric field, returning undefined for missing or non-numeric values. */
|
|
214
|
+
function numberField(value: JsonObject, key: string): number | undefined {
|
|
215
|
+
const field = value[key];
|
|
216
|
+
|
|
217
|
+
// oxlint-disable-next-line anti-slop/no-runtime-typeof -- validates an untrusted JSON field
|
|
218
|
+
return typeof field === "number" && Number.isFinite(field) ? field : undefined;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** Returns a random floating-point metric in the supplied range. */
|
|
222
|
+
function randomMetric(low: number, high: number): number {
|
|
223
|
+
return Math.round((low + Math.random() * (high - low)) * 10) / 10;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Builds realistic-looking WAF metrics and the matching fingerprint metric map. */
|
|
227
|
+
function buildMetrics(hasToken: boolean): BuiltMetrics {
|
|
228
|
+
const fingerprint: FingerprintMetrics = {
|
|
229
|
+
fp2: 0,
|
|
230
|
+
browser: 0,
|
|
231
|
+
capabilities: 0,
|
|
232
|
+
gpu: 0,
|
|
233
|
+
dnt: 0,
|
|
234
|
+
math: 0,
|
|
235
|
+
screen: 0,
|
|
236
|
+
navigator: 0,
|
|
237
|
+
auto: 0,
|
|
238
|
+
stealth: 0,
|
|
239
|
+
subtle: 0,
|
|
240
|
+
canvas: 0,
|
|
241
|
+
formdetector: 0,
|
|
242
|
+
be: 0,
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
const metrics: JsonValue[] = [];
|
|
246
|
+
|
|
247
|
+
for (const collector of COLLECTORS) {
|
|
248
|
+
const value = randomMetric(collector.low, collector.high);
|
|
249
|
+
|
|
250
|
+
fingerprint[collector.name] = Math.trunc(value);
|
|
251
|
+
metrics.push({ name: collector.metric, value, unit: "2" });
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const encoding = randomMetric(0.5, 3);
|
|
255
|
+
const encryption = randomMetric(2, 8);
|
|
256
|
+
|
|
257
|
+
const acquisition =
|
|
258
|
+
COLLECTORS.reduce((total, collector) => total + fingerprint[collector.name], 0) +
|
|
259
|
+
encoding +
|
|
260
|
+
encryption +
|
|
261
|
+
randomMetric(2, 6);
|
|
262
|
+
|
|
263
|
+
const challenge = randomMetric(2, 8);
|
|
264
|
+
const cookie = randomMetric(0.1, 1);
|
|
265
|
+
const total = acquisition + challenge + cookie;
|
|
266
|
+
|
|
267
|
+
metrics.push(
|
|
268
|
+
{ name: "2", value: encoding, unit: "2" },
|
|
269
|
+
{ name: "3", value: encryption, unit: "2" },
|
|
270
|
+
{ name: "7", value: hasToken ? 1 : 0, unit: "4" },
|
|
271
|
+
{ name: "1", value: acquisition, unit: "2" },
|
|
272
|
+
{ name: "4", value: challenge, unit: "2" },
|
|
273
|
+
{ name: "5", value: cookie, unit: "2" },
|
|
274
|
+
{ name: "6", value: total, unit: "2" },
|
|
275
|
+
{ name: "8", value: 1, unit: "4" },
|
|
276
|
+
);
|
|
277
|
+
|
|
278
|
+
return { metrics, fingerprint };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Builds a browser fingerprint object accepted by the WAF signal decoder. */
|
|
282
|
+
function buildSignal(site: string, fingerprint: FingerprintMetrics) {
|
|
283
|
+
const now = Date.now();
|
|
284
|
+
const canvasHash = randomInt(100_000_000, 1_000_000_000);
|
|
285
|
+
const histogram = Array.from({ length: 256 }, () => randomInt(1, 50));
|
|
286
|
+
|
|
287
|
+
return {
|
|
288
|
+
metrics: { ...fingerprint },
|
|
289
|
+
start: now,
|
|
290
|
+
flashVersion: null,
|
|
291
|
+
plugins: PLUGINS.map((plugin) => ({ name: plugin.name, str: plugin.str })),
|
|
292
|
+
dupedPlugins: `${PLUGIN_STRING}||${SCREEN}`,
|
|
293
|
+
screenInfo: SCREEN,
|
|
294
|
+
referrer: "",
|
|
295
|
+
userAgent: BROWSER_USER_AGENT,
|
|
296
|
+
location: `${site}/`,
|
|
297
|
+
webDriver: false,
|
|
298
|
+
capabilities: {
|
|
299
|
+
css: {
|
|
300
|
+
textShadow: 1,
|
|
301
|
+
WebkitTextStroke: 1,
|
|
302
|
+
boxShadow: 1,
|
|
303
|
+
borderRadius: 1,
|
|
304
|
+
borderImage: 1,
|
|
305
|
+
opacity: 1,
|
|
306
|
+
transform: 1,
|
|
307
|
+
transition: 1,
|
|
308
|
+
},
|
|
309
|
+
js: {
|
|
310
|
+
audio: true,
|
|
311
|
+
geolocation: true,
|
|
312
|
+
localStorage: "supported",
|
|
313
|
+
touch: false,
|
|
314
|
+
video: true,
|
|
315
|
+
webWorker: true,
|
|
316
|
+
},
|
|
317
|
+
elapsed: fingerprint.capabilities,
|
|
318
|
+
},
|
|
319
|
+
gpu: {
|
|
320
|
+
vendor: "Google Inc. (NVIDIA Corporation)",
|
|
321
|
+
model: "ANGLE (NVIDIA Corporation, NVIDIA GeForce RTX 5070/PCIe/SSE2, OpenGL ES 3.2)",
|
|
322
|
+
extensions: GPU_EXTENSIONS,
|
|
323
|
+
},
|
|
324
|
+
dnt: null,
|
|
325
|
+
math: MATH_VALUES,
|
|
326
|
+
automation: {
|
|
327
|
+
wd: { properties: { document: [], window: [], navigator: [] } },
|
|
328
|
+
phantom: { properties: { window: [] } },
|
|
329
|
+
},
|
|
330
|
+
stealth: { t1: 0, t2: 0, i: 1, mte: 0, mtd: false },
|
|
331
|
+
crypto: {
|
|
332
|
+
crypto: 1,
|
|
333
|
+
subtle: 1,
|
|
334
|
+
encrypt: true,
|
|
335
|
+
decrypt: true,
|
|
336
|
+
wrapKey: true,
|
|
337
|
+
unwrapKey: true,
|
|
338
|
+
sign: true,
|
|
339
|
+
verify: true,
|
|
340
|
+
digest: true,
|
|
341
|
+
deriveBits: true,
|
|
342
|
+
deriveKey: true,
|
|
343
|
+
getRandomValues: true,
|
|
344
|
+
randomUUID: true,
|
|
345
|
+
},
|
|
346
|
+
canvas: { hash: canvasHash, emailHash: null, histogramBins: histogram },
|
|
347
|
+
formDetected: false,
|
|
348
|
+
numForms: 0,
|
|
349
|
+
numFormElements: 0,
|
|
350
|
+
be: { si: false },
|
|
351
|
+
end: now + 1,
|
|
352
|
+
errors: [],
|
|
353
|
+
version: "2.4.0",
|
|
354
|
+
id: randomUUID(),
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/** Builds and encrypts the browser signal payload. */
|
|
359
|
+
function encodeSignal(site: string, hasToken: boolean) {
|
|
360
|
+
const built = buildMetrics(hasToken);
|
|
361
|
+
const signal = buildSignal(site, built.fingerprint);
|
|
362
|
+
const plaintextJson = JSON.stringify(signal);
|
|
363
|
+
const checksum = crc32(plaintextJson).toString(16).toUpperCase().padStart(8, "0");
|
|
364
|
+
const iv = randomBytes(12);
|
|
365
|
+
const cipher = createCipheriv("aes-256-gcm", SIGNAL_KEY, iv);
|
|
366
|
+
|
|
367
|
+
const ciphertext = Buffer.concat([
|
|
368
|
+
cipher.update(`${checksum}#${plaintextJson}`, "utf8"),
|
|
369
|
+
cipher.final(),
|
|
370
|
+
]);
|
|
371
|
+
|
|
372
|
+
const tag = cipher.getAuthTag();
|
|
373
|
+
|
|
374
|
+
return {
|
|
375
|
+
encrypted: `${iv.toString("base64")}::${tag.toString("hex")}::${ciphertext.toString("hex")}`,
|
|
376
|
+
checksum,
|
|
377
|
+
metrics: built.metrics,
|
|
378
|
+
signal,
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/** Builds the signal entry used in challenge submissions. */
|
|
383
|
+
function signalEntry(encrypted: string): JsonValue[] {
|
|
384
|
+
return [{ name: "Zoey", value: { Present: encrypted } }];
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** Run CPU-intensive work off the main thread and terminate it on cancellation. */
|
|
388
|
+
function solveAwsWafChallenge(
|
|
389
|
+
inputs: ChallengeInputs,
|
|
390
|
+
checksum: string,
|
|
391
|
+
signal: AbortSignal,
|
|
392
|
+
): Promise<string | undefined> {
|
|
393
|
+
signal.throwIfAborted();
|
|
394
|
+
|
|
395
|
+
const work: AwsWafWork = {
|
|
396
|
+
type: inputs.challengeType,
|
|
397
|
+
input: inputs.challenge.input,
|
|
398
|
+
checksum,
|
|
399
|
+
difficulty: inputs.difficulty,
|
|
400
|
+
memory: inputs.memory,
|
|
401
|
+
};
|
|
402
|
+
|
|
403
|
+
const worker = new Worker(new URL("../../workers/aws-waf-worker.js", import.meta.url), {
|
|
404
|
+
workerData: work,
|
|
405
|
+
resourceLimits: { maxOldGenerationSizeMb: 64, maxYoungGenerationSizeMb: 16 },
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
return new Promise((resolve, reject) => {
|
|
409
|
+
let settled = false;
|
|
410
|
+
|
|
411
|
+
const timer = setTimeout(
|
|
412
|
+
() => finish(() => reject(new Error("AWS WAF worker timed out."))),
|
|
413
|
+
60_000,
|
|
414
|
+
);
|
|
415
|
+
|
|
416
|
+
function finish(callback: () => void): void {
|
|
417
|
+
if (settled) return;
|
|
418
|
+
settled = true;
|
|
419
|
+
clearTimeout(timer);
|
|
420
|
+
signal.removeEventListener("abort", onAbort);
|
|
421
|
+
void worker.terminate();
|
|
422
|
+
callback();
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
function onAbort(): void {
|
|
426
|
+
finish(() => reject(signal.reason));
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
worker.on(
|
|
430
|
+
"message",
|
|
431
|
+
(message: { kind: "solved"; solution?: string } | { kind: "error"; message: string }) => {
|
|
432
|
+
if (message.kind === "solved") finish(() => resolve(message.solution));
|
|
433
|
+
else finish(() => reject(new Error(message.message)));
|
|
434
|
+
},
|
|
435
|
+
);
|
|
436
|
+
worker.on("error", (error) => finish(() => reject(error)));
|
|
437
|
+
worker.on("exit", (code) =>
|
|
438
|
+
finish(() => reject(new Error(`AWS WAF worker exited with code ${code}.`))),
|
|
439
|
+
);
|
|
440
|
+
|
|
441
|
+
if (signal.aborted) onAbort();
|
|
442
|
+
else signal.addEventListener("abort", onAbort, { once: true });
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/** Parses the base64 challenge input into challenge metadata. */
|
|
447
|
+
function decodeChallengeInput(input: string): JsonObject | undefined {
|
|
448
|
+
try {
|
|
449
|
+
return parseJsonObject(Buffer.from(input, "base64").toString("utf8"));
|
|
450
|
+
} catch {
|
|
451
|
+
return undefined;
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/** Reads challenge inputs from the `/inputs` response. */
|
|
456
|
+
function parseChallengeInputs(value: JsonObject): ChallengeInputs | undefined {
|
|
457
|
+
const challenge = value["challenge"];
|
|
458
|
+
|
|
459
|
+
if (!isJsonObject(challenge)) return undefined;
|
|
460
|
+
|
|
461
|
+
const input = stringField(challenge, "input");
|
|
462
|
+
|
|
463
|
+
if (!input) return undefined;
|
|
464
|
+
|
|
465
|
+
const decoded = decodeChallengeInput(input);
|
|
466
|
+
|
|
467
|
+
const challengeType =
|
|
468
|
+
(decoded ? stringField(decoded, "challenge_type") : undefined) ??
|
|
469
|
+
stringField(value, "challenge_type");
|
|
470
|
+
|
|
471
|
+
if (!challengeType) return undefined;
|
|
472
|
+
|
|
473
|
+
const difficulty =
|
|
474
|
+
(decoded ? numberField(decoded, "difficulty") : undefined) ??
|
|
475
|
+
numberField(value, "difficulty") ??
|
|
476
|
+
1;
|
|
477
|
+
|
|
478
|
+
const memory =
|
|
479
|
+
(decoded ? numberField(decoded, "memory") : undefined) ?? numberField(value, "memory") ?? 128;
|
|
480
|
+
|
|
481
|
+
return {
|
|
482
|
+
challenge: {
|
|
483
|
+
input,
|
|
484
|
+
hmac: stringField(challenge, "hmac"),
|
|
485
|
+
region: stringField(challenge, "region"),
|
|
486
|
+
},
|
|
487
|
+
challengeType: normalizeChallengeType(challengeType),
|
|
488
|
+
difficulty,
|
|
489
|
+
memory,
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/** Extracts the challenge URL and goku properties from a WAF page. */
|
|
494
|
+
export function extractAwsWafChallenge(
|
|
495
|
+
html: string,
|
|
496
|
+
site: URL,
|
|
497
|
+
): { challengeUrl: string; gokuProps?: JsonObject } | undefined {
|
|
498
|
+
const sameOrigin = CHALLENGE_SAME_ORIGIN.exec(html);
|
|
499
|
+
const external = CHALLENGE_EXTERNAL.exec(html);
|
|
500
|
+
const sdk = CHALLENGE_SDK.exec(html);
|
|
501
|
+
const match = sameOrigin ?? external ?? sdk;
|
|
502
|
+
|
|
503
|
+
if (!match) return undefined;
|
|
504
|
+
|
|
505
|
+
const challengeUrl = sameOrigin ? new URL(match[1], site).href : external ? match[1] : match[1];
|
|
506
|
+
const gokuMatch = GOKU_PROPS.exec(html);
|
|
507
|
+
const gokuProps = gokuMatch ? parseJsonObject(gokuMatch[1]) : undefined;
|
|
508
|
+
|
|
509
|
+
return { challengeUrl, gokuProps };
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/** Builds the browser headers used to request the protected page. */
|
|
513
|
+
function browserHeaders() {
|
|
514
|
+
return {
|
|
515
|
+
accept:
|
|
516
|
+
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
517
|
+
"accept-language": "en-US,en;q=0.9",
|
|
518
|
+
"sec-ch-ua": `"Not:A-Brand";v="8", "Chromium";v="${BROWSER_VERSION}", "Google Chrome";v="${BROWSER_VERSION}"`,
|
|
519
|
+
"sec-ch-ua-mobile": "?0",
|
|
520
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
521
|
+
"sec-fetch-dest": "document",
|
|
522
|
+
"sec-fetch-mode": "navigate",
|
|
523
|
+
"sec-fetch-site": "none",
|
|
524
|
+
"sec-fetch-user": "?1",
|
|
525
|
+
"upgrade-insecure-requests": "1",
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/** Builds the headers used for challenge API calls. */
|
|
530
|
+
function apiHeaders(site: string) {
|
|
531
|
+
return {
|
|
532
|
+
accept: "*/*",
|
|
533
|
+
"accept-language": "en-US,en;q=0.9",
|
|
534
|
+
"cache-control": "no-cache",
|
|
535
|
+
origin: site,
|
|
536
|
+
pragma: "no-cache",
|
|
537
|
+
priority: "u=1, i",
|
|
538
|
+
referer: `${site}/`,
|
|
539
|
+
"sec-ch-ua": `"Not:A-Brand";v="8", "Chromium";v="${BROWSER_VERSION}", "Google Chrome";v="${BROWSER_VERSION}"`,
|
|
540
|
+
"sec-ch-ua-mobile": "?0",
|
|
541
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
542
|
+
"sec-fetch-dest": "empty",
|
|
543
|
+
"sec-fetch-mode": "cors",
|
|
544
|
+
"sec-fetch-site": "cross-site",
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/** Reads and decodes a bounded response body. */
|
|
549
|
+
async function readText(response: IncomingMessage, signal: AbortSignal): Promise<string> {
|
|
550
|
+
const bytes = await readResponseBytes(response, FETCH_MAX_BYTES, signal);
|
|
551
|
+
const decompressed = decompressResponse(bytes, responseHeader(response, "content-encoding"));
|
|
552
|
+
|
|
553
|
+
return decodeResponse(decompressed, responseHeader(response, "content-type") ?? "text/plain");
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/** Performs a GET through the validated redirect and cookie boundary. */
|
|
557
|
+
async function getText(
|
|
558
|
+
url: string,
|
|
559
|
+
network: RedirectDependencies,
|
|
560
|
+
signal: AbortSignal,
|
|
561
|
+
headers: Readonly<Record<string, string>>,
|
|
562
|
+
): Promise<TextResponse> {
|
|
563
|
+
const { target, response } = await requestFollowingRedirects(url, signal, network, headers);
|
|
564
|
+
const text = await readText(response, signal);
|
|
565
|
+
|
|
566
|
+
return { url: target.url, status: response.statusCode ?? 0, text };
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/** Performs a POST to a validated target without following redirects. */
|
|
570
|
+
async function postText(
|
|
571
|
+
url: string,
|
|
572
|
+
network: RedirectDependencies,
|
|
573
|
+
signal: AbortSignal,
|
|
574
|
+
headers: Readonly<Record<string, string>>,
|
|
575
|
+
body: string,
|
|
576
|
+
): Promise<TextResponse> {
|
|
577
|
+
const validateUrl = network.validateUrl ?? validateRemoteUrl;
|
|
578
|
+
const target = await validateUrl(url);
|
|
579
|
+
const cookieHeader = network.cookies?.headerFor(target.url);
|
|
580
|
+
|
|
581
|
+
const requestHeaders = { ...headers };
|
|
582
|
+
|
|
583
|
+
if (network.userAgent) requestHeaders["User-Agent"] = network.userAgent;
|
|
584
|
+
|
|
585
|
+
if (cookieHeader) requestHeaders.Cookie = cookieHeader;
|
|
586
|
+
|
|
587
|
+
const request =
|
|
588
|
+
network.request ??
|
|
589
|
+
((value, requestSignal, requestHeadersInit, options) =>
|
|
590
|
+
requestPinned(value, requestSignal, {
|
|
591
|
+
headers: requestHeadersInit,
|
|
592
|
+
method: options?.method,
|
|
593
|
+
body: options?.body,
|
|
594
|
+
}));
|
|
595
|
+
|
|
596
|
+
const response = await request(target, signal, requestHeaders, { method: "POST", body });
|
|
597
|
+
const text = await readText(response, signal);
|
|
598
|
+
|
|
599
|
+
return { url: target.url, status: response.statusCode ?? 0, text };
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/** Builds the multipart body used by bandwidth challenges. */
|
|
603
|
+
function buildMultipart(
|
|
604
|
+
domain: string,
|
|
605
|
+
challenge: ChallengeData,
|
|
606
|
+
solution: string,
|
|
607
|
+
checksum: string,
|
|
608
|
+
encrypted: string,
|
|
609
|
+
metrics: JsonValue[],
|
|
610
|
+
gokuProps: JsonObject | undefined,
|
|
611
|
+
) {
|
|
612
|
+
const boundary = `----WebKitFormBoundary${randomBytes(16).toString("hex")}`;
|
|
613
|
+
|
|
614
|
+
const metadata = {
|
|
615
|
+
challenge: {
|
|
616
|
+
input: challenge.input,
|
|
617
|
+
hmac: challenge.hmac ?? null,
|
|
618
|
+
region: challenge.region ?? null,
|
|
619
|
+
},
|
|
620
|
+
solution: null,
|
|
621
|
+
signals: signalEntry(encrypted),
|
|
622
|
+
checksum,
|
|
623
|
+
existing_token: null,
|
|
624
|
+
client: "Browser",
|
|
625
|
+
domain,
|
|
626
|
+
metrics,
|
|
627
|
+
goku_props: gokuProps ?? null,
|
|
628
|
+
};
|
|
629
|
+
|
|
630
|
+
const body = [
|
|
631
|
+
`--${boundary}`,
|
|
632
|
+
'Content-Disposition: form-data; name="solution_data"',
|
|
633
|
+
"",
|
|
634
|
+
solution,
|
|
635
|
+
`--${boundary}`,
|
|
636
|
+
'Content-Disposition: form-data; name="solution_metadata"',
|
|
637
|
+
"",
|
|
638
|
+
JSON.stringify(metadata),
|
|
639
|
+
`--${boundary}--`,
|
|
640
|
+
"",
|
|
641
|
+
].join("\r\n");
|
|
642
|
+
|
|
643
|
+
return { body, contentType: `multipart/form-data; boundary=${boundary}` };
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
/** Builds the JSON body used by verify challenges. */
|
|
647
|
+
function buildVerificationPayload(
|
|
648
|
+
domain: string,
|
|
649
|
+
challenge: ChallengeData,
|
|
650
|
+
solution: string,
|
|
651
|
+
checksum: string,
|
|
652
|
+
encrypted: string,
|
|
653
|
+
metrics: JsonValue[],
|
|
654
|
+
existingToken: string | undefined,
|
|
655
|
+
gokuProps: JsonObject | undefined,
|
|
656
|
+
) {
|
|
657
|
+
return {
|
|
658
|
+
challenge: {
|
|
659
|
+
input: challenge.input,
|
|
660
|
+
hmac: challenge.hmac ?? null,
|
|
661
|
+
region: challenge.region ?? null,
|
|
662
|
+
},
|
|
663
|
+
solution,
|
|
664
|
+
signals: signalEntry(encrypted),
|
|
665
|
+
checksum,
|
|
666
|
+
existing_token: existingToken ?? null,
|
|
667
|
+
client: "Browser",
|
|
668
|
+
domain,
|
|
669
|
+
metrics,
|
|
670
|
+
goku_props: gokuProps ?? null,
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/** Reads a token from a challenge response, including nested response objects. */
|
|
675
|
+
function parseToken(value: JsonObject): string | undefined {
|
|
676
|
+
const direct = stringField(value, "token");
|
|
677
|
+
|
|
678
|
+
if (direct) return direct;
|
|
679
|
+
|
|
680
|
+
const nested = value["response"];
|
|
681
|
+
|
|
682
|
+
return isJsonObject(nested) ? stringField(nested, "token") : undefined;
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/** Reads replacement challenge inputs from a challenge response. */
|
|
686
|
+
function parseReplacementInputs(value: JsonObject): ChallengeInputs | undefined {
|
|
687
|
+
const nested = value["response"];
|
|
688
|
+
|
|
689
|
+
if (isJsonObject(nested)) {
|
|
690
|
+
const inputs = nested["inputs"];
|
|
691
|
+
|
|
692
|
+
if (isJsonObject(inputs)) return parseChallengeInputs(inputs);
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
const inputs = value["inputs"];
|
|
696
|
+
|
|
697
|
+
return isJsonObject(inputs) ? parseChallengeInputs(inputs) : undefined;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
/** Runs the native AWS WAF challenge flow and returns a token. */
|
|
701
|
+
async function resolveAwsWafNative(
|
|
702
|
+
context: AwsWafResolveContext,
|
|
703
|
+
): Promise<AwsWafSolution | undefined> {
|
|
704
|
+
const network: RedirectDependencies = {
|
|
705
|
+
...context.network,
|
|
706
|
+
userAgent: BROWSER_USER_AGENT,
|
|
707
|
+
persistentHeaders: browserHeaders(),
|
|
708
|
+
cookies: context.network?.cookies ?? new CookieJar(),
|
|
709
|
+
};
|
|
710
|
+
|
|
711
|
+
const site = context.url.origin;
|
|
712
|
+
const domain = context.url.hostname;
|
|
713
|
+
|
|
714
|
+
const page = await getText(context.url.href, network, context.signal, browserHeaders());
|
|
715
|
+
|
|
716
|
+
if (page.status !== 200 && page.status !== 202) return undefined;
|
|
717
|
+
|
|
718
|
+
const discovery = extractAwsWafChallenge(page.text, context.url);
|
|
719
|
+
|
|
720
|
+
if (!discovery) return undefined;
|
|
721
|
+
|
|
722
|
+
const inputsResponse = await getText(
|
|
723
|
+
`${discovery.challengeUrl}/inputs?client=browser`,
|
|
724
|
+
network,
|
|
725
|
+
context.signal,
|
|
726
|
+
apiHeaders(site),
|
|
727
|
+
);
|
|
728
|
+
|
|
729
|
+
if (inputsResponse.status !== 200) return undefined;
|
|
730
|
+
|
|
731
|
+
let inputs = parseChallengeInputs(parseJsonObject(inputsResponse.text) ?? {});
|
|
732
|
+
|
|
733
|
+
if (!inputs) return undefined;
|
|
734
|
+
|
|
735
|
+
let token: string | undefined;
|
|
736
|
+
|
|
737
|
+
for (let round = 0; round < 2; round += 1) {
|
|
738
|
+
const signal = encodeSignal(site, token !== undefined);
|
|
739
|
+
const solution = await solveAwsWafChallenge(inputs, signal.checksum, context.signal);
|
|
740
|
+
|
|
741
|
+
if (!solution) return undefined;
|
|
742
|
+
|
|
743
|
+
const endpoint = challengeEndpoint(inputs.challengeType);
|
|
744
|
+
let body: string;
|
|
745
|
+
let contentType = "text/plain;charset=UTF-8";
|
|
746
|
+
|
|
747
|
+
if (endpoint === "mp_verify") {
|
|
748
|
+
const multipart = buildMultipart(
|
|
749
|
+
domain,
|
|
750
|
+
inputs.challenge,
|
|
751
|
+
solution,
|
|
752
|
+
signal.checksum,
|
|
753
|
+
signal.encrypted,
|
|
754
|
+
signal.metrics,
|
|
755
|
+
discovery.gokuProps,
|
|
756
|
+
);
|
|
757
|
+
|
|
758
|
+
body = multipart.body;
|
|
759
|
+
contentType = multipart.contentType;
|
|
760
|
+
} else {
|
|
761
|
+
body = JSON.stringify(
|
|
762
|
+
buildVerificationPayload(
|
|
763
|
+
domain,
|
|
764
|
+
inputs.challenge,
|
|
765
|
+
solution,
|
|
766
|
+
signal.checksum,
|
|
767
|
+
signal.encrypted,
|
|
768
|
+
signal.metrics,
|
|
769
|
+
token,
|
|
770
|
+
discovery.gokuProps,
|
|
771
|
+
),
|
|
772
|
+
);
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
const headers = { ...apiHeaders(site), "content-type": contentType };
|
|
776
|
+
|
|
777
|
+
const response = await postText(
|
|
778
|
+
`${discovery.challengeUrl}/${endpoint}`,
|
|
779
|
+
network,
|
|
780
|
+
context.signal,
|
|
781
|
+
headers,
|
|
782
|
+
body,
|
|
783
|
+
);
|
|
784
|
+
|
|
785
|
+
if (response.status !== 200) return undefined;
|
|
786
|
+
|
|
787
|
+
const parsed = parseJsonObject(response.text);
|
|
788
|
+
|
|
789
|
+
if (!parsed) return undefined;
|
|
790
|
+
|
|
791
|
+
token = parseToken(parsed) ?? token;
|
|
792
|
+
|
|
793
|
+
if (token) break;
|
|
794
|
+
|
|
795
|
+
const replacement = parseReplacementInputs(parsed);
|
|
796
|
+
|
|
797
|
+
if (replacement) inputs = replacement;
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
return token ? { token, userAgent: BROWSER_USER_AGENT } : undefined;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
/** Native in-process AWS WAF resolver. */
|
|
804
|
+
export const nativeAwsWafResolver: AwsWafResolver = {
|
|
805
|
+
resolve: resolveAwsWafNative,
|
|
806
|
+
};
|