@zeldrisho/pi-web-fetch 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -1
- package/README.md +2 -4
- package/package.json +1 -1
- package/src/acquisition/fetch.ts +86 -1
- package/src/acquisition/llms.ts +1 -1
- package/src/challenges/aws-waf-resolver.ts +806 -0
- package/src/challenges/aws-waf-solver.ts +65 -0
- package/src/challenges/aws-waf-worker.ts +14 -0
- package/src/challenges/aws-waf.ts +34 -0
- package/src/index.ts +7 -0
- package/src/network/cookies.ts +41 -0
- package/src/network/network-redirects.ts +15 -2
- package/src/network/network-transport.ts +21 -3
- package/workers/aws-waf-solver.js +56 -0
- package/workers/aws-waf-worker.js +14 -0
package/CHANGELOG.md
CHANGED
|
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
|
|
8
8
|
## [Unreleased]
|
|
9
9
|
|
|
10
|
+
## [0.16.0] - 2026-10-28
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
|
|
14
|
+
- Automatically resolve supported AWS WAF challenges when fetching pages.
|
|
15
|
+
|
|
10
16
|
## [0.15.0] - 2026-10-28
|
|
11
17
|
|
|
12
18
|
### Added
|
|
@@ -289,7 +295,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
289
295
|
|
|
290
296
|
- Add bounded public page fetching ([b979496](https://github.com/zeldrisho/pi-packages/commit/b979496b32de1cead172ba570307e4a4a7b3421d))
|
|
291
297
|
|
|
292
|
-
[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.
|
|
298
|
+
[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.16.0...HEAD
|
|
299
|
+
[0.16.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.15.0...pi-web-fetch-v0.16.0
|
|
293
300
|
[0.15.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.14.1...pi-web-fetch-v0.15.0
|
|
294
301
|
[0.14.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.14.0...pi-web-fetch-v0.14.1
|
|
295
302
|
[0.14.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.13.0...pi-web-fetch-v0.14.0
|
package/README.md
CHANGED
|
@@ -23,15 +23,13 @@ The `web_fetch` tool accepts public HTTP and HTTPS URLs and supports HTML, Markd
|
|
|
23
23
|
| `query` | Optional; selects matching sections from a long page. Sections remain in source order. |
|
|
24
24
|
| `offset` | Optional character offset for continuing through extracted content; takes precedence over a URL fragment. |
|
|
25
25
|
|
|
26
|
-
Output is bounded to 6,000 characters by default. For long pages, use the returned `nextOffset` to continue.
|
|
26
|
+
Output is bounded to 6,000 characters by default. For long pages, use the returned `nextOffset` to continue. Use `query` to focus on matching sections, or omit it to read the complete document. Pi's preview is collapsed by default; press `Ctrl+O` to expand it.
|
|
27
27
|
|
|
28
28
|
For example, a URL such as `https://github.com/shadcn-ui/lint#get-started` starts at the matching heading. GitHub `blob` links and bare gists are converted to raw source URLs; repository roots use their README. GitHub directory listings are not supported.
|
|
29
29
|
|
|
30
30
|
### Page alternatives and bot challenges
|
|
31
31
|
|
|
32
|
-
The tool can discover advertised Markdown versions and `/llms.txt` indexes.
|
|
33
|
-
|
|
34
|
-
Pages with HTML nested beyond 512 elements are rejected.
|
|
32
|
+
The tool can discover advertised Markdown versions and `/llms.txt` indexes. An index URL appears in the result when found; follow it if the page is insufficient. The tool can solve Anubis and supported AWS WAF proof-of-work challenges. If an AWS WAF or Cloudflare challenge remains blocked, it may fall back to the site's documentation index. Unsupported challenges and JavaScript CAPTCHAs return a blocked result; there is no browser fallback.
|
|
35
33
|
|
|
36
34
|
### Caching
|
|
37
35
|
|
package/package.json
CHANGED
package/src/acquisition/fetch.ts
CHANGED
|
@@ -14,6 +14,13 @@ import {
|
|
|
14
14
|
parseAnubisChallenge,
|
|
15
15
|
solveAnubisChallenge,
|
|
16
16
|
} from "../challenges/anubis.ts";
|
|
17
|
+
import {
|
|
18
|
+
AWS_WAF_TOKEN_COOKIE,
|
|
19
|
+
isAwsWafChallengeResponse,
|
|
20
|
+
type AwsWafResolver,
|
|
21
|
+
type AwsWafSolution,
|
|
22
|
+
} from "../challenges/aws-waf.ts";
|
|
23
|
+
import { nativeAwsWafResolver } from "../challenges/aws-waf-resolver.ts";
|
|
17
24
|
import {
|
|
18
25
|
sliceCompleteDocument,
|
|
19
26
|
type CompleteDocument,
|
|
@@ -89,6 +96,10 @@ export interface FetchRemoteDependencies extends RedirectDependencies {
|
|
|
89
96
|
timeoutMs?: number;
|
|
90
97
|
/** When false, an Anubis interstitial is reported as blocked instead of solved. Defaults to true. */
|
|
91
98
|
solveAnubisChallenges?: boolean;
|
|
99
|
+
/** When false, an AWS WAF challenge is reported as blocked instead of solved. Defaults to true. */
|
|
100
|
+
solveAwsWafChallenges?: boolean;
|
|
101
|
+
/** Resolver used for AWS WAF challenges; defaults to the in-process resolver. */
|
|
102
|
+
awsWafResolver?: AwsWafResolver;
|
|
92
103
|
}
|
|
93
104
|
|
|
94
105
|
/**
|
|
@@ -190,7 +201,7 @@ export class ChallengeBlockedError extends Error {
|
|
|
190
201
|
function assertSuccessfulResponse(response: IncomingMessage): void {
|
|
191
202
|
const status = response.statusCode ?? 0;
|
|
192
203
|
|
|
193
|
-
if (
|
|
204
|
+
if (isAwsWafChallengeResponse(response)) {
|
|
194
205
|
response.resume();
|
|
195
206
|
throw new ChallengeBlockedError(
|
|
196
207
|
`web_fetch was blocked by an AWS WAF anti-bot challenge (HTTP ${status}).`,
|
|
@@ -493,6 +504,37 @@ function assertAbsoluteHttpUrlForFetch(value: string): URL {
|
|
|
493
504
|
return url;
|
|
494
505
|
}
|
|
495
506
|
|
|
507
|
+
/**
|
|
508
|
+
* Attempts one WAF token resolution while preserving caller cancellation.
|
|
509
|
+
* Solver failures are intentionally converted into the normal blocked path.
|
|
510
|
+
*/
|
|
511
|
+
async function resolveAwsWafToken(
|
|
512
|
+
resolver: AwsWafResolver,
|
|
513
|
+
target: ValidatedTarget,
|
|
514
|
+
signal: AbortSignal,
|
|
515
|
+
network: RedirectDependencies,
|
|
516
|
+
budgetMs: number,
|
|
517
|
+
): Promise<AwsWafSolution | undefined> {
|
|
518
|
+
if (budgetMs <= 0) return undefined;
|
|
519
|
+
|
|
520
|
+
const controller = new AbortController();
|
|
521
|
+
const timeout = setTimeout(() => controller.abort(), budgetMs);
|
|
522
|
+
const resolverSignal = AbortSignal.any([signal, controller.signal]);
|
|
523
|
+
|
|
524
|
+
try {
|
|
525
|
+
return await awaitWithAbort(
|
|
526
|
+
resolver.resolve({ url: target.url, signal: resolverSignal, network }),
|
|
527
|
+
resolverSignal,
|
|
528
|
+
);
|
|
529
|
+
} catch (error) {
|
|
530
|
+
if (signal.aborted) throw error;
|
|
531
|
+
|
|
532
|
+
return undefined;
|
|
533
|
+
} finally {
|
|
534
|
+
clearTimeout(timeout);
|
|
535
|
+
}
|
|
536
|
+
}
|
|
537
|
+
|
|
496
538
|
/**
|
|
497
539
|
* Fetches a URL and returns the complete document with extracted content.
|
|
498
540
|
*
|
|
@@ -513,6 +555,7 @@ async function fetchDocument(
|
|
|
513
555
|
assertAbsoluteHttpUrlForFetch(normalizeGitHubRawUrl(rawUrl));
|
|
514
556
|
const controller = new AbortController();
|
|
515
557
|
const timeoutMs = dependencies.timeoutMs ?? REQUEST_TIMEOUT_MS;
|
|
558
|
+
const deadline = Date.now() + timeoutMs;
|
|
516
559
|
const extractHtml = dependencies.extractHtml ?? extractHtmlToMarkdown;
|
|
517
560
|
let timedOut = false;
|
|
518
561
|
|
|
@@ -579,6 +622,48 @@ async function fetchDocument(
|
|
|
579
622
|
}
|
|
580
623
|
}
|
|
581
624
|
|
|
625
|
+
if (
|
|
626
|
+
isAwsWafChallengeResponse(response) &&
|
|
627
|
+
requestDependencies.solveAwsWafChallenges !== false
|
|
628
|
+
) {
|
|
629
|
+
const resolver = requestDependencies.awsWafResolver ?? nativeAwsWafResolver;
|
|
630
|
+
|
|
631
|
+
if (resolver) {
|
|
632
|
+
response.resume();
|
|
633
|
+
|
|
634
|
+
const solution = await resolveAwsWafToken(
|
|
635
|
+
resolver,
|
|
636
|
+
target,
|
|
637
|
+
controller.signal,
|
|
638
|
+
requestDependencies,
|
|
639
|
+
Math.floor(Math.max(0, deadline - Date.now()) / 2),
|
|
640
|
+
);
|
|
641
|
+
|
|
642
|
+
if (solution) {
|
|
643
|
+
const stored = requestDependencies.cookies?.set(
|
|
644
|
+
AWS_WAF_TOKEN_COOKIE,
|
|
645
|
+
solution.token,
|
|
646
|
+
target.url,
|
|
647
|
+
);
|
|
648
|
+
|
|
649
|
+
if (stored) {
|
|
650
|
+
// Retry once inside the same validated redirect and cookie boundary.
|
|
651
|
+
// Keep the browser identity that earned the token, and report a
|
|
652
|
+
// second challenge as blocked instead of looping.
|
|
653
|
+
const retryDependencies = solution.userAgent
|
|
654
|
+
? { ...requestDependencies, userAgent: solution.userAgent }
|
|
655
|
+
: requestDependencies;
|
|
656
|
+
|
|
657
|
+
({ target, response } = await requestFollowingRedirects(
|
|
658
|
+
target.url,
|
|
659
|
+
controller.signal,
|
|
660
|
+
retryDependencies,
|
|
661
|
+
));
|
|
662
|
+
}
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
}
|
|
666
|
+
|
|
582
667
|
if (response.statusCode === 304 && cached) {
|
|
583
668
|
response.resume();
|
|
584
669
|
const etag = responseHeader(response, "etag") ?? cached.validators?.etag;
|
package/src/acquisition/llms.ts
CHANGED
|
@@ -191,7 +191,7 @@ async function probeUsableRawText(
|
|
|
191
191
|
const document = await fetchCompleteDocument(
|
|
192
192
|
candidate.toString(),
|
|
193
193
|
sharedSignal,
|
|
194
|
-
{ ...dependencies, solveAnubisChallenges: false },
|
|
194
|
+
{ ...dependencies, solveAnubisChallenges: false, solveAwsWafChallenges: false },
|
|
195
195
|
// An index probe never accepts an empty body.
|
|
196
196
|
false,
|
|
197
197
|
// A probe wants exactly this representation; retrying as HTML only burns
|
|
@@ -0,0 +1,806 @@
|
|
|
1
|
+
import { createCipheriv, randomBytes, randomInt, randomUUID } from "node:crypto";
|
|
2
|
+
import { Worker } from "node:worker_threads";
|
|
3
|
+
import type { AwsWafWork } from "./aws-waf-solver.ts";
|
|
4
|
+
import type { IncomingMessage } from "node:http";
|
|
5
|
+
import { crc32 } from "node:zlib";
|
|
6
|
+
import type { AwsWafResolveContext, AwsWafResolver, AwsWafSolution } from "./aws-waf.ts";
|
|
7
|
+
import { CookieJar } from "../network/cookies.ts";
|
|
8
|
+
import {
|
|
9
|
+
requestFollowingRedirects,
|
|
10
|
+
type RedirectDependencies,
|
|
11
|
+
} from "../network/network-redirects.ts";
|
|
12
|
+
import { validateRemoteUrl } from "../network/network-policy.ts";
|
|
13
|
+
import {
|
|
14
|
+
FETCH_MAX_BYTES,
|
|
15
|
+
decodeResponse,
|
|
16
|
+
decompressResponse,
|
|
17
|
+
readResponseBytes,
|
|
18
|
+
requestPinned,
|
|
19
|
+
responseHeader,
|
|
20
|
+
} from "../network/network-transport.ts";
|
|
21
|
+
|
|
22
|
+
/** AWS WAF browser-challenge signal encryption key used by the current challenge family. */
|
|
23
|
+
const SIGNAL_KEY = Buffer.from(
|
|
24
|
+
"6f71a512b1e035eaab53d8be73120d3fb68a0ca346b9560aab3e5cdf753d5e98",
|
|
25
|
+
"hex",
|
|
26
|
+
);
|
|
27
|
+
|
|
28
|
+
/** Browser identity used for the challenge handshake and the retry. */
|
|
29
|
+
const BROWSER_USER_AGENT =
|
|
30
|
+
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36";
|
|
31
|
+
|
|
32
|
+
const BROWSER_VERSION = "144";
|
|
33
|
+
|
|
34
|
+
const SCREEN = "1920-1080-1080-24-*-*-*";
|
|
35
|
+
|
|
36
|
+
const PLUGINS = [
|
|
37
|
+
{ name: "PDF Viewer", str: "PDF Viewer " },
|
|
38
|
+
{ name: "Chrome PDF Viewer", str: "Chrome PDF Viewer " },
|
|
39
|
+
{ name: "Chromium PDF Viewer", str: "Chromium PDF Viewer " },
|
|
40
|
+
{ name: "Microsoft Edge PDF Viewer", str: "Microsoft Edge PDF Viewer " },
|
|
41
|
+
{ name: "WebKit built-in PDF", str: "WebKit built-in PDF " },
|
|
42
|
+
];
|
|
43
|
+
|
|
44
|
+
const PLUGIN_STRING = PLUGINS.map((plugin) => plugin.str).join("");
|
|
45
|
+
|
|
46
|
+
const MATH_VALUES = {
|
|
47
|
+
tan: "-1.4214488238747245",
|
|
48
|
+
sin: "0.8178819121159085",
|
|
49
|
+
cos: "-0.5753861119575491",
|
|
50
|
+
};
|
|
51
|
+
|
|
52
|
+
const GPU_EXTENSIONS = [
|
|
53
|
+
"ANGLE_instanced_arrays",
|
|
54
|
+
"EXT_blend_minmax",
|
|
55
|
+
"EXT_clip_control",
|
|
56
|
+
"EXT_color_buffer_half_float",
|
|
57
|
+
"EXT_depth_clamp",
|
|
58
|
+
"EXT_disjoint_timer_query",
|
|
59
|
+
"EXT_float_blend",
|
|
60
|
+
"EXT_frag_depth",
|
|
61
|
+
"EXT_polygon_offset_clamp",
|
|
62
|
+
"EXT_shader_texture_lod",
|
|
63
|
+
"EXT_texture_compression_bptc",
|
|
64
|
+
"EXT_texture_compression_rgtc",
|
|
65
|
+
"EXT_texture_filter_anisotropic",
|
|
66
|
+
"EXT_texture_mirror_clamp_to_edge",
|
|
67
|
+
"EXT_sRGB",
|
|
68
|
+
"KHR_parallel_shader_compile",
|
|
69
|
+
"OES_element_index_uint",
|
|
70
|
+
"OES_fbo_render_mipmap",
|
|
71
|
+
"OES_standard_derivatives",
|
|
72
|
+
"OES_texture_float",
|
|
73
|
+
"OES_texture_float_linear",
|
|
74
|
+
"OES_texture_half_float",
|
|
75
|
+
"OES_texture_half_float_linear",
|
|
76
|
+
"OES_vertex_array_object",
|
|
77
|
+
"WEBGL_blend_func_extended",
|
|
78
|
+
"WEBGL_color_buffer_float",
|
|
79
|
+
"WEBGL_compressed_texture_astc",
|
|
80
|
+
"WEBGL_compressed_texture_etc",
|
|
81
|
+
"WEBGL_compressed_texture_etc1",
|
|
82
|
+
"WEBGL_compressed_texture_s3tc",
|
|
83
|
+
"WEBGL_compressed_texture_s3tc_srgb",
|
|
84
|
+
"WEBGL_debug_renderer_info",
|
|
85
|
+
"WEBGL_debug_shaders",
|
|
86
|
+
"WEBGL_depth_texture",
|
|
87
|
+
"WEBGL_lose_context",
|
|
88
|
+
"WEBGL_multi_draw",
|
|
89
|
+
"WEBGL_polygon_mode",
|
|
90
|
+
];
|
|
91
|
+
|
|
92
|
+
/** Maps challenge type hashes to their proof-of-work family. */
|
|
93
|
+
function normalizeChallengeType(value: string): string {
|
|
94
|
+
if (value.startsWith("ha9faaffd")) return "NetworkBandwidth";
|
|
95
|
+
|
|
96
|
+
if (value.startsWith("h72f957df")) return "HashcashScrypt";
|
|
97
|
+
|
|
98
|
+
if (value.startsWith("h7b0c470f")) return "SHA256";
|
|
99
|
+
|
|
100
|
+
return value;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** Returns the submission endpoint for a challenge type. */
|
|
104
|
+
function challengeEndpoint(challengeType: string): string {
|
|
105
|
+
return challengeType === "NetworkBandwidth" ? "mp_verify" : "verify";
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
const CHALLENGE_SAME_ORIGIN = /(\/__challenge_[A-Za-z0-9]+\/[a-f0-9]+\/[a-f0-9]+)/;
|
|
109
|
+
|
|
110
|
+
const CHALLENGE_EXTERNAL =
|
|
111
|
+
/(https:\/\/[a-z0-9]+\.[a-z0-9]+\.[a-z0-9-]+\.token\.awswaf\.com\/[^/\s"]+\/[^/\s"]+\/[^/\s"]+)/;
|
|
112
|
+
|
|
113
|
+
const CHALLENGE_SDK =
|
|
114
|
+
/(https:\/\/[a-z0-9]+\.edge\.sdk\.awswaf\.com\/[a-z0-9]+\/[a-z0-9]+)\/challenge\.js/;
|
|
115
|
+
|
|
116
|
+
const GOKU_PROPS = /window\.gokuProps\s*=\s*(\{[^}]+\})/;
|
|
117
|
+
|
|
118
|
+
type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
|
|
119
|
+
|
|
120
|
+
type JsonObject = { [key: string]: JsonValue };
|
|
121
|
+
|
|
122
|
+
interface TextResponse {
|
|
123
|
+
url: URL;
|
|
124
|
+
status: number;
|
|
125
|
+
text: string;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
interface FingerprintMetrics {
|
|
129
|
+
fp2: number;
|
|
130
|
+
browser: number;
|
|
131
|
+
capabilities: number;
|
|
132
|
+
gpu: number;
|
|
133
|
+
dnt: number;
|
|
134
|
+
math: number;
|
|
135
|
+
screen: number;
|
|
136
|
+
navigator: number;
|
|
137
|
+
auto: number;
|
|
138
|
+
stealth: number;
|
|
139
|
+
subtle: number;
|
|
140
|
+
canvas: number;
|
|
141
|
+
formdetector: number;
|
|
142
|
+
be: number;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
interface Collector {
|
|
146
|
+
name: keyof FingerprintMetrics;
|
|
147
|
+
metric: string;
|
|
148
|
+
low: number;
|
|
149
|
+
high: number;
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
const COLLECTORS: readonly Collector[] = [
|
|
153
|
+
{ name: "fp2", metric: "100", low: 0.5, high: 3 },
|
|
154
|
+
{ name: "browser", metric: "101", low: 0, high: 1 },
|
|
155
|
+
{ name: "capabilities", metric: "102", low: 2, high: 8 },
|
|
156
|
+
{ name: "gpu", metric: "103", low: 3, high: 12 },
|
|
157
|
+
{ name: "dnt", metric: "104", low: 0, high: 1 },
|
|
158
|
+
{ name: "math", metric: "105", low: 0, high: 1 },
|
|
159
|
+
{ name: "screen", metric: "106", low: 0, high: 1 },
|
|
160
|
+
{ name: "navigator", metric: "107", low: 0, high: 1 },
|
|
161
|
+
{ name: "auto", metric: "108", low: 0, high: 1 },
|
|
162
|
+
{ name: "stealth", metric: "undefined", low: 1, high: 4 },
|
|
163
|
+
{ name: "subtle", metric: "110", low: 0, high: 1 },
|
|
164
|
+
{ name: "canvas", metric: "111", low: 80, high: 200 },
|
|
165
|
+
{ name: "formdetector", metric: "112", low: 0, high: 3 },
|
|
166
|
+
{ name: "be", metric: "undefined", low: 0, high: 1 },
|
|
167
|
+
];
|
|
168
|
+
|
|
169
|
+
interface ChallengeData {
|
|
170
|
+
input: string;
|
|
171
|
+
hmac?: string;
|
|
172
|
+
region?: string;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
interface ChallengeInputs {
|
|
176
|
+
challenge: ChallengeData;
|
|
177
|
+
challengeType: string;
|
|
178
|
+
difficulty: number;
|
|
179
|
+
memory: number;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
interface BuiltMetrics {
|
|
183
|
+
metrics: JsonValue[];
|
|
184
|
+
fingerprint: FingerprintMetrics;
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** Returns whether a value is a plain JSON object. */
|
|
188
|
+
// oxlint-disable-next-line anti-slop/no-unknown-parameters -- parses untrusted challenge JSON at the I/O boundary
|
|
189
|
+
function isJsonObject(value: unknown): value is JsonObject {
|
|
190
|
+
// oxlint-disable-next-line anti-slop/no-runtime-typeof -- validates untrusted challenge JSON before use
|
|
191
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/** Parses untrusted JSON text into a plain object. */
|
|
195
|
+
function parseJsonObject(text: string): JsonObject | undefined {
|
|
196
|
+
try {
|
|
197
|
+
const parsed: unknown = JSON.parse(text);
|
|
198
|
+
|
|
199
|
+
return isJsonObject(parsed) ? parsed : undefined;
|
|
200
|
+
} catch {
|
|
201
|
+
return undefined;
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/** Reads a string field, returning undefined for missing or non-string values. */
|
|
206
|
+
function stringField(value: JsonObject, key: string): string | undefined {
|
|
207
|
+
const field = value[key];
|
|
208
|
+
|
|
209
|
+
// oxlint-disable-next-line anti-slop/no-runtime-typeof -- validates an untrusted JSON field
|
|
210
|
+
return typeof field === "string" ? field : undefined;
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/** Reads a numeric field, returning undefined for missing or non-numeric values. */
|
|
214
|
+
function numberField(value: JsonObject, key: string): number | undefined {
|
|
215
|
+
const field = value[key];
|
|
216
|
+
|
|
217
|
+
// oxlint-disable-next-line anti-slop/no-runtime-typeof -- validates an untrusted JSON field
|
|
218
|
+
return typeof field === "number" && Number.isFinite(field) ? field : undefined;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/** Returns a random floating-point metric in the supplied range. */
|
|
222
|
+
function randomMetric(low: number, high: number): number {
|
|
223
|
+
return Math.round((low + Math.random() * (high - low)) * 10) / 10;
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/** Builds realistic-looking WAF metrics and the matching fingerprint metric map. */
|
|
227
|
+
function buildMetrics(hasToken: boolean): BuiltMetrics {
|
|
228
|
+
const fingerprint: FingerprintMetrics = {
|
|
229
|
+
fp2: 0,
|
|
230
|
+
browser: 0,
|
|
231
|
+
capabilities: 0,
|
|
232
|
+
gpu: 0,
|
|
233
|
+
dnt: 0,
|
|
234
|
+
math: 0,
|
|
235
|
+
screen: 0,
|
|
236
|
+
navigator: 0,
|
|
237
|
+
auto: 0,
|
|
238
|
+
stealth: 0,
|
|
239
|
+
subtle: 0,
|
|
240
|
+
canvas: 0,
|
|
241
|
+
formdetector: 0,
|
|
242
|
+
be: 0,
|
|
243
|
+
};
|
|
244
|
+
|
|
245
|
+
const metrics: JsonValue[] = [];
|
|
246
|
+
|
|
247
|
+
for (const collector of COLLECTORS) {
|
|
248
|
+
const value = randomMetric(collector.low, collector.high);
|
|
249
|
+
|
|
250
|
+
fingerprint[collector.name] = Math.trunc(value);
|
|
251
|
+
metrics.push({ name: collector.metric, value, unit: "2" });
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
const encoding = randomMetric(0.5, 3);
|
|
255
|
+
const encryption = randomMetric(2, 8);
|
|
256
|
+
|
|
257
|
+
const acquisition =
|
|
258
|
+
COLLECTORS.reduce((total, collector) => total + fingerprint[collector.name], 0) +
|
|
259
|
+
encoding +
|
|
260
|
+
encryption +
|
|
261
|
+
randomMetric(2, 6);
|
|
262
|
+
|
|
263
|
+
const challenge = randomMetric(2, 8);
|
|
264
|
+
const cookie = randomMetric(0.1, 1);
|
|
265
|
+
const total = acquisition + challenge + cookie;
|
|
266
|
+
|
|
267
|
+
metrics.push(
|
|
268
|
+
{ name: "2", value: encoding, unit: "2" },
|
|
269
|
+
{ name: "3", value: encryption, unit: "2" },
|
|
270
|
+
{ name: "7", value: hasToken ? 1 : 0, unit: "4" },
|
|
271
|
+
{ name: "1", value: acquisition, unit: "2" },
|
|
272
|
+
{ name: "4", value: challenge, unit: "2" },
|
|
273
|
+
{ name: "5", value: cookie, unit: "2" },
|
|
274
|
+
{ name: "6", value: total, unit: "2" },
|
|
275
|
+
{ name: "8", value: 1, unit: "4" },
|
|
276
|
+
);
|
|
277
|
+
|
|
278
|
+
return { metrics, fingerprint };
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** Builds a browser fingerprint object accepted by the WAF signal decoder. */
|
|
282
|
+
function buildSignal(site: string, fingerprint: FingerprintMetrics) {
|
|
283
|
+
const now = Date.now();
|
|
284
|
+
const canvasHash = randomInt(100_000_000, 1_000_000_000);
|
|
285
|
+
const histogram = Array.from({ length: 256 }, () => randomInt(1, 50));
|
|
286
|
+
|
|
287
|
+
return {
|
|
288
|
+
metrics: { ...fingerprint },
|
|
289
|
+
start: now,
|
|
290
|
+
flashVersion: null,
|
|
291
|
+
plugins: PLUGINS.map((plugin) => ({ name: plugin.name, str: plugin.str })),
|
|
292
|
+
dupedPlugins: `${PLUGIN_STRING}||${SCREEN}`,
|
|
293
|
+
screenInfo: SCREEN,
|
|
294
|
+
referrer: "",
|
|
295
|
+
userAgent: BROWSER_USER_AGENT,
|
|
296
|
+
location: `${site}/`,
|
|
297
|
+
webDriver: false,
|
|
298
|
+
capabilities: {
|
|
299
|
+
css: {
|
|
300
|
+
textShadow: 1,
|
|
301
|
+
WebkitTextStroke: 1,
|
|
302
|
+
boxShadow: 1,
|
|
303
|
+
borderRadius: 1,
|
|
304
|
+
borderImage: 1,
|
|
305
|
+
opacity: 1,
|
|
306
|
+
transform: 1,
|
|
307
|
+
transition: 1,
|
|
308
|
+
},
|
|
309
|
+
js: {
|
|
310
|
+
audio: true,
|
|
311
|
+
geolocation: true,
|
|
312
|
+
localStorage: "supported",
|
|
313
|
+
touch: false,
|
|
314
|
+
video: true,
|
|
315
|
+
webWorker: true,
|
|
316
|
+
},
|
|
317
|
+
elapsed: fingerprint.capabilities,
|
|
318
|
+
},
|
|
319
|
+
gpu: {
|
|
320
|
+
vendor: "Google Inc. (NVIDIA Corporation)",
|
|
321
|
+
model: "ANGLE (NVIDIA Corporation, NVIDIA GeForce RTX 5070/PCIe/SSE2, OpenGL ES 3.2)",
|
|
322
|
+
extensions: GPU_EXTENSIONS,
|
|
323
|
+
},
|
|
324
|
+
dnt: null,
|
|
325
|
+
math: MATH_VALUES,
|
|
326
|
+
automation: {
|
|
327
|
+
wd: { properties: { document: [], window: [], navigator: [] } },
|
|
328
|
+
phantom: { properties: { window: [] } },
|
|
329
|
+
},
|
|
330
|
+
stealth: { t1: 0, t2: 0, i: 1, mte: 0, mtd: false },
|
|
331
|
+
crypto: {
|
|
332
|
+
crypto: 1,
|
|
333
|
+
subtle: 1,
|
|
334
|
+
encrypt: true,
|
|
335
|
+
decrypt: true,
|
|
336
|
+
wrapKey: true,
|
|
337
|
+
unwrapKey: true,
|
|
338
|
+
sign: true,
|
|
339
|
+
verify: true,
|
|
340
|
+
digest: true,
|
|
341
|
+
deriveBits: true,
|
|
342
|
+
deriveKey: true,
|
|
343
|
+
getRandomValues: true,
|
|
344
|
+
randomUUID: true,
|
|
345
|
+
},
|
|
346
|
+
canvas: { hash: canvasHash, emailHash: null, histogramBins: histogram },
|
|
347
|
+
formDetected: false,
|
|
348
|
+
numForms: 0,
|
|
349
|
+
numFormElements: 0,
|
|
350
|
+
be: { si: false },
|
|
351
|
+
end: now + 1,
|
|
352
|
+
errors: [],
|
|
353
|
+
version: "2.4.0",
|
|
354
|
+
id: randomUUID(),
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
/** Builds and encrypts the browser signal payload. */
|
|
359
|
+
function encodeSignal(site: string, hasToken: boolean) {
|
|
360
|
+
const built = buildMetrics(hasToken);
|
|
361
|
+
const signal = buildSignal(site, built.fingerprint);
|
|
362
|
+
const plaintextJson = JSON.stringify(signal);
|
|
363
|
+
const checksum = crc32(plaintextJson).toString(16).toUpperCase().padStart(8, "0");
|
|
364
|
+
const iv = randomBytes(12);
|
|
365
|
+
const cipher = createCipheriv("aes-256-gcm", SIGNAL_KEY, iv);
|
|
366
|
+
|
|
367
|
+
const ciphertext = Buffer.concat([
|
|
368
|
+
cipher.update(`${checksum}#${plaintextJson}`, "utf8"),
|
|
369
|
+
cipher.final(),
|
|
370
|
+
]);
|
|
371
|
+
|
|
372
|
+
const tag = cipher.getAuthTag();
|
|
373
|
+
|
|
374
|
+
return {
|
|
375
|
+
encrypted: `${iv.toString("base64")}::${tag.toString("hex")}::${ciphertext.toString("hex")}`,
|
|
376
|
+
checksum,
|
|
377
|
+
metrics: built.metrics,
|
|
378
|
+
signal,
|
|
379
|
+
};
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
/** Builds the signal entry used in challenge submissions. */
|
|
383
|
+
function signalEntry(encrypted: string): JsonValue[] {
|
|
384
|
+
return [{ name: "Zoey", value: { Present: encrypted } }];
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** Run CPU-intensive work off the main thread and terminate it on cancellation. */
|
|
388
|
+
function solveAwsWafChallenge(
|
|
389
|
+
inputs: ChallengeInputs,
|
|
390
|
+
checksum: string,
|
|
391
|
+
signal: AbortSignal,
|
|
392
|
+
): Promise<string | undefined> {
|
|
393
|
+
signal.throwIfAborted();
|
|
394
|
+
|
|
395
|
+
const work: AwsWafWork = {
|
|
396
|
+
type: inputs.challengeType,
|
|
397
|
+
input: inputs.challenge.input,
|
|
398
|
+
checksum,
|
|
399
|
+
difficulty: inputs.difficulty,
|
|
400
|
+
memory: inputs.memory,
|
|
401
|
+
};
|
|
402
|
+
|
|
403
|
+
const worker = new Worker(new URL("../../workers/aws-waf-worker.js", import.meta.url), {
|
|
404
|
+
workerData: work,
|
|
405
|
+
resourceLimits: { maxOldGenerationSizeMb: 64, maxYoungGenerationSizeMb: 16 },
|
|
406
|
+
});
|
|
407
|
+
|
|
408
|
+
return new Promise((resolve, reject) => {
|
|
409
|
+
let settled = false;
|
|
410
|
+
|
|
411
|
+
const timer = setTimeout(
|
|
412
|
+
() => finish(() => reject(new Error("AWS WAF worker timed out."))),
|
|
413
|
+
60_000,
|
|
414
|
+
);
|
|
415
|
+
|
|
416
|
+
function finish(callback: () => void): void {
|
|
417
|
+
if (settled) return;
|
|
418
|
+
settled = true;
|
|
419
|
+
clearTimeout(timer);
|
|
420
|
+
signal.removeEventListener("abort", onAbort);
|
|
421
|
+
void worker.terminate();
|
|
422
|
+
callback();
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
function onAbort(): void {
|
|
426
|
+
finish(() => reject(signal.reason));
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
worker.on(
|
|
430
|
+
"message",
|
|
431
|
+
(message: { kind: "solved"; solution?: string } | { kind: "error"; message: string }) => {
|
|
432
|
+
if (message.kind === "solved") finish(() => resolve(message.solution));
|
|
433
|
+
else finish(() => reject(new Error(message.message)));
|
|
434
|
+
},
|
|
435
|
+
);
|
|
436
|
+
worker.on("error", (error) => finish(() => reject(error)));
|
|
437
|
+
worker.on("exit", (code) =>
|
|
438
|
+
finish(() => reject(new Error(`AWS WAF worker exited with code ${code}.`))),
|
|
439
|
+
);
|
|
440
|
+
|
|
441
|
+
if (signal.aborted) onAbort();
|
|
442
|
+
else signal.addEventListener("abort", onAbort, { once: true });
|
|
443
|
+
});
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/** Parses the base64 challenge input into challenge metadata. */
|
|
447
|
+
function decodeChallengeInput(input: string): JsonObject | undefined {
|
|
448
|
+
try {
|
|
449
|
+
return parseJsonObject(Buffer.from(input, "base64").toString("utf8"));
|
|
450
|
+
} catch {
|
|
451
|
+
return undefined;
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
/** Reads challenge inputs from the `/inputs` response. */
|
|
456
|
+
function parseChallengeInputs(value: JsonObject): ChallengeInputs | undefined {
|
|
457
|
+
const challenge = value["challenge"];
|
|
458
|
+
|
|
459
|
+
if (!isJsonObject(challenge)) return undefined;
|
|
460
|
+
|
|
461
|
+
const input = stringField(challenge, "input");
|
|
462
|
+
|
|
463
|
+
if (!input) return undefined;
|
|
464
|
+
|
|
465
|
+
const decoded = decodeChallengeInput(input);
|
|
466
|
+
|
|
467
|
+
const challengeType =
|
|
468
|
+
(decoded ? stringField(decoded, "challenge_type") : undefined) ??
|
|
469
|
+
stringField(value, "challenge_type");
|
|
470
|
+
|
|
471
|
+
if (!challengeType) return undefined;
|
|
472
|
+
|
|
473
|
+
const difficulty =
|
|
474
|
+
(decoded ? numberField(decoded, "difficulty") : undefined) ??
|
|
475
|
+
numberField(value, "difficulty") ??
|
|
476
|
+
1;
|
|
477
|
+
|
|
478
|
+
const memory =
|
|
479
|
+
(decoded ? numberField(decoded, "memory") : undefined) ?? numberField(value, "memory") ?? 128;
|
|
480
|
+
|
|
481
|
+
return {
|
|
482
|
+
challenge: {
|
|
483
|
+
input,
|
|
484
|
+
hmac: stringField(challenge, "hmac"),
|
|
485
|
+
region: stringField(challenge, "region"),
|
|
486
|
+
},
|
|
487
|
+
challengeType: normalizeChallengeType(challengeType),
|
|
488
|
+
difficulty,
|
|
489
|
+
memory,
|
|
490
|
+
};
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
/** Extracts the challenge URL and goku properties from a WAF page. */
|
|
494
|
+
export function extractAwsWafChallenge(
|
|
495
|
+
html: string,
|
|
496
|
+
site: URL,
|
|
497
|
+
): { challengeUrl: string; gokuProps?: JsonObject } | undefined {
|
|
498
|
+
const sameOrigin = CHALLENGE_SAME_ORIGIN.exec(html);
|
|
499
|
+
const external = CHALLENGE_EXTERNAL.exec(html);
|
|
500
|
+
const sdk = CHALLENGE_SDK.exec(html);
|
|
501
|
+
const match = sameOrigin ?? external ?? sdk;
|
|
502
|
+
|
|
503
|
+
if (!match) return undefined;
|
|
504
|
+
|
|
505
|
+
const challengeUrl = sameOrigin ? new URL(match[1], site).href : external ? match[1] : match[1];
|
|
506
|
+
const gokuMatch = GOKU_PROPS.exec(html);
|
|
507
|
+
const gokuProps = gokuMatch ? parseJsonObject(gokuMatch[1]) : undefined;
|
|
508
|
+
|
|
509
|
+
return { challengeUrl, gokuProps };
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/** Builds the browser headers used to request the protected page. */
|
|
513
|
+
function browserHeaders() {
|
|
514
|
+
return {
|
|
515
|
+
accept:
|
|
516
|
+
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
|
517
|
+
"accept-language": "en-US,en;q=0.9",
|
|
518
|
+
"sec-ch-ua": `"Not:A-Brand";v="8", "Chromium";v="${BROWSER_VERSION}", "Google Chrome";v="${BROWSER_VERSION}"`,
|
|
519
|
+
"sec-ch-ua-mobile": "?0",
|
|
520
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
521
|
+
"sec-fetch-dest": "document",
|
|
522
|
+
"sec-fetch-mode": "navigate",
|
|
523
|
+
"sec-fetch-site": "none",
|
|
524
|
+
"sec-fetch-user": "?1",
|
|
525
|
+
"upgrade-insecure-requests": "1",
|
|
526
|
+
};
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
/** Builds the headers used for challenge API calls. */
|
|
530
|
+
function apiHeaders(site: string) {
|
|
531
|
+
return {
|
|
532
|
+
accept: "*/*",
|
|
533
|
+
"accept-language": "en-US,en;q=0.9",
|
|
534
|
+
"cache-control": "no-cache",
|
|
535
|
+
origin: site,
|
|
536
|
+
pragma: "no-cache",
|
|
537
|
+
priority: "u=1, i",
|
|
538
|
+
referer: `${site}/`,
|
|
539
|
+
"sec-ch-ua": `"Not:A-Brand";v="8", "Chromium";v="${BROWSER_VERSION}", "Google Chrome";v="${BROWSER_VERSION}"`,
|
|
540
|
+
"sec-ch-ua-mobile": "?0",
|
|
541
|
+
"sec-ch-ua-platform": '"Windows"',
|
|
542
|
+
"sec-fetch-dest": "empty",
|
|
543
|
+
"sec-fetch-mode": "cors",
|
|
544
|
+
"sec-fetch-site": "cross-site",
|
|
545
|
+
};
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
/** Reads and decodes a bounded response body. */
|
|
549
|
+
async function readText(response: IncomingMessage, signal: AbortSignal): Promise<string> {
|
|
550
|
+
const bytes = await readResponseBytes(response, FETCH_MAX_BYTES, signal);
|
|
551
|
+
const decompressed = decompressResponse(bytes, responseHeader(response, "content-encoding"));
|
|
552
|
+
|
|
553
|
+
return decodeResponse(decompressed, responseHeader(response, "content-type") ?? "text/plain");
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/** Performs a GET through the validated redirect and cookie boundary. */
|
|
557
|
+
async function getText(
|
|
558
|
+
url: string,
|
|
559
|
+
network: RedirectDependencies,
|
|
560
|
+
signal: AbortSignal,
|
|
561
|
+
headers: Readonly<Record<string, string>>,
|
|
562
|
+
): Promise<TextResponse> {
|
|
563
|
+
const { target, response } = await requestFollowingRedirects(url, signal, network, headers);
|
|
564
|
+
const text = await readText(response, signal);
|
|
565
|
+
|
|
566
|
+
return { url: target.url, status: response.statusCode ?? 0, text };
|
|
567
|
+
}
|
|
568
|
+
|
|
569
|
+
/** Performs a POST to a validated target without following redirects. */
|
|
570
|
+
async function postText(
|
|
571
|
+
url: string,
|
|
572
|
+
network: RedirectDependencies,
|
|
573
|
+
signal: AbortSignal,
|
|
574
|
+
headers: Readonly<Record<string, string>>,
|
|
575
|
+
body: string,
|
|
576
|
+
): Promise<TextResponse> {
|
|
577
|
+
const validateUrl = network.validateUrl ?? validateRemoteUrl;
|
|
578
|
+
const target = await validateUrl(url);
|
|
579
|
+
const cookieHeader = network.cookies?.headerFor(target.url);
|
|
580
|
+
|
|
581
|
+
const requestHeaders = { ...headers };
|
|
582
|
+
|
|
583
|
+
if (network.userAgent) requestHeaders["User-Agent"] = network.userAgent;
|
|
584
|
+
|
|
585
|
+
if (cookieHeader) requestHeaders.Cookie = cookieHeader;
|
|
586
|
+
|
|
587
|
+
const request =
|
|
588
|
+
network.request ??
|
|
589
|
+
((value, requestSignal, requestHeadersInit, options) =>
|
|
590
|
+
requestPinned(value, requestSignal, {
|
|
591
|
+
headers: requestHeadersInit,
|
|
592
|
+
method: options?.method,
|
|
593
|
+
body: options?.body,
|
|
594
|
+
}));
|
|
595
|
+
|
|
596
|
+
const response = await request(target, signal, requestHeaders, { method: "POST", body });
|
|
597
|
+
const text = await readText(response, signal);
|
|
598
|
+
|
|
599
|
+
return { url: target.url, status: response.statusCode ?? 0, text };
|
|
600
|
+
}
|
|
601
|
+
|
|
602
|
+
/** Builds the multipart body used by bandwidth challenges. */
|
|
603
|
+
function buildMultipart(
|
|
604
|
+
domain: string,
|
|
605
|
+
challenge: ChallengeData,
|
|
606
|
+
solution: string,
|
|
607
|
+
checksum: string,
|
|
608
|
+
encrypted: string,
|
|
609
|
+
metrics: JsonValue[],
|
|
610
|
+
gokuProps: JsonObject | undefined,
|
|
611
|
+
) {
|
|
612
|
+
const boundary = `----WebKitFormBoundary${randomBytes(16).toString("hex")}`;
|
|
613
|
+
|
|
614
|
+
const metadata = {
|
|
615
|
+
challenge: {
|
|
616
|
+
input: challenge.input,
|
|
617
|
+
hmac: challenge.hmac ?? null,
|
|
618
|
+
region: challenge.region ?? null,
|
|
619
|
+
},
|
|
620
|
+
solution: null,
|
|
621
|
+
signals: signalEntry(encrypted),
|
|
622
|
+
checksum,
|
|
623
|
+
existing_token: null,
|
|
624
|
+
client: "Browser",
|
|
625
|
+
domain,
|
|
626
|
+
metrics,
|
|
627
|
+
goku_props: gokuProps ?? null,
|
|
628
|
+
};
|
|
629
|
+
|
|
630
|
+
const body = [
|
|
631
|
+
`--${boundary}`,
|
|
632
|
+
'Content-Disposition: form-data; name="solution_data"',
|
|
633
|
+
"",
|
|
634
|
+
solution,
|
|
635
|
+
`--${boundary}`,
|
|
636
|
+
'Content-Disposition: form-data; name="solution_metadata"',
|
|
637
|
+
"",
|
|
638
|
+
JSON.stringify(metadata),
|
|
639
|
+
`--${boundary}--`,
|
|
640
|
+
"",
|
|
641
|
+
].join("\r\n");
|
|
642
|
+
|
|
643
|
+
return { body, contentType: `multipart/form-data; boundary=${boundary}` };
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
/** Builds the JSON body used by verify challenges. */
|
|
647
|
+
function buildVerificationPayload(
|
|
648
|
+
domain: string,
|
|
649
|
+
challenge: ChallengeData,
|
|
650
|
+
solution: string,
|
|
651
|
+
checksum: string,
|
|
652
|
+
encrypted: string,
|
|
653
|
+
metrics: JsonValue[],
|
|
654
|
+
existingToken: string | undefined,
|
|
655
|
+
gokuProps: JsonObject | undefined,
|
|
656
|
+
) {
|
|
657
|
+
return {
|
|
658
|
+
challenge: {
|
|
659
|
+
input: challenge.input,
|
|
660
|
+
hmac: challenge.hmac ?? null,
|
|
661
|
+
region: challenge.region ?? null,
|
|
662
|
+
},
|
|
663
|
+
solution,
|
|
664
|
+
signals: signalEntry(encrypted),
|
|
665
|
+
checksum,
|
|
666
|
+
existing_token: existingToken ?? null,
|
|
667
|
+
client: "Browser",
|
|
668
|
+
domain,
|
|
669
|
+
metrics,
|
|
670
|
+
goku_props: gokuProps ?? null,
|
|
671
|
+
};
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
/** Reads a token from a challenge response, including nested response objects. */
|
|
675
|
+
function parseToken(value: JsonObject): string | undefined {
|
|
676
|
+
const direct = stringField(value, "token");
|
|
677
|
+
|
|
678
|
+
if (direct) return direct;
|
|
679
|
+
|
|
680
|
+
const nested = value["response"];
|
|
681
|
+
|
|
682
|
+
return isJsonObject(nested) ? stringField(nested, "token") : undefined;
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
/** Reads replacement challenge inputs from a challenge response. */
|
|
686
|
+
function parseReplacementInputs(value: JsonObject): ChallengeInputs | undefined {
|
|
687
|
+
const nested = value["response"];
|
|
688
|
+
|
|
689
|
+
if (isJsonObject(nested)) {
|
|
690
|
+
const inputs = nested["inputs"];
|
|
691
|
+
|
|
692
|
+
if (isJsonObject(inputs)) return parseChallengeInputs(inputs);
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
const inputs = value["inputs"];
|
|
696
|
+
|
|
697
|
+
return isJsonObject(inputs) ? parseChallengeInputs(inputs) : undefined;
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
/** Runs the native AWS WAF challenge flow and returns a token. */
|
|
701
|
+
async function resolveAwsWafNative(
|
|
702
|
+
context: AwsWafResolveContext,
|
|
703
|
+
): Promise<AwsWafSolution | undefined> {
|
|
704
|
+
const network: RedirectDependencies = {
|
|
705
|
+
...context.network,
|
|
706
|
+
userAgent: BROWSER_USER_AGENT,
|
|
707
|
+
persistentHeaders: browserHeaders(),
|
|
708
|
+
cookies: context.network?.cookies ?? new CookieJar(),
|
|
709
|
+
};
|
|
710
|
+
|
|
711
|
+
const site = context.url.origin;
|
|
712
|
+
const domain = context.url.hostname;
|
|
713
|
+
|
|
714
|
+
const page = await getText(context.url.href, network, context.signal, browserHeaders());
|
|
715
|
+
|
|
716
|
+
if (page.status !== 200 && page.status !== 202) return undefined;
|
|
717
|
+
|
|
718
|
+
const discovery = extractAwsWafChallenge(page.text, context.url);
|
|
719
|
+
|
|
720
|
+
if (!discovery) return undefined;
|
|
721
|
+
|
|
722
|
+
const inputsResponse = await getText(
|
|
723
|
+
`${discovery.challengeUrl}/inputs?client=browser`,
|
|
724
|
+
network,
|
|
725
|
+
context.signal,
|
|
726
|
+
apiHeaders(site),
|
|
727
|
+
);
|
|
728
|
+
|
|
729
|
+
if (inputsResponse.status !== 200) return undefined;
|
|
730
|
+
|
|
731
|
+
let inputs = parseChallengeInputs(parseJsonObject(inputsResponse.text) ?? {});
|
|
732
|
+
|
|
733
|
+
if (!inputs) return undefined;
|
|
734
|
+
|
|
735
|
+
let token: string | undefined;
|
|
736
|
+
|
|
737
|
+
for (let round = 0; round < 2; round += 1) {
|
|
738
|
+
const signal = encodeSignal(site, token !== undefined);
|
|
739
|
+
const solution = await solveAwsWafChallenge(inputs, signal.checksum, context.signal);
|
|
740
|
+
|
|
741
|
+
if (!solution) return undefined;
|
|
742
|
+
|
|
743
|
+
const endpoint = challengeEndpoint(inputs.challengeType);
|
|
744
|
+
let body: string;
|
|
745
|
+
let contentType = "text/plain;charset=UTF-8";
|
|
746
|
+
|
|
747
|
+
if (endpoint === "mp_verify") {
|
|
748
|
+
const multipart = buildMultipart(
|
|
749
|
+
domain,
|
|
750
|
+
inputs.challenge,
|
|
751
|
+
solution,
|
|
752
|
+
signal.checksum,
|
|
753
|
+
signal.encrypted,
|
|
754
|
+
signal.metrics,
|
|
755
|
+
discovery.gokuProps,
|
|
756
|
+
);
|
|
757
|
+
|
|
758
|
+
body = multipart.body;
|
|
759
|
+
contentType = multipart.contentType;
|
|
760
|
+
} else {
|
|
761
|
+
body = JSON.stringify(
|
|
762
|
+
buildVerificationPayload(
|
|
763
|
+
domain,
|
|
764
|
+
inputs.challenge,
|
|
765
|
+
solution,
|
|
766
|
+
signal.checksum,
|
|
767
|
+
signal.encrypted,
|
|
768
|
+
signal.metrics,
|
|
769
|
+
token,
|
|
770
|
+
discovery.gokuProps,
|
|
771
|
+
),
|
|
772
|
+
);
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
const headers = { ...apiHeaders(site), "content-type": contentType };
|
|
776
|
+
|
|
777
|
+
const response = await postText(
|
|
778
|
+
`${discovery.challengeUrl}/${endpoint}`,
|
|
779
|
+
network,
|
|
780
|
+
context.signal,
|
|
781
|
+
headers,
|
|
782
|
+
body,
|
|
783
|
+
);
|
|
784
|
+
|
|
785
|
+
if (response.status !== 200) return undefined;
|
|
786
|
+
|
|
787
|
+
const parsed = parseJsonObject(response.text);
|
|
788
|
+
|
|
789
|
+
if (!parsed) return undefined;
|
|
790
|
+
|
|
791
|
+
token = parseToken(parsed) ?? token;
|
|
792
|
+
|
|
793
|
+
if (token) break;
|
|
794
|
+
|
|
795
|
+
const replacement = parseReplacementInputs(parsed);
|
|
796
|
+
|
|
797
|
+
if (replacement) inputs = replacement;
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
return token ? { token, userAgent: BROWSER_USER_AGENT } : undefined;
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
/** Native in-process AWS WAF resolver. */
|
|
804
|
+
export const nativeAwsWafResolver: AwsWafResolver = {
|
|
805
|
+
resolve: resolveAwsWafNative,
|
|
806
|
+
};
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
import { createHash, scryptSync } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
/** Bounded proof-of-work request sent to the AWS WAF worker. */
|
|
4
|
+
export interface AwsWafWork {
|
|
5
|
+
type: string;
|
|
6
|
+
input: string;
|
|
7
|
+
checksum: string;
|
|
8
|
+
difficulty: number;
|
|
9
|
+
memory: number;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** Returns whether the first `difficulty` bits of a hash are zero. */
|
|
13
|
+
export function hasLeadingZeroBits(hash: Uint8Array, difficulty: number): boolean {
|
|
14
|
+
for (let bit = 0; bit < difficulty; bit += 1) {
|
|
15
|
+
const byte = hash[Math.floor(bit / 8)];
|
|
16
|
+
|
|
17
|
+
if (byte === undefined || (byte & (1 << (7 - (bit % 8)))) !== 0) return false;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
return true;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Solves the bandwidth challenge using a bounded zero-filled buffer. */
|
|
24
|
+
export function solveAwsWafBandwidth(difficulty: number): string {
|
|
25
|
+
const sizes = [1_024, 1_024, 10_240, 102_400, 1_048_576, 10_485_760];
|
|
26
|
+
|
|
27
|
+
return Buffer.alloc(sizes[Math.min(5, Math.max(0, difficulty))] ?? 1_024).toString("base64");
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Computes bounded proof-of-work inside a disposable worker. */
|
|
31
|
+
export function solveAwsWafWork(work: AwsWafWork): string | undefined {
|
|
32
|
+
if (!Number.isInteger(work.difficulty) || work.difficulty < 1 || work.difficulty > 32)
|
|
33
|
+
return undefined;
|
|
34
|
+
|
|
35
|
+
if (work.type === "NetworkBandwidth") return solveAwsWafBandwidth(work.difficulty);
|
|
36
|
+
|
|
37
|
+
if (work.type === "SHA256") {
|
|
38
|
+
for (let nonce = 0; nonce < 2_000_000; nonce += 1) {
|
|
39
|
+
const digest = createHash("sha256").update(`${work.input}${work.checksum}${nonce}`).digest();
|
|
40
|
+
|
|
41
|
+
if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
if (work.type === "HashcashScrypt") {
|
|
46
|
+
const cost = work.memory > 0 ? work.memory : 128;
|
|
47
|
+
|
|
48
|
+
// Reject large or invalid costs before invoking OpenSSL.
|
|
49
|
+
if (!Number.isInteger(cost) || cost < 2 || cost > 16_384 || (cost & (cost - 1)) !== 0)
|
|
50
|
+
return undefined;
|
|
51
|
+
|
|
52
|
+
for (let nonce = 0; nonce < 100_000; nonce += 1) {
|
|
53
|
+
const digest = scryptSync(`${work.input}${work.checksum}${nonce}`, work.checksum, 32, {
|
|
54
|
+
N: cost,
|
|
55
|
+
r: 8,
|
|
56
|
+
p: 1,
|
|
57
|
+
maxmem: 64 * 1_024 * 1_024,
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
return undefined;
|
|
65
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/* v8 ignore file -- exercised through worker integration tests. */
|
|
2
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
3
|
+
import { solveAwsWafWork, type AwsWafWork } from "./aws-waf-solver.ts";
|
|
4
|
+
|
|
5
|
+
const work: AwsWafWork = workerData;
|
|
6
|
+
|
|
7
|
+
try {
|
|
8
|
+
parentPort?.postMessage({ kind: "solved", solution: solveAwsWafWork(work) });
|
|
9
|
+
} catch (error) {
|
|
10
|
+
parentPort?.postMessage({
|
|
11
|
+
kind: "error",
|
|
12
|
+
message: error instanceof Error ? error.message : "AWS WAF worker failed.",
|
|
13
|
+
});
|
|
14
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import type { IncomingMessage } from "node:http";
|
|
2
|
+
import type { RedirectDependencies } from "../network/network-redirects.ts";
|
|
3
|
+
|
|
4
|
+
/** Cookie that carries an AWS WAF challenge token. */
|
|
5
|
+
export const AWS_WAF_TOKEN_COOKIE = "aws-waf-token";
|
|
6
|
+
|
|
7
|
+
/** A resolved AWS WAF token and the browser identity it was issued for. */
|
|
8
|
+
export interface AwsWafSolution {
|
|
9
|
+
token: string;
|
|
10
|
+
userAgent?: string;
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** A request to resolve one AWS WAF challenge. */
|
|
14
|
+
export interface AwsWafResolveContext {
|
|
15
|
+
/** The page URL withheld by the challenge. */
|
|
16
|
+
url: URL;
|
|
17
|
+
/** Cancels the solver request. */
|
|
18
|
+
signal: AbortSignal;
|
|
19
|
+
/** Network dependencies used by the resolver. */
|
|
20
|
+
network?: RedirectDependencies;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/** Resolves an AWS WAF challenge to a token accepted by the target origin. */
|
|
24
|
+
export interface AwsWafResolver {
|
|
25
|
+
resolve(context: AwsWafResolveContext): Promise<AwsWafSolution | undefined>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Returns whether a response carries an AWS WAF challenge. */
|
|
29
|
+
export function isAwsWafChallengeResponse(response: Pick<IncomingMessage, "headers">): boolean {
|
|
30
|
+
const value = response.headers["x-amzn-waf-action"];
|
|
31
|
+
const action = Array.isArray(value) ? value[0] : value;
|
|
32
|
+
|
|
33
|
+
return action?.trim().toLowerCase() === "challenge";
|
|
34
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -20,6 +20,13 @@ export {
|
|
|
20
20
|
type AnubisSolveContext,
|
|
21
21
|
} from "./challenges/anubis.ts";
|
|
22
22
|
|
|
23
|
+
export {
|
|
24
|
+
AWS_WAF_TOKEN_COOKIE,
|
|
25
|
+
isAwsWafChallengeResponse,
|
|
26
|
+
type AwsWafResolveContext,
|
|
27
|
+
type AwsWafResolver,
|
|
28
|
+
} from "./challenges/aws-waf.ts";
|
|
29
|
+
|
|
23
30
|
export { CookieJar } from "./network/cookies.ts";
|
|
24
31
|
|
|
25
32
|
export type { FetchResult } from "./document/content.ts";
|
package/src/network/cookies.ts
CHANGED
|
@@ -211,6 +211,47 @@ export class CookieJar {
|
|
|
211
211
|
this.#evict();
|
|
212
212
|
}
|
|
213
213
|
|
|
214
|
+
/**
|
|
215
|
+
* Stores a challenge token returned by a resolver.
|
|
216
|
+
*
|
|
217
|
+
* The token is untrusted input. It is accepted only for the target origin,
|
|
218
|
+
* scoped to the whole origin, and held in memory for the current acquisition.
|
|
219
|
+
*
|
|
220
|
+
* @param name - Cookie name
|
|
221
|
+
* @param value - Cookie value
|
|
222
|
+
* @param url - The page URL the cookie belongs to
|
|
223
|
+
* @returns Whether the cookie was accepted
|
|
224
|
+
*/
|
|
225
|
+
set(name: string, value: string, url: URL): boolean {
|
|
226
|
+
const token = value.trim();
|
|
227
|
+
|
|
228
|
+
if (
|
|
229
|
+
name.length === 0 ||
|
|
230
|
+
name.length > MAX_COOKIE_NAME_CHARACTERS ||
|
|
231
|
+
!COOKIE_NAME.test(name) ||
|
|
232
|
+
token.length === 0 ||
|
|
233
|
+
token.length > MAX_COOKIE_VALUE_CHARACTERS ||
|
|
234
|
+
hasInvalidCookieCharacter(token) ||
|
|
235
|
+
/[\s";,\\]/u.test(token)
|
|
236
|
+
) {
|
|
237
|
+
return false;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const key = `${url.origin}\0${name}\0/`;
|
|
241
|
+
this.#cookies.delete(key);
|
|
242
|
+
this.#cookies.set(key, {
|
|
243
|
+
name,
|
|
244
|
+
value: token,
|
|
245
|
+
origin: url.origin,
|
|
246
|
+
path: "/",
|
|
247
|
+
secure: url.protocol === "https:",
|
|
248
|
+
createdAt: Date.now(),
|
|
249
|
+
});
|
|
250
|
+
this.#evict();
|
|
251
|
+
|
|
252
|
+
return true;
|
|
253
|
+
}
|
|
254
|
+
|
|
214
255
|
/**
|
|
215
256
|
* Returns a Cookie header value for a URL, excluding expired, insecure, and
|
|
216
257
|
* cross-origin cookies.
|
|
@@ -29,11 +29,16 @@ export interface RedirectDependencies {
|
|
|
29
29
|
target: ValidatedTarget,
|
|
30
30
|
signal: AbortSignal,
|
|
31
31
|
headers?: Readonly<Record<string, string>>,
|
|
32
|
+
options?: { method?: string; body?: string | Uint8Array },
|
|
32
33
|
) => Promise<IncomingMessage>;
|
|
33
34
|
random?: () => number;
|
|
34
35
|
sleep?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
|
|
35
36
|
/** Origin-scoped cookies for the current acquisition, including challenge handshakes. */
|
|
36
37
|
cookies?: CookieJar;
|
|
38
|
+
/** User-Agent override that persists across redirects. */
|
|
39
|
+
userAgent?: string;
|
|
40
|
+
/** Headers that survive a redirect, used by challenge handshakes that need browser identity. */
|
|
41
|
+
persistentHeaders?: Readonly<Record<string, string>>;
|
|
37
42
|
}
|
|
38
43
|
|
|
39
44
|
/** Creates the consistent abort error used by redirect and retry coordination. */
|
|
@@ -144,7 +149,14 @@ async function coordinatedRequest(
|
|
|
144
149
|
const release = await acquireOrigin(target.url.origin, signal);
|
|
145
150
|
let response: IncomingMessage;
|
|
146
151
|
const cookieHeader = dependencies.cookies?.headerFor(target.url);
|
|
147
|
-
|
|
152
|
+
|
|
153
|
+
const headersWithUserAgent = dependencies.userAgent
|
|
154
|
+
? { "User-Agent": dependencies.userAgent, ...headers }
|
|
155
|
+
: headers;
|
|
156
|
+
|
|
157
|
+
const requestHeaders = cookieHeader
|
|
158
|
+
? { Cookie: cookieHeader, ...headersWithUserAgent }
|
|
159
|
+
: headersWithUserAgent;
|
|
148
160
|
|
|
149
161
|
try {
|
|
150
162
|
response = await request(target, signal, requestHeaders);
|
|
@@ -197,7 +209,8 @@ export async function requestFollowingRedirects(
|
|
|
197
209
|
response.resume();
|
|
198
210
|
target = await awaitWithAbort(validateUrl(new URL(location, target.url)), signal);
|
|
199
211
|
// Validators describe the originally cached resource and must not leak to a redirect target.
|
|
200
|
-
|
|
212
|
+
// Challenge handshakes opt into a fixed browser identity instead.
|
|
213
|
+
requestHeaders = dependencies.persistentHeaders ?? {};
|
|
201
214
|
}
|
|
202
215
|
|
|
203
216
|
throw new Error("web_fetch followed too many redirects.");
|
|
@@ -8,6 +8,9 @@ import type { ValidatedTarget } from "./network-policy.ts";
|
|
|
8
8
|
/** Maximum size in bytes for raw HTTP response bodies. */
|
|
9
9
|
export const FETCH_MAX_BYTES = 5 * 1_024 * 1_024;
|
|
10
10
|
|
|
11
|
+
/** Fixed User-Agent used for page requests and challenge solvers. */
|
|
12
|
+
export const WEB_FETCH_USER_AGENT = "Mozilla/5.0 (compatible; PiWebFetch/1.0; +https://pi.dev)";
|
|
13
|
+
|
|
11
14
|
/**
|
|
12
15
|
* Per-address connect deadline. A hanging address is abandoned after this
|
|
13
16
|
* budget so the next validated address can be tried inside the overall
|
|
@@ -67,6 +70,8 @@ async function requestOnce(
|
|
|
67
70
|
signal: AbortSignal,
|
|
68
71
|
attemptTimeoutMs: number,
|
|
69
72
|
headers: Readonly<Record<string, string>>,
|
|
73
|
+
method = "GET",
|
|
74
|
+
body?: string | Uint8Array,
|
|
70
75
|
): Promise<IncomingMessage> {
|
|
71
76
|
const family = isIP(address);
|
|
72
77
|
|
|
@@ -114,11 +119,12 @@ async function requestOnce(
|
|
|
114
119
|
target.url,
|
|
115
120
|
{
|
|
116
121
|
lookup,
|
|
122
|
+
method,
|
|
117
123
|
signal: controller.signal,
|
|
118
124
|
headers: {
|
|
119
125
|
Accept:
|
|
120
126
|
"text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, application/xhtml+xml;q=0.7, application/json;q=0.7, application/xml;q=0.7, */*;q=0.1",
|
|
121
|
-
"User-Agent":
|
|
127
|
+
"User-Agent": WEB_FETCH_USER_AGENT,
|
|
122
128
|
...headers,
|
|
123
129
|
},
|
|
124
130
|
},
|
|
@@ -139,7 +145,7 @@ async function requestOnce(
|
|
|
139
145
|
if (attemptExpired) finish(() => reject(unreachableError()));
|
|
140
146
|
else finish(() => reject(error));
|
|
141
147
|
});
|
|
142
|
-
outgoing.end();
|
|
148
|
+
outgoing.end(body);
|
|
143
149
|
});
|
|
144
150
|
}
|
|
145
151
|
|
|
@@ -148,6 +154,10 @@ export interface RequestPinnedOptions {
|
|
|
148
154
|
attemptTimeoutMs?: number;
|
|
149
155
|
/** Additional fixed request headers, used for conditional cache revalidation. */
|
|
150
156
|
headers?: Readonly<Record<string, string>>;
|
|
157
|
+
/** HTTP method; defaults to GET. */
|
|
158
|
+
method?: string;
|
|
159
|
+
/** Request body for non-GET requests. */
|
|
160
|
+
body?: string | Uint8Array;
|
|
151
161
|
}
|
|
152
162
|
|
|
153
163
|
/**
|
|
@@ -169,7 +179,15 @@ export async function requestPinned(
|
|
|
169
179
|
|
|
170
180
|
for (const address of addresses) {
|
|
171
181
|
try {
|
|
172
|
-
return await requestOnce(
|
|
182
|
+
return await requestOnce(
|
|
183
|
+
target,
|
|
184
|
+
address,
|
|
185
|
+
signal,
|
|
186
|
+
attemptTimeoutMs,
|
|
187
|
+
options.headers ?? {},
|
|
188
|
+
options.method,
|
|
189
|
+
options.body,
|
|
190
|
+
);
|
|
173
191
|
} catch (error) {
|
|
174
192
|
lastError = error;
|
|
175
193
|
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
import { createHash, scryptSync } from "node:crypto";
|
|
2
|
+
|
|
3
|
+
/** Returns whether the first `difficulty` bits of a hash are zero. */
|
|
4
|
+
export function hasLeadingZeroBits(hash, difficulty) {
|
|
5
|
+
for (let bit = 0; bit < difficulty; bit += 1) {
|
|
6
|
+
const byte = hash[Math.floor(bit / 8)];
|
|
7
|
+
|
|
8
|
+
if (byte === undefined || (byte & (1 << (7 - (bit % 8)))) !== 0) return false;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
return true;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/** Solves the bandwidth challenge using a bounded zero-filled buffer. */
|
|
15
|
+
export function solveAwsWafBandwidth(difficulty) {
|
|
16
|
+
const sizes = [1_024, 1_024, 10_240, 102_400, 1_048_576, 10_485_760];
|
|
17
|
+
|
|
18
|
+
return Buffer.alloc(sizes[Math.min(5, Math.max(0, difficulty))] ?? 1_024).toString("base64");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Computes bounded proof-of-work inside a disposable worker. */
|
|
22
|
+
export function solveAwsWafWork(work) {
|
|
23
|
+
if (!Number.isInteger(work.difficulty) || work.difficulty < 1 || work.difficulty > 32)
|
|
24
|
+
return undefined;
|
|
25
|
+
|
|
26
|
+
if (work.type === "NetworkBandwidth") return solveAwsWafBandwidth(work.difficulty);
|
|
27
|
+
|
|
28
|
+
if (work.type === "SHA256") {
|
|
29
|
+
for (let nonce = 0; nonce < 2_000_000; nonce += 1) {
|
|
30
|
+
const digest = createHash("sha256").update(`${work.input}${work.checksum}${nonce}`).digest();
|
|
31
|
+
|
|
32
|
+
if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
if (work.type === "HashcashScrypt") {
|
|
37
|
+
const cost = work.memory > 0 ? work.memory : 128;
|
|
38
|
+
|
|
39
|
+
// Reject large or invalid costs before invoking OpenSSL.
|
|
40
|
+
if (!Number.isInteger(cost) || cost < 2 || cost > 16_384 || (cost & (cost - 1)) !== 0)
|
|
41
|
+
return undefined;
|
|
42
|
+
|
|
43
|
+
for (let nonce = 0; nonce < 100_000; nonce += 1) {
|
|
44
|
+
const digest = scryptSync(`${work.input}${work.checksum}${nonce}`, work.checksum, 32, {
|
|
45
|
+
N: cost,
|
|
46
|
+
r: 8,
|
|
47
|
+
p: 1,
|
|
48
|
+
maxmem: 64 * 1_024 * 1_024,
|
|
49
|
+
});
|
|
50
|
+
|
|
51
|
+
if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
return undefined;
|
|
56
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/* v8 ignore file -- exercised through worker integration tests. */
|
|
2
|
+
import { parentPort, workerData } from "node:worker_threads";
|
|
3
|
+
import { solveAwsWafWork } from "./aws-waf-solver.js";
|
|
4
|
+
|
|
5
|
+
const work = workerData;
|
|
6
|
+
|
|
7
|
+
try {
|
|
8
|
+
parentPort?.postMessage({ kind: "solved", solution: solveAwsWafWork(work) });
|
|
9
|
+
} catch (error) {
|
|
10
|
+
parentPort?.postMessage({
|
|
11
|
+
kind: "error",
|
|
12
|
+
message: error instanceof Error ? error.message : "AWS WAF worker failed.",
|
|
13
|
+
});
|
|
14
|
+
}
|