@zeldrisho/pi-web-fetch 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/CHANGELOG.md CHANGED
@@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
7
7
 
8
8
  ## [Unreleased]
9
9
 
10
+ ## [0.16.0] - 2026-10-28
11
+
12
+ ### Added
13
+
14
+ - Automatically resolve supported AWS WAF challenges when fetching pages.
15
+
10
16
  ## [0.15.0] - 2026-10-28
11
17
 
12
18
  ### Added
@@ -289,7 +295,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
289
295
 
290
296
  - Add bounded public page fetching ([b979496](https://github.com/zeldrisho/pi-packages/commit/b979496b32de1cead172ba570307e4a4a7b3421d))
291
297
 
292
- [Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.15.0...HEAD
298
+ [Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.16.0...HEAD
299
+ [0.16.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.15.0...pi-web-fetch-v0.16.0
293
300
  [0.15.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.14.1...pi-web-fetch-v0.15.0
294
301
  [0.14.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.14.0...pi-web-fetch-v0.14.1
295
302
  [0.14.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.13.0...pi-web-fetch-v0.14.0
package/README.md CHANGED
@@ -23,15 +23,13 @@ The `web_fetch` tool accepts public HTTP and HTTPS URLs and supports HTML, Markd
23
23
  | `query` | Optional; selects matching sections from a long page. Sections remain in source order. |
24
24
  | `offset` | Optional character offset for continuing through extracted content; takes precedence over a URL fragment. |
25
25
 
26
- Output is bounded to 6,000 characters by default. For long pages, use the returned `nextOffset` to continue. A query selects a focused view of the fetched page; omit it to read the complete document. Large fenced source files can be focused in line-aligned excerpts; excerpts are not necessarily valid standalone source. OpenAPI and Swagger JSON or YAML documents and JavaScript bundles containing literal specs can be reported as `openapi-spec`; detection is best-effort and limited to source bodies up to 512 KiB. Pi's preview renders Markdown with the active theme and is collapsed by default (`Ctrl+O` expands it). The call preview shows the URL; the focus query and character offset are not repeated there.
26
+ Output is bounded to 6,000 characters by default. For long pages, use the returned `nextOffset` to continue. Use `query` to focus on matching sections, or omit it to read the complete document. Pi's preview is collapsed by default; press `Ctrl+O` to expand it.
27
27
 
28
28
  For example, a URL such as `https://github.com/shadcn-ui/lint#get-started` starts at the matching heading. GitHub `blob` links and bare gists are converted to raw source URLs; repository roots use their README. GitHub directory listings are not supported.
29
29
 
30
30
  ### Page alternatives and bot challenges
31
31
 
32
- The tool can discover advertised Markdown versions and `/llms.txt` indexes. When an index is discovered, its URL is shown in the result so you can follow it if the fetched page is insufficient; when the index replaces the requested page, the result explains that fallback. Markdown files served with a generic binary content type are read as text. Anubis proof-of-work challenges are supported, and a page withheld by an AWS WAF or Cloudflare bot challenge falls back to the site's documentation index when one exists. Other unsupported challenges and JavaScript CAPTCHAs return a blocked result. There is no browser fallback. Challenge cookies are not reused between fetches.
33
-
34
- Pages with HTML nested beyond 512 elements are rejected.
32
+ The tool can discover advertised Markdown versions and `/llms.txt` indexes. An index URL appears in the result when found; follow it if the page is insufficient. The tool can solve Anubis and supported AWS WAF proof-of-work challenges. If an AWS WAF or Cloudflare challenge remains blocked, it may fall back to the site's documentation index. Unsupported challenges and JavaScript CAPTCHAs return a blocked result; there is no browser fallback.
35
33
 
36
34
  ### Caching
37
35
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@zeldrisho/pi-web-fetch",
3
- "version": "0.15.0",
3
+ "version": "0.16.0",
4
4
  "description": "Pi extension for secure, bounded public web page fetching and Markdown extraction",
5
5
  "keywords": [
6
6
  "pi",
@@ -14,6 +14,13 @@ import {
14
14
  parseAnubisChallenge,
15
15
  solveAnubisChallenge,
16
16
  } from "../challenges/anubis.ts";
17
+ import {
18
+ AWS_WAF_TOKEN_COOKIE,
19
+ isAwsWafChallengeResponse,
20
+ type AwsWafResolver,
21
+ type AwsWafSolution,
22
+ } from "../challenges/aws-waf.ts";
23
+ import { nativeAwsWafResolver } from "../challenges/aws-waf-resolver.ts";
17
24
  import {
18
25
  sliceCompleteDocument,
19
26
  type CompleteDocument,
@@ -89,6 +96,10 @@ export interface FetchRemoteDependencies extends RedirectDependencies {
89
96
  timeoutMs?: number;
90
97
  /** When false, an Anubis interstitial is reported as blocked instead of solved. Defaults to true. */
91
98
  solveAnubisChallenges?: boolean;
99
+ /** When false, an AWS WAF challenge is reported as blocked instead of solved. Defaults to true. */
100
+ solveAwsWafChallenges?: boolean;
101
+ /** Resolver used for AWS WAF challenges; defaults to the in-process resolver. */
102
+ awsWafResolver?: AwsWafResolver;
92
103
  }
93
104
 
94
105
  /**
@@ -190,7 +201,7 @@ export class ChallengeBlockedError extends Error {
190
201
  function assertSuccessfulResponse(response: IncomingMessage): void {
191
202
  const status = response.statusCode ?? 0;
192
203
 
193
- if (responseHeader(response, "x-amzn-waf-action")?.trim().toLowerCase() === "challenge") {
204
+ if (isAwsWafChallengeResponse(response)) {
194
205
  response.resume();
195
206
  throw new ChallengeBlockedError(
196
207
  `web_fetch was blocked by an AWS WAF anti-bot challenge (HTTP ${status}).`,
@@ -493,6 +504,37 @@ function assertAbsoluteHttpUrlForFetch(value: string): URL {
493
504
  return url;
494
505
  }
495
506
 
507
+ /**
508
+ * Attempts one WAF token resolution while preserving caller cancellation.
509
+ * Solver failures are intentionally converted into the normal blocked path.
510
+ */
511
+ async function resolveAwsWafToken(
512
+ resolver: AwsWafResolver,
513
+ target: ValidatedTarget,
514
+ signal: AbortSignal,
515
+ network: RedirectDependencies,
516
+ budgetMs: number,
517
+ ): Promise<AwsWafSolution | undefined> {
518
+ if (budgetMs <= 0) return undefined;
519
+
520
+ const controller = new AbortController();
521
+ const timeout = setTimeout(() => controller.abort(), budgetMs);
522
+ const resolverSignal = AbortSignal.any([signal, controller.signal]);
523
+
524
+ try {
525
+ return await awaitWithAbort(
526
+ resolver.resolve({ url: target.url, signal: resolverSignal, network }),
527
+ resolverSignal,
528
+ );
529
+ } catch (error) {
530
+ if (signal.aborted) throw error;
531
+
532
+ return undefined;
533
+ } finally {
534
+ clearTimeout(timeout);
535
+ }
536
+ }
537
+
496
538
  /**
497
539
  * Fetches a URL and returns the complete document with extracted content.
498
540
  *
@@ -513,6 +555,7 @@ async function fetchDocument(
513
555
  assertAbsoluteHttpUrlForFetch(normalizeGitHubRawUrl(rawUrl));
514
556
  const controller = new AbortController();
515
557
  const timeoutMs = dependencies.timeoutMs ?? REQUEST_TIMEOUT_MS;
558
+ const deadline = Date.now() + timeoutMs;
516
559
  const extractHtml = dependencies.extractHtml ?? extractHtmlToMarkdown;
517
560
  let timedOut = false;
518
561
 
@@ -579,6 +622,48 @@ async function fetchDocument(
579
622
  }
580
623
  }
581
624
 
625
+ if (
626
+ isAwsWafChallengeResponse(response) &&
627
+ requestDependencies.solveAwsWafChallenges !== false
628
+ ) {
629
+ const resolver = requestDependencies.awsWafResolver ?? nativeAwsWafResolver;
630
+
631
+ if (resolver) {
632
+ response.resume();
633
+
634
+ const solution = await resolveAwsWafToken(
635
+ resolver,
636
+ target,
637
+ controller.signal,
638
+ requestDependencies,
639
+ Math.floor(Math.max(0, deadline - Date.now()) / 2),
640
+ );
641
+
642
+ if (solution) {
643
+ const stored = requestDependencies.cookies?.set(
644
+ AWS_WAF_TOKEN_COOKIE,
645
+ solution.token,
646
+ target.url,
647
+ );
648
+
649
+ if (stored) {
650
+ // Retry once inside the same validated redirect and cookie boundary.
651
+ // Keep the browser identity that earned the token, and report a
652
+ // second challenge as blocked instead of looping.
653
+ const retryDependencies = solution.userAgent
654
+ ? { ...requestDependencies, userAgent: solution.userAgent }
655
+ : requestDependencies;
656
+
657
+ ({ target, response } = await requestFollowingRedirects(
658
+ target.url,
659
+ controller.signal,
660
+ retryDependencies,
661
+ ));
662
+ }
663
+ }
664
+ }
665
+ }
666
+
582
667
  if (response.statusCode === 304 && cached) {
583
668
  response.resume();
584
669
  const etag = responseHeader(response, "etag") ?? cached.validators?.etag;
@@ -191,7 +191,7 @@ async function probeUsableRawText(
191
191
  const document = await fetchCompleteDocument(
192
192
  candidate.toString(),
193
193
  sharedSignal,
194
- { ...dependencies, solveAnubisChallenges: false },
194
+ { ...dependencies, solveAnubisChallenges: false, solveAwsWafChallenges: false },
195
195
  // An index probe never accepts an empty body.
196
196
  false,
197
197
  // A probe wants exactly this representation; retrying as HTML only burns
@@ -0,0 +1,806 @@
1
+ import { createCipheriv, randomBytes, randomInt, randomUUID } from "node:crypto";
2
+ import { Worker } from "node:worker_threads";
3
+ import type { AwsWafWork } from "./aws-waf-solver.ts";
4
+ import type { IncomingMessage } from "node:http";
5
+ import { crc32 } from "node:zlib";
6
+ import type { AwsWafResolveContext, AwsWafResolver, AwsWafSolution } from "./aws-waf.ts";
7
+ import { CookieJar } from "../network/cookies.ts";
8
+ import {
9
+ requestFollowingRedirects,
10
+ type RedirectDependencies,
11
+ } from "../network/network-redirects.ts";
12
+ import { validateRemoteUrl } from "../network/network-policy.ts";
13
+ import {
14
+ FETCH_MAX_BYTES,
15
+ decodeResponse,
16
+ decompressResponse,
17
+ readResponseBytes,
18
+ requestPinned,
19
+ responseHeader,
20
+ } from "../network/network-transport.ts";
21
+
22
+ /** AWS WAF browser-challenge signal encryption key used by the current challenge family. */
23
+ const SIGNAL_KEY = Buffer.from(
24
+ "6f71a512b1e035eaab53d8be73120d3fb68a0ca346b9560aab3e5cdf753d5e98",
25
+ "hex",
26
+ );
27
+
28
+ /** Browser identity used for the challenge handshake and the retry. */
29
+ const BROWSER_USER_AGENT =
30
+ "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36";
31
+
32
+ const BROWSER_VERSION = "144";
33
+
34
+ const SCREEN = "1920-1080-1080-24-*-*-*";
35
+
36
+ const PLUGINS = [
37
+ { name: "PDF Viewer", str: "PDF Viewer " },
38
+ { name: "Chrome PDF Viewer", str: "Chrome PDF Viewer " },
39
+ { name: "Chromium PDF Viewer", str: "Chromium PDF Viewer " },
40
+ { name: "Microsoft Edge PDF Viewer", str: "Microsoft Edge PDF Viewer " },
41
+ { name: "WebKit built-in PDF", str: "WebKit built-in PDF " },
42
+ ];
43
+
44
+ const PLUGIN_STRING = PLUGINS.map((plugin) => plugin.str).join("");
45
+
46
+ const MATH_VALUES = {
47
+ tan: "-1.4214488238747245",
48
+ sin: "0.8178819121159085",
49
+ cos: "-0.5753861119575491",
50
+ };
51
+
52
+ const GPU_EXTENSIONS = [
53
+ "ANGLE_instanced_arrays",
54
+ "EXT_blend_minmax",
55
+ "EXT_clip_control",
56
+ "EXT_color_buffer_half_float",
57
+ "EXT_depth_clamp",
58
+ "EXT_disjoint_timer_query",
59
+ "EXT_float_blend",
60
+ "EXT_frag_depth",
61
+ "EXT_polygon_offset_clamp",
62
+ "EXT_shader_texture_lod",
63
+ "EXT_texture_compression_bptc",
64
+ "EXT_texture_compression_rgtc",
65
+ "EXT_texture_filter_anisotropic",
66
+ "EXT_texture_mirror_clamp_to_edge",
67
+ "EXT_sRGB",
68
+ "KHR_parallel_shader_compile",
69
+ "OES_element_index_uint",
70
+ "OES_fbo_render_mipmap",
71
+ "OES_standard_derivatives",
72
+ "OES_texture_float",
73
+ "OES_texture_float_linear",
74
+ "OES_texture_half_float",
75
+ "OES_texture_half_float_linear",
76
+ "OES_vertex_array_object",
77
+ "WEBGL_blend_func_extended",
78
+ "WEBGL_color_buffer_float",
79
+ "WEBGL_compressed_texture_astc",
80
+ "WEBGL_compressed_texture_etc",
81
+ "WEBGL_compressed_texture_etc1",
82
+ "WEBGL_compressed_texture_s3tc",
83
+ "WEBGL_compressed_texture_s3tc_srgb",
84
+ "WEBGL_debug_renderer_info",
85
+ "WEBGL_debug_shaders",
86
+ "WEBGL_depth_texture",
87
+ "WEBGL_lose_context",
88
+ "WEBGL_multi_draw",
89
+ "WEBGL_polygon_mode",
90
+ ];
91
+
92
+ /** Maps challenge type hashes to their proof-of-work family. */
93
+ function normalizeChallengeType(value: string): string {
94
+ if (value.startsWith("ha9faaffd")) return "NetworkBandwidth";
95
+
96
+ if (value.startsWith("h72f957df")) return "HashcashScrypt";
97
+
98
+ if (value.startsWith("h7b0c470f")) return "SHA256";
99
+
100
+ return value;
101
+ }
102
+
103
+ /** Returns the submission endpoint for a challenge type. */
104
+ function challengeEndpoint(challengeType: string): string {
105
+ return challengeType === "NetworkBandwidth" ? "mp_verify" : "verify";
106
+ }
107
+
108
+ const CHALLENGE_SAME_ORIGIN = /(\/__challenge_[A-Za-z0-9]+\/[a-f0-9]+\/[a-f0-9]+)/;
109
+
110
+ const CHALLENGE_EXTERNAL =
111
+ /(https:\/\/[a-z0-9]+\.[a-z0-9]+\.[a-z0-9-]+\.token\.awswaf\.com\/[^/\s"]+\/[^/\s"]+\/[^/\s"]+)/;
112
+
113
+ const CHALLENGE_SDK =
114
+ /(https:\/\/[a-z0-9]+\.edge\.sdk\.awswaf\.com\/[a-z0-9]+\/[a-z0-9]+)\/challenge\.js/;
115
+
116
+ const GOKU_PROPS = /window\.gokuProps\s*=\s*(\{[^}]+\})/;
117
+
118
+ type JsonValue = string | number | boolean | null | JsonValue[] | { [key: string]: JsonValue };
119
+
120
+ type JsonObject = { [key: string]: JsonValue };
121
+
122
+ interface TextResponse {
123
+ url: URL;
124
+ status: number;
125
+ text: string;
126
+ }
127
+
128
+ interface FingerprintMetrics {
129
+ fp2: number;
130
+ browser: number;
131
+ capabilities: number;
132
+ gpu: number;
133
+ dnt: number;
134
+ math: number;
135
+ screen: number;
136
+ navigator: number;
137
+ auto: number;
138
+ stealth: number;
139
+ subtle: number;
140
+ canvas: number;
141
+ formdetector: number;
142
+ be: number;
143
+ }
144
+
145
+ interface Collector {
146
+ name: keyof FingerprintMetrics;
147
+ metric: string;
148
+ low: number;
149
+ high: number;
150
+ }
151
+
152
+ const COLLECTORS: readonly Collector[] = [
153
+ { name: "fp2", metric: "100", low: 0.5, high: 3 },
154
+ { name: "browser", metric: "101", low: 0, high: 1 },
155
+ { name: "capabilities", metric: "102", low: 2, high: 8 },
156
+ { name: "gpu", metric: "103", low: 3, high: 12 },
157
+ { name: "dnt", metric: "104", low: 0, high: 1 },
158
+ { name: "math", metric: "105", low: 0, high: 1 },
159
+ { name: "screen", metric: "106", low: 0, high: 1 },
160
+ { name: "navigator", metric: "107", low: 0, high: 1 },
161
+ { name: "auto", metric: "108", low: 0, high: 1 },
162
+ { name: "stealth", metric: "undefined", low: 1, high: 4 },
163
+ { name: "subtle", metric: "110", low: 0, high: 1 },
164
+ { name: "canvas", metric: "111", low: 80, high: 200 },
165
+ { name: "formdetector", metric: "112", low: 0, high: 3 },
166
+ { name: "be", metric: "undefined", low: 0, high: 1 },
167
+ ];
168
+
169
+ interface ChallengeData {
170
+ input: string;
171
+ hmac?: string;
172
+ region?: string;
173
+ }
174
+
175
+ interface ChallengeInputs {
176
+ challenge: ChallengeData;
177
+ challengeType: string;
178
+ difficulty: number;
179
+ memory: number;
180
+ }
181
+
182
+ interface BuiltMetrics {
183
+ metrics: JsonValue[];
184
+ fingerprint: FingerprintMetrics;
185
+ }
186
+
187
+ /** Returns whether a value is a plain JSON object. */
188
+ // oxlint-disable-next-line anti-slop/no-unknown-parameters -- parses untrusted challenge JSON at the I/O boundary
189
+ function isJsonObject(value: unknown): value is JsonObject {
190
+ // oxlint-disable-next-line anti-slop/no-runtime-typeof -- validates untrusted challenge JSON before use
191
+ return typeof value === "object" && value !== null && !Array.isArray(value);
192
+ }
193
+
194
+ /** Parses untrusted JSON text into a plain object. */
195
+ function parseJsonObject(text: string): JsonObject | undefined {
196
+ try {
197
+ const parsed: unknown = JSON.parse(text);
198
+
199
+ return isJsonObject(parsed) ? parsed : undefined;
200
+ } catch {
201
+ return undefined;
202
+ }
203
+ }
204
+
205
+ /** Reads a string field, returning undefined for missing or non-string values. */
206
+ function stringField(value: JsonObject, key: string): string | undefined {
207
+ const field = value[key];
208
+
209
+ // oxlint-disable-next-line anti-slop/no-runtime-typeof -- validates an untrusted JSON field
210
+ return typeof field === "string" ? field : undefined;
211
+ }
212
+
213
+ /** Reads a numeric field, returning undefined for missing or non-numeric values. */
214
+ function numberField(value: JsonObject, key: string): number | undefined {
215
+ const field = value[key];
216
+
217
+ // oxlint-disable-next-line anti-slop/no-runtime-typeof -- validates an untrusted JSON field
218
+ return typeof field === "number" && Number.isFinite(field) ? field : undefined;
219
+ }
220
+
221
+ /** Returns a random floating-point metric in the supplied range. */
222
+ function randomMetric(low: number, high: number): number {
223
+ return Math.round((low + Math.random() * (high - low)) * 10) / 10;
224
+ }
225
+
226
+ /** Builds realistic-looking WAF metrics and the matching fingerprint metric map. */
227
+ function buildMetrics(hasToken: boolean): BuiltMetrics {
228
+ const fingerprint: FingerprintMetrics = {
229
+ fp2: 0,
230
+ browser: 0,
231
+ capabilities: 0,
232
+ gpu: 0,
233
+ dnt: 0,
234
+ math: 0,
235
+ screen: 0,
236
+ navigator: 0,
237
+ auto: 0,
238
+ stealth: 0,
239
+ subtle: 0,
240
+ canvas: 0,
241
+ formdetector: 0,
242
+ be: 0,
243
+ };
244
+
245
+ const metrics: JsonValue[] = [];
246
+
247
+ for (const collector of COLLECTORS) {
248
+ const value = randomMetric(collector.low, collector.high);
249
+
250
+ fingerprint[collector.name] = Math.trunc(value);
251
+ metrics.push({ name: collector.metric, value, unit: "2" });
252
+ }
253
+
254
+ const encoding = randomMetric(0.5, 3);
255
+ const encryption = randomMetric(2, 8);
256
+
257
+ const acquisition =
258
+ COLLECTORS.reduce((total, collector) => total + fingerprint[collector.name], 0) +
259
+ encoding +
260
+ encryption +
261
+ randomMetric(2, 6);
262
+
263
+ const challenge = randomMetric(2, 8);
264
+ const cookie = randomMetric(0.1, 1);
265
+ const total = acquisition + challenge + cookie;
266
+
267
+ metrics.push(
268
+ { name: "2", value: encoding, unit: "2" },
269
+ { name: "3", value: encryption, unit: "2" },
270
+ { name: "7", value: hasToken ? 1 : 0, unit: "4" },
271
+ { name: "1", value: acquisition, unit: "2" },
272
+ { name: "4", value: challenge, unit: "2" },
273
+ { name: "5", value: cookie, unit: "2" },
274
+ { name: "6", value: total, unit: "2" },
275
+ { name: "8", value: 1, unit: "4" },
276
+ );
277
+
278
+ return { metrics, fingerprint };
279
+ }
280
+
281
+ /** Builds a browser fingerprint object accepted by the WAF signal decoder. */
282
+ function buildSignal(site: string, fingerprint: FingerprintMetrics) {
283
+ const now = Date.now();
284
+ const canvasHash = randomInt(100_000_000, 1_000_000_000);
285
+ const histogram = Array.from({ length: 256 }, () => randomInt(1, 50));
286
+
287
+ return {
288
+ metrics: { ...fingerprint },
289
+ start: now,
290
+ flashVersion: null,
291
+ plugins: PLUGINS.map((plugin) => ({ name: plugin.name, str: plugin.str })),
292
+ dupedPlugins: `${PLUGIN_STRING}||${SCREEN}`,
293
+ screenInfo: SCREEN,
294
+ referrer: "",
295
+ userAgent: BROWSER_USER_AGENT,
296
+ location: `${site}/`,
297
+ webDriver: false,
298
+ capabilities: {
299
+ css: {
300
+ textShadow: 1,
301
+ WebkitTextStroke: 1,
302
+ boxShadow: 1,
303
+ borderRadius: 1,
304
+ borderImage: 1,
305
+ opacity: 1,
306
+ transform: 1,
307
+ transition: 1,
308
+ },
309
+ js: {
310
+ audio: true,
311
+ geolocation: true,
312
+ localStorage: "supported",
313
+ touch: false,
314
+ video: true,
315
+ webWorker: true,
316
+ },
317
+ elapsed: fingerprint.capabilities,
318
+ },
319
+ gpu: {
320
+ vendor: "Google Inc. (NVIDIA Corporation)",
321
+ model: "ANGLE (NVIDIA Corporation, NVIDIA GeForce RTX 5070/PCIe/SSE2, OpenGL ES 3.2)",
322
+ extensions: GPU_EXTENSIONS,
323
+ },
324
+ dnt: null,
325
+ math: MATH_VALUES,
326
+ automation: {
327
+ wd: { properties: { document: [], window: [], navigator: [] } },
328
+ phantom: { properties: { window: [] } },
329
+ },
330
+ stealth: { t1: 0, t2: 0, i: 1, mte: 0, mtd: false },
331
+ crypto: {
332
+ crypto: 1,
333
+ subtle: 1,
334
+ encrypt: true,
335
+ decrypt: true,
336
+ wrapKey: true,
337
+ unwrapKey: true,
338
+ sign: true,
339
+ verify: true,
340
+ digest: true,
341
+ deriveBits: true,
342
+ deriveKey: true,
343
+ getRandomValues: true,
344
+ randomUUID: true,
345
+ },
346
+ canvas: { hash: canvasHash, emailHash: null, histogramBins: histogram },
347
+ formDetected: false,
348
+ numForms: 0,
349
+ numFormElements: 0,
350
+ be: { si: false },
351
+ end: now + 1,
352
+ errors: [],
353
+ version: "2.4.0",
354
+ id: randomUUID(),
355
+ };
356
+ }
357
+
358
+ /** Builds and encrypts the browser signal payload. */
359
+ function encodeSignal(site: string, hasToken: boolean) {
360
+ const built = buildMetrics(hasToken);
361
+ const signal = buildSignal(site, built.fingerprint);
362
+ const plaintextJson = JSON.stringify(signal);
363
+ const checksum = crc32(plaintextJson).toString(16).toUpperCase().padStart(8, "0");
364
+ const iv = randomBytes(12);
365
+ const cipher = createCipheriv("aes-256-gcm", SIGNAL_KEY, iv);
366
+
367
+ const ciphertext = Buffer.concat([
368
+ cipher.update(`${checksum}#${plaintextJson}`, "utf8"),
369
+ cipher.final(),
370
+ ]);
371
+
372
+ const tag = cipher.getAuthTag();
373
+
374
+ return {
375
+ encrypted: `${iv.toString("base64")}::${tag.toString("hex")}::${ciphertext.toString("hex")}`,
376
+ checksum,
377
+ metrics: built.metrics,
378
+ signal,
379
+ };
380
+ }
381
+
382
+ /** Builds the signal entry used in challenge submissions. */
383
+ function signalEntry(encrypted: string): JsonValue[] {
384
+ return [{ name: "Zoey", value: { Present: encrypted } }];
385
+ }
386
+
387
+ /** Run CPU-intensive work off the main thread and terminate it on cancellation. */
388
+ function solveAwsWafChallenge(
389
+ inputs: ChallengeInputs,
390
+ checksum: string,
391
+ signal: AbortSignal,
392
+ ): Promise<string | undefined> {
393
+ signal.throwIfAborted();
394
+
395
+ const work: AwsWafWork = {
396
+ type: inputs.challengeType,
397
+ input: inputs.challenge.input,
398
+ checksum,
399
+ difficulty: inputs.difficulty,
400
+ memory: inputs.memory,
401
+ };
402
+
403
+ const worker = new Worker(new URL("../../workers/aws-waf-worker.js", import.meta.url), {
404
+ workerData: work,
405
+ resourceLimits: { maxOldGenerationSizeMb: 64, maxYoungGenerationSizeMb: 16 },
406
+ });
407
+
408
+ return new Promise((resolve, reject) => {
409
+ let settled = false;
410
+
411
+ const timer = setTimeout(
412
+ () => finish(() => reject(new Error("AWS WAF worker timed out."))),
413
+ 60_000,
414
+ );
415
+
416
+ function finish(callback: () => void): void {
417
+ if (settled) return;
418
+ settled = true;
419
+ clearTimeout(timer);
420
+ signal.removeEventListener("abort", onAbort);
421
+ void worker.terminate();
422
+ callback();
423
+ }
424
+
425
+ function onAbort(): void {
426
+ finish(() => reject(signal.reason));
427
+ }
428
+
429
+ worker.on(
430
+ "message",
431
+ (message: { kind: "solved"; solution?: string } | { kind: "error"; message: string }) => {
432
+ if (message.kind === "solved") finish(() => resolve(message.solution));
433
+ else finish(() => reject(new Error(message.message)));
434
+ },
435
+ );
436
+ worker.on("error", (error) => finish(() => reject(error)));
437
+ worker.on("exit", (code) =>
438
+ finish(() => reject(new Error(`AWS WAF worker exited with code ${code}.`))),
439
+ );
440
+
441
+ if (signal.aborted) onAbort();
442
+ else signal.addEventListener("abort", onAbort, { once: true });
443
+ });
444
+ }
445
+
446
+ /** Parses the base64 challenge input into challenge metadata. */
447
+ function decodeChallengeInput(input: string): JsonObject | undefined {
448
+ try {
449
+ return parseJsonObject(Buffer.from(input, "base64").toString("utf8"));
450
+ } catch {
451
+ return undefined;
452
+ }
453
+ }
454
+
455
+ /** Reads challenge inputs from the `/inputs` response. */
456
+ function parseChallengeInputs(value: JsonObject): ChallengeInputs | undefined {
457
+ const challenge = value["challenge"];
458
+
459
+ if (!isJsonObject(challenge)) return undefined;
460
+
461
+ const input = stringField(challenge, "input");
462
+
463
+ if (!input) return undefined;
464
+
465
+ const decoded = decodeChallengeInput(input);
466
+
467
+ const challengeType =
468
+ (decoded ? stringField(decoded, "challenge_type") : undefined) ??
469
+ stringField(value, "challenge_type");
470
+
471
+ if (!challengeType) return undefined;
472
+
473
+ const difficulty =
474
+ (decoded ? numberField(decoded, "difficulty") : undefined) ??
475
+ numberField(value, "difficulty") ??
476
+ 1;
477
+
478
+ const memory =
479
+ (decoded ? numberField(decoded, "memory") : undefined) ?? numberField(value, "memory") ?? 128;
480
+
481
+ return {
482
+ challenge: {
483
+ input,
484
+ hmac: stringField(challenge, "hmac"),
485
+ region: stringField(challenge, "region"),
486
+ },
487
+ challengeType: normalizeChallengeType(challengeType),
488
+ difficulty,
489
+ memory,
490
+ };
491
+ }
492
+
493
+ /** Extracts the challenge URL and goku properties from a WAF page. */
494
+ export function extractAwsWafChallenge(
495
+ html: string,
496
+ site: URL,
497
+ ): { challengeUrl: string; gokuProps?: JsonObject } | undefined {
498
+ const sameOrigin = CHALLENGE_SAME_ORIGIN.exec(html);
499
+ const external = CHALLENGE_EXTERNAL.exec(html);
500
+ const sdk = CHALLENGE_SDK.exec(html);
501
+ const match = sameOrigin ?? external ?? sdk;
502
+
503
+ if (!match) return undefined;
504
+
505
+ const challengeUrl = sameOrigin ? new URL(match[1], site).href : external ? match[1] : match[1];
506
+ const gokuMatch = GOKU_PROPS.exec(html);
507
+ const gokuProps = gokuMatch ? parseJsonObject(gokuMatch[1]) : undefined;
508
+
509
+ return { challengeUrl, gokuProps };
510
+ }
511
+
512
+ /** Builds the browser headers used to request the protected page. */
513
+ function browserHeaders() {
514
+ return {
515
+ accept:
516
+ "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
517
+ "accept-language": "en-US,en;q=0.9",
518
+ "sec-ch-ua": `"Not:A-Brand";v="8", "Chromium";v="${BROWSER_VERSION}", "Google Chrome";v="${BROWSER_VERSION}"`,
519
+ "sec-ch-ua-mobile": "?0",
520
+ "sec-ch-ua-platform": '"Windows"',
521
+ "sec-fetch-dest": "document",
522
+ "sec-fetch-mode": "navigate",
523
+ "sec-fetch-site": "none",
524
+ "sec-fetch-user": "?1",
525
+ "upgrade-insecure-requests": "1",
526
+ };
527
+ }
528
+
529
+ /** Builds the headers used for challenge API calls. */
530
+ function apiHeaders(site: string) {
531
+ return {
532
+ accept: "*/*",
533
+ "accept-language": "en-US,en;q=0.9",
534
+ "cache-control": "no-cache",
535
+ origin: site,
536
+ pragma: "no-cache",
537
+ priority: "u=1, i",
538
+ referer: `${site}/`,
539
+ "sec-ch-ua": `"Not:A-Brand";v="8", "Chromium";v="${BROWSER_VERSION}", "Google Chrome";v="${BROWSER_VERSION}"`,
540
+ "sec-ch-ua-mobile": "?0",
541
+ "sec-ch-ua-platform": '"Windows"',
542
+ "sec-fetch-dest": "empty",
543
+ "sec-fetch-mode": "cors",
544
+ "sec-fetch-site": "cross-site",
545
+ };
546
+ }
547
+
548
+ /** Reads and decodes a bounded response body. */
549
+ async function readText(response: IncomingMessage, signal: AbortSignal): Promise<string> {
550
+ const bytes = await readResponseBytes(response, FETCH_MAX_BYTES, signal);
551
+ const decompressed = decompressResponse(bytes, responseHeader(response, "content-encoding"));
552
+
553
+ return decodeResponse(decompressed, responseHeader(response, "content-type") ?? "text/plain");
554
+ }
555
+
556
+ /** Performs a GET through the validated redirect and cookie boundary. */
557
+ async function getText(
558
+ url: string,
559
+ network: RedirectDependencies,
560
+ signal: AbortSignal,
561
+ headers: Readonly<Record<string, string>>,
562
+ ): Promise<TextResponse> {
563
+ const { target, response } = await requestFollowingRedirects(url, signal, network, headers);
564
+ const text = await readText(response, signal);
565
+
566
+ return { url: target.url, status: response.statusCode ?? 0, text };
567
+ }
568
+
569
+ /** Performs a POST to a validated target without following redirects. */
570
+ async function postText(
571
+ url: string,
572
+ network: RedirectDependencies,
573
+ signal: AbortSignal,
574
+ headers: Readonly<Record<string, string>>,
575
+ body: string,
576
+ ): Promise<TextResponse> {
577
+ const validateUrl = network.validateUrl ?? validateRemoteUrl;
578
+ const target = await validateUrl(url);
579
+ const cookieHeader = network.cookies?.headerFor(target.url);
580
+
581
+ const requestHeaders = { ...headers };
582
+
583
+ if (network.userAgent) requestHeaders["User-Agent"] = network.userAgent;
584
+
585
+ if (cookieHeader) requestHeaders.Cookie = cookieHeader;
586
+
587
+ const request =
588
+ network.request ??
589
+ ((value, requestSignal, requestHeadersInit, options) =>
590
+ requestPinned(value, requestSignal, {
591
+ headers: requestHeadersInit,
592
+ method: options?.method,
593
+ body: options?.body,
594
+ }));
595
+
596
+ const response = await request(target, signal, requestHeaders, { method: "POST", body });
597
+ const text = await readText(response, signal);
598
+
599
+ return { url: target.url, status: response.statusCode ?? 0, text };
600
+ }
601
+
602
+ /** Builds the multipart body used by bandwidth challenges. */
603
+ function buildMultipart(
604
+ domain: string,
605
+ challenge: ChallengeData,
606
+ solution: string,
607
+ checksum: string,
608
+ encrypted: string,
609
+ metrics: JsonValue[],
610
+ gokuProps: JsonObject | undefined,
611
+ ) {
612
+ const boundary = `----WebKitFormBoundary${randomBytes(16).toString("hex")}`;
613
+
614
+ const metadata = {
615
+ challenge: {
616
+ input: challenge.input,
617
+ hmac: challenge.hmac ?? null,
618
+ region: challenge.region ?? null,
619
+ },
620
+ solution: null,
621
+ signals: signalEntry(encrypted),
622
+ checksum,
623
+ existing_token: null,
624
+ client: "Browser",
625
+ domain,
626
+ metrics,
627
+ goku_props: gokuProps ?? null,
628
+ };
629
+
630
+ const body = [
631
+ `--${boundary}`,
632
+ 'Content-Disposition: form-data; name="solution_data"',
633
+ "",
634
+ solution,
635
+ `--${boundary}`,
636
+ 'Content-Disposition: form-data; name="solution_metadata"',
637
+ "",
638
+ JSON.stringify(metadata),
639
+ `--${boundary}--`,
640
+ "",
641
+ ].join("\r\n");
642
+
643
+ return { body, contentType: `multipart/form-data; boundary=${boundary}` };
644
+ }
645
+
646
+ /** Builds the JSON body used by verify challenges. */
647
+ function buildVerificationPayload(
648
+ domain: string,
649
+ challenge: ChallengeData,
650
+ solution: string,
651
+ checksum: string,
652
+ encrypted: string,
653
+ metrics: JsonValue[],
654
+ existingToken: string | undefined,
655
+ gokuProps: JsonObject | undefined,
656
+ ) {
657
+ return {
658
+ challenge: {
659
+ input: challenge.input,
660
+ hmac: challenge.hmac ?? null,
661
+ region: challenge.region ?? null,
662
+ },
663
+ solution,
664
+ signals: signalEntry(encrypted),
665
+ checksum,
666
+ existing_token: existingToken ?? null,
667
+ client: "Browser",
668
+ domain,
669
+ metrics,
670
+ goku_props: gokuProps ?? null,
671
+ };
672
+ }
673
+
674
+ /** Reads a token from a challenge response, including nested response objects. */
675
+ function parseToken(value: JsonObject): string | undefined {
676
+ const direct = stringField(value, "token");
677
+
678
+ if (direct) return direct;
679
+
680
+ const nested = value["response"];
681
+
682
+ return isJsonObject(nested) ? stringField(nested, "token") : undefined;
683
+ }
684
+
685
+ /** Reads replacement challenge inputs from a challenge response. */
686
+ function parseReplacementInputs(value: JsonObject): ChallengeInputs | undefined {
687
+ const nested = value["response"];
688
+
689
+ if (isJsonObject(nested)) {
690
+ const inputs = nested["inputs"];
691
+
692
+ if (isJsonObject(inputs)) return parseChallengeInputs(inputs);
693
+ }
694
+
695
+ const inputs = value["inputs"];
696
+
697
+ return isJsonObject(inputs) ? parseChallengeInputs(inputs) : undefined;
698
+ }
699
+
700
+ /** Runs the native AWS WAF challenge flow and returns a token. */
701
+ async function resolveAwsWafNative(
702
+ context: AwsWafResolveContext,
703
+ ): Promise<AwsWafSolution | undefined> {
704
+ const network: RedirectDependencies = {
705
+ ...context.network,
706
+ userAgent: BROWSER_USER_AGENT,
707
+ persistentHeaders: browserHeaders(),
708
+ cookies: context.network?.cookies ?? new CookieJar(),
709
+ };
710
+
711
+ const site = context.url.origin;
712
+ const domain = context.url.hostname;
713
+
714
+ const page = await getText(context.url.href, network, context.signal, browserHeaders());
715
+
716
+ if (page.status !== 200 && page.status !== 202) return undefined;
717
+
718
+ const discovery = extractAwsWafChallenge(page.text, context.url);
719
+
720
+ if (!discovery) return undefined;
721
+
722
+ const inputsResponse = await getText(
723
+ `${discovery.challengeUrl}/inputs?client=browser`,
724
+ network,
725
+ context.signal,
726
+ apiHeaders(site),
727
+ );
728
+
729
+ if (inputsResponse.status !== 200) return undefined;
730
+
731
+ let inputs = parseChallengeInputs(parseJsonObject(inputsResponse.text) ?? {});
732
+
733
+ if (!inputs) return undefined;
734
+
735
+ let token: string | undefined;
736
+
737
+ for (let round = 0; round < 2; round += 1) {
738
+ const signal = encodeSignal(site, token !== undefined);
739
+ const solution = await solveAwsWafChallenge(inputs, signal.checksum, context.signal);
740
+
741
+ if (!solution) return undefined;
742
+
743
+ const endpoint = challengeEndpoint(inputs.challengeType);
744
+ let body: string;
745
+ let contentType = "text/plain;charset=UTF-8";
746
+
747
+ if (endpoint === "mp_verify") {
748
+ const multipart = buildMultipart(
749
+ domain,
750
+ inputs.challenge,
751
+ solution,
752
+ signal.checksum,
753
+ signal.encrypted,
754
+ signal.metrics,
755
+ discovery.gokuProps,
756
+ );
757
+
758
+ body = multipart.body;
759
+ contentType = multipart.contentType;
760
+ } else {
761
+ body = JSON.stringify(
762
+ buildVerificationPayload(
763
+ domain,
764
+ inputs.challenge,
765
+ solution,
766
+ signal.checksum,
767
+ signal.encrypted,
768
+ signal.metrics,
769
+ token,
770
+ discovery.gokuProps,
771
+ ),
772
+ );
773
+ }
774
+
775
+ const headers = { ...apiHeaders(site), "content-type": contentType };
776
+
777
+ const response = await postText(
778
+ `${discovery.challengeUrl}/${endpoint}`,
779
+ network,
780
+ context.signal,
781
+ headers,
782
+ body,
783
+ );
784
+
785
+ if (response.status !== 200) return undefined;
786
+
787
+ const parsed = parseJsonObject(response.text);
788
+
789
+ if (!parsed) return undefined;
790
+
791
+ token = parseToken(parsed) ?? token;
792
+
793
+ if (token) break;
794
+
795
+ const replacement = parseReplacementInputs(parsed);
796
+
797
+ if (replacement) inputs = replacement;
798
+ }
799
+
800
+ return token ? { token, userAgent: BROWSER_USER_AGENT } : undefined;
801
+ }
802
+
803
+ /** Native in-process AWS WAF resolver. */
804
+ export const nativeAwsWafResolver: AwsWafResolver = {
805
+ resolve: resolveAwsWafNative,
806
+ };
@@ -0,0 +1,65 @@
1
+ import { createHash, scryptSync } from "node:crypto";
2
+
3
+ /** Bounded proof-of-work request sent to the AWS WAF worker. */
4
+ export interface AwsWafWork {
5
+ type: string;
6
+ input: string;
7
+ checksum: string;
8
+ difficulty: number;
9
+ memory: number;
10
+ }
11
+
12
+ /** Returns whether the first `difficulty` bits of a hash are zero. */
13
+ export function hasLeadingZeroBits(hash: Uint8Array, difficulty: number): boolean {
14
+ for (let bit = 0; bit < difficulty; bit += 1) {
15
+ const byte = hash[Math.floor(bit / 8)];
16
+
17
+ if (byte === undefined || (byte & (1 << (7 - (bit % 8)))) !== 0) return false;
18
+ }
19
+
20
+ return true;
21
+ }
22
+
23
+ /** Solves the bandwidth challenge using a bounded zero-filled buffer. */
24
+ export function solveAwsWafBandwidth(difficulty: number): string {
25
+ const sizes = [1_024, 1_024, 10_240, 102_400, 1_048_576, 10_485_760];
26
+
27
+ return Buffer.alloc(sizes[Math.min(5, Math.max(0, difficulty))] ?? 1_024).toString("base64");
28
+ }
29
+
30
+ /** Computes bounded proof-of-work inside a disposable worker. */
31
+ export function solveAwsWafWork(work: AwsWafWork): string | undefined {
32
+ if (!Number.isInteger(work.difficulty) || work.difficulty < 1 || work.difficulty > 32)
33
+ return undefined;
34
+
35
+ if (work.type === "NetworkBandwidth") return solveAwsWafBandwidth(work.difficulty);
36
+
37
+ if (work.type === "SHA256") {
38
+ for (let nonce = 0; nonce < 2_000_000; nonce += 1) {
39
+ const digest = createHash("sha256").update(`${work.input}${work.checksum}${nonce}`).digest();
40
+
41
+ if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
42
+ }
43
+ }
44
+
45
+ if (work.type === "HashcashScrypt") {
46
+ const cost = work.memory > 0 ? work.memory : 128;
47
+
48
+ // Reject large or invalid costs before invoking OpenSSL.
49
+ if (!Number.isInteger(cost) || cost < 2 || cost > 16_384 || (cost & (cost - 1)) !== 0)
50
+ return undefined;
51
+
52
+ for (let nonce = 0; nonce < 100_000; nonce += 1) {
53
+ const digest = scryptSync(`${work.input}${work.checksum}${nonce}`, work.checksum, 32, {
54
+ N: cost,
55
+ r: 8,
56
+ p: 1,
57
+ maxmem: 64 * 1_024 * 1_024,
58
+ });
59
+
60
+ if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
61
+ }
62
+ }
63
+
64
+ return undefined;
65
+ }
@@ -0,0 +1,14 @@
1
+ /* v8 ignore file -- exercised through worker integration tests. */
2
+ import { parentPort, workerData } from "node:worker_threads";
3
+ import { solveAwsWafWork, type AwsWafWork } from "./aws-waf-solver.ts";
4
+
5
+ const work: AwsWafWork = workerData;
6
+
7
+ try {
8
+ parentPort?.postMessage({ kind: "solved", solution: solveAwsWafWork(work) });
9
+ } catch (error) {
10
+ parentPort?.postMessage({
11
+ kind: "error",
12
+ message: error instanceof Error ? error.message : "AWS WAF worker failed.",
13
+ });
14
+ }
@@ -0,0 +1,34 @@
1
+ import type { IncomingMessage } from "node:http";
2
+ import type { RedirectDependencies } from "../network/network-redirects.ts";
3
+
4
+ /** Cookie that carries an AWS WAF challenge token. */
5
+ export const AWS_WAF_TOKEN_COOKIE = "aws-waf-token";
6
+
7
+ /** A resolved AWS WAF token and the browser identity it was issued for. */
8
+ export interface AwsWafSolution {
9
+ token: string;
10
+ userAgent?: string;
11
+ }
12
+
13
+ /** A request to resolve one AWS WAF challenge. */
14
+ export interface AwsWafResolveContext {
15
+ /** The page URL withheld by the challenge. */
16
+ url: URL;
17
+ /** Cancels the solver request. */
18
+ signal: AbortSignal;
19
+ /** Network dependencies used by the resolver. */
20
+ network?: RedirectDependencies;
21
+ }
22
+
23
+ /** Resolves an AWS WAF challenge to a token accepted by the target origin. */
24
+ export interface AwsWafResolver {
25
+ resolve(context: AwsWafResolveContext): Promise<AwsWafSolution | undefined>;
26
+ }
27
+
28
+ /** Returns whether a response carries an AWS WAF challenge. */
29
+ export function isAwsWafChallengeResponse(response: Pick<IncomingMessage, "headers">): boolean {
30
+ const value = response.headers["x-amzn-waf-action"];
31
+ const action = Array.isArray(value) ? value[0] : value;
32
+
33
+ return action?.trim().toLowerCase() === "challenge";
34
+ }
package/src/index.ts CHANGED
@@ -20,6 +20,13 @@ export {
20
20
  type AnubisSolveContext,
21
21
  } from "./challenges/anubis.ts";
22
22
 
23
+ export {
24
+ AWS_WAF_TOKEN_COOKIE,
25
+ isAwsWafChallengeResponse,
26
+ type AwsWafResolveContext,
27
+ type AwsWafResolver,
28
+ } from "./challenges/aws-waf.ts";
29
+
23
30
  export { CookieJar } from "./network/cookies.ts";
24
31
 
25
32
  export type { FetchResult } from "./document/content.ts";
@@ -211,6 +211,47 @@ export class CookieJar {
211
211
  this.#evict();
212
212
  }
213
213
 
214
+ /**
215
+ * Stores a challenge token returned by a resolver.
216
+ *
217
+ * The token is untrusted input. It is accepted only for the target origin,
218
+ * scoped to the whole origin, and held in memory for the current acquisition.
219
+ *
220
+ * @param name - Cookie name
221
+ * @param value - Cookie value
222
+ * @param url - The page URL the cookie belongs to
223
+ * @returns Whether the cookie was accepted
224
+ */
225
+ set(name: string, value: string, url: URL): boolean {
226
+ const token = value.trim();
227
+
228
+ if (
229
+ name.length === 0 ||
230
+ name.length > MAX_COOKIE_NAME_CHARACTERS ||
231
+ !COOKIE_NAME.test(name) ||
232
+ token.length === 0 ||
233
+ token.length > MAX_COOKIE_VALUE_CHARACTERS ||
234
+ hasInvalidCookieCharacter(token) ||
235
+ /[\s";,\\]/u.test(token)
236
+ ) {
237
+ return false;
238
+ }
239
+
240
+ const key = `${url.origin}\0${name}\0/`;
241
+ this.#cookies.delete(key);
242
+ this.#cookies.set(key, {
243
+ name,
244
+ value: token,
245
+ origin: url.origin,
246
+ path: "/",
247
+ secure: url.protocol === "https:",
248
+ createdAt: Date.now(),
249
+ });
250
+ this.#evict();
251
+
252
+ return true;
253
+ }
254
+
214
255
  /**
215
256
  * Returns a Cookie header value for a URL, excluding expired, insecure, and
216
257
  * cross-origin cookies.
@@ -29,11 +29,16 @@ export interface RedirectDependencies {
29
29
  target: ValidatedTarget,
30
30
  signal: AbortSignal,
31
31
  headers?: Readonly<Record<string, string>>,
32
+ options?: { method?: string; body?: string | Uint8Array },
32
33
  ) => Promise<IncomingMessage>;
33
34
  random?: () => number;
34
35
  sleep?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
35
36
  /** Origin-scoped cookies for the current acquisition, including challenge handshakes. */
36
37
  cookies?: CookieJar;
38
+ /** User-Agent override that persists across redirects. */
39
+ userAgent?: string;
40
+ /** Headers that survive a redirect, used by challenge handshakes that need browser identity. */
41
+ persistentHeaders?: Readonly<Record<string, string>>;
37
42
  }
38
43
 
39
44
  /** Creates the consistent abort error used by redirect and retry coordination. */
@@ -144,7 +149,14 @@ async function coordinatedRequest(
144
149
  const release = await acquireOrigin(target.url.origin, signal);
145
150
  let response: IncomingMessage;
146
151
  const cookieHeader = dependencies.cookies?.headerFor(target.url);
147
- const requestHeaders = cookieHeader ? { Cookie: cookieHeader, ...headers } : headers;
152
+
153
+ const headersWithUserAgent = dependencies.userAgent
154
+ ? { "User-Agent": dependencies.userAgent, ...headers }
155
+ : headers;
156
+
157
+ const requestHeaders = cookieHeader
158
+ ? { Cookie: cookieHeader, ...headersWithUserAgent }
159
+ : headersWithUserAgent;
148
160
 
149
161
  try {
150
162
  response = await request(target, signal, requestHeaders);
@@ -197,7 +209,8 @@ export async function requestFollowingRedirects(
197
209
  response.resume();
198
210
  target = await awaitWithAbort(validateUrl(new URL(location, target.url)), signal);
199
211
  // Validators describe the originally cached resource and must not leak to a redirect target.
200
- requestHeaders = {};
212
+ // Challenge handshakes opt into a fixed browser identity instead.
213
+ requestHeaders = dependencies.persistentHeaders ?? {};
201
214
  }
202
215
 
203
216
  throw new Error("web_fetch followed too many redirects.");
@@ -8,6 +8,9 @@ import type { ValidatedTarget } from "./network-policy.ts";
8
8
  /** Maximum size in bytes for raw HTTP response bodies. */
9
9
  export const FETCH_MAX_BYTES = 5 * 1_024 * 1_024;
10
10
 
11
+ /** Fixed User-Agent used for page requests and challenge solvers. */
12
+ export const WEB_FETCH_USER_AGENT = "Mozilla/5.0 (compatible; PiWebFetch/1.0; +https://pi.dev)";
13
+
11
14
  /**
12
15
  * Per-address connect deadline. A hanging address is abandoned after this
13
16
  * budget so the next validated address can be tried inside the overall
@@ -67,6 +70,8 @@ async function requestOnce(
67
70
  signal: AbortSignal,
68
71
  attemptTimeoutMs: number,
69
72
  headers: Readonly<Record<string, string>>,
73
+ method = "GET",
74
+ body?: string | Uint8Array,
70
75
  ): Promise<IncomingMessage> {
71
76
  const family = isIP(address);
72
77
 
@@ -114,11 +119,12 @@ async function requestOnce(
114
119
  target.url,
115
120
  {
116
121
  lookup,
122
+ method,
117
123
  signal: controller.signal,
118
124
  headers: {
119
125
  Accept:
120
126
  "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, application/xhtml+xml;q=0.7, application/json;q=0.7, application/xml;q=0.7, */*;q=0.1",
121
- "User-Agent": "Mozilla/5.0 (compatible; PiWebFetch/1.0; +https://pi.dev)",
127
+ "User-Agent": WEB_FETCH_USER_AGENT,
122
128
  ...headers,
123
129
  },
124
130
  },
@@ -139,7 +145,7 @@ async function requestOnce(
139
145
  if (attemptExpired) finish(() => reject(unreachableError()));
140
146
  else finish(() => reject(error));
141
147
  });
142
- outgoing.end();
148
+ outgoing.end(body);
143
149
  });
144
150
  }
145
151
 
@@ -148,6 +154,10 @@ export interface RequestPinnedOptions {
148
154
  attemptTimeoutMs?: number;
149
155
  /** Additional fixed request headers, used for conditional cache revalidation. */
150
156
  headers?: Readonly<Record<string, string>>;
157
+ /** HTTP method; defaults to GET. */
158
+ method?: string;
159
+ /** Request body for non-GET requests. */
160
+ body?: string | Uint8Array;
151
161
  }
152
162
 
153
163
  /**
@@ -169,7 +179,15 @@ export async function requestPinned(
169
179
 
170
180
  for (const address of addresses) {
171
181
  try {
172
- return await requestOnce(target, address, signal, attemptTimeoutMs, options.headers ?? {});
182
+ return await requestOnce(
183
+ target,
184
+ address,
185
+ signal,
186
+ attemptTimeoutMs,
187
+ options.headers ?? {},
188
+ options.method,
189
+ options.body,
190
+ );
173
191
  } catch (error) {
174
192
  lastError = error;
175
193
 
@@ -0,0 +1,56 @@
1
+ import { createHash, scryptSync } from "node:crypto";
2
+
3
+ /** Returns whether the first `difficulty` bits of a hash are zero. */
4
+ export function hasLeadingZeroBits(hash, difficulty) {
5
+ for (let bit = 0; bit < difficulty; bit += 1) {
6
+ const byte = hash[Math.floor(bit / 8)];
7
+
8
+ if (byte === undefined || (byte & (1 << (7 - (bit % 8)))) !== 0) return false;
9
+ }
10
+
11
+ return true;
12
+ }
13
+
14
+ /** Solves the bandwidth challenge using a bounded zero-filled buffer. */
15
+ export function solveAwsWafBandwidth(difficulty) {
16
+ const sizes = [1_024, 1_024, 10_240, 102_400, 1_048_576, 10_485_760];
17
+
18
+ return Buffer.alloc(sizes[Math.min(5, Math.max(0, difficulty))] ?? 1_024).toString("base64");
19
+ }
20
+
21
+ /** Computes bounded proof-of-work inside a disposable worker. */
22
+ export function solveAwsWafWork(work) {
23
+ if (!Number.isInteger(work.difficulty) || work.difficulty < 1 || work.difficulty > 32)
24
+ return undefined;
25
+
26
+ if (work.type === "NetworkBandwidth") return solveAwsWafBandwidth(work.difficulty);
27
+
28
+ if (work.type === "SHA256") {
29
+ for (let nonce = 0; nonce < 2_000_000; nonce += 1) {
30
+ const digest = createHash("sha256").update(`${work.input}${work.checksum}${nonce}`).digest();
31
+
32
+ if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
33
+ }
34
+ }
35
+
36
+ if (work.type === "HashcashScrypt") {
37
+ const cost = work.memory > 0 ? work.memory : 128;
38
+
39
+ // Reject large or invalid costs before invoking OpenSSL.
40
+ if (!Number.isInteger(cost) || cost < 2 || cost > 16_384 || (cost & (cost - 1)) !== 0)
41
+ return undefined;
42
+
43
+ for (let nonce = 0; nonce < 100_000; nonce += 1) {
44
+ const digest = scryptSync(`${work.input}${work.checksum}${nonce}`, work.checksum, 32, {
45
+ N: cost,
46
+ r: 8,
47
+ p: 1,
48
+ maxmem: 64 * 1_024 * 1_024,
49
+ });
50
+
51
+ if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
52
+ }
53
+ }
54
+
55
+ return undefined;
56
+ }
@@ -0,0 +1,14 @@
1
+ /* v8 ignore file -- exercised through worker integration tests. */
2
+ import { parentPort, workerData } from "node:worker_threads";
3
+ import { solveAwsWafWork } from "./aws-waf-solver.js";
4
+
5
+ const work = workerData;
6
+
7
+ try {
8
+ parentPort?.postMessage({ kind: "solved", solution: solveAwsWafWork(work) });
9
+ } catch (error) {
10
+ parentPort?.postMessage({
11
+ kind: "error",
12
+ message: error instanceof Error ? error.message : "AWS WAF worker failed.",
13
+ });
14
+ }