@zeldrisho/pi-web-fetch 0.15.0 → 0.16.1

package/CHANGELOG.md

@@ -7,6 +7,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.16.1] - 2026-10-28
+
+### Changed
+
+- Show focus queries in tool calls, and display expansion and continuation notices with clearer spacing.
+
+## [0.16.0] - 2026-10-28
+
+### Added
+
+- Automatically resolve supported AWS WAF challenges when fetching pages.
+
 ## [0.15.0] - 2026-10-28
 
 ### Added
@@ -289,7 +301,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add bounded public page fetching ([b979496](https://github.com/zeldrisho/pi-packages/commit/b979496b32de1cead172ba570307e4a4a7b3421d))
 
-[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.15.0...HEAD
+[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.16.1...HEAD
+[0.16.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.16.0...pi-web-fetch-v0.16.1
+[0.16.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.15.0...pi-web-fetch-v0.16.0
 [0.15.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.14.1...pi-web-fetch-v0.15.0
 [0.14.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.14.0...pi-web-fetch-v0.14.1
 [0.14.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.13.0...pi-web-fetch-v0.14.0

package/README.md

@@ -23,15 +23,13 @@ The `web_fetch` tool accepts public HTTP and HTTPS URLs and supports HTML, Markd
 | `query`   | Optional; selects matching sections from a long page. Sections remain in source order.                    |
 | `offset`  | Optional character offset for continuing through extracted content; takes precedence over a URL fragment. |
 
-Output is bounded to 6,000 characters by default. For long pages, use the returned `nextOffset` to continue. A query selects a focused view of the fetched page; omit it to read the complete document. Large fenced source files can be focused in line-aligned excerpts; excerpts are not necessarily valid standalone source. OpenAPI and Swagger JSON or YAML documents and JavaScript bundles containing literal specs can be reported as `openapi-spec`; detection is best-effort and limited to source bodies up to 512 KiB. Pi's preview renders Markdown with the active theme and is collapsed by default (`Ctrl+O` expands it). The call preview shows the URL; the focus query and character offset are not repeated there.
+Output is bounded to 6,000 characters by default. For long pages, use the returned `nextOffset` to continue. Use `query` to focus on matching sections, or omit it to read the complete document. Pi's preview is collapsed by default; press `Ctrl+O` to expand it.
 
-For example, a URL such as `https://github.com/shadcn-ui/lint#get-started` starts at the matching heading. GitHub `blob` links and bare gists are converted to raw source URLs; repository roots use their README. GitHub directory listings are not supported.
+For example, a URL such as `https://github.com/zeldrisho/pi-packages/blob/main/packages/pi-web-fetch/README.md#usage` starts at the matching heading. GitHub `blob` links and bare gists are converted to raw source URLs; repository roots use their README. GitHub directory listings are not supported.
 
 ### Page alternatives and bot challenges
 
-The tool can discover advertised Markdown versions and `/llms.txt` indexes. When an index is discovered, its URL is shown in the result so you can follow it if the fetched page is insufficient; when the index replaces the requested page, the result explains that fallback. Markdown files served with a generic binary content type are read as text. Anubis proof-of-work challenges are supported, and a page withheld by an AWS WAF or Cloudflare bot challenge falls back to the site's documentation index when one exists. Other unsupported challenges and JavaScript CAPTCHAs return a blocked result. There is no browser fallback. Challenge cookies are not reused between fetches.
-
-Pages with HTML nested beyond 512 elements are rejected.
+The tool can discover advertised Markdown versions and `/llms.txt` indexes. An index URL appears in the result when found; follow it if the page is insufficient. The tool can solve Anubis and supported AWS WAF proof-of-work challenges. If an AWS WAF or Cloudflare challenge remains blocked, it may fall back to the site's documentation index. Unsupported challenges and JavaScript CAPTCHAs return a blocked result; there is no browser fallback.
 
 ### Caching
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zeldrisho/pi-web-fetch",
-  "version": "0.15.0",
+  "version": "0.16.1",
   "description": "Pi extension for secure, bounded public web page fetching and Markdown extraction",
   "keywords": [
     "pi",

package/src/acquisition/fetch.ts

@@ -14,6 +14,13 @@ import {
   parseAnubisChallenge,
   solveAnubisChallenge,
 } from "../challenges/anubis.ts";
+import {
+  AWS_WAF_TOKEN_COOKIE,
+  isAwsWafChallengeResponse,
+  type AwsWafResolver,
+  type AwsWafSolution,
+} from "../challenges/aws-waf.ts";
+import { nativeAwsWafResolver } from "../challenges/aws-waf-resolver.ts";
 import {
   sliceCompleteDocument,
   type CompleteDocument,
@@ -89,6 +96,10 @@ export interface FetchRemoteDependencies extends RedirectDependencies {
   timeoutMs?: number;
   /** When false, an Anubis interstitial is reported as blocked instead of solved. Defaults to true. */
   solveAnubisChallenges?: boolean;
+  /** When false, an AWS WAF challenge is reported as blocked instead of solved. Defaults to true. */
+  solveAwsWafChallenges?: boolean;
+  /** Resolver used for AWS WAF challenges; defaults to the in-process resolver. */
+  awsWafResolver?: AwsWafResolver;
 }
 
 /**
@@ -190,7 +201,7 @@ export class ChallengeBlockedError extends Error {
 function assertSuccessfulResponse(response: IncomingMessage): void {
   const status = response.statusCode ?? 0;
 
-  if (responseHeader(response, "x-amzn-waf-action")?.trim().toLowerCase() === "challenge") {
+  if (isAwsWafChallengeResponse(response)) {
     response.resume();
     throw new ChallengeBlockedError(
       `web_fetch was blocked by an AWS WAF anti-bot challenge (HTTP ${status}).`,
@@ -493,6 +504,37 @@ function assertAbsoluteHttpUrlForFetch(value: string): URL {
   return url;
 }
 
+/**
+ * Attempts one WAF token resolution while preserving caller cancellation.
+ * Solver failures are intentionally converted into the normal blocked path.
+ */
+async function resolveAwsWafToken(
+  resolver: AwsWafResolver,
+  target: ValidatedTarget,
+  signal: AbortSignal,
+  network: RedirectDependencies,
+  budgetMs: number,
+): Promise<AwsWafSolution | undefined> {
+  if (budgetMs <= 0) return undefined;
+
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort(), budgetMs);
+  const resolverSignal = AbortSignal.any([signal, controller.signal]);
+
+  try {
+    return await awaitWithAbort(
+      resolver.resolve({ url: target.url, signal: resolverSignal, network }),
+      resolverSignal,
+    );
+  } catch (error) {
+    if (signal.aborted) throw error;
+
+    return undefined;
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
 /**
  * Fetches a URL and returns the complete document with extracted content.
  *
@@ -513,6 +555,7 @@ async function fetchDocument(
   assertAbsoluteHttpUrlForFetch(normalizeGitHubRawUrl(rawUrl));
   const controller = new AbortController();
   const timeoutMs = dependencies.timeoutMs ?? REQUEST_TIMEOUT_MS;
+  const deadline = Date.now() + timeoutMs;
   const extractHtml = dependencies.extractHtml ?? extractHtmlToMarkdown;
   let timedOut = false;
 
@@ -579,6 +622,48 @@ async function fetchDocument(
       }
     }
 
+    if (
+      isAwsWafChallengeResponse(response) &&
+      requestDependencies.solveAwsWafChallenges !== false
+    ) {
+      const resolver = requestDependencies.awsWafResolver ?? nativeAwsWafResolver;
+
+      if (resolver) {
+        response.resume();
+
+        const solution = await resolveAwsWafToken(
+          resolver,
+          target,
+          controller.signal,
+          requestDependencies,
+          Math.floor(Math.max(0, deadline - Date.now()) / 2),
+        );
+
+        if (solution) {
+          const stored = requestDependencies.cookies?.set(
+            AWS_WAF_TOKEN_COOKIE,
+            solution.token,
+            target.url,
+          );
+
+          if (stored) {
+            // Retry once inside the same validated redirect and cookie boundary.
+            // Keep the browser identity that earned the token, and report a
+            // second challenge as blocked instead of looping.
+            const retryDependencies = solution.userAgent
+              ? { ...requestDependencies, userAgent: solution.userAgent }
+              : requestDependencies;
+
+            ({ target, response } = await requestFollowingRedirects(
+              target.url,
+              controller.signal,
+              retryDependencies,
+            ));
+          }
+        }
+      }
+    }
+
     if (response.statusCode === 304 && cached) {
       response.resume();
       const etag = responseHeader(response, "etag") ?? cached.validators?.etag;

package/src/acquisition/llms.ts

@@ -191,7 +191,7 @@ async function probeUsableRawText(
         const document = await fetchCompleteDocument(
           candidate.toString(),
           sharedSignal,
-          { ...dependencies, solveAnubisChallenges: false },
+          { ...dependencies, solveAnubisChallenges: false, solveAwsWafChallenges: false },
           // An index probe never accepts an empty body.
           false,
           // A probe wants exactly this representation; retrying as HTML only burns

package/src/cache.ts

@@ -39,6 +39,26 @@ export function stableKeyHash(key: string): string {
   return createHash("sha256").update(key).digest("hex");
 }
 
+/** Hash canonical JSON, sorting object keys recursively while preserving array order. */
+// oxlint-disable-next-line anti-slop/no-unknown-parameters -- JSON serialization defines the accepted values.
+export function canonicalKey(value: unknown): string {
+  // oxlint-disable-next-line anti-slop/no-unknown-parameters -- JSON replacer receives arbitrary JSON values.
+  const json = JSON.stringify(value, (_key, entry: unknown) => {
+    // oxlint-disable-next-line anti-slop/no-runtime-typeof -- recursively sort JSON objects.
+    if (entry !== null && typeof entry === "object" && !Array.isArray(entry)) {
+      return Object.fromEntries(
+        Object.entries(entry).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)),
+      );
+    }
+
+    return entry;
+  });
+
+  if (json === undefined) throw new TypeError("Cache key must be JSON serializable");
+
+  return stableKeyHash(json);
+}
+
 /**
  * Resolves a cache entry's on-disk path and refuses to escape the cache
  * directory, guarding against a `keyToPath` that returns `..` segments. The