@zeldrisho/pi-web-fetch 0.14.1 → 0.16.0

package/src/challenges/aws-waf-solver.ts

@@ -0,0 +1,65 @@
+import { createHash, scryptSync } from "node:crypto";
+
+/** Bounded proof-of-work request sent to the AWS WAF worker. */
+export interface AwsWafWork {
+  type: string;
+  input: string;
+  checksum: string;
+  difficulty: number;
+  memory: number;
+}
+
+/** Returns whether the first `difficulty` bits of a hash are zero. */
+export function hasLeadingZeroBits(hash: Uint8Array, difficulty: number): boolean {
+  for (let bit = 0; bit < difficulty; bit += 1) {
+    const byte = hash[Math.floor(bit / 8)];
+
+    if (byte === undefined || (byte & (1 << (7 - (bit % 8)))) !== 0) return false;
+  }
+
+  return true;
+}
+
+/** Solves the bandwidth challenge using a bounded zero-filled buffer. */
+export function solveAwsWafBandwidth(difficulty: number): string {
+  const sizes = [1_024, 1_024, 10_240, 102_400, 1_048_576, 10_485_760];
+
+  return Buffer.alloc(sizes[Math.min(5, Math.max(0, difficulty))] ?? 1_024).toString("base64");
+}
+
+/** Computes bounded proof-of-work inside a disposable worker. */
+export function solveAwsWafWork(work: AwsWafWork): string | undefined {
+  if (!Number.isInteger(work.difficulty) || work.difficulty < 1 || work.difficulty > 32)
+    return undefined;
+
+  if (work.type === "NetworkBandwidth") return solveAwsWafBandwidth(work.difficulty);
+
+  if (work.type === "SHA256") {
+    for (let nonce = 0; nonce < 2_000_000; nonce += 1) {
+      const digest = createHash("sha256").update(`${work.input}${work.checksum}${nonce}`).digest();
+
+      if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
+    }
+  }
+
+  if (work.type === "HashcashScrypt") {
+    const cost = work.memory > 0 ? work.memory : 128;
+
+    // Reject large or invalid costs before invoking OpenSSL.
+    if (!Number.isInteger(cost) || cost < 2 || cost > 16_384 || (cost & (cost - 1)) !== 0)
+      return undefined;
+
+    for (let nonce = 0; nonce < 100_000; nonce += 1) {
+      const digest = scryptSync(`${work.input}${work.checksum}${nonce}`, work.checksum, 32, {
+        N: cost,
+        r: 8,
+        p: 1,
+        maxmem: 64 * 1_024 * 1_024,
+      });
+
+      if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
+    }
+  }
+
+  return undefined;
+}

package/src/challenges/aws-waf-worker.ts

@@ -0,0 +1,14 @@
+/* v8 ignore file -- exercised through worker integration tests. */
+import { parentPort, workerData } from "node:worker_threads";
+import { solveAwsWafWork, type AwsWafWork } from "./aws-waf-solver.ts";
+
+const work: AwsWafWork = workerData;
+
+try {
+  parentPort?.postMessage({ kind: "solved", solution: solveAwsWafWork(work) });
+} catch (error) {
+  parentPort?.postMessage({
+    kind: "error",
+    message: error instanceof Error ? error.message : "AWS WAF worker failed.",
+  });
+}

package/src/challenges/aws-waf.ts

@@ -0,0 +1,34 @@
+import type { IncomingMessage } from "node:http";
+import type { RedirectDependencies } from "../network/network-redirects.ts";
+
+/** Cookie that carries an AWS WAF challenge token. */
+export const AWS_WAF_TOKEN_COOKIE = "aws-waf-token";
+
+/** A resolved AWS WAF token and the browser identity it was issued for. */
+export interface AwsWafSolution {
+  token: string;
+  userAgent?: string;
+}
+
+/** A request to resolve one AWS WAF challenge. */
+export interface AwsWafResolveContext {
+  /** The page URL withheld by the challenge. */
+  url: URL;
+  /** Cancels the solver request. */
+  signal: AbortSignal;
+  /** Network dependencies used by the resolver. */
+  network?: RedirectDependencies;
+}
+
+/** Resolves an AWS WAF challenge to a token accepted by the target origin. */
+export interface AwsWafResolver {
+  resolve(context: AwsWafResolveContext): Promise<AwsWafSolution | undefined>;
+}
+
+/** Returns whether a response carries an AWS WAF challenge. */
+export function isAwsWafChallengeResponse(response: Pick<IncomingMessage, "headers">): boolean {
+  const value = response.headers["x-amzn-waf-action"];
+  const action = Array.isArray(value) ? value[0] : value;
+
+  return action?.trim().toLowerCase() === "challenge";
+}

package/src/document/content.ts

@@ -33,6 +33,10 @@ export interface CompleteDocument {
   cachedAt?: number;
   /** True when this document is the site's /llms.txt served instead of a low-quality page. */
   llmsTxtFallback?: boolean;
+  /** Code-fence language when the body was returned as reformatted source. */
+  sourceLanguage?: string;
+  /** Whether the response is a standalone OpenAPI or Swagger document. */
+  openapiSpec?: boolean;
   /** Set when the site publishes a usable /llms.txt index and this document is not that index. */
   llmsTxtIndexUrl?: string;
   /** Absolute URL advertised via `rel="describedby"` (header or HTML link) for this page. */

package/src/document/focus.ts

@@ -22,27 +22,130 @@ interface Section {
   tokens: string[];
 }
 
+interface SplitSections {
+  sections: Section[];
+  /** Separator that rejoins selected sections; windowed source stays contiguous. */
+  separator: "\n" | "\n\n";
+}
+
+/** Target size of one section when a block has no headings and no blank lines. */
+const WINDOW_TARGET_CHARACTERS = 8_000;
+
+/** Fewest lines a headingless block needs before it is split into windows. */
+const WINDOW_MIN_LINES = 80;
+
+/** Fewest characters a headingless block needs before it is split into windows. */
+const WINDOW_MIN_CHARACTERS = 12_000;
+
 function tokens(value: string): string[] {
   return (value.toLowerCase().match(WORD_PATTERN) ?? []).filter((token) => token.length > 1);
 }
 
+/**
+ * Splits one headingless, blank-line-free block into line-aligned windows.
+ *
+ * Formatted source and log files have neither headings nor blank lines, so treating the
+ * whole body as a single section would make focus select everything. Windows are bounded
+ * by characters but never cut a line, and short blocks keep their single section.
+ */
+function splitIntoWindows(block: string): string[] {
+  const lines = block.split(/\r?\n/);
+
+  if (lines.length < WINDOW_MIN_LINES && block.length < WINDOW_MIN_CHARACTERS) return [block];
+
+  const windows: string[] = [];
+  let current: string[] = [];
+  let size = 0;
+
+  for (const line of lines) {
+    if (size > 0 && size + line.length + 1 > WINDOW_TARGET_CHARACTERS) {
+      windows.push(current.join("\n"));
+      current = [];
+      size = 0;
+    }
+
+    current.push(line);
+    size += line.length + 1;
+  }
+
+  const last = current.join("\n");
+
+  if (last) windows.push(last);
+
+  return windows.filter(Boolean);
+}
+
 /**
  * Splits Markdown into sections. Headings stay attached to their following content; a
- * headingless document falls back to paragraphs so focused extraction remains useful for
- * plain-text and basic-extractor responses.
+ * headingless document falls back to paragraphs, and a headingless block without blank
+ * lines falls back to line windows so focused extraction remains useful for plain-text,
+ * log, and reformatted-source responses.
  */
-function splitSections(markdown: string): Section[] {
+function splitSections(markdown: string): SplitSections {
+  // Window a standalone fenced source body without leaving partial Markdown fences.
+  const fenced = /^( {0,3})(`{3,}|~{3,})([^\n\r]*)\r?\n([\s\S]*?)\r?\n\1\2\s*$/.exec(markdown);
+
+  const hasInteriorClose = fenced?.[4].split(/\r?\n/).some((line) => {
+    const close = /^[ \t]{0,3}(`{3,}|~{3,})[ \t]*$/.exec(line);
+
+    return close && close[1][0] === fenced[2][0] && close[1].length >= fenced[2].length;
+  });
+
+  if (fenced && !hasInteriorClose) {
+    const windows = splitIntoWindows(fenced[4]);
+
+    if (windows.length > 1) {
+      return {
+        sections: windows.map((window) => ({
+          markdown: `${fenced[1]}${fenced[2]}${fenced[3]}\n${window}\n${fenced[1]}${fenced[2]}`,
+          tokens: tokens(window),
+        })),
+        separator: "\n\n",
+      };
+    }
+
+    return { sections: [{ markdown, tokens: tokens(fenced[4]) }], separator: "\n\n" };
+  }
+
   const blocks = markdown
     .split(/\n{2,}/)
     .map((block) => block.trim())
     .filter(Boolean);
 
-  if (blocks.length === 0) return [];
+  if (blocks.length === 0) return { sections: [], separator: "\n\n" };
 
   const lines = markdown.split(/\r?\n/);
-  const hasHeadings = lines.some((line) => HEADING_PATTERN.test(line));
+  let fence: { marker: string; length: number } | undefined;
+
+  const headingLines = lines.map((line) => {
+    const match = /^[ \t]{0,3}(`{3,}|~{3,})(.*)$/.exec(line);
+
+    if (match) {
+      if (!fence) fence = { marker: match[1][0], length: match[1].length };
+      else if (match[1][0] === fence.marker && match[1].length >= fence.length && !match[2].trim())
+        fence = undefined;
+
+      return false;
+    }
+
+    return !fence && HEADING_PATTERN.test(line);
+  });
+
+  const hasHeadings = headingLines.some(Boolean);
+
+  if (!hasHeadings) {
+    // Keep mixed fenced Markdown intact rather than splitting a fence at blank lines.
+    if (lines.some((line) => /^[ \t]{0,3}(`{3,}|~{3,})/.test(line))) {
+      return { sections: [{ markdown, tokens: tokens(markdown) }], separator: "\n\n" };
+    }
 
-  if (!hasHeadings) return blocks.map((block) => ({ markdown: block, tokens: tokens(block) }));
+    const windowed = blocks.flatMap(splitIntoWindows);
+
+    return {
+      sections: windowed.map((block) => ({ markdown: block, tokens: tokens(block) })),
+      separator: blocks.length === 1 && windowed.length > 1 ? "\n" : "\n\n",
+    };
+  }
 
   const sections: string[] = [];
   let current: string[] = [];
@@ -54,14 +157,17 @@ function splitSections(markdown: string): Section[] {
     current = [];
   };
 
-  for (const line of lines) {
-    if (HEADING_PATTERN.test(line)) flush();
+  for (const [index, line] of lines.entries()) {
+    if (headingLines[index]) flush();
     current.push(line);
   }
 
   flush();
 
-  return sections.map((section) => ({ markdown: section, tokens: tokens(section) }));
+  return {
+    sections: sections.map((section) => ({ markdown: section, tokens: tokens(section) })),
+    separator: "\n\n",
+  };
 }
 
 /**
@@ -72,7 +178,7 @@ function splitSections(markdown: string): Section[] {
 export function focusMarkdown(markdown: string, query: string): FocusResult {
   const normalizedQuery = query.trim();
   const queryTerms = [...new Set(tokens(normalizedQuery))];
-  const sections = splitSections(markdown);
+  const { sections, separator } = splitSections(markdown);
   const originalCharacters = markdown.length;
 
   if (queryTerms.length === 0 || sections.length === 0) {
@@ -126,7 +232,7 @@ export function focusMarkdown(markdown: string, query: string): FocusResult {
     return score > 0;
   });
 
-  const focused = selected.map((section) => section.markdown).join("\n\n");
+  const focused = selected.map((section) => section.markdown).join(separator);
 
   return {
     markdown: focused,

package/src/document/openapi.ts

@@ -0,0 +1,80 @@
+import { parse } from "acorn";
+import { simple } from "acorn-walk";
+import { parseDocument } from "yaml";
+import { Type } from "typebox";
+import { Check } from "typebox/value";
+
+const schema = Type.Union([
+  Type.Object({
+    openapi: Type.String({ pattern: "^3\\.\\d+\\.\\d+$" }),
+    info: Type.Object({}),
+    paths: Type.Object({}),
+  }),
+  Type.Object({ swagger: Type.Literal("2.0"), info: Type.Object({}), paths: Type.Object({}) }),
+]);
+
+// Parsing is synchronous and untrusted. Larger inputs remain readable without classification.
+const MAX_CLASSIFICATION_CHARACTERS = 512 * 1024;
+
+/** Conservatively recognizes a document or a literal object embedded in JavaScript. Never executes source. */
+export function isOpenapiSource(raw: string, contentType: string): boolean {
+  if (raw.length > MAX_CLASSIFICATION_CHARACTERS) return false;
+
+  try {
+    if (contentType === "application/json") return Check(schema, JSON.parse(raw));
+
+    if (contentType === "text/yaml") {
+      const document = parseDocument(raw, { uniqueKeys: true });
+
+      return document.errors.length === 0 && Check(schema, document.toJS({ maxAliasCount: 0 }));
+    }
+
+    if (contentType !== "text/javascript") return false;
+    const ast = parse(raw, { ecmaVersion: "latest", sourceType: "module" });
+    let found = false;
+    simple(ast, {
+      ObjectExpression(node) {
+        let version = false;
+        let info = false;
+        let paths = false;
+
+        for (const property of node.properties) {
+          if (property.type !== "Property" || property.computed || property.kind !== "init")
+            continue;
+
+          const key =
+            property.key.type === "Identifier"
+              ? property.key.name
+              : property.key.type === "Literal"
+                ? property.key.value
+                : undefined;
+
+          if ((key === "openapi" || key === "swagger") && property.value.type === "Literal") {
+            const value = property.value.value;
+            version ||=
+              key === "swagger"
+                ? value === "2.0"
+                : Check(Type.String({ pattern: "^3\\.\\d+\\.\\d+$" }), value);
+          }
+
+          if (key === "info" && property.value.type === "ObjectExpression") info = true;
+
+          if (key === "paths" && property.value.type === "ObjectExpression") paths = true;
+        }
+
+        if (version && info && paths) found = true;
+      },
+    });
+
+    return found;
+  } catch {
+    return false;
+  }
+}
+
+/** Reconstructs detection from older cached canonical representations, without a network request. */
+export function isCachedOpenapi(markdown: string, contentType: string): boolean {
+  const match = /^```(?:json|js|yaml)\r?\n([\s\S]*)\r?\n```\s*$/.exec(markdown);
+
+  return isOpenapiSource(match ? match[1] : markdown, contentType);
+}

package/src/document/source-format.ts

@@ -0,0 +1,180 @@
+/**
+ * Source text that is read as code rather than prose, mapped to the code-fence
+ * language used when one of these bodies is reformatted.
+ */
+const SOURCE_LANGUAGE_BY_CONTENT_TYPE = {
+  "application/json": "json",
+  "text/javascript": "js",
+  "text/yaml": "yaml",
+} as const;
+
+type SourceContentType = keyof typeof SOURCE_LANGUAGE_BY_CONTENT_TYPE;
+
+/** Narrows a media type to the source types that are reformatted. */
+function isSourceContentType(value: string): value is SourceContentType {
+  return Object.hasOwn(SOURCE_LANGUAGE_BY_CONTENT_TYPE, value);
+}
+
+/** Shortest body reformatted; below this a page is returned verbatim. */
+const MINIFIED_MIN_CHARACTERS = 2_000;
+
+/**
+ * Largest body reformatted. Reformatting triples transient memory for a document too
+ * large to cache, so bigger bodies keep their original single-line layout.
+ */
+const MINIFIED_MAX_CHARACTERS = 2 * 1_024 * 1_024;
+
+/**
+ * Shortest average line that still counts as machine-generated. Bundled source is
+ * packed into very long lines even when it keeps a few real line breaks inside block
+ * scalars, while hand-written source keeps short lines.
+ */
+const MINIFIED_MIN_AVERAGE_LINE_CHARACTERS = 300;
+
+/** Deepest nesting that still receives extra indentation. */
+const MAX_FORMAT_DEPTH = 24;
+
+const INDENT = "  ";
+
+const QUOTE_CHARACTERS = new Set(['"', "'", "`"]);
+
+const OPENING_CHARACTERS = new Set(["{", "["]);
+
+const CLOSING_CHARACTERS = new Set(["}", "]"]);
+
+const LINE_ENDING_CHARACTERS = new Set([",", ";"]);
+
+/**
+ * Returns whether a body is machine-generated source rather than hand-written code.
+ *
+ * Only minified bodies are reformatted: a hand-written file already carries the
+ * author's line structure, which is more useful than any generated layout.
+ */
+function isMinified(value: string): boolean {
+  if (value.length < MINIFIED_MIN_CHARACTERS || value.length > MINIFIED_MAX_CHARACTERS)
+    return false;
+
+  let lineBreaks = 0;
+
+  for (let index = 0; index < value.length; index += 1) {
+    if (value[index] === "\n") lineBreaks += 1;
+  }
+
+  return value.length / (lineBreaks + 1) >= MINIFIED_MIN_AVERAGE_LINE_CHARACTERS;
+}
+
+/** Returns the fence language for a content type, or undefined when it is not source. */
+export function sourceLanguageFor(contentType: string): string | undefined {
+  return isSourceContentType(contentType)
+    ? SOURCE_LANGUAGE_BY_CONTENT_TYPE[contentType]
+    : undefined;
+}
+
+/**
+ * Adds line breaks and indentation to minified source without interpreting it.
+ *
+ * Layout whitespace is normalized and every other character is preserved in order.
+ * String literals are tracked only far enough to keep their punctuation from changing
+ * the layout, so no value is parsed, evaluated, or re-serialized.
+ *
+ * @param value - The minified source
+ * @returns Indented source, or undefined when the input was not minified
+ */
+export function indentMinifiedSource(value: string): string | undefined {
+  if (!isMinified(value)) return undefined;
+
+  const lines: string[] = [];
+  let current = "";
+  let depth = 0;
+  let quote: string | undefined;
+  let escaped = false;
+
+  const indent = (): string => INDENT.repeat(Math.min(depth, MAX_FORMAT_DEPTH));
+
+  const flush = (): void => {
+    const line = current.trimEnd();
+
+    if (line) lines.push(line);
+    current = "";
+  };
+
+  for (const character of value) {
+    if (quote !== undefined) {
+      current += character;
+
+      if (escaped) escaped = false;
+      else if (character === "\\") escaped = true;
+      else if (character === quote) quote = undefined;
+
+      continue;
+    }
+
+    if (QUOTE_CHARACTERS.has(character)) {
+      quote = character;
+      current += character;
+
+      continue;
+    }
+
+    if (OPENING_CHARACTERS.has(character)) {
+      current += character;
+      flush();
+      depth += 1;
+      current = indent();
+
+      continue;
+    }
+
+    if (CLOSING_CHARACTERS.has(character)) {
+      flush();
+      depth = Math.max(0, depth - 1);
+      current = indent() + character;
+
+      continue;
+    }
+
+    if (LINE_ENDING_CHARACTERS.has(character)) {
+      current += character;
+      flush();
+      current = indent();
+
+      continue;
+    }
+
+    if (character === "\n") {
+      flush();
+      current = indent();
+
+      continue;
+    }
+
+    if (character !== "\r") current += character;
+  }
+
+  flush();
+
+  return lines.join("\n");
+}
+
+/**
+ * Reformats minified JavaScript, JSON, or YAML as a fenced code block.
+ *
+ * @param raw - The decoded response body
+ * @param contentType - The normalized response media type
+ * @returns A fenced, indented Markdown block, or undefined when the body is not
+ *   source that benefits from reformatting
+ */
+export function formatMinifiedSource(
+  raw: string,
+  contentType: string,
+): { markdown: string; language: string } | undefined {
+  const language = sourceLanguageFor(contentType);
+
+  if (language === undefined) return undefined;
+
+  const indented = indentMinifiedSource(raw);
+
+  if (indented === undefined) return undefined;
+
+  return { language, markdown: `\`\`\`${language}\n${indented}\n\`\`\`` };
+}

package/src/index.ts

@@ -20,6 +20,13 @@ export {
   type AnubisSolveContext,
 } from "./challenges/anubis.ts";
 
+export {
+  AWS_WAF_TOKEN_COOKIE,
+  isAwsWafChallengeResponse,
+  type AwsWafResolveContext,
+  type AwsWafResolver,
+} from "./challenges/aws-waf.ts";
+
 export { CookieJar } from "./network/cookies.ts";
 
 export type { FetchResult } from "./document/content.ts";

package/src/network/cookies.ts

@@ -211,6 +211,47 @@ export class CookieJar {
     this.#evict();
   }
 
+  /**
+   * Stores a challenge token returned by a resolver.
+   *
+   * The token is untrusted input. It is accepted only for the target origin,
+   * scoped to the whole origin, and held in memory for the current acquisition.
+   *
+   * @param name - Cookie name
+   * @param value - Cookie value
+   * @param url - The page URL the cookie belongs to
+   * @returns Whether the cookie was accepted
+   */
+  set(name: string, value: string, url: URL): boolean {
+    const token = value.trim();
+
+    if (
+      name.length === 0 ||
+      name.length > MAX_COOKIE_NAME_CHARACTERS ||
+      !COOKIE_NAME.test(name) ||
+      token.length === 0 ||
+      token.length > MAX_COOKIE_VALUE_CHARACTERS ||
+      hasInvalidCookieCharacter(token) ||
+      /[\s";,\\]/u.test(token)
+    ) {
+      return false;
+    }
+
+    const key = `${url.origin}\0${name}\0/`;
+    this.#cookies.delete(key);
+    this.#cookies.set(key, {
+      name,
+      value: token,
+      origin: url.origin,
+      path: "/",
+      secure: url.protocol === "https:",
+      createdAt: Date.now(),
+    });
+    this.#evict();
+
+    return true;
+  }
+
   /**
    * Returns a Cookie header value for a URL, excluding expired, insecure, and
    * cross-origin cookies.

package/src/network/network-redirects.ts

@@ -29,11 +29,16 @@ export interface RedirectDependencies {
     target: ValidatedTarget,
     signal: AbortSignal,
     headers?: Readonly<Record<string, string>>,
+    options?: { method?: string; body?: string | Uint8Array },
   ) => Promise<IncomingMessage>;
   random?: () => number;
   sleep?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
   /** Origin-scoped cookies for the current acquisition, including challenge handshakes. */
   cookies?: CookieJar;
+  /** User-Agent override that persists across redirects. */
+  userAgent?: string;
+  /** Headers that survive a redirect, used by challenge handshakes that need browser identity. */
+  persistentHeaders?: Readonly<Record<string, string>>;
 }
 
 /** Creates the consistent abort error used by redirect and retry coordination. */
@@ -144,7 +149,14 @@ async function coordinatedRequest(
     const release = await acquireOrigin(target.url.origin, signal);
     let response: IncomingMessage;
     const cookieHeader = dependencies.cookies?.headerFor(target.url);
-    const requestHeaders = cookieHeader ? { Cookie: cookieHeader, ...headers } : headers;
+
+    const headersWithUserAgent = dependencies.userAgent
+      ? { "User-Agent": dependencies.userAgent, ...headers }
+      : headers;
+
+    const requestHeaders = cookieHeader
+      ? { Cookie: cookieHeader, ...headersWithUserAgent }
+      : headersWithUserAgent;
 
     try {
       response = await request(target, signal, requestHeaders);
@@ -197,7 +209,8 @@ export async function requestFollowingRedirects(
     response.resume();
     target = await awaitWithAbort(validateUrl(new URL(location, target.url)), signal);
     // Validators describe the originally cached resource and must not leak to a redirect target.
-    requestHeaders = {};
+    // Challenge handshakes opt into a fixed browser identity instead.
+    requestHeaders = dependencies.persistentHeaders ?? {};
   }
 
   throw new Error("web_fetch followed too many redirects.");