@zeldrisho/pi-web-fetch 0.14.1 → 0.16.0

package/src/network/network-transport.ts

@@ -8,6 +8,9 @@ import type { ValidatedTarget } from "./network-policy.ts";
 /** Maximum size in bytes for raw HTTP response bodies. */
 export const FETCH_MAX_BYTES = 5 * 1_024 * 1_024;
 
+/** Fixed User-Agent used for page requests and challenge solvers. */
+export const WEB_FETCH_USER_AGENT = "Mozilla/5.0 (compatible; PiWebFetch/1.0; +https://pi.dev)";
+
 /**
  * Per-address connect deadline. A hanging address is abandoned after this
  * budget so the next validated address can be tried inside the overall
@@ -67,6 +70,8 @@ async function requestOnce(
   signal: AbortSignal,
   attemptTimeoutMs: number,
   headers: Readonly<Record<string, string>>,
+  method = "GET",
+  body?: string | Uint8Array,
 ): Promise<IncomingMessage> {
   const family = isIP(address);
 
@@ -114,11 +119,12 @@ async function requestOnce(
       target.url,
       {
         lookup,
+        method,
         signal: controller.signal,
         headers: {
           Accept:
             "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, application/xhtml+xml;q=0.7, application/json;q=0.7, application/xml;q=0.7, */*;q=0.1",
-          "User-Agent": "Mozilla/5.0 (compatible; PiWebFetch/1.0; +https://pi.dev)",
+          "User-Agent": WEB_FETCH_USER_AGENT,
           ...headers,
         },
       },
@@ -139,7 +145,7 @@ async function requestOnce(
       if (attemptExpired) finish(() => reject(unreachableError()));
       else finish(() => reject(error));
     });
-    outgoing.end();
+    outgoing.end(body);
   });
 }
 
@@ -148,6 +154,10 @@ export interface RequestPinnedOptions {
   attemptTimeoutMs?: number;
   /** Additional fixed request headers, used for conditional cache revalidation. */
   headers?: Readonly<Record<string, string>>;
+  /** HTTP method; defaults to GET. */
+  method?: string;
+  /** Request body for non-GET requests. */
+  body?: string | Uint8Array;
 }
 
 /**
@@ -169,7 +179,15 @@ export async function requestPinned(
 
   for (const address of addresses) {
     try {
-      return await requestOnce(target, address, signal, attemptTimeoutMs, options.headers ?? {});
+      return await requestOnce(
+        target,
+        address,
+        signal,
+        attemptTimeoutMs,
+        options.headers ?? {},
+        options.method,
+        options.body,
+      );
     } catch (error) {
       lastError = error;
 

package/src/service.ts

@@ -13,6 +13,7 @@ import { sliceCompleteDocument, type CompleteDocument } from "./document/content
 import { revalidateCompleteDocument, type FetchRemoteDependencies } from "./acquisition/fetch.ts";
 import { fetchDocumentWithLlmsTxtSupport } from "./acquisition/llms.ts";
 import { focusMarkdown, type FocusDetails } from "./document/focus.ts";
+import { isCachedOpenapi } from "./document/openapi.ts";
 import { resolveFragmentOffset, urlWithoutFragment } from "./document/fragments.ts";
 import { InflightCoalescer } from "./inflight";
 import { createDocumentOutline, type DocumentOutline } from "./document/outline.ts";
@@ -43,6 +44,7 @@ const encoder = new TextEncoder();
 export type ContentKind =
   | "repository-readme"
   | "code-file"
+  | "openapi-spec"
   | "directory-listing"
   | "llms-index"
   | "article"
@@ -73,14 +75,24 @@ function safeUrl(value: string): URL | undefined {
  * @param url - The fetched URL
  * @param extractor - Extraction method used
  * @param shellSuspected - Whether the page appears to be an app shell
+ * @param sourceLanguage - Code-fence language when the body was reformatted as source
  * @returns Content classification
  */
 export function classifyContentKind(
   url: string,
   extractor: CompleteDocument["extractor"],
   shellSuspected: boolean,
+  sourceLanguage?: string,
+  openapiSpec = false,
 ): ContentKind {
   if (shellSuspected) return "markup-shell";
+
+  if (openapiSpec) return "openapi-spec";
+
+  // A reformatted body is code whatever host served it: the response was read as
+  // source, not as page prose.
+  if (sourceLanguage !== undefined) return "code-file";
+
   const parsed = safeUrl(url);
   const host = parsed?.hostname ?? "";
   const path = parsed?.pathname ?? "";
@@ -423,6 +435,7 @@ export async function executeWebFetch(
 
   const fresh = Boolean(
     document &&
+    !document.llmsTxtFallback &&
     !document.extractionDiagnostics?.botWall &&
     (document.cachedAt === undefined || now - document.cachedAt < CACHE_TTL_MS),
   );
@@ -576,7 +589,15 @@ export async function executeWebFetch(
   const rawFinalUrl = result.url;
   const finalUrl = redactUrlForDisplay(rawFinalUrl);
   const shellSuspected = result.shellSuspected;
-  const contentKind = classifyContentKind(rawFinalUrl, result.extractor, shellSuspected);
+
+  const contentKind = classifyContentKind(
+    rawFinalUrl,
+    result.extractor,
+    shellSuspected,
+    result.sourceLanguage,
+    document.openapiSpec ?? isCachedOpenapi(document.markdown, document.contentType),
+  );
+
   const confidence = classifyConfidence(result.extractor, shellSuspected, result.markdown.length);
 
   const requestedPath = safeUrl(params.url)?.pathname.replace(/\/+$/u, "").toLowerCase();
@@ -603,10 +624,7 @@ export async function executeWebFetch(
             ]
           : []),
         ...(showLlmsNotice && result.llmsTxtIndexUrl
-          ? [
-              `Documentation index available: ${redactUrlForDisplay(result.llmsTxtIndexUrl)}. Use it if this page does not cover the topic.`,
-              "",
-            ]
+          ? [`Index: ${redactUrlForDisplay(result.llmsTxtIndexUrl)}`, ""]
           : []),
         ...(resolvedFragment.fragment !== undefined && !fragmentMatched
           ? [

package/workers/aws-waf-solver.js

@@ -0,0 +1,56 @@
+import { createHash, scryptSync } from "node:crypto";
+
+/** Returns whether the first `difficulty` bits of a hash are zero. */
+export function hasLeadingZeroBits(hash, difficulty) {
+  for (let bit = 0; bit < difficulty; bit += 1) {
+    const byte = hash[Math.floor(bit / 8)];
+
+    if (byte === undefined || (byte & (1 << (7 - (bit % 8)))) !== 0) return false;
+  }
+
+  return true;
+}
+
+/** Solves the bandwidth challenge using a bounded zero-filled buffer. */
+export function solveAwsWafBandwidth(difficulty) {
+  const sizes = [1_024, 1_024, 10_240, 102_400, 1_048_576, 10_485_760];
+
+  return Buffer.alloc(sizes[Math.min(5, Math.max(0, difficulty))] ?? 1_024).toString("base64");
+}
+
+/** Computes bounded proof-of-work inside a disposable worker. */
+export function solveAwsWafWork(work) {
+  if (!Number.isInteger(work.difficulty) || work.difficulty < 1 || work.difficulty > 32)
+    return undefined;
+
+  if (work.type === "NetworkBandwidth") return solveAwsWafBandwidth(work.difficulty);
+
+  if (work.type === "SHA256") {
+    for (let nonce = 0; nonce < 2_000_000; nonce += 1) {
+      const digest = createHash("sha256").update(`${work.input}${work.checksum}${nonce}`).digest();
+
+      if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
+    }
+  }
+
+  if (work.type === "HashcashScrypt") {
+    const cost = work.memory > 0 ? work.memory : 128;
+
+    // Reject large or invalid costs before invoking OpenSSL.
+    if (!Number.isInteger(cost) || cost < 2 || cost > 16_384 || (cost & (cost - 1)) !== 0)
+      return undefined;
+
+    for (let nonce = 0; nonce < 100_000; nonce += 1) {
+      const digest = scryptSync(`${work.input}${work.checksum}${nonce}`, work.checksum, 32, {
+        N: cost,
+        r: 8,
+        p: 1,
+        maxmem: 64 * 1_024 * 1_024,
+      });
+
+      if (hasLeadingZeroBits(digest, work.difficulty)) return String(nonce);
+    }
+  }
+
+  return undefined;
+}

package/workers/aws-waf-worker.js

@@ -0,0 +1,14 @@
+/* v8 ignore file -- exercised through worker integration tests. */
+import { parentPort, workerData } from "node:worker_threads";
+import { solveAwsWafWork } from "./aws-waf-solver.js";
+
+const work = workerData;
+
+try {
+  parentPort?.postMessage({ kind: "solved", solution: solveAwsWafWork(work) });
+} catch (error) {
+  parentPort?.postMessage({
+    kind: "error",
+    message: error instanceof Error ? error.message : "AWS WAF worker failed.",
+  });
+}