@zeldrisho/pi-web-fetch 0.5.1 → 0.5.3

package/CHANGELOG.md

@@ -1,4 +1,22 @@
 # Changelog
+
+## [0.5.3](https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.5.2...pi-web-fetch-v0.5.3) (2026-08-12)
+
+### Bug fixes
+
+- **web-fetch:** Fall back across validated addresses before timing out ([32a000b](https://github.com/zeldrisho/pi-packages/commit/32a000bf367268d4ff38e7b45cb73d45ea894f5d))
+- **web-fetch:** Cancel attempts when the caller signal is already aborted ([2f62c95](https://github.com/zeldrisho/pi-packages/commit/2f62c95906e171cc974c9f5cdbd66cf3ecd65b56))
+
+### Maintenance
+
+- **deps:** Upgrade Vite+ toolchain to 0.2.9 ([9921cf3](https://github.com/zeldrisho/pi-packages/commit/9921cf3ffbed29f9c08ca3ab595a5096fadf2be0))
+
+## [0.5.2](https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.5.1...pi-web-fetch-v0.5.2) (2026-08-10)
+
+### Bug fixes
+
+- **web-fetch:** Discard malformed schema metadata ([2ac7b02](https://github.com/zeldrisho/pi-packages/commit/2ac7b02f8952950aa60d4d22f080ab0774cdf50a))
+
 ## [0.5.1](https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.5.0...pi-web-fetch-v0.5.1) (2026-08-03)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zeldrisho/pi-web-fetch",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "description": "Pi extension for secure, bounded public web page fetching and Markdown extraction",
   "keywords": [
     "pi",
@@ -35,11 +35,12 @@
     "linkedom": "^0.18.13"
   },
   "devDependencies": {
-    "@earendil-works/pi-coding-agent": "^0.80.10",
-    "@earendil-works/pi-tui": "^0.80.10",
+    "@earendil-works/pi-coding-agent": "^0.84.0",
+    "@earendil-works/pi-tui": "^0.84.0",
     "typebox": "^1.1.24",
-    "typescript": "^5.0.0",
-    "vite-plus": "0.2.5"
+    "typescript": "^7.0.0",
+    "vite": "npm:@voidzero-dev/vite-plus-core@0.2.9",
+    "vite-plus": "0.2.9"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",

package/src/extract.ts

@@ -2,6 +2,24 @@ import { parseHTML } from "linkedom";
 
 const RAW_ID_SELECTOR_SAFE = /^-?[_a-zA-Z][-_a-zA-Z0-9]*$/;
 
+/** Removes schema.org scripts that Defuddle would report directly to the process console. */
+function removeMalformedSchemaOrgData(document: Document): void {
+  for (const script of document.querySelectorAll<HTMLScriptElement>(
+    'script[type="application/ld+json"]',
+  )) {
+    const jsonContent = (script.textContent || "")
+      .replace(/\/\*[\s\S]*?\*\/|^\s*\/\/.*$/gm, "")
+      .replace(/^\s*<!\[CDATA\[([\s\S]*?)\]\]>\s*$/, "$1")
+      .replace(/^\s*(\*\/|\/\*)\s*|\s*(\*\/|\/\*)\s*$/g, "")
+      .trim();
+    try {
+      if (JSON.parse(jsonContent) === null) script.remove();
+    } catch {
+      script.remove();
+    }
+  }
+}
+
 /**
  * Replaces element IDs that are unsafe for CSS selectors and updates matching fragment links.
  *
@@ -71,8 +89,10 @@ export async function extractHtmlToMarkdown(
   try {
     const { Defuddle } = await import("defuddle/node");
     const { document } = parseHTML(html);
-    normalizeSelectorUnsafeIds(document as unknown as Document);
-    const result = await Defuddle(document as unknown as Document, baseUrl.toString(), {
+    const defuddleDocument = document as unknown as Document;
+    removeMalformedSchemaOrgData(defuddleDocument);
+    normalizeSelectorUnsafeIds(defuddleDocument);
+    const result = await Defuddle(defuddleDocument, baseUrl.toString(), {
       markdown: true,
       useAsync: false,
     });

package/src/network-policy.ts

@@ -70,12 +70,24 @@ for (const [network, prefix] of GLOBALLY_REACHABLE_IPV6_EXCEPTIONS) {
 
 export interface ValidatedTarget {
   url: URL;
+  /** The preferred address used for the first connection attempt. */
   address: string;
   family: 4 | 6;
+  /**
+   * Every address resolved for the hostname, ordered for connection attempts.
+   * Absent for single-address targets constructed without DNS resolution.
+   */
+  addresses?: string[];
 }
 
 export type ResolveAddresses = (hostname: string) => Promise<string[]>;
 
+/**
+ * Determines whether an IP address belongs to a blocked or reserved address range.
+ *
+ * @param address - The IP address to evaluate
+ * @returns `true` if the address is invalid or blocked, `false` if it is allowed
+ */
 export function isPrivateAddress(address: string): boolean {
   const family = isIP(address);
   if (family === 4) return blockedIPv4Addresses.check(address, "ipv4");
@@ -86,6 +98,23 @@ export function isPrivateAddress(address: string): boolean {
   return true;
 }
 
+/**
+ * Orders addresses with IPv4 addresses before IPv6 addresses while preserving their original order within each family.
+ *
+ * @returns A copy of the addresses ordered with IPv4 addresses first.
+ */
+export function preferIpv4First(addresses: string[]): string[] {
+  return [...addresses].sort((a, b) => Number(isIP(b) === 4) - Number(isIP(a) === 4));
+}
+
+/**
+ * Validates an HTTP or HTTPS URL and resolves it to an allowed network target.
+ *
+ * @param value - The URL to validate.
+ * @param resolveHostname - Optional hostname resolver.
+ * @returns The validated URL, preferred connection address, address family, and ordered resolved addresses.
+ * @throws If the URL uses an unsupported scheme, contains credentials, targets a local hostname, resolves to a private or reserved address, or cannot be resolved to IPv4 or IPv6.
+ */
 export async function validateRemoteUrl(
   value: string | URL,
   resolveHostname?: ResolveAddresses,
@@ -115,8 +144,9 @@ export async function validateRemoteUrl(
   if (addresses.length === 0 || addresses.some(isPrivateAddress)) {
     throw new Error(`web_fetch blocks private or reserved network targets (${hostname}).`);
   }
-  const address = addresses[0];
+  const ordered = preferIpv4First(addresses);
+  const address = ordered[0];
   const family = isIP(address);
   if (family !== 4 && family !== 6) throw new Error(`web_fetch could not resolve ${hostname}.`);
-  return { url, address, family };
+  return { url, address, family, addresses: ordered };
 }

package/src/network-transport.ts

@@ -1,13 +1,28 @@
 import { request as httpRequest, type IncomingMessage } from "node:http";
 import { request as httpsRequest } from "node:https";
-import type { LookupFunction } from "node:net";
+import { isIP, type LookupFunction } from "node:net";
 import { formatSize } from "@earendil-works/pi-coding-agent";
 import type { ValidatedTarget } from "./network-policy";
 
 export const FETCH_MAX_BYTES = 5 * 1_024 * 1_024;
 
+/**
+ * Per-address connect deadline. A hanging address is abandoned after this
+ * budget so the next validated address can be tried inside the overall
+ * request timeout.
+ */
+export const CONNECT_ATTEMPT_TIMEOUT_MS = 4_000;
+
 const encoder = new TextEncoder();
 
+/**
+ * Formats an error message for a response that exceeds the raw download limit.
+ *
+ * @param receivedBytes - The number of bytes received or reported by the response.
+ * @param maxBytes - The maximum allowed number of bytes.
+ * @param sizeIsExact - Whether `receivedBytes` is the exact response size.
+ * @returns A message describing the response size and configured limit.
+ */
 function responseTooLargeMessage(
   receivedBytes: number,
   maxBytes: number,
@@ -22,33 +37,110 @@ function responseTooLargeMessage(
   ].join(" ");
 }
 
-export async function requestPinned(
+/**
+ * Sends a request to a specific validated network address.
+ *
+ * @param target - The validated request target.
+ * @param address - The IPv4 or IPv6 address to use for the connection.
+ * @param signal - Signal used to cancel the request.
+ * @param attemptTimeoutMs - Maximum time allowed for the connection attempt.
+ * @returns The received response.
+ * @throws If `address` is not a valid IPv4 or IPv6 address.
+ */
+async function requestOnce(
   target: ValidatedTarget,
+  address: string,
   signal: AbortSignal,
+  attemptTimeoutMs: number,
 ): Promise<IncomingMessage> {
+  const family = isIP(address);
+  if (family !== 4 && family !== 6) throw new Error(`web_fetch could not resolve ${address}.`);
   const lookup: LookupFunction = (_hostname, options, callback) => {
-    if (options.all) callback(null, [{ address: target.address, family: target.family }]);
-    else callback(null, target.address, target.family);
+    if (options.all) callback(null, [{ address, family }]);
+    else callback(null, address, family);
   };
   const request = target.url.protocol === "https:" ? httpsRequest : httpRequest;
   return await new Promise((resolve, reject) => {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), attemptTimeoutMs);
+    const forwardAbort = () => controller.abort();
+    if (signal.aborted) controller.abort();
+    else signal.addEventListener("abort", forwardAbort, { once: true });
+    let settled = false;
+    const finish = (callback: () => void) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      signal.removeEventListener("abort", forwardAbort);
+      callback();
+    };
     const outgoing = request(
       target.url,
       {
         lookup,
-        signal,
+        signal: controller.signal,
         headers: {
           Accept: "text/markdown, text/html, text/plain, application/json;q=0.9, */*;q=0.1",
           "User-Agent": "Mozilla/5.0 (compatible; PiWebFetch/1.0; +https://pi.dev)",
         },
       },
-      resolve,
+      (response) => {
+        if (controller.signal.aborted) {
+          // The attempt deadline fired before response headers: drop the socket
+          // and treat the address as unreachable so the next one is tried.
+          response.destroy();
+          finish(() =>
+            reject(
+              new Error(`web_fetch could not reach ${address} within ${attemptTimeoutMs} ms.`),
+            ),
+          );
+        } else finish(() => resolve(response));
+      },
     );
-    outgoing.once("error", reject);
+    outgoing.once("error", (error) => finish(() => reject(error)));
     outgoing.end();
   });
 }
 
+export interface RequestPinnedOptions {
+  /** Per-address connect deadline used to fall back to the next address. */
+  attemptTimeoutMs?: number;
+}
+
+/**
+ * Sends a request to the target using its validated addresses in sequence.
+ *
+ * @param options - Optional settings for individual connection attempts.
+ * @param options.attemptTimeoutMs - Maximum time allowed for each connection attempt in milliseconds.
+ * @returns The first successful HTTP response.
+ * @throws The final connection error if all addresses fail, or the abort error if the signal is aborted.
+ */
+export async function requestPinned(
+  target: ValidatedTarget,
+  signal: AbortSignal,
+  options: RequestPinnedOptions = {},
+): Promise<IncomingMessage> {
+  const attemptTimeoutMs = options.attemptTimeoutMs ?? CONNECT_ATTEMPT_TIMEOUT_MS;
+  const addresses = target.addresses?.length ? target.addresses : [target.address];
+  let lastError: unknown;
+  for (const address of addresses) {
+    try {
+      return await requestOnce(target, address, signal, attemptTimeoutMs);
+    } catch (error) {
+      lastError = error;
+      if (signal.aborted) throw error;
+    }
+  }
+  throw lastError ?? new Error("web_fetch could not connect.");
+}
+
+/**
+ * Retrieves a response header value by name.
+ *
+ * @param response - The response containing the header
+ * @param name - The header name to retrieve
+ * @returns The first header value, or `undefined` when the header is absent
+ */
 export function responseHeader(response: IncomingMessage, name: string): string | undefined {
   const value = response.headers[name];
   return Array.isArray(value) ? value[0] : value;