@zeldrisho/pi-web-fetch 0.8.2 → 0.9.1

package/src/network-redirects.ts

@@ -5,15 +5,20 @@ import { requestPinned, responseHeader } from "./network-transport";
 
 /** Maximum number of HTTP redirects to follow before aborting. */
 export const FETCH_MAX_REDIRECTS = 5;
+
 const MAX_ORIGIN_CONCURRENCY = 4;
+
 const MAX_RETRIES = 2;
+
 const MAX_RETRY_DELAY_MS = 10_000;
+
 const BASE_RETRY_DELAY_MS = 250;
 
 interface OriginState {
   active: number;
   waiting: Array<() => void>;
 }
+
 const origins = new Map<string, OriginState>();
 
 /** Optional dependencies for redirect handling (used for testing). */
@@ -28,21 +33,27 @@ export interface RedirectDependencies {
   sleep?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
 }
 
+/** Creates the consistent abort error used by redirect and retry coordination. */
 function abortError(): Error {
   const error = new Error("Operation aborted.");
   error.name = "AbortError";
+
   return error;
 }
 
+/** Reserves an origin-concurrency slot and returns a function that releases it. */
 async function acquireOrigin(origin: string, signal: AbortSignal): Promise<() => void> {
   const state = origins.get(origin) ?? { active: 0, waiting: [] };
   origins.set(origin, state);
   let reserved = false;
+
   const release = () => {
     state.active -= 1;
     state.waiting.shift()?.();
+
     if (state.active === 0 && state.waiting.length === 0) origins.delete(origin);
   };
+
   if (state.active >= MAX_ORIGIN_CONCURRENCY) {
     await new Promise<void>((resolve, reject) => {
       const enter = () => {
@@ -51,11 +62,14 @@ async function acquireOrigin(origin: string, signal: AbortSignal): Promise<() =>
         reserved = true;
         resolve();
       };
+
       const cancel = () => {
         const index = state.waiting.indexOf(enter);
+
         if (index >= 0) state.waiting.splice(index, 1);
         reject(abortError());
       };
+
       if (signal.aborted) cancel();
       else {
         state.waiting.push(enter);
@@ -63,40 +77,52 @@ async function acquireOrigin(origin: string, signal: AbortSignal): Promise<() =>
       }
     });
   }
+
   if (signal.aborted) {
     if (reserved) release();
     throw abortError();
   }
+
   if (!reserved) state.active += 1;
+
   return release;
 }
 
+/** Waits for a retry delay while allowing the caller to cancel immediately. */
 async function defaultSleep(milliseconds: number, signal: AbortSignal): Promise<void> {
   await new Promise<void>((resolve, reject) => {
     const timer = setTimeout(done, milliseconds);
+
     function done(): void {
       signal.removeEventListener("abort", cancel);
       resolve();
     }
+
     function cancel(): void {
       clearTimeout(timer);
       reject(abortError());
     }
+
     if (signal.aborted) cancel();
     else signal.addEventListener("abort", cancel, { once: true });
   });
 }
 
+/** Parses a Retry-After header into a bounded delay in milliseconds. */
 function retryAfterMilliseconds(value: string | undefined, now = Date.now()): number | undefined {
   if (!value) return undefined;
   const seconds = Number(value.trim());
+
   if (Number.isFinite(seconds) && seconds >= 0)
     return Math.min(seconds * 1_000, MAX_RETRY_DELAY_MS);
   const date = Date.parse(value);
+
   if (!Number.isFinite(date)) return undefined;
+
   return Math.min(Math.max(0, date - now), MAX_RETRY_DELAY_MS);
 }
 
+/** Sends one origin-limited request and retries transient rate-limit responses. */
 async function coordinatedRequest(
   target: ValidatedTarget,
   signal: AbortSignal,
@@ -107,17 +133,22 @@ async function coordinatedRequest(
     dependencies.request ??
     ((value, requestSignal, requestHeaders) =>
       requestPinned(value, requestSignal, { headers: requestHeaders }));
+
   const random = dependencies.random ?? Math.random;
   const sleep = dependencies.sleep ?? defaultSleep;
+
   for (let attempt = 0; ; attempt += 1) {
     const release = await acquireOrigin(target.url.origin, signal);
     let response: IncomingMessage;
+
     try {
       response = await request(target, signal, headers);
     } finally {
       release();
     }
+
     const status = response.statusCode ?? 0;
+
     if (![429, 503].includes(status) || attempt >= MAX_RETRIES) return response;
     // The retry decision uses headers only. Drop an untrusted error body so it cannot
     // consume unbounded bandwidth while the caller is backing off.
@@ -146,10 +177,13 @@ export async function requestFollowingRedirects(
   for (let redirects = 0; redirects <= FETCH_MAX_REDIRECTS; redirects += 1) {
     const response = await coordinatedRequest(target, signal, requestHeaders, dependencies);
     const status = response.statusCode ?? 0;
+
     if (![301, 302, 303, 307, 308].includes(status)) return { target, response };
 
     const location = responseHeader(response, "location");
+
     if (!location) throw new Error("web_fetch received a redirect without a Location header.");
+
     if (redirects === FETCH_MAX_REDIRECTS)
       throw new Error("web_fetch followed too many redirects.");
     response.resume();
@@ -157,5 +191,6 @@ export async function requestFollowingRedirects(
     // Validators describe the originally cached resource and must not leak to a redirect target.
     requestHeaders = {};
   }
+
   throw new Error("web_fetch followed too many redirects.");
 }

package/src/network-transport.ts

@@ -20,6 +20,7 @@ const encoder = new TextEncoder();
 function abortedError(): Error {
   const error = new Error("Operation aborted.");
   error.name = "AbortError";
+
   return error;
 }
 
@@ -39,6 +40,7 @@ function responseTooLargeMessage(
   const size = sizeIsExact
     ? `is ${formatSize(receivedBytes)}`
     : `has reached at least ${formatSize(receivedBytes)}`;
+
   return [
     `web_fetch response ${size}, exceeding the ${formatSize(maxBytes)} raw download limit.`,
     "maxCharacters only controls returned output.",
@@ -63,7 +65,9 @@ async function requestOnce(
   headers: Readonly<Record<string, string>>,
 ): Promise<IncomingMessage> {
   const family = isIP(address);
+
   if (family !== 4 && family !== 6) throw new Error(`web_fetch could not resolve ${address}.`);
+
   const lookup: LookupFunction = (_hostname, options, callback) => {
     // Match dns.lookup's asynchronous callback contract. Calling back inline
     // lets an immediately unreachable address emit a socket error before
@@ -73,18 +77,24 @@ async function requestOnce(
       else callback(null, address, family);
     });
   };
+
   const request = target.url.protocol === "https:" ? httpsRequest : httpRequest;
+
   return await new Promise((resolve, reject) => {
     const controller = new AbortController();
     let attemptExpired = false;
+
     const timer = setTimeout(() => {
       attemptExpired = true;
       controller.abort();
     }, attemptTimeoutMs);
+
     const forwardAbort = () => controller.abort();
+
     if (signal.aborted) controller.abort();
     else signal.addEventListener("abort", forwardAbort, { once: true });
     let settled = false;
+
     const finish = (callback: () => void) => {
       if (settled) return;
       settled = true;
@@ -92,8 +102,10 @@ async function requestOnce(
       signal.removeEventListener("abort", forwardAbort);
       callback();
     };
+
     const unreachableError = () =>
       new Error(`web_fetch could not reach ${address} within ${attemptTimeoutMs} ms.`);
+
     const outgoing = request(
       target.url,
       {
@@ -114,6 +126,7 @@ async function requestOnce(
         } else finish(() => resolve(response));
       },
     );
+
     outgoing.once("error", (error) => {
       // The per-attempt deadline surfaces as a raw AbortError from the HTTP
       // client; report it as an unreachable address instead so the next
@@ -148,14 +161,17 @@ export async function requestPinned(
   const attemptTimeoutMs = options.attemptTimeoutMs ?? CONNECT_ATTEMPT_TIMEOUT_MS;
   const addresses = target.addresses?.length ? target.addresses : [target.address];
   let lastError: unknown;
+
   for (const address of addresses) {
     try {
       return await requestOnce(target, address, signal, attemptTimeoutMs, options.headers ?? {});
     } catch (error) {
       lastError = error;
+
       if (signal.aborted) throw error;
     }
   }
+
   throw lastError ?? new Error("web_fetch could not connect.");
 }
 
@@ -168,9 +184,29 @@ export async function requestPinned(
  */
 export function responseHeader(response: IncomingMessage, name: string): string | undefined {
   const value = response.headers[name];
+
   return Array.isArray(value) ? value[0] : value;
 }
 
+export function responseHeaderValues(response: IncomingMessage, name: string): string[] {
+  const values: string[] = [];
+  const lowerName = name.toLowerCase();
+
+  const rawHeaders = response.rawHeaders ?? [];
+
+  for (let index = 0; index + 1 < rawHeaders.length; index += 2) {
+    if (rawHeaders[index].toLowerCase() === lowerName) {
+      values.push(rawHeaders[index + 1]);
+    }
+  }
+
+  if (values.length > 0) return values;
+
+  const value = response.headers[name];
+
+  return value === undefined ? [] : Array.isArray(value) ? value : [value];
+}
+
 /**
  * Read and buffer the complete response body.
  *
@@ -189,27 +225,33 @@ export async function readResponseBytes(
   signal?: AbortSignal,
 ): Promise<Uint8Array> {
   const declared = Number(responseHeader(response, "content-length"));
+
   if (Number.isFinite(declared) && declared > maxBytes) {
     response.destroy();
     throw new Error(responseTooLargeMessage(declared, maxBytes, true));
   }
+
   // Once response headers arrive the connect deadline and caller signal are no
   // longer wired to the socket, so a stalled body would otherwise hang the
   // fetch forever. Keep the caller signal attached for the whole body read and
   // drop the socket when it fires.
   const forwardAbort = () => response.destroy();
+
   if (signal?.aborted) response.destroy();
   else signal?.addEventListener("abort", forwardAbort, { once: true });
   const chunks: Uint8Array[] = [];
   let total = 0;
+
   try {
     for await (const value of response) {
       const chunk = value instanceof Uint8Array ? value : encoder.encode(value);
       total += chunk.byteLength;
+
       if (total > maxBytes) {
         response.destroy();
         throw new Error(responseTooLargeMessage(total, maxBytes, false));
       }
+
       chunks.push(chunk);
     }
   } catch (error) {
@@ -218,15 +260,18 @@ export async function readResponseBytes(
   } finally {
     signal?.removeEventListener("abort", forwardAbort);
   }
+
   // Destroying the socket can end the stream without an error; detect a
   // mid-read abort here as well so truncated bodies never look complete.
   if (signal?.aborted) throw abortedError();
   const output = new Uint8Array(total);
   let offset = 0;
+
   for (const chunk of chunks) {
     output.set(chunk, offset);
     offset += chunk.byteLength;
   }
+
   return output;
 }
 
@@ -242,6 +287,7 @@ export async function readResponseBytes(
  */
 export function decodeResponse(bytes: Uint8Array, contentTypeHeader: string): string {
   const charset = contentTypeHeader.match(/(?:^|;)\s*charset\s*=\s*["']?([^;"'\s]+)/i)?.[1];
+
   try {
     return new TextDecoder(charset || "utf-8").decode(bytes);
   } catch {

package/src/network.ts

@@ -5,6 +5,7 @@ export {
   type ResolveAddresses,
   type ValidatedTarget,
 } from "./network-policy";
+
 export {
   decodeResponse,
   FETCH_MAX_BYTES,

package/src/outline.ts

@@ -1,6 +1,9 @@
 const FENCE = /^[ \t]{0,3}(`{3,}|~{3,})/;
+
 const MAX_FALLBACK_HEADING_CHARACTERS = 60;
+
 export const MAX_OUTLINE_HEADINGS = 12;
+
 export const MAX_OUTLINE_HEADING_CHARACTERS = 120;
 
 /** One bounded heading exposed as untrusted document-shape metadata. */
@@ -41,15 +44,20 @@ interface MarkdownFence {
   length: number;
 }
 
+/** Tracks whether the current Markdown line opens or closes a fenced code block. */
 function updateFence(line: string, fence: MarkdownFence | undefined): MarkdownFence | undefined {
   const match = FENCE.exec(line);
+
   if (!match) return fence;
   const marker = match[1];
+
   if (!fence) return { character: marker[0], length: marker.length };
+
   const closes =
     marker[0] === fence.character &&
     marker.length >= fence.length &&
     !line.slice(match[0].length).trim();
+
   return closes ? undefined : fence;
 }
 
@@ -66,15 +74,20 @@ function isSpaceOrTab(character: string | undefined): boolean {
 /** Parses the bounded ATX prefix with a linear scan to avoid backtracking on remote input. */
 function parseAtxHeading(line: string): ParsedAtxHeading | undefined {
   let index = 0;
+
   while (index < 3 && isSpaceOrTab(line[index])) index += 1;
+
   if (isSpaceOrTab(line[index])) return undefined;
 
   const markerStart = index;
+
   while (line[index] === "#") index += 1;
   const level = index - markerStart;
+
   if (level < 1 || level > 6 || !isSpaceOrTab(line[index])) return undefined;
 
   while (isSpaceOrTab(line[index])) index += 1;
+
   return { level, text: line.slice(index) };
 }
 
@@ -86,6 +99,7 @@ function parseAtxHeading(line: string): ParsedAtxHeading | undefined {
  */
 function countWords(value: string): number {
   const trimmed = value.trim();
+
   return trimmed ? trimmed.split(/\s+/).length : 0;
 }
 
@@ -97,10 +111,12 @@ function countWords(value: string): number {
  */
 function countDocumentWords(lines: string[]): number {
   let fence: MarkdownFence | undefined;
+
   return lines.reduce((total, line) => {
     const wasInsideFence = Boolean(fence);
     fence = updateFence(line, fence);
     const heading = wasInsideFence || fence ? undefined : parseAtxHeading(line);
+
     return total + countWords(heading ? cleanHeading(heading.text) : line);
   }, 0);
 }
@@ -113,12 +129,16 @@ function countDocumentWords(lines: string[]): number {
  */
 function cleanHeading(value: string): string {
   let contentEnd = value.length;
+
   while (contentEnd > 0 && isSpaceOrTab(value[contentEnd - 1])) contentEnd -= 1;
 
   let markerStart = contentEnd;
+
   while (markerStart > 0 && value[markerStart - 1] === "#") markerStart -= 1;
+
   if (markerStart < contentEnd && markerStart > 0 && isSpaceOrTab(value[markerStart - 1])) {
     contentEnd = markerStart - 1;
+
     while (contentEnd > 0 && isSpaceOrTab(value[contentEnd - 1])) contentEnd -= 1;
   }
 
@@ -139,16 +159,21 @@ function collectAtxHeadings(lines: string[]): HeadingCollection {
   for (const [lineNumber, line] of lines.entries()) {
     const wasInsideFence = Boolean(fence);
     fence = updateFence(line, fence);
+
     if (wasInsideFence || fence) continue;
     const heading = parseAtxHeading(line);
+
     if (!heading) continue;
     const text = cleanHeading(heading.text);
+
     if (!text) continue;
     total += 1;
+
     if (locations.length <= MAX_OUTLINE_HEADINGS) {
       locations.push({ level: heading.level, text, line: lineNumber, inferred: false });
     }
   }
+
   return { locations, total };
 }
 
@@ -162,6 +187,7 @@ function collectAtxHeadings(lines: string[]): HeadingCollection {
 function looksLikeFallbackHeading(line: string, nextLine: string): boolean {
   if (/^\s{4,}/.test(line)) return false;
   const candidate = line.trim();
+
   if (
     !candidate ||
     candidate.length > MAX_FALLBACK_HEADING_CHARACTERS ||
@@ -173,8 +199,10 @@ function looksLikeFallbackHeading(line: string, nextLine: string): boolean {
   ) {
     return false;
   }
+
   if (!/^[A-Z]/.test(candidate) && !/[`()._]/.test(candidate)) return false;
   const following = nextLine.trim();
+
   return following.length > MAX_FALLBACK_HEADING_CHARACTERS || /[.!?]$/.test(following);
 }
 
@@ -190,25 +218,32 @@ function collectFallbackHeadings(lines: string[]): HeadingCollection {
   const nextNonEmptyIndexes = new Int32Array(lines.length);
   nextNonEmptyIndexes.fill(-1);
   let nextNonEmptyIndex = -1;
+
   for (let lineNumber = lines.length - 1; lineNumber >= 0; lineNumber -= 1) {
     nextNonEmptyIndexes[lineNumber] = nextNonEmptyIndex;
+
     if (lines[lineNumber].trim()) nextNonEmptyIndex = lineNumber;
   }
+
   let fence: MarkdownFence | undefined;
 
   for (const [lineNumber, line] of lines.entries()) {
     const wasInsideFence = Boolean(fence);
     fence = updateFence(line, fence);
+
     if (wasInsideFence || fence) continue;
     const followingIndex = nextNonEmptyIndexes[lineNumber];
     const following = followingIndex === -1 ? "" : lines[followingIndex];
+
     if (looksLikeFallbackHeading(line, following)) {
       total += 1;
+
       if (locations.length <= MAX_OUTLINE_HEADINGS) {
         locations.push({ level: 2, text: cleanHeading(line), line: lineNumber, inferred: true });
       }
     }
   }
+
   return { locations, total };
 }
 
@@ -222,8 +257,10 @@ export function createDocumentOutline(markdown: string): DocumentOutline {
   const lines = markdown.split("\n");
   const explicit = collectAtxHeadings(lines);
   const collection = explicit.total > 0 ? explicit : collectFallbackHeadings(lines);
+
   const headings = collection.locations.slice(0, MAX_OUTLINE_HEADINGS).map((heading, index) => {
     const next = collection.locations[index + 1]?.line ?? lines.length;
+
     return {
       level: heading.level,
       text: heading.text,
@@ -231,6 +268,7 @@ export function createDocumentOutline(markdown: string): DocumentOutline {
       inferred: heading.inferred,
     };
   });
+
   return {
     totalWords: countDocumentWords(lines),
     totalHeadings: collection.total,

package/src/redact.ts

@@ -17,14 +17,17 @@ const SENSITIVE_QUERY_KEYS = new Set([
   "token",
 ]);
 
+/** Replaces sensitive query-parameter values and reports whether anything changed. */
 function redactSensitiveParams(params: URLSearchParams): boolean {
   let redacted = false;
+
   for (const key of params.keys()) {
     if (SENSITIVE_QUERY_KEYS.has(key.toLowerCase())) {
       params.set(key, "REDACTED");
       redacted = true;
     }
   }
+
   return redacted;
 }
 
@@ -37,6 +40,7 @@ function redactSensitiveParams(params: URLSearchParams): boolean {
  */
 export function redactUrlForDisplay(value: string | URL): string {
   let url: URL;
+
   try {
     url = value instanceof URL ? new URL(value.href) : new URL(value);
   } catch {
@@ -46,9 +50,12 @@ export function redactUrlForDisplay(value: string | URL): string {
   url.username = "";
   url.password = "";
   redactSensitiveParams(url.searchParams);
+
   if (url.hash.length > 1) {
     const fragmentParams = new URLSearchParams(url.hash.slice(1));
+
     if (redactSensitiveParams(fragmentParams)) url.hash = fragmentParams.toString();
   }
+
   return url.href;
 }

package/src/render.ts

@@ -13,14 +13,17 @@ export function formatCollapsibleOutput(
   theme: RenderTheme,
 ): string {
   const lines = output.replace(/\r\n/g, "\n").split("\n");
+
   while (lines.at(-1) === "") lines.pop();
   const totalLines = lines.length;
   const visibleLines = expanded ? lines : lines.slice(0, COLLAPSED_LINES);
   let text = visibleLines.map((line) => theme.fg("toolOutput", line)).join("\n");
 
   const remaining = totalLines - visibleLines.length;
+
   if (remaining > 0) {
     text += `${theme.fg("muted", `\n... (${remaining} more lines, ${totalLines} total,`)} ${keyHint("app.tools.expand", "to expand")}${theme.fg("muted", ")")}`;
   }
+
   return `\n${text}`;
 }