@zeldrisho/pi-web-fetch 0.8.1 → 0.9.0

package/src/focus.ts

@@ -1,4 +1,5 @@
 const WORD_PATTERN = /[\p{L}\p{N}_+#]+/gu;
+
 const HEADING_PATTERN = /^#{1,6}\s+\S/;
 
 /** Metadata describing deterministic query-focused extraction. */
@@ -35,24 +36,31 @@ function splitSections(markdown: string): Section[] {
     .split(/\n{2,}/)
     .map((block) => block.trim())
     .filter(Boolean);
+
   if (blocks.length === 0) return [];
 
   const lines = markdown.split(/\r?\n/);
   const hasHeadings = lines.some((line) => HEADING_PATTERN.test(line));
+
   if (!hasHeadings) return blocks.map((block) => ({ markdown: block, tokens: tokens(block) }));
 
   const sections: string[] = [];
   let current: string[] = [];
+
   const flush = () => {
     const section = current.join("\n").trim();
+
     if (section) sections.push(section);
     current = [];
   };
+
   for (const line of lines) {
     if (HEADING_PATTERN.test(line)) flush();
     current.push(line);
   }
+
   flush();
+
   return sections.map((section) => ({ markdown: section, tokens: tokens(section) }));
 }
 
@@ -83,7 +91,9 @@ export function focusMarkdown(markdown: string, query: string): FocusResult {
 
   const averageLength =
     sections.reduce((sum, section) => sum + section.tokens.length, 0) / sections.length || 1;
+
   const documentFrequency = new Map<string, number>();
+
   for (const term of queryTerms) {
     documentFrequency.set(
       term,
@@ -93,24 +103,31 @@ export function focusMarkdown(markdown: string, query: string): FocusResult {
 
   const selected = sections.filter((section) => {
     const frequencies = new Map<string, number>();
+
     for (const token of section.tokens) frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
     const lengthNormalization = 1 - 0.75 + 0.75 * (section.tokens.length / averageLength);
     let score = 0;
+
     for (const term of queryTerms) {
       const frequency = frequencies.get(term) ?? 0;
+
       if (frequency === 0) continue;
       const frequencyInDocuments = documentFrequency.get(term) ?? 0;
+
       const inverseDocumentFrequency = Math.log(
         1 + (sections.length - frequencyInDocuments + 0.5) / (frequencyInDocuments + 0.5),
       );
+
       score +=
         inverseDocumentFrequency *
         ((frequency * (1.2 + 1)) / (frequency + 1.2 * lengthNormalization));
     }
+
     return score > 0;
   });
 
   const focused = selected.map((section) => section.markdown).join("\n\n");
+
   return {
     markdown: focused,
     details: {

package/src/index.ts

@@ -15,7 +15,9 @@ import { formatCollapsibleOutput } from "./render";
 import { executeWebFetch } from "./service";
 
 export { ExpiringLruCache } from "./cache";
+
 export type { FetchResult } from "./content";
+
 export {
   diagnoseExtraction,
   extractDocumentLinks,
@@ -23,10 +25,15 @@ export {
   type ExtractedLinks,
   type ExtractionDiagnostics,
 } from "./evidence";
+
 export { fetchRemoteContent, type FetchRemoteDependencies } from "./fetch";
+
 export { focusMarkdown, type FocusDetails, type FocusResult } from "./focus";
+
 export { createDocumentOutline, type DocumentOutline, type OutlineHeading } from "./outline";
+
 export { redactUrlForDisplay } from "./redact";
+
 export {
   executeWebFetch,
   type CacheStatus,
@@ -34,6 +41,7 @@ export {
   type WebFetchParameters,
   type WebFetchTruncationDetails,
 } from "./service";
+
 export {
   FETCH_MAX_BYTES,
   isPrivateAddress,
@@ -85,7 +93,7 @@ export default function (pi: ExtensionAPI) {
     name: "web_fetch",
     label: "Web Fetch",
     description:
-      "Fetch a public HTTP(S) page and return a bounded Markdown content chunk with continuation metadata.",
+      "Fetch a public HTTP(S) page and return a bounded Markdown content chunk with continuation metadata. URL fragments start at the matching heading or anchor; explicit offsets take precedence.",
     promptSnippet: "Read a public web page as bounded Markdown",
     promptGuidelines: [
       "Use web_fetch for a user-provided URL or to inspect relevant sources found with web_search.",
@@ -110,6 +118,7 @@ export default function (pi: ExtensionAPI) {
       if (isPartial) return new Text(theme.fg("warning", "Fetching…"), 0, 0);
 
       const content = result.content.find((item) => item.type === "text");
+
       return new Text(
         content?.type === "text"
           ? formatCollapsibleOutput(content.text, expanded, theme)

package/src/inflight.ts

@@ -19,12 +19,16 @@ function waitForCaller<T>(
   cancelledMessage: string,
 ): Promise<T> {
   if (!signal) return operation;
+
   return new Promise((resolve, reject) => {
     const abort = (): void => reject(new Error(cancelledMessage));
+
     if (signal.aborted) {
       abort();
+
       return;
     }
+
     signal.addEventListener("abort", abort, { once: true });
     operation.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
   });
@@ -60,7 +64,10 @@ export class InflightCoalescer<K, V> {
     signal: AbortSignal | undefined,
     cancelledMessage: string,
   ): Promise<V> {
+    if (signal?.aborted) throw new Error(cancelledMessage);
+
     let entry = this.#entries.get(key);
+
     if (!entry) {
       if (this.#entries.size >= this.maxEntries) return operation(signal);
       const controller = new AbortController();
@@ -79,16 +86,19 @@ export class InflightCoalescer<K, V> {
     }
 
     entry.waiters += 1;
+
     try {
       return await waitForCaller(entry.promise, signal, cancelledMessage);
     } finally {
       entry.waiters -= 1;
+
       if (entry.waiters === 0 && !entry.settled) entry.controller.abort();
     }
   }
 
   #settle(key: K, entry: InflightEntry<V>): void {
     entry.settled = true;
+
     if (this.#entries.get(key) === entry) this.#entries.delete(key);
   }
 }

package/src/network-policy.ts

@@ -64,15 +64,19 @@ const GLOBALLY_REACHABLE_IPV6_EXCEPTIONS = [
 ] as const;
 
 const blockedIPv4Addresses = new BlockList();
+
 const blockedIPv6Addresses = new BlockList();
+
 const allowedIPv6Addresses = new BlockList();
 
 for (const [network, prefix] of BLOCKED_IPV4_RANGES) {
   blockedIPv4Addresses.addSubnet(network, prefix, "ipv4");
 }
+
 for (const [network, prefix] of BLOCKED_IPV6_RANGES) {
   blockedIPv6Addresses.addSubnet(network, prefix, "ipv6");
 }
+
 for (const [network, prefix] of GLOBALLY_REACHABLE_IPV6_EXCEPTIONS) {
   allowedIPv6Addresses.addSubnet(network, prefix, "ipv6");
 }
@@ -106,11 +110,15 @@ export type ResolveAddresses = (hostname: string) => Promise<string[]>;
  */
 export function isPrivateAddress(address: string): boolean {
   const family = isIP(address);
+
   if (family === 4) return blockedIPv4Addresses.check(address, "ipv4");
+
   if (family === 6) {
     if (allowedIPv6Addresses.check(address, "ipv6")) return false;
+
     return blockedIPv6Addresses.check(address, "ipv6");
   }
+
   return true;
 }
 
@@ -136,8 +144,10 @@ export async function validateRemoteUrl(
   resolveHostname?: ResolveAddresses,
 ): Promise<ValidatedTarget> {
   const url = value instanceof URL ? value : new URL(value);
+
   if (url.protocol !== "http:" && url.protocol !== "https:")
     throw new Error("web_fetch only supports HTTP and HTTPS URLs.");
+
   if (url.username || url.password)
     throw new Error("web_fetch blocks URLs containing credentials.");
 
@@ -145,24 +155,31 @@ export async function validateRemoteUrl(
     .toLowerCase()
     .replace(/^\[|\]$/g, "")
     .replace(/\.$/, "");
+
   if (!hostname || hostname === "localhost" || hostname.endsWith(".localhost")) {
     throw new Error("web_fetch blocks local hostnames.");
   }
 
   let addresses: string[];
+
   if (isIP(hostname)) addresses = [hostname];
   else if (resolveHostname) addresses = await resolveHostname(hostname);
   else {
     const records = await dnsLookup(hostname, { all: true, verbatim: true });
     addresses = [];
+
     for (const record of records) addresses.push(record.address);
   }
+
   if (addresses.length === 0 || addresses.some(isPrivateAddress)) {
     throw new Error(`web_fetch blocks private or reserved network targets (${hostname}).`);
   }
+
   const ordered = preferIpv4First(addresses);
   const address = ordered[0];
   const family = isIP(address);
+
   if (family !== 4 && family !== 6) throw new Error(`web_fetch could not resolve ${hostname}.`);
+
   return { url, address, family, addresses: ordered };
 }

package/src/network-redirects.ts

@@ -5,15 +5,20 @@ import { requestPinned, responseHeader } from "./network-transport";
 
 /** Maximum number of HTTP redirects to follow before aborting. */
 export const FETCH_MAX_REDIRECTS = 5;
+
 const MAX_ORIGIN_CONCURRENCY = 4;
+
 const MAX_RETRIES = 2;
+
 const MAX_RETRY_DELAY_MS = 10_000;
+
 const BASE_RETRY_DELAY_MS = 250;
 
 interface OriginState {
   active: number;
   waiting: Array<() => void>;
 }
+
 const origins = new Map<string, OriginState>();
 
 /** Optional dependencies for redirect handling (used for testing). */
@@ -28,21 +33,27 @@ export interface RedirectDependencies {
   sleep?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
 }
 
+/** Creates the consistent abort error used by redirect and retry coordination. */
 function abortError(): Error {
   const error = new Error("Operation aborted.");
   error.name = "AbortError";
+
   return error;
 }
 
+/** Reserves an origin-concurrency slot and returns a function that releases it. */
 async function acquireOrigin(origin: string, signal: AbortSignal): Promise<() => void> {
   const state = origins.get(origin) ?? { active: 0, waiting: [] };
   origins.set(origin, state);
   let reserved = false;
+
   const release = () => {
     state.active -= 1;
     state.waiting.shift()?.();
+
     if (state.active === 0 && state.waiting.length === 0) origins.delete(origin);
   };
+
   if (state.active >= MAX_ORIGIN_CONCURRENCY) {
     await new Promise<void>((resolve, reject) => {
       const enter = () => {
@@ -51,11 +62,14 @@ async function acquireOrigin(origin: string, signal: AbortSignal): Promise<() =>
         reserved = true;
         resolve();
       };
+
       const cancel = () => {
         const index = state.waiting.indexOf(enter);
+
         if (index >= 0) state.waiting.splice(index, 1);
         reject(abortError());
       };
+
       if (signal.aborted) cancel();
       else {
         state.waiting.push(enter);
@@ -63,40 +77,52 @@ async function acquireOrigin(origin: string, signal: AbortSignal): Promise<() =>
       }
     });
   }
+
   if (signal.aborted) {
     if (reserved) release();
     throw abortError();
   }
+
   if (!reserved) state.active += 1;
+
   return release;
 }
 
+/** Waits for a retry delay while allowing the caller to cancel immediately. */
 async function defaultSleep(milliseconds: number, signal: AbortSignal): Promise<void> {
   await new Promise<void>((resolve, reject) => {
     const timer = setTimeout(done, milliseconds);
+
     function done(): void {
       signal.removeEventListener("abort", cancel);
       resolve();
     }
+
     function cancel(): void {
       clearTimeout(timer);
       reject(abortError());
     }
+
     if (signal.aborted) cancel();
     else signal.addEventListener("abort", cancel, { once: true });
   });
 }
 
+/** Parses a Retry-After header into a bounded delay in milliseconds. */
 function retryAfterMilliseconds(value: string | undefined, now = Date.now()): number | undefined {
   if (!value) return undefined;
   const seconds = Number(value.trim());
+
   if (Number.isFinite(seconds) && seconds >= 0)
     return Math.min(seconds * 1_000, MAX_RETRY_DELAY_MS);
   const date = Date.parse(value);
+
   if (!Number.isFinite(date)) return undefined;
+
   return Math.min(Math.max(0, date - now), MAX_RETRY_DELAY_MS);
 }
 
+/** Sends one origin-limited request and retries transient rate-limit responses. */
 async function coordinatedRequest(
   target: ValidatedTarget,
   signal: AbortSignal,
@@ -107,17 +133,22 @@ async function coordinatedRequest(
     dependencies.request ??
     ((value, requestSignal, requestHeaders) =>
       requestPinned(value, requestSignal, { headers: requestHeaders }));
+
   const random = dependencies.random ?? Math.random;
   const sleep = dependencies.sleep ?? defaultSleep;
+
   for (let attempt = 0; ; attempt += 1) {
     const release = await acquireOrigin(target.url.origin, signal);
     let response: IncomingMessage;
+
     try {
       response = await request(target, signal, headers);
     } finally {
       release();
     }
+
     const status = response.statusCode ?? 0;
+
     if (![429, 503].includes(status) || attempt >= MAX_RETRIES) return response;
     // The retry decision uses headers only. Drop an untrusted error body so it cannot
     // consume unbounded bandwidth while the caller is backing off.
@@ -146,10 +177,13 @@ export async function requestFollowingRedirects(
   for (let redirects = 0; redirects <= FETCH_MAX_REDIRECTS; redirects += 1) {
     const response = await coordinatedRequest(target, signal, requestHeaders, dependencies);
     const status = response.statusCode ?? 0;
+
     if (![301, 302, 303, 307, 308].includes(status)) return { target, response };
 
     const location = responseHeader(response, "location");
+
     if (!location) throw new Error("web_fetch received a redirect without a Location header.");
+
     if (redirects === FETCH_MAX_REDIRECTS)
       throw new Error("web_fetch followed too many redirects.");
     response.resume();
@@ -157,5 +191,6 @@ export async function requestFollowingRedirects(
     // Validators describe the originally cached resource and must not leak to a redirect target.
     requestHeaders = {};
   }
+
   throw new Error("web_fetch followed too many redirects.");
 }

package/src/network-transport.ts

@@ -20,6 +20,7 @@ const encoder = new TextEncoder();
 function abortedError(): Error {
   const error = new Error("Operation aborted.");
   error.name = "AbortError";
+
   return error;
 }
 
@@ -39,6 +40,7 @@ function responseTooLargeMessage(
   const size = sizeIsExact
     ? `is ${formatSize(receivedBytes)}`
     : `has reached at least ${formatSize(receivedBytes)}`;
+
   return [
     `web_fetch response ${size}, exceeding the ${formatSize(maxBytes)} raw download limit.`,
     "maxCharacters only controls returned output.",
@@ -63,7 +65,9 @@ async function requestOnce(
   headers: Readonly<Record<string, string>>,
 ): Promise<IncomingMessage> {
   const family = isIP(address);
+
   if (family !== 4 && family !== 6) throw new Error(`web_fetch could not resolve ${address}.`);
+
   const lookup: LookupFunction = (_hostname, options, callback) => {
     // Match dns.lookup's asynchronous callback contract. Calling back inline
     // lets an immediately unreachable address emit a socket error before
@@ -73,18 +77,24 @@ async function requestOnce(
       else callback(null, address, family);
     });
   };
+
   const request = target.url.protocol === "https:" ? httpsRequest : httpRequest;
+
   return await new Promise((resolve, reject) => {
     const controller = new AbortController();
     let attemptExpired = false;
+
     const timer = setTimeout(() => {
       attemptExpired = true;
       controller.abort();
     }, attemptTimeoutMs);
+
     const forwardAbort = () => controller.abort();
+
     if (signal.aborted) controller.abort();
     else signal.addEventListener("abort", forwardAbort, { once: true });
     let settled = false;
+
     const finish = (callback: () => void) => {
       if (settled) return;
       settled = true;
@@ -92,8 +102,10 @@ async function requestOnce(
       signal.removeEventListener("abort", forwardAbort);
       callback();
     };
+
     const unreachableError = () =>
       new Error(`web_fetch could not reach ${address} within ${attemptTimeoutMs} ms.`);
+
     const outgoing = request(
       target.url,
       {
@@ -114,6 +126,7 @@ async function requestOnce(
         } else finish(() => resolve(response));
       },
     );
+
     outgoing.once("error", (error) => {
       // The per-attempt deadline surfaces as a raw AbortError from the HTTP
       // client; report it as an unreachable address instead so the next
@@ -148,14 +161,17 @@ export async function requestPinned(
   const attemptTimeoutMs = options.attemptTimeoutMs ?? CONNECT_ATTEMPT_TIMEOUT_MS;
   const addresses = target.addresses?.length ? target.addresses : [target.address];
   let lastError: unknown;
+
   for (const address of addresses) {
     try {
       return await requestOnce(target, address, signal, attemptTimeoutMs, options.headers ?? {});
     } catch (error) {
       lastError = error;
+
       if (signal.aborted) throw error;
     }
   }
+
   throw lastError ?? new Error("web_fetch could not connect.");
 }
 
@@ -168,6 +184,7 @@ export async function requestPinned(
  */
 export function responseHeader(response: IncomingMessage, name: string): string | undefined {
   const value = response.headers[name];
+
   return Array.isArray(value) ? value[0] : value;
 }
 
@@ -189,27 +206,33 @@ export async function readResponseBytes(
   signal?: AbortSignal,
 ): Promise<Uint8Array> {
   const declared = Number(responseHeader(response, "content-length"));
+
   if (Number.isFinite(declared) && declared > maxBytes) {
     response.destroy();
     throw new Error(responseTooLargeMessage(declared, maxBytes, true));
   }
+
   // Once response headers arrive the connect deadline and caller signal are no
   // longer wired to the socket, so a stalled body would otherwise hang the
   // fetch forever. Keep the caller signal attached for the whole body read and
   // drop the socket when it fires.
   const forwardAbort = () => response.destroy();
+
   if (signal?.aborted) response.destroy();
   else signal?.addEventListener("abort", forwardAbort, { once: true });
   const chunks: Uint8Array[] = [];
   let total = 0;
+
   try {
     for await (const value of response) {
       const chunk = value instanceof Uint8Array ? value : encoder.encode(value);
       total += chunk.byteLength;
+
       if (total > maxBytes) {
         response.destroy();
         throw new Error(responseTooLargeMessage(total, maxBytes, false));
       }
+
       chunks.push(chunk);
     }
   } catch (error) {
@@ -218,15 +241,18 @@ export async function readResponseBytes(
   } finally {
     signal?.removeEventListener("abort", forwardAbort);
   }
+
   // Destroying the socket can end the stream without an error; detect a
   // mid-read abort here as well so truncated bodies never look complete.
   if (signal?.aborted) throw abortedError();
   const output = new Uint8Array(total);
   let offset = 0;
+
   for (const chunk of chunks) {
     output.set(chunk, offset);
     offset += chunk.byteLength;
   }
+
   return output;
 }
 
@@ -242,6 +268,7 @@ export async function readResponseBytes(
  */
 export function decodeResponse(bytes: Uint8Array, contentTypeHeader: string): string {
   const charset = contentTypeHeader.match(/(?:^|;)\s*charset\s*=\s*["']?([^;"'\s]+)/i)?.[1];
+
   try {
     return new TextDecoder(charset || "utf-8").decode(bytes);
   } catch {

package/src/network.ts

@@ -5,6 +5,7 @@ export {
   type ResolveAddresses,
   type ValidatedTarget,
 } from "./network-policy";
+
 export {
   decodeResponse,
   FETCH_MAX_BYTES,