@zeldrisho/pi-web-fetch 0.8.2 → 0.9.1

package/src/fetch.ts

@@ -11,6 +11,7 @@ import {
   FETCH_MAX_BYTES,
   readResponseBytes,
   responseHeader,
+  responseHeaderValues,
 } from "./network-transport";
 
 /**
@@ -27,31 +28,43 @@ import {
 export function normalizeGitHubRawUrl(rawUrl: string): string {
   try {
     const url = new URL(rawUrl);
+
     if (url.protocol !== "https:") return rawUrl;
+
     if (url.hostname === "github.com") {
       if (url.pathname.includes("/blob/")) {
         return `https://raw.githubusercontent.com${url.pathname.replace("/blob/", "/")}${url.search}`;
       }
+
       // Parse pathname segments: /owner/repo/tree/ref/path
       const segments = url.pathname.split("/").filter(Boolean);
+
       if (segments.length >= 4 && segments[2] === "tree") {
         const lastSegment = segments[segments.length - 1];
+
         if (lastSegment.includes(".")) {
           // Reconstruct path: /owner/repo/ref/path (remove "tree" segment)
           const newPath = `/${segments[0]}/${segments[1]}/${segments.slice(3).join("/")}`;
+
           return `https://raw.githubusercontent.com${newPath}${url.search}`;
         }
+
         return rawUrl;
       }
+
       return rawUrl;
     }
+
     if (url.hostname === "gist.github.com") {
       const segments = url.pathname.split("/").filter(Boolean);
+
       // Only rewrite bare gist pages. Subpages such as `/revisions`, `/forks`, or an
       // explicit `/raw` path already point at a usable resource and stay untouched.
       if (segments.length !== 2) return rawUrl;
+
       return `https://gist.github.com/${segments[0]}/${segments[1]}/raw${url.search}`;
     }
+
     return rawUrl;
   } catch {
     return rawUrl;
@@ -90,10 +103,14 @@ export function parseLinkHeaderForAgentHints(
   let current = "";
   let inQuotes = false;
   let inAngleBrackets = false;
+
   for (const character of header) {
     if (character === '"') inQuotes = !inQuotes;
+
     if (character === "<") inAngleBrackets = true;
+
     if (character === ">" && !inQuotes) inAngleBrackets = false;
+
     if (character === "," && !inQuotes && !inAngleBrackets) {
       directives.push(current);
       current = "";
@@ -101,39 +118,52 @@ export function parseLinkHeaderForAgentHints(
       current += character;
     }
   }
+
   directives.push(current);
 
   let describedBy: string | undefined;
   let markdownAlternate: string | undefined;
+
   for (const directive of directives) {
     const match = /^\s*<([^>]*)>\s*(.*)$/.exec(directive);
+
     if (!match) continue;
     const [, rawTarget, rawParameters] = match;
     const parameters = new Map<string, string>();
+
     for (const parameter of rawParameters.split(";")) {
       const equals = parameter.indexOf("=");
+
       if (equals === -1) continue;
       const key = parameter.slice(0, equals).trim().toLowerCase();
+
       const value = parameter
         .slice(equals + 1)
         .trim()
         .replace(/^"|"$/g, "");
+
       parameters.set(key, value);
     }
+
     const relations = (parameters.get("rel") ?? "").toLowerCase().split(/\s+/);
     const type = (parameters.get("type") ?? "").toLowerCase();
     let resolved: string | undefined;
+
     try {
       resolved = new URL(rawTarget.trim(), baseUrl).href;
     } catch {
       continue;
     }
+
     if (!resolved) continue;
+
     if (!describedBy && relations.includes("describedby")) describedBy = resolved;
+
     if (!markdownAlternate && relations.includes("alternate") && type === "text/markdown") {
       markdownAlternate = resolved;
     }
   }
+
   return { describedBy, markdownAlternate };
 }
 
@@ -142,6 +172,43 @@ export interface FetchRemoteDependencies extends RedirectDependencies {
   timeoutMs?: number;
 }
 
+/**
+ * Returns a usable media type from a response header.
+ *
+ * Some servers incorrectly emit an equivalent media type more than once (for example,
+ * `text/html,text/html; charset=utf-8`). Treat that as the declared type, but reject
+ * conflicting media types rather than guessing which representation to parse.
+ */
+function normalizeContentType(header: string): string | undefined {
+  const mediaTypes: string[] = [];
+  let current = "";
+  let quoted = false;
+
+  for (const character of header) {
+    if (character === '"') quoted = !quoted;
+
+    if (character === "," && !quoted) {
+      mediaTypes.push(current);
+      current = "";
+    } else current += character;
+  }
+
+  mediaTypes.push(current);
+  const normalized: string[] = [];
+
+  for (const value of mediaTypes) {
+    const mediaType = value.split(";", 1)[0].trim().toLowerCase();
+
+    if (mediaType) normalized.push(mediaType);
+  }
+
+  if (normalized.length === 0 || normalized.some((value) => value !== normalized[0])) {
+    return undefined;
+  }
+
+  return normalized[0];
+}
+
 /**
  * Converts a successful HTTP response into a complete document.
  *
@@ -160,47 +227,62 @@ async function documentFromResponse(
   extractHtml: typeof extractHtmlToMarkdown,
 ): Promise<CompleteDocument> {
   const status = response.statusCode ?? 0;
+
   if (status < 200 || status >= 300) {
     response.resume();
+
     const authenticationHint = [401, 403, 404].includes(status)
       ? " The page may be missing, private, or require authentication."
       : "";
+
     throw new Error(`web_fetch returned HTTP ${status}.${authenticationHint}`);
   }
 
-  const contentTypeHeader = responseHeader(response, "content-type") ?? "text/plain";
-  const contentType = contentTypeHeader.split(";", 1)[0].trim().toLowerCase();
+  const contentTypeHeader =
+    responseHeaderValues(response, "content-type").join(", ") || "text/plain";
+
+  const contentType = normalizeContentType(contentTypeHeader);
+
   const linkHints = parseLinkHeaderForAgentHints(
     responseHeader(response, "link") ?? "",
     target.url,
   );
+
   const allowed =
-    contentType.startsWith("text/") ||
-    [
-      "application/json",
-      "application/markdown",
-      "application/x-markdown",
-      "application/xml",
-      "application/xhtml+xml",
-    ].includes(contentType);
+    contentType !== undefined &&
+    (contentType.startsWith("text/") ||
+      [
+        "application/json",
+        "application/markdown",
+        "application/x-markdown",
+        "application/xml",
+        "application/xhtml+xml",
+      ].includes(contentType));
+
   if (!allowed) {
     response.destroy();
-    throw new Error(`web_fetch does not support ${contentType || "this content type"}.`);
+    throw new Error(
+      `web_fetch does not support ${contentType ?? (contentTypeHeader || "this content type")}.`,
+    );
   }
 
   const raw = decodeResponse(
     await readResponseBytes(response, FETCH_MAX_BYTES, signal),
     contentTypeHeader,
   );
+
   let markdown: string;
   let title: string | undefined;
+  let fragmentOffsets: Record<string, number> | undefined;
   let extractor: CompleteDocument["extractor"] = "raw";
   let describedBy: string | undefined = linkHints.describedBy;
   let markdownAlternateUrl: string | undefined = linkHints.markdownAlternate;
+
   if (contentType === "text/html" || contentType === "application/xhtml+xml") {
     const extracted = await awaitWithAbort(extractHtml(raw, target.url), signal);
     markdown = extracted.markdown;
     title = extracted.title;
+    fragmentOffsets = extracted.fragmentOffsets;
     extractor = extracted.extractor;
     describedBy = describedBy ?? extracted.describedByLink;
     markdownAlternateUrl = markdownAlternateUrl ?? extracted.markdownAlternateLink;
@@ -214,9 +296,11 @@ async function documentFromResponse(
 
   const isHtml = contentType === "text/html" || contentType === "application/xhtml+xml";
   const extractionDiagnostics = isHtml ? diagnoseExtraction(raw, markdown) : undefined;
+
   const shellSuspected = extractionDiagnostics
     ? hasExtractionWarning(extractionDiagnostics)
     : false;
+
   const etag = responseHeader(response, "etag");
   const lastModified = responseHeader(response, "last-modified");
 
@@ -224,6 +308,7 @@ async function documentFromResponse(
     url: target.url.toString(),
     contentType,
     markdown: markdown.replace(/<\/untrusted_web_content>/gi, "&lt;/untrusted_web_content&gt;"),
+    fragmentOffsets,
     title,
     extractor,
     shellSuspected,
@@ -245,14 +330,17 @@ async function documentFromResponse(
  */
 function assertAbsoluteHttpUrlForFetch(value: string): URL {
   let url: URL;
+
   try {
     url = new URL(value);
   } catch {
     throw new Error("web_fetch received an invalid URL.");
   }
+
   if (url.protocol !== "http:" && url.protocol !== "https:") {
     throw new Error(`web_fetch only supports http(s) URLs: ${redactUrlForDisplay(url)}`);
   }
+
   return url;
 }
 
@@ -276,14 +364,18 @@ async function fetchDocument(
   const timeoutMs = dependencies.timeoutMs ?? REQUEST_TIMEOUT_MS;
   const extractHtml = dependencies.extractHtml ?? extractHtmlToMarkdown;
   let timedOut = false;
+
   const timeout = setTimeout(() => {
     timedOut = true;
     controller.abort();
   }, timeoutMs);
+
   const cancel = () => controller.abort();
   signal?.addEventListener("abort", cancel, { once: true });
   const conditionalHeaders: Record<string, string> = {};
+
   if (cached?.validators?.etag) conditionalHeaders["If-None-Match"] = cached.validators.etag;
+
   if (cached?.validators?.lastModified)
     conditionalHeaders["If-Modified-Since"] = cached.validators.lastModified;
 
@@ -294,11 +386,14 @@ async function fetchDocument(
       dependencies,
       conditionalHeaders,
     );
+
     if (response.statusCode === 304 && cached) {
       response.resume();
       const etag = responseHeader(response, "etag") ?? cached.validators?.etag;
+
       const lastModified =
         responseHeader(response, "last-modified") ?? cached.validators?.lastModified;
+
       return {
         document: {
           ...cached,
@@ -308,12 +403,14 @@ async function fetchDocument(
         revalidated: true,
       };
     }
+
     return {
       document: await documentFromResponse(target, response, controller.signal, extractHtml),
       revalidated: false,
     };
   } catch (error) {
     if (timedOut) throw new Error(`web_fetch timed out after ${timeoutMs / 1000} seconds.`);
+
     if (signal?.aborted) throw new Error("web_fetch was cancelled.");
     throw error;
   } finally {

package/src/focus.ts

@@ -1,4 +1,5 @@
 const WORD_PATTERN = /[\p{L}\p{N}_+#]+/gu;
+
 const HEADING_PATTERN = /^#{1,6}\s+\S/;
 
 /** Metadata describing deterministic query-focused extraction. */
@@ -35,24 +36,31 @@ function splitSections(markdown: string): Section[] {
     .split(/\n{2,}/)
     .map((block) => block.trim())
     .filter(Boolean);
+
   if (blocks.length === 0) return [];
 
   const lines = markdown.split(/\r?\n/);
   const hasHeadings = lines.some((line) => HEADING_PATTERN.test(line));
+
   if (!hasHeadings) return blocks.map((block) => ({ markdown: block, tokens: tokens(block) }));
 
   const sections: string[] = [];
   let current: string[] = [];
+
   const flush = () => {
     const section = current.join("\n").trim();
+
     if (section) sections.push(section);
     current = [];
   };
+
   for (const line of lines) {
     if (HEADING_PATTERN.test(line)) flush();
     current.push(line);
   }
+
   flush();
+
   return sections.map((section) => ({ markdown: section, tokens: tokens(section) }));
 }
 
@@ -83,7 +91,9 @@ export function focusMarkdown(markdown: string, query: string): FocusResult {
 
   const averageLength =
     sections.reduce((sum, section) => sum + section.tokens.length, 0) / sections.length || 1;
+
   const documentFrequency = new Map<string, number>();
+
   for (const term of queryTerms) {
     documentFrequency.set(
       term,
@@ -93,24 +103,31 @@ export function focusMarkdown(markdown: string, query: string): FocusResult {
 
   const selected = sections.filter((section) => {
     const frequencies = new Map<string, number>();
+
     for (const token of section.tokens) frequencies.set(token, (frequencies.get(token) ?? 0) + 1);
     const lengthNormalization = 1 - 0.75 + 0.75 * (section.tokens.length / averageLength);
     let score = 0;
+
     for (const term of queryTerms) {
       const frequency = frequencies.get(term) ?? 0;
+
       if (frequency === 0) continue;
       const frequencyInDocuments = documentFrequency.get(term) ?? 0;
+
       const inverseDocumentFrequency = Math.log(
         1 + (sections.length - frequencyInDocuments + 0.5) / (frequencyInDocuments + 0.5),
       );
+
       score +=
         inverseDocumentFrequency *
         ((frequency * (1.2 + 1)) / (frequency + 1.2 * lengthNormalization));
     }
+
     return score > 0;
   });
 
   const focused = selected.map((section) => section.markdown).join("\n\n");
+
   return {
     markdown: focused,
     details: {

package/src/index.ts

@@ -15,7 +15,9 @@ import { formatCollapsibleOutput } from "./render";
 import { executeWebFetch } from "./service";
 
 export { ExpiringLruCache } from "./cache";
+
 export type { FetchResult } from "./content";
+
 export {
   diagnoseExtraction,
   extractDocumentLinks,
@@ -23,10 +25,15 @@ export {
   type ExtractedLinks,
   type ExtractionDiagnostics,
 } from "./evidence";
+
 export { fetchRemoteContent, type FetchRemoteDependencies } from "./fetch";
+
 export { focusMarkdown, type FocusDetails, type FocusResult } from "./focus";
+
 export { createDocumentOutline, type DocumentOutline, type OutlineHeading } from "./outline";
+
 export { redactUrlForDisplay } from "./redact";
+
 export {
   executeWebFetch,
   type CacheStatus,
@@ -34,6 +41,7 @@ export {
   type WebFetchParameters,
   type WebFetchTruncationDetails,
 } from "./service";
+
 export {
   FETCH_MAX_BYTES,
   isPrivateAddress,
@@ -85,7 +93,7 @@ export default function (pi: ExtensionAPI) {
     name: "web_fetch",
     label: "Web Fetch",
     description:
-      "Fetch a public HTTP(S) page and return a bounded Markdown content chunk with continuation metadata.",
+      "Fetch a public HTTP(S) page and return a bounded Markdown content chunk with continuation metadata. URL fragments start at the matching heading or anchor; explicit offsets take precedence.",
     promptSnippet: "Read a public web page as bounded Markdown",
     promptGuidelines: [
       "Use web_fetch for a user-provided URL or to inspect relevant sources found with web_search.",
@@ -110,6 +118,7 @@ export default function (pi: ExtensionAPI) {
       if (isPartial) return new Text(theme.fg("warning", "Fetching…"), 0, 0);
 
       const content = result.content.find((item) => item.type === "text");
+
       return new Text(
         content?.type === "text"
           ? formatCollapsibleOutput(content.text, expanded, theme)

package/src/inflight.ts

@@ -19,12 +19,16 @@ function waitForCaller<T>(
   cancelledMessage: string,
 ): Promise<T> {
   if (!signal) return operation;
+
   return new Promise((resolve, reject) => {
     const abort = (): void => reject(new Error(cancelledMessage));
+
     if (signal.aborted) {
       abort();
+
       return;
     }
+
     signal.addEventListener("abort", abort, { once: true });
     operation.then(resolve, reject).finally(() => signal.removeEventListener("abort", abort));
   });
@@ -60,7 +64,10 @@ export class InflightCoalescer<K, V> {
     signal: AbortSignal | undefined,
     cancelledMessage: string,
   ): Promise<V> {
+    if (signal?.aborted) throw new Error(cancelledMessage);
+
     let entry = this.#entries.get(key);
+
     if (!entry) {
       if (this.#entries.size >= this.maxEntries) return operation(signal);
       const controller = new AbortController();
@@ -79,16 +86,19 @@ export class InflightCoalescer<K, V> {
     }
 
     entry.waiters += 1;
+
     try {
       return await waitForCaller(entry.promise, signal, cancelledMessage);
     } finally {
       entry.waiters -= 1;
+
       if (entry.waiters === 0 && !entry.settled) entry.controller.abort();
     }
   }
 
   #settle(key: K, entry: InflightEntry<V>): void {
     entry.settled = true;
+
     if (this.#entries.get(key) === entry) this.#entries.delete(key);
   }
 }

package/src/network-policy.ts

@@ -64,15 +64,19 @@ const GLOBALLY_REACHABLE_IPV6_EXCEPTIONS = [
 ] as const;
 
 const blockedIPv4Addresses = new BlockList();
+
 const blockedIPv6Addresses = new BlockList();
+
 const allowedIPv6Addresses = new BlockList();
 
 for (const [network, prefix] of BLOCKED_IPV4_RANGES) {
   blockedIPv4Addresses.addSubnet(network, prefix, "ipv4");
 }
+
 for (const [network, prefix] of BLOCKED_IPV6_RANGES) {
   blockedIPv6Addresses.addSubnet(network, prefix, "ipv6");
 }
+
 for (const [network, prefix] of GLOBALLY_REACHABLE_IPV6_EXCEPTIONS) {
   allowedIPv6Addresses.addSubnet(network, prefix, "ipv6");
 }
@@ -106,11 +110,15 @@ export type ResolveAddresses = (hostname: string) => Promise<string[]>;
  */
 export function isPrivateAddress(address: string): boolean {
   const family = isIP(address);
+
   if (family === 4) return blockedIPv4Addresses.check(address, "ipv4");
+
   if (family === 6) {
     if (allowedIPv6Addresses.check(address, "ipv6")) return false;
+
     return blockedIPv6Addresses.check(address, "ipv6");
   }
+
   return true;
 }
 
@@ -136,8 +144,10 @@ export async function validateRemoteUrl(
   resolveHostname?: ResolveAddresses,
 ): Promise<ValidatedTarget> {
   const url = value instanceof URL ? value : new URL(value);
+
   if (url.protocol !== "http:" && url.protocol !== "https:")
     throw new Error("web_fetch only supports HTTP and HTTPS URLs.");
+
   if (url.username || url.password)
     throw new Error("web_fetch blocks URLs containing credentials.");
 
@@ -145,24 +155,31 @@ export async function validateRemoteUrl(
     .toLowerCase()
     .replace(/^\[|\]$/g, "")
     .replace(/\.$/, "");
+
   if (!hostname || hostname === "localhost" || hostname.endsWith(".localhost")) {
     throw new Error("web_fetch blocks local hostnames.");
   }
 
   let addresses: string[];
+
   if (isIP(hostname)) addresses = [hostname];
   else if (resolveHostname) addresses = await resolveHostname(hostname);
   else {
     const records = await dnsLookup(hostname, { all: true, verbatim: true });
     addresses = [];
+
     for (const record of records) addresses.push(record.address);
   }
+
   if (addresses.length === 0 || addresses.some(isPrivateAddress)) {
     throw new Error(`web_fetch blocks private or reserved network targets (${hostname}).`);
   }
+
   const ordered = preferIpv4First(addresses);
   const address = ordered[0];
   const family = isIP(address);
+
   if (family !== 4 && family !== 6) throw new Error(`web_fetch could not resolve ${hostname}.`);
+
   return { url, address, family, addresses: ordered };
 }