@zeldrisho/pi-web-fetch 0.11.0 → 0.12.0

package/CHANGELOG.md

@@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.12.0] - 2026-10-23
+
+### Changed
+
+- Show discovered `llms.txt` indexes on every fetch result so the index remains visible when fetching multiple pages from the same origin.
+- Prefer native Markdown representations during content negotiation while retaining supported text formats.
+
+### Fixed
+
+- Suggest links to pages containing a requested fragment when it is absent from the current page.
+- Handle empty and text-only HTML fragments without a null-document parser crash.
+- Resolve section wrapper fragments when extracted headings are omitted.
+
+## [0.11.1] - 2026-10-20
+
+### Changed
+
+- Simplify the model-facing fetch tool and parameter descriptions.
+
+### Fixed
+
+- Retry 404 and 406 responses to Markdown-negotiated requests with HTML preferred, allowing pages whose HTML representation is available to fetch successfully.
+
 ## [0.11.0] - 2026-10-19
 
 ### Added
@@ -223,7 +246,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add bounded public page fetching ([b979496](https://github.com/zeldrisho/pi-packages/commit/b979496b32de1cead172ba570307e4a4a7b3421d))
 
-[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.11.0...HEAD
+[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.12.0...HEAD
+[0.12.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.11.1...pi-web-fetch-v0.12.0
+[0.11.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.11.0...pi-web-fetch-v0.11.1
 [0.11.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.10.2...pi-web-fetch-v0.11.0
 [0.10.2]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.10.1...pi-web-fetch-v0.10.2
 [0.10.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.10.0...pi-web-fetch-v0.10.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zeldrisho/pi-web-fetch",
-  "version": "0.11.0",
+  "version": "0.12.0",
   "description": "Pi extension for secure, bounded public web page fetching and Markdown extraction",
   "keywords": [
     "pi",

package/src/acquisition/fetch.ts

@@ -8,7 +8,7 @@ import {
 
 export { normalizeGitHubRawUrl, parseLinkHeaderForAgentHints } from "./fetch-metadata.ts";
 
-import { awaitWithAbort } from "../abort";
+import { awaitWithAbort } from "../abort.ts";
 import {
   hasAnubisMarkers,
   parseAnubisChallenge,
@@ -31,7 +31,7 @@ import {
   type RedirectDependencies,
 } from "../network/network-redirects.ts";
 import type { ValidatedTarget } from "../network/network-policy.ts";
-import { redactUrlForDisplay } from "../redact";
+import { redactUrlForDisplay } from "../redact.ts";
 import {
   decodeResponse,
   decompressResponse,
@@ -478,13 +478,29 @@ async function fetchDocument(
   };
 
   try {
+    const requestUrl = normalizeGitHubRawUrl(rawUrl);
+
     let { target, response } = await requestFollowingRedirects(
-      normalizeGitHubRawUrl(rawUrl),
+      requestUrl,
       controller.signal,
       requestDependencies,
       conditionalHeaders,
     );
 
+    // Some servers route Markdown negotiation to a missing representation even
+    // though their HTML page exists (for example, Redis documentation URLs).
+    // Retry only these representation-related statuses, through the same
+    // validated redirect/network boundary, with HTML preferred.
+    if ([404, 406].includes(response.statusCode ?? 0)) {
+      response.resume();
+      ({ target, response } = await requestFollowingRedirects(
+        target.url,
+        controller.signal,
+        requestDependencies,
+        { Accept: "text/html, text/plain;q=0.9, */*;q=0.1" },
+      ));
+    }
+
     if (
       response.statusCode !== undefined &&
       responseHeader(response, "cf-mitigated")?.trim().toLowerCase() === "challenge"

package/src/document/extract.ts

@@ -82,11 +82,13 @@ function isElementNode(node: ChildNode): node is Element {
  * HTML5 permits those omissions, but body-based extraction (Defuddle and the plain-text
  * fallback) then sees nothing. Rebuild a well-formed `<html>` document, moving the
  * fragment's top-level nodes into `<head>` or `<body>` as appropriate.
+ * Empty and text-only documents also receive these wrappers, with text moved
+ * into the body. Documents already rooted at `<html>` are left unchanged.
  */
 function normalizeFragmentDocument(document: Document): void {
   const root = document.documentElement;
 
-  if (!root || root.tagName === "HTML") return;
+  if (root?.tagName === "HTML") return;
 
   const html = document.createElement("html");
   const head = document.createElement("head");
@@ -94,9 +96,18 @@ function normalizeFragmentDocument(document: Document): void {
   html.appendChild(head);
   html.appendChild(body);
 
-  const nodes = root.tagName === "BODY" ? [...root.childNodes] : [...document.childNodes];
+  const nodes = root
+    ? root.tagName === "BODY"
+      ? [...root.childNodes]
+      : [...document.childNodes]
+    : [...document.childNodes];
 
   for (const node of nodes) {
+    if (node.nodeType === 3) {
+      body.appendChild(node);
+      continue;
+    }
+
     if (!isElementNode(node)) continue;
     const tagName = node.tagName;
 

package/src/document/fragment-index.ts

@@ -36,6 +36,53 @@ interface FragmentOffsets {
   [fragment: string]: number;
 }
 
+interface NormalizedMarkdown {
+  text: string;
+  offsets: number[];
+}
+
+/**
+ * Collapses whitespace and lowercases Markdown while retaining the source
+ * offset of every resulting character.
+ */
+function normalizeMarkdown(markdown: string): NormalizedMarkdown {
+  const characters: string[] = [];
+  const offsets: number[] = [];
+  let index = 0;
+
+  while (index < markdown.length) {
+    const character = markdown[index] ?? "";
+
+    if (/\s/u.test(character)) {
+      let end = index + 1;
+
+      while (end < markdown.length && /\s/u.test(markdown[end] ?? "")) end += 1;
+
+      characters.push(" ");
+      offsets.push(index);
+      index = end;
+      continue;
+    }
+
+    for (const lowered of character.toLowerCase()) {
+      characters.push(lowered);
+      offsets.push(index);
+    }
+
+    index += 1;
+  }
+
+  return { text: characters.join(""), offsets };
+}
+
+/**
+ * Maps HTML ids, named anchors, and Markdown heading slugs to extracted Markdown positions.
+ * HTML anchors match headings or retained text, including section descendants;
+ * anchors with no matching content are omitted.
+ *
+ * @param originalIds - Original ids for elements renamed during extraction; override current ids
+ * @returns Zero-based UTF-16 code-unit offsets into `markdown`, keyed by fragment without `#`
+ */
 export function buildFragmentOffsets(
   document: Document,
   markdown: string,
@@ -58,6 +105,7 @@ export function buildFragmentOffsets(
 
   const offsets: Record<string, number> = {};
   const used = new Set<number>();
+  let normalizedMarkdown: NormalizedMarkdown | undefined;
 
   for (const element of document.querySelectorAll<HTMLElement>("[id], a[name]")) {
     const id =
@@ -82,7 +130,28 @@ export function buildFragmentOffsets(
       if (text) {
         const textOffset = markdown.toLowerCase().indexOf(text.toLowerCase());
 
-        if (textOffset >= 0) offsets[id] = textOffset;
+        if (textOffset >= 0) {
+          offsets[id] = textOffset;
+          continue;
+        }
+      }
+
+      // Section wrapper IDs often precede headings omitted by extraction. Fall
+      // back to the first retained paragraph/list/code block inside the section.
+      for (const descendant of element.querySelectorAll<HTMLElement>("p, li, pre, blockquote")) {
+        const content = (descendant.textContent ?? "").replace(/\s+/g, " ").trim();
+
+        if (content.length < 12) continue;
+        normalizedMarkdown ??= normalizeMarkdown(markdown);
+        const normalizedContent = content.toLowerCase();
+        const position = normalizedMarkdown.text.indexOf(normalizedContent);
+
+        if (position >= 0) {
+          const offset = normalizedMarkdown.offsets[position];
+
+          if (offset !== undefined) offsets[id] = offset;
+          break;
+        }
       }
     }
   }

package/src/index.ts

@@ -3,7 +3,7 @@ import { Text } from "@earendil-works/pi-tui";
 import { Type } from "typebox";
 import { redactUrlForDisplay } from "./redact";
 import { formatCollapsibleOutput } from "./render";
-import { executeWebFetch, resetLlmsNoticeOrigins } from "./service";
+import { executeWebFetch } from "./service";
 
 export { ExpiringLruCache } from "./cache";
 
@@ -61,10 +61,8 @@ export {
 
 export const webFetchParameters = Type.Object({
   url: Type.String(),
-  query: Type.Optional(Type.String({ description: "Return only matching sections (long pages)." })),
-  offset: Type.Optional(
-    Type.Integer({ description: "Character offset; use nextOffset to continue." }),
-  ),
+  query: Type.Optional(Type.String({ description: "Find sections matching this text." })),
+  offset: Type.Optional(Type.Integer({ description: "Start at this character offset." })),
 });
 
 /**
@@ -73,14 +71,10 @@ export const webFetchParameters = Type.Object({
  * @param pi - The extension API instance
  */
 export default function (pi: ExtensionAPI) {
-  // SAFETY: The host provides `on`; minimal extension test doubles may omit it.
-  (pi as ExtensionAPI & { on?: ExtensionAPI["on"] }).on?.("session_start", () =>
-    resetLlmsNoticeOrigins(),
-  );
   pi.registerTool({
     name: "web_fetch",
     label: "Web Fetch",
-    description: "Fetch a public HTTP(S) page as Markdown (6,000 chars per call).",
+    description: "Read a public webpage as Markdown.",
     parameters: webFetchParameters,
 
     renderCall(args, theme) {

package/src/network/network-redirects.ts

@@ -1,5 +1,5 @@
 import type { IncomingMessage } from "node:http";
-import { awaitWithAbort } from "../abort";
+import { awaitWithAbort } from "../abort.ts";
 import type { CookieJar } from "./cookies.ts";
 import { validateRemoteUrl, type ValidatedTarget } from "./network-policy.ts";
 import { requestPinned, responseHeader } from "./network-transport.ts";

package/src/network/network-transport.ts

@@ -51,12 +51,15 @@ function responseTooLargeMessage(
 /**
  * Sends a request to a specific validated network address.
  *
+ * The default Accept header prefers Markdown over other supported text formats.
+ *
  * @param target - The validated request target.
  * @param address - The IPv4 or IPv6 address to use for the connection.
  * @param signal - Signal used to cancel the request.
- * @param attemptTimeoutMs - Maximum time allowed for the connection attempt.
- * @returns The received response.
- * @throws If `address` is not a valid IPv4 or IPv6 address.
+ * @param attemptTimeoutMs - Deadline in milliseconds until response headers arrive.
+ * @param headers - Request headers that can override the default Accept and User-Agent headers.
+ * @returns The response once headers arrive, regardless of HTTP status; its body is not read.
+ * @throws If the address is invalid, the deadline expires, the signal aborts, or the HTTP request fails.
  */
 async function requestOnce(
   target: ValidatedTarget,
@@ -113,7 +116,8 @@ async function requestOnce(
         lookup,
         signal: controller.signal,
         headers: {
-          Accept: "text/markdown, text/html, text/plain, application/json;q=0.9, */*;q=0.1",
+          Accept:
+            "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, application/xhtml+xml;q=0.7, application/json;q=0.7, application/xml;q=0.7, */*;q=0.1",
           "User-Agent": "Mozilla/5.0 (compatible; PiWebFetch/1.0; +https://pi.dev)",
           ...headers,
         },

package/src/service.ts

@@ -193,13 +193,6 @@ export interface WebFetchDetails {
   fragment?: { requested: string; matched: boolean; offset?: number };
 }
 
-const llmsNoticeOrigins = new Set<string>();
-
-/** Clears per-origin notice history so subsequent fetches can show llms.txt index notices again. */
-export function resetLlmsNoticeOrigins(): void {
-  llmsNoticeOrigins.clear();
-}
-
 interface WebFetchUpdate {
   content: Array<{ type: "text"; text: string }>;
   details: Record<string, never>;
@@ -286,7 +279,24 @@ interface FetchAcquisition {
 
 const inflightFetches = new InflightCoalescer<string, FetchAcquisition>(MAX_INFLIGHT_REQUESTS);
 
-/** Executes a validated web fetch, including caching, focusing, and bounded pagination. */
+/**
+ * Executes a validated web fetch, including caching, focusing, and bounded pagination.
+ *
+ * A URL fragment selects the starting heading, anchor, or source range unless
+ * an explicit offset is supplied. A query selects sections before pagination.
+ * Missing fragments may suggest up to three links to other pages with that
+ * fragment. Discovered llms.txt index URLs appear on every unblocked result,
+ * including cache hits. Detected bot-wall documents return a blocked result
+ * without page content.
+ *
+ * @param params - Fetch options; offsets and character limits count UTF-16 code units
+ * @param params.offset - Integer from 0 to 20,000,000; applies to the focused view when querying
+ * @param params.maxCharacters - Integer from 1,000 to 30,000; defaults to 6,000 before line/byte limits
+ * @param signal - Cancels acquisition; fresh cache hits do not consult this signal
+ * @param onUpdate - Receives cache or acquisition progress before content is returned
+ * @returns Bounded text output and metadata, including cache, fragment, and continuation details
+ * @throws For invalid parameter limits; acquisition, cancellation, and progress callback errors propagate
+ */
 export async function executeWebFetch(
   params: WebFetchParameters,
   signal: AbortSignal | undefined,
@@ -404,6 +414,35 @@ export async function executeWebFetch(
   const resolvedFragment = resolveFragmentOffset(document, params.url);
   const fragmentOffset = resolvedFragment.offset;
   const fragmentMatched = resolvedFragment.fragment !== undefined && fragmentOffset !== undefined;
+
+  const fragmentSuggestions =
+    resolvedFragment.fragment !== undefined && !fragmentMatched
+      ? [
+          ...(document.links?.internal ?? []).map((link) => link.url),
+          ...(document.links?.external ?? []).map((link) => link.url),
+          ...[...document.markdown.matchAll(/\]\(([^)]+)\)/g)].map((match) => match[1]),
+        ]
+          .filter((url) => {
+            try {
+              const linkedUrl = new URL(url, document.url);
+
+              if (decodeURIComponent(linkedUrl.hash.slice(1)) !== resolvedFragment.fragment) {
+                return false;
+              }
+
+              linkedUrl.hash = "";
+              const sourceUrl = new URL(document.url);
+              sourceUrl.hash = "";
+
+              return linkedUrl.href !== sourceUrl.href;
+            } catch {
+              return false;
+            }
+          })
+          .slice(0, 3)
+          .map((url) => redactUrlForDisplay(new URL(url, document.url).href))
+      : [];
+
   const startingOffset = explicitOffset ?? fragmentOffset ?? FETCH_DEFAULT_OFFSET;
 
   const hasFragmentRange =
@@ -460,13 +499,7 @@ export async function executeWebFetch(
   const contentKind = classifyContentKind(rawFinalUrl, result.extractor, shellSuspected);
   const confidence = classifyConfidence(result.extractor, shellSuspected, result.markdown.length);
 
-  const llmsNoticeOrigin = result.llmsTxtIndexUrl
-    ? new URL(result.llmsTxtIndexUrl).origin
-    : undefined;
-
-  const showLlmsNotice = Boolean(llmsNoticeOrigin && !llmsNoticeOrigins.has(llmsNoticeOrigin));
-
-  if (showLlmsNotice && llmsNoticeOrigin) llmsNoticeOrigins.add(llmsNoticeOrigin);
+  const showLlmsNotice = Boolean(result.llmsTxtIndexUrl);
 
   const output = blocked
     ? "Blocked by an anti-bot challenge. Requested page content is unavailable."
@@ -489,7 +522,13 @@ export async function executeWebFetch(
           ? [`llms.txt index: ${redactUrlForDisplay(result.llmsTxtIndexUrl)}`, ""]
           : []),
         ...(resolvedFragment.fragment !== undefined && !fragmentMatched
-          ? ["[The requested URL fragment was not found; showing the page from the beginning.]", ""]
+          ? [
+              "[The requested URL fragment was not found; showing the page from the beginning.]",
+              ...(fragmentSuggestions.length
+                ? [`[This page links to the requested fragment: ${fragmentSuggestions.join(", ")}]`]
+                : []),
+              "",
+            ]
           : []),
         ...(fragmentMatched ? ["[Starting at the requested URL fragment.]", ""] : []),
         ...(focused