@zeldrisho/pi-web-fetch 0.7.0 → 0.7.1

package/CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.7.1] - 2026-08-28
+
+### Fixed
+
+- Prevent immediately unreachable pinned addresses from emitting uncaught socket errors and terminating Pi under concurrent fetch load
+- Fix `https://github.com/voidzero-dev/vite-plus/releases` rendering in pi: harden `runDefuddle` to swallow bare-pathname `ERR_INVALID_URL` detached rejections, guarantee absolute `http(s)` URLs at every extraction/fetch boundary, strip `nav`/`header`/`footer`/`aside` chrome before Defuddle, deduplicate adjacent identical markdown links, and bump `defuddle` to `^0.19.3`
+- Fix GitHub file fetches returning `https://github.com/llms.txt` instead of raw content: bypass `llms.txt` probing and `describedby`/`alternate` fallbacks for `github.com`, `raw.githubusercontent.com`, `gist.github.com` and `gist.githubusercontent.com`, and rewrite `github.com/*/tree/<ref>/*.*` file URLs to `raw.githubusercontent.com` like existing `blob` handling so `tree` file links (e.g. `.../permission-gate.ts`) return clean code
+
 ## [0.7.0] - 2026-08-24
 
 ### Added
@@ -102,7 +110,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add bounded public page fetching ([b979496](https://github.com/zeldrisho/pi-packages/commit/b979496b32de1cead172ba570307e4a4a7b3421d))
 
-[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.7.0...HEAD
+[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.7.1...HEAD
+[0.7.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.7.0...pi-web-fetch-v0.7.1
 [0.7.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.6.1...pi-web-fetch-v0.7.0
 [0.6.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.6.0...pi-web-fetch-v0.6.1
 [0.6.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.5.4...pi-web-fetch-v0.6.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zeldrisho/pi-web-fetch",
-  "version": "0.7.0",
+  "version": "0.7.1",
   "description": "Pi extension for secure, bounded public web page fetching and Markdown extraction",
   "keywords": [
     "pi",
@@ -31,7 +31,7 @@
     "access": "public"
   },
   "dependencies": {
-    "defuddle": "^0.19.2",
+    "defuddle": "^0.19.3",
     "linkedom": "^0.18.13"
   },
   "devDependencies": {

package/src/content.ts

@@ -4,6 +4,12 @@ const CONTENT_LINE_BUDGET = Math.max(1, DEFAULT_MAX_LINES - 10);
 const CONTENT_BYTE_BUDGET = Math.max(1_024, DEFAULT_MAX_BYTES - 2_048);
 const encoder = new TextEncoder();
 
+/**
+ * A complete fetched document with extracted content and metadata.
+ *
+ * Contains the full markdown representation along with metadata about
+ * the extraction method and any special handling that was applied.
+ */
 export interface CompleteDocument {
   url: string;
   contentType: string;
@@ -24,6 +30,12 @@ export interface CompleteDocument {
   markdownAlternateFallback?: boolean;
 }
 
+/**
+ * A paginated chunk of fetched content with offset tracking.
+ *
+ * Extends CompleteDocument with pagination metadata to support
+ * reading large documents in bounded chunks.
+ */
 export interface FetchResult extends CompleteDocument {
   offset: number;
   nextOffset?: number;
@@ -55,6 +67,18 @@ function boundedContentChunk(value: string, offset: number, maxCharacters: numbe
   return sliceByByteLength(chunk, CONTENT_BYTE_BUDGET);
 }
 
+/**
+ * Slice a complete document into a bounded content chunk.
+ *
+ * Extracts a chunk of content starting at the given offset, respecting
+ * character, line, and byte limits. Adds pagination metadata and
+ * continuation instructions when content is truncated.
+ *
+ * @param document - Complete document to slice
+ * @param offset - Character offset to start reading from
+ * @param maxCharacters - Maximum characters to include in the chunk
+ * @returns Paginated fetch result with content chunk and metadata
+ */
 export function sliceCompleteDocument(
   document: CompleteDocument,
   offset: number,

package/src/extract.ts

@@ -76,6 +76,78 @@ export function htmlToMarkdownFallback(html: string): string {
     .trim();
 }
 
+/** Asserts that a URL is absolute http(s) and returns it, or throws for bare pathnames. */
+export function assertAbsoluteHttpUrl(value: string | URL): URL {
+  const url = value instanceof URL ? value : new URL(value);
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new TypeError(`Expected absolute http(s) URL, got ${url.protocol}: ${url.href}`);
+  }
+  return url;
+}
+
+/**
+ * Type guard that checks if an unknown value is a string.
+ *
+ * @param value - The value to check
+ * @returns `true` if the value is a string
+ */
+// oxlint-disable-next-line anti-slop/no-unknown-parameters -- type-guard helper intentionally narrows unknown at call site
+function isString(value: unknown): value is string {
+  return typeof value === "string";
+}
+
+/** Returns true when a rejection looks like `ERR_INVALID_URL` for a bare pathname. */
+function isBarePathnameInvalidUrl(cause: unknown): boolean {
+  if (!(cause instanceof Error)) return false;
+  if (!/Invalid URL/i.test(cause.message)) return false;
+  // SAFETY: Node's ERR_INVALID_URL extends Error with an `input` string property holding the offending URL.
+  const input = (cause as NodeJS.ErrnoException & { input?: unknown }).input;
+  if (isString(input) && input.startsWith("/")) return true;
+  // SAFETY: Error.stack is an optional string populated by V8; we coerce it for pattern matching.
+  const detail = `${cause.message}\n${(cause as Error).stack ?? ""}`;
+  // Node's ERR_INVALID_URL prints `input: '/path'` in the message/stack even when
+  // `input` is not on the error instance in some Node versions.
+  return /input:\s*'\/[^']*'/.test(detail);
+}
+
+/**
+ * Removes chrome wrappers that pollute Defuddle's article extraction on GitHub releases.
+ *
+ * @param document - The document from which to remove chrome wrappers
+ * @param baseUrl - The absolute URL of the page, used to determine if chrome should be stripped
+ */
+function stripChromeWrappers(document: Document, baseUrl: URL): void {
+  // Only strip chrome wrappers for GitHub release pages
+  const isGitHubRelease =
+    baseUrl.hostname === "github.com" && /\/releases(?:\/|$)/.test(baseUrl.pathname);
+  if (!isGitHubRelease) return;
+  for (const element of document.querySelectorAll("nav, header, footer, aside")) {
+    element.remove();
+  }
+}
+
+/**
+ * Collapses consecutive duplicate markdown link lines (e.g. duplicated [Latest] on GitHub releases).
+ *
+ * @param markdown - The markdown string to process
+ * @returns The markdown with adjacent duplicate links removed
+ */
+function deduplicateAdjacentLinks(markdown: string): string {
+  const lines = markdown.split("\n");
+  const out: string[] = [];
+  let previousTrimmed: string | undefined;
+  for (const line of lines) {
+    const trimmed = line.trim();
+    const isLinkLine =
+      /^\[[^\]]+\]\(<[^>]+>\)$/.test(trimmed) || /^\[[^\]]+\]\([^)]+\)$/.test(trimmed);
+    if (isLinkLine && trimmed === previousTrimmed) continue;
+    out.push(line);
+    previousTrimmed = isLinkLine ? trimmed : undefined;
+  }
+  // Also collapse duplicate adjacent blocks like a repeated vite-plus tag link paragraph.
+  return out.join("\n").replace(/(\[[^\n]+?\([^)]+\)[^\n]*\n)\1/g, "$1");
+}
+
 /**
  * Runs Defuddle over an already-normalized document with Markdown extraction enabled.
  *
@@ -94,9 +166,10 @@ export function htmlToMarkdownFallback(html: string): string {
  * To keep `extractHtmlToMarkdown` from ever propagating such a failure, this
  * helper arms a scoped `unhandledRejection` listener for the lifetime of the
  * call. The listener is scoped, not process-wide in effect: it only treats a
- * rejection as a Defuddle failure when its message or stack mentions Defuddle,
- * so unrelated rejections from other concurrent work are ignored and do not
- * force a spurious fallback to the basic extractor. After Defuddle resolves we
+ * rejection as a Defuddle failure when its message or stack mentions Defuddle
+ * or looks like a bare-pathname `ERR_INVALID_URL` occurring within the armed
+ * window, so unrelated rejections from other concurrent work are ignored and do
+ * not force a spurious fallback to the basic extractor. After Defuddle resolves we
  * flush a microtask and a macrotask so any rejection Defuddle scheduled settles
  * inside the armed window; a rejection observed there is re-thrown so the
  * caller falls back. Deeply-nested timers in Defuddle are out of scope and would
@@ -112,13 +185,11 @@ async function runDefuddle(
 ): Promise<DefuddleResponse | undefined> {
   let escapedRejection: unknown = undefined;
   let armed = true;
-  // Only attribute a rejection to Defuddle when it mentions Defuddle. This keeps
-  // the guard scoped so unrelated concurrent rejections are ignored.
   const captureUnhandled = (cause: unknown): void => {
     if (!armed || escapedRejection !== undefined) return;
     const detail =
       cause instanceof Error ? `${cause.message}\n${cause.stack ?? ""}` : String(cause);
-    if (/defuddle/i.test(detail)) escapedRejection = cause;
+    if (/defuddle/i.test(detail) || isBarePathnameInvalidUrl(cause)) escapedRejection = cause;
   };
   process.on("unhandledRejection", captureUnhandled);
   try {
@@ -200,15 +271,17 @@ export async function extractHtmlToMarkdown(
 }> {
   let advertised: AdvertisedLinks = {};
   try {
+    assertAbsoluteHttpUrl(baseUrl);
     const { document } = parseHTML(html);
     removeMalformedSchemaOrgData(document);
     normalizeSelectorUnsafeIds(document);
+    stripChromeWrappers(document, baseUrl);
     advertised = readAdvertisedLinks(document, baseUrl);
     // Pass the full absolute URL so Defuddle resolves relative links (e.g.
     // `/owner/repo/releases`) and metadata against the real origin instead of
     // dropping the scheme and host and constructing `new URL(pathname)`.
     const result = await runDefuddle(document, baseUrl.href);
-    const markdown = result?.content?.trim() ?? "";
+    const markdown = deduplicateAdjacentLinks(result?.content?.trim() ?? "");
     const trimmedTitle = result?.title?.trim();
     if (markdown) {
       return {

package/src/fetch.ts

@@ -27,8 +27,21 @@ export function normalizeGitHubRawUrl(rawUrl: string): string {
     const url = new URL(rawUrl);
     if (url.protocol !== "https:") return rawUrl;
     if (url.hostname === "github.com") {
-      if (!url.pathname.includes("/blob/")) return rawUrl;
-      return `https://raw.githubusercontent.com${url.pathname.replace("/blob/", "/")}${url.search}`;
+      if (url.pathname.includes("/blob/")) {
+        return `https://raw.githubusercontent.com${url.pathname.replace("/blob/", "/")}${url.search}`;
+      }
+      // Parse pathname segments: /owner/repo/tree/ref/path
+      const segments = url.pathname.split("/").filter(Boolean);
+      if (segments.length >= 4 && segments[2] === "tree") {
+        const lastSegment = segments[segments.length - 1];
+        if (lastSegment.includes(".")) {
+          // Reconstruct path: /owner/repo/ref/path (remove "tree" segment)
+          const newPath = `/${segments[0]}/${segments[1]}/${segments.slice(3).join("/")}`;
+          return `https://raw.githubusercontent.com${newPath}${url.search}`;
+        }
+        return rawUrl;
+      }
+      return rawUrl;
     }
     if (url.hostname === "gist.github.com") {
       const segments = url.pathname.split("/").filter(Boolean);
@@ -242,11 +255,41 @@ async function documentFromResponse(
   };
 }
 
+/**
+ * Validates that a URL string is an absolute http or https URL.
+ *
+ * @param value - The URL string to validate
+ * @returns The parsed URL object
+ * @throws If the URL is invalid or not http(s)
+ */
+function assertAbsoluteHttpUrlForFetch(value: string): URL {
+  let url: URL;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error(`Invalid URL: ${value}`);
+  }
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error(`web_fetch only supports http(s) URLs: ${value}`);
+  }
+  return url;
+}
+
+/**
+ * Fetches a URL and returns the complete document with extracted content.
+ *
+ * @param rawUrl - The URL to fetch
+ * @param signal - Optional abort signal to cancel the request
+ * @param dependencies - Dependencies for network policy validation, transport, and HTML extraction
+ * @returns The complete document with extracted markdown content
+ * @throws If the request times out, is cancelled, or encounters an error
+ */
 export async function fetchCompleteDocument(
   rawUrl: string,
   signal: AbortSignal | undefined,
   dependencies: FetchRemoteDependencies,
 ): Promise<CompleteDocument> {
+  assertAbsoluteHttpUrlForFetch(normalizeGitHubRawUrl(rawUrl));
   const controller = new AbortController();
   const timeoutMs = dependencies.timeoutMs ?? REQUEST_TIMEOUT_MS;
   const extractHtml = dependencies.extractHtml ?? extractHtmlToMarkdown;
@@ -278,6 +321,16 @@ export async function fetchCompleteDocument(
   }
 }
 
+/**
+ * Fetches a URL and returns a slice of the content starting at the given offset.
+ *
+ * @param rawUrl - The URL to fetch
+ * @param offset - The character offset to start slicing from
+ * @param maxCharacters - The maximum number of characters to return
+ * @param signal - Optional abort signal to cancel the request
+ * @param dependencies - Dependencies for network policy validation, transport, and HTML extraction
+ * @returns The sliced fetch result with content and metadata
+ */
 export async function fetchRemoteContent(
   rawUrl: string,
   offset: number,

package/src/index.ts

@@ -51,6 +51,14 @@ export const webFetchParameters = Type.Object({
   ),
 });
 
+/**
+ * Pi web fetch extension that registers the web_fetch tool.
+ *
+ * Provides a tool for fetching public HTTP(S) pages and converting them to
+ * Markdown with support for pagination via offset/nextOffset parameters.
+ *
+ * @param pi - The extension API instance
+ */
 export default function (pi: ExtensionAPI) {
   pi.registerTool({
     name: "web_fetch",

package/src/limits.ts

@@ -1,6 +1,17 @@
+/** Default starting offset for content extraction (characters). */
 export const FETCH_DEFAULT_OFFSET = 0;
+
+/** Minimum allowed value for maxCharacters parameter. */
 export const FETCH_MIN_MAX_CHARACTERS = 1_000;
+
+/** Default maximum characters to return per fetch. */
 export const FETCH_DEFAULT_MAX_CHARACTERS = 6_000;
+
+/** Maximum allowed characters to return per fetch. */
 export const FETCH_MAX_CHARACTERS = 30_000;
+
+/** Maximum allowed offset for content extraction (characters). */
 export const FETCH_MAX_OFFSET_CHARACTERS = 20_000_000;
+
+/** Maximum allowed length for input URLs (characters). */
 export const FETCH_MAX_URL_CHARACTERS = 2_048;

package/src/network-policy.ts

@@ -1,11 +1,13 @@
 import { lookup as dnsLookup } from "node:dns/promises";
 import { BlockList, isIP } from "node:net";
 
-// Reviewed 2026-08-03 against the IANA IPv4 and IPv6 Special-Purpose Address Registries:
-// https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
-// https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
-// Keep non-global, documentation, benchmarking, multicast, and reserved space blocked. The explicit
-// endpoints make registry reviews and boundary tests auditable without changing the pinning flow.
+/**
+ * IPv4 address ranges that are blocked from network requests.
+ *
+ * Reviewed 2026-08-03 against the IANA IPv4 Special-Purpose Address Registry:
+ * https://www.iana.org/assignments/iana-ipv4-special-registry/iana-ipv4-special-registry.xhtml
+ * Includes non-global, documentation, benchmarking, multicast, and reserved ranges.
+ */
 export const BLOCKED_IPV4_RANGES = [
   ["0.0.0.0", 8, "0.0.0.0", "0.255.255.255", "current network"],
   ["10.0.0.0", 8, "10.0.0.0", "10.255.255.255", "private use"],
@@ -27,6 +29,13 @@ export const BLOCKED_IPV4_RANGES = [
   ["240.0.0.0", 4, "240.0.0.0", "255.255.255.255", "reserved"],
 ] as const;
 
+/**
+ * IPv6 address ranges that are blocked from network requests.
+ *
+ * Reviewed 2026-08-03 against the IANA IPv6 Special-Purpose Address Registry:
+ * https://www.iana.org/assignments/iana-ipv6-special-registry/iana-ipv6-special-registry.xhtml
+ * Includes non-global, documentation, benchmarking, multicast, and reserved ranges.
+ */
 export const BLOCKED_IPV6_RANGES = [
   ["::", 128, "::", "::", "unspecified"],
   ["::1", 128, "::1", "::1", "loopback"],
@@ -68,6 +77,12 @@ for (const [network, prefix] of GLOBALLY_REACHABLE_IPV6_EXCEPTIONS) {
   allowedIPv6Addresses.addSubnet(network, prefix, "ipv6");
 }
 
+/**
+ * A validated and resolved network target ready for connection.
+ *
+ * Contains the target URL, preferred address, and all resolved addresses
+ * for the hostname.
+ */
 export interface ValidatedTarget {
   url: URL;
   /** The preferred address used for the first connection attempt. */
@@ -80,6 +95,7 @@ export interface ValidatedTarget {
   addresses?: string[];
 }
 
+/** Function type for resolving a hostname to IP addresses. */
 export type ResolveAddresses = (hostname: string) => Promise<string[]>;
 
 /**

package/src/network-redirects.ts

@@ -3,8 +3,10 @@ import { awaitWithAbort } from "./abort";
 import { validateRemoteUrl, type ValidatedTarget } from "./network-policy";
 import { requestPinned, responseHeader } from "./network-transport";
 
+/** Maximum number of HTTP redirects to follow before aborting. */
 export const FETCH_MAX_REDIRECTS = 5;
 
+/** Optional dependencies for redirect handling (used for testing). */
 export interface RedirectDependencies {
   validateUrl?: (value: string | URL) => Promise<ValidatedTarget>;
   request?: (target: ValidatedTarget, signal: AbortSignal) => Promise<IncomingMessage>;

package/src/network-transport.ts

@@ -4,6 +4,7 @@ import { isIP, type LookupFunction } from "node:net";
 import { formatSize } from "@earendil-works/pi-coding-agent";
 import type { ValidatedTarget } from "./network-policy";
 
+/** Maximum size in bytes for raw HTTP response bodies. */
 export const FETCH_MAX_BYTES = 5 * 1_024 * 1_024;
 
 /**
@@ -63,8 +64,13 @@ async function requestOnce(
   const family = isIP(address);
   if (family !== 4 && family !== 6) throw new Error(`web_fetch could not resolve ${address}.`);
   const lookup: LookupFunction = (_hostname, options, callback) => {
-    if (options.all) callback(null, [{ address, family }]);
-    else callback(null, address, family);
+    // Match dns.lookup's asynchronous callback contract. Calling back inline
+    // lets an immediately unreachable address emit a socket error before
+    // node:http has finished installing its forwarding listeners.
+    queueMicrotask(() => {
+      if (options.all) callback(null, [{ address, family }]);
+      else callback(null, address, family);
+    });
   };
   const request = target.url.protocol === "https:" ? httpsRequest : httpRequest;
   return await new Promise((resolve, reject) => {
@@ -161,6 +167,18 @@ export function responseHeader(response: IncomingMessage, name: string): string
   return Array.isArray(value) ? value[0] : value;
 }
 
+/**
+ * Read and buffer the complete response body.
+ *
+ * Enforces the maximum byte limit and respects abort signals during the read.
+ * Destroys the response stream if limits are exceeded or abort is signaled.
+ *
+ * @param response - HTTP response to read
+ * @param maxBytes - Maximum allowed response size in bytes
+ * @param signal - Optional abort signal to cancel the read
+ * @returns Buffered response body as bytes
+ * @throws When response exceeds maxBytes or abort signal fires
+ */
 export async function readResponseBytes(
   response: IncomingMessage,
   maxBytes: number,
@@ -208,6 +226,16 @@ export async function readResponseBytes(
   return output;
 }
 
+/**
+ * Decode HTTP response bytes to a string using the charset from Content-Type.
+ *
+ * Extracts the charset parameter from the Content-Type header and uses it
+ * to decode the response. Falls back to UTF-8 if charset is missing or invalid.
+ *
+ * @param bytes - Raw response bytes to decode
+ * @param contentTypeHeader - Value of the Content-Type response header
+ * @returns Decoded response string
+ */
 export function decodeResponse(bytes: Uint8Array, contentTypeHeader: string): string {
   const charset = contentTypeHeader.match(/(?:^|;)\s*charset\s*=\s*["']?([^;"'\s]+)/i)?.[1];
   try {

package/src/service.ts

@@ -47,6 +47,17 @@ function safeUrl(value: string): URL | undefined {
   }
 }
 
+/**
+ * Classify the kind of content returned by a fetch operation.
+ *
+ * Determines the content type based on URL patterns, extractor used,
+ * and whether an app shell was detected.
+ *
+ * @param url - The fetched URL
+ * @param extractor - Extraction method used
+ * @param shellSuspected - Whether the page appears to be an app shell
+ * @returns Content classification
+ */
 export function classifyContentKind(
   url: string,
   extractor: CompleteDocument["extractor"],
@@ -69,6 +80,17 @@ export function classifyContentKind(
   return "unknown";
 }
 
+/**
+ * Classify confidence level for fetched content quality.
+ *
+ * Determines how confident we are that the extracted content accurately
+ * represents the source page, based on extraction method and content length.
+ *
+ * @param extractor - Extraction method used
+ * @param shellSuspected - Whether the page appears to be an app shell
+ * @param markdownLength - Length of extracted markdown content
+ * @returns Confidence level (high, medium, or low)
+ */
 export function classifyConfidence(
   extractor: CompleteDocument["extractor"],
   shellSuspected: boolean,
@@ -87,10 +109,30 @@ export function classifyConfidence(
  * Returns them ordered shallow → deep, and an empty list for non-HTTP(S) targets or when
  * the page itself is already a `llms.txt` path, so a fallback never retries itself.
  */
+function isGitHubLikeHost(hostname: string): boolean {
+  return (
+    hostname === "github.com" ||
+    hostname === "raw.githubusercontent.com" ||
+    hostname === "gist.github.com" ||
+    hostname === "gist.githubusercontent.com"
+  );
+}
+
+/**
+ * Build candidate /llms.txt URLs to probe for a given page.
+ *
+ * Generates a list of potential /llms.txt index URLs by checking the site root
+ * and ancestor directories up to a maximum depth. Returns URLs ordered from
+ * shallowest to deepest.
+ *
+ * @param rawUrl - The original page URL
+ * @returns Array of candidate /llms.txt URLs to probe (empty for non-HTTP/S or GitHub)
+ */
 export function buildLlmsTxtCandidateUrls(rawUrl: string): URL[] {
   try {
     const url = new URL(rawUrl);
     if (url.protocol !== "https:" && url.protocol !== "http:") return [];
+    if (isGitHubLikeHost(url.hostname)) return [];
     if (url.pathname === "/llms.txt" || url.pathname.endsWith("/llms.txt")) return [];
     // Ancestor directories only, one level deep: for `/r2/buckets/x` probe
     // `/llms.txt` and `/r2/llms.txt`, never deeper. A trailing slash marks the final
@@ -232,6 +274,13 @@ async function fetchDocumentWithLlmsTxtSupport(
   signal: AbortSignal | undefined,
   dependencies: FetchRemoteDependencies,
 ): Promise<CompleteDocument> {
+  try {
+    if (isGitHubLikeHost(new URL(rawUrl).hostname)) {
+      return fetchCompleteDocument(rawUrl, signal, dependencies);
+    }
+  } catch {
+    // Invalid URL falls through to fetchCompleteDocument which will throw.
+  }
   const candidates = buildLlmsTxtCandidateUrls(rawUrl);
   const [primary, blindIndex] = await Promise.all([
     fetchCompleteDocument(rawUrl, signal, dependencies),
@@ -277,18 +326,21 @@ function resolveCacheDirectory(name: string): string {
   return base;
 }
 
+/** Parameters for a web fetch operation. */
 export interface WebFetchParameters {
   url: string;
   offset?: number;
   maxCharacters?: number;
 }
 
+/** Details about content truncation and pagination strategy. */
 export interface WebFetchTruncationDetails {
   truncated: boolean;
   strategy: "continuation" | "none";
   nextOffset?: number;
 }
 
+/** Comprehensive metadata about a web fetch result. */
 export interface WebFetchDetails {
   url: string;
   requestedUrl: string;