@zeldrisho/pi-web-fetch 0.9.0 → 0.9.2

package/CHANGELOG.md

@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.9.2] - 2026-09-23
+
+### Fixed
+
+- Resolve GitHub Markdown heading fragments and source line, line-range, and column anchors in raw files
+
+## [0.9.1] - 2026-09-22
+
+### Changed
+
+- Upgrade Defuddle to `^0.19.4` for improved metadata and Markdown extraction, HTML sanitization, SVG safety, and support for declarative shadow DOM and MathML content
+
+### Fixed
+
+- Extract HTML correctly when servers send duplicated equivalent `Content-Type` media types, and reject conflicting media-type declarations instead of returning raw HTML
+
 ## [0.9.0] - 2026-09-17
 
 ### Added
@@ -148,7 +164,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add bounded public page fetching ([b979496](https://github.com/zeldrisho/pi-packages/commit/b979496b32de1cead172ba570307e4a4a7b3421d))
 
-[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.9.0...HEAD
+[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.9.2...HEAD
+[0.9.2]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.9.1...pi-web-fetch-v0.9.2
+[0.9.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.9.0...pi-web-fetch-v0.9.1
 [0.9.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.8.2...pi-web-fetch-v0.9.0
 [0.8.2]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.8.1...pi-web-fetch-v0.8.2
 [0.8.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.8.0...pi-web-fetch-v0.8.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zeldrisho/pi-web-fetch",
-  "version": "0.9.0",
+  "version": "0.9.2",
   "description": "Pi extension for secure, bounded public web page fetching and Markdown extraction",
   "keywords": [
     "pi",
@@ -29,16 +29,16 @@
     "access": "public"
   },
   "dependencies": {
-    "defuddle": "^0.19.3",
+    "defuddle": "^0.19.4",
     "linkedom": "^0.18.13"
   },
   "devDependencies": {
-    "@earendil-works/pi-coding-agent": "^0.84.2",
-    "@earendil-works/pi-tui": "^0.84.2",
-    "typebox": "^1.3.16",
+    "@earendil-works/pi-coding-agent": "^0.85.1",
+    "@earendil-works/pi-tui": "^0.85.1",
+    "typebox": "^1.3.30",
     "typescript": "^7.0.2",
-    "vite": "npm:@voidzero-dev/vite-plus-core@0.3.2",
-    "vite-plus": "0.3.2"
+    "vite": "npm:@voidzero-dev/vite-plus-core@0.3.3",
+    "vite-plus": "0.3.3"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",

package/src/fetch.ts

@@ -11,6 +11,7 @@ import {
   FETCH_MAX_BYTES,
   readResponseBytes,
   responseHeader,
+  responseHeaderValues,
 } from "./network-transport";
 
 /**
@@ -77,6 +78,87 @@ export function detectAppShell(raw: string, markdown: string): boolean {
 
 const REQUEST_TIMEOUT_MS = 20_000;
 
+/** Indexes GitHub-style Markdown heading slugs without treating fenced examples as headings. */
+function markdownFragmentOffsets(markdown: string) {
+  const offsets: Record<string, number> = {};
+  const duplicates = new Map<string, number>();
+  const lines = markdown.split("\n");
+  let offset = 0;
+  let fence: { marker: string; length: number } | undefined;
+
+  const addHeading = (text: string, headingOffset: number) => {
+    let withoutHtml = "";
+
+    for (let index = 0; index < text.length; index += 1) {
+      if (text[index] !== "<") {
+        withoutHtml += text[index];
+        continue;
+      }
+
+      const tagEnd = text.indexOf(">", index + 1);
+
+      if (tagEnd < 0) {
+        withoutHtml += text.slice(index + 1);
+        break;
+      }
+
+      index = tagEnd;
+    }
+
+    const base = withoutHtml
+      .replace(/!\[([^\]]*)\]\([^)]*\)/g, "$1")
+      .replace(/\[([^\]]+)\]\([^)]*\)/g, "$1")
+      .replace(/[`*_~]/g, "")
+      .normalize("NFKD")
+      .replace(/[\u0300-\u036f]/g, "")
+      .toLowerCase()
+      .replace(/[^\p{Letter}\p{Number} _-]/gu, "")
+      .trim()
+      .replace(/[ _]+/g, "-");
+
+    if (!base) return;
+
+    const duplicate = duplicates.get(base) ?? 0;
+    duplicates.set(base, duplicate + 1);
+    offsets[duplicate ? `${base}-${duplicate}` : base] = headingOffset;
+  };
+
+  for (let index = 0; index < lines.length; index += 1) {
+    const line = lines[index];
+    const fenceMatch = /^\s{0,3}(`{3,}|~{3,})/.exec(line);
+
+    if (fenceMatch) {
+      const marker = fenceMatch[1][0];
+
+      if (!fence) fence = { marker, length: fenceMatch[1].length };
+      else if (fence.marker === marker && fenceMatch[1].length >= fence.length) fence = undefined;
+    } else if (!fence) {
+      const atxHeading = /^\s{0,3}#{1,6}\s+(.+?)\s*#*\s*$/.exec(line);
+      const setextHeading = /^\s{0,3}(=+|-+)\s*$/.test(lines[index + 1] ?? "");
+
+      if (atxHeading) addHeading(atxHeading[1], offset);
+      else if (setextHeading) addHeading(line.trim(), offset);
+    }
+
+    offset += line.length + 1;
+  }
+
+  return offsets;
+}
+
+/** Indexes GitHub line anchors such as `#L78` in plain-text source files. */
+function lineFragmentOffsets(text: string) {
+  const offsets: Record<string, number> = {};
+  let offset = 0;
+
+  for (const [index, line] of text.split("\n").entries()) {
+    offsets[`L${index + 1}`] = offset;
+    offset += line.length + 1;
+  }
+
+  return offsets;
+}
+
 /** Agent-discovery hints advertised in an HTTP `Link:` header. */
 export interface LinkHeaderAgentHints {
   describedBy?: string;
@@ -171,6 +253,43 @@ export interface FetchRemoteDependencies extends RedirectDependencies {
   timeoutMs?: number;
 }
 
+/**
+ * Returns a usable media type from a response header.
+ *
+ * Some servers incorrectly emit an equivalent media type more than once (for example,
+ * `text/html,text/html; charset=utf-8`). Treat that as the declared type, but reject
+ * conflicting media types rather than guessing which representation to parse.
+ */
+function normalizeContentType(header: string): string | undefined {
+  const mediaTypes: string[] = [];
+  let current = "";
+  let quoted = false;
+
+  for (const character of header) {
+    if (character === '"') quoted = !quoted;
+
+    if (character === "," && !quoted) {
+      mediaTypes.push(current);
+      current = "";
+    } else current += character;
+  }
+
+  mediaTypes.push(current);
+  const normalized: string[] = [];
+
+  for (const value of mediaTypes) {
+    const mediaType = value.split(";", 1)[0].trim().toLowerCase();
+
+    if (mediaType) normalized.push(mediaType);
+  }
+
+  if (normalized.length === 0 || normalized.some((value) => value !== normalized[0])) {
+    return undefined;
+  }
+
+  return normalized[0];
+}
+
 /**
  * Converts a successful HTTP response into a complete document.
  *
@@ -200,8 +319,10 @@ async function documentFromResponse(
     throw new Error(`web_fetch returned HTTP ${status}.${authenticationHint}`);
   }
 
-  const contentTypeHeader = responseHeader(response, "content-type") ?? "text/plain";
-  const contentType = contentTypeHeader.split(";", 1)[0].trim().toLowerCase();
+  const contentTypeHeader =
+    responseHeaderValues(response, "content-type").join(", ") || "text/plain";
+
+  const contentType = normalizeContentType(contentTypeHeader);
 
   const linkHints = parseLinkHeaderForAgentHints(
     responseHeader(response, "link") ?? "",
@@ -209,18 +330,21 @@ async function documentFromResponse(
   );
 
   const allowed =
-    contentType.startsWith("text/") ||
-    [
-      "application/json",
-      "application/markdown",
-      "application/x-markdown",
-      "application/xml",
-      "application/xhtml+xml",
-    ].includes(contentType);
+    contentType !== undefined &&
+    (contentType.startsWith("text/") ||
+      [
+        "application/json",
+        "application/markdown",
+        "application/x-markdown",
+        "application/xml",
+        "application/xhtml+xml",
+      ].includes(contentType));
 
   if (!allowed) {
     response.destroy();
-    throw new Error(`web_fetch does not support ${contentType || "this content type"}.`);
+    throw new Error(
+      `web_fetch does not support ${contentType ?? (contentTypeHeader || "this content type")}.`,
+    );
   }
 
   const raw = decodeResponse(
@@ -249,7 +373,19 @@ async function documentFromResponse(
     } catch {
       markdown = raw;
     }
-  } else markdown = raw.trim();
+  } else {
+    markdown = raw;
+
+    const markdownContent = /\bmarkdown\b|\.md(?:$|[?#])/i.test(
+      contentTypeHeader + target.url.pathname,
+    );
+
+    if (contentType.startsWith("text/") || markdownContent) {
+      fragmentOffsets = lineFragmentOffsets(markdown);
+
+      if (markdownContent) Object.assign(fragmentOffsets, markdownFragmentOffsets(markdown));
+    }
+  }
 
   const isHtml = contentType === "text/html" || contentType === "application/xhtml+xml";
   const extractionDiagnostics = isHtml ? diagnoseExtraction(raw, markdown) : undefined;

package/src/network-transport.ts

@@ -188,6 +188,25 @@ export function responseHeader(response: IncomingMessage, name: string): string
   return Array.isArray(value) ? value[0] : value;
 }
 
+export function responseHeaderValues(response: IncomingMessage, name: string): string[] {
+  const values: string[] = [];
+  const lowerName = name.toLowerCase();
+
+  const rawHeaders = response.rawHeaders ?? [];
+
+  for (let index = 0; index + 1 < rawHeaders.length; index += 2) {
+    if (rawHeaders[index].toLowerCase() === lowerName) {
+      values.push(rawHeaders[index + 1]);
+    }
+  }
+
+  if (values.length > 0) return values;
+
+  const value = response.headers[name];
+
+  return value === undefined ? [] : Array.isArray(value) ? value : [value];
+}
+
 /**
  * Read and buffer the complete response body.
  *

package/src/service.ts

@@ -568,8 +568,25 @@ function urlWithoutFragment(rawUrl: string): string {
 interface ResolvedFragment {
   fragment?: string;
   offset?: number;
+  endOffset?: number;
 }
 
+/** Converts a one-based source line and column into a Markdown character offset. */
+function sourceOffset(markdown: string, lineNumber: number, columnNumber = 1): number | undefined {
+  if (lineNumber < 1 || columnNumber < 1) return undefined;
+
+  const lines = markdown.split("\n");
+
+  if (lineNumber > lines.length) return undefined;
+
+  let offset = 0;
+
+  for (let index = 0; index < lineNumber - 1; index += 1) offset += lines[index].length + 1;
+
+  return offset + Math.min(columnNumber - 1, lines[lineNumber - 1].length);
+}
+
+/** Resolves heading and source-location URL fragments against a fetched document. */
 function resolveFragmentOffset(document: CompleteDocument, rawUrl: string): ResolvedFragment {
   let fragment: string | undefined;
 
@@ -582,6 +599,36 @@ function resolveFragmentOffset(document: CompleteDocument, rawUrl: string): Reso
   }
 
   if (!fragment) return {};
+
+  const lineAnchor = /^L(\d+)(?:C(\d+))?(?:-L?(\d+)(?:C(\d+))?)?$/i.exec(fragment);
+
+  if (lineAnchor) {
+    const [, firstLine, firstColumn, lastLine, lastColumn] = lineAnchor;
+    const offset = sourceOffset(document.markdown, Number(firstLine), Number(firstColumn ?? 1));
+
+    if (offset === undefined) return { fragment };
+
+    if (lastLine) {
+      const endLine = Number(lastLine);
+      const endColumn = lastColumn === undefined ? undefined : Number(lastColumn);
+      const lines = document.markdown.split("\n");
+      const line = lines[endLine - 1];
+
+      if (line === undefined) return { fragment };
+
+      const endOffset =
+        endColumn === undefined
+          ? sourceOffset(document.markdown, endLine, line.length + 1)
+          : sourceOffset(document.markdown, endLine, endColumn + 1);
+
+      return endOffset !== undefined && endOffset >= offset
+        ? { fragment, offset, endOffset }
+        : { fragment };
+    }
+
+    return { fragment, offset };
+  }
+
   const offsets = document.fragmentOffsets ?? {};
 
   return { fragment, offset: offsets[fragment] ?? offsets[fragment.toLowerCase()] };
@@ -688,19 +735,31 @@ export async function executeWebFetch(
   const fragmentMatched = resolvedFragment.fragment !== undefined && fragmentOffset !== undefined;
   const startingOffset = explicitOffset ?? fragmentOffset ?? FETCH_DEFAULT_OFFSET;
 
+  const hasFragmentRange =
+    explicitOffset === undefined &&
+    fragmentOffset !== undefined &&
+    resolvedFragment.endOffset !== undefined;
+
   const fragmentDocument =
     explicitOffset === undefined && fragmentOffset !== undefined
-      ? { ...document, markdown: document.markdown.slice(fragmentOffset) }
+      ? {
+          ...document,
+          markdown: hasFragmentRange
+            ? document.markdown.slice(fragmentOffset, resolvedFragment.endOffset)
+            : document.markdown.slice(fragmentOffset),
+        }
       : document;
 
   const focused =
     params.query === undefined ? undefined : focusMarkdown(fragmentDocument.markdown, params.query);
 
-  const outputDocument = focused ? { ...fragmentDocument, markdown: focused.markdown } : document;
+  const outputDocument = focused
+    ? { ...fragmentDocument, markdown: focused.markdown }
+    : fragmentDocument;
 
   const resultOffset = focused
     ? (explicitOffset ?? 0)
-    : (explicitOffset ?? fragmentOffset ?? startingOffset);
+    : (explicitOffset ?? (fragmentMatched ? 0 : startingOffset));
 
   const result = sliceCompleteDocument(outputDocument, resultOffset, maxCharacters);