@zeldrisho/pi-web-fetch 0.9.0 → 0.9.1

package/CHANGELOG.md

@@ -7,6 +7,16 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.9.1] - 2026-09-22
+
+### Changed
+
+- Upgrade Defuddle to `^0.19.4` for improved metadata and Markdown extraction, HTML sanitization, SVG safety, and support for declarative shadow DOM and MathML content
+
+### Fixed
+
+- Extract HTML correctly when servers send duplicated equivalent `Content-Type` media types, and reject conflicting media-type declarations instead of returning raw HTML
+
 ## [0.9.0] - 2026-09-17
 
 ### Added
@@ -148,7 +158,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add bounded public page fetching ([b979496](https://github.com/zeldrisho/pi-packages/commit/b979496b32de1cead172ba570307e4a4a7b3421d))
 
-[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.9.0...HEAD
+[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.9.1...HEAD
+[0.9.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.9.0...pi-web-fetch-v0.9.1
 [0.9.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.8.2...pi-web-fetch-v0.9.0
 [0.8.2]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.8.1...pi-web-fetch-v0.8.2
 [0.8.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.8.0...pi-web-fetch-v0.8.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zeldrisho/pi-web-fetch",
-  "version": "0.9.0",
+  "version": "0.9.1",
   "description": "Pi extension for secure, bounded public web page fetching and Markdown extraction",
   "keywords": [
     "pi",
@@ -29,16 +29,16 @@
     "access": "public"
   },
   "dependencies": {
-    "defuddle": "^0.19.3",
+    "defuddle": "^0.19.4",
     "linkedom": "^0.18.13"
   },
   "devDependencies": {
-    "@earendil-works/pi-coding-agent": "^0.84.2",
-    "@earendil-works/pi-tui": "^0.84.2",
-    "typebox": "^1.3.16",
+    "@earendil-works/pi-coding-agent": "^0.85.1",
+    "@earendil-works/pi-tui": "^0.85.1",
+    "typebox": "^1.3.30",
     "typescript": "^7.0.2",
-    "vite": "npm:@voidzero-dev/vite-plus-core@0.3.2",
-    "vite-plus": "0.3.2"
+    "vite": "npm:@voidzero-dev/vite-plus-core@0.3.3",
+    "vite-plus": "0.3.3"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",

package/src/fetch.ts

@@ -11,6 +11,7 @@ import {
   FETCH_MAX_BYTES,
   readResponseBytes,
   responseHeader,
+  responseHeaderValues,
 } from "./network-transport";
 
 /**
@@ -171,6 +172,43 @@ export interface FetchRemoteDependencies extends RedirectDependencies {
   timeoutMs?: number;
 }
 
+/**
+ * Returns a usable media type from a response header.
+ *
+ * Some servers incorrectly emit an equivalent media type more than once (for example,
+ * `text/html,text/html; charset=utf-8`). Treat that as the declared type, but reject
+ * conflicting media types rather than guessing which representation to parse.
+ */
+function normalizeContentType(header: string): string | undefined {
+  const mediaTypes: string[] = [];
+  let current = "";
+  let quoted = false;
+
+  for (const character of header) {
+    if (character === '"') quoted = !quoted;
+
+    if (character === "," && !quoted) {
+      mediaTypes.push(current);
+      current = "";
+    } else current += character;
+  }
+
+  mediaTypes.push(current);
+  const normalized: string[] = [];
+
+  for (const value of mediaTypes) {
+    const mediaType = value.split(";", 1)[0].trim().toLowerCase();
+
+    if (mediaType) normalized.push(mediaType);
+  }
+
+  if (normalized.length === 0 || normalized.some((value) => value !== normalized[0])) {
+    return undefined;
+  }
+
+  return normalized[0];
+}
+
 /**
  * Converts a successful HTTP response into a complete document.
  *
@@ -200,8 +238,10 @@ async function documentFromResponse(
     throw new Error(`web_fetch returned HTTP ${status}.${authenticationHint}`);
   }
 
-  const contentTypeHeader = responseHeader(response, "content-type") ?? "text/plain";
-  const contentType = contentTypeHeader.split(";", 1)[0].trim().toLowerCase();
+  const contentTypeHeader =
+    responseHeaderValues(response, "content-type").join(", ") || "text/plain";
+
+  const contentType = normalizeContentType(contentTypeHeader);
 
   const linkHints = parseLinkHeaderForAgentHints(
     responseHeader(response, "link") ?? "",
@@ -209,18 +249,21 @@ async function documentFromResponse(
   );
 
   const allowed =
-    contentType.startsWith("text/") ||
-    [
-      "application/json",
-      "application/markdown",
-      "application/x-markdown",
-      "application/xml",
-      "application/xhtml+xml",
-    ].includes(contentType);
+    contentType !== undefined &&
+    (contentType.startsWith("text/") ||
+      [
+        "application/json",
+        "application/markdown",
+        "application/x-markdown",
+        "application/xml",
+        "application/xhtml+xml",
+      ].includes(contentType));
 
   if (!allowed) {
     response.destroy();
-    throw new Error(`web_fetch does not support ${contentType || "this content type"}.`);
+    throw new Error(
+      `web_fetch does not support ${contentType ?? (contentTypeHeader || "this content type")}.`,
+    );
   }
 
   const raw = decodeResponse(

package/src/network-transport.ts

@@ -188,6 +188,25 @@ export function responseHeader(response: IncomingMessage, name: string): string
   return Array.isArray(value) ? value[0] : value;
 }
 
+export function responseHeaderValues(response: IncomingMessage, name: string): string[] {
+  const values: string[] = [];
+  const lowerName = name.toLowerCase();
+
+  const rawHeaders = response.rawHeaders ?? [];
+
+  for (let index = 0; index + 1 < rawHeaders.length; index += 2) {
+    if (rawHeaders[index].toLowerCase() === lowerName) {
+      values.push(rawHeaders[index + 1]);
+    }
+  }
+
+  if (values.length > 0) return values;
+
+  const value = response.headers[name];
+
+  return value === undefined ? [] : Array.isArray(value) ? value : [value];
+}
+
 /**
  * Read and buffer the complete response body.
  *