@zeldrisho/pi-web-fetch 0.8.2 → 0.9.1

package/CHANGELOG.md

@@ -7,6 +7,22 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [0.9.1] - 2026-09-22
+
+### Changed
+
+- Upgrade Defuddle to `^0.19.4` for improved metadata and Markdown extraction, HTML sanitization, SVG safety, and support for declarative shadow DOM and MathML content
+
+### Fixed
+
+- Extract HTML correctly when servers send duplicated equivalent `Content-Type` media types, and reject conflicting media-type declarations instead of returning raw HTML
+
+## [0.9.0] - 2026-09-17
+
+### Added
+
+- Support URL fragments for starting fetched content at matching headings or anchors; explicit offsets take precedence and fragment variants share cached page acquisitions
+
 ## [0.8.2] - 2026-09-12
 
 ### Fixed
@@ -142,7 +158,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Add bounded public page fetching ([b979496](https://github.com/zeldrisho/pi-packages/commit/b979496b32de1cead172ba570307e4a4a7b3421d))
 
-[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.8.2...HEAD
+[Unreleased]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.9.1...HEAD
+[0.9.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.9.0...pi-web-fetch-v0.9.1
+[0.9.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.8.2...pi-web-fetch-v0.9.0
 [0.8.2]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.8.1...pi-web-fetch-v0.8.2
 [0.8.1]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.8.0...pi-web-fetch-v0.8.1
 [0.8.0]: https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.7.1...pi-web-fetch-v0.8.0

package/README.md

@@ -14,7 +14,7 @@ pi install -l npm:@zeldrisho/pi-web-fetch
 
 `web_fetch` accepts public HTTP and HTTPS URLs and supports HTML, Markdown, plain text, JSON, and XML. HTML is converted with Defuddle, with a basic text extractor as fallback. URLs with credentials, local/private/reserved targets, unsafe redirects, responses over 5 MiB, or unsupported content types are rejected.
 
-`maxCharacters` bounds returned Markdown (default 6,000); it does not change the raw download limit. `offset` is a character offset into extracted content. For long pages, pass `query` to get a deterministic, source-ordered focused view selected locally after the complete page is fetched and cached. `details.focus` reports matched, total, and omitted sections plus character counts. Omit `query` to read the complete document. Use a returned `nextOffset` for continuation.
+`maxCharacters` bounds returned Markdown (default 6,000); it does not change the raw download limit. `offset` is a character offset into extracted content. A URL fragment such as `https://github.com/shadcn-ui/lint#get-started` starts at the matching heading or anchor (the complete page is still fetched and cached); an explicit `offset` takes precedence. For long pages, pass `query` to get a deterministic, source-ordered focused view selected locally after the complete page is fetched and cached. `details.focus` reports matched, total, and omitted sections plus character counts. Omit `query` to read the complete document. Use a returned `nextOffset` for continuation.
 
 Results include `details.truncation`: complete output has `{ truncated: false, strategy: "none" }`; continuation output has `strategy: "continuation"` and a valid `nextOffset`. Compatibility fields `details.truncated` and `details.nextOffset` remain available. Pi's interactive preview is collapsed by default (`Ctrl+O` expands it); tool output is also capped at 2,000 lines / 50 KiB.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zeldrisho/pi-web-fetch",
-  "version": "0.8.2",
+  "version": "0.9.1",
   "description": "Pi extension for secure, bounded public web page fetching and Markdown extraction",
   "keywords": [
     "pi",
@@ -22,25 +22,23 @@
   },
   "files": [
     "src",
-    "README.md",
-    "CHANGELOG.md",
-    "LICENSE"
+    "CHANGELOG.md"
   ],
   "type": "module",
   "publishConfig": {
     "access": "public"
   },
   "dependencies": {
-    "defuddle": "^0.19.3",
+    "defuddle": "^0.19.4",
     "linkedom": "^0.18.13"
   },
   "devDependencies": {
-    "@earendil-works/pi-coding-agent": "^0.84.2",
-    "@earendil-works/pi-tui": "^0.84.2",
-    "typebox": "^1.3.16",
+    "@earendil-works/pi-coding-agent": "^0.85.1",
+    "@earendil-works/pi-tui": "^0.85.1",
+    "typebox": "^1.3.30",
     "typescript": "^7.0.2",
-    "vite": "npm:@voidzero-dev/vite-plus-core@0.3.0",
-    "vite-plus": "0.3.0"
+    "vite": "npm:@voidzero-dev/vite-plus-core@0.3.3",
+    "vite-plus": "0.3.3"
   },
   "peerDependencies": {
     "@earendil-works/pi-coding-agent": "*",

package/src/abort.ts

@@ -8,12 +8,14 @@
 export function awaitWithAbort<T>(operation: Promise<T>, signal: AbortSignal): Promise<T> {
   return new Promise((resolve, reject) => {
     let settled = false;
+
     const finish = (callback: () => void): void => {
       if (settled) return;
       settled = true;
       signal.removeEventListener("abort", abort);
       callback();
     };
+
     const abort = (): void => {
       const error = new Error("Operation aborted.");
       error.name = "AbortError";
@@ -24,6 +26,7 @@ export function awaitWithAbort<T>(operation: Promise<T>, signal: AbortSignal): P
       (value) => finish(() => resolve(value)),
       (error) => finish(() => reject(error)),
     );
+
     if (signal.aborted) abort();
     else signal.addEventListener("abort", abort, { once: true });
   });

package/src/cache.ts

@@ -48,9 +48,11 @@ export function stableKeyHash(key: string): string {
 export function resolveCachePath(directory: string, keyPath: string): string {
   const base = resolve(directory);
   const full = resolve(base, keyPath);
+
   if (full !== base && !full.startsWith(base + sep)) {
     throw new Error(`Refusing to write cache entry outside ${base}: ${keyPath}`);
   }
+
   return full;
 }
 
@@ -63,10 +65,12 @@ export function resolveCachePath(directory: string, keyPath: string): string {
 function encodeExpiresAt(expiresAt: number): Uint8Array {
   const out = new Uint8Array(8);
   let value = BigInt(Math.round(expiresAt));
+
   for (let index = 7; index >= 0; index -= 1) {
     out[index] = Number(value & 0xffn);
     value >>= 8n;
   }
+
   return out;
 }
 
@@ -78,7 +82,9 @@ function encodeExpiresAt(expiresAt: number): Uint8Array {
  */
 function decodeExpiresAt(bytes: Uint8Array): number {
   let value = 0n;
+
   for (let index = 0; index < 8; index += 1) value = (value << 8n) | BigInt(bytes[index]);
+
   return Number(value);
 }
 
@@ -123,19 +129,26 @@ export class ExpiringLruCache<K, V> {
    */
   get(key: K): V | undefined {
     const entry = this.#entries.get(key);
+
     if (entry) {
       if (entry.expiresAt <= this.now()) {
         this.#delete(key);
+
         return undefined;
       }
+
       this.#entries.delete(key);
       this.#entries.set(key, entry);
+
       return entry.value;
     }
+
     if (this.persistence) {
       const loaded = this.#loadFromDisk(key);
+
       if (loaded !== undefined) return loaded.value;
     }
+
     return undefined;
   }
 
@@ -150,13 +163,16 @@ export class ExpiringLruCache<K, V> {
   set(key: K, value: V, expiresAt: number): boolean {
     this.#delete(key);
     const size = this.sizeOf(value);
+
     if (size > this.maxBytes) return false;
 
     this.#entries.set(key, { expiresAt, size, value });
     this.#byteSize += size;
     this.#evict();
     const stored = this.#entries.has(key);
+
     if (stored && this.persistence) this.#writeToDisk(key, value, expiresAt);
+
     return stored;
   }
 
@@ -164,6 +180,7 @@ export class ExpiringLruCache<K, V> {
     let bytes: Uint8Array;
     const path = resolveCachePath(this.persistence!.directory, this.persistence!.keyToPath(key));
     let fileSize: number;
+
     try {
       // Serialized metadata can exceed the value budget, but never allow an
       // unbounded cache file to be read into memory.
@@ -171,37 +188,50 @@ export class ExpiringLruCache<K, V> {
     } catch {
       return undefined;
     }
+
     if (fileSize < 8 || fileSize > this.maxBytes * 2 + 65_536) {
       this.#removeFromDisk(key);
+
       return undefined;
     }
+
     try {
       bytes = readFileSync(path);
     } catch {
       return undefined;
     }
+
     let entry: ExpiringCacheEntry<V>;
+
     try {
       const expiresAt = decodeExpiresAt(bytes);
       const value = this.persistence!.deserialize(bytes.subarray(8));
+
       if (this.persistence!.validate && !this.persistence!.validate(value)) {
         throw new Error("invalid cache entry");
       }
+
       const size = this.sizeOf(value);
+
       if (!Number.isFinite(size) || size < 0 || size > this.maxBytes)
         throw new Error("oversized cache entry");
       entry = { expiresAt, size, value };
     } catch {
       this.#removeFromDisk(key);
+
       return undefined;
     }
+
     if (entry.expiresAt <= this.now()) {
       this.#removeFromDisk(key);
+
       return undefined;
     }
+
     this.#entries.set(key, entry);
     this.#byteSize += entry.size;
     this.#evict();
+
     return this.#entries.get(key);
   }
 
@@ -211,10 +241,12 @@ export class ExpiringLruCache<K, V> {
       mkdirSync(directory, { recursive: true, mode: 0o700 });
       chmodSync(directory, 0o700);
       const payload = this.persistence!.serialize(value);
+
       if (payload.byteLength > this.maxBytes * 2 + 65_536)
         throw new Error("oversized cache payload");
       const path = resolveCachePath(directory, this.persistence!.keyToPath(key));
       const temp = `${path}.${process.pid}.${Math.random().toString(36).slice(2)}.tmp`;
+
       try {
         writeFileSync(temp, Buffer.concat([encodeExpiresAt(expiresAt), payload]), { mode: 0o600 });
         chmodSync(temp, 0o600);
@@ -241,27 +273,35 @@ export class ExpiringLruCache<K, V> {
 
   #maintainDisk(): void {
     const persistence = this.persistence!;
+
     try {
       const files = readdirSync(persistence.directory);
       const candidates: Array<{ path: string; size: number; mtime: number }> = [];
+
       for (const file of files) {
         const path = resolveCachePath(persistence.directory, file);
+
         if (file.endsWith(".tmp")) {
           try {
             unlinkSync(path);
           } catch {
             // Best effort cleanup.
           }
+
           continue;
         }
+
         try {
           const stat = statSync(path);
+
           if (!stat.isFile() || stat.size < 8 || stat.size > this.maxBytes * 2 + 65_536) {
             unlinkSync(path);
             continue;
           }
+
           const descriptor = openSync(path, "r");
           const header = new Uint8Array(8);
+
           try {
             if (
               readSync(descriptor, header, 0, 8, 0) !== 8 ||
@@ -273,17 +313,22 @@ export class ExpiringLruCache<K, V> {
           } finally {
             closeSync(descriptor);
           }
+
           candidates.push({ path, size: stat.size, mtime: stat.mtimeMs });
         } catch {
           // Ignore races and inaccessible files in this best-effort sweep.
         }
       }
+
       candidates.sort((a, b) => a.mtime - b.mtime);
       let total = candidates.reduce((sum, candidate) => sum + candidate.size, 0);
+
       while (candidates.length > this.maxEntries || total > this.maxBytes * 2 + 65_536) {
         const oldest = candidates.shift();
+
         if (!oldest) break;
         total -= oldest.size;
+
         try {
           unlinkSync(oldest.path);
         } catch {
@@ -298,6 +343,7 @@ export class ExpiringLruCache<K, V> {
   #evict(): void {
     while (this.#entries.size > this.maxEntries || this.#byteSize > this.maxBytes) {
       const oldest = this.#entries.keys().next().value;
+
       if (oldest === undefined) break;
       this.#delete(oldest);
     }
@@ -305,12 +351,16 @@ export class ExpiringLruCache<K, V> {
 
   #delete(key: K): void {
     const entry = this.#entries.get(key);
+
     if (!entry) {
       if (this.persistence) this.#removeFromDisk(key);
+
       return;
     }
+
     this.#entries.delete(key);
     this.#byteSize -= entry.size;
+
     if (this.persistence) this.#removeFromDisk(key);
   }
 }

package/src/content.ts

@@ -2,7 +2,9 @@ import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "@earendil-works/pi-coding-
 import type { ExtractedLinks, ExtractionDiagnostics } from "./evidence";
 
 const CONTENT_LINE_BUDGET = Math.max(1, DEFAULT_MAX_LINES - 10);
+
 const CONTENT_BYTE_BUDGET = Math.max(1_024, DEFAULT_MAX_BYTES - 2_048);
+
 const encoder = new TextEncoder();
 
 /**
@@ -14,6 +16,8 @@ const encoder = new TextEncoder();
 export interface CompleteDocument {
   url: string;
   contentType: string;
+  /** Character offsets for document fragments, derived from HTML anchors/headings. */
+  fragmentOffsets?: Record<string, number>;
   markdown: string;
   title?: string;
   extractor: "defuddle" | "basic" | "raw";
@@ -52,27 +56,37 @@ export interface FetchResult extends CompleteDocument {
   truncated: boolean;
 }
 
+/** Returns the longest prefix whose UTF-8 representation fits within the byte limit. */
 function sliceByByteLength(value: string, maxBytes: number): string {
   if (encoder.encode(value).byteLength <= maxBytes) return value;
   let low = 0;
   let high = value.length;
+
   while (low < high) {
     const middle = Math.ceil((low + high) / 2);
+
     if (encoder.encode(value.slice(0, middle)).byteLength <= maxBytes) low = middle;
     else high = middle - 1;
   }
+
   if (low > 0 && /[\uD800-\uDBFF]/.test(value[low - 1])) low -= 1;
+
   return value.slice(0, low);
 }
 
+/** Slices content while enforcing the configured character, line, and byte budgets. */
 function boundedContentChunk(value: string, offset: number, maxCharacters: number): string {
   let chunk = value.slice(offset, offset + maxCharacters);
   let newline = -1;
+
   for (let lines = 1; lines < CONTENT_LINE_BUDGET; lines += 1) {
     newline = chunk.indexOf("\n", newline + 1);
+
     if (newline === -1) break;
   }
+
   if (newline !== -1) chunk = chunk.slice(0, newline);
+
   return sliceByByteLength(chunk, CONTENT_BYTE_BUDGET);
 }
 
@@ -97,11 +111,13 @@ export function sliceCompleteDocument(
   let markdown = boundedContentChunk(document.markdown, offset, maxCharacters);
   const end = offset + markdown.length;
   const truncated = end < totalCharacters;
+
   if (truncated) {
     markdown += `\n\n[Content truncated. Continue with offset=${end} to read the next chunk.]`;
   } else if (offset > 0) {
     markdown += "\n\n[End of page content.]";
   }
+
   return {
     ...document,
     markdown,

package/src/evidence.ts

@@ -1,6 +1,7 @@
 import { parseHTML } from "linkedom";
 
 const MAX_LINKS = 16;
+
 const MAX_ANCHOR_CHARACTERS = 120;
 
 /** Bounded, machine-readable reasons that an HTML extraction may be incomplete. */
@@ -28,12 +29,14 @@ export interface ExtractedLinks {
 }
 
 const JAVASCRIPT_MARKERS = [/please\s+enable\s+javascript/i, /javascript\s+(?:is\s+)?required/i];
+
 const BOT_MARKERS = [
   /are\s+you\s+a\s+robot/i,
   /verify\s+you\s+are\s+human/i,
   /checking\s+your\s+browser/i,
   /<title>\s*just\s+a\s+moment/i,
 ];
+
 const CONSENT_MARKERS = [
   /manage\s+(?:your\s+)?consent/i,
   /your\s+(?:privacy\s+)?consent/i,
@@ -78,6 +81,7 @@ export function hasExtractionWarning(diagnostics: ExtractionDiagnostics): boolea
  */
 function normalizedAnchor(value: string): string {
   const text = value.replace(/\s+/g, " ").trim();
+
   return text.length > MAX_ANCHOR_CHARACTERS
     ? `${text.slice(0, MAX_ANCHOR_CHARACTERS - 1)}…`
     : text;
@@ -94,25 +98,31 @@ export function extractDocumentLinks(raw: string, pageUrl: URL): ExtractedLinks
 
   for (const element of document.querySelectorAll<HTMLAnchorElement>("a[href]")) {
     const href = element.getAttribute("href");
+
     if (!href) continue;
     let target: URL;
+
     try {
       target = new URL(href, pageUrl);
     } catch {
       continue;
     }
+
     if (!["http:", "https:"].includes(target.protocol) || target.username || target.password)
       continue;
     target.hash = "";
     const anchorText = normalizedAnchor(element.textContent ?? "");
     const key = `${target.href}\0${anchorText}`;
+
     if (seen.has(key)) continue;
     seen.add(key);
     const isInternal = target.origin === pageUrl.origin;
     const bucket = isInternal ? internal : external;
+
     if (bucket.length < MAX_LINKS) bucket.push({ url: target.href, anchorText });
     else if (isInternal) omittedInternal += 1;
     else omittedExternal += 1;
   }
+
   return { internal, external, omittedInternal, omittedExternal };
 }