@zeldrisho/pi-web-fetch 0.4.0 → 0.5.0

package/CHANGELOG.md

@@ -1,4 +1,16 @@
 # Changelog
+## [0.5.0](https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.4.0...pi-web-fetch-v0.5.0) (2026-07-28)
+
+
+### Bug fixes
+
+- Apply CodeRabbit auto-fixes ([b9149ac](https://github.com/zeldrisho/pi-packages/commit/b9149ac91016bb25b49806af5c99b8486420dd74))
+
+
+### Features
+
+- **web-fetch:** Support larger documentation pages ([39cdb32](https://github.com/zeldrisho/pi-packages/commit/39cdb3213a9eee9e0c905319631ef3228e31c8d8))
+
 
 ## [0.4.0](https://github.com/zeldrisho/pi-packages/compare/pi-web-fetch-v0.3.1...pi-web-fetch-v0.4.0) (2026-07-25)
 

package/README.md

@@ -12,9 +12,9 @@ pi install npm:@zeldrisho/pi-web-fetch
 
 The `web_fetch` tool accepts public HTTP and HTTPS URLs. It supports textual content such as HTML, Markdown, plain text, JSON, and XML. HTML pages are converted to Markdown with Defuddle; a basic text extractor is used as a fallback when Defuddle cannot extract the page.
 
-For safety, the tool blocks URLs containing credentials, local hostnames, private or reserved network targets, unsafe redirects, responses larger than its configured limit, and unsupported content types.
+For safety, the tool blocks URLs containing credentials, local hostnames, private or reserved network targets, unsafe redirects, raw responses larger than 5 MiB, and unsupported content types. The `maxCharacters` parameter controls returned Markdown length; it does not change the raw download limit.
 
-In Pi's interactive UI, fetched content uses Pi's standard collapsed preview; use the configured tool-expansion shortcut (`Ctrl+O` by default) to show all visible tool output. Output sent to the agent remains bounded. When a result is truncated, call the tool again with the returned `nextOffset` as `offset` to continue reading. Fetched and extracted pages are cached in byte-bounded memory for a limited time so continuation requests can reuse the same content. Concurrent requests for the same URL share one fetch; cancelling one caller does not cancel work still needed by another.
+In Pi's interactive UI, fetched content uses Pi's standard collapsed preview; use the configured tool-expansion shortcut (`Ctrl+O` by default) to show all visible tool output. Output sent to the agent remains bounded. The `offset` parameter is a character offset into extracted content, not a byte range into the remote response. When a result is truncated, call the tool again with the returned `nextOffset` as `offset` to continue reading. Fetched and extracted pages are cached in byte-bounded memory for a limited time so continuation requests can reuse the same content. Concurrent requests for the same URL share one fetch; cancelling one caller does not cancel work still needed by another.
 
 Every result includes `details.truncation`. Complete output reports `{ truncated: false, strategy: "none" }`. Truncated output reports `strategy: "continuation"` and a valid `nextOffset`. The existing top-level `details.truncated` and `details.nextOffset` fields remain available.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@zeldrisho/pi-web-fetch",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "Pi extension for secure, bounded public web page fetching and Markdown extraction",
   "keywords": [
     "pi",

package/src/index.ts

@@ -2,7 +2,6 @@ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
 import { Type } from "typebox";
 import { executeWebFetch } from "./service";
-import { FETCH_MAX_BYTES } from "./network";
 import { formatCollapsibleOutput } from "./render";
 
 export { ExpiringLruCache } from "./cache";
@@ -15,6 +14,7 @@ export {
   type WebFetchTruncationDetails,
 } from "./service";
 export {
+  FETCH_MAX_BYTES,
   isPrivateAddress,
   requestPinned,
   validateRemoteUrl,
@@ -22,6 +22,7 @@ export {
 } from "./network";
 
 const FETCH_DEFAULT_MAX_CHARACTERS = 6_000;
+const FETCH_MAX_OFFSET_CHARACTERS = 20_000_000;
 
 export default function (pi: ExtensionAPI) {
   pi.registerTool({
@@ -44,9 +45,9 @@ export default function (pi: ExtensionAPI) {
       offset: Type.Optional(
         Type.Integer({
           minimum: 0,
-          maximum: FETCH_MAX_BYTES,
+          maximum: FETCH_MAX_OFFSET_CHARACTERS,
           description:
-            "Character offset to start reading from (default: 0; use nextOffset to continue)",
+            "Extracted-content character offset to start reading from (default: 0; use nextOffset to continue)",
         }),
       ),
       maxCharacters: Type.Optional(

package/src/network-transport.ts

@@ -4,10 +4,24 @@ import type { LookupFunction } from "node:net";
 import { formatSize } from "@earendil-works/pi-coding-agent";
 import type { ValidatedTarget } from "./network-policy";
 
-export const FETCH_MAX_BYTES = 1_000_000;
+export const FETCH_MAX_BYTES = 5 * 1_024 * 1_024;
 
 const encoder = new TextEncoder();
 
+function responseTooLargeMessage(
+  receivedBytes: number,
+  maxBytes: number,
+  sizeIsExact: boolean,
+): string {
+  const size = sizeIsExact
+    ? `is ${formatSize(receivedBytes)}`
+    : `has reached at least ${formatSize(receivedBytes)}`;
+  return [
+    `web_fetch response ${size}, exceeding the ${formatSize(maxBytes)} raw download limit.`,
+    "maxCharacters only controls returned output.",
+  ].join(" ");
+}
+
 export async function requestPinned(
   target: ValidatedTarget,
   signal: AbortSignal,
@@ -45,8 +59,10 @@ export async function readResponseBytes(
   maxBytes: number,
 ): Promise<Uint8Array> {
   const declared = Number(responseHeader(response, "content-length"));
-  if (Number.isFinite(declared) && declared > maxBytes)
-    throw new Error(`web_fetch response exceeds ${formatSize(maxBytes)}.`);
+  if (Number.isFinite(declared) && declared > maxBytes) {
+    response.destroy();
+    throw new Error(responseTooLargeMessage(declared, maxBytes, true));
+  }
   const chunks: Uint8Array[] = [];
   let total = 0;
   for await (const value of response) {
@@ -54,7 +70,7 @@ export async function readResponseBytes(
     total += chunk.byteLength;
     if (total > maxBytes) {
       response.destroy();
-      throw new Error(`web_fetch response exceeds ${formatSize(maxBytes)}.`);
+      throw new Error(responseTooLargeMessage(total, maxBytes, false));
     }
     chunks.push(chunk);
   }