@mammothb/pi-webfetch 0.1.0 → 0.2.0

package/index.ts

@@ -1,35 +1,6 @@
 import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
-import { Type } from "typebox";
+import { createWebfetchTool } from "./src/webfetch.js";
 
 export default function (pi: ExtensionAPI) {
-  pi.registerTool({
-    name: "webfetch",
-    label: "Web Fetch",
-    description:
-      "Fetches content from a specified URL and converts it to the requested format (markdown by default).",
-    parameters: Type.Object({
-      url: Type.String({ description: "The URL to fetch content from" }),
-      format: Type.Optional(
-        Type.String({
-          description:
-            "The format to return the content in - text, markdown, or html (default: 'markdown')",
-        }),
-      ),
-    }),
-    async execute(_toolCallId, params, _signal, _onUpdate, _ctx) {
-      const url = params.url;
-      const format = params.format ?? "markdown";
-
-      // TODO: implement actual fetch + conversion
-      return {
-        content: [
-          {
-            type: "text",
-            text: `Fetched ${url} as ${format} (placeholder).`,
-          },
-        ],
-        details: { url, format },
-      };
-    },
-  });
+  pi.registerTool(createWebfetchTool());
 }

package/package.json

@@ -1,19 +1,38 @@
 {
   "name": "@mammothb/pi-webfetch",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "A pi extension that adds a webfetch tool for fetching and converting web content",
   "keywords": [
     "pi-package"
   ],
   "license": "MIT",
+  "files": [
+    "index.ts",
+    "src"
+  ],
   "pi": {
     "extensions": [
       "./index.ts"
     ]
   },
+  "dependencies": {
+    "htmlparser2": "12.0.0",
+    "turndown": "7.2.4"
+  },
+  "scripts": {
+    "test": "vitest run",
+    "test:coverage": "vitest run --coverage"
+  },
+  "devDependencies": {
+    "@types/turndown": "5.0.6",
+    "@vitest/coverage-v8": "4.1.7",
+    "typescript": "6.0.3",
+    "vitest": "4.1.7"
+  },
   "peerDependencies": {
-    "@earendil-works/pi-coding-agent": "*",
     "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*",
     "typebox": "*"
   }
 }

package/src/lib/headers.ts

@@ -0,0 +1,28 @@
+import type { Format, Header } from "./types";
+
+export function buildHeaders(format: Format): Header {
+  let acceptHeader = "*/*";
+  switch (format) {
+    case "markdown":
+      acceptHeader =
+        "text/markdown;q=1.0, text/x-markdown;q=0.9, text/plain;q=0.8, text/html;q=0.7, */*;q=0.1";
+      break;
+    case "text":
+      acceptHeader =
+        "text/plain;q=1.0, text/markdown;q=0.9, text/html;q=0.8, */*;q=0.1";
+      break;
+    case "html":
+      acceptHeader =
+        "text/html;q=1.0, application/xhtml+xml;q=0.9, text/plain;q=0.8, text/markdown;q=0.7, */*;q=0.1";
+      break;
+    default:
+      acceptHeader =
+        "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8";
+  }
+  return {
+    "User-Agent":
+      "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/143.0.0.0 Safari/537.36",
+    Accept: acceptHeader,
+    "Accept-Language": "en-US,en;q=0.9",
+  };
+}

package/src/lib/processors.ts

@@ -0,0 +1,57 @@
+import { Parser } from "htmlparser2";
+import TurndownService from "turndown";
+
+export function toMarkdown(contentType: string, html: string): string {
+  if (!contentType.includes("text/html")) {
+    return html;
+  }
+  const turndownService = new TurndownService({
+    headingStyle: "atx",
+    hr: "---",
+    bulletListMarker: "-",
+    codeBlockStyle: "fenced",
+    emDelimiter: "*",
+  });
+  turndownService.remove(["link", "meta", "script", "style"]);
+  return turndownService.turndown(html);
+}
+
+export function toText(contentType: string, html: string): string {
+  if (!contentType.includes("text/html")) {
+    return html;
+  }
+
+  const tagsToSkip = [
+    "script",
+    "style",
+    "noscript",
+    "iframe",
+    "object",
+    "embed",
+  ];
+  let text = "";
+  let skipDepth = 0;
+
+  const parser = new Parser({
+    onopentag(name, _attribs, _isImplied) {
+      if (skipDepth > 0 || tagsToSkip.includes(name)) {
+        skipDepth++;
+      }
+    },
+    ontext(data) {
+      if (skipDepth === 0) {
+        text += data;
+      }
+    },
+    onclosetag(_name, _isImplied) {
+      if (skipDepth > 0) {
+        skipDepth--;
+      }
+    },
+  });
+
+  parser.write(html);
+  parser.end();
+
+  return text.trim();
+}

package/src/lib/types.ts

@@ -0,0 +1,13 @@
+import { StringEnum } from "@earendil-works/pi-ai";
+import type { Static } from "typebox";
+
+export const FormatSchema = StringEnum(["text", "markdown", "html"] as const, {
+  description:
+    "The format to return the content in - text, markdown, or html (default: 'markdown')",
+});
+export type Format = Static<typeof FormatSchema>;
+
+export type Header = Record<
+  "User-Agent" | "Accept" | "Accept-Language",
+  string
+>;

package/src/webfetch.ts

@@ -0,0 +1,389 @@
+import type { ImageContent, TextContent } from "@earendil-works/pi-ai";
+import {
+  getMarkdownTheme,
+  keyText,
+  type Theme,
+  type ToolDefinition,
+} from "@earendil-works/pi-coding-agent";
+import { Container, Markdown, Spacer, Text } from "@earendil-works/pi-tui";
+import Type from "typebox";
+import { buildHeaders } from "./lib/headers.js";
+import { toMarkdown, toText } from "./lib/processors.js";
+import { FormatSchema, type Header } from "./lib/types.js";
+
+const COLLAPSED_PREVIEW_LINES = 7;
+const DEFAULT_TIMEOUT = 30; // 30 seconds
+const MAX_TIMEOUT = 120; // 2 minutes
+const MAX_RESPONSE_SIZE = 5 * 1024 * 1024; // 5 MB
+const USER_AGENT = "opencode";
+
+interface WebfetchDetails {
+  url: string;
+  contentType: string;
+  format: string;
+  displayTitle: string;
+  size?: number;
+  isImage?: boolean;
+  imageDataUrl?: string;
+  error?: boolean;
+  errorSummary?: string;
+}
+
+const Parameters = Type.Object({
+  url: Type.String({
+    description: "The URL to fetch content from",
+    pattern: "^https?://.*",
+  }),
+  format: Type.Optional(FormatSchema),
+  timeout: Type.Optional(
+    Type.Number({
+      description: "Optional timeout in seconds (max 120)",
+      exclusiveMinimum: 0,
+      maximum: 120,
+    }),
+  ),
+});
+
+async function fetchWithRetry(
+  url: string,
+  headers: Header,
+  signal: AbortSignal | undefined,
+  timeoutMs: number,
+): Promise<{ body: ArrayBuffer; contentType: string }> {
+  // Set up timeout
+  const controller = new AbortController();
+  const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
+  // Forward external signal
+  const onAbort = () => controller.abort();
+  if (signal) {
+    if (signal.aborted) {
+      throw new Error("Request aborted");
+    }
+    signal.addEventListener("abort", onAbort, { once: true });
+  }
+
+  try {
+    const doFetch = async (userAgent: string) => {
+      return await fetch(url, {
+        method: "GET",
+        headers: { ...headers, "User-Agent": userAgent },
+        signal: controller.signal,
+        redirect: "follow",
+      });
+    };
+
+    let response = await doFetch(headers["User-Agent"]);
+    // Retry with honest UA if blocked by Cloudflare bot detection
+    if (isBlockedByCloudflare(response)) {
+      response = await doFetch(USER_AGENT);
+    }
+
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+    }
+
+    const contentLength = response.headers.get("content-length");
+    if (contentLength && parseInt(contentLength, 10) > MAX_RESPONSE_SIZE) {
+      throw new Error(
+        `Response too large (exceeds ${formatSize(MAX_RESPONSE_SIZE)} limit)`,
+      );
+    }
+
+    const arrayBuffer = await response.arrayBuffer();
+    if (arrayBuffer.byteLength > MAX_RESPONSE_SIZE) {
+      throw new Error(
+        `Response too large (exceeds ${formatSize(MAX_RESPONSE_SIZE)} limit)`,
+      );
+    }
+
+    return {
+      body: arrayBuffer,
+      contentType: response.headers.get("content-type") || "text/html",
+    };
+  } catch (error) {
+    if (controller.signal.aborted && !signal?.aborted) {
+      throw new Error("Request timed out");
+    }
+    throw error;
+  } finally {
+    clearTimeout(timeoutId);
+    if (signal) {
+      signal.removeEventListener("abort", onAbort);
+    }
+  }
+}
+
+function formatTitle(details: WebfetchDetails): string {
+  return details.displayTitle ?? details.url ?? "Unknown URL";
+}
+
+function formatSize(bytes: number | undefined): string {
+  if (bytes === undefined) {
+    return "unknown size";
+  }
+  if (bytes < 1024) return `${bytes} B`;
+  if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
+  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
+}
+
+function isBlockedByCloudflare(response: Response): boolean {
+  return (
+    response.status === 403 &&
+    response.headers.get("cf-mitigated") === "challenge"
+  );
+}
+
+function isImageAttachment(mime: string): boolean {
+  return (
+    mime.startsWith("image/") &&
+    mime !== "image/svg+xml" &&
+    mime !== "image/vnd.fastbidsheet"
+  );
+}
+
+function isTextContent(c: TextContent | ImageContent): c is TextContent {
+  return c.type === "text";
+}
+
+function renderWebfetchResult(
+  details: WebfetchDetails,
+  textContent: string,
+  expanded: boolean,
+  theme: Theme,
+): Container {
+  const container = new Container();
+
+  const title = formatTitle(details);
+  const format = details.format ? ` [${details.format}]` : "";
+  container.addChild(
+    new Text(
+      theme.fg("syntaxKeyword", "url: ") +
+        theme.fg("syntaxString", title + format),
+    ),
+  );
+
+  if (details.size !== undefined) {
+    container.addChild(
+      new Text(
+        theme.fg("syntaxKeyword", "size: ") +
+          theme.fg("syntaxString", formatSize(details.size)),
+      ),
+    );
+  }
+
+  if (!textContent) {
+    return container;
+  }
+
+  container.addChild(new Spacer(1));
+
+  if (expanded) {
+    const format = details.format ?? "markdown";
+    if (format === "markdown") {
+      container.addChild(new Markdown(textContent, 0, 0, getMarkdownTheme()));
+    } else {
+      const highlighted = `\`\`\`${format}\n${textContent}\n\`\`\``;
+      container.addChild(new Markdown(highlighted, 0, 0, getMarkdownTheme()));
+    }
+  } else {
+    const lines = textContent
+      .split("\n")
+      .filter(
+        (line, index, arr) =>
+          line.length > 0 || index === 0 || index < arr.length - 1,
+      );
+    const previewLines = lines.slice(0, COLLAPSED_PREVIEW_LINES);
+    const remaining = Math.max(0, lines.length - previewLines.length);
+
+    const preview = previewLines.join("\n");
+    const format = details.format ?? "markdown";
+    if (format === "markdown") {
+      container.addChild(new Markdown(preview, 0, 0, getMarkdownTheme()));
+    } else {
+      container.addChild(new Text(preview));
+    }
+
+    if (remaining > 0) {
+      container.addChild(new Spacer(1));
+      const expandKey = keyText("app.tools.expand") || "Ctrl+O";
+      container.addChild(
+        new Text(
+          theme.fg("muted", `... (${remaining} more lines, `) +
+            theme.fg("dim", expandKey) +
+            theme.fg("muted", " to expand)"),
+        ),
+      );
+    }
+  }
+  return container;
+}
+
+export function createWebfetchTool(): ToolDefinition<
+  typeof Parameters,
+  WebfetchDetails
+> {
+  return {
+    name: "webfetch",
+    label: "Web Fetch",
+    description: `- Fetches content from a specified URL
+- Takes a URL and optional format as input
+- Fetches the URL content, converts to requested format (markdown by default)
+- Returns the content in the specified format
+- Use this tool when you need to retrieve and analyze web content
+
+Usage notes:
+  - IMPORTANT: if another tool is present that offers better web fetching capabilities, is more targeted to the task, or has fewer restrictions, prefer using that tool instead of this one.
+  - The URL must be a fully-formed valid URL
+  - HTTP URLs will be automatically upgraded to HTTPS
+  - Format options: "markdown" (default), "text", or "html"
+  - This tool is read-only and does not modify any files
+  - Results may be summarized if the content is very large`,
+    promptSnippet: "Fetch content from a URL",
+    parameters: Parameters,
+    async execute(_toolCallId, params, signal, _onUpdate, _ctx) {
+      const url = params.url;
+      const format = params.format ?? "markdown";
+
+      if (signal?.aborted) {
+        return {
+          content: [{ type: "text", text: "Cancelled" }],
+          details: {
+            url,
+            contentType: "",
+            format,
+            displayTitle: url,
+            error: true,
+            errorSummary: "Request cancelled",
+          },
+        };
+      }
+
+      const timeoutMs =
+        Math.min(params.timeout ?? DEFAULT_TIMEOUT, MAX_TIMEOUT) * 1000;
+
+      const headers = buildHeaders(format);
+      const { body, contentType } = await fetchWithRetry(
+        url,
+        headers,
+        signal,
+        timeoutMs,
+      );
+
+      const mime = contentType.split(";")[0]?.trim().toLowerCase() || "";
+      const displayTitle = `${url} (${contentType})`;
+
+      // Handle image
+      if (isImageAttachment(mime)) {
+        const base64Content = Buffer.from(body).toString("base64");
+        return {
+          content: [
+            {
+              type: "text",
+              text: `Image fetched successfully: ${url}\nMIME type: ${mime}\nSize: ${body.byteLength} bytes`,
+            },
+            {
+              type: "image",
+              data: base64Content,
+              mimeType: mime,
+            },
+          ],
+          details: {
+            url,
+            contentType: mime,
+            format,
+            displayTitle,
+            size: body.byteLength,
+            isImage: true,
+            imageDataUrl: `data:${mime};base64,${base64Content}`,
+          },
+        };
+      }
+
+      // Handle text
+      const text = new TextDecoder().decode(body);
+      switch (format) {
+        case "markdown": {
+          return {
+            content: [{ type: "text", text: toMarkdown(contentType, text) }],
+            details: {
+              url,
+              contentType,
+              format: "markdown",
+              displayTitle,
+              size: body.byteLength,
+            },
+          };
+        }
+        case "text": {
+          return {
+            content: [{ type: "text", text: toText(contentType, text) }],
+            details: {
+              url,
+              contentType,
+              format: "text",
+              displayTitle,
+              size: body.byteLength,
+            },
+          };
+        }
+        case "html": {
+          return {
+            content: [{ type: "text", text: text }],
+            details: {
+              url,
+              contentType,
+              format: "html",
+              displayTitle,
+              size: body.byteLength,
+            },
+          };
+        }
+        default: {
+          return {
+            content: [{ type: "text", text: text }],
+            details: {
+              url,
+              contentType,
+              format,
+              displayTitle,
+              size: body.byteLength,
+            },
+          };
+        }
+      }
+    },
+    renderResult(result, options, theme, _context) {
+      const details = result.details;
+
+      if (options.isPartial && !details.url) {
+        return new Text(theme.fg("muted", "Fetching..."));
+      }
+
+      if (details.isImage) {
+        return new Text(
+          theme.fg(
+            "muted",
+            `Image: ${formatTitle(details)} (${formatSize(details.size)})`,
+          ),
+        );
+      }
+
+      if (details.error) {
+        return new Text(
+          theme.fg("error", details.errorSummary ?? "Request failed"),
+        );
+      }
+
+      const textContent = result.content
+        .filter(isTextContent)
+        .map((c) => c.text)
+        .join("\n");
+      return renderWebfetchResult(
+        details,
+        textContent,
+        options.expanded,
+        theme,
+      );
+    },
+  };
+}

package/biome.json

@@ -1,35 +0,0 @@
-{
-  "$schema": "https://biomejs.dev/schemas/2.4.14/schema.json",
-  "formatter": {
-    "enabled": true,
-    "indentStyle": "space"
-  },
-  "linter": {
-    "enabled": true,
-    "rules": {
-      "recommended": true
-    }
-  },
-  "files": {
-    "includes": [
-      "**",
-      "!dist",
-      "!coverage",
-      "!packages/*/dist",
-      "!packages/*/coverage",
-      "!node_modules",
-      "!forks/wreq-js"
-    ]
-  },
-  "javascript": {
-    "formatter": {
-      "quoteStyle": "double",
-      "semicolons": "always"
-    }
-  },
-  "json": {
-    "formatter": {
-      "indentStyle": "space"
-    }
-  }
-}