@aaroncql/pim-agent 0.0.1

package/src/extensions/web-fetch/fetch.test.ts

@@ -0,0 +1,244 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { SpillCache } from "../../shared/SpillCache";
+import {
+  executeFetch,
+  formatOutcome,
+  truncationFooter,
+  validatePublicUrl,
+} from "./fetch";
+import { WEB_FETCH_INLINE_BYTES } from "./schema";
+import type { JinaReaderClient } from "./JinaReaderClient";
+import type { WebViewFetchClient } from "./WebViewFetchClient";
+
+let previousPimHomeDir: string | undefined;
+let testPimHomeDir: string | undefined;
+
+beforeAll(async () => {
+  previousPimHomeDir = process.env.PIM_HOME_DIR;
+  testPimHomeDir = await mkdtemp(join(tmpdir(), "pim-fetch-home-"));
+  process.env.PIM_HOME_DIR = testPimHomeDir;
+});
+
+afterAll(async () => {
+  if (previousPimHomeDir === undefined) {
+    delete process.env.PIM_HOME_DIR;
+  } else {
+    process.env.PIM_HOME_DIR = previousPimHomeDir;
+  }
+  if (testPimHomeDir) {
+    await rm(testPimHomeDir, { recursive: true, force: true });
+  }
+});
+
+describe("validatePublicUrl", () => {
+  test("accepts http and https", () => {
+    expect(validatePublicUrl("https://example.com/path")).toBe(
+      "https://example.com/path"
+    );
+    expect(validatePublicUrl("http://example.com")).toBe("http://example.com/");
+  });
+
+  test("rejects non-http schemes", () => {
+    expect(() => validatePublicUrl("ftp://example.com")).toThrow(/http/);
+    expect(() => validatePublicUrl("file:///etc/passwd")).toThrow(/http/);
+  });
+
+  test("rejects malformed URLs", () => {
+    expect(() => validatePublicUrl("not a url")).toThrow(/valid/);
+  });
+
+  test("rejects embedded credentials", () => {
+    expect(() => validatePublicUrl("https://user:pw@example.com")).toThrow(
+      /credentials/
+    );
+  });
+
+  test("rejects localhost and .local", () => {
+    expect(() => validatePublicUrl("http://localhost/")).toThrow(/public/);
+    expect(() => validatePublicUrl("http://printer.local/")).toThrow(/public/);
+  });
+
+  test("rejects RFC1918 IPv4", () => {
+    expect(() => validatePublicUrl("http://10.0.0.1/")).toThrow(/public/);
+    expect(() => validatePublicUrl("http://192.168.1.1/")).toThrow(/public/);
+    expect(() => validatePublicUrl("http://172.16.0.1/")).toThrow(/public/);
+    expect(() => validatePublicUrl("http://127.0.0.1/")).toThrow(/public/);
+    expect(() => validatePublicUrl("http://169.254.0.1/")).toThrow(/public/);
+  });
+
+  test("rejects IPv6 loopback and link-local", () => {
+    expect(() => validatePublicUrl("http://[::1]/")).toThrow(/public/);
+    expect(() => validatePublicUrl("http://[fe80::1]/")).toThrow(/public/);
+    expect(() => validatePublicUrl("http://[fc00::1]/")).toThrow(/public/);
+  });
+
+  test("accepts public IPs", () => {
+    expect(validatePublicUrl("http://8.8.8.8/")).toBe("http://8.8.8.8/");
+  });
+});
+
+describe("executeFetch", () => {
+  test("returns remote markdown when available", async () => {
+    const jina = {
+      fetchUrl: async () => ({
+        title: "Remote",
+        url: "https://example.test/remote",
+        content: "remote markdown",
+      }),
+    } as unknown as JinaReaderClient;
+    const webView = {
+      fetchMarkdown: async () => {
+        throw new Error("Rendered markdown should not be attempted.");
+      },
+    } as unknown as WebViewFetchClient;
+
+    const outcome = await executeFetch({
+      jina,
+      webView,
+      url: "https://example.test/",
+      format: "markdown",
+    });
+
+    expect(outcome.format).toBe("markdown");
+    expect(outcome.text).toContain("remote markdown");
+  });
+
+  test("falls back to rendered markdown when remote markdown fails", async () => {
+    const jina = {
+      fetchUrl: async () => {
+        throw new Error("Request timed out after 20000ms.");
+      },
+    } as unknown as JinaReaderClient;
+    const webView = {
+      fetchMarkdown: async () => ({
+        title: "Rendered",
+        url: "https://example.test/rendered",
+        content: "# rendered markdown",
+      }),
+      fetchHtml: async () => {
+        throw new Error("HTML should not be attempted for markdown mode.");
+      },
+    } as unknown as WebViewFetchClient;
+
+    const outcome = await executeFetch({
+      jina,
+      webView,
+      url: "https://example.test/",
+      format: "markdown",
+    });
+
+    expect(outcome.format).toBe("markdown");
+    expect(outcome.text).toContain("# rendered markdown");
+  });
+
+  test("throws rendered markdown error when fallback fails", async () => {
+    const jina = {
+      fetchUrl: async () => {
+        throw new Error("remote unavailable");
+      },
+    } as unknown as JinaReaderClient;
+    const webView = {
+      fetchMarkdown: async () => {
+        throw new Error("Request failed: unavailable");
+      },
+    } as unknown as WebViewFetchClient;
+
+    await expect(
+      executeFetch({
+        jina,
+        webView,
+        url: "https://example.test/",
+        format: "markdown",
+      })
+    ).rejects.toThrow("Failed to fetch: Request failed: unavailable");
+  });
+
+  test("returns raw rendered HTML for HTML mode", async () => {
+    const jina = {
+      fetchUrl: async () => {
+        throw new Error("Remote markdown should not be attempted.");
+      },
+    } as unknown as JinaReaderClient;
+    const webView = {
+      fetchHtml: async () => ({
+        title: "HTML",
+        url: "https://example.test/html",
+        content: "<html><body>Hello</body></html>",
+      }),
+    } as unknown as WebViewFetchClient;
+
+    const outcome = await executeFetch({
+      jina,
+      webView,
+      url: "https://example.test/",
+      format: "html",
+    });
+
+    expect(outcome.format).toBe("html");
+    expect(outcome.text).toContain("<html><body>Hello</body></html>");
+  });
+});
+
+describe("formatOutcome", () => {
+  const page = {
+    title: "Example",
+    url: "https://example.test/page",
+    content: "hello world",
+  };
+
+  test("formats untruncated page without spilling", async () => {
+    const outcome = await formatOutcome(page, "markdown");
+    expect(outcome.text).toBe(
+      [
+        "title: Example",
+        "url: https://example.test/page",
+        "format: markdown",
+        "content:",
+        "hello world",
+      ].join("\n")
+    );
+    expect(outcome.truncated).toBe(false);
+    expect(outcome.returnedBytes).toBe(11);
+    expect(outcome.totalBytes).toBe(11);
+    expect(outcome.format).toBe("markdown");
+    expect(outcome.path).toBeNull();
+  });
+
+  test("spills the full body and points the footer at the resume line over the inline budget", async () => {
+    // 1 KiB newline-terminated lines: the 32 KiB head holds exactly 32 lines,
+    // so the footer should resume reading at line 33.
+    const line = `${"x".repeat(1023)}\n`;
+    const content = line.repeat(40);
+    const long = { ...page, content };
+    const outcome = await formatOutcome(long, "html");
+    expect(outcome.truncated).toBe(true);
+    expect(outcome.returnedBytes).toBe(WEB_FETCH_INLINE_BYTES);
+    expect(outcome.totalBytes).toBe(content.length);
+    expect(outcome.path).toBeTruthy();
+    expect(outcome.path!.startsWith(join(SpillCache.dir(), "fetch-"))).toBe(
+      true
+    );
+    expect(outcome.path!.endsWith(".html")).toBe(true);
+    expect(outcome.text).toContain(
+      `use read with path=${outcome.path} and start=33 for the rest.]`
+    );
+    expect(await Bun.file(outcome.path!).text()).toBe(content);
+  });
+});
+
+describe("truncationFooter", () => {
+  test("points at the spill file with a resume line when one was written", () => {
+    expect(truncationFooter(100, 2000, "/tmp/pim/cache/fetch-abc.md", 7)).toBe(
+      "[web_fetch tool: showing first 100 bytes of 2000; use read with path=/tmp/pim/cache/fetch-abc.md and start=7 for the rest.]"
+    );
+  });
+
+  test("signals the rest is unavailable when the spill failed", () => {
+    expect(truncationFooter(100, 2000, null, 7)).toBe(
+      "[web_fetch tool: showing first 100 bytes of 2000; full content unavailable.]"
+    );
+  });
+});

package/src/extensions/web-fetch/fetch.ts

@@ -0,0 +1,249 @@
+import { isIP } from "node:net";
+import { Lines } from "../../shared/Lines";
+import { OutputBudget } from "../../shared/OutputBudget";
+import { SpillCache } from "../../shared/SpillCache";
+import type { JinaReaderClient } from "./JinaReaderClient";
+import {
+  WEB_FETCH_INLINE_BYTES,
+  type WebFetchFormat,
+  type WebFetchResolvedFormat,
+} from "./schema";
+import type { WebViewFetchClient } from "./WebViewFetchClient";
+
+export type WebFetchPage = {
+  readonly title: string;
+  readonly url: string;
+  readonly content: string;
+};
+
+export type WebFetchOutcome = {
+  readonly text: string;
+  readonly title: string;
+  readonly url: string;
+  readonly format: WebFetchResolvedFormat;
+  readonly returnedBytes: number;
+  readonly totalBytes: number;
+  readonly truncated: boolean;
+  readonly path: string | null;
+};
+
+export function truncationFooter(
+  returnedBytes: number,
+  totalBytes: number,
+  spillPath: string | null,
+  nextStart: number
+): string {
+  const base = `[web_fetch tool: showing first ${returnedBytes} bytes of ${totalBytes}`;
+  if (spillPath) {
+    return `${base}; use read with path=${spillPath} and start=${nextStart} for the rest.]`;
+  }
+  return `${base}; full content unavailable.]`;
+}
+
+export function validatePublicUrl(value: string): string {
+  let url: URL;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new Error(`URL must be a valid public HTTP or HTTPS URL: ${value}`);
+  }
+
+  if (url.protocol !== "http:" && url.protocol !== "https:") {
+    throw new Error(`URL must use http:// or https://: ${value}`);
+  }
+
+  if (url.username.length > 0 || url.password.length > 0) {
+    throw new Error("URL must not contain embedded credentials.");
+  }
+
+  if (!isPublicHostname(url.hostname)) {
+    throw new Error(
+      `URL must use a public hostname or IP address (no localhost, .local, or RFC1918 ranges): ${url.hostname}`
+    );
+  }
+
+  return url.href;
+}
+
+export async function formatOutcome(
+  page: WebFetchPage,
+  format: WebFetchResolvedFormat
+): Promise<WebFetchOutcome> {
+  const { body, returnedBytes, totalBytes, truncated } =
+    OutputBudget.truncateUtf8(page.content, WEB_FETCH_INLINE_BYTES);
+  const path = truncated
+    ? await SpillCache.write(
+        "fetch",
+        format === "html" ? "html" : "md",
+        page.content
+      )
+    : null;
+
+  const lines = [
+    `title: ${page.title}`,
+    `url: ${page.url}`,
+    `format: ${format}`,
+    "content:",
+    body,
+  ];
+
+  if (truncated) {
+    const nextStart = Lines.continuationLine(body);
+    lines.push(
+      "",
+      truncationFooter(returnedBytes, totalBytes, path, nextStart)
+    );
+  }
+
+  return {
+    text: lines.join("\n"),
+    title: page.title,
+    url: page.url,
+    format,
+    returnedBytes,
+    totalBytes,
+    truncated,
+    path,
+  };
+}
+
+export type ExecuteFetchInput = {
+  readonly jina: JinaReaderClient;
+  readonly webView: WebViewFetchClient;
+  readonly url: string;
+  readonly format: WebFetchFormat;
+  readonly signal?: AbortSignal;
+};
+
+export async function executeFetch(
+  input: ExecuteFetchInput
+): Promise<WebFetchOutcome> {
+  const { jina, webView, url, format, signal } = input;
+  const fetchInput = {
+    url,
+    ...(signal === undefined ? {} : { signal }),
+  };
+
+  if (format === "html") {
+    const page = await webView.fetchHtml(fetchInput);
+    return formatOutcome(page, "html");
+  }
+
+  try {
+    const page = await jina.fetchUrl(fetchInput);
+    return await formatOutcome(page, "markdown");
+  } catch (error) {
+    if (signal?.aborted) {
+      throw error;
+    }
+  }
+
+  try {
+    const page = await webView.fetchMarkdown(fetchInput);
+    return await formatOutcome(page, "markdown");
+  } catch (error) {
+    if (signal?.aborted) {
+      throw error;
+    }
+    throw new Error(`Failed to fetch: ${describeError(error)}`);
+  }
+}
+
+function isPublicHostname(hostname: string): boolean {
+  const normalized = normalizeHostname(hostname);
+
+  if (normalized === "localhost" || normalized.endsWith(".local")) {
+    return false;
+  }
+
+  const version = isIP(normalized);
+
+  if (version === 4) {
+    return isPublicIpv4(normalized);
+  }
+
+  if (version === 6) {
+    return isPublicIpv6(normalized);
+  }
+
+  return normalized.length > 0;
+}
+
+function normalizeHostname(hostname: string): string {
+  const lower = hostname.toLowerCase().replace(/\.+$/u, "");
+
+  if (lower.startsWith("[") && lower.endsWith("]")) {
+    return lower.slice(1, -1);
+  }
+
+  return lower;
+}
+
+function isPublicIpv4(ip: string): boolean {
+  const octets = ip.split(".").map((octet) => Number(octet));
+  const [a = 0, b = 0, c = 0] = octets;
+
+  if (a === 0 || a === 10 || a === 127) {
+    return false;
+  }
+  if (a === 169 && b === 254) {
+    return false;
+  }
+  if (a === 172 && b >= 16 && b <= 31) {
+    return false;
+  }
+  if (a === 192 && b === 168) {
+    return false;
+  }
+  if (a === 100 && b >= 64 && b <= 127) {
+    return false;
+  }
+  if (a === 192 && b === 0 && c === 0) {
+    return false;
+  }
+  if (a === 198 && (b === 18 || b === 19)) {
+    return false;
+  }
+
+  return a < 224;
+}
+
+function isPublicIpv6(ip: string): boolean {
+  const embedded = readEmbeddedIpv4(ip);
+
+  if (embedded !== undefined && !isPublicIpv4(embedded)) {
+    return false;
+  }
+
+  if (ip === "::" || ip === "::1") {
+    return false;
+  }
+
+  const first = readFirstIpv6Segment(ip);
+
+  if ((first & 0xfe00) === 0xfc00) {
+    return false;
+  }
+  if ((first & 0xffc0) === 0xfe80) {
+    return false;
+  }
+
+  return (first & 0xff00) !== 0xff00;
+}
+
+function readEmbeddedIpv4(ip: string): string | undefined {
+  const match = /(?:^|:)(\d{1,3}(?:\.\d{1,3}){3})$/u.exec(ip);
+  return match?.[1];
+}
+
+function readFirstIpv6Segment(ip: string): number {
+  const first =
+    ip
+      .split(":")
+      .find((segment) => segment.length > 0 && !segment.includes(".")) ?? "0";
+  return Number.parseInt(first, 16);
+}
+
+function describeError(error: unknown): string {
+  return error instanceof Error ? error.message : String(error);
+}

package/src/extensions/web-fetch/index.ts

@@ -0,0 +1,107 @@
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import { PimSettings } from "../../shared/PimSettings";
+import { Renderer } from "../../shared/Renderer";
+import { SpillCache } from "../../shared/SpillCache";
+import { Tools } from "../../shared/Tools";
+import { executeFetch, validatePublicUrl, type WebFetchOutcome } from "./fetch";
+import { JinaReaderClient } from "./JinaReaderClient";
+import { formatTitle, type WebFetchTitleOutcome } from "./render";
+import { type WebFetchInput, webFetchSchema } from "./schema";
+import { WebViewFetchClient } from "./WebViewFetchClient";
+
+const PREVIEW_LINES = 10;
+
+type WebFetchRenderState = {
+  outcome?: WebFetchTitleOutcome;
+};
+
+async function createJina(): Promise<JinaReaderClient> {
+  const apiKey = await PimSettings.getJinaApiKey();
+  return new JinaReaderClient(apiKey ? { apiKey } : {});
+}
+
+export default function (pi: ExtensionAPI): void {
+  SpillCache.installSweeper();
+
+  let jinaPromise: Promise<JinaReaderClient> | undefined;
+  const getJina = () => (jinaPromise ??= createJina());
+  const webView = new WebViewFetchClient();
+
+  Tools.register(pi, {
+    name: "web_fetch",
+    label: "web_fetch",
+    description: "Fetch a web page as markdown or HTML.",
+    parameters: webFetchSchema,
+    renderShell: "self",
+    executionMode: "parallel",
+    async execute(_id, params, signal) {
+      const { url, format } = params as WebFetchInput;
+
+      if (signal?.aborted) {
+        throw new Error("Web fetch aborted before execution.");
+      }
+
+      const safeUrl = validatePublicUrl(url);
+
+      const jina = await getJina();
+      const outcome = await executeFetch({
+        jina,
+        webView,
+        url: safeUrl,
+        format: format ?? "markdown",
+        ...(signal === undefined ? {} : { signal }),
+      });
+
+      return {
+        content: [{ type: "text", text: outcome.text }],
+        details: {
+          url: outcome.url,
+          title: outcome.title,
+          format: outcome.format,
+          returnedBytes: outcome.returnedBytes,
+          totalBytes: outcome.totalBytes,
+          truncated: outcome.truncated,
+          path: outcome.path,
+        },
+      };
+    },
+    renderCall(args, theme, context) {
+      const input = (args ?? {}) as Partial<WebFetchInput>;
+      const state = context.state as WebFetchRenderState;
+      return Renderer.renderToolCallTitle({
+        label: "Web Fetch",
+        title: formatTitle(input.url, input.format, state.outcome),
+        theme,
+        context,
+      });
+    },
+    renderResult(result, options, theme, context) {
+      const state = context.state as WebFetchRenderState;
+
+      if (!options.isPartial && state.outcome === undefined) {
+        const details = result.details as
+          | Pick<WebFetchOutcome, "format" | "totalBytes">
+          | undefined;
+
+        if (
+          details?.format !== undefined &&
+          typeof details.totalBytes === "number"
+        ) {
+          state.outcome = {
+            format: details.format,
+            totalBytes: details.totalBytes,
+          };
+          context.invalidate();
+        }
+      }
+
+      return Renderer.renderBorderedResult({
+        result,
+        options,
+        theme,
+        context,
+        previewLines: PREVIEW_LINES,
+      });
+    },
+  });
+}

package/src/extensions/web-fetch/render.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, test } from "bun:test";
+import { formatTitle } from "./render";
+
+describe("formatTitle", () => {
+  test("returns placeholder and default format when url is undefined", () => {
+    expect(formatTitle(undefined, undefined, undefined)).toBe("... (Markdown)");
+  });
+
+  test("renders default markdown pre-result", () => {
+    expect(formatTitle("https://example.com", undefined, undefined)).toBe(
+      "https://example.com (Markdown)"
+    );
+  });
+
+  test("renders requested HTML pre-result", () => {
+    expect(formatTitle("https://example.com", "html", undefined)).toBe(
+      "https://example.com (HTML)"
+    );
+  });
+
+  test("renders size + format label after result arrives", () => {
+    expect(
+      formatTitle("https://example.com", undefined, {
+        format: "markdown",
+        totalBytes: 23 * 1024,
+      })
+    ).toBe("https://example.com (23KB Markdown)");
+  });
+
+  test("strips trailing zeros and supports two decimals", () => {
+    expect(
+      formatTitle("https://example.com", undefined, {
+        format: "html",
+        totalBytes: 5355,
+      })
+    ).toBe("https://example.com (5.23KB HTML)");
+  });
+
+  test("renders bytes for tiny payloads", () => {
+    expect(
+      formatTitle("https://example.com", undefined, {
+        format: "markdown",
+        totalBytes: 512,
+      })
+    ).toBe("https://example.com (512B Markdown)");
+  });
+
+  test("renders MB for large payloads", () => {
+    expect(
+      formatTitle("https://example.com", undefined, {
+        format: "html",
+        totalBytes: 2.5 * 1024 * 1024,
+      })
+    ).toBe("https://example.com (2.5MB HTML)");
+  });
+});

package/src/extensions/web-fetch/render.ts

@@ -0,0 +1,39 @@
+import type { WebFetchFormat, WebFetchResolvedFormat } from "./schema";
+
+export type WebFetchTitleOutcome = {
+  readonly format: WebFetchResolvedFormat;
+  readonly totalBytes: number;
+};
+
+export function formatTitle(
+  url: string | undefined,
+  format: WebFetchFormat | undefined,
+  outcome: WebFetchTitleOutcome | undefined
+): string {
+  const u = url ?? "...";
+  const label = formatLabel(outcome?.format ?? format ?? "markdown");
+
+  if (outcome !== undefined) {
+    return `${u} (${formatSize(outcome.totalBytes)} ${label})`;
+  }
+
+  return `${u} (${label})`;
+}
+
+function formatLabel(format: WebFetchResolvedFormat): string {
+  return format === "html" ? "HTML" : "Markdown";
+}
+
+function formatSize(bytes: number): string {
+  if (bytes < 1024) {
+    return `${bytes}B`;
+  }
+  if (bytes < 1024 * 1024) {
+    return `${trimZeros((bytes / 1024).toFixed(2))}KB`;
+  }
+  return `${trimZeros((bytes / (1024 * 1024)).toFixed(2))}MB`;
+}
+
+function trimZeros(value: string): string {
+  return value.replace(/\.?0+$/u, "");
+}