@j6e/pi-md-web-surfer 0.9.1

package/README.md

@@ -0,0 +1,105 @@
+# pi-md-web-surfer
+
+A [pi](https://github.com/nichochar/pi-coding-agent) extension that adds `fetch_markdown` and `web_search` tools — fetch web pages as clean markdown and search the web.
+
+Inspired by the [skill](https://github.com/ruliana/pi-fetch-markdown) of the same name.
+
+## Tools
+
+### `fetch_markdown`
+
+Fetch any web page as clean, token-efficient markdown.
+
+Three strategies, tried in order:
+
+1. **Content Negotiation** — Sends `Accept: text/markdown` header. Sites using Cloudflare's [Markdown for Agents](https://blog.cloudflare.com/markdown-for-agents/) return markdown directly (~80% fewer tokens than HTML).
+2. **Direct text/plain** — For raw `.md` files, `text/plain` responses, etc. The body is already clean text, so we use it directly.
+3. **Jina Reader** (fallback) — For sites that return HTML, routes through `r.jina.ai` which renders the page and extracts clean markdown.
+
+**Usage:**
+```
+fetch_markdown(url, info?)
+```
+
+### `web_search`
+
+Search the web and return results as clean markdown via Jina Search.
+
+**Requires a Jina API key.** Get one free at [jina.ai/?newKey](https://jina.ai/?newKey), then configure it with `/md-web-surfer-config` or the `JINA_API_KEY` environment variable.
+
+**Usage:**
+```
+web_search(query, num?, info?, retainImages?, withLinksSummary?)
+```
+
+- `query` — Search query
+- `num` — Number of results (1–20, default 5)
+- `info` — Include metadata (method, tokens)
+- `retainImages` — Image retention mode: `"none"` (default), `"all"`, or `"alt_text"` (keep alt text with auto-generation)
+- `withLinksSummary` — Include a summary section for hyperlinks (default: false)
+
+## Configuration
+
+### `/md-web-surfer-config`
+
+Interactive command to set your Jina API key. Writes to `~/.config/pi-md-web-surfer/config.json` (chmod 0600).
+
+```
+/md-web-surfer-config        # prompt for key
+/md-web-surfer-config --show # show current key status (masked)
+```
+
+### API key resolution
+
+First match wins:
+
+1. `JINA_API_KEY` environment variable
+2. `jinaApiKey` field in `~/.config/pi-md-web-surfer/config.json`
+
+## Installation
+
+### Quick test (one session)
+
+```bash
+pi -e ./src/index.ts
+```
+
+### Persistent (auto-discovered)
+
+```bash
+ln -s $(pwd) ~/.pi/agent/extensions/pi-md-web-surfer
+```
+
+Then run `pi` normally. The tools appear alongside `read`, `write`, `edit`, etc.
+
+### As a pi package
+
+```bash
+pi install /path/to/pi-md-web-surfer
+# or once published:
+# pi install npm:@j6e/pi-md-web-surfer
+```
+
+## Development
+
+```bash
+npm install
+npm test           # run tests once
+npm run test:watch # TDD mode
+npm run check      # typecheck
+```
+
+Smoke test against real URLs:
+
+```bash
+node --experimental-strip-types scripts/smoke-test.ts
+```
+
+## Requirements
+
+- Node.js 22+ (for `--experimental-strip-types`)
+- pi-coding-agent 0.74+
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,42 @@
+{
+  "name": "@j6e/pi-md-web-surfer",
+  "version": "0.9.1",
+  "description": "Pi extension: fetch web pages as markdown and search the web, powered by Jina AI",
+  "license": "MIT",
+  "author": "Joan G. Esquerdo",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/j6e/pi-md-web-surfer.git"
+  },
+  "bugs": {
+    "url": "https://github.com/j6e/pi-md-web-surfer/issues"
+  },
+  "homepage": "https://github.com/j6e/pi-md-web-surfer#readme",
+  "keywords": [
+    "pi-package"
+  ],
+  "type": "module",
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "check": "tsc --noEmit"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*",
+    "typebox": "*"
+  },
+  "devDependencies": {
+    "@types/node": "^22.0.0",
+    "typescript": "^5.6.0",
+    "vitest": "^2.1.0"
+  },
+  "files": [
+    "src"
+  ],
+  "pi": {
+    "extensions": [
+      "./src/index.ts"
+    ]
+  }
+}

package/src/config.test.ts

@@ -0,0 +1,69 @@
+import { describe, it, expect, beforeEach, afterEach } from "vitest";
+import {
+  getConfigPath,
+  readConfig,
+  writeConfig,
+  clearConfig,
+  resolveJinaApiKey,
+  maskKey,
+} from "./config.js";
+
+describe("config", () => {
+  const originalEnv = process.env.JINA_API_KEY;
+
+  beforeEach(() => {
+    delete process.env.JINA_API_KEY;
+    clearConfig();
+  });
+
+  afterEach(() => {
+    if (originalEnv !== undefined) {
+      process.env.JINA_API_KEY = originalEnv;
+    } else {
+      delete process.env.JINA_API_KEY;
+    }
+    clearConfig();
+  });
+
+  it("reads empty config when file does not exist", () => {
+    const config = readConfig();
+    expect(config).toEqual({});
+  });
+
+  it("writes and reads config", () => {
+    writeConfig({ jinaApiKey: "jina_test_123" });
+    const config = readConfig();
+    expect(config.jinaApiKey).toBe("jina_test_123");
+  });
+
+  it("overwrites existing config", () => {
+    writeConfig({ jinaApiKey: "old" });
+    writeConfig({ jinaApiKey: "new" });
+    const config = readConfig();
+    expect(config.jinaApiKey).toBe("new");
+  });
+
+  it("resolves from env var first", () => {
+    process.env.JINA_API_KEY = "env_key";
+    writeConfig({ jinaApiKey: "file_key" });
+    expect(resolveJinaApiKey()).toBe("env_key");
+  });
+
+  it("falls back to config file when env is unset", () => {
+    writeConfig({ jinaApiKey: "file_key" });
+    expect(resolveJinaApiKey()).toBe("file_key");
+  });
+
+  it("returns null when no key is available", () => {
+    expect(resolveJinaApiKey()).toBeNull();
+  });
+
+  it("masks keys correctly", () => {
+    expect(maskKey("short")).toBe("****");
+    expect(maskKey("jina_1234567890abcd")).toBe("jina...abcd");
+  });
+
+  it("config path includes package name", () => {
+    expect(getConfigPath()).toContain("pi-md-web-surfer");
+  });
+});

package/src/config.ts

@@ -0,0 +1,63 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
+import {
+  readFileSync,
+  writeFileSync,
+  mkdirSync,
+  existsSync,
+  unlinkSync,
+} from "node:fs";
+
+const CONFIG_DIR = join(homedir(), ".config", "pi-md-web-surfer");
+const CONFIG_PATH = join(CONFIG_DIR, "config.json");
+
+export interface Config {
+  jinaApiKey?: string;
+}
+
+export function getConfigDir(): string {
+  return CONFIG_DIR;
+}
+
+export function getConfigPath(): string {
+  return CONFIG_PATH;
+}
+
+export function readConfig(): Config {
+  try {
+    const raw = readFileSync(CONFIG_PATH, "utf8");
+    return JSON.parse(raw) as Config;
+  } catch {
+    return {};
+  }
+}
+
+export function writeConfig(config: Config): void {
+  mkdirSync(CONFIG_DIR, { recursive: true, mode: 0o700 });
+  writeFileSync(CONFIG_PATH, JSON.stringify(config, null, 2), {
+    mode: 0o600,
+  });
+}
+
+export function clearConfig(): void {
+  try {
+    unlinkSync(CONFIG_PATH);
+  } catch {
+    // ignore
+  }
+}
+
+export function resolveJinaApiKey(): string | null {
+  const envKey = process.env.JINA_API_KEY;
+  if (envKey) return envKey;
+
+  const config = readConfig();
+  if (config.jinaApiKey) return config.jinaApiKey;
+
+  return null;
+}
+
+export function maskKey(key: string): string {
+  if (key.length <= 8) return "****";
+  return key.slice(0, 4) + "..." + key.slice(-4);
+}

package/src/fetch-markdown.test.ts

@@ -0,0 +1,272 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+vi.mock("node:os", () => ({ tmpdir: vi.fn(() => "/tmp") }));
+vi.mock("node:fs/promises", () => ({ writeFile: vi.fn(() => Promise.resolve()) }));
+
+vi.mock("@earendil-works/pi-coding-agent", async () => {
+  return {
+    truncateHead: vi.fn((content: string) => ({
+      content,
+      truncated: false,
+      truncatedBy: undefined,
+      firstLineExceedsLimit: false,
+      lastLinePartial: false,
+      maxLines: 2000,
+      maxBytes: 50000,
+      outputLines: content.split("\n").length,
+      totalLines: content.split("\n").length,
+      outputBytes: content.length,
+      totalBytes: content.length,
+    })),
+    formatSize: vi.fn((bytes: number) => `${bytes}B`),
+    DEFAULT_MAX_LINES: 2000,
+    DEFAULT_MAX_BYTES: 50000,
+  };
+});
+
+import { fetchMarkdown } from "./fetch-markdown.js";
+import { truncateHead } from "@earendil-works/pi-coding-agent";
+import { writeFile } from "node:fs/promises";
+
+describe("fetchMarkdown", () => {
+  beforeEach(() => {
+    vi.stubGlobal("fetch", vi.fn());
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    vi.clearAllMocks();
+  });
+
+  it("returns markdown via content negotiation when server supports it", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Hello\n\nWorld", {
+        status: 200,
+        headers: new Headers({
+          "content-type": "text/markdown; charset=utf-8",
+          "x-markdown-tokens": "42",
+          "content-signal": "blog-post",
+        }),
+      }),
+    );
+
+    const result = await fetchMarkdown("https://example.com");
+
+    expect(result.text).toBe("# Hello\n\nWorld");
+    expect(result.details.method).toBe("content-negotiation");
+    expect(result.details.url).toBe("https://example.com");
+  });
+
+  it("falls back to Jina Reader when content negotiation fails", async () => {
+    vi.mocked(fetch)
+      .mockResolvedValueOnce(new Response("Not Found", { status: 404 }))
+      .mockResolvedValueOnce(
+        new Response("# Jina Result", {
+          status: 200,
+          headers: new Headers({ "content-type": "text/markdown" }),
+        }),
+      );
+
+    const result = await fetchMarkdown("https://example.com");
+
+    expect(result.text).toBe("# Jina Result");
+    expect(result.details.method).toBe("jina-reader");
+    expect(fetch).toHaveBeenNthCalledWith(
+      2,
+      "https://r.jina.ai/https://example.com",
+      expect.any(Object),
+    );
+  });
+
+  it("falls back when server returns HTML despite markdown content-type", async () => {
+    vi.mocked(fetch)
+      .mockResolvedValueOnce(
+        new Response("<!DOCTYPE html><html><body>oops</body></html>", {
+          status: 200,
+          headers: new Headers({ "content-type": "text/markdown" }),
+        }),
+      )
+      .mockResolvedValueOnce(
+        new Response("# Clean Markdown", {
+          status: 200,
+          headers: new Headers({ "content-type": "text/markdown" }),
+        }),
+      );
+
+    const result = await fetchMarkdown("https://example.com");
+
+    expect(result.text).toBe("# Clean Markdown");
+    expect(result.details.method).toBe("jina-reader");
+  });
+
+  it("falls back when content-type is not text/markdown", async () => {
+    vi.mocked(fetch)
+      .mockResolvedValueOnce(
+        new Response("<html></html>", {
+          status: 200,
+          headers: new Headers({ "content-type": "text/html" }),
+        }),
+      )
+      .mockResolvedValueOnce(
+        new Response("# Fallback", {
+          status: 200,
+          headers: new Headers({ "content-type": "text/markdown" }),
+        }),
+      );
+
+    const result = await fetchMarkdown("https://example.com");
+
+    expect(result.text).toBe("# Fallback");
+    expect(result.details.method).toBe("jina-reader");
+  });
+
+  it("throws when both methods fail", async () => {
+    vi.mocked(fetch)
+      .mockResolvedValueOnce(new Response("Error", { status: 500 }))
+      .mockResolvedValueOnce(new Response("Error", { status: 500 }));
+
+    await expect(fetchMarkdown("https://example.com")).rejects.toThrow(
+      "Both content negotiation and Jina Reader failed for https://example.com",
+    );
+  });
+
+  it("throws when Jina returns HTML", async () => {
+    vi.mocked(fetch)
+      .mockResolvedValueOnce(new Response("Error", { status: 500 }))
+      .mockResolvedValueOnce(
+        new Response("<!DOCTYPE html><html></html>", { status: 200 }),
+      );
+
+    await expect(fetchMarkdown("https://example.com")).rejects.toThrow(
+      "Jina Reader returned empty or HTML for https://example.com",
+    );
+  });
+
+  it("throws when Jina returns empty body", async () => {
+    vi.mocked(fetch)
+      .mockResolvedValueOnce(new Response("Error", { status: 500 }))
+      .mockResolvedValueOnce(new Response("", { status: 200 }));
+
+    await expect(fetchMarkdown("https://example.com")).rejects.toThrow(
+      "Jina Reader returned empty or HTML for https://example.com",
+    );
+  });
+
+  it("includes metadata when info flag is set", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Content", {
+        status: 200,
+        headers: new Headers({
+          "content-type": "text/markdown",
+          "x-markdown-tokens": "1234",
+          "content-signal": "blog-post",
+        }),
+      }),
+    );
+
+    const result = await fetchMarkdown("https://example.com", { info: true });
+
+    expect(result.details.tokens).toBe("1234");
+    expect(result.details.contentSignal).toBe("blog-post");
+  });
+
+  it("omits metadata when info flag is false", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Content", {
+        status: 200,
+        headers: new Headers({
+          "content-type": "text/markdown",
+          "x-markdown-tokens": "1234",
+        }),
+      }),
+    );
+
+    const result = await fetchMarkdown("https://example.com");
+
+    expect(result.details.tokens).toBeUndefined();
+    expect(result.details.contentSignal).toBeUndefined();
+  });
+
+  it("handles truncated output by writing temp file", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Very long content", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/markdown" }),
+      }),
+    );
+
+    vi.mocked(truncateHead).mockReturnValueOnce({
+      content: "# Very",
+      truncated: true,
+      truncatedBy: "bytes",
+      firstLineExceedsLimit: false,
+      lastLinePartial: false,
+      maxLines: 2000,
+      maxBytes: 50000,
+      outputLines: 1,
+      totalLines: 100,
+      outputBytes: 6,
+      totalBytes: 10000,
+    });
+
+    const result = await fetchMarkdown("https://example.com");
+
+    expect(result.text).toContain("[Output truncated:");
+    expect(result.text).toContain("Full output saved to:");
+    expect(result.details.fullOutputPath).toMatch(/\.md$/);
+    expect(writeFile).toHaveBeenCalledWith(
+      expect.stringMatching(/\.md$/),
+      "# Very long content",
+      "utf8",
+    );
+  });
+
+  it("uses text/plain responses directly without Jina", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Raw Markdown\n\nContent", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/plain; charset=utf-8" }),
+      }),
+    );
+
+    const result = await fetchMarkdown(
+      "https://raw.githubusercontent.com/user/repo/main/README.md",
+    );
+
+    expect(result.text).toBe("# Raw Markdown\n\nContent");
+    expect(result.details.method).toBe("direct");
+    expect(fetch).toHaveBeenCalledTimes(1); // no Jina fallback
+  });
+
+  it("uses text/plain for non-markdown URLs too", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("just plain text", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/plain" }),
+      }),
+    );
+
+    const result = await fetchMarkdown("https://example.com/notes.txt");
+
+    expect(result.text).toBe("just plain text");
+    expect(result.details.method).toBe("direct");
+    expect(fetch).toHaveBeenCalledTimes(1); // no Jina fallback
+  });
+
+  it("passes abort signal to fetch", async () => {
+    const controller = new AbortController();
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Content", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/markdown" }),
+      }),
+    );
+
+    await fetchMarkdown("https://example.com", { signal: controller.signal });
+
+    expect(fetch).toHaveBeenCalledWith(
+      "https://example.com",
+      expect.objectContaining({ signal: controller.signal }),
+    );
+  });
+});

package/src/fetch-markdown.ts

@@ -0,0 +1,108 @@
+import {
+  truncateHead,
+  formatSize,
+  DEFAULT_MAX_BYTES,
+  DEFAULT_MAX_LINES,
+} from "@earendil-works/pi-coding-agent";
+import { tmpdir } from "node:os";
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+
+export interface FetchOptions {
+  info?: boolean;
+  signal?: AbortSignal;
+}
+
+export interface FetchResult {
+  text: string;
+  details: Record<string, unknown>;
+}
+
+export async function fetchMarkdown(
+  url: string,
+  options: FetchOptions = {},
+): Promise<FetchResult> {
+  const { info, signal } = options;
+
+  // ─── Method 1: Content negotiation ───
+  const negotiate = await fetch(url, {
+    signal,
+    headers: {
+      Accept: "text/markdown, text/html;q=0.9",
+      "User-Agent": "Mozilla/5.0 (compatible; pi-agent/1.0)",
+    },
+    redirect: "follow",
+  });
+
+  let body = "";
+  let method = "jina-reader";
+  let tokens: string | undefined;
+  let contentSignal: string | undefined;
+
+  const ct = negotiate.headers.get("content-type") ?? "";
+  if (negotiate.ok && ct.includes("text/markdown")) {
+    body = await negotiate.text();
+    if (!body.slice(0, 500).match(/<!DOCTYPE|<html/i)) {
+      method = "content-negotiation";
+      tokens = negotiate.headers.get("x-markdown-tokens") ?? undefined;
+      contentSignal = negotiate.headers.get("content-signal") ?? undefined;
+    } else {
+      body = "";
+    }
+  }
+
+  // ─── Method 1b: text/plain served directly ───
+  if (negotiate.ok && !body && ct.includes("text/plain")) {
+    body = await negotiate.text();
+    if (!body.slice(0, 500).match(/<!DOCTYPE|<html/i)) {
+      method = "direct";
+    } else {
+      body = "";
+    }
+  }
+
+  // ─── Method 2: Jina Reader fallback ───
+  if (!body) {
+    const jina = await fetch(`https://r.jina.ai/${url}`, {
+      signal,
+      headers: {
+        Accept: "text/markdown",
+        "User-Agent": "Mozilla/5.0 (compatible; pi-agent/1.0)",
+      },
+    });
+    if (!jina.ok) {
+      throw new Error(
+        `Both content negotiation and Jina Reader failed for ${url}`,
+      );
+    }
+    body = await jina.text();
+    if (!body || body.slice(0, 300).match(/<!DOCTYPE|<html/i)) {
+      throw new Error(`Jina Reader returned empty or HTML for ${url}`);
+    }
+  }
+
+  // ─── Truncate if needed ───
+  const truncated = truncateHead(body, {
+    maxLines: DEFAULT_MAX_LINES,
+    maxBytes: DEFAULT_MAX_BYTES,
+  });
+
+  let text = truncated.content;
+  const details: Record<string, unknown> = { method, url };
+
+  if (info) {
+    details.tokens = tokens ?? "unknown";
+    details.contentSignal = contentSignal ?? "none";
+  }
+
+  if (truncated.truncated) {
+    const tmp = join(tmpdir(), `fetch-markdown-${Date.now()}.md`);
+    await writeFile(tmp, body, "utf8");
+    text += `\n\n[Output truncated: ${truncated.outputLines} of ${truncated.totalLines} lines`;
+    text += ` (${formatSize(truncated.outputBytes)} of ${formatSize(truncated.totalBytes)}).`;
+    text += ` Full output saved to: ${tmp}]`;
+    details.fullOutputPath = tmp;
+  }
+
+  return { text, details };
+}

package/src/index.test.ts

@@ -0,0 +1,73 @@
+import { describe, it, expect, vi } from "vitest";
+import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import extensionFactory from "./index.js";
+
+describe("extension", () => {
+  it("registers fetch_markdown tool", () => {
+    const registerTool = vi.fn();
+    const registerCommand = vi.fn();
+    const mockPi = {
+      registerTool,
+      registerCommand,
+    } as unknown as ExtensionAPI;
+
+    extensionFactory(mockPi);
+
+    const definitions = registerTool.mock.calls.map(
+      (call) =>
+        call[0] as {
+          name: string;
+          label: string;
+          description: string;
+        },
+    );
+
+    const markdownTool = definitions.find((d) => d.name === "fetch_markdown");
+    expect(markdownTool).toBeDefined();
+    expect(markdownTool!.label).toBe("Fetch Markdown");
+    expect(markdownTool!.description).toContain("content negotiation");
+  });
+
+  it("registers web_search tool", () => {
+    const registerTool = vi.fn();
+    const registerCommand = vi.fn();
+    const mockPi = {
+      registerTool,
+      registerCommand,
+    } as unknown as ExtensionAPI;
+
+    extensionFactory(mockPi);
+
+    const definitions = registerTool.mock.calls.map(
+      (call) =>
+        call[0] as {
+          name: string;
+          label: string;
+          description: string;
+        },
+    );
+
+    const searchTool = definitions.find((d) => d.name === "web_search");
+    expect(searchTool).toBeDefined();
+    expect(searchTool!.label).toBe("Web Search");
+    expect(searchTool!.description).toContain("Jina Search");
+  });
+
+  it("registers md-web-surfer-config command", () => {
+    const registerTool = vi.fn();
+    const registerCommand = vi.fn();
+    const mockPi = {
+      registerTool,
+      registerCommand,
+    } as unknown as ExtensionAPI;
+
+    extensionFactory(mockPi);
+
+    expect(registerCommand).toHaveBeenCalledWith(
+      "md-web-surfer-config",
+      expect.objectContaining({
+        description: expect.stringContaining("Jina API key"),
+      }),
+    );
+  });
+});

package/src/index.ts

@@ -0,0 +1,267 @@
+import type { ExtensionAPI, Theme } from "@earendil-works/pi-coding-agent";
+import { highlightCode, keyHint } from "@earendil-works/pi-coding-agent";
+import { Text } from "@earendil-works/pi-tui";
+import { Type } from "typebox";
+import { fetchMarkdown } from "./fetch-markdown.js";
+import { webSearch } from "./web-search.js";
+import {
+  resolveJinaApiKey,
+  readConfig,
+  writeConfig,
+  maskKey,
+} from "./config.js";
+
+function replaceTabs(text: string): string {
+  return text.replace(/\t/g, "   ");
+}
+
+function trimTrailingEmptyLines(lines: string[]): string[] {
+  let end = lines.length;
+  while (end > 0 && lines[end - 1] === "") {
+    end--;
+  }
+  return lines.slice(0, end);
+}
+
+function renderMarkdownOutput(
+  output: string,
+  options: { expanded: boolean; isPartial: boolean },
+  theme: Theme,
+): string {
+  const highlighted = highlightCode(replaceTabs(output), "markdown");
+  const lines = trimTrailingEmptyLines(highlighted);
+
+  const maxLines = options.expanded || options.isPartial ? lines.length : 10;
+  const displayLines = lines.slice(0, maxLines);
+  const remaining = lines.length - maxLines;
+
+  let resultText = `\n${displayLines.map((line) => replaceTabs(line)).join("\n")}`;
+
+  if (remaining > 0) {
+    resultText += `${theme.fg("muted", `\n... (${remaining} more lines,`)} ${keyHint("app.tools.expand", "to expand")})`;
+  }
+
+  return resultText;
+}
+
+export default function (pi: ExtensionAPI) {
+  // ─── fetch_markdown ───
+  pi.registerTool({
+    name: "fetch_markdown",
+    label: "Fetch Markdown",
+    description:
+      "Fetch any web page as clean, token-efficient markdown. Tries content negotiation (Accept: text/markdown) first, then falls back to Jina Reader.",
+    promptSnippet: "Fetch a URL and return its content as markdown",
+    promptGuidelines: [
+      "Use fetch_markdown when you need to read, summarize, or analyze web page content.",
+      "Use fetch_markdown instead of bash+curl for fetching web pages.",
+    ],
+    parameters: Type.Object({
+      url: Type.String({ description: "URL to fetch" }),
+      info: Type.Optional(
+        Type.Boolean({
+          description: "Include metadata (method, tokens, content-signal)",
+        }),
+      ),
+    }),
+
+    async execute(_toolCallId, params, signal) {
+      const result = await fetchMarkdown(params.url, {
+        info: params.info,
+        signal,
+      });
+      return {
+        content: [{ type: "text", text: result.text }],
+        details: result.details,
+      };
+    },
+
+    renderCall(args, theme, context) {
+      const text =
+        (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
+      const url = typeof args.url === "string" ? args.url : "";
+      const urlDisplay = url
+        ? theme.fg("accent", url)
+        : theme.fg("error", "[invalid arg]");
+      text.setText(
+        `${theme.fg("toolTitle", theme.bold("fetch_markdown"))} ${urlDisplay}`,
+      );
+      return text;
+    },
+
+    renderResult(result, options, theme, context) {
+      const text =
+        (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
+
+      if (options.isPartial) {
+        text.setText(theme.fg("warning", "Fetching..."));
+        return text;
+      }
+
+      const output = result.content
+        .filter((c): c is { type: "text"; text: string } => c.type === "text")
+        .map((c) => c.text)
+        .join("\n");
+
+      text.setText(
+        renderMarkdownOutput(output, options, theme),
+      );
+      return text;
+    },
+  });
+
+  // ─── web_search ───
+  pi.registerTool({
+    name: "web_search",
+    label: "Web Search",
+    description:
+      "Search the web and return results as clean markdown via Jina Search.",
+    promptSnippet: "Search the web for current information",
+    promptGuidelines: [
+      "Use web_search when the user asks about recent events, current facts, or information that may not be in your training data.",
+      "Use web_search before fetch_markdown when you don't know the exact URL.",
+    ],
+    parameters: Type.Object({
+      query: Type.String({ description: "Search query" }),
+      num: Type.Optional(
+        Type.Number({
+          description: "Number of results (1–20, default 5)",
+          minimum: 1,
+          maximum: 20,
+        }),
+      ),
+      info: Type.Optional(
+        Type.Boolean({
+          description: "Include metadata (method, tokens)",
+        }),
+      ),
+      retainImages: Type.Optional(
+        Type.Union(
+          [
+            Type.Literal("none"),
+            Type.Literal("all"),
+            Type.Literal("alt_text"),
+          ],
+          {
+            description:
+              "Image retention mode: none (default), all, alt_text (keep alt text with auto-generation)",
+          },
+        ),
+      ),
+      withLinksSummary: Type.Optional(
+        Type.Boolean({
+          description: "Include a summary section for hyperlinks (default: false)",
+        }),
+      ),
+    }),
+
+    async execute(_toolCallId, params, signal) {
+      const result = await webSearch(params.query, {
+        num: params.num,
+        info: params.info,
+        signal,
+        retainImages: params.retainImages,
+        withLinksSummary: params.withLinksSummary,
+      });
+      return {
+        content: [{ type: "text", text: result.text }],
+        details: result.details,
+      };
+    },
+
+    renderCall(args, theme, context) {
+      const text =
+        (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
+      const query = typeof args.query === "string" ? args.query : "";
+      const queryDisplay = query
+        ? theme.fg("accent", `"${query}"`)
+        : theme.fg("error", "[invalid arg]");
+      text.setText(
+        `${theme.fg("toolTitle", theme.bold("web_search"))} ${queryDisplay}`,
+      );
+      return text;
+    },
+
+    renderResult(result, options, theme, context) {
+      const text =
+        (context.lastComponent as Text | undefined) ?? new Text("", 0, 0);
+
+      if (options.isPartial) {
+        text.setText(theme.fg("warning", "Searching..."));
+        return text;
+      }
+
+      const output = result.content
+        .filter((c): c is { type: "text"; text: string } => c.type === "text")
+        .map((c) => c.text)
+        .join("\n");
+
+      text.setText(renderMarkdownOutput(output, options, theme));
+      return text;
+    },
+  });
+
+  // ─── /md-web-surfer-config ───
+  pi.registerCommand("md-web-surfer-config", {
+    description: "Configure Jina API key for web search",
+    handler: async (args, ctx) => {
+      const show = args.trim() === "--show";
+      const envKey = process.env.JINA_API_KEY;
+      const config = readConfig();
+      const configKey = config.jinaApiKey;
+
+      if (show) {
+        const lines: string[] = [];
+        if (envKey) {
+          lines.push(`Env var JINA_API_KEY: ${maskKey(envKey)}`);
+        } else {
+          lines.push("Env var JINA_API_KEY: not set");
+        }
+        if (configKey) {
+          lines.push(`Config file: ${maskKey(configKey)}`);
+        } else {
+          lines.push("Config file: no key stored");
+        }
+        if (!ctx.hasUI) {
+          console.log(lines.join("\n"));
+          return;
+        }
+        for (const line of lines) {
+          ctx.ui.notify(line, "info");
+        }
+        return;
+      }
+
+      if (!ctx.hasUI) {
+        console.log(
+          "Run /md-web-surfer-config interactively to set your API key, or set the JINA_API_KEY environment variable.",
+        );
+        return;
+      }
+
+      const currentKey = resolveJinaApiKey();
+      if (currentKey) {
+        const ok = await ctx.ui.confirm(
+          "API Key",
+          `Current key: ${maskKey(currentKey)}. Replace it?`,
+        );
+        if (!ok) {
+          ctx.ui.notify("Key unchanged", "info");
+          return;
+        }
+      }
+
+      const key = await ctx.ui.input(
+        "Jina API Key",
+        "Get one free at https://jina.ai/?newKey",
+      );
+      if (!key) {
+        ctx.ui.notify("No key provided", "warning");
+        return;
+      }
+
+      writeConfig({ jinaApiKey: key.trim() });
+      ctx.ui.notify("API key saved", "info");
+    },
+  });
+}

package/src/web-search.test.ts

@@ -0,0 +1,221 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+vi.mock("node:os", () => ({ tmpdir: vi.fn(() => "/tmp") }));
+vi.mock("node:fs/promises", () => ({ writeFile: vi.fn(() => Promise.resolve()) }));
+
+vi.mock("@earendil-works/pi-coding-agent", async () => {
+  return {
+    truncateHead: vi.fn((content: string) => ({
+      content,
+      truncated: false,
+      truncatedBy: undefined,
+      firstLineExceedsLimit: false,
+      lastLinePartial: false,
+      maxLines: 2000,
+      maxBytes: 50000,
+      outputLines: content.split("\n").length,
+      totalLines: content.split("\n").length,
+      outputBytes: content.length,
+      totalBytes: content.length,
+    })),
+    formatSize: vi.fn((bytes: number) => `${bytes}B`),
+    DEFAULT_MAX_LINES: 2000,
+    DEFAULT_MAX_BYTES: 50000,
+  };
+});
+
+vi.mock("./config.js", () => ({
+  resolveJinaApiKey: vi.fn(),
+}));
+
+import { webSearch } from "./web-search.js";
+import { resolveJinaApiKey } from "./config.js";
+import { truncateHead } from "@earendil-works/pi-coding-agent";
+
+describe("webSearch", () => {
+  beforeEach(() => {
+    vi.stubGlobal("fetch", vi.fn());
+    vi.mocked(resolveJinaApiKey).mockReturnValue("jina_test_key");
+  });
+
+  afterEach(() => {
+    vi.unstubAllGlobals();
+    vi.clearAllMocks();
+  });
+
+  it("returns search results with correct defaults", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Search Results\n\n1. [Example](https://example.com)", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/markdown" }),
+      }),
+    );
+
+    const result = await webSearch("hello world");
+
+    expect(result.text).toContain("Search Results");
+    expect(result.details.method).toBe("jina-search");
+    expect(result.details.query).toBe("hello world");
+    expect(result.details.num).toBe(5);
+    expect(result.details.retainImages).toBe("none");
+    expect(result.details.withLinksSummary).toBe(false);
+
+    const call = vi.mocked(fetch).mock.calls[0];
+    expect(call[0]).toContain("s.jina.ai/hello%20world");
+    expect(call[0]).toContain("num=5");
+    expect(call[0]).toContain("retainImages=none");
+    expect(call[0]).toContain("withLinksSummary=false");
+    expect(call[0]).not.toContain("withImagesSummary");
+    expect(call[1]?.headers).toMatchObject({
+      Authorization: "Bearer jina_test_key",
+    });
+  });
+
+  it("throws when no API key is configured", async () => {
+    vi.mocked(resolveJinaApiKey).mockReturnValue(null);
+
+    await expect(webSearch("test")).rejects.toThrow(
+      "Jina Search requires an API key",
+    );
+  });
+
+  it("throws on non-ok response", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("Rate limited", { status: 429 }),
+    );
+
+    await expect(webSearch("test")).rejects.toThrow(
+      "Jina Search failed: 429",
+    );
+  });
+
+  it("throws when response is HTML", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("<!DOCTYPE html><html></html>", { status: 200 }),
+    );
+
+    await expect(webSearch("test")).rejects.toThrow(
+      "Jina Search returned empty or HTML response",
+    );
+  });
+
+  it("respects num parameter and clamps to 1-20", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Results", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/markdown" }),
+      }),
+    );
+
+    await webSearch("test", { num: 50 });
+    expect(vi.mocked(fetch).mock.calls[0][0]).toContain("num=20");
+
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Results", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/markdown" }),
+      }),
+    );
+    await webSearch("test", { num: 0 });
+    expect(vi.mocked(fetch).mock.calls[1][0]).toContain("num=1");
+  });
+
+  it("passes abort signal to fetch", async () => {
+    const controller = new AbortController();
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Results", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/markdown" }),
+      }),
+    );
+
+    await webSearch("test", { signal: controller.signal });
+
+    expect(vi.mocked(fetch).mock.calls[0][1]).toMatchObject({
+      signal: controller.signal,
+    });
+  });
+
+  it("includes metadata when info flag is set", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Results", {
+        status: 200,
+        headers: new Headers({
+          "content-type": "text/markdown",
+          "x-markdown-tokens": "15000",
+        }),
+      }),
+    );
+
+    const result = await webSearch("test", { info: true });
+    expect(result.details.tokens).toBe("15000");
+  });
+
+  it("passes custom retainImages and withLinksSummary", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Results", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/markdown" }),
+      }),
+    );
+
+    const result = await webSearch("test", {
+      retainImages: "all",
+      withLinksSummary: true,
+    });
+
+    expect(result.details.retainImages).toBe("all");
+    expect(result.details.withLinksSummary).toBe(true);
+
+    const call = vi.mocked(fetch).mock.calls[0][0] as string;
+    expect(call).toContain("retainImages=all");
+    expect(call).toContain("withLinksSummary=true");
+  });
+
+  it("maps alt_text to alt_p for Jina API", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Results", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/markdown" }),
+      }),
+    );
+
+    const result = await webSearch("test", {
+      retainImages: "alt_text",
+    });
+
+    expect(result.details.retainImages).toBe("alt_text");
+
+    const call = vi.mocked(fetch).mock.calls[0][0] as string;
+    expect(call).toContain("retainImages=alt_p");
+  });
+
+  it("handles truncated output", async () => {
+    vi.mocked(fetch).mockResolvedValueOnce(
+      new Response("# Very long results", {
+        status: 200,
+        headers: new Headers({ "content-type": "text/markdown" }),
+      }),
+    );
+
+    vi.mocked(truncateHead).mockReturnValueOnce({
+      content: "# Very",
+      truncated: true,
+      truncatedBy: "bytes",
+      firstLineExceedsLimit: false,
+      lastLinePartial: false,
+      maxLines: 2000,
+      maxBytes: 50000,
+      outputLines: 1,
+      totalLines: 100,
+      outputBytes: 6,
+      totalBytes: 10000,
+    });
+
+    const result = await webSearch("test");
+
+    expect(result.text).toContain("[Output truncated:");
+    expect(result.text).toContain("Full output saved to:");
+    expect(result.details.fullOutputPath).toMatch(/\.md$/);
+  });
+});

package/src/web-search.ts

@@ -0,0 +1,103 @@
+import {
+  truncateHead,
+  formatSize,
+  DEFAULT_MAX_BYTES,
+  DEFAULT_MAX_LINES,
+} from "@earendil-works/pi-coding-agent";
+import { tmpdir } from "node:os";
+import { writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { resolveJinaApiKey } from "./config.js";
+
+export interface SearchOptions {
+  num?: number;
+  info?: boolean;
+  signal?: AbortSignal;
+  retainImages?: "none" | "all" | "alt_text";
+  withLinksSummary?: boolean;
+}
+
+export interface SearchResult {
+  text: string;
+  details: Record<string, unknown>;
+}
+
+export async function webSearch(
+  query: string,
+  options: SearchOptions = {},
+): Promise<SearchResult> {
+  const {
+    num = 5,
+    info,
+    signal,
+    retainImages = "none",
+    withLinksSummary = false,
+  } = options;
+
+  const apiKey = resolveJinaApiKey();
+  if (!apiKey) {
+    throw new Error(
+      "Jina Search requires an API key. Run /md-web-surfer-config to set one, or set the JINA_API_KEY environment variable.",
+    );
+  }
+
+  const count = Math.min(Math.max(num, 1), 20);
+  const jinaRetain = retainImages === "alt_text" ? "alt_p" : retainImages;
+
+  const params = new URLSearchParams({
+    num: String(count),
+    respondWith: "markdown",
+    retainImages: jinaRetain,
+    withLinksSummary: String(withLinksSummary),
+  });
+  const url = `https://s.jina.ai/${encodeURIComponent(query)}?${params.toString()}`;
+
+  const response = await fetch(url, {
+    signal,
+    headers: {
+      Accept: "text/markdown",
+      Authorization: `Bearer ${apiKey}`,
+      "User-Agent": "Mozilla/5.0 (compatible; pi-agent/1.0)",
+    },
+  });
+
+  if (!response.ok) {
+    throw new Error(
+      `Jina Search failed: ${response.status} ${response.statusText}`,
+    );
+  }
+
+  let body = await response.text();
+  if (!body || body.slice(0, 300).match(/<!DOCTYPE|<html/i)) {
+    throw new Error("Jina Search returned empty or HTML response");
+  }
+
+  const truncated = truncateHead(body, {
+    maxLines: DEFAULT_MAX_LINES,
+    maxBytes: DEFAULT_MAX_BYTES,
+  });
+
+  let text = truncated.content;
+  const details: Record<string, unknown> = {
+    method: "jina-search",
+    query,
+    num: count,
+    retainImages,
+    withLinksSummary,
+  };
+
+  if (info) {
+    details.tokens = response.headers.get("x-markdown-tokens") ?? "unknown";
+  }
+
+  if (truncated.truncated) {
+    const tmp = join(tmpdir(), `fetch-search-${Date.now()}.md`);
+    await writeFile(tmp, body, "utf8");
+    text += `\n\n[Output truncated: ${truncated.outputLines} of ${truncated.totalLines} lines`;
+    text += ` (${formatSize(truncated.outputBytes)} of ${formatSize(truncated.totalBytes)}).`;
+    text += ` Full output saved to: ${tmp}]`;
+    details.fullOutputPath = tmp;
+  }
+
+  return { text, details };
+}