@rjshrjndrn/pi-fetch 0.1.0

package/Makefile

@@ -0,0 +1,63 @@
+.PHONY: install test typecheck build publish patch minor major clean help
+
+# ── Config ───────────────────────────────────────────────────────────────────
+
+PACKAGE_NAME := $(shell node -p "require('./package.json').name" 2>/dev/null)
+VERSION      := $(shell node -p "require('./package.json').version" 2>/dev/null)
+
+# ── Default ──────────────────────────────────────────────────────────────────
+
+help:
+	@echo "$(PACKAGE_NAME) v$(VERSION)"
+	@echo ""
+	@echo "  make install     Install dependencies"
+	@echo "  make test        Run tests"
+	@echo "  make typecheck   Type-check TypeScript (no emit)"
+	@echo "  make build       install + typecheck + test"
+	@echo "  make patch       Bump patch version and publish"
+	@echo "  make minor       Bump minor version and publish"
+	@echo "  make major       Bump major version and publish"
+	@echo "  make publish     Publish current version to npm"
+	@echo "  make clean       Remove node_modules"
+
+# ── Core tasks ───────────────────────────────────────────────────────────────
+
+install:
+	npm install
+
+test:
+	npm test
+
+typecheck:
+	npx tsc --noEmit --strict --skipLibCheck --moduleResolution bundler --module esnext \
+		--target esnext --allowImportingTsExtensions \
+		extensions/index.ts $(wildcard src/*.ts)
+
+build: install typecheck test
+	@echo "✓ Build complete — $(PACKAGE_NAME) v$(VERSION)"
+
+# ── Publishing ───────────────────────────────────────────────────────────────
+
+publish: clean build
+	npm publish --access public
+	@echo "✓ Published $(PACKAGE_NAME) v$(VERSION)"
+
+patch: clean build
+	npm version patch
+	npm publish --access public
+	@echo "✓ Published $(PACKAGE_NAME) v$(shell node -p "require('./package.json').version")"
+
+minor: clean build
+	npm version minor
+	npm publish --access public
+	@echo "✓ Published $(PACKAGE_NAME) v$(shell node -p "require('./package.json').version")"
+
+major: clean build
+	npm version major
+	npm publish --access public
+	@echo "✓ Published $(PACKAGE_NAME) v$(shell node -p "require('./package.json').version")"
+
+# ── Cleanup ──────────────────────────────────────────────────────────────────
+
+clean:
+	rm -rf node_modules

package/README.md

@@ -0,0 +1,73 @@
+# pi-fetch
+
+Web content extraction for [pi](https://github.com/badlogic/pi-mono). Fetch any URL as clean Markdown — no headless browser required.
+
+Powered by [Defuddle](https://github.com/kepano/defuddle) by [Steph Ango](https://github.com/kepano) (creator of [Obsidian Web Clipper](https://obsidian.md/clipper)).
+
+## How it works
+
+Registers a `web_fetch` tool that the LLM can call with any URL. Under the hood:
+
+1. **[JSDOM](https://github.com/jsdom/jsdom)** fetches and parses the HTML (lightweight, no browser engine)
+2. **[Defuddle](https://github.com/kepano/defuddle)** extracts the main content, stripping navigation, ads, sidebars, cookie banners, and clutter
+3. Returns clean **Markdown** with metadata (title, author, description, word count)
+
+```
+web_fetch https://www.npmjs.com/package/defuddle
+
+# defuddle
+
+**Author:** kepano · **Domain:** npmjs.com · **Words:** 342
+
+> Get the main content of any page as Markdown.
+
+---
+
+Defuddle extracts the main content from web pages...
+```
+
+Output is automatically truncated to stay within pi's context limits.
+
+## Install
+
+```bash
+# As a pi package
+pi install npm:pi-fetch
+
+# Or test locally
+pi -e ./extensions/index.ts
+```
+
+## Usage
+
+Once installed, just ask pi to fetch a URL:
+
+```
+fetch https://docs.example.com/getting-started
+```
+
+Or the LLM will use `web_fetch` automatically when it needs to read a webpage.
+
+## Limitations
+
+- **No JavaScript rendering** — JSDOM does not execute JavaScript. SPAs that require JS to render content will return empty or minimal results. For those, you'll still need a headless browser.
+- **Some sites block non-browser requests** — sites with aggressive bot detection may reject the request.
+- **Output truncation** — very large pages are truncated to 50KB / 2000 lines to protect context window.
+
+## Development
+
+```bash
+cd pi-fetch
+npm install
+npm test
+```
+
+## Credits
+
+- **[Defuddle](https://github.com/kepano/defuddle)** by [Steph Ango (kepano)](https://github.com/kepano) — content extraction engine
+- **[JSDOM](https://github.com/jsdom/jsdom)** — HTML parsing without a browser
+- **[pi](https://github.com/badlogic/pi-mono)** by [Mario Zechner (badlogic)](https://github.com/badlogic) — agent framework
+
+## License
+
+MIT

package/extensions/index.ts

@@ -0,0 +1,55 @@
+import type { ExtensionAPI } from "@mariozechner/pi-coding-agent"
+import {
+  truncateHead,
+  DEFAULT_MAX_BYTES,
+  DEFAULT_MAX_LINES,
+  formatSize,
+} from "@mariozechner/pi-coding-agent"
+import { Type } from "@sinclair/typebox"
+import { fetchPage, formatResult } from "../src/fetch.js"
+
+export default function (pi: ExtensionAPI) {
+  pi.registerTool({
+    name: "web_fetch",
+    label: "Web Fetch",
+    description:
+      "Fetch a webpage and extract its main content as clean Markdown. " +
+      "Strips navigation, ads, sidebars, and clutter. " +
+      "Returns title, author, description, word count, and the article content.",
+    promptSnippet:
+      "Fetch a URL and return its readable content as clean Markdown",
+    promptGuidelines: [
+      "Use web_fetch when the user provides a URL to read, analyze, or summarize.",
+      "Prefer web_fetch over bash curl for reading webpage content — it extracts clean text and saves tokens.",
+    ],
+    parameters: Type.Object({
+      url: Type.String({ description: "URL of the webpage to fetch" }),
+    }),
+
+    async execute(_toolCallId, params, signal, _onUpdate, _ctx) {
+      const result = await fetchPage(params.url, signal ?? undefined)
+      const output = formatResult(result)
+
+      // Truncate if output exceeds context limits
+      const truncation = truncateHead(output, {
+        maxLines: DEFAULT_MAX_LINES,
+        maxBytes: DEFAULT_MAX_BYTES,
+      })
+
+      let text = truncation.content
+      if (truncation.truncated) {
+        text += `\n\n[Truncated: showing ${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}]`
+      }
+
+      return {
+        content: [{ type: "text", text }],
+        details: {
+          url: result.url,
+          title: result.title,
+          wordCount: result.wordCount,
+          truncated: truncation.truncated,
+        },
+      }
+    },
+  })
+}

package/package.json

@@ -0,0 +1,36 @@
+{
+  "name": "@rjshrjndrn/pi-fetch",
+  "version": "0.1.0",
+  "description": "Web content extraction for pi — fetch any URL as clean Markdown using Defuddle",
+  "keywords": [
+    "pi-package",
+    "pi-extension",
+    "web-fetch",
+    "markdown",
+    "defuddle",
+    "readability"
+  ],
+  "license": "MIT",
+  "type": "module",
+  "pi": {
+    "extensions": [
+      "./extensions"
+    ]
+  },
+  "dependencies": {
+    "defuddle": "^0.14.0"
+  },
+  "peerDependencies": {
+    "@mariozechner/pi-coding-agent": "*",
+    "@sinclair/typebox": "*"
+  },
+  "devDependencies": {
+    "@types/node": "^20.0.0",
+    "typescript": "^5.4.0",
+    "vitest": "^1.6.0"
+  },
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}

package/src/fetch.ts

@@ -0,0 +1,87 @@
+export interface FetchResult {
+  url: string
+  title: string | null
+  author: string | null
+  description: string | null
+  domain: string | null
+  published: string | null
+  wordCount: number | null
+  content: string
+}
+
+const USER_AGENT =
+  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36"
+
+/**
+ * Fetch a URL and extract its main content as Markdown via Defuddle.
+ *
+ * Uses native fetch + Defuddle's Node API (no headless browser) to
+ * strip navigation, ads, sidebars, and clutter.
+ */
+export async function fetchPage(
+  url: string,
+  signal?: AbortSignal
+): Promise<FetchResult> {
+  const { Defuddle } = await import("defuddle/node")
+
+  let html: string
+  try {
+    const response = await fetch(url, {
+      signal,
+      headers: { "User-Agent": USER_AGENT },
+    })
+    if (!response.ok) {
+      throw new Error(`HTTP ${response.status} ${response.statusText}`)
+    }
+    html = await response.text()
+  } catch (err: any) {
+    if (err?.name === "AbortError") throw err
+    throw new Error(`Failed to fetch ${url}: ${err?.message ?? err}`)
+  }
+
+  const result = await Defuddle(html, url, { markdown: true })
+
+  return {
+    url,
+    title: result.title ?? null,
+    author: result.author ?? null,
+    description: result.description ?? null,
+    domain: result.domain ?? null,
+    published: result.published ?? null,
+    wordCount: result.wordCount ?? null,
+    content: result.content ?? "",
+  }
+}
+
+/**
+ * Format a FetchResult into a human/LLM-readable Markdown string.
+ */
+export function formatResult(result: FetchResult): string {
+  const lines: string[] = []
+
+  if (result.title) lines.push(`# ${result.title}`)
+  const meta: string[] = []
+  if (result.author) meta.push(`**Author:** ${result.author}`)
+  if (result.domain) meta.push(`**Domain:** ${result.domain}`)
+  if (result.published) meta.push(`**Published:** ${result.published}`)
+  if (result.wordCount) meta.push(`**Words:** ${result.wordCount}`)
+  if (meta.length > 0) {
+    lines.push("")
+    lines.push(meta.join(" · "))
+  }
+
+  if (result.description) {
+    lines.push("")
+    lines.push(`> ${result.description}`)
+  }
+
+  if (lines.length > 0) {
+    lines.push("")
+    lines.push("---")
+  }
+
+  lines.push("")
+  lines.push(result.content || "(no content extracted)")
+
+  return lines.join("\n")
+}

package/tests/fetch.test.ts

@@ -0,0 +1,82 @@
+import { describe, it, expect } from "vitest"
+import { formatResult, type FetchResult } from "../src/fetch.js"
+
+describe("formatResult", () => {
+  it("formats a full result with all metadata", () => {
+    const result: FetchResult = {
+      url: "https://example.com/article",
+      title: "Test Article",
+      author: "Jane Doe",
+      description: "A test article about testing",
+      domain: "example.com",
+      published: "2026-01-15",
+      wordCount: 500,
+      content: "This is the article body.",
+    }
+
+    const output = formatResult(result)
+    expect(output).toContain("# Test Article")
+    expect(output).toContain("**Author:** Jane Doe")
+    expect(output).toContain("**Domain:** example.com")
+    expect(output).toContain("**Published:** 2026-01-15")
+    expect(output).toContain("**Words:** 500")
+    expect(output).toContain("> A test article about testing")
+    expect(output).toContain("---")
+    expect(output).toContain("This is the article body.")
+  })
+
+  it("handles minimal result with no metadata", () => {
+    const result: FetchResult = {
+      url: "https://example.com",
+      title: null,
+      author: null,
+      description: null,
+      domain: null,
+      published: null,
+      wordCount: null,
+      content: "Just content.",
+    }
+
+    const output = formatResult(result)
+    expect(output).not.toContain("#")
+    expect(output).not.toContain("**Author:**")
+    expect(output).not.toContain("---")
+    expect(output).toContain("Just content.")
+  })
+
+  it("handles empty content", () => {
+    const result: FetchResult = {
+      url: "https://example.com",
+      title: "Empty Page",
+      author: null,
+      description: null,
+      domain: null,
+      published: null,
+      wordCount: null,
+      content: "",
+    }
+
+    const output = formatResult(result)
+    expect(output).toContain("# Empty Page")
+    expect(output).toContain("(no content extracted)")
+  })
+
+  it("includes only available metadata fields", () => {
+    const result: FetchResult = {
+      url: "https://example.com",
+      title: "Partial",
+      author: "Author Only",
+      description: null,
+      domain: null,
+      published: null,
+      wordCount: 100,
+      content: "Body text.",
+    }
+
+    const output = formatResult(result)
+    expect(output).toContain("**Author:** Author Only")
+    expect(output).toContain("**Words:** 100")
+    expect(output).not.toContain("**Domain:**")
+    expect(output).not.toContain("**Published:**")
+  })
+})