@aprimediet/webtools 1.0.0

package/README.md

@@ -0,0 +1,76 @@
+# @aprimediet/webtools
+
+Self-hosted **web tools** for the [pi coding agent](https://www.npmjs.com/package/@earendil-works/pi-coding-agent): fetch, extract, search, and crawl — using only npm/npx/docker backends. **No MCP, no hosted scraping APIs.** The engine is **Playwright**; extraction is **Mozilla Readability + Turndown**; search drives the browser over a public engine's HTML.
+
+## Tools
+
+| Tool | Params | Returns |
+|---|---|---|
+| `web_fetch` | `url`, `render?`(auto/static/browser), `format?`(markdown/text/html), `timeoutMs?`, `maxBytes?` | page content as markdown/text/html + `{finalUrl,status,title,bytes,renderUsed,truncated}` |
+| `web_extract` | `url`, `html?`, `render?`, `format?`(markdown/text), `includeMetadata?`, `selectors?`(name→CSS) | main article markdown + `{title,byline,siteName,publishedTime,lang,excerpt,fields}` |
+| `web_search` | `query`, `limit?`, `engine?`(duckduckgo/bing/brave) | `[{title,url,snippet}]` scraped from the engine's HTML |
+| `web_crawl` | `startUrl`, `maxPages?`(20, cap 100), `maxDepth?`(2), `sameOrigin?`, `include?`/`exclude?`, `render?`, `format?`, `concurrency?`(3, cap 5) | combined per-page markdown + `{pageCount,pages[]}` (streams progress) |
+
+`render:auto` fetches statically and **escalates to the headless browser** if the page looks like a JS shell. The browser is launched **once** and reused, then closed on session shutdown.
+
+## Setup (self-hosted backends)
+
+Third-party libs (`playwright`, `@mozilla/readability`, `jsdom`, `turndown`, `turndown-plugin-gfm`) install automatically with the package. **Playwright browser binaries do not** — install them once:
+
+```bash
+npx playwright install --with-deps chromium
+```
+
+Or point at a **Docker / remote browser** (zero local install) and set:
+
+```bash
+export PLAYWRIGHT_CDP=http://localhost:9222         # Chrome DevTools Protocol endpoint
+# or
+export PLAYWRIGHT_WS_ENDPOINT=ws://localhost:3000   # a Playwright server / browserless container
+```
+
+## Configuration (env + flags)
+
+| Env | Flag | Meaning |
+|---|---|---|
+| `WEBTOOLS_RENDER` | `--render-default` | default render mode (`auto`) |
+| `WEBTOOLS_SEARCH_ENGINE` | `--search-engine` | default engine (`duckduckgo`) |
+| `WEBTOOLS_USER_AGENT` | `--user-agent` | UA string |
+| `WEBTOOLS_ALLOW_PRIVATE` | `--allow-private` | allow private/loopback hosts (off by default) |
+| `WEBTOOLS_IGNORE_ROBOTS` | — | crawl ignores robots.txt (off by default) |
+| `PLAYWRIGHT_CDP` / `PLAYWRIGHT_WS_ENDPOINT` | — | connect to a remote/Docker browser |
+
+`/webtools` prints the current config and chosen browser backend.
+
+## Safety
+
+- **SSRF guard:** `web_fetch`/`web_extract`/`web_crawl` resolve the host and **block loopback/private/link-local ranges** (`127/8`, `10/8`, `172.16/12`, `192.168/16`, `169.254/16`, `::1`, `fc00::/7`, `*.local`) unless `WEBTOOLS_ALLOW_PRIVATE=1`.
+- **Caps:** per-request timeout (30s) and `maxBytes` (~5 MB); crawl caps per-page (30 KB) and total output (200 KB), honors robots.txt, rate-limits, and a concurrency cap.
+- **Search** scrapes a public engine's HTML, so it's best-effort: on a consent/captcha/changed page it **throws** a clear error (try another `engine`).
+
+## Install / run
+
+```bash
+pi install npm:@aprimediet/webtools
+pi list
+
+# Quick try without installing
+pi -e ./extensions/webtools/index.ts
+
+# Hot-reload during dev
+/reload
+```
+
+## Layout
+
+```
+webtools/                 # @aprimediet/webtools
+├── package.json          # pi manifest + deps
+├── index.ts              # factory: 4 tools + flags + /webtools + cleanup
+├── browser.ts            # SSRF guard + shared Playwright + fetchHtml
+├── extract.ts            # Readability + selectors
+├── search.ts             # Playwright SERP scraping (ddg/bing/brave)
+├── crawl.ts              # BFS crawler
+├── markdown.ts           # Turndown/jsdom HTML→markdown/text helpers
+└── config.ts             # env/flag-driven config
+```

package/browser.ts

@@ -0,0 +1,242 @@
+/**
+ * The fetch engine: SSRF guard + a shared (lazy) Playwright browser + `fetchHtml`
+ * with static / browser / auto render modes.
+ *
+ * Dependency-free of pi so it can be unit-tested directly with Node.
+ */
+
+import { isIP } from "node:net";
+import { lookup } from "node:dns/promises";
+import type { Browser } from "playwright";
+import { chromium } from "playwright";
+import { config, type RenderMode } from "./config.ts";
+
+// ---------------------------------------------------------------- SSRF guard
+
+function isPrivateV4(ip: string): boolean {
+	return (
+		/^127\./.test(ip) || // loopback
+		/^10\./.test(ip) || // private
+		/^192\.168\./.test(ip) || // private
+		/^169\.254\./.test(ip) || // link-local / metadata
+		/^172\.(1[6-9]|2\d|3[01])\./.test(ip) || // private
+		/^0\./.test(ip) || // this host
+		ip === "0.0.0.0"
+	);
+}
+
+function isPrivateV6(ip: string): boolean {
+	const s = ip.toLowerCase();
+	return (
+		s === "::1" || // loopback
+		s === "::" ||
+		s.startsWith("fe80:") || // link-local
+		s.startsWith("fc") || // unique-local fc00::/7
+		s.startsWith("fd") ||
+		s.startsWith("::ffff:127.") || // mapped loopback
+		s.startsWith("::ffff:10.") ||
+		s.startsWith("::ffff:192.168.") ||
+		s.startsWith("::ffff:169.254.")
+	);
+}
+
+export function isPrivateAddress(ip: string): boolean {
+	return isIP(ip) === 6 ? isPrivateV6(ip) : isPrivateV4(ip);
+}
+
+/** Throw unless `url` is a public http(s) URL (unless WEBTOOLS_ALLOW_PRIVATE). */
+export async function assertPublicUrl(url: string): Promise<URL> {
+	let u: URL;
+	try {
+		u = new URL(url);
+	} catch {
+		throw new Error(`Invalid URL: ${url}`);
+	}
+	if (u.protocol !== "http:" && u.protocol !== "https:") {
+		throw new Error(`Blocked non-http(s) URL: ${url}`);
+	}
+	if (config.allowPrivate) return u;
+
+	const host = u.hostname.replace(/^\[|\]$/g, "");
+	if (host === "localhost" || host.endsWith(".localhost") || host.endsWith(".local")) {
+		throw new Error(`Blocked local hostname "${host}". Set WEBTOOLS_ALLOW_PRIVATE=1 to override.`);
+	}
+
+	let ips: string[];
+	if (isIP(host)) {
+		ips = [host];
+	} else {
+		const resolved = await lookup(host, { all: true }).catch(() => [] as { address: string }[]);
+		ips = resolved.map((r) => r.address);
+	}
+	for (const ip of ips) {
+		if (isPrivateAddress(ip)) {
+			throw new Error(
+				`Blocked private/loopback address ${ip} for ${host}. Set WEBTOOLS_ALLOW_PRIVATE=1 to override.`,
+			);
+		}
+	}
+	return u;
+}
+
+// ------------------------------------------------------------ shared browser
+
+let browserPromise: Promise<Browser> | undefined;
+
+function launchBrowser(): Promise<Browser> {
+	if (config.cdp) return chromium.connectOverCDP(config.cdp);
+	if (config.wsEndpoint) return chromium.connect(config.wsEndpoint);
+	return chromium.launch({ headless: true });
+}
+
+export async function getBrowser(): Promise<Browser> {
+	if (!browserPromise) browserPromise = launchBrowser();
+	try {
+		return await browserPromise;
+	} catch (err) {
+		browserPromise = undefined;
+		throw new Error(
+			`Failed to start Playwright. Install browsers with "npx playwright install --with-deps chromium" or set PLAYWRIGHT_CDP. Cause: ${(err as Error).message}`,
+		);
+	}
+}
+
+export async function closeBrowser(): Promise<void> {
+	if (!browserPromise) return;
+	const p = browserPromise;
+	browserPromise = undefined;
+	const b = await p.catch(() => undefined);
+	if (b) await b.close().catch(() => {});
+}
+
+// ------------------------------------------------------------------- fetchHtml
+
+export interface FetchOptions {
+	render?: RenderMode;
+	timeoutMs?: number;
+	maxBytes?: number;
+}
+
+export interface FetchResult {
+	finalUrl: string;
+	status: number;
+	html: string;
+	contentType: string;
+	renderUsed: "static" | "browser";
+	truncated: boolean;
+	bytes: number;
+}
+
+function throwIfAborted(signal?: AbortSignal): void {
+	if (signal?.aborted) throw new Error("web fetch aborted");
+}
+
+/** A small/empty body or a known SPA mount point ⇒ escalate to the browser. */
+function looksLikeShell(html: string): boolean {
+	if (!html) return true;
+	const lower = html.toLowerCase();
+	const body = lower.match(/<body[^>]*>([\s\S]*?)<\/body>/)?.[1] ?? lower;
+	const text = body
+		.replace(/<script[\s\S]*?<\/script>/g, "")
+		.replace(/<style[\s\S]*?<\/style>/g, "")
+		.replace(/<[^>]+>/g, " ")
+		.replace(/\s+/g, " ")
+		.trim();
+	if (text.length < 200) return true;
+	return /<div id="(root|app|__next|__nuxt)">\s*<\/div>/.test(lower);
+}
+
+async function staticFetch(url: string, opts: Required<FetchOptions>, signal?: AbortSignal): Promise<FetchResult> {
+	throwIfAborted(signal);
+	const ctrl = new AbortController();
+	const onAbort = () => ctrl.abort();
+	signal?.addEventListener("abort", onAbort, { once: true });
+	const timer = setTimeout(() => ctrl.abort(), opts.timeoutMs);
+	try {
+		const res = await fetch(url, {
+			redirect: "follow",
+			signal: ctrl.signal,
+			headers: { "user-agent": config.userAgent, accept: "text/html,application/xhtml+xml,*/*" },
+		});
+		const contentType = res.headers.get("content-type") ?? "";
+		const reader = res.body?.getReader();
+		const decoder = new TextDecoder();
+		let html = "";
+		let bytes = 0;
+		let truncated = false;
+		if (reader) {
+			for (;;) {
+				const { done, value } = await reader.read();
+				if (done) break;
+				bytes += value.byteLength;
+				if (bytes > opts.maxBytes) {
+					html += decoder.decode(value.slice(0, Math.max(0, value.byteLength - (bytes - opts.maxBytes))));
+					truncated = true;
+					await reader.cancel().catch(() => {});
+					break;
+				}
+				html += decoder.decode(value, { stream: true });
+			}
+			if (!truncated) html += decoder.decode();
+		} else {
+			html = await res.text();
+			bytes = Buffer.byteLength(html);
+		}
+		return { finalUrl: res.url || url, status: res.status, html, contentType, renderUsed: "static", truncated, bytes };
+	} catch (err) {
+		if (signal?.aborted) throw new Error("web fetch aborted");
+		throw new Error(`static fetch failed for ${url}: ${(err as Error).message}`);
+	} finally {
+		clearTimeout(timer);
+		signal?.removeEventListener("abort", onAbort);
+	}
+}
+
+async function browserFetch(url: string, opts: Required<FetchOptions>, signal?: AbortSignal): Promise<FetchResult> {
+	throwIfAborted(signal);
+	const browser = await getBrowser();
+	const context = await browser.newContext({ userAgent: config.userAgent });
+	const page = await context.newPage();
+	try {
+		const response = await page.goto(url, { waitUntil: "domcontentloaded", timeout: opts.timeoutMs });
+		throwIfAborted(signal);
+		const html = await page.content();
+		let bytes = Buffer.byteLength(html);
+		let truncated = false;
+		let body = html;
+		if (bytes > opts.maxBytes) {
+			body = html.slice(0, opts.maxBytes);
+			bytes = Buffer.byteLength(body);
+			truncated = true;
+		}
+		return {
+			finalUrl: page.url(),
+			status: response?.status() ?? 0,
+			html: body,
+			contentType: response?.headers()["content-type"] ?? "text/html",
+			renderUsed: "browser",
+			truncated,
+			bytes,
+		};
+	} finally {
+		await page.close().catch(() => {});
+		await context.close().catch(() => {});
+	}
+}
+
+export async function fetchHtml(url: string, options: FetchOptions = {}, signal?: AbortSignal): Promise<FetchResult> {
+	await assertPublicUrl(url);
+	const opts: Required<FetchOptions> = {
+		render: options.render ?? config.renderDefault,
+		timeoutMs: options.timeoutMs ?? config.timeoutMs,
+		maxBytes: options.maxBytes ?? config.maxBytes,
+	};
+
+	if (opts.render === "static") return staticFetch(url, opts, signal);
+	if (opts.render === "browser") return browserFetch(url, opts, signal);
+
+	// auto: try static, escalate to the browser if it looks like a JS shell or non-HTML.
+	const s = await staticFetch(url, opts, signal).catch(() => undefined);
+	if (s && /html/i.test(s.contentType || "html") && !looksLikeShell(s.html)) return s;
+	return browserFetch(url, opts, signal);
+}

package/config.ts

@@ -0,0 +1,39 @@
+/**
+ * Shared runtime configuration for @aprimediet/webtools.
+ *
+ * Defaults come from environment variables (WEBTOOLS_* / PLAYWRIGHT_*); index.ts
+ * may override fields from CLI flags on session start. Kept dependency-free so the
+ * logic modules can be unit-tested outside pi.
+ */
+
+export type RenderMode = "auto" | "static" | "browser";
+export type SearchEngine = "duckduckgo" | "bing" | "brave";
+
+const truthy = (v: string | undefined): boolean => /^(1|true|yes|on)$/i.test(v ?? "");
+
+export const DEFAULT_USER_AGENT =
+	"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36 webtools-bot";
+
+export interface WebToolsConfig {
+	userAgent: string;
+	renderDefault: RenderMode;
+	searchEngine: SearchEngine;
+	allowPrivate: boolean;
+	ignoreRobots: boolean;
+	cdp: string;
+	wsEndpoint: string;
+	timeoutMs: number;
+	maxBytes: number;
+}
+
+export const config: WebToolsConfig = {
+	userAgent: process.env.WEBTOOLS_USER_AGENT || DEFAULT_USER_AGENT,
+	renderDefault: (process.env.WEBTOOLS_RENDER as RenderMode) || "auto",
+	searchEngine: (process.env.WEBTOOLS_SEARCH_ENGINE as SearchEngine) || "duckduckgo",
+	allowPrivate: truthy(process.env.WEBTOOLS_ALLOW_PRIVATE),
+	ignoreRobots: truthy(process.env.WEBTOOLS_IGNORE_ROBOTS),
+	cdp: process.env.PLAYWRIGHT_CDP || "",
+	wsEndpoint: process.env.PLAYWRIGHT_WS_ENDPOINT || "",
+	timeoutMs: 30_000,
+	maxBytes: 5 * 1024 * 1024,
+};

package/crawl.ts

@@ -0,0 +1,178 @@
+/**
+ * Breadth-first crawler built on `fetchHtml` + `extractArticle`.
+ * Same-origin by default, depth/page-capped, robots-aware, concurrency-limited.
+ */
+
+import { assertPublicUrl, fetchHtml, type FetchResult } from "./browser.ts";
+import { config, type RenderMode } from "./config.ts";
+import { extractArticle } from "./extract.ts";
+import { extractLinks } from "./markdown.ts";
+
+export interface CrawlOptions {
+	startUrl: string;
+	maxPages?: number;
+	maxDepth?: number;
+	sameOrigin?: boolean;
+	include?: string;
+	exclude?: string;
+	render?: RenderMode;
+	format?: "markdown" | "text";
+	concurrency?: number;
+}
+
+export interface CrawlPage {
+	url: string;
+	title?: string;
+	depth: number;
+	markdown: string;
+	bytes: number;
+}
+
+const HARD_MAX_PAGES = 100;
+const MAX_CONCURRENCY = 5;
+const TOTAL_OUTPUT_CAP = 200 * 1024;
+const PER_PAGE_CAP = 30 * 1024;
+const PER_ORIGIN_DELAY_MS = 250;
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/** Minimal robots.txt parser for `User-agent: *` Disallow rules. */
+function parseDisallow(robotsTxt: string): string[] {
+	const lines = robotsTxt.split(/\r?\n/);
+	const disallow: string[] = [];
+	let appliesToAll = false;
+	for (const line of lines) {
+		const clean = line.replace(/#.*$/, "").trim();
+		if (!clean) continue;
+		const [rawKey, ...rest] = clean.split(":");
+		const key = rawKey.toLowerCase().trim();
+		const value = rest.join(":").trim();
+		if (key === "user-agent") appliesToAll = value === "*";
+		else if (key === "disallow" && appliesToAll && value) disallow.push(value);
+	}
+	return disallow;
+}
+
+async function loadRobots(origin: string, signal?: AbortSignal): Promise<string[]> {
+	if (config.ignoreRobots) return [];
+	try {
+		const res = await fetchHtml(`${origin}/robots.txt`, { render: "static", maxBytes: 256 * 1024 }, signal);
+		return res.status >= 200 && res.status < 300 ? parseDisallow(res.html) : [];
+	} catch {
+		return [];
+	}
+}
+
+function isAllowed(url: string, disallow: string[]): boolean {
+	if (!disallow.length) return true;
+	const path = new URL(url).pathname;
+	return !disallow.some((rule) => path.startsWith(rule));
+}
+
+function matches(pattern: string | undefined, url: string): boolean {
+	if (!pattern) return false;
+	try {
+		return new RegExp(pattern).test(url);
+	} catch {
+		return url.includes(pattern);
+	}
+}
+
+export async function webCrawl(
+	options: CrawlOptions,
+	signal?: AbortSignal,
+	onProgress?: (pages: CrawlPage[]) => void,
+): Promise<{ pages: CrawlPage[]; markdown: string; truncated: boolean }> {
+	const start = await assertPublicUrl(options.startUrl);
+	const maxPages = Math.min(options.maxPages ?? 20, HARD_MAX_PAGES);
+	const maxDepth = options.maxDepth ?? 2;
+	const sameOrigin = options.sameOrigin ?? true;
+	const concurrency = Math.min(Math.max(options.concurrency ?? 3, 1), MAX_CONCURRENCY);
+	const render = options.render;
+	const format = options.format ?? "markdown";
+
+	const disallowByOrigin = new Map<string, string[]>();
+	disallowByOrigin.set(start.origin, await loadRobots(start.origin, signal));
+
+	const queue: { url: string; depth: number }[] = [{ url: start.toString(), depth: 0 }];
+	const visited = new Set<string>();
+	const pages: CrawlPage[] = [];
+
+	while (queue.length && pages.length < maxPages) {
+		if (signal?.aborted) throw new Error("web crawl aborted");
+
+		const batch: { url: string; depth: number }[] = [];
+		while (queue.length && batch.length < concurrency && pages.length + batch.length < maxPages) {
+			const item = queue.shift();
+			if (!item || visited.has(item.url)) continue;
+			visited.add(item.url);
+
+			const origin = new URL(item.url).origin;
+			if (!disallowByOrigin.has(origin)) disallowByOrigin.set(origin, await loadRobots(origin, signal));
+			if (!isAllowed(item.url, disallowByOrigin.get(origin) ?? [])) continue;
+
+			batch.push(item);
+		}
+		if (!batch.length) break;
+
+		const results = await Promise.all(
+			batch.map(async (item): Promise<{ page: CrawlPage; links: string[] } | null> => {
+				if (signal?.aborted) throw new Error("web crawl aborted");
+				let fetched: FetchResult;
+				try {
+					fetched = await fetchHtml(item.url, { render }, signal);
+				} catch {
+					return null;
+				}
+				let markdown = "";
+				let title: string | undefined;
+				try {
+					const ex = extractArticle(fetched.html, fetched.finalUrl, { format });
+					markdown = ex.content;
+					title = ex.meta.title;
+				} catch {
+					/* keep empty markdown */
+				}
+				if (markdown.length > PER_PAGE_CAP) markdown = `${markdown.slice(0, PER_PAGE_CAP)}\n\n…[page truncated]`;
+				const links = extractLinks(fetched.html, fetched.finalUrl);
+				return {
+					page: { url: fetched.finalUrl, title, depth: item.depth, markdown, bytes: Buffer.byteLength(markdown) },
+					links,
+				};
+			}),
+		);
+
+		for (const r of results) {
+			if (!r) continue;
+			pages.push(r.page);
+			onProgress?.(pages);
+			if (r.page.depth >= maxDepth) continue;
+			for (const link of r.links) {
+				if (visited.has(link)) continue;
+				try {
+					if (sameOrigin && new URL(link).origin !== start.origin) continue;
+				} catch {
+					continue;
+				}
+				if (options.include && !matches(options.include, link)) continue;
+				if (options.exclude && matches(options.exclude, link)) continue;
+				queue.push({ url: link, depth: r.page.depth + 1 });
+			}
+		}
+
+		await sleep(PER_ORIGIN_DELAY_MS);
+	}
+
+	// Assemble combined markdown, capped.
+	let markdown = "";
+	let truncated = false;
+	for (const p of pages) {
+		const section = `\n\n## ${p.title ?? p.url}\n${p.url}\n\n${p.markdown}\n`;
+		if (Buffer.byteLength(markdown) + Buffer.byteLength(section) > TOTAL_OUTPUT_CAP) {
+			truncated = true;
+			break;
+		}
+		markdown += section;
+	}
+	return { pages, markdown: markdown.trim(), truncated };
+}

package/extract.ts

@@ -0,0 +1,90 @@
+/**
+ * Readable-article extraction (Mozilla Readability) + optional CSS-selector fields.
+ *
+ * Dependency-free of pi so it can be unit-tested directly with Node.
+ */
+
+import { Readability } from "@mozilla/readability";
+import { JSDOM } from "jsdom";
+import { htmlToMarkdown, pageToMarkdown, pageToText } from "./markdown.ts";
+
+export interface ExtractOptions {
+	format?: "markdown" | "text";
+	includeMetadata?: boolean;
+	selectors?: Record<string, string>;
+}
+
+export interface ExtractMeta {
+	title?: string;
+	byline?: string;
+	siteName?: string;
+	publishedTime?: string;
+	lang?: string;
+	length?: number;
+	excerpt?: string;
+}
+
+export interface ExtractResult {
+	content: string;
+	meta: ExtractMeta;
+	fields?: Record<string, string | string[]>;
+	fallback: boolean;
+}
+
+function collectFields(doc: Document, selectors: Record<string, string>): Record<string, string | string[]> {
+	const fields: Record<string, string | string[]> = {};
+	for (const [name, sel] of Object.entries(selectors)) {
+		try {
+			const els = [...doc.querySelectorAll(sel)];
+			const vals = els.map((e) => (e.textContent ?? "").replace(/\s+/g, " ").trim()).filter(Boolean);
+			fields[name] = vals.length <= 1 ? (vals[0] ?? "") : vals;
+		} catch {
+			fields[name] = "";
+		}
+	}
+	return fields;
+}
+
+/**
+ * Extract the main article from `html`. Selectors are evaluated BEFORE Readability
+ * (which mutates/strips the DOM). Falls back to full-page conversion when the page
+ * is not article-like.
+ */
+export function extractArticle(html: string, url: string, options: ExtractOptions = {}): ExtractResult {
+	const format = options.format ?? "markdown";
+	const dom = new JSDOM(html, { url });
+	const doc = dom.window.document;
+
+	const fields = options.selectors ? collectFields(doc, options.selectors) : undefined;
+	const docLang = doc.documentElement.getAttribute("lang") ?? undefined;
+
+	let article: ReturnType<Readability["parse"]> = null;
+	try {
+		// Readability mutates the document, so clone for a clean parse.
+		article = new Readability(doc.cloneNode(true) as Document).parse();
+	} catch {
+		article = null;
+	}
+
+	if (article?.content) {
+		const content = format === "text" ? (article.textContent ?? "").trim() : htmlToMarkdown(article.content);
+		const meta: ExtractMeta =
+			options.includeMetadata === false
+				? {}
+				: {
+						title: article.title ?? undefined,
+						byline: article.byline ?? undefined,
+						siteName: article.siteName ?? undefined,
+						publishedTime: (article as { publishedTime?: string }).publishedTime,
+						lang: article.lang ?? docLang,
+						length: article.length ?? undefined,
+						excerpt: article.excerpt ?? undefined,
+					};
+		return { content, meta, fields, fallback: false };
+	}
+
+	// Fallback: non-article page → clean full-page conversion.
+	const content = format === "text" ? pageToText(html, url) : pageToMarkdown(html, url);
+	const meta: ExtractMeta = options.includeMetadata === false ? {} : { title: doc.title?.trim() || undefined, lang: docLang };
+	return { content, meta, fields, fallback: true };
+}

package/index.ts

@@ -0,0 +1,248 @@
+/**
+ * @aprimediet/webtools
+ *
+ * Self-hosted web tools for the pi coding agent — fetch, extract, search, crawl.
+ * Engine: Playwright (+ static fetch). Extraction: Mozilla Readability + Turndown.
+ * Search: Playwright scrapes a public engine's HTML. No MCP, no hosted APIs.
+ */
+
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { StringEnum } from "@earendil-works/pi-ai";
+import { Text } from "@earendil-works/pi-tui";
+import { Type } from "typebox";
+
+import { closeBrowser, fetchHtml } from "./browser.ts";
+import { config, type RenderMode, type SearchEngine } from "./config.ts";
+import { webCrawl } from "./crawl.ts";
+import { extractArticle } from "./extract.ts";
+import { extractTitle, pageToMarkdown, pageToText, sanitizeHtml } from "./markdown.ts";
+import { webSearch } from "./search.ts";
+
+const RENDER = StringEnum(["auto", "static", "browser"] as const);
+const FETCH_FORMAT = StringEnum(["markdown", "text", "html"] as const);
+const TEXT_FORMAT = StringEnum(["markdown", "text"] as const);
+const ENGINE = StringEnum(["duckduckgo", "bing", "brave"] as const);
+
+export default function webtoolsExtension(pi: ExtensionAPI): void {
+	// ----------------------------------------------------------- web_fetch
+	pi.registerTool({
+		name: "web_fetch",
+		label: "Web Fetch",
+		description:
+			"Fetch a URL and return its content as clean markdown, text, or html. Renders JavaScript when needed (render:auto/browser). Use for raw page content; prefer web_extract for article bodies.",
+		promptSnippet: "Fetch a web page as markdown/text/html",
+		parameters: Type.Object({
+			url: Type.String({ description: "The URL to fetch (http/https)" }),
+			render: Type.Optional(RENDER),
+			format: Type.Optional(FETCH_FORMAT),
+			timeoutMs: Type.Optional(Type.Number({ description: "Per-request timeout (ms)" })),
+			maxBytes: Type.Optional(Type.Number({ description: "Max bytes to read" })),
+		}),
+		async execute(_id, params, signal) {
+			const res = await fetchHtml(
+				params.url,
+				{ render: params.render as RenderMode, timeoutMs: params.timeoutMs, maxBytes: params.maxBytes },
+				signal,
+			);
+			const format = params.format ?? "markdown";
+			const text =
+				format === "html"
+					? sanitizeHtml(res.html, res.finalUrl)
+					: format === "text"
+						? pageToText(res.html, res.finalUrl)
+						: pageToMarkdown(res.html, res.finalUrl);
+			const title = extractTitle(res.html, res.finalUrl);
+			return {
+				content: [{ type: "text" as const, text: text || "(empty document)" }],
+				details: {
+					finalUrl: res.finalUrl,
+					status: res.status,
+					title,
+					contentType: res.contentType,
+					renderUsed: res.renderUsed,
+					bytes: res.bytes,
+					truncated: res.truncated,
+				},
+			};
+		},
+		renderResult(result, _opts, theme) {
+			const d = result.details as { title?: string; finalUrl?: string; bytes?: number; renderUsed?: string };
+			return new Text(
+				`${theme.fg("toolTitle", theme.bold("web_fetch "))}${theme.fg("muted", d.title ?? d.finalUrl ?? "")} ${theme.fg("dim", `(${d.bytes ?? 0}b · ${d.renderUsed ?? "?"})`)}`,
+				0,
+				0,
+			);
+		},
+	});
+
+	// --------------------------------------------------------- web_extract
+	pi.registerTool({
+		name: "web_extract",
+		label: "Web Extract",
+		description:
+			"Extract the main readable article (content + title/byline/date/lang metadata) from a URL or supplied HTML. Optionally pull structured fields via CSS selectors. Deterministic — no LLM, no service.",
+		promptSnippet: "Extract the main article + metadata from a page",
+		parameters: Type.Object({
+			url: Type.String({ description: "The URL to extract from" }),
+			html: Type.Optional(Type.String({ description: "Extract from this HTML instead of fetching" })),
+			render: Type.Optional(RENDER),
+			format: Type.Optional(TEXT_FORMAT),
+			includeMetadata: Type.Optional(Type.Boolean({ description: "Include title/byline/etc. (default true)" })),
+			selectors: Type.Optional(
+				Type.Record(Type.String(), Type.String(), {
+					description: "Map of field name → CSS selector for structured extraction",
+				}),
+			),
+		}),
+		async execute(_id, params, signal) {
+			let html = params.html;
+			let finalUrl = params.url;
+			if (!html) {
+				const res = await fetchHtml(params.url, { render: params.render as RenderMode }, signal);
+				html = res.html;
+				finalUrl = res.finalUrl;
+			}
+			const ex = extractArticle(html, finalUrl, {
+				format: params.format as "markdown" | "text" | undefined,
+				includeMetadata: params.includeMetadata,
+				selectors: params.selectors as Record<string, string> | undefined,
+			});
+			const note = ex.fallback ? "\n\n_(no article detected — full-page extraction)_" : "";
+			return {
+				content: [{ type: "text" as const, text: (ex.content || "(no content)") + note }],
+				details: {
+					url: finalUrl,
+					...ex.meta,
+					fields: ex.fields,
+					fallback: ex.fallback,
+				},
+			};
+		},
+		renderResult(result, _opts, theme) {
+			const d = result.details as { title?: string; url?: string; fallback?: boolean };
+			return new Text(
+				`${theme.fg("toolTitle", theme.bold("web_extract "))}${theme.fg("muted", d.title ?? d.url ?? "")}${d.fallback ? theme.fg("dim", " (full-page)") : ""}`,
+				0,
+				0,
+			);
+		},
+	});
+
+	// ---------------------------------------------------------- web_search
+	pi.registerTool({
+		name: "web_search",
+		label: "Web Search",
+		description:
+			"Search the web by driving a headless browser over a public search engine's HTML results. Returns a list of {title, url, snippet}. Use to find URLs, then web_fetch/web_extract them.",
+		promptSnippet: "Search the web for URLs",
+		parameters: Type.Object({
+			query: Type.String({ description: "The search query" }),
+			limit: Type.Optional(Type.Number({ description: "Max results (default 10)" })),
+			engine: Type.Optional(ENGINE),
+		}),
+		async execute(_id, params, signal) {
+			const engine = (params.engine as SearchEngine) ?? config.searchEngine;
+			const limit = Math.max(1, Math.min(params.limit ?? 10, 25));
+			const { results } = await webSearch(params.query, limit, engine, signal);
+			const text = results.map((r, i) => `${i + 1}. [${r.title}](${r.url})${r.snippet ? ` — ${r.snippet}` : ""}`).join("\n");
+			return {
+				content: [{ type: "text" as const, text }],
+				details: { engine, query: params.query, results },
+			};
+		},
+		renderResult(result, _opts, theme) {
+			const d = result.details as { engine?: string; results?: unknown[] };
+			return new Text(
+				`${theme.fg("toolTitle", theme.bold("web_search "))}${theme.fg("muted", `${d.results?.length ?? 0} results`)} ${theme.fg("dim", `via ${d.engine ?? "?"}`)}`,
+				0,
+				0,
+			);
+		},
+	});
+
+	// ----------------------------------------------------------- web_crawl
+	pi.registerTool({
+		name: "web_crawl",
+		label: "Web Crawl",
+		description:
+			"Breadth-first crawl from a start URL (same-origin by default), extracting each page to markdown. Depth/page-capped, robots-aware, concurrency-limited. Use to ingest a docs section.",
+		promptSnippet: "Crawl a site section and extract pages",
+		parameters: Type.Object({
+			startUrl: Type.String({ description: "The URL to start crawling from" }),
+			maxPages: Type.Optional(Type.Number({ description: "Max pages (default 20, cap 100)" })),
+			maxDepth: Type.Optional(Type.Number({ description: "Max link depth (default 2)" })),
+			sameOrigin: Type.Optional(Type.Boolean({ description: "Restrict to the start origin (default true)" })),
+			include: Type.Optional(Type.String({ description: "Only crawl URLs matching this regex/substring" })),
+			exclude: Type.Optional(Type.String({ description: "Skip URLs matching this regex/substring" })),
+			render: Type.Optional(RENDER),
+			format: Type.Optional(TEXT_FORMAT),
+			concurrency: Type.Optional(Type.Number({ description: "Parallel fetches (default 3, cap 5)" })),
+		}),
+		async execute(_id, params, signal, onUpdate) {
+			const { pages, markdown, truncated } = await webCrawl(
+				params as Parameters<typeof webCrawl>[0],
+				signal,
+				(p) =>
+					onUpdate?.({
+						content: [{ type: "text", text: `crawled ${p.length} page(s)…` }],
+						details: { pages: p.map((x) => ({ url: x.url, title: x.title, depth: x.depth, bytes: x.bytes })) },
+					}),
+			);
+			const header = `Crawled ${pages.length} page(s) from ${params.startUrl}${truncated ? " (output truncated)" : ""}.\n`;
+			return {
+				content: [{ type: "text" as const, text: header + markdown }],
+				details: {
+					startUrl: params.startUrl,
+					pageCount: pages.length,
+					truncated,
+					pages: pages.map((p) => ({ url: p.url, title: p.title, depth: p.depth, bytes: p.bytes })),
+				},
+			};
+		},
+		renderResult(result, _opts, theme) {
+			const d = result.details as { pageCount?: number; startUrl?: string };
+			return new Text(
+				`${theme.fg("toolTitle", theme.bold("web_crawl "))}${theme.fg("muted", `${d.pageCount ?? 0} pages`)} ${theme.fg("dim", d.startUrl ?? "")}`,
+				0,
+				0,
+			);
+		},
+	});
+
+	// ------------------------------------------------------------- flags
+	pi.registerFlag("render-default", { description: "Default render mode: auto, static, browser", type: "string" });
+	pi.registerFlag("search-engine", { description: "Default search engine: duckduckgo, bing, brave", type: "string" });
+	pi.registerFlag("user-agent", { description: "HTTP/browser User-Agent string", type: "string" });
+	pi.registerFlag("allow-private", { description: "Allow fetching private/loopback addresses (SSRF)", type: "boolean" });
+
+	function applyFlags(): void {
+		const render = pi.getFlag("render-default");
+		if (typeof render === "string" && render) config.renderDefault = render as RenderMode;
+		const engine = pi.getFlag("search-engine");
+		if (typeof engine === "string" && engine) config.searchEngine = engine as SearchEngine;
+		const ua = pi.getFlag("user-agent");
+		if (typeof ua === "string" && ua) config.userAgent = ua;
+		if (pi.getFlag("allow-private") === true) config.allowPrivate = true;
+	}
+
+	pi.on("session_start", async () => applyFlags());
+
+	// ------------------------------------------------------- /webtools cmd
+	pi.registerCommand("webtools", {
+		description: "Show webtools config and backend availability",
+		handler: async (_args, ctx: ExtensionContext) => {
+			const lines = [
+				`render default : ${config.renderDefault}`,
+				`search engine  : ${config.searchEngine}`,
+				`user agent     : ${config.userAgent}`,
+				`allow private  : ${config.allowPrivate}`,
+				`ignore robots  : ${config.ignoreRobots}`,
+				`browser        : ${config.cdp ? `CDP ${config.cdp}` : config.wsEndpoint ? `WS ${config.wsEndpoint}` : "local Playwright (npx playwright install chromium)"}`,
+			];
+			if (ctx.hasUI) ctx.ui.notify(lines.join("\n"), "info");
+		},
+	});
+
+	// --------------------------------------------------------- cleanup
+	pi.on("session_shutdown", async () => closeBrowser());
+}

package/markdown.ts

@@ -0,0 +1,99 @@
+/**
+ * HTML → Markdown / text helpers (Turndown + jsdom).
+ *
+ * Dependency-free of pi so it can be unit-tested directly with Node.
+ */
+
+import { JSDOM } from "jsdom";
+import TurndownService from "turndown";
+// @ts-expect-error - no bundled types for the gfm plugin
+import { gfm } from "turndown-plugin-gfm";
+
+let turndown: TurndownService | undefined;
+
+function getTurndown(): TurndownService {
+	if (!turndown) {
+		turndown = new TurndownService({
+			headingStyle: "atx",
+			codeBlockStyle: "fenced",
+			bulletListMarker: "-",
+		});
+		try {
+			turndown.use(gfm); // GFM tables / strikethrough / task lists
+		} catch {
+			/* plugin optional */
+		}
+		// Drop noise outright.
+		turndown.remove(["script", "style", "noscript", "iframe", "svg", "head"]);
+	}
+	return turndown;
+}
+
+/** Convert an HTML fragment (e.g. Readability output) to Markdown. */
+export function htmlToMarkdown(html: string): string {
+	if (!html) return "";
+	return getTurndown().turndown(html).trim();
+}
+
+function stripNoise(doc: Document): void {
+	for (const sel of ["script", "style", "noscript", "iframe", "svg", "template"]) {
+		for (const el of [...doc.querySelectorAll(sel)]) el.remove();
+	}
+}
+
+/** Full-page HTML → Markdown: strip scripts/styles, convert the body. */
+export function pageToMarkdown(html: string, url: string): string {
+	const doc = new JSDOM(html, { url }).window.document;
+	stripNoise(doc);
+	const body = doc.body ?? doc.documentElement;
+	return htmlToMarkdown(body.innerHTML);
+}
+
+/** Full-page HTML → collapsed plain text. */
+export function pageToText(html: string, url: string): string {
+	const doc = new JSDOM(html, { url }).window.document;
+	stripNoise(doc);
+	const body = doc.body ?? doc.documentElement;
+	return (body.textContent ?? "").replace(/\s+/g, " ").trim();
+}
+
+/** Sanitized HTML: scripts/styles removed, body markup returned. */
+export function sanitizeHtml(html: string, url: string): string {
+	const doc = new JSDOM(html, { url }).window.document;
+	stripNoise(doc);
+	const body = doc.body ?? doc.documentElement;
+	return body.innerHTML.trim();
+}
+
+/** Best-effort <title>. */
+export function extractTitle(html: string, url: string): string | undefined {
+	try {
+		const doc = new JSDOM(html, { url }).window.document;
+		return doc.title?.trim() || undefined;
+	} catch {
+		return undefined;
+	}
+}
+
+/** Same-origin/absolute links found in the page (deduped, hash-stripped). */
+export function extractLinks(html: string, baseUrl: string): string[] {
+	const out = new Set<string>();
+	try {
+		const doc = new JSDOM(html, { url: baseUrl }).window.document;
+		for (const a of [...doc.querySelectorAll("a[href]")]) {
+			const raw = a.getAttribute("href");
+			if (!raw || raw.startsWith("#") || raw.startsWith("mailto:") || raw.startsWith("javascript:")) continue;
+			try {
+				const u = new URL(raw, baseUrl);
+				if (u.protocol !== "http:" && u.protocol !== "https:") continue;
+				u.hash = "";
+				out.add(u.toString());
+			} catch {
+				/* skip bad href */
+			}
+		}
+	} catch {
+		/* skip */
+	}
+	return [...out];
+}

package/package.json

@@ -0,0 +1,25 @@
+{
+  "name": "@aprimediet/webtools",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "Self-hosted web fetch / extract / search / crawl tools for the pi coding agent (Playwright + Readability, no MCP).",
+  "keywords": ["pi-package"],
+  "pi": {
+    "extensions": ["./index.ts"]
+  },
+  "files": ["*.ts", "README.md"],
+  "dependencies": {
+    "playwright": "^1.48.0",
+    "@mozilla/readability": "^0.5.0",
+    "jsdom": "^25.0.0",
+    "turndown": "^7.2.0",
+    "turndown-plugin-gfm": "^1.0.2"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-agent-core": "*",
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-tui": "*",
+    "typebox": "*"
+  }
+}

package/search.ts

@@ -0,0 +1,141 @@
+/**
+ * Playwright-only web search: drive the shared headless browser over a public
+ * search engine's HTML results and scrape the rows. No search API, no service.
+ */
+
+import { config, type SearchEngine } from "./config.ts";
+import { getBrowser } from "./browser.ts";
+
+export interface SearchHit {
+	title: string;
+	url: string;
+	snippet: string;
+}
+
+interface EngineSpec {
+	url: (q: string) => string;
+	row: string;
+	titleSel: string;
+	snippetSel: string;
+	/** Some engines wrap result hrefs in a redirect; how to recover the real URL. */
+	unwrap?: "ddg" | "bing";
+}
+
+// Keep selectors in one table so they're trivial to update when a SERP changes.
+const ENGINES: Record<SearchEngine, EngineSpec> = {
+	duckduckgo: {
+		url: (q) => `https://html.duckduckgo.com/html/?q=${encodeURIComponent(q)}`,
+		row: ".result",
+		titleSel: "a.result__a",
+		snippetSel: ".result__snippet",
+		unwrap: "ddg",
+	},
+	bing: {
+		url: (q) => `https://www.bing.com/search?q=${encodeURIComponent(q)}`,
+		row: "li.b_algo",
+		titleSel: "h2 a",
+		snippetSel: ".b_caption p",
+		unwrap: "bing",
+	},
+	brave: {
+		url: (q) => `https://search.brave.com/search?q=${encodeURIComponent(q)}`,
+		row: "[data-type=web]",
+		titleSel: "a",
+		snippetSel: ".snippet-description",
+	},
+};
+
+function unwrapDdg(href: string): string {
+	try {
+		const u = new URL(href, "https://duckduckgo.com");
+		const real = u.searchParams.get("uddg");
+		return real ? real : u.toString();
+	} catch {
+		return href;
+	}
+}
+
+/** Bing wraps results in `bing.com/ck/a?...&u=a1<base64url(realUrl)>`. */
+function unwrapBing(href: string): string {
+	try {
+		const u = new URL(href, "https://www.bing.com");
+		const param = u.searchParams.get("u");
+		if (param) {
+			const enc = param.startsWith("a1") ? param.slice(2) : param;
+			const decoded = Buffer.from(enc.replace(/-/g, "+").replace(/_/g, "/"), "base64").toString("utf8");
+			if (/^https?:\/\//.test(decoded)) return decoded;
+		}
+		return u.toString();
+	} catch {
+		return href;
+	}
+}
+
+function unwrapHref(kind: EngineSpec["unwrap"], href: string): string {
+	if (kind === "ddg") return unwrapDdg(href);
+	if (kind === "bing") return unwrapBing(href);
+	return href;
+}
+
+export async function webSearch(
+	query: string,
+	limit: number,
+	engineName: SearchEngine,
+	signal?: AbortSignal,
+): Promise<{ engine: SearchEngine; results: SearchHit[] }> {
+	const spec = ENGINES[engineName];
+	if (!spec) throw new Error(`Unknown search engine: ${engineName}`);
+
+	const browser = await getBrowser();
+	const context = await browser.newContext({ userAgent: config.userAgent });
+	const page = await context.newPage();
+	try {
+		await page.goto(spec.url(query), { waitUntil: "domcontentloaded", timeout: config.timeoutMs });
+		if (signal?.aborted) throw new Error("web search aborted");
+
+		const raw = (await page
+			.$$eval(
+				spec.row,
+				(rows, sel) =>
+					rows.map((el) => {
+						const a = el.querySelector(sel.titleSel) as HTMLAnchorElement | null;
+						const snip = el.querySelector(sel.snippetSel);
+						return {
+							title: (a?.textContent ?? "").replace(/\s+/g, " ").trim(),
+							href: a?.getAttribute("href") ?? "",
+							snippet: (snip?.textContent ?? "").replace(/\s+/g, " ").trim(),
+						};
+					}),
+				{ titleSel: spec.titleSel, snippetSel: spec.snippetSel },
+			)
+			.catch(() => [] as { title: string; href: string; snippet: string }[])) as {
+			title: string;
+			href: string;
+			snippet: string;
+		}[];
+
+		const results: SearchHit[] = [];
+		for (const r of raw) {
+			if (!r.href || !r.title) continue;
+			let url = unwrapHref(spec.unwrap, r.href);
+			try {
+				url = new URL(url, spec.url(query)).toString();
+			} catch {
+				continue;
+			}
+			if (!/^https?:/.test(url)) continue;
+			results.push({ title: r.title, url, snippet: r.snippet });
+			if (results.length >= limit) break;
+		}
+
+		if (results.length === 0) {
+			throw new Error(
+				`web_search: no results from "${engineName}". The page may be a consent/captcha/"unusual traffic" page, or its markup changed. Try engine:"bing" or "brave".`,
+			);
+		}
+		return { engine: engineName, results };
+	} finally {
+		await page.close().catch(() => {});
+		await context.close().catch(() => {});
+	}
+}