@pi-lab/webfetch 1.0.0 → 1.0.1

package/README.md

@@ -16,3 +16,18 @@ pi install npm:@pi-lab/webfetch
 - **Redirect handling** — same-domain redirects are followed automatically (up to 10 hops); cross-domain redirects are surfaced to the model so it can decide whether to follow.
 - **Binary downloads** — non-text responses (PDFs, images, etc.) are saved to `~/.pi/agent/pi-lab/tmp/webfetch/` and the file path is returned.
 - **LRU cache** — processed Markdown is cached in memory so paginating the same URL doesn't re-fetch.
+- **Built-in fetch optimizations** — enabled by default. Site-specific rules can rewrite or parse difficult pages before generic extraction. Reddit links are rewritten to `old.reddit.com`; X/Twitter posts are extracted from the page's `window.__INITIAL_STATE__` script and formatted as clean Markdown.
+
+## Configuration
+
+Disable the built-in optimization framework in pi settings:
+
+```json
+{
+  "webfetch": {
+    "optimizations": false
+  }
+}
+```
+
+User settings live at `~/.pi/agent/settings.json`; project settings live at `<cwd>/.pi/settings.json` and override user settings.

package/dist/index.mjs

@@ -1,20 +1,66 @@
+import { homedir } from "node:os";
+import { join } from "node:path";
+import { readFileSync } from "node:fs";
 import { Type } from "@sinclair/typebox";
 import { keyHint } from "@earendil-works/pi-coding-agent";
 import { Text } from "@earendil-works/pi-tui";
 import { LRUCache } from "lru-cache";
 import { mkdir, writeFile } from "node:fs/promises";
-import { join } from "node:path";
 import { Readability } from "@mozilla/readability";
 import { parseHTML } from "linkedom";
 import TurndownService from "turndown";
-import { homedir } from "node:os";
+//#region ../utils/src/paths.ts
+function getPiLabGlobalDir(home = homedir()) {
+	return join(home, ".pi", "agent", "pi-lab");
+}
+function getPiLabGlobalTmpDir(name, home = homedir()) {
+	const tmpDir = join(getPiLabGlobalDir(home), "tmp");
+	return name ? join(tmpDir, name) : tmpDir;
+}
+//#endregion
+//#region ../utils/src/settings.ts
+function readJsonFile(filePath) {
+	try {
+		return JSON.parse(readFileSync(filePath, "utf8"));
+	} catch (error) {
+		if (error.code === "ENOENT") return {};
+		throw error;
+	}
+}
+function readPiProjectSettings(cwd = process.cwd()) {
+	return readJsonFile(join(cwd, ".pi", "settings.json"));
+}
+function readPiUserSettings(home = homedir()) {
+	return readJsonFile(join(home, ".pi", "agent", "settings.json"));
+}
+function mergePiSettings(userSettings = {}, projectSettings = {}) {
+	return deepMerge(userSettings, projectSettings);
+}
+function readMergedPiSettings(options = {}) {
+	return mergePiSettings(readPiUserSettings(options.home), readPiProjectSettings(options.cwd));
+}
+function deepMerge(base, override) {
+	const result = { ...base };
+	for (const [key, value] of Object.entries(override)) {
+		if (value === void 0) continue;
+		const existing = result[key];
+		if (isPlainObject(existing) && isPlainObject(value)) result[key] = deepMerge(existing, value);
+		else result[key] = value;
+	}
+	return result;
+}
+function isPlainObject(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+//#endregion
 //#region src/config.ts
 const DEFAULT_CONFIG = {
 	maxPageLength: 2e4,
 	cache: {
 		maxSizeBytes: 50 * 1024 * 1024,
 		ttlMs: 900 * 1e3
-	}
+	},
+	optimizations: true
 };
 function mergeConfig(partial) {
 	if (!partial) return DEFAULT_CONFIG;
@@ -23,9 +69,16 @@ function mergeConfig(partial) {
 		cache: {
 			...DEFAULT_CONFIG.cache,
 			...partial.cache
-		}
+		},
+		optimizations: partial.optimizations ?? DEFAULT_CONFIG.optimizations
 	};
 }
+function loadWebFetchConfig(settings = {}) {
+	const webfetch = settings.webfetch;
+	if (typeof webfetch !== "object" || webfetch === null || Array.isArray(webfetch)) return mergeConfig();
+	const optimizations = webfetch.optimizations;
+	return mergeConfig({ optimizations: typeof optimizations === "boolean" ? optimizations : void 0 });
+}
 //#endregion
 //#region src/cache.ts
 var WebFetchCache = class {
@@ -234,20 +287,206 @@ function processPlainText(text) {
 	return text;
 }
 //#endregion
-//#region ../utils/src/paths.ts
-function getPiLabGlobalDir(home = homedir()) {
-	return join(home, ".pi", "agent", "pi-lab");
-}
-function getPiLabGlobalTmpDir(name, home = homedir()) {
-	const tmpDir = join(getPiLabGlobalDir(home), "tmp");
-	return name ? join(tmpDir, name) : tmpDir;
-}
-//#endregion
 //#region src/paths.ts
 function getBinaryTempDir(home = homedir()) {
 	return getPiLabGlobalTmpDir("webfetch", home);
 }
 //#endregion
+//#region src/optimizers/reddit.ts
+function isRedditHost(hostname) {
+	return hostname === "reddit.com" || hostname === "www.reddit.com";
+}
+const redditOptimizer = {
+	id: "reddit",
+	match(url) {
+		try {
+			return isRedditHost(new URL(url).hostname.toLowerCase());
+		} catch {
+			return false;
+		}
+	},
+	rewriteUrl(url) {
+		const parsed = new URL(url);
+		parsed.hostname = "old.reddit.com";
+		return parsed.toString();
+	}
+};
+//#endregion
+//#region src/optimizers/x.ts
+function isXHost(hostname) {
+	return hostname === "x.com" || hostname === "www.x.com" || hostname === "twitter.com" || hostname === "www.twitter.com";
+}
+function isObject(value) {
+	return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function entitiesMap(value) {
+	if (!isObject(value)) return {};
+	const nested = value.entities;
+	return isObject(nested) ? nested : value;
+}
+function stringValue(value) {
+	return typeof value === "string" && value.trim() ? value : void 0;
+}
+function numberValue(value) {
+	return typeof value === "number" && Number.isFinite(value) ? value : void 0;
+}
+function extractStatusId(url) {
+	try {
+		return new URL(url).pathname.match(/\/status(?:es)?\/(\d+)/)?.[1];
+	} catch {
+		return;
+	}
+}
+function extractInitialStateJson(script) {
+	const start = script.indexOf("window.__INITIAL_STATE__=");
+	if (start === -1) return void 0;
+	const jsonStart = script.indexOf("{", start + 25);
+	if (jsonStart === -1) return void 0;
+	let depth = 0;
+	let inString = false;
+	let escaped = false;
+	for (let i = jsonStart; i < script.length; i++) {
+		const char = script[i];
+		if (inString) {
+			if (escaped) escaped = false;
+			else if (char === "\\") escaped = true;
+			else if (char === "\"") inString = false;
+			continue;
+		}
+		if (char === "\"") inString = true;
+		else if (char === "{") depth++;
+		else if (char === "}") {
+			depth--;
+			if (depth === 0) return script.slice(jsonStart, i + 1);
+		}
+	}
+}
+function parseInitialState(html) {
+	const { document } = parseHTML(html);
+	for (const script of document.querySelectorAll("script:not([src])")) {
+		const json = extractInitialStateJson(script.textContent ?? "");
+		if (!json) continue;
+		try {
+			const parsed = JSON.parse(json);
+			return isObject(parsed) ? parsed : void 0;
+		} catch {
+			return;
+		}
+	}
+}
+function decodeHtmlEntities(text) {
+	let result = text;
+	for (let i = 0; i < 3; i++) {
+		const decoded = result.replace(/&#x([0-9a-f]+);/gi, (_, hex) => String.fromCodePoint(Number.parseInt(hex, 16))).replace(/&#(\d+);/g, (_, dec) => String.fromCodePoint(Number.parseInt(dec, 10))).replace(/&quot;/g, "\"").replace(/&#39;/g, "'").replace(/&lt;/g, "<").replace(/&gt;/g, ">").replace(/&amp;/g, "&");
+		if (decoded === result) break;
+		result = decoded;
+	}
+	return result;
+}
+function mediaUrls(tweet) {
+	const extended = isObject(tweet.extended_entities) ? tweet.extended_entities : void 0;
+	const entities = isObject(tweet.entities) ? tweet.entities : void 0;
+	const media = Array.isArray(extended?.media) ? extended.media : Array.isArray(entities?.media) ? entities.media : [];
+	const urls = /* @__PURE__ */ new Set();
+	for (const item of media) {
+		if (!isObject(item)) continue;
+		const url = stringValue(item.media_url_https) ?? stringValue(item.media_url);
+		if (url) urls.add(url);
+	}
+	return [...urls];
+}
+function formatStats(tweet) {
+	const stats = [
+		["Replies", numberValue(tweet.reply_count)],
+		["Retweets", numberValue(tweet.retweet_count)],
+		["Quotes", numberValue(tweet.quote_count)],
+		["Likes", numberValue(tweet.favorite_count)]
+	].filter((entry) => entry[1] !== void 0);
+	if (stats.length === 0) return void 0;
+	return stats.map(([label, value]) => `${label}: ${value}`).join(" · ");
+}
+function selectTweet(tweets, url) {
+	const statusId = extractStatusId(url);
+	if (statusId && isObject(tweets[statusId])) return tweets[statusId];
+	return Object.values(tweets).find(isObject);
+}
+function renderTweetMarkdown(tweet, users) {
+	const text = stringValue(tweet.full_text) ?? stringValue(tweet.text);
+	if (!text) return void 0;
+	const userId = stringValue(tweet.user);
+	const user = userId ? users[userId] : void 0;
+	const userObj = isObject(user) ? user : void 0;
+	const name = stringValue(userObj?.name) ?? "Unknown author";
+	const screenName = stringValue(userObj?.screen_name);
+	const createdAt = stringValue(tweet.created_at);
+	const stats = formatStats(tweet);
+	const media = mediaUrls(tweet);
+	let body = decodeHtmlEntities(text).replace(/\s+https:\/\/t\.co\/\S+\s*$/g, "").trim();
+	if (!body) return void 0;
+	const lines = [];
+	lines.push(screenName ? `# Tweet by ${name} (@${screenName})` : `# Tweet by ${name}`);
+	if (createdAt) lines.push("", `Posted: ${createdAt}`);
+	if (stats) lines.push(stats);
+	lines.push("", body);
+	if (media.length > 0) lines.push("", "Media:", ...media.map((url) => `- ${url}`));
+	return lines.join("\n");
+}
+function optimizeXHtml({ url, html }) {
+	const state = parseInitialState(html);
+	const tweets = entitiesMap(state?.entities?.tweets);
+	const users = entitiesMap(state?.entities?.users);
+	const tweet = selectTweet(tweets, url);
+	if (!tweet) return void 0;
+	const markdown = renderTweetMarkdown(tweet, users);
+	if (!markdown) return void 0;
+	return {
+		markdown,
+		scripts: [],
+		method: "optimized"
+	};
+}
+//#endregion
+//#region src/optimizers/index.ts
+const BUILT_IN_OPTIMIZERS = [redditOptimizer, {
+	id: "x",
+	match(url) {
+		try {
+			return isXHost(new URL(url).hostname.toLowerCase());
+		} catch {
+			return false;
+		}
+	},
+	async processHtml(input) {
+		return optimizeXHtml(input);
+	}
+}];
+function findOptimizer(url, config) {
+	if (!config.optimizations) return void 0;
+	return BUILT_IN_OPTIMIZERS.find((optimizer) => optimizer.match(url));
+}
+function applyFetchOptimizations(url, config) {
+	const optimizer = findOptimizer(url, config);
+	if (!optimizer?.rewriteUrl) return { url };
+	const rewritten = optimizer.rewriteUrl(url);
+	if (!rewritten || rewritten === url) return { url };
+	return {
+		url: normalizeUrl(rewritten),
+		optimizerId: optimizer.id
+	};
+}
+async function processHtmlWithOptimizations({ url, html, config, defaultProcess }) {
+	const optimizer = findOptimizer(url, config);
+	if (optimizer?.processHtml) {
+		const optimized = await optimizer.processHtml({
+			url,
+			html,
+			defaultProcess
+		});
+		if (optimized) return optimized;
+	}
+	return defaultProcess();
+}
+//#endregion
 //#region src/tool.ts
 function formatScriptIndex(scripts) {
 	if (scripts.length === 0) return "";
@@ -328,6 +567,7 @@ function registerWebFetchTool(pi, config) {
 			} catch {
 				throw new Error(`Invalid URL: ${url}`);
 			}
+			normalizedUrl = applyFetchOptimizations(normalizedUrl, config).url;
 			const tempDir = getBinaryTempDir();
 			let entry = cache.get(normalizedUrl);
 			if (!entry) {
@@ -377,7 +617,12 @@ function registerWebFetchTool(pi, config) {
 					details: {}
 				});
 				if (result.contentType === "text/html") {
-					const processed = await processHtml(result.content, normalizedUrl);
+					const processed = await processHtmlWithOptimizations({
+						url: normalizedUrl,
+						html: result.content,
+						config,
+						defaultProcess: () => processHtml(result.content, normalizedUrl)
+					});
 					entry = {
 						markdown: processed.markdown,
 						scripts: processed.scripts
@@ -481,10 +726,11 @@ function registerWebFetchTool(pi, config) {
 * - Mozilla Readability for HTML → Markdown extraction
 * - Inline script index — use `script=N` to read a specific inline script
 * - LRU cache (50 MB, 15 min TTL) keyed on normalized URL
+* - Built-in fetch optimizations (enabled by default), including Reddit URL rewrite
 * - Pagination via offset/max_length parameters
 */
 function src_default(pi) {
-	registerWebFetchTool(pi, mergeConfig());
+	registerWebFetchTool(pi, loadWebFetchConfig(readMergedPiSettings()));
 }
 //#endregion
-export { DEFAULT_CONFIG, src_default as default, mergeConfig, registerWebFetchTool };
+export { DEFAULT_CONFIG, src_default as default, loadWebFetchConfig, mergeConfig, registerWebFetchTool };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@pi-lab/webfetch",
-  "version": "1.0.0",
+  "version": "1.0.1",
   "description": "WebFetch tool extension for pi coding agent",
   "keywords": [
     "pi-package"