@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/mdn.ts

@@ -0,0 +1,174 @@
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface MDNSection {
+	type: string;
+	value: {
+		id?: string;
+		title?: string;
+		content?: string;
+		isH3?: boolean;
+		code?: string;
+		language?: string;
+		items?: Array<{ term: string; description: string }>;
+		rows?: string[][];
+	};
+}
+
+interface MDNDoc {
+	doc: {
+		title: string;
+		summary: string;
+		mdn_url: string;
+		body: MDNSection[];
+		browserCompat?: unknown;
+	};
+}
+
+/**
+ * Convert MDN body sections to markdown
+ */
+function convertMDNBody(sections: MDNSection[]): string {
+	const parts: string[] = [];
+
+	for (const section of sections) {
+		const { type, value } = section;
+
+		switch (type) {
+			case "prose":
+				if (value.content) {
+					const markdown = htmlToBasicMarkdown(value.content);
+					if (value.title) {
+						const level = value.isH3 ? "###" : "##";
+						parts.push(`${level} ${value.title}\n\n${markdown}`);
+					} else {
+						parts.push(markdown);
+					}
+				}
+				break;
+
+			case "browser_compatibility":
+				if (value.title) {
+					parts.push(`## ${value.title}\n\n(See browser compatibility data at MDN)`);
+				}
+				break;
+
+			case "specifications":
+				if (value.title) {
+					parts.push(`## ${value.title}\n\n(See specifications at MDN)`);
+				}
+				break;
+
+			case "code_example":
+				if (value.title) {
+					parts.push(`### ${value.title}`);
+				}
+				if (value.code) {
+					const lang = value.language || "";
+					parts.push(`\`\`\`${lang}\n${value.code}\n\`\`\``);
+				}
+				break;
+
+			case "definition_list":
+				if (value.items) {
+					for (const item of value.items) {
+						parts.push(`**${item.term}**`);
+						const desc = htmlToBasicMarkdown(item.description);
+						parts.push(desc);
+					}
+				}
+				break;
+
+			case "table":
+				if (value.rows && value.rows.length > 0) {
+					// Simple markdown table
+					const header = value.rows[0].map((cell) => htmlToBasicMarkdown(cell)).join(" | ");
+					const separator = value.rows[0].map(() => "---").join(" | ");
+					const bodyRows = value.rows
+						.slice(1)
+						.map((row) => row.map((cell) => htmlToBasicMarkdown(cell)).join(" | "));
+
+					parts.push(`| ${header} |`);
+					parts.push(`| ${separator} |`);
+					for (const row of bodyRows) {
+						parts.push(`| ${row} |`);
+					}
+				}
+				break;
+
+			default:
+				// Skip unknown types
+				break;
+		}
+	}
+
+	return parts.join("\n\n");
+}
+
+export const handleMDN: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
+	const urlObj = new URL(url);
+
+	// Only handle developer.mozilla.org
+	if (!urlObj.hostname.includes("developer.mozilla.org")) {
+		return null;
+	}
+
+	// Only handle docs paths
+	if (!urlObj.pathname.includes("/docs/")) {
+		return null;
+	}
+
+	const notes: string[] = [];
+
+	// Construct JSON API URL
+	const jsonUrl = url.replace(/\/?$/, "/index.json");
+
+	try {
+		const result = await loadPage(jsonUrl, { timeout, signal, headers: { Accept: "application/json" } });
+
+		if (!result.ok) {
+			notes.push(`Failed to fetch MDN JSON API (status ${result.status || "unknown"})`);
+			return null;
+		}
+
+		const data: MDNDoc = JSON.parse(result.content);
+		const { doc } = data;
+
+		if (!doc || !doc.title) {
+			notes.push("Invalid MDN JSON structure");
+			return null;
+		}
+
+		// Build markdown content
+		const parts: string[] = [];
+
+		parts.push(`# ${doc.title}`);
+
+		if (doc.summary) {
+			const summary = htmlToBasicMarkdown(doc.summary);
+			parts.push(summary);
+		}
+
+		if (doc.body && doc.body.length > 0) {
+			const bodyMarkdown = convertMDNBody(doc.body);
+			parts.push(bodyMarkdown);
+		}
+
+		const rawContent = parts.join("\n\n");
+		const { content, truncated } = finalizeOutput(rawContent);
+
+		return {
+			url,
+			finalUrl: doc.mdn_url || result.finalUrl,
+			contentType: "text/markdown",
+			method: "mdn",
+			content,
+			fetchedAt: new Date().toISOString(),
+			truncated,
+			notes,
+		};
+	} catch (err) {
+		notes.push(`MDN handler error: ${err instanceof Error ? err.message : String(err)}`);
+		return null;
+	}
+};

package/src/core/tools/web-scrapers/media.test.ts

@@ -0,0 +1,138 @@
+import { describe, expect, it } from "bun:test";
+import { handleHuggingFace } from "./huggingface";
+import { handleSpotify } from "./spotify";
+import { handleVimeo } from "./vimeo";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleVimeo", () => {
+	it("returns null for non-Vimeo URLs", async () => {
+		const result = await handleVimeo("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for invalid Vimeo URLs", async () => {
+		const result = await handleVimeo("https://vimeo.com/invalid", 10);
+		expect(result).toBeNull();
+	});
+
+	it("fetches video metadata via oEmbed", async () => {
+		const result = await handleVimeo("https://vimeo.com/1084537", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("vimeo");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("Video ID");
+		expect(result?.notes).toContain("Fetched via Vimeo oEmbed API");
+	});
+
+	it("handles player.vimeo.com URLs", async () => {
+		const result = await handleVimeo("https://player.vimeo.com/video/1084537", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("vimeo");
+		expect(result?.content).toContain("Video ID");
+	});
+
+	it("handles vimeo.com/user/video format", async () => {
+		const result = await handleVimeo("https://vimeo.com/staffpicks/1084537", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("vimeo");
+	});
+});
+
+describe.skipIf(SKIP)("handleSpotify", () => {
+	it("returns null for non-Spotify URLs", async () => {
+		const result = await handleSpotify("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for invalid Spotify URLs", async () => {
+		const result = await handleSpotify("https://open.spotify.com/invalid/xyz", 10);
+		expect(result).toBeNull();
+	});
+
+	it("identifies track URLs", async () => {
+		const result = await handleSpotify("https://open.spotify.com/track/4cOdK2wGLETKBW3PvgPWqT", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("spotify");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("Type");
+		expect(result?.content).toContain("track");
+	});
+
+	it("identifies album URLs", async () => {
+		const result = await handleSpotify("https://open.spotify.com/album/2ODvWsOgouMbaA5xf0RkJe", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("spotify");
+		expect(result?.content).toContain("album");
+	});
+
+	it("identifies playlist URLs", async () => {
+		const result = await handleSpotify("https://open.spotify.com/playlist/37i9dQZF1DXcBWIGoYBM5M", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("spotify");
+		expect(result?.content).toContain("playlist");
+	});
+
+	it("identifies podcast episode URLs", async () => {
+		const result = await handleSpotify("https://open.spotify.com/episode/0Q86acNRm6V9GYx55SXKwf", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("spotify");
+		expect(result?.content).toContain("podcast-episode");
+	});
+
+	it("identifies podcast show URLs", async () => {
+		const result = await handleSpotify("https://open.spotify.com/show/2MAi0BvDc6GTFvKFPXnkCL", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("spotify");
+		expect(result?.content).toContain("podcast-show");
+	});
+});
+
+describe.skipIf(SKIP)("handleHuggingFace", () => {
+	it("returns null for non-HF URLs", async () => {
+		const result = await handleHuggingFace("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for invalid HF URLs", async () => {
+		const result = await handleHuggingFace("https://huggingface.co", 10);
+		expect(result).toBeNull();
+	});
+
+	it("fetches model info", async () => {
+		const result = await handleHuggingFace("https://huggingface.co/bert-base-uncased", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("huggingface");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("bert-base-uncased");
+	});
+
+	it("fetches dataset info", async () => {
+		const result = await handleHuggingFace("https://huggingface.co/datasets/squad", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("huggingface");
+		expect(result?.content).toContain("squad");
+	});
+
+	it("fetches space info", async () => {
+		const result = await handleHuggingFace("https://huggingface.co/spaces/gradio/hello_world", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("huggingface");
+		expect(result?.content).toContain("gradio/hello_world");
+	});
+
+	it("fetches model without org prefix", async () => {
+		// Some models like bert-base-uncased don't have an org prefix
+		const result = await handleHuggingFace("https://huggingface.co/bert-base-uncased", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("huggingface");
+		expect(result?.content).toContain("bert-base-uncased");
+	});
+
+	it("handles org/model format", async () => {
+		const result = await handleHuggingFace("https://huggingface.co/google/bert_uncased_L-2_H-128_A-2", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("huggingface");
+		expect(result?.content).toContain("google/bert_uncased_L-2_H-128_A-2");
+	});
+});

package/src/core/tools/web-scrapers/metacpan.ts

@@ -0,0 +1,253 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface ModuleResponse {
+	name: string;
+	version: string;
+	abstract?: string;
+	author: string;
+	distribution: string;
+	release: string;
+	path: string;
+	pod?: string;
+}
+
+interface ReleaseResponse {
+	name: string;
+	version: string;
+	abstract?: string;
+	author: string;
+	distribution: string;
+	license?: string[];
+	stat?: { mtime: number };
+	download_url?: string;
+	dependency?: Array<{
+		module: string;
+		version: string;
+		phase: string;
+		relationship: string;
+	}>;
+	metadata?: {
+		resources?: {
+			repository?: { url?: string; web?: string };
+			homepage?: string;
+			bugtracker?: { web?: string };
+		};
+	};
+}
+
+/**
+ * Handle MetaCPAN URLs via fastapi.metacpan.org
+ */
+export const handleMetaCPAN: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "metacpan.org" && parsed.hostname !== "www.metacpan.org") return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Match /pod/Module::Name pattern
+		const podMatch = parsed.pathname.match(/^\/pod\/(.+?)(?:\/|$)/);
+		if (podMatch) {
+			const moduleName = decodeURIComponent(podMatch[1]);
+			return await fetchModule(url, moduleName, timeout, fetchedAt, signal);
+		}
+
+		// Match /release/AUTHOR/Distribution pattern
+		const releaseMatch = parsed.pathname.match(/^\/release\/([^/]+)\/([^/]+)/);
+		if (releaseMatch) {
+			const distribution = decodeURIComponent(releaseMatch[2]);
+			return await fetchRelease(url, distribution, timeout, fetchedAt, signal);
+		}
+
+		// Match /release/Distribution pattern (without author)
+		const simpleReleaseMatch = parsed.pathname.match(/^\/release\/([^/]+)$/);
+		if (simpleReleaseMatch) {
+			const distribution = decodeURIComponent(simpleReleaseMatch[1]);
+			return await fetchRelease(url, distribution, timeout, fetchedAt, signal);
+		}
+
+		return null;
+	} catch {}
+
+	return null;
+};
+
+async function fetchModule(
+	url: string,
+	moduleName: string,
+	timeout: number,
+	fetchedAt: string,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> {
+	const apiUrl = `https://fastapi.metacpan.org/v1/module/${moduleName}`;
+	const result = await loadPage(apiUrl, { timeout, signal });
+
+	if (!result.ok) return null;
+
+	let module: ModuleResponse;
+	try {
+		module = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	// Fetch additional release info for dependencies and metadata
+	const releaseUrl = `https://fastapi.metacpan.org/v1/release/${module.distribution}`;
+	const releaseResult = await loadPage(releaseUrl, { timeout: Math.min(timeout, 5), signal });
+
+	let release: ReleaseResponse | null = null;
+	if (releaseResult.ok) {
+		try {
+			release = JSON.parse(releaseResult.content);
+		} catch {}
+	}
+
+	const md = formatModuleMarkdown(module, release);
+	const output = finalizeOutput(md);
+
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/markdown",
+		method: "metacpan",
+		content: output.content,
+		fetchedAt,
+		truncated: output.truncated,
+		notes: ["Fetched via MetaCPAN API"],
+	};
+}
+
+async function fetchRelease(
+	url: string,
+	distribution: string,
+	timeout: number,
+	fetchedAt: string,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> {
+	const apiUrl = `https://fastapi.metacpan.org/v1/release/${distribution}`;
+	const result = await loadPage(apiUrl, { timeout, signal });
+
+	if (!result.ok) return null;
+
+	let release: ReleaseResponse;
+	try {
+		release = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	const md = formatReleaseMarkdown(release);
+	const output = finalizeOutput(md);
+
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/markdown",
+		method: "metacpan",
+		content: output.content,
+		fetchedAt,
+		truncated: output.truncated,
+		notes: ["Fetched via MetaCPAN API"],
+	};
+}
+
+function formatModuleMarkdown(module: ModuleResponse, release: ReleaseResponse | null): string {
+	let md = `# ${module.name}\n\n`;
+	if (module.abstract) md += `${module.abstract}\n\n`;
+
+	md += `**Version:** ${module.version}`;
+	md += ` · **Distribution:** ${module.distribution}`;
+	md += ` · **Author:** [${module.author}](https://metacpan.org/author/${module.author})\n`;
+
+	if (release) {
+		if (release.license?.length) {
+			md += `**License:** ${release.license.join(", ")}\n`;
+		}
+
+		const resources = release.metadata?.resources;
+		if (resources?.repository?.web || resources?.repository?.url) {
+			const repoUrl = resources.repository.web || resources.repository.url;
+			md += `**Repository:** ${repoUrl}\n`;
+		}
+		if (resources?.homepage) {
+			md += `**Homepage:** ${resources.homepage}\n`;
+		}
+		if (resources?.bugtracker?.web) {
+			md += `**Issues:** ${resources.bugtracker.web}\n`;
+		}
+
+		// Show runtime dependencies
+		const runtimeDeps = release.dependency?.filter(
+			(d) => d.phase === "runtime" && d.relationship === "requires" && d.module !== "perl",
+		);
+		if (runtimeDeps?.length) {
+			md += `\n## Dependencies\n\n`;
+			for (const dep of runtimeDeps.slice(0, 20)) {
+				md += `- **${dep.module}**`;
+				if (dep.version && dep.version !== "0") md += ` >= ${dep.version}`;
+				md += "\n";
+			}
+			if (runtimeDeps.length > 20) {
+				md += `\n*...and ${runtimeDeps.length - 20} more*\n`;
+			}
+		}
+	}
+
+	md += `\n## Installation\n\n\`\`\`bash\ncpanm ${module.name}\n\`\`\`\n`;
+
+	return md;
+}
+
+function formatReleaseMarkdown(release: ReleaseResponse): string {
+	let md = `# ${release.distribution}\n\n`;
+	if (release.abstract) md += `${release.abstract}\n\n`;
+
+	md += `**Version:** ${release.version}`;
+	md += ` · **Author:** [${release.author}](https://metacpan.org/author/${release.author})\n`;
+
+	if (release.license?.length) {
+		md += `**License:** ${release.license.join(", ")}\n`;
+	}
+
+	if (release.stat?.mtime) {
+		const date = new Date(release.stat.mtime * 1000).toISOString().split("T")[0];
+		md += `**Released:** ${date}\n`;
+	}
+
+	const resources = release.metadata?.resources;
+	if (resources?.repository?.web || resources?.repository?.url) {
+		const repoUrl = resources.repository.web || resources.repository.url;
+		md += `**Repository:** ${repoUrl}\n`;
+	}
+	if (resources?.homepage) {
+		md += `**Homepage:** ${resources.homepage}\n`;
+	}
+	if (resources?.bugtracker?.web) {
+		md += `**Issues:** ${resources.bugtracker.web}\n`;
+	}
+
+	// Show runtime dependencies
+	const runtimeDeps = release.dependency?.filter(
+		(d) => d.phase === "runtime" && d.relationship === "requires" && d.module !== "perl",
+	);
+	if (runtimeDeps?.length) {
+		md += `\n## Dependencies\n\n`;
+		for (const dep of runtimeDeps.slice(0, 20)) {
+			md += `- **${dep.module}**`;
+			if (dep.version && dep.version !== "0") md += ` >= ${dep.version}`;
+			md += "\n";
+		}
+		if (runtimeDeps.length > 20) {
+			md += `\n*...and ${runtimeDeps.length - 20} more*\n`;
+		}
+	}
+
+	md += `\n## Installation\n\n\`\`\`bash\ncpanm ${release.distribution}\n\`\`\`\n`;
+
+	return md;
+}