@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/documentation.test.ts

@@ -0,0 +1,85 @@
+import { describe, expect, it } from "bun:test";
+import { handleMDN } from "./mdn";
+import { handleReadTheDocs } from "./readthedocs";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleMDN", () => {
+	it("returns null for non-MDN URLs", async () => {
+		const result = await handleMDN("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for non-docs MDN URLs", async () => {
+		const result = await handleMDN("https://developer.mozilla.org/en-US/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for MDN blog URLs", async () => {
+		const result = await handleMDN("https://developer.mozilla.org/en-US/blog/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches Array.map documentation", async () => {
+		const result = await handleMDN(
+			"https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/map",
+			20,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("mdn");
+		expect(result?.content).toContain("map");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+	});
+
+	it("fetches Promise documentation", async () => {
+		const result = await handleMDN(
+			"https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Promise",
+			20,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("mdn");
+		expect(result?.content).toContain("Promise");
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches CSS documentation", async () => {
+		const result = await handleMDN("https://developer.mozilla.org/en-US/docs/Web/CSS/display", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("mdn");
+		expect(result?.content).toContain("display");
+	});
+});
+
+describe.skipIf(SKIP)("handleReadTheDocs", () => {
+	it("returns null for non-RTD URLs", async () => {
+		const result = await handleReadTheDocs("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for github.com URLs", async () => {
+		const result = await handleReadTheDocs("https://github.com/user/repo", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches requests docs", async () => {
+		const result = await handleReadTheDocs("https://requests.readthedocs.io/en/latest/", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("readthedocs");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("returns null for non-readthedocs sites", async () => {
+		// These sites use Sphinx/RTD theme but aren't hosted on readthedocs.io
+		expect(await handleReadTheDocs("https://www.sphinx-doc.org/en/master/", 20)).toBeNull();
+		expect(await handleReadTheDocs("https://docs.pytest.org/en/stable/", 20)).toBeNull();
+		expect(await handleReadTheDocs("https://pip.pypa.io/en/stable/", 20)).toBeNull();
+	});
+
+	it("handles readthedocs.io subdomain", async () => {
+		const result = await handleReadTheDocs("https://flask.palletsprojects.readthedocs.io/en/latest/", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("readthedocs");
+	});
+});

package/src/core/tools/web-scrapers/fdroid.ts

@@ -0,0 +1,158 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+type LocalizedText = string | Record<string, string>;
+
+type FdroidPackage = {
+	packageName?: string;
+	name?: LocalizedText;
+	summary?: LocalizedText;
+	description?: LocalizedText;
+	author?: string | { name?: string; email?: string };
+	authorName?: string;
+	authorEmail?: string;
+	license?: string;
+	categories?: string[];
+	antiFeatures?: string[];
+	sourceCode?: string;
+	packages?: Array<{
+		versionName?: string;
+		versionCode?: number;
+		added?: number;
+		antiFeatures?: string[];
+	}>;
+	suggestedVersionCode?: number;
+	suggestedVersionName?: string;
+};
+
+function pickLocalizedText(value?: LocalizedText): string | undefined {
+	if (!value) return undefined;
+	if (typeof value === "string") return value;
+	const preferred = value["en-US"] ?? value.en_US ?? value.en;
+	if (preferred) return preferred;
+	const first = Object.values(value).find((entry) => typeof entry === "string");
+	return first;
+}
+
+function normalizeAuthor(data: FdroidPackage): string | undefined {
+	if (data.authorName) return data.authorName;
+	if (typeof data.author === "string") return data.author;
+	if (data.author && typeof data.author !== "string" && typeof data.author.name === "string") return data.author.name;
+	if (data.authorEmail) return data.authorEmail;
+	return undefined;
+}
+
+function normalizeAuthorEmail(data: FdroidPackage): string | undefined {
+	if (data.authorEmail) return data.authorEmail;
+	if (data.author && typeof data.author !== "string" && typeof data.author.email === "string")
+		return data.author.email;
+	return undefined;
+}
+
+function collectAntiFeatures(data: FdroidPackage): string[] {
+	const values = new Set<string>();
+	for (const feature of data.antiFeatures ?? []) values.add(feature);
+	for (const pkg of data.packages ?? []) {
+		for (const feature of pkg.antiFeatures ?? []) values.add(feature);
+	}
+	return Array.from(values);
+}
+
+function resolveSuggestedVersion(data: FdroidPackage): string | undefined {
+	if (data.suggestedVersionName) return data.suggestedVersionName;
+	if (data.suggestedVersionCode) {
+		const match = data.packages?.find((pkg) => pkg.versionCode === data.suggestedVersionCode);
+		if (match?.versionName) return match.versionName;
+	}
+	return data.packages?.[0]?.versionName;
+}
+
+/**
+ * Handle F-Droid URLs via API
+ */
+export const handleFdroid: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "f-droid.org" && parsed.hostname !== "www.f-droid.org") return null;
+
+		// Extract package name from /packages/{packageName} or /en/packages/{packageName}
+		const match = parsed.pathname.match(/^\/(?:en\/)?packages\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://f-droid.org/api/v1/packages/${encodeURIComponent(packageName)}`;
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let data: FdroidPackage;
+		try {
+			data = JSON.parse(result.content) as FdroidPackage;
+		} catch {
+			return null;
+		}
+
+		const displayName = pickLocalizedText(data.name) ?? packageName;
+		const summary = pickLocalizedText(data.summary);
+		const description = pickLocalizedText(data.description);
+		const author = normalizeAuthor(data);
+		const authorEmail = normalizeAuthorEmail(data);
+		const antiFeatures = collectAntiFeatures(data);
+		const latestVersion = resolveSuggestedVersion(data);
+
+		let md = `# ${displayName}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		md += `**Package:** ${packageName}`;
+		if (latestVersion) md += ` · **Latest:** ${latestVersion}`;
+		if (data.license) md += ` · **License:** ${data.license}`;
+		md += "\n";
+
+		if (author) {
+			md += `**Author:** ${author}`;
+			if (authorEmail && authorEmail !== author) md += ` <${authorEmail}>`;
+			md += "\n";
+		}
+
+		if (data.sourceCode) md += `**Source Code:** ${data.sourceCode}\n`;
+		if (data.categories?.length) md += `**Categories:** ${data.categories.join(", ")}\n`;
+		if (antiFeatures.length) md += `**Anti-Features:** ${antiFeatures.join(", ")}\n`;
+
+		if (description) {
+			md += `\n## Description\n\n${description}\n`;
+		}
+
+		if (data.packages?.length) {
+			md += "\n## Version History\n\n";
+			for (const version of data.packages.slice(0, 10)) {
+				const label = version.versionName ?? "unknown";
+				const code = version.versionCode ? ` (${version.versionCode})` : "";
+				md += `- ${label}${code}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "fdroid",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via F-Droid API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/finance-media.test.ts

@@ -0,0 +1,144 @@
+import { describe, expect, it } from "bun:test";
+import { handleArtifactHub } from "./artifacthub";
+import { handleCoinGecko } from "./coingecko";
+import { handleDiscogs } from "./discogs";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleCoinGecko", () => {
+	it("returns null for non-CoinGecko URLs", async () => {
+		const result = await handleCoinGecko("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for CoinGecko homepage", async () => {
+		const result = await handleCoinGecko("https://www.coingecko.com/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for CoinGecko categories page", async () => {
+		const result = await handleCoinGecko("https://www.coingecko.com/en/categories", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches Bitcoin data", async () => {
+		const result = await handleCoinGecko("https://www.coingecko.com/en/coins/bitcoin", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("coingecko");
+		expect(result?.content).toContain("Bitcoin");
+		expect(result?.content).toContain("BTC");
+		expect(result?.content).toContain("Price");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches Ethereum data", async () => {
+		const result = await handleCoinGecko("https://www.coingecko.com/en/coins/ethereum", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("coingecko");
+		expect(result?.content).toContain("Ethereum");
+		expect(result?.content).toContain("ETH");
+		expect(result?.content).toContain("Market Cap");
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("handles URL without locale prefix", async () => {
+		const result = await handleCoinGecko("https://www.coingecko.com/coins/bitcoin", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("coingecko");
+	});
+});
+
+describe.skipIf(SKIP)("handleDiscogs", () => {
+	it("returns null for non-Discogs URLs", async () => {
+		const result = await handleDiscogs("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for Discogs homepage", async () => {
+		const result = await handleDiscogs("https://www.discogs.com/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for Discogs search page", async () => {
+		const result = await handleDiscogs("https://www.discogs.com/search/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches Daft Punk Discovery release", async () => {
+		// Release 249504: Daft Punk - Discovery
+		const result = await handleDiscogs("https://www.discogs.com/release/249504-Daft-Punk-Discovery", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("discogs");
+		expect(result?.content).toContain("Daft Punk");
+		expect(result?.content).toContain("Discovery");
+		expect(result?.content).toContain("Tracklist");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches master release", async () => {
+		// Master 33395: Daft Punk - Discovery (master)
+		const result = await handleDiscogs("https://www.discogs.com/master/33395-Daft-Punk-Discovery", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("discogs");
+		expect(result?.content).toContain("Daft Punk");
+		expect(result?.content).toContain("Master Release");
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("handles release URL with just ID", async () => {
+		const result = await handleDiscogs("https://www.discogs.com/release/249504", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("discogs");
+	});
+});
+
+describe.skipIf(SKIP)("handleArtifactHub", () => {
+	it("returns null for non-ArtifactHub URLs", async () => {
+		const result = await handleArtifactHub("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for ArtifactHub homepage", async () => {
+		const result = await handleArtifactHub("https://artifacthub.io/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for ArtifactHub search page", async () => {
+		const result = await handleArtifactHub("https://artifacthub.io/packages/search", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches bitnami/nginx helm chart", async () => {
+		const result = await handleArtifactHub("https://artifacthub.io/packages/helm/bitnami/nginx", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("artifacthub");
+		expect(result?.content).toContain("nginx");
+		expect(result?.content).toContain("Helm Chart");
+		expect(result?.content).toContain("Version");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches prometheus-community/prometheus helm chart", async () => {
+		const result = await handleArtifactHub(
+			"https://artifacthub.io/packages/helm/prometheus-community/prometheus",
+			20,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("artifacthub");
+		expect(result?.content).toContain("prometheus");
+		expect(result?.content).toContain("Repository");
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("handles www subdomain", async () => {
+		const result = await handleArtifactHub("https://www.artifacthub.io/packages/helm/bitnami/nginx", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("artifacthub");
+	});
+});

package/src/core/tools/web-scrapers/firefox-addons.ts

@@ -0,0 +1,214 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, htmlToBasicMarkdown, loadPage } from "./types";
+
+type LocalizedText = string | Record<string, string | null | undefined> | null | undefined;
+
+type AddonFile = {
+	permissions?: string[];
+	host_permissions?: string[];
+	optional_permissions?: string[];
+	optional_host_permissions?: string[];
+};
+
+type AddonLicense = {
+	name?: LocalizedText;
+	slug?: string;
+	url?: string;
+};
+
+type AddonVersion = {
+	version?: string;
+	license?: AddonLicense;
+	file?: AddonFile;
+};
+
+type AddonHomepage = {
+	url?: LocalizedText;
+	outgoing?: LocalizedText;
+};
+
+type AddonData = {
+	name?: LocalizedText;
+	summary?: LocalizedText;
+	description?: LocalizedText;
+	default_locale?: string;
+	authors?: Array<{ name?: string | null }>;
+	average_daily_users?: number;
+	weekly_downloads?: number;
+	ratings?: { average?: number; count?: number };
+	current_version?: AddonVersion;
+	categories?: string[] | Record<string, string[]>;
+	homepage?: AddonHomepage;
+	url?: string;
+};
+
+function getLocalizedText(value: LocalizedText, defaultLocale?: string): string | undefined {
+	if (!value) return undefined;
+	if (typeof value === "string") return value;
+
+	const localized = value as Record<string, string | null | undefined>;
+	if (defaultLocale && localized[defaultLocale]) return localized[defaultLocale] ?? undefined;
+	if (localized["en-US"]) return localized["en-US"] ?? undefined;
+
+	for (const entry of Object.values(localized)) {
+		if (entry) return entry;
+	}
+
+	return undefined;
+}
+
+function normalizeCategories(categories?: string[] | Record<string, string[]>): string[] {
+	if (!categories) return [];
+	if (Array.isArray(categories)) return categories.filter(Boolean);
+
+	const values: string[] = [];
+	for (const list of Object.values(categories)) {
+		if (Array.isArray(list)) {
+			for (const item of list) {
+				if (item) values.push(item);
+			}
+		}
+	}
+
+	const seen = new Set<string>();
+	return values.filter((item) => {
+		if (seen.has(item)) return false;
+		seen.add(item);
+		return true;
+	});
+}
+
+function collectPermissions(file?: AddonFile): string[] {
+	if (!file) return [];
+	const permissions: string[] = [];
+	const seen = new Set<string>();
+
+	const add = (items?: string[]) => {
+		for (const item of items ?? []) {
+			if (!item || seen.has(item)) continue;
+			seen.add(item);
+			permissions.push(item);
+		}
+	};
+
+	add(file.permissions);
+	add(file.host_permissions);
+	add(file.optional_permissions);
+	add(file.optional_host_permissions);
+
+	return permissions;
+}
+
+export const handleFirefoxAddons: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "addons.mozilla.org") return null;
+
+		const segments = parsed.pathname.split("/").filter(Boolean);
+		const addonIndex = segments.indexOf("addon");
+		if (addonIndex === -1) return null;
+
+		const slug = segments[addonIndex + 1] ? decodeURIComponent(segments[addonIndex + 1]) : "";
+		if (!slug) return null;
+
+		const apiUrl = `https://addons.mozilla.org/api/v5/addons/addon/${encodeURIComponent(slug)}/`;
+		const result = await loadPage(apiUrl, { timeout, headers: { Accept: "application/json" }, signal });
+		if (!result.ok) return null;
+
+		let data: AddonData;
+		try {
+			data = JSON.parse(result.content) as AddonData;
+		} catch {
+			return null;
+		}
+
+		const fetchedAt = new Date().toISOString();
+		const defaultLocale = data.default_locale || "en-US";
+
+		const name = getLocalizedText(data.name, defaultLocale) ?? slug;
+		const summary = getLocalizedText(data.summary, defaultLocale);
+		const descriptionRaw = getLocalizedText(data.description, defaultLocale);
+		const description = descriptionRaw ? htmlToBasicMarkdown(descriptionRaw) : undefined;
+
+		const authors = (data.authors ?? [])
+			.map((author) => author.name ?? "")
+			.map((author) => author.trim())
+			.filter(Boolean);
+
+		const ratingAverage = data.ratings?.average;
+		const ratingCount = data.ratings?.count;
+		const users = data.average_daily_users ?? data.weekly_downloads;
+		const version = data.current_version?.version;
+		const categories = normalizeCategories(data.categories);
+
+		const licenseName =
+			getLocalizedText(data.current_version?.license?.name, defaultLocale) ?? data.current_version?.license?.slug;
+		const licenseUrl = data.current_version?.license?.url;
+
+		const homepage =
+			getLocalizedText(data.homepage?.url, defaultLocale) ??
+			getLocalizedText(data.homepage?.outgoing, defaultLocale);
+
+		const permissions = collectPermissions(data.current_version?.file);
+
+		let md = `# ${name}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		if (authors.length > 0) {
+			md += `**Author${authors.length > 1 ? "s" : ""}:** ${authors.join(", ")}\n`;
+		}
+
+		if (ratingAverage !== undefined) {
+			md += `**Rating:** ${ratingAverage.toFixed(2)}`;
+			if (ratingCount !== undefined) md += ` (${formatCount(ratingCount)} reviews)`;
+			md += "\n";
+		}
+
+		if (users !== undefined) md += `**Users:** ${formatCount(users)}\n`;
+		if (version) md += `**Version:** ${version}\n`;
+		if (categories.length > 0) md += `**Categories:** ${categories.join(", ")}\n`;
+
+		if (licenseName && licenseUrl) {
+			md += `**License:** [${licenseName}](${licenseUrl})\n`;
+		} else if (licenseName) {
+			md += `**License:** ${licenseName}\n`;
+		} else if (licenseUrl) {
+			md += `**License:** ${licenseUrl}\n`;
+		}
+
+		if (homepage) md += `**Homepage:** ${homepage}\n`;
+
+		if (description) {
+			md += `\n## Description\n\n${description}\n`;
+		}
+
+		if (permissions.length > 0) {
+			const preview = permissions.slice(0, 40);
+			md += `\n## Permissions (${permissions.length})\n\n`;
+			for (const permission of preview) {
+				md += `- ${permission}\n`;
+			}
+			if (permissions.length > preview.length) {
+				md += `\n*...and ${permissions.length - preview.length} more*\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: data.url ?? result.finalUrl ?? url,
+			contentType: "text/markdown",
+			method: "firefox-addons",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Firefox Add-ons API"],
+		};
+	} catch {}
+
+	return null;
+};