@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/snapcraft.ts

@@ -0,0 +1,200 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface SnapcraftPublisher {
+	"display-name"?: string;
+	username?: string;
+	id?: string;
+	validation?: string;
+}
+
+interface SnapcraftChannel {
+	name?: string;
+	track?: string;
+	risk?: string;
+	branch?: string | null;
+	architecture?: string;
+	"released-at"?: string;
+}
+
+interface SnapcraftDownload {
+	size?: number;
+	url?: string;
+	"sha3-384"?: string;
+}
+
+interface SnapcraftChannelMapEntry {
+	channel?: SnapcraftChannel;
+	version?: string;
+	revision?: number | string;
+	download?: SnapcraftDownload;
+	type?: string;
+	"created-at"?: string;
+}
+
+interface SnapcraftSnap {
+	name?: string;
+	title?: string;
+	summary?: string;
+	description?: string;
+	publisher?: SnapcraftPublisher;
+	version?: string;
+	confinement?: string;
+	base?: string;
+	downloads?: number;
+	download?: number;
+}
+
+interface SnapcraftResponse {
+	name?: string;
+	title?: string;
+	summary?: string;
+	description?: string;
+	publisher?: SnapcraftPublisher;
+	version?: string;
+	confinement?: string;
+	base?: string;
+	downloads?: number;
+	download?: number;
+	snap?: SnapcraftSnap;
+	"channel-map"?: SnapcraftChannelMapEntry[];
+}
+
+function formatPublisher(publisher?: SnapcraftPublisher): string | null {
+	if (!publisher) return null;
+	const displayName = publisher["display-name"] ?? publisher.username ?? publisher.id;
+	if (!displayName) return null;
+	if (publisher.username && displayName !== publisher.username) {
+		return `${displayName} (@${publisher.username})`;
+	}
+	return displayName;
+}
+
+function formatChannelName(channel?: SnapcraftChannel): string | null {
+	if (!channel) return null;
+	if (channel.name?.includes("/")) return channel.name;
+	if (channel.track && channel.risk) {
+		const branch = channel.branch ? `/${channel.branch}` : "";
+		return `${channel.track}/${channel.risk}${branch}`;
+	}
+	return channel.name ?? null;
+}
+
+function pickVersionFromChannels(entries: SnapcraftChannelMapEntry[]): string | undefined {
+	const stable = entries.find((entry) => entry.channel?.risk === "stable" && entry.version);
+	if (stable?.version) return stable.version;
+	const first = entries.find((entry) => entry.version);
+	return first?.version;
+}
+
+function extractDownloads(snapInfo: SnapcraftSnap | SnapcraftResponse, data: SnapcraftResponse): number | null {
+	const candidates = [snapInfo.downloads, snapInfo.download, data.downloads, data.download];
+	for (const value of candidates) {
+		if (typeof value === "number" && Number.isFinite(value)) return value;
+	}
+	return null;
+}
+
+export const handleSnapcraft: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "snapcraft.io" && parsed.hostname !== "www.snapcraft.io") return null;
+
+		const installMatch = parsed.pathname.match(/^\/install\/([^/]+)\/?$/);
+		const directMatch = parsed.pathname.match(/^\/([^/]+)\/?$/);
+		if (!installMatch && !directMatch) return null;
+
+		const snapName = decodeURIComponent((installMatch ?? directMatch)![1]);
+		const fetchedAt = new Date().toISOString();
+
+		const apiUrl = `https://api.snapcraft.io/v2/snaps/info/${encodeURIComponent(snapName)}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: {
+				Accept: "application/json",
+				"Snap-Device-Series": "16",
+			},
+		});
+		if (!result.ok) return null;
+
+		let data: SnapcraftResponse;
+		try {
+			data = JSON.parse(result.content) as SnapcraftResponse;
+		} catch {
+			return null;
+		}
+
+		const snapInfo = data.snap ?? data;
+		const name = snapInfo.title ?? snapInfo.name ?? data.name ?? snapName;
+		const summary = snapInfo.summary ?? data.summary;
+		const description = snapInfo.description ?? data.description;
+		const publisher = formatPublisher(snapInfo.publisher ?? data.publisher);
+		const confinement = snapInfo.confinement ?? data.confinement;
+		const base = snapInfo.base ?? data.base;
+
+		const channelMap = data["channel-map"] ?? [];
+		let version = snapInfo.version ?? data.version;
+		if (!version && channelMap.length > 0) {
+			version = pickVersionFromChannels(channelMap);
+		}
+
+		const downloads = extractDownloads(snapInfo, data);
+
+		const channels = new Map<string, { version?: string; architectures: Set<string> }>();
+		for (const entry of channelMap) {
+			const channelName = formatChannelName(entry.channel);
+			if (!channelName) continue;
+			const existing = channels.get(channelName) ?? { architectures: new Set<string>() };
+			if (!existing.version && entry.version) existing.version = entry.version;
+			if (entry.channel?.architecture) existing.architectures.add(entry.channel.architecture);
+			channels.set(channelName, existing);
+		}
+
+		let md = `# ${name}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		md += `**Version:** ${version ?? "unknown"}`;
+		if (confinement) md += ` · **Confinement:** ${confinement}`;
+		if (base) md += ` · **Base:** ${base}`;
+		md += "\n";
+		if (publisher) md += `**Publisher:** ${publisher}\n`;
+		if (downloads !== null) md += `**Downloads:** ${formatCount(downloads)}\n`;
+		md += "\n";
+
+		if (channels.size > 0) {
+			md += "## Channels\n\n";
+			const sortedChannels = Array.from(channels.entries()).sort((a, b) => a[0].localeCompare(b[0]));
+			for (const [channelName, info] of sortedChannels) {
+				const arches = Array.from(info.architectures).sort();
+				const versionSuffix = info.version ? `: ${info.version}` : "";
+				const archSuffix = arches.length > 0 ? ` (${arches.join(", ")})` : "";
+				md += `- ${channelName}${versionSuffix}${archSuffix}\n`;
+			}
+			md += "\n";
+		}
+
+		const descriptionText = description ?? summary;
+		if (descriptionText) {
+			md += `## Description\n\n${descriptionText}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "snapcraft",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Snapcraft API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/social-extended.test.ts

@@ -0,0 +1,192 @@
+import { describe, expect, it } from "bun:test";
+import { handleBluesky } from "./bluesky";
+import { handleMastodon } from "./mastodon";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleMastodon", () => {
+	it("returns null for non-Mastodon URLs", async () => {
+		const result = await handleMastodon("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for URLs without @user pattern", async () => {
+		const result = await handleMastodon("https://mastodon.social/about", 20);
+		expect(result).toBeNull();
+	});
+
+	it(
+		"fetches a Mastodon profile",
+		async () => {
+			// @Gargron is Eugen Rochko, creator of Mastodon - very stable
+			const result = await handleMastodon("https://mastodon.social/@Gargron", 20);
+			expect(result).not.toBeNull();
+			expect(result?.method).toBe("mastodon");
+			expect(result?.contentType).toBe("text/markdown");
+			expect(result?.content).toContain("Gargron");
+			expect(result?.content).toContain("@Gargron");
+			expect(result?.content).toContain("**Followers:**");
+			expect(result?.content).toContain("**Following:**");
+			expect(result?.content).toContain("**Posts:**");
+			expect(result?.fetchedAt).toBeTruthy();
+			expect(result?.truncated).toBeDefined();
+			expect(result?.notes?.[0]).toContain("Mastodon API");
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"fetches a Mastodon post",
+		async () => {
+			// Gargron's post ID 1 - the first ever Mastodon post
+			const result = await handleMastodon("https://mastodon.social/@Gargron/1", 20);
+			// Post 1 may not exist anymore; check gracefully
+			if (result !== null) {
+				expect(result.method).toBe("mastodon");
+				expect(result.contentType).toBe("text/markdown");
+				expect(result.content).toContain("Post by");
+				expect(result.content).toContain("@Gargron");
+				expect(result.fetchedAt).toBeTruthy();
+				expect(result.truncated).toBeDefined();
+				expect(result.notes?.[0]).toContain("Mastodon API");
+			}
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"handles a stable pinned post",
+		async () => {
+			// Use a well-known post from mastodon.social - Gargron's announcement post
+			const result = await handleMastodon("https://mastodon.social/@Gargron/109318821117356215", 20);
+			// May not exist, check gracefully
+			if (result !== null) {
+				expect(result.method).toBe("mastodon");
+				expect(result.contentType).toBe("text/markdown");
+				expect(result.content).toContain("@Gargron");
+				expect(result.content).toContain("replies");
+				expect(result.content).toContain("boosts");
+				expect(result.content).toContain("favorites");
+				expect(result.fetchedAt).toBeTruthy();
+			}
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"includes recent posts in profile",
+		async () => {
+			const result = await handleMastodon("https://mastodon.social/@Gargron", 20);
+			expect(result).not.toBeNull();
+			// May include recent posts section
+			if (result?.content?.includes("## Recent Posts")) {
+				expect(result.content).toMatch(/###\s+\w+/); // Date header
+			}
+		},
+		{ timeout: 30000 },
+	);
+
+	it("returns null for non-Mastodon instance with @user pattern", async () => {
+		// A site that has @user pattern but isn't Mastodon
+		const result = await handleMastodon("https://twitter.com/@jack", 20);
+		expect(result).toBeNull();
+	});
+});
+
+describe.skipIf(SKIP)("handleBluesky", () => {
+	it("returns null for non-Bluesky URLs", async () => {
+		const result = await handleBluesky("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for bsky.app URLs without profile path", async () => {
+		const result = await handleBluesky("https://bsky.app/about", 20);
+		expect(result).toBeNull();
+	});
+
+	it(
+		"fetches a Bluesky profile",
+		async () => {
+			// bsky.app official account - stable
+			const result = await handleBluesky("https://bsky.app/profile/bsky.app", 20);
+			expect(result).not.toBeNull();
+			expect(result?.method).toBe("bluesky-api");
+			expect(result?.contentType).toBe("text/markdown");
+			expect(result?.content).toContain("bsky.app");
+			expect(result?.content).toContain("@bsky.app");
+			expect(result?.content).toContain("**Followers:**");
+			expect(result?.content).toContain("**Following:**");
+			expect(result?.content).toContain("**Posts:**");
+			expect(result?.content).toContain("**DID:**");
+			expect(result?.fetchedAt).toBeTruthy();
+			expect(result?.truncated).toBeDefined();
+			expect(result?.notes).toContain("Fetched via AT Protocol API");
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"fetches Jay Graber's profile",
+		async () => {
+			// Jay Graber - CEO of Bluesky, very stable
+			const result = await handleBluesky("https://bsky.app/profile/jay.bsky.team", 20);
+			expect(result).not.toBeNull();
+			expect(result?.method).toBe("bluesky-api");
+			expect(result?.contentType).toBe("text/markdown");
+			expect(result?.content).toContain("@jay.bsky.team");
+			expect(result?.content).toContain("**Followers:**");
+			expect(result?.fetchedAt).toBeTruthy();
+			expect(result?.truncated).toBeDefined();
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"fetches a Bluesky post",
+		async () => {
+			// A post from bsky.app - use a well-known stable post
+			const result = await handleBluesky("https://bsky.app/profile/bsky.app/post/3juzlwllznd24", 20);
+			// Post may not exist, check gracefully
+			if (result !== null) {
+				expect(result.method).toBe("bluesky-api");
+				expect(result.contentType).toBe("text/markdown");
+				expect(result.content).toContain("# Bluesky Post");
+				expect(result.content).toContain("@bsky.app");
+				expect(result.fetchedAt).toBeTruthy();
+				expect(result.truncated).toBeDefined();
+				expect(result.notes?.[0]).toContain("AT URI");
+			}
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"includes post stats",
+		async () => {
+			const result = await handleBluesky("https://bsky.app/profile/bsky.app/post/3juzlwllznd24", 20);
+			// Stats include likes, reposts, replies
+			if (result?.content) {
+				// Should have some engagement markers
+				const hasStats =
+					result.content.includes("❤️") || result.content.includes("🔁") || result.content.includes("💬");
+				expect(hasStats || result.content.includes("# Bluesky Post")).toBe(true);
+			}
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"handles www.bsky.app URLs",
+		async () => {
+			const result = await handleBluesky("https://www.bsky.app/profile/bsky.app", 20);
+			expect(result).not.toBeNull();
+			expect(result?.method).toBe("bluesky-api");
+		},
+		{ timeout: 30000 },
+	);
+
+	it("returns null for invalid profile handle", async () => {
+		const result = await handleBluesky("https://bsky.app/profile/", 20);
+		expect(result).toBeNull();
+	});
+});

package/src/core/tools/web-scrapers/social.test.ts

@@ -0,0 +1,259 @@
+import { describe, expect, it } from "bun:test";
+import { handleReddit } from "./reddit";
+import { handleStackOverflow } from "./stackoverflow";
+import { handleTwitter } from "./twitter";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleTwitter", () => {
+	it("returns null for non-Twitter URLs", async () => {
+		const result = await handleTwitter("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it(
+		"handles twitter.com status URLs",
+		async () => {
+			const result = await handleTwitter("https://twitter.com/jack/status/20", 10000);
+			expect(result).not.toBeNull();
+			expect(result?.method).toMatch(/^twitter/);
+			expect(result?.contentType).toMatch(/^text\/(markdown|plain)$/);
+			// Either successful fetch or blocked/unavailable message
+			if (result?.method === "twitter-nitter") {
+				expect(result?.content).toContain("Tweet by");
+				expect(result?.notes?.[0]).toContain("Via Nitter");
+			} else if (result?.method === "twitter-blocked") {
+				expect(result?.content).toContain("blocks automated access");
+				expect(result?.notes?.[0]).toContain("Nitter instances unavailable");
+			}
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"handles x.com status URLs",
+		async () => {
+			const result = await handleTwitter("https://x.com/elonmusk/status/1", 10000);
+			expect(result).not.toBeNull();
+			expect(result?.method).toMatch(/^twitter/);
+			expect(result?.contentType).toMatch(/^text\/(markdown|plain)$/);
+			// Either successful fetch or blocked/unavailable message
+			if (result?.method === "twitter-nitter") {
+				expect(result?.finalUrl).toContain("nitter");
+			} else if (result?.method === "twitter-blocked") {
+				expect(result?.content).toContain("blocks automated access");
+			}
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"handles www.twitter.com URLs",
+		async () => {
+			const result = await handleTwitter("https://www.twitter.com/twitter/status/1", 10000);
+			expect(result).not.toBeNull();
+			expect(result?.method).toMatch(/^twitter/);
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"handles www.x.com URLs",
+		async () => {
+			const result = await handleTwitter("https://www.x.com/twitter/status/1", 10000);
+			expect(result).not.toBeNull();
+			expect(result?.method).toMatch(/^twitter/);
+		},
+		{ timeout: 30000 },
+	);
+
+	it(
+		"may fail due to Nitter availability",
+		async () => {
+			// Test that failure returns helpful message instead of null
+			const result = await handleTwitter("https://twitter.com/nonexistent/status/999999999999999999", 10000);
+			expect(result).not.toBeNull();
+			// Should return blocked message when Nitter fails
+			if (result?.method === "twitter-blocked") {
+				expect(result?.content).toContain("Nitter instances were unavailable");
+				expect(result?.content).toContain("Try:");
+			}
+		},
+		{ timeout: 30000 },
+	);
+});
+
+describe.skipIf(SKIP)("handleReddit", () => {
+	it("returns null for non-Reddit URLs", async () => {
+		const result = await handleReddit("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("fetches subreddit", async () => {
+		const result = await handleReddit("https://www.reddit.com/r/programming/", 20000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("reddit");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("# r/programming");
+		expect(result?.content).toMatch(/\*\*.*\*\*/); // Contains bold formatting
+		expect(result?.notes).toContain("Fetched via Reddit JSON API");
+	});
+
+	it("fetches individual post", async () => {
+		// Use a more reliable recent post URL
+		const result = await handleReddit("https://www.reddit.com/r/programming/", 20000);
+		// Individual post may fail if post doesn't exist, check if we get data
+		if (result !== null) {
+			expect(result.method).toBe("reddit");
+			expect(result.contentType).toBe("text/markdown");
+			expect(result.content).toContain("# r/");
+			expect(result.notes).toContain("Fetched via Reddit JSON API");
+		}
+	});
+
+	it("includes comments in post when available", async () => {
+		const result = await handleReddit("https://www.reddit.com/r/programming/", 20000);
+		// Comments test - just verify structure if post with comments is found
+		if (result?.content?.includes("## Top Comments")) {
+			expect(result.content).toContain("### u/");
+			expect(result.content).toContain("points");
+		}
+	});
+
+	it("handles old.reddit.com", async () => {
+		const result = await handleReddit("https://old.reddit.com/r/programming/", 20000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("reddit");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("# r/");
+		expect(result?.notes).toContain("Fetched via Reddit JSON API");
+	});
+
+	it("handles reddit.com without www", async () => {
+		const result = await handleReddit("https://reddit.com/r/programming/", 20000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("reddit");
+	});
+
+	it("handles URLs with query parameters", async () => {
+		const result = await handleReddit("https://www.reddit.com/r/programming/?sort=top", 20000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("reddit");
+		expect(result?.content).toContain("# r/");
+	});
+
+	it("returns null for malformed Reddit URLs", async () => {
+		const result = await handleReddit("https://www.reddit.com/invalid", 20000);
+		// May return null or empty result
+		if (result !== null) {
+			expect(result.content).toBeDefined();
+		}
+	});
+});
+
+describe.skipIf(SKIP)("handleStackOverflow", () => {
+	it("returns null for non-SO URLs", async () => {
+		const result = await handleStackOverflow("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for SO URLs without question ID", async () => {
+		const result = await handleStackOverflow("https://stackoverflow.com/", 10);
+		expect(result).toBeNull();
+	});
+
+	it("fetches a known question", async () => {
+		// Use a well-known question that definitely exists
+		const result = await handleStackOverflow(
+			"https://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster",
+			20000,
+		);
+		// API may fail or rate limit, check gracefully
+		if (result !== null) {
+			expect(result.method).toBe("stackoverflow");
+			expect(result.contentType).toBe("text/markdown");
+			expect(result.content).toContain("# ");
+			expect(result.content).toContain("**Score:");
+			expect(result.content).toContain("**Tags:");
+			expect(result.content).toContain("## Question");
+			expect(result.notes).toContain("Fetched via Stack Exchange API");
+		}
+	});
+
+	it("includes answers", async () => {
+		const result = await handleStackOverflow(
+			"https://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster",
+			20000,
+		);
+		if (result?.content?.includes("## Answers")) {
+			expect(result.content).toContain("### Score:");
+		}
+	});
+
+	it("shows accepted answer marker when present", async () => {
+		const result = await handleStackOverflow(
+			"https://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster",
+			20000,
+		);
+		// Some questions may have accepted answers
+		if (result?.content?.includes("(Accepted)")) {
+			expect(result.content).toContain("## Answers");
+		}
+	});
+
+	it("handles stackoverflow.com", async () => {
+		const result = await handleStackOverflow(
+			"https://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster-than-processing-an-unsorted-array",
+			20000,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("stackoverflow");
+		expect(result?.content).toContain("# ");
+		expect(result?.content).toContain("## Question");
+	});
+
+	it("handles other StackExchange sites", async () => {
+		const result = await handleStackOverflow("https://math.stackexchange.com/questions/1000/", 20000);
+		// API may fail, check gracefully
+		if (result !== null) {
+			expect(result.method).toBe("stackoverflow");
+			expect(result.contentType).toBe("text/markdown");
+			expect(result.content).toContain("# ");
+			expect(result.notes).toContain("Fetched via Stack Exchange API");
+		}
+	});
+
+	it("extracts question ID from URL", async () => {
+		const result = await handleStackOverflow(
+			"https://stackoverflow.com/questions/1234567/some-long-question-title",
+			20000,
+		);
+		// Should attempt to fetch, may or may not exist
+		// Either returns valid result or null
+		if (result !== null) {
+			expect(result.method).toBe("stackoverflow");
+		}
+	});
+
+	it("handles URLs without trailing slash", async () => {
+		const result = await handleStackOverflow("https://stackoverflow.com/questions/11227809", 20000);
+		// API may fail, check gracefully
+		if (result !== null) {
+			expect(result.method).toBe("stackoverflow");
+		}
+	});
+
+	it("includes question metadata", async () => {
+		const result = await handleStackOverflow(
+			"https://stackoverflow.com/questions/11227809/why-is-processing-a-sorted-array-faster",
+			20000,
+		);
+		// API may fail, check gracefully
+		if (result !== null) {
+			expect(result.content).toContain("**Score:");
+			expect(result.content).toContain("**Answers:");
+			expect(result.content).toContain("**Tags:");
+			expect(result.content).toContain("**Asked by:");
+		}
+	});
+});