@oh-my-pi/pi-coding-agent 3.30.0 → 3.31.0

package/src/core/tools/web-scrapers/index.ts

@@ -0,0 +1,250 @@
+/**
+ * Web Fetch Special Handlers Index
+ *
+ * Exports all special handlers for site-specific content extraction.
+ */
+
+import { handleArtifactHub } from "./artifacthub";
+import { handleArxiv } from "./arxiv";
+import { handleAur } from "./aur";
+import { handleBiorxiv } from "./biorxiv";
+import { handleBluesky } from "./bluesky";
+import { handleBrew } from "./brew";
+import { handleCheatSh } from "./cheatsh";
+import { handleChocolatey } from "./chocolatey";
+import { handleChooseALicense } from "./choosealicense";
+import { handleCisaKev } from "./cisa-kev";
+import { handleClojars } from "./clojars";
+import { handleCoinGecko } from "./coingecko";
+import { handleCratesIo } from "./crates-io";
+import { handleCrossref } from "./crossref";
+import { handleDevTo } from "./devto";
+import { handleDiscogs } from "./discogs";
+import { handleDiscourse } from "./discourse";
+import { handleDockerHub } from "./dockerhub";
+import { handleFdroid } from "./fdroid";
+import { handleFirefoxAddons } from "./firefox-addons";
+import { handleFlathub } from "./flathub";
+import { fetchGitHubApi, handleGitHub } from "./github";
+import { handleGitHubGist } from "./github-gist";
+import { handleGitLab } from "./gitlab";
+import { handleGoPkg } from "./go-pkg";
+import { handleHackage } from "./hackage";
+import { handleHackerNews } from "./hackernews";
+import { handleHex } from "./hex";
+import { handleHuggingFace } from "./huggingface";
+import { handleIacr } from "./iacr";
+import { handleJetBrainsMarketplace } from "./jetbrains-marketplace";
+import { handleLemmy } from "./lemmy";
+import { handleLobsters } from "./lobsters";
+import { handleMastodon } from "./mastodon";
+import { handleMaven } from "./maven";
+import { handleMDN } from "./mdn";
+import { handleMetaCPAN } from "./metacpan";
+import { handleMusicBrainz } from "./musicbrainz";
+import { handleNpm } from "./npm";
+import { handleNuGet } from "./nuget";
+import { handleNvd } from "./nvd";
+import { handleOllama } from "./ollama";
+import { handleOpenVsx } from "./open-vsx";
+import { handleOpenCorporates } from "./opencorporates";
+import { handleOpenLibrary } from "./openlibrary";
+import { handleOrcid } from "./orcid";
+import { handleOsv } from "./osv";
+import { handlePackagist } from "./packagist";
+import { handlePubDev } from "./pub-dev";
+import { handlePubMed } from "./pubmed";
+import { handlePyPI } from "./pypi";
+import { handleRawg } from "./rawg";
+import { handleReadTheDocs } from "./readthedocs";
+import { handleReddit } from "./reddit";
+import { handleRepology } from "./repology";
+import { handleRfc } from "./rfc";
+import { handleRubyGems } from "./rubygems";
+import { handleSearchcode } from "./searchcode";
+import { handleSecEdgar } from "./sec-edgar";
+import { handleSemanticScholar } from "./semantic-scholar";
+import { handleSnapcraft } from "./snapcraft";
+import { handleSourcegraph } from "./sourcegraph";
+import { handleSpdx } from "./spdx";
+import { handleSpotify } from "./spotify";
+import { handleStackOverflow } from "./stackoverflow";
+import { handleTerraform } from "./terraform";
+import { handleTldr } from "./tldr";
+import { handleTwitter } from "./twitter";
+import type { SpecialHandler } from "./types";
+import { handleVimeo } from "./vimeo";
+import { handleVscodeMarketplace } from "./vscode-marketplace";
+import { handleW3c } from "./w3c";
+import { handleWikidata } from "./wikidata";
+import { handleWikipedia } from "./wikipedia";
+import { handleYouTube } from "./youtube";
+
+export type { RenderResult, SpecialHandler } from "./types";
+
+export {
+	fetchGitHubApi,
+	handleArtifactHub,
+	handleArxiv,
+	handleAur,
+	handleBiorxiv,
+	handleBluesky,
+	handleBrew,
+	handleCheatSh,
+	handleCisaKev,
+	handleChocolatey,
+	handleClojars,
+	handleChooseALicense,
+	handleCoinGecko,
+	handleCratesIo,
+	handleCrossref,
+	handleDevTo,
+	handleDiscogs,
+	handleDiscourse,
+	handleDockerHub,
+	handleFdroid,
+	handleFlathub,
+	handleFirefoxAddons,
+	handleGitHub,
+	handleGitHubGist,
+	handleGitLab,
+	handleGoPkg,
+	handleHackage,
+	handleHackerNews,
+	handleHex,
+	handleHuggingFace,
+	handleIacr,
+	handleJetBrainsMarketplace,
+	handleLemmy,
+	handleLobsters,
+	handleMastodon,
+	handleMaven,
+	handleMDN,
+	handleMetaCPAN,
+	handleMusicBrainz,
+	handleNpm,
+	handleNuGet,
+	handleNvd,
+	handleOllama,
+	handleOpenCorporates,
+	handleOpenLibrary,
+	handleOrcid,
+	handleOpenVsx,
+	handleOsv,
+	handlePackagist,
+	handlePubDev,
+	handlePubMed,
+	handlePyPI,
+	handleRawg,
+	handleReadTheDocs,
+	handleReddit,
+	handleRepology,
+	handleRfc,
+	handleRubyGems,
+	handleSecEdgar,
+	handleSearchcode,
+	handleSemanticScholar,
+	handleSnapcraft,
+	handleSourcegraph,
+	handleSpotify,
+	handleSpdx,
+	handleStackOverflow,
+	handleTerraform,
+	handleTldr,
+	handleTwitter,
+	handleVimeo,
+	handleVscodeMarketplace,
+	handleW3c,
+	handleWikidata,
+	handleWikipedia,
+	handleYouTube,
+};
+
+export const specialHandlers: SpecialHandler[] = [
+	// Git hosting
+	handleGitHubGist,
+	handleGitHub,
+	handleGitLab,
+	// Video/Media
+	handleYouTube,
+	handleVimeo,
+	handleSpotify,
+	handleDiscogs,
+	handleMusicBrainz,
+	// Games
+	handleRawg,
+	// Social/News
+	handleTwitter,
+	handleBluesky,
+	handleMastodon,
+	handleLemmy,
+	handleHackerNews,
+	handleLobsters,
+	handleReddit,
+	handleDiscourse,
+	// Developer content
+	handleStackOverflow,
+	handleDevTo,
+	handleMDN,
+	handleReadTheDocs,
+	handleSearchcode,
+	handleSourcegraph,
+	handleTldr,
+	handleCheatSh,
+	// Package registries
+	handleNpm,
+	handleFirefoxAddons,
+	handleVscodeMarketplace,
+	handleNuGet,
+	handleChocolatey,
+	handleClojars,
+	handleBrew,
+	handlePyPI,
+	handleCratesIo,
+	handleDockerHub,
+	handleFdroid,
+	handleFlathub,
+	handleGoPkg,
+	handleHex,
+	handlePackagist,
+	handlePubDev,
+	handleMaven,
+	handleJetBrainsMarketplace,
+	handleOpenVsx,
+	handleArtifactHub,
+	handleRubyGems,
+	handleTerraform,
+	handleAur,
+	handleHackage,
+	handleMetaCPAN,
+	handleRepology,
+	handleSnapcraft,
+	// ML/AI
+	handleHuggingFace,
+	handleOllama,
+	// Academic
+	handleArxiv,
+	handleBiorxiv,
+	handleCrossref,
+	handleIacr,
+	handleOrcid,
+	handleSemanticScholar,
+	handlePubMed,
+	handleRfc,
+	// Security
+	handleCisaKev,
+	handleNvd,
+	handleOsv,
+	// Crypto
+	handleCoinGecko,
+	// Business
+	handleOpenCorporates,
+	handleSecEdgar,
+	// Reference
+	handleOpenLibrary,
+	handleChooseALicense,
+	handleW3c,
+	handleSpdx,
+	handleWikidata,
+	handleWikipedia,
+];

package/src/core/tools/web-scrapers/jetbrains-marketplace.ts

@@ -0,0 +1,169 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface PluginVendor {
+	name?: string;
+	publicName?: string;
+	url?: string;
+}
+
+interface PluginTag {
+	name?: string;
+}
+
+type PluginRating =
+	| number
+	| {
+			rating?: number;
+			value?: number;
+			score?: number;
+			votes?: number;
+			totalVotes?: number;
+			count?: number;
+	  };
+
+interface PluginData {
+	id?: number;
+	name?: string;
+	description?: string;
+	preview?: string;
+	vendor?: PluginVendor;
+	rating?: PluginRating;
+	ratingCount?: number;
+	downloads?: number;
+	tags?: PluginTag[];
+	urls?: {
+		url?: string;
+		docUrl?: string;
+		sourceCodeUrl?: string;
+		bugtrackerUrl?: string;
+	};
+}
+
+interface UpdateData {
+	version?: string;
+	since?: string;
+	until?: string;
+	sinceUntil?: string;
+	channel?: string;
+	downloads?: number;
+	compatibleVersions?: Record<string, string>;
+	cdate?: string | number;
+}
+
+const MARKETPLACE_HOSTS = new Set(["plugins.jetbrains.com"]);
+
+function extractRating(plugin: PluginData): { value: number | null; votes: number | null } {
+	const rating = plugin.rating;
+	if (typeof rating === "number" && Number.isFinite(rating)) {
+		return { value: rating, votes: plugin.ratingCount ?? null };
+	}
+	if (rating && typeof rating === "object") {
+		const value = rating.rating ?? rating.value ?? rating.score ?? null;
+		const votes = rating.votes ?? rating.totalVotes ?? rating.count ?? plugin.ratingCount ?? null;
+		return { value: typeof value === "number" ? value : null, votes: typeof votes === "number" ? votes : null };
+	}
+	return { value: null, votes: plugin.ratingCount ?? null };
+}
+
+function formatBuildCompatibility(update: UpdateData): string | null {
+	if (update.sinceUntil) return update.sinceUntil;
+	if (update.since && update.until) return `${update.since} - ${update.until}`;
+	if (update.since) return `${update.since}+`;
+	return null;
+}
+
+export const handleJetBrainsMarketplace: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!MARKETPLACE_HOSTS.has(parsed.hostname)) return null;
+
+		const match = parsed.pathname.match(/^\/plugin\/(\d+)(?:-[^/]+)?(?:\/|$)/);
+		if (!match) return null;
+
+		const pluginId = match[1];
+		const fetchedAt = new Date().toISOString();
+
+		const pluginUrl = `https://plugins.jetbrains.com/api/plugins/${pluginId}`;
+		const updatesUrl = `https://plugins.jetbrains.com/api/plugins/${pluginId}/updates?size=1`;
+
+		const [pluginResult, updatesResult] = await Promise.all([
+			loadPage(pluginUrl, { timeout, signal }),
+			loadPage(updatesUrl, { timeout, signal }),
+		]);
+
+		if (!pluginResult.ok || !updatesResult.ok) return null;
+
+		let plugin: PluginData;
+		let updates: UpdateData[];
+
+		try {
+			plugin = JSON.parse(pluginResult.content) as PluginData;
+			updates = JSON.parse(updatesResult.content) as UpdateData[];
+		} catch {
+			return null;
+		}
+
+		const update = updates[0];
+		if (!plugin?.name) return null;
+
+		const vendorName = plugin.vendor?.name ?? plugin.vendor?.publicName;
+		const descriptionSource = plugin.description ?? plugin.preview ?? "";
+		const description = descriptionSource ? htmlToBasicMarkdown(descriptionSource) : "";
+		const tags = (plugin.tags ?? []).map((tag) => tag.name).filter(Boolean) as string[];
+		const rating = extractRating(plugin);
+		const buildCompatibility = update ? formatBuildCompatibility(update) : null;
+
+		let md = `# ${plugin.name}\n\n`;
+		if (description) md += `${description}\n\n`;
+
+		md += `**Plugin ID:** ${pluginId}\n`;
+		if (vendorName) md += `**Vendor:** ${vendorName}\n`;
+		if (plugin.downloads !== undefined) {
+			md += `**Downloads:** ${formatCount(plugin.downloads)}\n`;
+		}
+		if (rating.value !== null) {
+			md += `**Rating:** ${rating.value.toFixed(2)}`;
+			if (rating.votes !== null) md += ` (${formatCount(rating.votes)} votes)`;
+			md += "\n";
+		}
+		if (tags.length) md += `**Tags:** ${tags.join(", ")}\n`;
+
+		if (update) {
+			md += "\n## Latest Release\n\n";
+			if (update.version) md += `**Version:** ${update.version}\n`;
+			if (update.channel) md += `**Channel:** ${update.channel}\n`;
+			if (buildCompatibility) md += `**Build Compatibility:** ${buildCompatibility}\n`;
+			if (update.downloads !== undefined) {
+				md += `**Release Downloads:** ${formatCount(update.downloads)}\n`;
+			}
+		}
+
+		const compatibility = update?.compatibleVersions ?? {};
+		const compatibilityEntries = Object.entries(compatibility).sort(([a], [b]) => a.localeCompare(b));
+		if (compatibilityEntries.length) {
+			md += "\n## IDE Compatibility\n\n";
+			for (const [product, version] of compatibilityEntries) {
+				md += `- ${product}: ${version}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "jetbrains-marketplace",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via JetBrains Marketplace API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/lemmy.ts

@@ -0,0 +1,220 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface LemmyCreator {
+	name: string;
+	actor_id?: string;
+}
+
+interface LemmyCommunity {
+	name: string;
+	actor_id?: string;
+}
+
+interface LemmyCounts {
+	score: number;
+	comments?: number;
+}
+
+interface LemmyPost {
+	id: number;
+	name: string;
+	body?: string;
+	url?: string;
+}
+
+interface LemmyPostView {
+	post: LemmyPost;
+	creator: LemmyCreator;
+	community: LemmyCommunity;
+	counts: LemmyCounts;
+}
+
+interface LemmyPostResponse {
+	post_view?: LemmyPostView;
+}
+
+interface LemmyComment {
+	id: number;
+	content?: string;
+	path?: string;
+	parent_id?: number | null;
+	post_id?: number;
+}
+
+interface LemmyCommentView {
+	comment: LemmyComment;
+	creator: LemmyCreator;
+	counts: LemmyCounts;
+}
+
+interface LemmyCommentListResponse {
+	comments?: LemmyCommentView[];
+}
+
+interface LemmyCommentResponse {
+	comment_view?: LemmyCommentView;
+}
+
+function parseJson<T>(content: string): T | null {
+	try {
+		return JSON.parse(content) as T;
+	} catch {
+		return null;
+	}
+}
+
+function formatCommunity(community: LemmyCommunity): string {
+	if (community.actor_id) {
+		try {
+			const host = new URL(community.actor_id).hostname;
+			return `!${community.name}@${host}`;
+		} catch {}
+	}
+	return `!${community.name}`;
+}
+
+function formatAuthor(creator: LemmyCreator): string {
+	if (creator.actor_id) {
+		try {
+			const host = new URL(creator.actor_id).hostname;
+			return `@${creator.name}@${host}`;
+		} catch {}
+	}
+	return creator.name;
+}
+
+function indentBlock(text: string, indent: string): string {
+	return text
+		.split("\n")
+		.map((line) => `${indent}${line}`)
+		.join("\n");
+}
+
+function renderComments(comments: LemmyCommentView[]): string {
+	const childrenByParent = new Map<number, LemmyCommentView[]>();
+
+	const commentIds = new Set(comments.map((view) => view.comment.id));
+
+	for (const commentView of comments) {
+		const parentId = commentView.comment.parent_id;
+		const resolvedParent = parentId && commentIds.has(parentId) ? parentId : 0;
+		const list = childrenByParent.get(resolvedParent);
+		if (list) {
+			list.push(commentView);
+		} else {
+			childrenByParent.set(resolvedParent, [commentView]);
+		}
+	}
+
+	const renderThread = (parentId: number, depth: number): string => {
+		const items = childrenByParent.get(parentId) ?? [];
+		let output = "";
+
+		for (const view of items) {
+			const author = view.creator?.name ? formatAuthor(view.creator) : "unknown";
+			const score = view.counts?.score ?? 0;
+			const content = (view.comment.content ?? "").trim();
+			const indent = "  ".repeat(depth);
+
+			output += `${indent}- **${author}** · ${score} points\n`;
+			if (content) {
+				output += `${indentBlock(content, `${indent}  `)}\n`;
+			}
+
+			output += renderThread(view.comment.id, depth + 1);
+			output += "\n";
+		}
+
+		return output;
+	};
+
+	return renderThread(0, 0).trim();
+}
+
+export const handleLemmy: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const match = parsed.pathname.match(/^\/(post|comment)\/(\d+)/);
+		if (!match) return null;
+
+		const kind = match[1];
+		const id = Number.parseInt(match[2], 10);
+		if (!Number.isFinite(id)) return null;
+
+		const baseUrl = parsed.origin;
+		const fetchedAt = new Date().toISOString();
+
+		let postId = id;
+		if (kind === "comment") {
+			const commentUrl = `${baseUrl}/api/v3/comment?id=${id}`;
+			const commentResult = await loadPage(commentUrl, { timeout, signal });
+			if (!commentResult.ok) return null;
+
+			const commentData = parseJson<LemmyCommentResponse>(commentResult.content);
+			const commentView = commentData?.comment_view;
+			const commentPostId = commentView?.comment?.post_id;
+			if (!commentPostId) return null;
+			postId = commentPostId;
+		}
+
+		const postUrl = `${baseUrl}/api/v3/post?id=${postId}`;
+		const commentsUrl = `${baseUrl}/api/v3/comment/list?post_id=${postId}`;
+
+		const [postResult, commentsResult] = await Promise.all([
+			loadPage(postUrl, { timeout, signal }),
+			loadPage(commentsUrl, { timeout, signal }),
+		]);
+
+		if (!postResult.ok || !commentsResult.ok) return null;
+
+		const postData = parseJson<LemmyPostResponse>(postResult.content);
+		const postView = postData?.post_view;
+		if (!postView) return null;
+
+		const commentsData = parseJson<LemmyCommentListResponse>(commentsResult.content);
+		const comments = commentsData?.comments ?? [];
+
+		let md = `# ${postView.post.name}\n\n`;
+
+		const communityLabel = formatCommunity(postView.community);
+		const authorLabel = formatAuthor(postView.creator);
+		const score = postView.counts?.score ?? 0;
+		const commentCount = postView.counts?.comments ?? comments.length;
+
+		md += `**Community:** ${communityLabel} · **Author:** ${authorLabel} · **Score:** ${score} · **Comments:** ${commentCount}\n`;
+		if (postView.post.url) {
+			md += `**Link:** ${postView.post.url}\n`;
+		}
+		md += "\n";
+
+		if (postView.post.body) {
+			md += `---\n\n${postView.post.body}\n\n`;
+		}
+
+		if (comments.length > 0) {
+			const threadedComments = renderComments(comments);
+			if (threadedComments) {
+				md += `---\n\n## Comments\n\n${threadedComments}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "lemmy-api",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Lemmy API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/{web-fetch-handlers → web-scrapers}/lobsters.ts

@@ -74,7 +74,7 @@ function renderComments(comments: LobstersComment[], maxDepth = 5): string {
 /**
  * Handle Lobste.rs URLs via JSON API
  */
-export const handleLobsters: SpecialHandler = async (url: string, timeout: number) => {
+export const handleLobsters: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
 	try {
 		const parsed = new URL(url);
 		if (!parsed.hostname.includes("lobste.rs")) return null;
@@ -87,7 +87,7 @@ export const handleLobsters: SpecialHandler = async (url: string, timeout: numbe
 		const storyMatch = parsed.pathname.match(/^\/s\/([^/]+)/);
 		if (storyMatch) {
 			jsonUrl = `https://lobste.rs/s/${storyMatch[1]}.json`;
-			const result = await loadPage(jsonUrl, { timeout });
+			const result = await loadPage(jsonUrl, { timeout, signal });
 			if (!result.ok) return null;
 
 			const story = JSON.parse(result.content) as LobstersStoryResponse;
@@ -140,7 +140,7 @@ export const handleLobsters: SpecialHandler = async (url: string, timeout: numbe
 
 			if (!jsonUrl) return null;
 
-			const result = await loadPage(jsonUrl, { timeout });
+			const result = await loadPage(jsonUrl, { timeout, signal });
 			if (!result.ok) return null;
 
 			const stories = JSON.parse(result.content) as LobstersStory[];