@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/lobsters.ts

@@ -0,0 +1,186 @@
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+// =============================================================================
+// Lobste.rs Types
+// =============================================================================
+
+interface LobstersStory {
+	short_id: string;
+	title: string;
+	url?: string;
+	description?: string;
+	submitter_user: {
+		username: string;
+	};
+	score: number;
+	comment_count: number;
+	created_at: string;
+	tags: string[];
+}
+
+interface LobstersComment {
+	short_id: string;
+	comment: string;
+	commenting_user: {
+		username: string;
+	};
+	score: number;
+	created_at: string;
+	indent_level: number;
+	comments?: LobstersComment[];
+}
+
+interface LobstersStoryResponse {
+	short_id: string;
+	title: string;
+	url?: string;
+	description?: string;
+	submitter_user: {
+		username: string;
+	};
+	score: number;
+	comment_count: number;
+	created_at: string;
+	tags: string[];
+	comments: LobstersComment[];
+}
+
+// =============================================================================
+// Handler
+// =============================================================================
+
+/**
+ * Render comments recursively
+ */
+function renderComments(comments: LobstersComment[], maxDepth = 5): string {
+	let md = "";
+	for (const comment of comments) {
+		if (comment.indent_level >= maxDepth) continue;
+
+		const indent = "  ".repeat(comment.indent_level);
+		md += `${indent}### ${comment.commenting_user.username} · ${comment.score} points\n\n`;
+		md += `${indent}${comment.comment.split("\n").join(`\n${indent}`)}\n\n`;
+
+		if (comment.comments && comment.comments.length > 0) {
+			md += renderComments(comment.comments, maxDepth);
+		}
+
+		md += `${indent}---\n\n`;
+	}
+	return md;
+}
+
+/**
+ * Handle Lobste.rs URLs via JSON API
+ */
+export const handleLobsters: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("lobste.rs")) return null;
+
+		const fetchedAt = new Date().toISOString();
+		let jsonUrl = "";
+		let md = "";
+
+		// Story page: lobste.rs/s/{short_id}/{slug}
+		const storyMatch = parsed.pathname.match(/^\/s\/([^/]+)/);
+		if (storyMatch) {
+			jsonUrl = `https://lobste.rs/s/${storyMatch[1]}.json`;
+			const result = await loadPage(jsonUrl, { timeout, signal });
+			if (!result.ok) return null;
+
+			const story = JSON.parse(result.content) as LobstersStoryResponse;
+
+			md = `# ${story.title}\n\n`;
+			md += `**${story.submitter_user.username}** · ${story.score} points · ${story.comment_count} comments`;
+			if (story.tags.length > 0) {
+				md += ` · [${story.tags.join(", ")}]`;
+			}
+			md += `\n`;
+			md += `*${new Date(story.created_at).toISOString().split("T")[0]}*\n\n`;
+
+			if (story.description) {
+				md += `---\n\n${story.description}\n\n`;
+			} else if (story.url) {
+				md += `**Link:** ${story.url}\n\n`;
+			}
+
+			// Add comments
+			if (story.comments && story.comments.length > 0) {
+				md += `---\n\n## Comments\n\n`;
+				md += renderComments(story.comments);
+			}
+
+			const output = finalizeOutput(md);
+			return {
+				url,
+				finalUrl: jsonUrl,
+				contentType: "text/markdown",
+				method: "lobsters",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes: ["Fetched via Lobste.rs JSON API"],
+			};
+		}
+
+		// Front page, newest, or tag page
+		if (parsed.pathname === "/" || parsed.pathname === "/newest" || parsed.pathname.startsWith("/t/")) {
+			if (parsed.pathname === "/") {
+				jsonUrl = "https://lobste.rs/hottest.json";
+			} else if (parsed.pathname === "/newest") {
+				jsonUrl = "https://lobste.rs/newest.json";
+			} else {
+				const tagMatch = parsed.pathname.match(/^\/t\/([^/]+)/);
+				if (tagMatch) {
+					jsonUrl = `https://lobste.rs/t/${tagMatch[1]}.json`;
+				}
+			}
+
+			if (!jsonUrl) return null;
+
+			const result = await loadPage(jsonUrl, { timeout, signal });
+			if (!result.ok) return null;
+
+			const stories = JSON.parse(result.content) as LobstersStory[];
+			const listingStories = stories.slice(0, 20);
+
+			const title =
+				parsed.pathname === "/"
+					? "Lobste.rs Front Page"
+					: parsed.pathname === "/newest"
+						? "Lobste.rs Newest"
+						: `Lobste.rs Tag: ${parsed.pathname.split("/")[2]}`;
+
+			md = `# ${title}\n\n`;
+
+			for (const story of listingStories) {
+				md += `- **${story.title}** (${story.score} pts, ${story.comment_count} comments)\n`;
+				md += `  by ${story.submitter_user.username}`;
+				if (story.tags.length > 0) {
+					md += ` · [${story.tags.join(", ")}]`;
+				}
+				md += `\n`;
+				if (story.url) {
+					md += `  ${story.url}\n`;
+				}
+				md += `  https://lobste.rs/s/${story.short_id}\n\n`;
+			}
+
+			const output = finalizeOutput(md);
+			return {
+				url,
+				finalUrl: jsonUrl,
+				contentType: "text/markdown",
+				method: "lobsters",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes: ["Fetched via Lobste.rs JSON API"],
+			};
+		}
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/mastodon.ts

@@ -0,0 +1,310 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface MastodonAccount {
+	id: string;
+	username: string;
+	acct: string;
+	display_name: string;
+	note: string;
+	url: string;
+	avatar: string;
+	header: string;
+	followers_count: number;
+	following_count: number;
+	statuses_count: number;
+	created_at: string;
+	bot: boolean;
+	fields?: Array<{ name: string; value: string }>;
+}
+
+interface MastodonMediaAttachment {
+	id: string;
+	type: "image" | "video" | "gifv" | "audio" | "unknown";
+	url: string;
+	preview_url?: string;
+	description?: string;
+}
+
+interface MastodonStatus {
+	id: string;
+	created_at: string;
+	content: string;
+	url: string;
+	account: MastodonAccount;
+	reblogs_count: number;
+	favourites_count: number;
+	replies_count: number;
+	reblog?: MastodonStatus;
+	media_attachments: MastodonMediaAttachment[];
+	spoiler_text?: string;
+	sensitive: boolean;
+	visibility: "public" | "unlisted" | "private" | "direct";
+	in_reply_to_id?: string;
+	poll?: {
+		options: Array<{ title: string; votes_count: number }>;
+		votes_count: number;
+		expired: boolean;
+	};
+}
+
+/**
+ * Check if a domain is a Mastodon instance by probing the API
+ */
+async function isMastodonInstance(hostname: string, timeout: number, signal?: AbortSignal): Promise<boolean> {
+	try {
+		const result = await loadPage(`https://${hostname}/api/v1/instance`, {
+			timeout: Math.min(timeout, 5),
+			headers: { Accept: "application/json" },
+			signal,
+		});
+		if (!result.ok) return false;
+		const data = JSON.parse(result.content);
+		// Mastodon instances return uri/domain field
+		return !!(data.uri || data.domain || data.title);
+	} catch {
+		return false;
+	}
+}
+
+/**
+ * Format a date string to readable format
+ */
+function formatDate(isoDate: string): string {
+	try {
+		const date = new Date(isoDate);
+		return date.toLocaleDateString("en-US", {
+			year: "numeric",
+			month: "short",
+			day: "numeric",
+			hour: "2-digit",
+			minute: "2-digit",
+		});
+	} catch {
+		return isoDate;
+	}
+}
+
+/**
+ * Format a status/post as markdown
+ */
+function formatStatus(status: MastodonStatus, isReblog = false): string {
+	// Handle reblogs (boosts)
+	if (status.reblog && !isReblog) {
+		let md = `🔁 **${status.account.display_name || status.account.username}** boosted:\n\n`;
+		md += formatStatus(status.reblog, true);
+		return md;
+	}
+
+	const account = status.account;
+	let md = "";
+
+	if (!isReblog) {
+		md += `# Post by ${account.display_name || account.username}\n\n`;
+	}
+
+	md += `**@${account.acct}**`;
+	if (account.bot) md += " 🤖";
+	md += ` · ${formatDate(status.created_at)}`;
+	if (status.visibility !== "public") md += ` · ${status.visibility}`;
+	md += "\n\n";
+
+	// Content warning / spoiler
+	if (status.spoiler_text) {
+		md += `> ⚠️ **CW:** ${status.spoiler_text}\n\n`;
+	}
+
+	// Main content (convert HTML to markdown)
+	const content = htmlToBasicMarkdown(status.content);
+	md += `${content}\n\n`;
+
+	// Poll
+	if (status.poll) {
+		md += "**Poll:**\n";
+		for (const option of status.poll.options) {
+			const pct =
+				status.poll.votes_count > 0 ? ((option.votes_count / status.poll.votes_count) * 100).toFixed(1) : "0";
+			md += `- ${option.title} (${pct}%, ${option.votes_count} votes)\n`;
+		}
+		md += `Total: ${status.poll.votes_count} votes${status.poll.expired ? " (closed)" : ""}\n\n`;
+	}
+
+	// Media attachments
+	if (status.media_attachments.length > 0) {
+		md += "**Attachments:**\n";
+		for (const media of status.media_attachments) {
+			const desc = media.description ? ` - ${media.description}` : "";
+			md += `- [${media.type}](${media.url})${desc}\n`;
+		}
+		md += "\n";
+	}
+
+	// Stats
+	md += `---\n`;
+	md += `💬 ${formatCount(status.replies_count)} replies · `;
+	md += `🔁 ${formatCount(status.reblogs_count)} boosts · `;
+	md += `⭐ ${formatCount(status.favourites_count)} favorites\n`;
+
+	return md;
+}
+
+/**
+ * Format an account/profile as markdown
+ */
+function formatAccount(account: MastodonAccount): string {
+	let md = `# ${account.display_name || account.username}\n\n`;
+
+	md += `**@${account.acct}**`;
+	if (account.bot) md += " 🤖 Bot";
+	md += "\n\n";
+
+	// Bio
+	if (account.note) {
+		const bio = htmlToBasicMarkdown(account.note);
+		if (bio && bio !== account.display_name) {
+			md += `${bio}\n\n`;
+		}
+	}
+
+	// Stats
+	md += `**Followers:** ${formatCount(account.followers_count)} · `;
+	md += `**Following:** ${formatCount(account.following_count)} · `;
+	md += `**Posts:** ${formatCount(account.statuses_count)}\n\n`;
+
+	md += `**Joined:** ${formatDate(account.created_at)}\n`;
+	md += `**Profile:** ${account.url}\n`;
+
+	// Profile fields (links, pronouns, etc.)
+	if (account.fields && account.fields.length > 0) {
+		md += "\n**Profile Fields:**\n";
+		for (const field of account.fields) {
+			const value = htmlToBasicMarkdown(field.value);
+			md += `- **${field.name}:** ${value}\n`;
+		}
+	}
+
+	return md;
+}
+
+/**
+ * Handle Mastodon/Fediverse URLs
+ */
+export const handleMastodon: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+
+		// Check for @user/postid or @user pattern
+		const postMatch = parsed.pathname.match(/^\/@([^/]+)\/(\d+)$/);
+		const profileMatch = parsed.pathname.match(/^\/@([^/]+)$/);
+
+		if (!postMatch && !profileMatch) return null;
+
+		// Verify this is a Mastodon instance
+		if (!(await isMastodonInstance(parsed.hostname, timeout, signal))) {
+			return null;
+		}
+
+		const fetchedAt = new Date().toISOString();
+		const instance = parsed.hostname;
+
+		if (postMatch) {
+			// Fetch status/post
+			const [, , statusId] = postMatch;
+			const apiUrl = `https://${instance}/api/v1/statuses/${statusId}`;
+
+			const result = await loadPage(apiUrl, {
+				timeout,
+				headers: { Accept: "application/json" },
+				signal,
+			});
+
+			if (!result.ok) return null;
+
+			let status: MastodonStatus;
+			try {
+				status = JSON.parse(result.content);
+			} catch {
+				return null;
+			}
+
+			const md = formatStatus(status);
+			const output = finalizeOutput(md);
+
+			return {
+				url,
+				finalUrl: status.url || url,
+				contentType: "text/markdown",
+				method: "mastodon",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes: [`Fetched via Mastodon API (${instance})`],
+			};
+		}
+
+		if (profileMatch) {
+			// Fetch account by username lookup
+			const [, username] = profileMatch;
+			const lookupUrl = `https://${instance}/api/v1/accounts/lookup?acct=${encodeURIComponent(username)}`;
+
+			const result = await loadPage(lookupUrl, {
+				timeout,
+				headers: { Accept: "application/json" },
+				signal,
+			});
+
+			if (!result.ok) return null;
+
+			let account: MastodonAccount;
+			try {
+				account = JSON.parse(result.content);
+			} catch {
+				return null;
+			}
+
+			// Fetch recent statuses
+			const statusesUrl = `https://${instance}/api/v1/accounts/${account.id}/statuses?limit=5&exclude_replies=true`;
+			const statusesResult = await loadPage(statusesUrl, {
+				timeout,
+				headers: { Accept: "application/json" },
+				signal,
+			});
+
+			let md = formatAccount(account);
+
+			if (statusesResult.ok) {
+				try {
+					const statuses: MastodonStatus[] = JSON.parse(statusesResult.content);
+					if (statuses.length > 0) {
+						md += "\n---\n\n## Recent Posts\n\n";
+						for (const status of statuses.slice(0, 5)) {
+							md += `### ${formatDate(status.created_at)}\n\n`;
+							const content = htmlToBasicMarkdown(status.content);
+							md += `${content}\n\n`;
+							md += `💬 ${status.replies_count} · 🔁 ${status.reblogs_count} · ⭐ ${status.favourites_count}\n\n`;
+						}
+					}
+				} catch {}
+			}
+
+			const output = finalizeOutput(md);
+
+			return {
+				url,
+				finalUrl: account.url || url,
+				contentType: "text/markdown",
+				method: "mastodon",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes: [`Fetched via Mastodon API (${instance})`],
+			};
+		}
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/maven.ts

@@ -0,0 +1,152 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface MavenDoc {
+	id: string;
+	g: string; // groupId
+	a: string; // artifactId
+	latestVersion: string;
+	repositoryId: string;
+	p: string; // packaging
+	timestamp: number;
+	versionCount: number;
+	text?: string[];
+	ec?: string[]; // extensions/classifiers
+}
+
+interface MavenResponse {
+	response: {
+		numFound: number;
+		docs: MavenDoc[];
+	};
+}
+
+/**
+ * Handle Maven Central URLs via Solr API
+ * Supports: search.maven.org/artifact/... and mvnrepository.com/artifact/...
+ */
+export const handleMaven: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const hostname = parsed.hostname;
+
+		// Check if this is a Maven URL
+		const isSearchMaven = hostname === "search.maven.org";
+		const isMvnRepository = hostname === "mvnrepository.com" || hostname === "www.mvnrepository.com";
+
+		if (!isSearchMaven && !isMvnRepository) return null;
+
+		let groupId: string | null = null;
+		let artifactId: string | null = null;
+		let version: string | null = null;
+
+		if (isSearchMaven) {
+			// Pattern: /artifact/{groupId}/{artifactId}[/{version}[/{packaging}]]
+			const match = parsed.pathname.match(/^\/artifact\/([^/]+)\/([^/]+)(?:\/([^/]+))?/);
+			if (!match) return null;
+			groupId = match[1];
+			artifactId = match[2];
+			version = match[3] || null;
+		} else if (isMvnRepository) {
+			// Pattern: /artifact/{groupId}/{artifactId}[/{version}]
+			const match = parsed.pathname.match(/^\/artifact\/([^/]+)\/([^/]+)(?:\/([^/]+))?/);
+			if (!match) return null;
+			groupId = match[1];
+			artifactId = match[2];
+			version = match[3] || null;
+		}
+
+		if (!groupId || !artifactId) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Query Maven Central API
+		const apiUrl = `https://search.maven.org/solrsearch/select?q=g:${encodeURIComponent(groupId)}+AND+a:${encodeURIComponent(artifactId)}&wt=json&rows=1`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let data: MavenResponse;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		if (data.response.numFound === 0) return null;
+
+		const doc = data.response.docs[0];
+		const displayVersion = version || doc.latestVersion;
+
+		let md = `# ${doc.g}:${doc.a}\n\n`;
+		md += `**Group ID:** ${doc.g}\n`;
+		md += `**Artifact ID:** ${doc.a}\n`;
+		md += `**Latest Version:** ${doc.latestVersion}`;
+		if (version && version !== doc.latestVersion) {
+			md += ` (viewing ${version})`;
+		}
+		md += "\n";
+
+		if (doc.p) md += `**Packaging:** ${doc.p}\n`;
+		if (doc.versionCount) md += `**Versions:** ${formatCount(doc.versionCount)}\n`;
+		if (doc.timestamp) {
+			const date = new Date(doc.timestamp);
+			md += `**Last Updated:** ${date.toISOString().split("T")[0]}\n`;
+		}
+
+		// Add dependency snippets
+		md += `\n## Maven Dependency\n\n`;
+		md += "```xml\n";
+		md += `<dependency>\n`;
+		md += `    <groupId>${doc.g}</groupId>\n`;
+		md += `    <artifactId>${doc.a}</artifactId>\n`;
+		md += `    <version>${displayVersion}</version>\n`;
+		md += `</dependency>\n`;
+		md += "```\n";
+
+		md += `\n## Gradle Dependency\n\n`;
+		md += "```groovy\n";
+		md += `implementation '${doc.g}:${doc.a}:${displayVersion}'\n`;
+		md += "```\n";
+
+		md += `\n## Gradle (Kotlin DSL)\n\n`;
+		md += "```kotlin\n";
+		md += `implementation("${doc.g}:${doc.a}:${displayVersion}")\n`;
+		md += "```\n";
+
+		// Add available classifiers/extensions if present
+		if (doc.ec && doc.ec.length > 0) {
+			const extensions = doc.ec.filter((e) => e && e !== "-");
+			if (extensions.length > 0) {
+				md += `\n## Available Extensions\n\n`;
+				md += `${extensions.map((e) => `- ${e}`).join("\n")}\n`;
+			}
+		}
+
+		md += `\n## Links\n\n`;
+		md += `- [Maven Central](https://search.maven.org/artifact/${doc.g}/${doc.a}/${displayVersion}/jar)\n`;
+		md += `- [MVN Repository](https://mvnrepository.com/artifact/${doc.g}/${doc.a}/${displayVersion})\n`;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "maven",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Maven Central API"],
+		};
+	} catch {}
+
+	return null;
+};