@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/discogs.ts

@@ -0,0 +1,308 @@
+/**
+ * Discogs URL handler for music releases and masters
+ *
+ * Uses the Discogs API to extract structured metadata about releases.
+ * API docs: https://www.discogs.com/developers
+ */
+
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface DiscogsArtist {
+	name: string;
+	anv?: string; // artist name variation
+	role?: string;
+	join?: string;
+}
+
+interface DiscogsTrack {
+	position: string;
+	title: string;
+	duration?: string;
+	artists?: DiscogsArtist[];
+	extraartists?: DiscogsArtist[];
+}
+
+interface DiscogsLabel {
+	name: string;
+	catno?: string; // catalog number
+}
+
+interface DiscogsFormat {
+	name: string;
+	qty?: string;
+	descriptions?: string[];
+}
+
+interface DiscogsRelease {
+	id: number;
+	title: string;
+	artists?: DiscogsArtist[];
+	year?: number;
+	released?: string;
+	country?: string;
+	genres?: string[];
+	styles?: string[];
+	labels?: DiscogsLabel[];
+	formats?: DiscogsFormat[];
+	tracklist?: DiscogsTrack[];
+	extraartists?: DiscogsArtist[];
+	notes?: string;
+	uri?: string;
+	master_id?: number;
+	master_url?: string;
+}
+
+interface DiscogsMaster {
+	id: number;
+	title: string;
+	artists?: DiscogsArtist[];
+	year?: number;
+	genres?: string[];
+	styles?: string[];
+	tracklist?: DiscogsTrack[];
+	notes?: string;
+	uri?: string;
+	main_release?: number;
+	main_release_url?: string;
+	versions_url?: string;
+	num_for_sale?: number;
+	lowest_price?: number;
+}
+
+/**
+ * Format artist names, handling name variations
+ */
+function formatArtists(artists: DiscogsArtist[] | undefined): string {
+	if (!artists?.length) return "Unknown Artist";
+	return artists
+		.map((a) => {
+			const name = a.anv || a.name;
+			const join = a.join || ", ";
+			return name + (a.join ? ` ${join} ` : "");
+		})
+		.join("")
+		.replace(/[,&]\s*$/, "")
+		.trim();
+}
+
+/**
+ * Format a single track
+ */
+function formatTrack(track: DiscogsTrack): string {
+	let line = track.position ? `${track.position}. ` : "- ";
+	line += track.title;
+	if (track.duration) line += ` (${track.duration})`;
+	if (track.artists?.length) {
+		line += ` - ${formatArtists(track.artists)}`;
+	}
+	return line;
+}
+
+/**
+ * Format credits/extraartists grouped by role
+ */
+function formatCredits(extraartists: DiscogsArtist[] | undefined): string {
+	if (!extraartists?.length) return "";
+
+	const byRole: Record<string, string[]> = {};
+	for (const artist of extraartists) {
+		const role = artist.role || "Other";
+		if (!byRole[role]) byRole[role] = [];
+		byRole[role].push(artist.anv || artist.name);
+	}
+
+	const lines: string[] = [];
+	for (const [role, names] of Object.entries(byRole)) {
+		lines.push(`- **${role}**: ${names.join(", ")}`);
+	}
+	return lines.join("\n");
+}
+
+/**
+ * Format formats (e.g., "2×LP, Album, Reissue")
+ */
+function formatFormats(formats: DiscogsFormat[] | undefined): string {
+	if (!formats?.length) return "";
+
+	return formats
+		.map((f) => {
+			const parts: string[] = [];
+			if (f.qty && parseInt(f.qty, 10) > 1) parts.push(`${f.qty}×`);
+			parts.push(f.name);
+			if (f.descriptions?.length) parts.push(f.descriptions.join(", "));
+			return parts.join(" ");
+		})
+		.join(" + ");
+}
+
+/**
+ * Format labels with catalog numbers
+ */
+function formatLabels(labels: DiscogsLabel[] | undefined): string {
+	if (!labels?.length) return "";
+	return labels
+		.map((l) => {
+			if (l.catno && l.catno !== "none") return `${l.name} (${l.catno})`;
+			return l.name;
+		})
+		.join(", ");
+}
+
+/**
+ * Build markdown for a release
+ */
+function buildReleaseMarkdown(release: DiscogsRelease): string {
+	const sections: string[] = [];
+
+	// Title with artist
+	const artist = formatArtists(release.artists);
+	sections.push(`# ${artist} - ${release.title}\n`);
+
+	// Metadata
+	const meta: string[] = [];
+	if (release.year) meta.push(`**Year**: ${release.year}`);
+	if (release.country) meta.push(`**Country**: ${release.country}`);
+
+	const format = formatFormats(release.formats);
+	if (format) meta.push(`**Format**: ${format}`);
+
+	const labels = formatLabels(release.labels);
+	if (labels) meta.push(`**Label**: ${labels}`);
+
+	if (release.genres?.length) meta.push(`**Genre**: ${release.genres.join(", ")}`);
+	if (release.styles?.length) meta.push(`**Style**: ${release.styles.join(", ")}`);
+
+	if (release.master_id) {
+		meta.push(`**Master Release**: [${release.master_id}](https://www.discogs.com/master/${release.master_id})`);
+	}
+
+	if (meta.length) sections.push(`${meta.join("\n")}\n`);
+
+	// Tracklist
+	if (release.tracklist?.length) {
+		sections.push("## Tracklist\n");
+		const tracks = release.tracklist.map(formatTrack);
+		sections.push(`${tracks.join("\n")}\n`);
+	}
+
+	// Credits
+	const credits = formatCredits(release.extraartists);
+	if (credits) {
+		sections.push("## Credits\n");
+		sections.push(`${credits}\n`);
+	}
+
+	// Notes
+	if (release.notes) {
+		sections.push("## Notes\n");
+		sections.push(`${release.notes}\n`);
+	}
+
+	return sections.join("\n");
+}
+
+/**
+ * Build markdown for a master release
+ */
+function buildMasterMarkdown(master: DiscogsMaster): string {
+	const sections: string[] = [];
+
+	// Title with artist
+	const artist = formatArtists(master.artists);
+	sections.push(`# ${artist} - ${master.title}\n`);
+	sections.push("*Master Release*\n");
+
+	// Metadata
+	const meta: string[] = [];
+	if (master.year) meta.push(`**Year**: ${master.year}`);
+	if (master.genres?.length) meta.push(`**Genre**: ${master.genres.join(", ")}`);
+	if (master.styles?.length) meta.push(`**Style**: ${master.styles.join(", ")}`);
+
+	if (master.main_release) {
+		meta.push(`**Main Release**: [${master.main_release}](https://www.discogs.com/release/${master.main_release})`);
+	}
+
+	if (master.num_for_sale !== undefined && master.num_for_sale > 0) {
+		meta.push(`**For Sale**: ${master.num_for_sale} copies`);
+		if (master.lowest_price !== undefined) {
+			meta.push(`**Lowest Price**: $${master.lowest_price.toFixed(2)}`);
+		}
+	}
+
+	if (meta.length) sections.push(`${meta.join("\n")}\n`);
+
+	// Tracklist
+	if (master.tracklist?.length) {
+		sections.push("## Tracklist\n");
+		const tracks = master.tracklist.map(formatTrack);
+		sections.push(`${tracks.join("\n")}\n`);
+	}
+
+	// Notes
+	if (master.notes) {
+		sections.push("## Notes\n");
+		sections.push(`${master.notes}\n`);
+	}
+
+	return sections.join("\n");
+}
+
+export const handleDiscogs: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("discogs.com")) return null;
+
+		// Match release or master URLs
+		// Patterns: /release/{id}, /master/{id}
+		// Also handles: /release/{id}-Artist-Title, /master/{id}-Artist-Title
+		const releaseMatch = parsed.pathname.match(/\/release\/(\d+)/);
+		const masterMatch = parsed.pathname.match(/\/master\/(\d+)/);
+
+		if (!releaseMatch && !masterMatch) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const isRelease = !!releaseMatch;
+		const id = isRelease ? releaseMatch[1] : masterMatch![1];
+
+		const apiUrl = isRelease ? `https://api.discogs.com/releases/${id}` : `https://api.discogs.com/masters/${id}`;
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: {
+				Accept: "application/json",
+				"User-Agent": "CodingAgent/1.0 +https://github.com/can1357/oh-my-pi",
+			},
+		});
+
+		if (!result.ok) return null;
+
+		let md: string;
+		if (isRelease) {
+			const release = JSON.parse(result.content) as DiscogsRelease;
+			md = buildReleaseMarkdown(release);
+		} else {
+			const master = JSON.parse(result.content) as DiscogsMaster;
+			md = buildMasterMarkdown(master);
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "discogs",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: [`Fetched via Discogs API (${isRelease ? "release" : "master"})`],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/discourse.ts

@@ -0,0 +1,221 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface DiscourseUser {
+	username?: string;
+	name?: string;
+}
+
+interface DiscoursePost {
+	id: number;
+	username?: string;
+	name?: string;
+	created_at?: string;
+	cooked?: string;
+	raw?: string;
+	like_count?: number;
+	post_number?: number;
+}
+
+interface DiscoursePostResponse extends DiscoursePost {
+	topic_id?: number;
+}
+
+interface DiscourseTopic {
+	id?: number;
+	title?: string;
+	fancy_title?: string;
+	posts_count?: number;
+	created_at?: string;
+	views?: number;
+	like_count?: number;
+	tags?: string[];
+	category_id?: number;
+	category_slug?: string;
+	category?: { id?: number; name?: string; slug?: string };
+	excerpt?: string;
+	details?: { created_by?: DiscourseUser };
+	post_stream?: { posts?: DiscoursePost[] };
+}
+
+const MAX_POSTS = 20;
+
+function normalizeBasePath(basePath: string): string {
+	if (!basePath || basePath === "/") return "";
+	return basePath.replace(/\/$/, "");
+}
+
+function parseTopicPath(pathname: string): { basePath: string; topicId: string } | null {
+	const match = pathname.match(/^(.*?)(?:\/t\/)(?:[^/]+\/)?(\d+)(?:\.json)?(?:\/|$)/);
+	if (!match) return null;
+	return { basePath: match[1] ?? "", topicId: match[2] };
+}
+
+function parsePostPath(pathname: string): { basePath: string; postId: string } | null {
+	const match = pathname.match(/^(.*?)(?:\/posts\/)(\d+)(?:\.json)?(?:\/|$)/);
+	if (!match) return null;
+	return { basePath: match[1] ?? "", postId: match[2] };
+}
+
+function formatAuthor(user?: DiscourseUser | null): string {
+	if (!user) return "unknown";
+	const name = user.name?.trim();
+	const username = user.username?.trim();
+	if (name && username && name !== username) return `${name} (@${username})`;
+	if (username) return `@${username}`;
+	if (name) return name;
+	return "unknown";
+}
+
+function formatIsoDate(value?: string): string {
+	if (!value) return "unknown";
+	const date = new Date(value);
+	if (Number.isNaN(date.getTime())) return value;
+	return date.toISOString().split("T")[0];
+}
+
+function formatCategory(topic: DiscourseTopic): string | null {
+	const parts: string[] = [];
+	const name = topic.category?.name ?? topic.category_slug;
+	if (name) parts.push(name);
+	const id = topic.category?.id ?? topic.category_id;
+	if (id != null) parts.push(`#${id}`);
+	return parts.length ? parts.join(" ") : null;
+}
+
+function formatPostBody(post: DiscoursePost): string {
+	const raw = post.raw?.trim();
+	if (raw) return raw;
+	const cooked = post.cooked?.trim();
+	if (!cooked) return "";
+	return htmlToBasicMarkdown(cooked);
+}
+
+function buildTopicUrl(baseUrl: string, topicId: string): string {
+	const topicUrl = new URL(`${baseUrl}/t/${topicId}.json`);
+	topicUrl.searchParams.set("include_raw", "1");
+	return topicUrl.toString();
+}
+
+function buildPostUrl(baseUrl: string, postId: string): string {
+	const postUrl = new URL(`${baseUrl}/posts/${postId}.json`);
+	postUrl.searchParams.set("include_raw", "1");
+	return postUrl.toString();
+}
+
+/**
+ * Handle Discourse forum URLs via API
+ */
+export const handleDiscourse: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const topicMatch = parseTopicPath(parsed.pathname);
+		const postMatch = topicMatch ? null : parsePostPath(parsed.pathname);
+		if (!topicMatch && !postMatch) return null;
+
+		const basePath = normalizeBasePath(topicMatch?.basePath ?? postMatch?.basePath ?? "");
+		const baseUrl = `${parsed.origin}${basePath}`;
+
+		let requestedPost: DiscoursePost | null = null;
+		let topicId = topicMatch?.topicId ?? null;
+
+		if (!topicId && postMatch) {
+			const postResult = await loadPage(buildPostUrl(baseUrl, postMatch.postId), { timeout, signal });
+			if (!postResult.ok) return null;
+
+			let postData: DiscoursePostResponse;
+			try {
+				postData = JSON.parse(postResult.content) as DiscoursePostResponse;
+			} catch {
+				return null;
+			}
+
+			if (!postData.topic_id) return null;
+			topicId = String(postData.topic_id);
+			requestedPost = postData;
+		}
+
+		if (!topicId) return null;
+
+		const topicResult = await loadPage(buildTopicUrl(baseUrl, topicId), { timeout, signal });
+		if (!topicResult.ok) return null;
+
+		let topic: DiscourseTopic;
+		try {
+			topic = JSON.parse(topicResult.content) as DiscourseTopic;
+		} catch {
+			return null;
+		}
+
+		const title = topic.title || topic.fancy_title;
+		if (!title) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		const posts: DiscoursePost[] = [...(topic.post_stream?.posts ?? [])];
+		if (requestedPost && !posts.some((post) => post.id === requestedPost?.id)) {
+			posts.unshift(requestedPost);
+		}
+
+		let md = `# ${title}\n\n`;
+
+		const metaParts: string[] = [];
+		if (topic.id != null) metaParts.push(`**Topic ID:** ${topic.id}`);
+		if (topic.posts_count != null) metaParts.push(`**Posts:** ${topic.posts_count}`);
+		if (topic.views != null) metaParts.push(`**Views:** ${topic.views}`);
+		if (topic.like_count != null) metaParts.push(`**Likes:** ${topic.like_count}`);
+		if (metaParts.length) md += `${metaParts.join(" | ")}\n`;
+
+		const categoryLabel = formatCategory(topic);
+		if (categoryLabel) md += `**Category:** ${categoryLabel}\n`;
+		if (topic.tags?.length) md += `**Tags:** ${topic.tags.join(", ")}\n`;
+
+		const createdBy = formatAuthor(topic.details?.created_by ?? null);
+		if (createdBy !== "unknown" || topic.created_at) {
+			md += `**Created by:** ${createdBy} - ${formatIsoDate(topic.created_at)}\n`;
+		}
+
+		md += "\n";
+
+		const description = topic.excerpt
+			? htmlToBasicMarkdown(topic.excerpt)
+			: posts.length
+				? formatPostBody(posts[0])
+				: "";
+		if (description) {
+			md += `## Description\n\n${description}\n\n`;
+		}
+
+		if (posts.length) {
+			md += "## Posts\n\n";
+			for (const post of posts.slice(0, MAX_POSTS)) {
+				const author = formatAuthor({ name: post.name, username: post.username });
+				const date = formatIsoDate(post.created_at);
+				const likes = post.like_count ?? 0;
+				const content = formatPostBody(post);
+				const postLabel = post.post_number != null ? `Post ${post.post_number}` : `Post ${post.id}`;
+
+				md += `### ${postLabel} - ${author} - ${date} - Likes: ${likes}\n\n`;
+				md += content ? `${content}\n\n---\n\n` : "_No content available._\n\n---\n\n";
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "discourse-api",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Discourse API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/dockerhub.ts

@@ -0,0 +1,160 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface DockerHubRepo {
+	name: string;
+	namespace: string;
+	description?: string;
+	star_count?: number;
+	pull_count?: number;
+	last_updated?: string;
+	is_official?: boolean;
+	is_automated?: boolean;
+	user?: string;
+}
+
+interface DockerHubTag {
+	name: string;
+	last_updated?: string;
+	full_size?: number;
+	digest?: string;
+	images?: Array<{
+		architecture?: string;
+		os?: string;
+		size?: number;
+	}>;
+}
+
+interface DockerHubTagsResponse {
+	results?: DockerHubTag[];
+}
+
+function formatSize(bytes: number): string {
+	if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(1)}GB`;
+	if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)}MB`;
+	if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)}KB`;
+	return `${bytes}B`;
+}
+
+/**
+ * Handle Docker Hub URLs via API
+ */
+export const handleDockerHub: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("hub.docker.com")) return null;
+
+		let namespace: string;
+		let repository: string;
+
+		// Official images: /_ /{image}
+		const officialMatch = parsed.pathname.match(/^\/_\/([^/]+)/);
+		if (officialMatch) {
+			namespace = "library";
+			repository = officialMatch[1];
+		} else {
+			// Regular images: /r/{namespace}/{repository}
+			const repoMatch = parsed.pathname.match(/^\/r\/([^/]+)\/([^/]+)/);
+			if (!repoMatch) return null;
+			namespace = repoMatch[1];
+			repository = repoMatch[2];
+		}
+
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch repository info and tags in parallel
+		const repoUrl = `https://hub.docker.com/v2/repositories/${namespace}/${repository}/`;
+		const tagsUrl = `https://hub.docker.com/v2/repositories/${namespace}/${repository}/tags/?page_size=10`;
+
+		const [repoResult, tagsResult] = await Promise.all([
+			loadPage(repoUrl, { timeout, headers: { Accept: "application/json" }, signal }),
+			loadPage(tagsUrl, { timeout: Math.min(timeout, 10), headers: { Accept: "application/json" }, signal }),
+		]);
+
+		if (!repoResult.ok) return null;
+
+		let repo: DockerHubRepo;
+		try {
+			repo = JSON.parse(repoResult.content);
+		} catch {
+			return null;
+		}
+
+		// Parse tags
+		let tags: DockerHubTag[] = [];
+		if (tagsResult.ok) {
+			try {
+				const tagsData = JSON.parse(tagsResult.content) as DockerHubTagsResponse;
+				tags = tagsData.results ?? [];
+			} catch {}
+		}
+
+		// Build markdown output
+		const fullName = namespace === "library" ? repo.name : `${namespace}/${repo.name}`;
+		let md = `# ${fullName}\n\n`;
+
+		if (repo.description) {
+			md += `${repo.description}\n\n`;
+		}
+
+		// Stats line
+		const stats: string[] = [];
+		if (repo.pull_count !== undefined) stats.push(`**Pulls:** ${formatCount(repo.pull_count)}`);
+		if (repo.star_count !== undefined) stats.push(`**Stars:** ${formatCount(repo.star_count)}`);
+		if (repo.is_official) stats.push("**Official Image**");
+		if (repo.is_automated) stats.push("**Automated Build**");
+		if (stats.length > 0) {
+			md += `${stats.join(" · ")}\n`;
+		}
+
+		if (repo.last_updated) {
+			const date = new Date(repo.last_updated);
+			md += `**Last Updated:** ${date.toISOString().split("T")[0]}\n`;
+		}
+
+		md += "\n";
+
+		// Docker pull command
+		md += "## Quick Start\n\n";
+		md += "```bash\n";
+		md += `docker pull ${fullName}\n`;
+		md += "```\n\n";
+
+		// Tags
+		if (tags.length > 0) {
+			md += "## Recent Tags\n\n";
+			md += "| Tag | Size | Architectures | Updated |\n";
+			md += "|-----|------|---------------|--------|\n";
+
+			for (const tag of tags) {
+				const size = tag.full_size ? formatSize(tag.full_size) : "-";
+				const archs =
+					tag.images
+						?.map((img) => img.architecture)
+						.filter(Boolean)
+						.join(", ") || "-";
+				const updated = tag.last_updated ? new Date(tag.last_updated).toISOString().split("T")[0] : "-";
+				md += `| \`${tag.name}\` | ${size} | ${archs} | ${updated} |\n`;
+			}
+			md += "\n";
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "dockerhub",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Docker Hub API"],
+		};
+	} catch {}
+
+	return null;
+};