@oh-my-pi/pi-coding-agent 3.30.0 → 3.31.0

package/src/core/tools/web-scrapers/crossref.ts

@@ -0,0 +1,149 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface CrossrefAuthor {
+	given?: string;
+	family?: string;
+	name?: string;
+}
+
+interface CrossrefDate {
+	"date-parts"?: number[][];
+}
+
+interface CrossrefMessage {
+	title?: string[];
+	author?: CrossrefAuthor[];
+	"container-title"?: string[];
+	"short-container-title"?: string[];
+	publisher?: string;
+	published?: CrossrefDate;
+	"published-print"?: CrossrefDate;
+	"published-online"?: CrossrefDate;
+	issued?: CrossrefDate;
+	created?: CrossrefDate;
+	DOI?: string;
+	abstract?: string;
+	type?: string;
+}
+
+interface CrossrefResponse {
+	message?: CrossrefMessage;
+}
+
+const DOI_HOSTS = new Set(["doi.org", "dx.doi.org", "www.doi.org"]);
+
+function extractDoi(pathname: string): string | null {
+	const raw = pathname.replace(/^\/+/, "");
+	if (!raw) return null;
+	return decodeURIComponent(raw);
+}
+
+function formatAuthors(authors?: CrossrefAuthor[]): string | null {
+	if (!authors || authors.length === 0) return null;
+	const names = authors
+		.map((author) => {
+			if (author.name) return author.name;
+			const parts = [author.given, author.family].filter(Boolean);
+			return parts.length > 0 ? parts.join(" ") : null;
+		})
+		.filter((name): name is string => Boolean(name));
+	if (names.length === 0) return null;
+	return names.join(", ");
+}
+
+function formatDate(date?: CrossrefDate): string | null {
+	const parts = date?.["date-parts"]?.[0];
+	if (!parts || parts.length === 0) return null;
+	const [year, month, day] = parts;
+	if (!year) return null;
+	const formatted = [
+		String(year),
+		month ? String(month).padStart(2, "0") : "",
+		day ? String(day).padStart(2, "0") : "",
+	].filter(Boolean);
+	return formatted.join("-");
+}
+
+function formatAbstract(abstract?: string): string | null {
+	if (!abstract) return null;
+	const normalized = abstract.replace(/<\/?jats:p[^>]*>/g, (match) => (match.startsWith("</") ? "</p>" : "<p>"));
+	const markdown = htmlToBasicMarkdown(normalized);
+	return markdown.trim().length > 0 ? markdown : null;
+}
+
+export const handleCrossref: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!DOI_HOSTS.has(parsed.hostname.toLowerCase())) return null;
+
+		const doi = extractDoi(parsed.pathname);
+		if (!doi) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://api.crossref.org/works/${encodeURIComponent(doi)}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: {
+				Accept: "application/json",
+			},
+		});
+
+		if (!result.ok) return null;
+
+		let data: CrossrefResponse;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const message = data.message;
+		if (!message) return null;
+
+		const title = message.title?.[0]?.trim() || "CrossRef Record";
+		const authors = formatAuthors(message.author);
+		const journal = message["container-title"]?.[0] || message["short-container-title"]?.[0];
+		const publisher = message.publisher;
+		const published =
+			formatDate(message.published) ||
+			formatDate(message["published-print"]) ||
+			formatDate(message["published-online"]) ||
+			formatDate(message.issued) ||
+			formatDate(message.created);
+		const doiValue = message.DOI || doi;
+		const abstract = formatAbstract(message.abstract);
+		const type = message.type?.replace(/-/g, " ");
+
+		let md = `# ${title}\n\n`;
+		if (authors) md += `**Authors:** ${authors}\n`;
+		if (journal) md += `**Journal:** ${journal}\n`;
+		if (publisher) md += `**Publisher:** ${publisher}\n`;
+		if (published) md += `**Published:** ${published}\n`;
+		md += `**DOI:** ${doiValue}\n`;
+		if (type) md += `**Type:** ${type}\n`;
+		md += "\n---\n\n";
+		md += "## Abstract\n\n";
+		md += abstract || "No abstract available.";
+		md += "\n";
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "crossref",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via CrossRef API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/{web-fetch-handlers → web-scrapers}/devto.ts

@@ -23,7 +23,11 @@ interface DevToArticle {
 /**
  * Handle dev.to URLs via API
  */
-export const handleDevTo: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+export const handleDevTo: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
 	try {
 		const parsed = new URL(url);
 		if (parsed.hostname !== "dev.to") return null;
@@ -39,7 +43,7 @@ export const handleDevTo: SpecialHandler = async (url: string, timeout: number):
 			const tag = pathParts[1];
 			const apiUrl = `https://dev.to/api/articles?tag=${encodeURIComponent(tag)}&per_page=20`;
 
-			const result = await loadPage(apiUrl, { timeout });
+			const result = await loadPage(apiUrl, { timeout, signal });
 			if (!result.ok) return null;
 
 			const articles = JSON.parse(result.content) as DevToArticle[];
@@ -82,7 +86,7 @@ export const handleDevTo: SpecialHandler = async (url: string, timeout: number):
 			const username = pathParts[0];
 			const apiUrl = `https://dev.to/api/articles?username=${encodeURIComponent(username)}&per_page=20`;
 
-			const result = await loadPage(apiUrl, { timeout });
+			const result = await loadPage(apiUrl, { timeout, signal });
 			if (!result.ok) return null;
 
 			const articles = JSON.parse(result.content) as DevToArticle[];
@@ -125,7 +129,7 @@ export const handleDevTo: SpecialHandler = async (url: string, timeout: number):
 			const slug = pathParts[1];
 			const apiUrl = `https://dev.to/api/articles/${encodeURIComponent(username)}/${encodeURIComponent(slug)}`;
 
-			const result = await loadPage(apiUrl, { timeout });
+			const result = await loadPage(apiUrl, { timeout, signal });
 			if (!result.ok) return null;
 
 			const article = JSON.parse(result.content) as DevToArticle;

package/src/core/tools/{web-fetch-handlers → web-scrapers}/discogs.ts

@@ -248,7 +248,11 @@ function buildMasterMarkdown(master: DiscogsMaster): string {
 	return sections.join("\n");
 }
 
-export const handleDiscogs: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+export const handleDiscogs: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
 	try {
 		const parsed = new URL(url);
 		if (!parsed.hostname.includes("discogs.com")) return null;
@@ -269,6 +273,7 @@ export const handleDiscogs: SpecialHandler = async (url: string, timeout: number
 
 		const result = await loadPage(apiUrl, {
 			timeout,
+			signal,
 			headers: {
 				Accept: "application/json",
 				"User-Agent": "CodingAgent/1.0 +https://github.com/can1357/oh-my-pi",

package/src/core/tools/web-scrapers/discourse.ts

@@ -0,0 +1,221 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface DiscourseUser {
+	username?: string;
+	name?: string;
+}
+
+interface DiscoursePost {
+	id: number;
+	username?: string;
+	name?: string;
+	created_at?: string;
+	cooked?: string;
+	raw?: string;
+	like_count?: number;
+	post_number?: number;
+}
+
+interface DiscoursePostResponse extends DiscoursePost {
+	topic_id?: number;
+}
+
+interface DiscourseTopic {
+	id?: number;
+	title?: string;
+	fancy_title?: string;
+	posts_count?: number;
+	created_at?: string;
+	views?: number;
+	like_count?: number;
+	tags?: string[];
+	category_id?: number;
+	category_slug?: string;
+	category?: { id?: number; name?: string; slug?: string };
+	excerpt?: string;
+	details?: { created_by?: DiscourseUser };
+	post_stream?: { posts?: DiscoursePost[] };
+}
+
+const MAX_POSTS = 20;
+
+function normalizeBasePath(basePath: string): string {
+	if (!basePath || basePath === "/") return "";
+	return basePath.replace(/\/$/, "");
+}
+
+function parseTopicPath(pathname: string): { basePath: string; topicId: string } | null {
+	const match = pathname.match(/^(.*?)(?:\/t\/)(?:[^/]+\/)?(\d+)(?:\.json)?(?:\/|$)/);
+	if (!match) return null;
+	return { basePath: match[1] ?? "", topicId: match[2] };
+}
+
+function parsePostPath(pathname: string): { basePath: string; postId: string } | null {
+	const match = pathname.match(/^(.*?)(?:\/posts\/)(\d+)(?:\.json)?(?:\/|$)/);
+	if (!match) return null;
+	return { basePath: match[1] ?? "", postId: match[2] };
+}
+
+function formatAuthor(user?: DiscourseUser | null): string {
+	if (!user) return "unknown";
+	const name = user.name?.trim();
+	const username = user.username?.trim();
+	if (name && username && name !== username) return `${name} (@${username})`;
+	if (username) return `@${username}`;
+	if (name) return name;
+	return "unknown";
+}
+
+function formatIsoDate(value?: string): string {
+	if (!value) return "unknown";
+	const date = new Date(value);
+	if (Number.isNaN(date.getTime())) return value;
+	return date.toISOString().split("T")[0];
+}
+
+function formatCategory(topic: DiscourseTopic): string | null {
+	const parts: string[] = [];
+	const name = topic.category?.name ?? topic.category_slug;
+	if (name) parts.push(name);
+	const id = topic.category?.id ?? topic.category_id;
+	if (id != null) parts.push(`#${id}`);
+	return parts.length ? parts.join(" ") : null;
+}
+
+function formatPostBody(post: DiscoursePost): string {
+	const raw = post.raw?.trim();
+	if (raw) return raw;
+	const cooked = post.cooked?.trim();
+	if (!cooked) return "";
+	return htmlToBasicMarkdown(cooked);
+}
+
+function buildTopicUrl(baseUrl: string, topicId: string): string {
+	const topicUrl = new URL(`${baseUrl}/t/${topicId}.json`);
+	topicUrl.searchParams.set("include_raw", "1");
+	return topicUrl.toString();
+}
+
+function buildPostUrl(baseUrl: string, postId: string): string {
+	const postUrl = new URL(`${baseUrl}/posts/${postId}.json`);
+	postUrl.searchParams.set("include_raw", "1");
+	return postUrl.toString();
+}
+
+/**
+ * Handle Discourse forum URLs via API
+ */
+export const handleDiscourse: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const topicMatch = parseTopicPath(parsed.pathname);
+		const postMatch = topicMatch ? null : parsePostPath(parsed.pathname);
+		if (!topicMatch && !postMatch) return null;
+
+		const basePath = normalizeBasePath(topicMatch?.basePath ?? postMatch?.basePath ?? "");
+		const baseUrl = `${parsed.origin}${basePath}`;
+
+		let requestedPost: DiscoursePost | null = null;
+		let topicId = topicMatch?.topicId ?? null;
+
+		if (!topicId && postMatch) {
+			const postResult = await loadPage(buildPostUrl(baseUrl, postMatch.postId), { timeout, signal });
+			if (!postResult.ok) return null;
+
+			let postData: DiscoursePostResponse;
+			try {
+				postData = JSON.parse(postResult.content) as DiscoursePostResponse;
+			} catch {
+				return null;
+			}
+
+			if (!postData.topic_id) return null;
+			topicId = String(postData.topic_id);
+			requestedPost = postData;
+		}
+
+		if (!topicId) return null;
+
+		const topicResult = await loadPage(buildTopicUrl(baseUrl, topicId), { timeout, signal });
+		if (!topicResult.ok) return null;
+
+		let topic: DiscourseTopic;
+		try {
+			topic = JSON.parse(topicResult.content) as DiscourseTopic;
+		} catch {
+			return null;
+		}
+
+		const title = topic.title || topic.fancy_title;
+		if (!title) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		const posts: DiscoursePost[] = [...(topic.post_stream?.posts ?? [])];
+		if (requestedPost && !posts.some((post) => post.id === requestedPost?.id)) {
+			posts.unshift(requestedPost);
+		}
+
+		let md = `# ${title}\n\n`;
+
+		const metaParts: string[] = [];
+		if (topic.id != null) metaParts.push(`**Topic ID:** ${topic.id}`);
+		if (topic.posts_count != null) metaParts.push(`**Posts:** ${topic.posts_count}`);
+		if (topic.views != null) metaParts.push(`**Views:** ${topic.views}`);
+		if (topic.like_count != null) metaParts.push(`**Likes:** ${topic.like_count}`);
+		if (metaParts.length) md += `${metaParts.join(" | ")}\n`;
+
+		const categoryLabel = formatCategory(topic);
+		if (categoryLabel) md += `**Category:** ${categoryLabel}\n`;
+		if (topic.tags?.length) md += `**Tags:** ${topic.tags.join(", ")}\n`;
+
+		const createdBy = formatAuthor(topic.details?.created_by ?? null);
+		if (createdBy !== "unknown" || topic.created_at) {
+			md += `**Created by:** ${createdBy} - ${formatIsoDate(topic.created_at)}\n`;
+		}
+
+		md += "\n";
+
+		const description = topic.excerpt
+			? htmlToBasicMarkdown(topic.excerpt)
+			: posts.length
+				? formatPostBody(posts[0])
+				: "";
+		if (description) {
+			md += `## Description\n\n${description}\n\n`;
+		}
+
+		if (posts.length) {
+			md += "## Posts\n\n";
+			for (const post of posts.slice(0, MAX_POSTS)) {
+				const author = formatAuthor({ name: post.name, username: post.username });
+				const date = formatIsoDate(post.created_at);
+				const likes = post.like_count ?? 0;
+				const content = formatPostBody(post);
+				const postLabel = post.post_number != null ? `Post ${post.post_number}` : `Post ${post.id}`;
+
+				md += `### ${postLabel} - ${author} - ${date} - Likes: ${likes}\n\n`;
+				md += content ? `${content}\n\n---\n\n` : "_No content available._\n\n---\n\n";
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "discourse-api",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Discourse API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/{web-fetch-handlers → web-scrapers}/dockerhub.ts

@@ -39,7 +39,11 @@ function formatSize(bytes: number): string {
 /**
  * Handle Docker Hub URLs via API
  */
-export const handleDockerHub: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+export const handleDockerHub: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
 	try {
 		const parsed = new URL(url);
 		if (!parsed.hostname.includes("hub.docker.com")) return null;
@@ -67,8 +71,8 @@ export const handleDockerHub: SpecialHandler = async (url: string, timeout: numb
 		const tagsUrl = `https://hub.docker.com/v2/repositories/${namespace}/${repository}/tags/?page_size=10`;
 
 		const [repoResult, tagsResult] = await Promise.all([
-			loadPage(repoUrl, { timeout, headers: { Accept: "application/json" } }),
-			loadPage(tagsUrl, { timeout: Math.min(timeout, 10), headers: { Accept: "application/json" } }),
+			loadPage(repoUrl, { timeout, headers: { Accept: "application/json" }, signal }),
+			loadPage(tagsUrl, { timeout: Math.min(timeout, 10), headers: { Accept: "application/json" }, signal }),
 		]);
 
 		if (!repoResult.ok) return null;

package/src/core/tools/web-scrapers/fdroid.ts

@@ -0,0 +1,158 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+type LocalizedText = string | Record<string, string>;
+
+type FdroidPackage = {
+	packageName?: string;
+	name?: LocalizedText;
+	summary?: LocalizedText;
+	description?: LocalizedText;
+	author?: string | { name?: string; email?: string };
+	authorName?: string;
+	authorEmail?: string;
+	license?: string;
+	categories?: string[];
+	antiFeatures?: string[];
+	sourceCode?: string;
+	packages?: Array<{
+		versionName?: string;
+		versionCode?: number;
+		added?: number;
+		antiFeatures?: string[];
+	}>;
+	suggestedVersionCode?: number;
+	suggestedVersionName?: string;
+};
+
+function pickLocalizedText(value?: LocalizedText): string | undefined {
+	if (!value) return undefined;
+	if (typeof value === "string") return value;
+	const preferred = value["en-US"] ?? value.en_US ?? value.en;
+	if (preferred) return preferred;
+	const first = Object.values(value).find((entry) => typeof entry === "string");
+	return first;
+}
+
+function normalizeAuthor(data: FdroidPackage): string | undefined {
+	if (data.authorName) return data.authorName;
+	if (typeof data.author === "string") return data.author;
+	if (data.author && typeof data.author !== "string" && typeof data.author.name === "string") return data.author.name;
+	if (data.authorEmail) return data.authorEmail;
+	return undefined;
+}
+
+function normalizeAuthorEmail(data: FdroidPackage): string | undefined {
+	if (data.authorEmail) return data.authorEmail;
+	if (data.author && typeof data.author !== "string" && typeof data.author.email === "string")
+		return data.author.email;
+	return undefined;
+}
+
+function collectAntiFeatures(data: FdroidPackage): string[] {
+	const values = new Set<string>();
+	for (const feature of data.antiFeatures ?? []) values.add(feature);
+	for (const pkg of data.packages ?? []) {
+		for (const feature of pkg.antiFeatures ?? []) values.add(feature);
+	}
+	return Array.from(values);
+}
+
+function resolveSuggestedVersion(data: FdroidPackage): string | undefined {
+	if (data.suggestedVersionName) return data.suggestedVersionName;
+	if (data.suggestedVersionCode) {
+		const match = data.packages?.find((pkg) => pkg.versionCode === data.suggestedVersionCode);
+		if (match?.versionName) return match.versionName;
+	}
+	return data.packages?.[0]?.versionName;
+}
+
+/**
+ * Handle F-Droid URLs via API
+ */
+export const handleFdroid: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "f-droid.org" && parsed.hostname !== "www.f-droid.org") return null;
+
+		// Extract package name from /packages/{packageName} or /en/packages/{packageName}
+		const match = parsed.pathname.match(/^\/(?:en\/)?packages\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://f-droid.org/api/v1/packages/${encodeURIComponent(packageName)}`;
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let data: FdroidPackage;
+		try {
+			data = JSON.parse(result.content) as FdroidPackage;
+		} catch {
+			return null;
+		}
+
+		const displayName = pickLocalizedText(data.name) ?? packageName;
+		const summary = pickLocalizedText(data.summary);
+		const description = pickLocalizedText(data.description);
+		const author = normalizeAuthor(data);
+		const authorEmail = normalizeAuthorEmail(data);
+		const antiFeatures = collectAntiFeatures(data);
+		const latestVersion = resolveSuggestedVersion(data);
+
+		let md = `# ${displayName}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		md += `**Package:** ${packageName}`;
+		if (latestVersion) md += ` · **Latest:** ${latestVersion}`;
+		if (data.license) md += ` · **License:** ${data.license}`;
+		md += "\n";
+
+		if (author) {
+			md += `**Author:** ${author}`;
+			if (authorEmail && authorEmail !== author) md += ` <${authorEmail}>`;
+			md += "\n";
+		}
+
+		if (data.sourceCode) md += `**Source Code:** ${data.sourceCode}\n`;
+		if (data.categories?.length) md += `**Categories:** ${data.categories.join(", ")}\n`;
+		if (antiFeatures.length) md += `**Anti-Features:** ${antiFeatures.join(", ")}\n`;
+
+		if (description) {
+			md += `\n## Description\n\n${description}\n`;
+		}
+
+		if (data.packages?.length) {
+			md += "\n## Version History\n\n";
+			for (const version of data.packages.slice(0, 10)) {
+				const label = version.versionName ?? "unknown";
+				const code = version.versionCode ? ` (${version.versionCode})` : "";
+				md += `- ${label}${code}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "fdroid",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via F-Droid API"],
+		};
+	} catch {}
+
+	return null;
+};