@oh-my-pi/pi-coding-agent 12.18.1 → 12.19.0

package/src/web/scrapers/snapcraft.ts

@@ -1,5 +1,5 @@
 import type { RenderResult, SpecialHandler } from "./types";
-import { finalizeOutput, formatCount, loadPage } from "./types";
+import { buildResult, formatNumber, loadPage, tryParseJson } from "./types";
 
 interface SnapcraftPublisher {
 	"display-name"?: string;
@@ -122,12 +122,8 @@ export const handleSnapcraft: SpecialHandler = async (
 		});
 		if (!result.ok) return null;
 
-		let data: SnapcraftResponse;
-		try {
-			data = JSON.parse(result.content) as SnapcraftResponse;
-		} catch {
-			return null;
-		}
+		const data = tryParseJson<SnapcraftResponse>(result.content);
+		if (!data) return null;
 
 		const snapInfo = data.snap ?? data;
 		const name = snapInfo.title ?? snapInfo.name ?? data.name ?? snapName;
@@ -163,7 +159,7 @@ export const handleSnapcraft: SpecialHandler = async (
 		if (base) md += ` · **Base:** ${base}`;
 		md += "\n";
 		if (publisher) md += `**Publisher:** ${publisher}\n`;
-		if (downloads !== null) md += `**Downloads:** ${formatCount(downloads)}\n`;
+		if (downloads !== null) md += `**Downloads:** ${formatNumber(downloads)}\n`;
 		md += "\n";
 
 		if (channels.size > 0) {
@@ -183,17 +179,7 @@ export const handleSnapcraft: SpecialHandler = async (
 			md += `## Description\n\n${descriptionText}\n`;
 		}
 
-		const output = finalizeOutput(md);
-		return {
-			url,
-			finalUrl: url,
-			contentType: "text/markdown",
-			method: "snapcraft",
-			content: output.content,
-			fetchedAt,
-			truncated: output.truncated,
-			notes: ["Fetched via Snapcraft API"],
-		};
+		return buildResult(md, { url, method: "snapcraft", fetchedAt, notes: ["Fetched via Snapcraft API"] });
 	} catch {}
 
 	return null;

package/src/web/scrapers/sourcegraph.ts

@@ -1,5 +1,5 @@
 import type { RenderResult, SpecialHandler } from "./types";
-import { finalizeOutput, loadPage } from "./types";
+import { buildResult, loadPage, tryParseJson } from "./types";
 
 const GRAPHQL_ENDPOINT = "https://sourcegraph.com/.api/graphql";
 const GRAPHQL_HEADERS = {
@@ -160,14 +160,6 @@ function parseSourcegraphUrl(url: string): SourcegraphTarget | null {
 	}
 }
 
-function safeParseJson<T>(content: string): T | null {
-	try {
-		return JSON.parse(content) as T;
-	} catch {
-		return null;
-	}
-}
-
 async function fetchGraphql<T>(
 	query: string,
 	variables: Record<string, unknown>,
@@ -184,7 +176,7 @@ async function fetchGraphql<T>(
 	});
 	if (!result.ok) return null;
 
-	const parsed = safeParseJson<{ data?: T; errors?: unknown }>(result.content);
+	const parsed = tryParseJson<{ data?: T; errors?: unknown }>(result.content);
 	if (!parsed?.data) return null;
 	if (Array.isArray(parsed.errors) && parsed.errors.length > 0) return null;
 	return parsed.data;
@@ -323,48 +315,18 @@ export const handleSourcegraph: SpecialHandler = async (
 			case "search": {
 				const result = await renderSearch(target.query, timeout, signal);
 				if (!result.ok) return null;
-				const output = finalizeOutput(result.content);
-				return {
-					url,
-					finalUrl: url,
-					contentType: "text/markdown",
-					method: "sourcegraph-search",
-					content: output.content,
-					fetchedAt,
-					truncated: output.truncated,
-					notes,
-				};
+				return buildResult(result.content, { url, method: "sourcegraph-search", fetchedAt, notes });
 			}
 			case "file": {
 				const rev = target.rev ?? "HEAD";
 				const result = await renderFile(target.repoName, target.filePath, rev, timeout, signal);
 				if (!result.ok) return null;
-				const output = finalizeOutput(result.content);
-				return {
-					url,
-					finalUrl: url,
-					contentType: "text/markdown",
-					method: "sourcegraph-file",
-					content: output.content,
-					fetchedAt,
-					truncated: output.truncated,
-					notes,
-				};
+				return buildResult(result.content, { url, method: "sourcegraph-file", fetchedAt, notes });
 			}
 			case "repo": {
 				const result = await renderRepo(target.repoName, timeout, signal);
 				if (!result.ok) return null;
-				const output = finalizeOutput(result.content);
-				return {
-					url,
-					finalUrl: url,
-					contentType: "text/markdown",
-					method: "sourcegraph-repo",
-					content: output.content,
-					fetchedAt,
-					truncated: output.truncated,
-					notes,
-				};
+				return buildResult(result.content, { url, method: "sourcegraph-repo", fetchedAt, notes });
 			}
 		}
 	} catch {}

package/src/web/scrapers/spdx.ts

@@ -1,5 +1,5 @@
 import type { RenderResult, SpecialHandler } from "./types";
-import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+import { buildResult, htmlToBasicMarkdown, loadPage, tryParseJson } from "./types";
 
 interface SpdxCrossRef {
 	url?: string;
@@ -67,12 +67,8 @@ export const handleSpdx: SpecialHandler = async (
 
 		if (!result.ok) return null;
 
-		let license: SpdxLicense;
-		try {
-			license = JSON.parse(result.content);
-		} catch {
-			return null;
-		}
+		const license = tryParseJson<SpdxLicense>(result.content);
+		if (!license) return null;
 
 		const title = license.name || license.licenseId || licenseId;
 		let md = `# ${title}\n\n`;
@@ -104,17 +100,7 @@ export const handleSpdx: SpecialHandler = async (
 			md += `\n## License Text\n\n\`\`\`\n${licenseText}\n\`\`\`\n`;
 		}
 
-		const output = finalizeOutput(md);
-		return {
-			url,
-			finalUrl: url,
-			contentType: "text/markdown",
-			method: "spdx-api",
-			content: output.content,
-			fetchedAt,
-			truncated: output.truncated,
-			notes: ["Fetched via SPDX license API"],
-		};
+		return buildResult(md, { url, method: "spdx-api", fetchedAt, notes: ["Fetched via SPDX license API"] });
 	} catch {}
 
 	return null;

package/src/web/scrapers/spotify.ts

@@ -5,7 +5,7 @@
  * from Spotify URLs without requiring authentication.
  */
 import type { SpecialHandler } from "./types";
-import { finalizeOutput, loadPage } from "./types";
+import { buildResult, formatMediaDuration, loadPage } from "./types";
 
 interface SpotifyOEmbedResponse {
 	title?: string;
@@ -71,21 +71,13 @@ function getContentType(url: string): string | null {
 }
 
 /**
- * Format duration from seconds
+ * Format duration from seconds string
  */
 function formatDuration(seconds: string | undefined): string | null {
 	if (!seconds) return null;
 	const num = parseInt(seconds, 10);
 	if (Number.isNaN(num)) return null;
-
-	const hours = Math.floor(num / 3600);
-	const minutes = Math.floor((num % 3600) / 60);
-	const secs = num % 60;
-
-	if (hours > 0) {
-		return `${hours}:${minutes.toString().padStart(2, "0")}:${secs.toString().padStart(2, "0")}`;
-	}
-	return `${minutes}:${secs.toString().padStart(2, "0")}`;
+	return formatMediaDuration(num);
 }
 
 /**
@@ -202,16 +194,5 @@ export const handleSpotify: SpecialHandler = async (url: string, timeout: number
 
 	// Format output
 	const output = formatOutput(contentType, oEmbedData, ogData, url);
-	const { content, truncated } = finalizeOutput(output);
-
-	return {
-		url,
-		finalUrl: url,
-		contentType: "text/markdown",
-		method: "spotify",
-		content,
-		fetchedAt: new Date().toISOString(),
-		truncated,
-		notes,
-	};
+	return buildResult(output, { url, method: "spotify", fetchedAt: new Date().toISOString(), notes });
 };

package/src/web/scrapers/stackoverflow.ts

@@ -1,5 +1,5 @@
 import type { RenderResult, SpecialHandler } from "./types";
-import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+import { buildResult, formatIsoDate, htmlToBasicMarkdown, loadPage, tryParseJson } from "./types";
 
 interface SOQuestion {
 	title: string;
@@ -79,8 +79,8 @@ export const handleStackOverflow: SpecialHandler = async (
 
 		if (!qResult.ok) return null;
 
-		const qData = JSON.parse(qResult.content) as { items: SOQuestion[] };
-		if (!qData.items?.length) return null;
+		const qData = tryParseJson<{ items: SOQuestion[] }>(qResult.content);
+		if (!qData?.items?.length) return null;
 
 		const question = qData.items[0];
 
@@ -88,7 +88,7 @@ export const handleStackOverflow: SpecialHandler = async (
 		md += `**Score:** ${question.score} · **Answers:** ${question.answer_count}`;
 		md += question.is_answered ? " (Answered)" : "";
 		md += `\n**Tags:** ${question.tags.join(", ")}\n`;
-		md += `**Asked by:** ${question.owner.display_name} · ${new Date(question.creation_date * 1000).toISOString().split("T")[0]}\n\n`;
+		md += `**Asked by:** ${question.owner.display_name} · ${formatIsoDate(question.creation_date * 1000)}\n\n`;
 		md += `---\n\n## Question\n\n${htmlToBasicMarkdown(question.body)}\n\n`;
 
 		// Fetch answers
@@ -96,8 +96,8 @@ export const handleStackOverflow: SpecialHandler = async (
 		const aResult = await loadPage(aUrl, { timeout, signal });
 
 		if (aResult.ok) {
-			const aData = JSON.parse(aResult.content) as { items: SOAnswer[] };
-			if (aData.items?.length) {
+			const aData = tryParseJson<{ items: SOAnswer[] }>(aResult.content);
+			if (aData?.items?.length) {
 				md += `---\n\n## Answers\n\n`;
 				for (const answer of aData.items.slice(0, 5)) {
 					const accepted = answer.is_accepted ? " (Accepted)" : "";
@@ -107,17 +107,12 @@ export const handleStackOverflow: SpecialHandler = async (
 			}
 		}
 
-		const output = finalizeOutput(md);
-		return {
+		return buildResult(md, {
 			url,
-			finalUrl: url,
-			contentType: "text/markdown",
 			method: "stackexchange",
-			content: output.content,
 			fetchedAt,
-			truncated: output.truncated,
 			notes: [`Fetched via Stack Exchange API (site=${site})`],
-		};
+		});
 	} catch {}
 
 	return null;

package/src/web/scrapers/terraform.ts

@@ -1,5 +1,5 @@
 import type { RenderResult, SpecialHandler } from "./types";
-import { finalizeOutput, formatCount, loadPage } from "./types";
+import { buildResult, formatNumber, loadPage, tryParseJson } from "./types";
 
 interface TerraformModule {
 	id: string;
@@ -113,12 +113,8 @@ async function handleModuleUrl(
 
 	if (!result.ok) return null;
 
-	let mod: TerraformModule;
-	try {
-		mod = JSON.parse(result.content);
-	} catch {
-		return null;
-	}
+	const mod = tryParseJson<TerraformModule>(result.content);
+	if (!mod) return null;
 
 	let md = `# ${mod.namespace}/${mod.name}/${mod.provider}\n\n`;
 
@@ -128,7 +124,7 @@ async function handleModuleUrl(
 	md += `**Version:** ${mod.version}`;
 	if (mod.verified) md += " ✓ Verified";
 	md += `\n`;
-	md += `**Downloads:** ${formatCount(mod.downloads)}\n`;
+	md += `**Downloads:** ${formatNumber(mod.downloads)}\n`;
 	if (mod.published_at) {
 		md += `**Published:** ${new Date(mod.published_at).toLocaleDateString()}\n`;
 	}
@@ -212,17 +208,7 @@ async function handleModuleUrl(
 		}
 	}
 
-	const output = finalizeOutput(md);
-	return {
-		url,
-		finalUrl: url,
-		contentType: "text/markdown",
-		method: "terraform",
-		content: output.content,
-		fetchedAt,
-		truncated: output.truncated,
-		notes: ["Fetched via Terraform Registry API"],
-	};
+	return buildResult(md, { url, method: "terraform", fetchedAt, notes: ["Fetched via Terraform Registry API"] });
 }
 
 async function handleProviderUrl(
@@ -242,12 +228,8 @@ async function handleProviderUrl(
 
 	if (!result.ok) return null;
 
-	let provider: TerraformProvider;
-	try {
-		provider = JSON.parse(result.content);
-	} catch {
-		return null;
-	}
+	const provider = tryParseJson<TerraformProvider>(result.content);
+	if (!provider) return null;
 
 	let md = `# ${provider.namespace}/${provider.name}\n\n`;
 
@@ -256,7 +238,7 @@ async function handleProviderUrl(
 	// Metadata
 	md += `**Version:** ${provider.version}\n`;
 	if (provider.tier) md += `**Tier:** ${provider.tier}\n`;
-	md += `**Downloads:** ${formatCount(provider.downloads)}\n`;
+	md += `**Downloads:** ${formatNumber(provider.downloads)}\n`;
 	if (provider.published_at) {
 		md += `**Published:** ${new Date(provider.published_at).toLocaleDateString()}\n`;
 	}
@@ -290,15 +272,5 @@ async function handleProviderUrl(
 		}
 	}
 
-	const output = finalizeOutput(md);
-	return {
-		url,
-		finalUrl: url,
-		contentType: "text/markdown",
-		method: "terraform",
-		content: output.content,
-		fetchedAt,
-		truncated: output.truncated,
-		notes: ["Fetched via Terraform Registry API"],
-	};
+	return buildResult(md, { url, method: "terraform", fetchedAt, notes: ["Fetched via Terraform Registry API"] });
 }

package/src/web/scrapers/tldr.ts

@@ -1,5 +1,5 @@
 import type { RenderResult, SpecialHandler } from "./types";
-import { finalizeOutput, loadPage } from "./types";
+import { buildResult, loadPage } from "./types";
 
 const TLDR_BASE = "https://raw.githubusercontent.com/tldr-pages/tldr/main/pages";
 const PLATFORMS = ["common", "linux", "osx"] as const;
@@ -30,17 +30,13 @@ export const handleTldr: SpecialHandler = async (
 			const result = await loadPage(rawUrl, { timeout, signal });
 
 			if (result.ok && result.content.trim()) {
-				const output = finalizeOutput(result.content);
-				return {
+				return buildResult(result.content, {
 					url,
 					finalUrl: rawUrl,
-					contentType: "text/markdown",
 					method: "tldr",
-					content: output.content,
 					fetchedAt,
-					truncated: output.truncated,
 					notes: [`Fetched from tldr-pages (${platform})`],
-				};
+				});
 			}
 		}
 

package/src/web/scrapers/twitter.ts

@@ -1,7 +1,7 @@
 import { parse as parseHtml } from "node-html-parser";
 import { ToolAbortError } from "../../tools/tool-errors";
 import type { RenderResult, SpecialHandler } from "./types";
-import { finalizeOutput, loadPage } from "./types";
+import { buildResult, loadPage } from "./types";
 
 const NITTER_INSTANCES = [
 	"nitter.privacyredirect.com",
@@ -58,17 +58,13 @@ export const handleTwitter: SpecialHandler = async (
 						}
 					}
 
-					const output = finalizeOutput(md);
-					return {
+					return buildResult(md, {
 						url,
 						finalUrl: nitterUrl,
-						contentType: "text/markdown",
 						method: "twitter-nitter",
-						content: output.content,
 						fetchedAt,
-						truncated: output.truncated,
 						notes: [`Via Nitter: ${instance}`],
-					};
+					});
 				}
 			}
 		}

package/src/web/scrapers/types.ts

@@ -4,6 +4,8 @@
 import { ptree } from "@oh-my-pi/pi-utils";
 import { ToolAbortError } from "../../tools/tool-errors";
 
+export { formatNumber } from "@oh-my-pi/pi-utils";
+
 export interface RenderResult {
 	url: string;
 	finalUrl: string;
@@ -18,7 +20,7 @@ export interface RenderResult {
 export type SpecialHandler = (url: string, timeout: number, signal?: AbortSignal) => Promise<RenderResult | null>;
 
 export const MAX_OUTPUT_CHARS = 500_000;
-const MAX_BYTES = 50 * 1024 * 1024;
+export const MAX_BYTES = 50 * 1024 * 1024;
 
 const USER_AGENTS = [
 	"curl/8.0",
@@ -151,50 +153,126 @@ export async function loadPage(url: string, options: LoadPageOptions = {}): Prom
 	return { content: "", contentType: "", finalUrl: url, ok: false };
 }
 
-/**
- * Format large numbers (1000 -> 1K, 1000000 -> 1M)
- */
-export function formatCount(n: number): string {
-	if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
-	if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
-	return String(n);
-}
-
 /**
  * Convert basic HTML to markdown
  */
 export function htmlToBasicMarkdown(html: string): string {
-	return html
-		.replace(/<pre><code[^>]*>/g, "\n```\n")
+	const stripped = html
+		.replace(/<pre[^>]*><code[^>]*>/g, "\n```\n")
 		.replace(/<\/code><\/pre>/g, "\n```\n")
-		.replace(/<code>/g, "`")
+		.replace(/<code[^>]*>/g, "`")
 		.replace(/<\/code>/g, "`")
-		.replace(/<strong>/g, "**")
+		.replace(/<strong[^>]*>/g, "**")
 		.replace(/<\/strong>/g, "**")
-		.replace(/<b>/g, "**")
+		.replace(/<b[^>]*>/g, "**")
 		.replace(/<\/b>/g, "**")
-		.replace(/<em>/g, "*")
+		.replace(/<em[^>]*>/g, "*")
 		.replace(/<\/em>/g, "*")
-		.replace(/<i>/g, "*")
+		.replace(/<i[^>]*>/g, "*")
 		.replace(/<\/i>/g, "*")
-		.replace(/<a href="([^"]+)"[^>]*>([^<]+)<\/a>/g, "[$2]($1)")
-		.replace(/<p>/g, "\n\n")
+		.replace(
+			/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g,
+			(_, href, text) => `[${text.replace(/<[^>]+>/g, "").trim()}](${href})`,
+		)
+		.replace(/<p[^>]*>/g, "\n\n")
 		.replace(/<\/p>/g, "")
 		.replace(/<br\s*\/?>/g, "\n")
-		.replace(/<li>/g, "- ")
+		.replace(/<li[^>]*>/g, "- ")
 		.replace(/<\/li>/g, "\n")
-		.replace(/<\/?[uo]l>/g, "\n")
-		.replace(/<h(\d)>/g, (_, n) => `\n${"#".repeat(parseInt(n, 10))} `)
+		.replace(/<\/?[uo]l[^>]*>/g, "\n")
+		.replace(/<h(\d)[^>]*>/g, (_, n) => `\n${"#".repeat(parseInt(n, 10))} `)
 		.replace(/<\/h\d>/g, "\n")
-		.replace(/<blockquote>/g, "\n> ")
+		.replace(/<blockquote[^>]*>/g, "\n> ")
 		.replace(/<\/blockquote>/g, "\n")
 		.replace(/<[^>]+>/g, "")
+		.replace(/\n{3,}/g, "\n\n")
+		.trim();
+	return decodeHtmlEntities(stripped);
+}
+
+/**
+ * Try to parse JSON, returning null on failure.
+ */
+export function tryParseJson<T = unknown>(content: string): T | null {
+	try {
+		return JSON.parse(content) as T;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Build a RenderResult from markdown content. Calls finalizeOutput internally.
+ */
+export function buildResult(
+	md: string,
+	opts: { url: string; finalUrl?: string; method: string; fetchedAt: string; notes?: string[]; contentType?: string },
+): RenderResult {
+	const output = finalizeOutput(md);
+	return {
+		url: opts.url,
+		finalUrl: opts.finalUrl ?? opts.url,
+		contentType: opts.contentType ?? "text/markdown",
+		method: opts.method,
+		content: output.content,
+		fetchedAt: opts.fetchedAt,
+		truncated: output.truncated,
+		notes: opts.notes ?? [],
+	};
+}
+
+/**
+ * Format a date value as YYYY-MM-DD. Returns empty string on invalid input.
+ */
+export function formatIsoDate(value?: string | number | Date): string {
+	if (value == null) return "";
+	if (typeof value === "string") {
+		const datePrefix = value.match(/^\d{4}-\d{2}-\d{2}/);
+		if (datePrefix) return datePrefix[0];
+	}
+	try {
+		return new Date(value).toISOString().split("T")[0];
+	} catch {
+		return "";
+	}
+}
+
+/**
+ * Decode common HTML entities.
+ */
+export function decodeHtmlEntities(text: string): string {
+	return text
 		.replace(/&lt;/g, "<")
 		.replace(/&gt;/g, ">")
 		.replace(/&amp;/g, "&")
 		.replace(/&quot;/g, '"')
-		.replace(/&#39;/g, "'")
-		.replace(/&nbsp;/g, " ")
-		.replace(/\n{3,}/g, "\n\n")
-		.trim();
+		.replace(/&#0?39;/g, "'")
+		.replace(/&#x27;/g, "'")
+		.replace(/&#x2F;/g, "/")
+		.replace(/&nbsp;/g, " ");
+}
+
+/**
+ * Format seconds into HH:MM:SS or MM:SS.
+ */
+export function formatMediaDuration(totalSeconds: number): string {
+	const hours = Math.floor(totalSeconds / 3600);
+	const minutes = Math.floor((totalSeconds % 3600) / 60);
+	const secs = Math.floor(totalSeconds % 60);
+	if (hours > 0) return `${hours}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
+	return `${minutes}:${String(secs).padStart(2, "0")}`;
+}
+
+/**
+ * Extract localized text, preferring en-US/en.
+ */
+export type LocalizedText = string | Record<string, string | null> | null | undefined;
+
+export function getLocalizedText(value: LocalizedText, defaultLocale?: string): string | undefined {
+	if (value == null) return undefined;
+	if (typeof value === "string") return value;
+	if (defaultLocale && value[defaultLocale]) return value[defaultLocale];
+	return (
+		value["en-US"] ?? value.en_US ?? value.en ?? Object.values(value).find(v => typeof v === "string") ?? undefined
+	);
 }