@oh-my-pi/pi-coding-agent 3.30.0 → 3.32.0

package/src/core/tools/web-scrapers/ollama.ts

@@ -0,0 +1,267 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface OllamaTagDetails {
+	parent_model?: string;
+	format?: string;
+	family?: string;
+	families?: string[] | null;
+	parameter_size?: string;
+	quantization_level?: string;
+}
+
+interface OllamaTagModel {
+	name?: string;
+	model?: string;
+	modified_at?: string;
+	size?: number;
+	digest?: string;
+	details?: OllamaTagDetails;
+}
+
+interface OllamaTagsResponse {
+	models?: OllamaTagModel[];
+}
+
+const VALID_HOSTNAMES = new Set(["ollama.com", "www.ollama.com"]);
+const RESERVED_ROOTS = new Set([
+	"models",
+	"blog",
+	"docs",
+	"download",
+	"cloud",
+	"signin",
+	"signout",
+	"search",
+	"api",
+	"terms",
+	"privacy",
+	"license",
+	"settings",
+]);
+
+function decodeHtmlEntities(value: string): string {
+	return value
+		.replace(/&/g, "&")
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&quot;/g, '"')
+		.replace(/&#39;/g, "'")
+		.replace(/&nbsp;/g, " ");
+}
+
+function extractMetaDescription(html: string): string | null {
+	const patterns = [
+		/<meta[^>]+name=["']description["'][^>]*content=["']([^"']+)["']/i,
+		/<meta[^>]+property=["']og:description["'][^>]*content=["']([^"']+)["']/i,
+		/<meta[^>]+property=["']twitter:description["'][^>]*content=["']([^"']+)["']/i,
+	];
+
+	for (const pattern of patterns) {
+		const match = html.match(pattern);
+		if (match?.[1]) {
+			return decodeHtmlEntities(match[1].trim());
+		}
+	}
+
+	return null;
+}
+
+function extractParameterSizes(html: string): string[] {
+	const sizes = new Set<string>();
+	const pattern = /x-test-size[^>]*>([^<]+)<\/span>/gi;
+	let match = pattern.exec(html);
+	while (match) {
+		const raw = match[1]?.trim();
+		if (raw) {
+			sizes.add(raw.toUpperCase());
+		}
+		match = pattern.exec(html);
+	}
+
+	return Array.from(sizes);
+}
+
+function extractTagsFromHtml(html: string, baseRef: string): string[] {
+	const tags = new Set<string>();
+	const pattern = /href=["']\/library\/([^"']+)["']/gi;
+	let match = pattern.exec(html);
+	while (match) {
+		const raw = match[1]?.trim();
+		if (raw) {
+			const decoded = decodeHtmlEntities(raw);
+			if (decoded === baseRef || decoded.startsWith(`${baseRef}:`)) {
+				tags.add(decoded);
+			}
+		}
+		match = pattern.exec(html);
+	}
+
+	return Array.from(tags);
+}
+
+function formatSize(bytes: number): string {
+	if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(1)}GB`;
+	if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)}MB`;
+	if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)}KB`;
+	return `${bytes}B`;
+}
+
+function buildModelPath(parts: string[]): string {
+	return parts.map((part) => encodeURIComponent(part)).join("/");
+}
+
+function parseOllamaUrl(url: string): { modelRef: string; baseRef: string; pageUrl: string } | null {
+	try {
+		const parsed = new URL(url);
+		if (!VALID_HOSTNAMES.has(parsed.hostname)) return null;
+
+		const parts = parsed.pathname.split("/").filter(Boolean);
+		if (parts.length === 0) return null;
+
+		if (parts[0] === "library" && parts.length >= 2) {
+			const modelRef = decodeURIComponent(parts[1]);
+			const baseRef = modelRef.split(":")[0] ?? modelRef;
+			const pageUrl = `${parsed.origin}/${buildModelPath(["library", baseRef])}`;
+			return { modelRef, baseRef, pageUrl };
+		}
+
+		if (parts.length >= 2 && !RESERVED_ROOTS.has(parts[0])) {
+			const namespace = decodeURIComponent(parts[0]);
+			const model = decodeURIComponent(parts[1]);
+			const modelBase = model.split(":")[0] ?? model;
+			const modelRef = `${namespace}/${model}`;
+			const baseRef = `${namespace}/${modelBase}`;
+			const pageUrl = `${parsed.origin}/${buildModelPath([namespace, modelBase])}`;
+			return { modelRef, baseRef, pageUrl };
+		}
+	} catch {}
+
+	return null;
+}
+
+function sortTags(tags: string[]): string[] {
+	return tags.sort((a, b) => {
+		const aLatest = a.endsWith(":latest");
+		const bLatest = b.endsWith(":latest");
+		if (aLatest && !bLatest) return -1;
+		if (!aLatest && bLatest) return 1;
+		return a.localeCompare(b);
+	});
+}
+
+function formatTagList(tags: string[], maxItems: number): string {
+	const limited = tags.slice(0, maxItems);
+	const formatted = limited.map((tag) => `\`${tag}\``).join(", ");
+	if (tags.length > maxItems) {
+		return `${formatted} (and ${tags.length - maxItems} more)`;
+	}
+	return formatted;
+}
+
+function collectParameterSizes(models: OllamaTagModel[], htmlSizes: string[]): string[] {
+	const sizes = new Set<string>();
+	for (const model of models) {
+		const param = model.details?.parameter_size?.trim();
+		if (param) sizes.add(param.toUpperCase());
+	}
+	for (const size of htmlSizes) {
+		sizes.add(size);
+	}
+	return Array.from(sizes);
+}
+
+export const handleOllama: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = parseOllamaUrl(url);
+		if (!parsed) return null;
+
+		const { modelRef, baseRef, pageUrl } = parsed;
+		const fetchedAt = new Date().toISOString();
+
+		const tagsUrl = "https://ollama.com/api/tags";
+		const [tagsResult, pageResult] = await Promise.all([
+			loadPage(tagsUrl, { timeout, signal, headers: { Accept: "application/json" } }),
+			loadPage(pageUrl, { timeout, signal }),
+		]);
+
+		let tagsData: OllamaTagsResponse | null = null;
+		if (tagsResult.ok) {
+			try {
+				tagsData = JSON.parse(tagsResult.content) as OllamaTagsResponse;
+			} catch {
+				tagsData = null;
+			}
+		}
+
+		const html = pageResult.ok ? pageResult.content : "";
+		const description = html ? extractMetaDescription(html) : null;
+		const htmlParameterSizes = html ? extractParameterSizes(html) : [];
+		const htmlTags = html ? extractTagsFromHtml(html, baseRef) : [];
+
+		const baseLower = baseRef.toLowerCase();
+		const models = tagsData?.models ?? [];
+		const matchingModels = models.filter((model) => {
+			const name = (model.model ?? model.name ?? "").toLowerCase();
+			return name === baseLower || name.startsWith(`${baseLower}:`);
+		});
+
+		const tagRef = modelRef.includes(":") ? modelRef : null;
+		const selectedTag = tagRef ? matchingModels.find((model) => (model.model ?? model.name ?? "") === tagRef) : null;
+
+		const availableTagsRaw = matchingModels
+			.map((model) => model.model ?? model.name ?? "")
+			.filter((tag) => tag.length > 0);
+		const availableTags = sortTags(Array.from(new Set(availableTagsRaw)));
+
+		const fallbackTags = sortTags(Array.from(new Set(htmlTags)));
+		const tagsToUse = availableTags.length > 0 ? availableTags : fallbackTags;
+
+		const parameterSizes = collectParameterSizes(selectedTag ? [selectedTag] : matchingModels, htmlParameterSizes);
+
+		const sizes = matchingModels
+			.map((model) => model.size)
+			.filter((size): size is number => typeof size === "number");
+		let sizeLine: string | null = null;
+
+		if (selectedTag?.size) {
+			sizeLine = formatSize(selectedTag.size);
+		} else if (sizes.length > 0) {
+			const minSize = Math.min(...sizes);
+			const maxSize = Math.max(...sizes);
+			sizeLine = minSize === maxSize ? formatSize(minSize) : `${formatSize(minSize)} - ${formatSize(maxSize)}`;
+		}
+
+		let md = `# ${baseRef}\n\n`;
+		if (description) md += `${description}\n\n`;
+
+		md += `**Model:** ${baseRef}\n`;
+		if (tagRef) md += `**Tag:** ${tagRef}\n`;
+		if (parameterSizes.length > 0) md += `**Parameters:** ${parameterSizes.join(", ")}\n`;
+		if (sizeLine) {
+			const label = sizeLine.includes(" - ") ? "Size Range" : "Size";
+			md += `**${label}:** ${sizeLine}\n`;
+		}
+		if (tagsToUse.length > 0) {
+			md += `**Available Tags:** ${formatTagList(tagsToUse, 40)}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: pageResult.ok ? pageResult.finalUrl : url,
+			contentType: "text/markdown",
+			method: "ollama",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Ollama API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/open-vsx.ts

@@ -0,0 +1,119 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface OpenVsxFileLinks {
+	readme?: string;
+}
+
+interface OpenVsxExtension {
+	name: string;
+	namespace: string;
+	version: string;
+	displayName?: string;
+	description?: string;
+	downloadCount?: number;
+	averageRating?: number;
+	reviewCount?: number;
+	repository?: string | { url?: string };
+	license?: string;
+	categories?: string[];
+	homepage?: string;
+	files?: OpenVsxFileLinks;
+}
+
+/**
+ * Handle Open VSX URLs via their API
+ */
+export const handleOpenVsx: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "open-vsx.org" && parsed.hostname !== "www.open-vsx.org") return null;
+
+		const match = parsed.pathname.match(/^\/extension\/([^/]+)\/([^/]+)(?:\/([^/]+))?\/?$/);
+		if (!match) return null;
+
+		const namespace = decodeURIComponent(match[1]);
+		const extension = decodeURIComponent(match[2]);
+		const version = match[3] ? decodeURIComponent(match[3]) : null;
+
+		const fetchedAt = new Date().toISOString();
+		const baseUrl = `https://open-vsx.org/api/${encodeURIComponent(namespace)}/${encodeURIComponent(extension)}`;
+		const apiUrl = version ? `${baseUrl}/${encodeURIComponent(version)}` : baseUrl;
+
+		const result = await loadPage(apiUrl, { timeout, signal });
+		if (!result.ok) return null;
+
+		let data: OpenVsxExtension;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		let readme: string | null = null;
+		const readmeUrl = data.files?.readme;
+		if (readmeUrl) {
+			try {
+				const readmeResult = await loadPage(readmeUrl, { timeout: Math.min(timeout, 10), signal });
+				if (readmeResult.ok) readme = readmeResult.content;
+			} catch {}
+		}
+
+		const displayName = data.displayName || data.name || `${namespace}/${extension}`;
+		const displayNamespace = data.namespace || namespace;
+		const displayVersion = data.version || version || "unknown";
+		const downloads = typeof data.downloadCount === "number" ? data.downloadCount : null;
+		const rating = typeof data.averageRating === "number" ? data.averageRating : null;
+		const reviews = typeof data.reviewCount === "number" ? data.reviewCount : null;
+		const repository = typeof data.repository === "string" ? data.repository : data.repository?.url || null;
+
+		let md = `# ${displayName}\n\n`;
+		if (data.description) md += `${data.description}\n\n`;
+
+		md += `**Namespace:** ${displayNamespace}\n`;
+		md += `**Extension:** ${data.name || extension}\n`;
+		md += `**Version:** ${displayVersion}`;
+		if (data.license) md += ` | **License:** ${data.license}`;
+		md += "\n";
+
+		if (downloads !== null) {
+			md += `**Downloads:** ${formatCount(downloads)}\n`;
+		}
+
+		if (rating !== null) {
+			const reviewSuffix = reviews !== null ? ` (${reviews} reviews)` : "";
+			md += `**Rating:** ${rating}${reviewSuffix}\n`;
+		}
+
+		if (repository) {
+			const cleanedRepo = repository.replace(/^git\+/, "").replace(/\.git$/, "");
+			md += `**Repository:** ${cleanedRepo}\n`;
+		}
+
+		if (data.homepage) md += `**Homepage:** ${data.homepage}\n`;
+		if (data.categories?.length) md += `**Categories:** ${data.categories.join(", ")}\n`;
+
+		if (readme) {
+			md += "\n---\n\n## README\n\n";
+			md += `${readme}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "open-vsx",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Open VSX API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/{web-fetch-handlers → web-scrapers}/opencorporates.ts

@@ -81,6 +81,7 @@ interface ApiResponse {
 export const handleOpenCorporates: SpecialHandler = async (
 	url: string,
 	timeout: number,
+	signal?: AbortSignal,
 ): Promise<RenderResult | null> => {
 	try {
 		const parsed = new URL(url);
@@ -100,6 +101,7 @@ export const handleOpenCorporates: SpecialHandler = async (
 		const result = await loadPage(apiUrl, {
 			timeout,
 			headers: { Accept: "application/json" },
+			signal,
 		});
 
 		if (!result.ok) return null;

package/src/core/tools/{web-fetch-handlers → web-scrapers}/openlibrary.ts

@@ -67,7 +67,11 @@ interface OpenLibraryBooksApiResponse {
 /**
  * Handle Open Library URLs via their API
  */
-export const handleOpenLibrary: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+export const handleOpenLibrary: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
 	try {
 		const parsed = new URL(url);
 		if (!parsed.hostname.includes("openlibrary.org")) return null;
@@ -83,11 +87,11 @@ export const handleOpenLibrary: SpecialHandler = async (url: string, timeout: nu
 		let md: string | null = null;
 
 		if (workMatch) {
-			md = await fetchWork(workMatch[1], timeout);
+			md = await fetchWork(workMatch[1], timeout, signal);
 		} else if (editionMatch) {
-			md = await fetchEdition(editionMatch[1], timeout);
+			md = await fetchEdition(editionMatch[1], timeout, signal);
 		} else if (isbnMatch) {
-			md = await fetchByIsbn(isbnMatch[1], timeout);
+			md = await fetchByIsbn(isbnMatch[1], timeout, signal);
 		}
 
 		if (!md) return null;
@@ -108,9 +112,9 @@ export const handleOpenLibrary: SpecialHandler = async (url: string, timeout: nu
 	return null;
 };
 
-async function fetchWork(workId: string, timeout: number): Promise<string | null> {
+async function fetchWork(workId: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
 	const apiUrl = `https://openlibrary.org/works/${workId}.json`;
-	const result = await loadPage(apiUrl, { timeout });
+	const result = await loadPage(apiUrl, { timeout, signal });
 	if (!result.ok) return null;
 
 	let work: OpenLibraryWork;
@@ -127,6 +131,7 @@ async function fetchWork(workId: string, timeout: number): Promise<string | null
 		const authorNames = await fetchAuthorNames(
 			work.authors.map((a) => a.author.key),
 			timeout,
+			signal,
 		);
 		if (authorNames.length) {
 			md += `**Authors:** ${authorNames.join(", ")}\n`;
@@ -157,9 +162,9 @@ async function fetchWork(workId: string, timeout: number): Promise<string | null
 	return md;
 }
 
-async function fetchEdition(editionId: string, timeout: number): Promise<string | null> {
+async function fetchEdition(editionId: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
 	const apiUrl = `https://openlibrary.org/books/${editionId}.json`;
-	const result = await loadPage(apiUrl, { timeout });
+	const result = await loadPage(apiUrl, { timeout, signal });
 	if (!result.ok) return null;
 
 	let edition: OpenLibraryEdition;
@@ -176,6 +181,7 @@ async function fetchEdition(editionId: string, timeout: number): Promise<string
 		const authorNames = await fetchAuthorNames(
 			edition.authors.map((a) => a.key),
 			timeout,
+			signal,
 		);
 		if (authorNames.length) {
 			md += `**Authors:** ${authorNames.join(", ")}\n`;
@@ -225,9 +231,9 @@ async function fetchEdition(editionId: string, timeout: number): Promise<string
 	return md;
 }
 
-async function fetchByIsbn(isbn: string, timeout: number): Promise<string | null> {
+async function fetchByIsbn(isbn: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
 	const apiUrl = `https://openlibrary.org/api/books?bibkeys=ISBN:${isbn}&format=json&jscmd=data`;
-	const result = await loadPage(apiUrl, { timeout });
+	const result = await loadPage(apiUrl, { timeout, signal });
 	if (!result.ok) return null;
 
 	let data: OpenLibraryBooksApiResponse;
@@ -281,7 +287,7 @@ async function fetchByIsbn(isbn: string, timeout: number): Promise<string | null
 	return md;
 }
 
-async function fetchAuthorNames(authorKeys: string[], timeout: number): Promise<string[]> {
+async function fetchAuthorNames(authorKeys: string[], timeout: number, signal?: AbortSignal): Promise<string[]> {
 	const names: string[] = [];
 
 	// Fetch authors in parallel (limit to first 5)
@@ -289,7 +295,7 @@ async function fetchAuthorNames(authorKeys: string[], timeout: number): Promise<
 		const authorKey = key.startsWith("/authors/") ? key : `/authors/${key}`;
 		const apiUrl = `https://openlibrary.org${authorKey}.json`;
 		try {
-			const result = await loadPage(apiUrl, { timeout: Math.min(timeout, 5) });
+			const result = await loadPage(apiUrl, { timeout: Math.min(timeout, 5), signal });
 			if (result.ok) {
 				const author = JSON.parse(result.content) as { name?: string };
 				return author.name || null;