@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/nvd.ts

@@ -0,0 +1,243 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface CvssV31 {
+	baseScore: number;
+	baseSeverity: string;
+	vectorString: string;
+}
+
+interface CvssV2 {
+	baseScore: number;
+	severity?: string;
+	vectorString: string;
+}
+
+interface CvssMetric {
+	cvssData: CvssV31 | CvssV2;
+	exploitabilityScore?: number;
+	impactScore?: number;
+}
+
+interface CpeMatch {
+	criteria: string;
+	vulnerable: boolean;
+	versionStartIncluding?: string;
+	versionEndExcluding?: string;
+	versionEndIncluding?: string;
+}
+
+interface Configuration {
+	nodes?: Array<{
+		operator?: string;
+		cpeMatch?: CpeMatch[];
+	}>;
+}
+
+interface Reference {
+	url: string;
+	source?: string;
+	tags?: string[];
+}
+
+interface Description {
+	lang: string;
+	value: string;
+}
+
+interface Weakness {
+	description: Description[];
+}
+
+interface CveItem {
+	id: string;
+	sourceIdentifier?: string;
+	published: string;
+	lastModified: string;
+	vulnStatus?: string;
+	descriptions: Description[];
+	metrics?: {
+		cvssMetricV31?: CvssMetric[];
+		cvssMetricV30?: CvssMetric[];
+		cvssMetricV2?: CvssMetric[];
+	};
+	weaknesses?: Weakness[];
+	configurations?: Configuration[];
+	references?: Reference[];
+}
+
+interface NvdResponse {
+	vulnerabilities?: Array<{ cve: CveItem }>;
+}
+
+/**
+ * Handle NVD (National Vulnerability Database) CVE URLs
+ */
+export const handleNvd: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("nvd.nist.gov")) return null;
+
+		// Extract CVE ID from /vuln/detail/{CVE-ID}
+		const match = parsed.pathname.match(/\/vuln\/detail\/(CVE-\d{4}-\d+)/i);
+		if (!match) return null;
+
+		const cveId = match[1].toUpperCase();
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from NVD API
+		const apiUrl = `https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=${cveId}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let data: NvdResponse;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const vuln = data.vulnerabilities?.[0]?.cve;
+		if (!vuln) return null;
+
+		let md = `# ${vuln.id}\n\n`;
+
+		// Status and dates
+		if (vuln.vulnStatus) {
+			md += `**Status:** ${vuln.vulnStatus}\n`;
+		}
+		md += `**Published:** ${formatDate(vuln.published)}`;
+		md += ` · **Modified:** ${formatDate(vuln.lastModified)}\n\n`;
+
+		// Description
+		const desc = vuln.descriptions.find((d) => d.lang === "en")?.value;
+		if (desc) {
+			md += `## Description\n\n${desc}\n\n`;
+		}
+
+		// CVSS Scores
+		const cvss31 = vuln.metrics?.cvssMetricV31?.[0];
+		const cvss30 = vuln.metrics?.cvssMetricV30?.[0];
+		const cvss2 = vuln.metrics?.cvssMetricV2?.[0];
+
+		if (cvss31 || cvss30 || cvss2) {
+			md += `## CVSS Scores\n\n`;
+
+			if (cvss31) {
+				const data = cvss31.cvssData as CvssV31;
+				md += `### CVSS 3.1\n\n`;
+				md += `- **Base Score:** ${data.baseScore} (${data.baseSeverity})\n`;
+				md += `- **Vector:** \`${data.vectorString}\`\n`;
+				if (cvss31.exploitabilityScore !== undefined) {
+					md += `- **Exploitability:** ${cvss31.exploitabilityScore}\n`;
+				}
+				if (cvss31.impactScore !== undefined) {
+					md += `- **Impact:** ${cvss31.impactScore}\n`;
+				}
+				md += "\n";
+			}
+
+			if (cvss30 && !cvss31) {
+				const data = cvss30.cvssData as CvssV31;
+				md += `### CVSS 3.0\n\n`;
+				md += `- **Base Score:** ${data.baseScore} (${data.baseSeverity})\n`;
+				md += `- **Vector:** \`${data.vectorString}\`\n`;
+				md += "\n";
+			}
+
+			if (cvss2) {
+				const data = cvss2.cvssData as CvssV2;
+				md += `### CVSS 2.0\n\n`;
+				md += `- **Base Score:** ${data.baseScore}`;
+				if (data.severity) md += ` (${data.severity})`;
+				md += `\n- **Vector:** \`${data.vectorString}\`\n\n`;
+			}
+		}
+
+		// Weaknesses (CWE)
+		const cwes = vuln.weaknesses
+			?.flatMap((w) => w.description)
+			.filter((d) => d.lang === "en" && d.value !== "NVD-CWE-Other" && d.value !== "NVD-CWE-noinfo");
+
+		if (cwes?.length) {
+			md += `## Weaknesses\n\n`;
+			for (const cwe of cwes) {
+				md += `- ${cwe.value}\n`;
+			}
+			md += "\n";
+		}
+
+		// Affected Products (CPE)
+		const cpes = extractCpes(vuln.configurations);
+		if (cpes.length > 0) {
+			md += `## Affected Products\n\n`;
+			const shown = cpes.slice(0, 20);
+			for (const cpe of shown) {
+				md += `- \`${cpe}\`\n`;
+			}
+			if (cpes.length > 20) {
+				md += `\n*...and ${cpes.length - 20} more*\n`;
+			}
+			md += "\n";
+		}
+
+		// References
+		if (vuln.references?.length) {
+			md += `## References\n\n`;
+			for (const ref of vuln.references.slice(0, 15)) {
+				const tags = ref.tags?.length ? ` (${ref.tags.join(", ")})` : "";
+				md += `- ${ref.url}${tags}\n`;
+			}
+			if (vuln.references.length > 15) {
+				md += `\n*...and ${vuln.references.length - 15} more references*\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "nvd",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via NVD API"],
+		};
+	} catch {}
+
+	return null;
+};
+
+function formatDate(iso: string): string {
+	try {
+		return new Date(iso).toISOString().split("T")[0];
+	} catch {
+		return iso;
+	}
+}
+
+function extractCpes(configurations?: Configuration[]): string[] {
+	if (!configurations) return [];
+
+	const cpes: string[] = [];
+	for (const config of configurations) {
+		for (const node of config.nodes ?? []) {
+			for (const match of node.cpeMatch ?? []) {
+				if (match.vulnerable && match.criteria) {
+					cpes.push(match.criteria);
+				}
+			}
+		}
+	}
+	return Array.from(new Set(cpes));
+}

package/src/core/tools/web-scrapers/ollama.ts

@@ -0,0 +1,267 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface OllamaTagDetails {
+	parent_model?: string;
+	format?: string;
+	family?: string;
+	families?: string[] | null;
+	parameter_size?: string;
+	quantization_level?: string;
+}
+
+interface OllamaTagModel {
+	name?: string;
+	model?: string;
+	modified_at?: string;
+	size?: number;
+	digest?: string;
+	details?: OllamaTagDetails;
+}
+
+interface OllamaTagsResponse {
+	models?: OllamaTagModel[];
+}
+
+const VALID_HOSTNAMES = new Set(["ollama.com", "www.ollama.com"]);
+const RESERVED_ROOTS = new Set([
+	"models",
+	"blog",
+	"docs",
+	"download",
+	"cloud",
+	"signin",
+	"signout",
+	"search",
+	"api",
+	"terms",
+	"privacy",
+	"license",
+	"settings",
+]);
+
+function decodeHtmlEntities(value: string): string {
+	return value
+		.replace(/&/g, "&")
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&quot;/g, '"')
+		.replace(/&#39;/g, "'")
+		.replace(/&nbsp;/g, " ");
+}
+
+function extractMetaDescription(html: string): string | null {
+	const patterns = [
+		/<meta[^>]+name=["']description["'][^>]*content=["']([^"']+)["']/i,
+		/<meta[^>]+property=["']og:description["'][^>]*content=["']([^"']+)["']/i,
+		/<meta[^>]+property=["']twitter:description["'][^>]*content=["']([^"']+)["']/i,
+	];
+
+	for (const pattern of patterns) {
+		const match = html.match(pattern);
+		if (match?.[1]) {
+			return decodeHtmlEntities(match[1].trim());
+		}
+	}
+
+	return null;
+}
+
+function extractParameterSizes(html: string): string[] {
+	const sizes = new Set<string>();
+	const pattern = /x-test-size[^>]*>([^<]+)<\/span>/gi;
+	let match = pattern.exec(html);
+	while (match) {
+		const raw = match[1]?.trim();
+		if (raw) {
+			sizes.add(raw.toUpperCase());
+		}
+		match = pattern.exec(html);
+	}
+
+	return Array.from(sizes);
+}
+
+function extractTagsFromHtml(html: string, baseRef: string): string[] {
+	const tags = new Set<string>();
+	const pattern = /href=["']\/library\/([^"']+)["']/gi;
+	let match = pattern.exec(html);
+	while (match) {
+		const raw = match[1]?.trim();
+		if (raw) {
+			const decoded = decodeHtmlEntities(raw);
+			if (decoded === baseRef || decoded.startsWith(`${baseRef}:`)) {
+				tags.add(decoded);
+			}
+		}
+		match = pattern.exec(html);
+	}
+
+	return Array.from(tags);
+}
+
+function formatSize(bytes: number): string {
+	if (bytes >= 1_000_000_000) return `${(bytes / 1_000_000_000).toFixed(1)}GB`;
+	if (bytes >= 1_000_000) return `${(bytes / 1_000_000).toFixed(1)}MB`;
+	if (bytes >= 1_000) return `${(bytes / 1_000).toFixed(1)}KB`;
+	return `${bytes}B`;
+}
+
+function buildModelPath(parts: string[]): string {
+	return parts.map((part) => encodeURIComponent(part)).join("/");
+}
+
+function parseOllamaUrl(url: string): { modelRef: string; baseRef: string; pageUrl: string } | null {
+	try {
+		const parsed = new URL(url);
+		if (!VALID_HOSTNAMES.has(parsed.hostname)) return null;
+
+		const parts = parsed.pathname.split("/").filter(Boolean);
+		if (parts.length === 0) return null;
+
+		if (parts[0] === "library" && parts.length >= 2) {
+			const modelRef = decodeURIComponent(parts[1]);
+			const baseRef = modelRef.split(":")[0] ?? modelRef;
+			const pageUrl = `${parsed.origin}/${buildModelPath(["library", baseRef])}`;
+			return { modelRef, baseRef, pageUrl };
+		}
+
+		if (parts.length >= 2 && !RESERVED_ROOTS.has(parts[0])) {
+			const namespace = decodeURIComponent(parts[0]);
+			const model = decodeURIComponent(parts[1]);
+			const modelBase = model.split(":")[0] ?? model;
+			const modelRef = `${namespace}/${model}`;
+			const baseRef = `${namespace}/${modelBase}`;
+			const pageUrl = `${parsed.origin}/${buildModelPath([namespace, modelBase])}`;
+			return { modelRef, baseRef, pageUrl };
+		}
+	} catch {}
+
+	return null;
+}
+
+function sortTags(tags: string[]): string[] {
+	return tags.sort((a, b) => {
+		const aLatest = a.endsWith(":latest");
+		const bLatest = b.endsWith(":latest");
+		if (aLatest && !bLatest) return -1;
+		if (!aLatest && bLatest) return 1;
+		return a.localeCompare(b);
+	});
+}
+
+function formatTagList(tags: string[], maxItems: number): string {
+	const limited = tags.slice(0, maxItems);
+	const formatted = limited.map((tag) => `\`${tag}\``).join(", ");
+	if (tags.length > maxItems) {
+		return `${formatted} (and ${tags.length - maxItems} more)`;
+	}
+	return formatted;
+}
+
+function collectParameterSizes(models: OllamaTagModel[], htmlSizes: string[]): string[] {
+	const sizes = new Set<string>();
+	for (const model of models) {
+		const param = model.details?.parameter_size?.trim();
+		if (param) sizes.add(param.toUpperCase());
+	}
+	for (const size of htmlSizes) {
+		sizes.add(size);
+	}
+	return Array.from(sizes);
+}
+
+export const handleOllama: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = parseOllamaUrl(url);
+		if (!parsed) return null;
+
+		const { modelRef, baseRef, pageUrl } = parsed;
+		const fetchedAt = new Date().toISOString();
+
+		const tagsUrl = "https://ollama.com/api/tags";
+		const [tagsResult, pageResult] = await Promise.all([
+			loadPage(tagsUrl, { timeout, signal, headers: { Accept: "application/json" } }),
+			loadPage(pageUrl, { timeout, signal }),
+		]);
+
+		let tagsData: OllamaTagsResponse | null = null;
+		if (tagsResult.ok) {
+			try {
+				tagsData = JSON.parse(tagsResult.content) as OllamaTagsResponse;
+			} catch {
+				tagsData = null;
+			}
+		}
+
+		const html = pageResult.ok ? pageResult.content : "";
+		const description = html ? extractMetaDescription(html) : null;
+		const htmlParameterSizes = html ? extractParameterSizes(html) : [];
+		const htmlTags = html ? extractTagsFromHtml(html, baseRef) : [];
+
+		const baseLower = baseRef.toLowerCase();
+		const models = tagsData?.models ?? [];
+		const matchingModels = models.filter((model) => {
+			const name = (model.model ?? model.name ?? "").toLowerCase();
+			return name === baseLower || name.startsWith(`${baseLower}:`);
+		});
+
+		const tagRef = modelRef.includes(":") ? modelRef : null;
+		const selectedTag = tagRef ? matchingModels.find((model) => (model.model ?? model.name ?? "") === tagRef) : null;
+
+		const availableTagsRaw = matchingModels
+			.map((model) => model.model ?? model.name ?? "")
+			.filter((tag) => tag.length > 0);
+		const availableTags = sortTags(Array.from(new Set(availableTagsRaw)));
+
+		const fallbackTags = sortTags(Array.from(new Set(htmlTags)));
+		const tagsToUse = availableTags.length > 0 ? availableTags : fallbackTags;
+
+		const parameterSizes = collectParameterSizes(selectedTag ? [selectedTag] : matchingModels, htmlParameterSizes);
+
+		const sizes = matchingModels
+			.map((model) => model.size)
+			.filter((size): size is number => typeof size === "number");
+		let sizeLine: string | null = null;
+
+		if (selectedTag?.size) {
+			sizeLine = formatSize(selectedTag.size);
+		} else if (sizes.length > 0) {
+			const minSize = Math.min(...sizes);
+			const maxSize = Math.max(...sizes);
+			sizeLine = minSize === maxSize ? formatSize(minSize) : `${formatSize(minSize)} - ${formatSize(maxSize)}`;
+		}
+
+		let md = `# ${baseRef}\n\n`;
+		if (description) md += `${description}\n\n`;
+
+		md += `**Model:** ${baseRef}\n`;
+		if (tagRef) md += `**Tag:** ${tagRef}\n`;
+		if (parameterSizes.length > 0) md += `**Parameters:** ${parameterSizes.join(", ")}\n`;
+		if (sizeLine) {
+			const label = sizeLine.includes(" - ") ? "Size Range" : "Size";
+			md += `**${label}:** ${sizeLine}\n`;
+		}
+		if (tagsToUse.length > 0) {
+			md += `**Available Tags:** ${formatTagList(tagsToUse, 40)}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: pageResult.ok ? pageResult.finalUrl : url,
+			contentType: "text/markdown",
+			method: "ollama",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Ollama API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/open-vsx.ts

@@ -0,0 +1,119 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface OpenVsxFileLinks {
+	readme?: string;
+}
+
+interface OpenVsxExtension {
+	name: string;
+	namespace: string;
+	version: string;
+	displayName?: string;
+	description?: string;
+	downloadCount?: number;
+	averageRating?: number;
+	reviewCount?: number;
+	repository?: string | { url?: string };
+	license?: string;
+	categories?: string[];
+	homepage?: string;
+	files?: OpenVsxFileLinks;
+}
+
+/**
+ * Handle Open VSX URLs via their API
+ */
+export const handleOpenVsx: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "open-vsx.org" && parsed.hostname !== "www.open-vsx.org") return null;
+
+		const match = parsed.pathname.match(/^\/extension\/([^/]+)\/([^/]+)(?:\/([^/]+))?\/?$/);
+		if (!match) return null;
+
+		const namespace = decodeURIComponent(match[1]);
+		const extension = decodeURIComponent(match[2]);
+		const version = match[3] ? decodeURIComponent(match[3]) : null;
+
+		const fetchedAt = new Date().toISOString();
+		const baseUrl = `https://open-vsx.org/api/${encodeURIComponent(namespace)}/${encodeURIComponent(extension)}`;
+		const apiUrl = version ? `${baseUrl}/${encodeURIComponent(version)}` : baseUrl;
+
+		const result = await loadPage(apiUrl, { timeout, signal });
+		if (!result.ok) return null;
+
+		let data: OpenVsxExtension;
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		let readme: string | null = null;
+		const readmeUrl = data.files?.readme;
+		if (readmeUrl) {
+			try {
+				const readmeResult = await loadPage(readmeUrl, { timeout: Math.min(timeout, 10), signal });
+				if (readmeResult.ok) readme = readmeResult.content;
+			} catch {}
+		}
+
+		const displayName = data.displayName || data.name || `${namespace}/${extension}`;
+		const displayNamespace = data.namespace || namespace;
+		const displayVersion = data.version || version || "unknown";
+		const downloads = typeof data.downloadCount === "number" ? data.downloadCount : null;
+		const rating = typeof data.averageRating === "number" ? data.averageRating : null;
+		const reviews = typeof data.reviewCount === "number" ? data.reviewCount : null;
+		const repository = typeof data.repository === "string" ? data.repository : data.repository?.url || null;
+
+		let md = `# ${displayName}\n\n`;
+		if (data.description) md += `${data.description}\n\n`;
+
+		md += `**Namespace:** ${displayNamespace}\n`;
+		md += `**Extension:** ${data.name || extension}\n`;
+		md += `**Version:** ${displayVersion}`;
+		if (data.license) md += ` | **License:** ${data.license}`;
+		md += "\n";
+
+		if (downloads !== null) {
+			md += `**Downloads:** ${formatCount(downloads)}\n`;
+		}
+
+		if (rating !== null) {
+			const reviewSuffix = reviews !== null ? ` (${reviews} reviews)` : "";
+			md += `**Rating:** ${rating}${reviewSuffix}\n`;
+		}
+
+		if (repository) {
+			const cleanedRepo = repository.replace(/^git\+/, "").replace(/\.git$/, "");
+			md += `**Repository:** ${cleanedRepo}\n`;
+		}
+
+		if (data.homepage) md += `**Homepage:** ${data.homepage}\n`;
+		if (data.categories?.length) md += `**Categories:** ${data.categories.join(", ")}\n`;
+
+		if (readme) {
+			md += "\n---\n\n## README\n\n";
+			md += `${readme}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "open-vsx",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Open VSX API"],
+		};
+	} catch {}
+
+	return null;
+};