@wahack/pi-coding-agent 15.11.0

package/src/web/scrapers/nuget.ts

@@ -0,0 +1,183 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, formatIsoDate, formatNumber, loadPage } from "./types";
+
+interface NuGetCatalogEntry {
+	id: string;
+	version: string;
+	description?: string;
+	authors?: string;
+	projectUrl?: string;
+	licenseUrl?: string;
+	licenseExpression?: string;
+	tags?: string[];
+	dependencyGroups?: Array<{
+		targetFramework?: string;
+		dependencies?: Array<{
+			id: string;
+			range: string;
+		}>;
+	}>;
+	published?: string;
+}
+
+interface NuGetRegistrationItem {
+	catalogEntry: NuGetCatalogEntry;
+	packageContent?: string;
+}
+
+interface NuGetRegistrationPage {
+	items?: NuGetRegistrationItem[];
+	"@id"?: string;
+}
+
+interface NuGetRegistrationIndex {
+	items: NuGetRegistrationPage[];
+}
+
+/**
+ * Handle NuGet URLs via API
+ */
+export const handleNuGet: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "www.nuget.org" && parsed.hostname !== "nuget.org") return null;
+
+		// Extract package name and optional version from /packages/name or /packages/name/version
+		const match = parsed.pathname.match(/^\/packages\/([^/]+)(?:\/([^/]+))?/i);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const requestedVersion = match[2] ? decodeURIComponent(match[2]) : null;
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from NuGet registration API (package name must be lowercase)
+		const apiUrl = `https://api.nuget.org/v3/registration5-gz-semver2/${packageName.toLowerCase()}/index.json`;
+		const result = await loadPage(apiUrl, { timeout, signal });
+
+		if (!result.ok) return null;
+
+		const index = tryParseJson<NuGetRegistrationIndex>(result.content);
+		if (!index) return null;
+
+		if (!index.items?.length) return null;
+
+		// Get the latest page (or fetch it if not inlined)
+		let latestPage = index.items[index.items.length - 1];
+
+		// If items are not inlined, fetch the page
+		if (!latestPage.items && latestPage["@id"]) {
+			const pageResult = await loadPage(latestPage["@id"], { timeout, signal });
+			if (!pageResult.ok) return null;
+			const fetched = tryParseJson<NuGetRegistrationPage>(pageResult.content);
+			if (!fetched) return null;
+			latestPage = fetched;
+		}
+
+		if (!latestPage.items?.length) return null;
+
+		// Find the requested version or get the latest
+		let targetEntry: NuGetCatalogEntry | null = null;
+
+		if (requestedVersion) {
+			// Search all pages for the requested version
+			for (const page of index.items) {
+				let pageItems = page.items;
+
+				// Fetch page if items not inlined
+				if (!pageItems && page["@id"]) {
+					const pageResult = await loadPage(page["@id"], { timeout: Math.min(timeout, 5), signal });
+					if (pageResult.ok) {
+						const fetchedPage = tryParseJson<NuGetRegistrationPage>(pageResult.content);
+						if (fetchedPage) pageItems = fetchedPage.items;
+					}
+				}
+
+				if (pageItems) {
+					const found = pageItems.find(
+						item => item.catalogEntry.version.toLowerCase() === requestedVersion.toLowerCase(),
+					);
+					if (found) {
+						targetEntry = found.catalogEntry;
+						break;
+					}
+				}
+			}
+		}
+
+		// If no specific version requested or not found, use the latest
+		if (!targetEntry) {
+			const latestItem = latestPage.items[latestPage.items.length - 1];
+			targetEntry = latestItem.catalogEntry;
+		}
+
+		// Fetch download stats via search API
+		let totalDownloads: number | null = null;
+		const searchUrl = `https://api.nuget.org/v3/query?q=packageid:${encodeURIComponent(packageName)}&prerelease=true&take=1`;
+		const searchResult = await loadPage(searchUrl, { timeout: Math.min(timeout, 5), signal });
+
+		if (searchResult.ok) {
+			const searchData = tryParseJson<{ data?: Array<{ totalDownloads?: number }> }>(searchResult.content);
+			if (searchData) totalDownloads = searchData.data?.[0]?.totalDownloads ?? null;
+		}
+
+		// Format markdown output
+		let md = `# ${targetEntry.id}\n\n`;
+		if (targetEntry.description) md += `${targetEntry.description}\n\n`;
+
+		md += `**Version:** ${targetEntry.version}`;
+		if (targetEntry.licenseExpression) {
+			md += ` · **License:** ${targetEntry.licenseExpression}`;
+		} else if (targetEntry.licenseUrl) {
+			md += ` · **License:** [View](${targetEntry.licenseUrl})`;
+		}
+		md += "\n";
+
+		if (totalDownloads !== null) {
+			md += `**Total Downloads:** ${formatNumber(totalDownloads)}\n`;
+		}
+
+		if (targetEntry.authors) md += `**Authors:** ${targetEntry.authors}\n`;
+		if (targetEntry.projectUrl) md += `**Project URL:** ${targetEntry.projectUrl}\n`;
+		if (targetEntry.tags?.length) md += `**Tags:** ${targetEntry.tags.join(", ")}\n`;
+		if (targetEntry.published) {
+			md += `**Published:** ${formatIsoDate(targetEntry.published)}\n`;
+		}
+
+		// Show dependencies by target framework
+		if (targetEntry.dependencyGroups?.length) {
+			const hasAnyDeps = targetEntry.dependencyGroups.some(g => g.dependencies?.length);
+			if (hasAnyDeps) {
+				md += `\n## Dependencies\n\n`;
+				for (const group of targetEntry.dependencyGroups) {
+					if (!group.dependencies?.length) continue;
+					const framework = group.targetFramework || "All Frameworks";
+					md += `### ${framework}\n\n`;
+					for (const dep of group.dependencies) {
+						md += `- ${dep.id} (${dep.range})\n`;
+					}
+					md += "\n";
+				}
+			}
+		}
+
+		// Show recent versions from the latest page
+		if (latestPage.items && latestPage.items.length > 1) {
+			md += `## Recent Versions\n\n`;
+			const recentVersions = latestPage.items.slice(-5).reverse();
+			for (const item of recentVersions) {
+				const entry = item.catalogEntry;
+				const pubDate = formatIsoDate(entry.published) || "unknown";
+				md += `- **${entry.version}** (${pubDate})\n`;
+			}
+		}
+
+		return buildResult(md, { url, method: "nuget", fetchedAt, notes: ["Fetched via NuGet API"] });
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/nvd.ts

@@ -0,0 +1,222 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, formatIsoDate, loadPage } from "./types";
+
+interface CvssV31 {
+	baseScore: number;
+	baseSeverity: string;
+	vectorString: string;
+}
+
+interface CvssV2 {
+	baseScore: number;
+	severity?: string;
+	vectorString: string;
+}
+
+interface CvssMetric {
+	cvssData: CvssV31 | CvssV2;
+	exploitabilityScore?: number;
+	impactScore?: number;
+}
+
+interface CpeMatch {
+	criteria: string;
+	vulnerable: boolean;
+	versionStartIncluding?: string;
+	versionEndExcluding?: string;
+	versionEndIncluding?: string;
+}
+
+interface Configuration {
+	nodes?: Array<{
+		operator?: string;
+		cpeMatch?: CpeMatch[];
+	}>;
+}
+
+interface Reference {
+	url: string;
+	source?: string;
+	tags?: string[];
+}
+
+interface Description {
+	lang: string;
+	value: string;
+}
+
+interface Weakness {
+	description: Description[];
+}
+
+interface CveItem {
+	id: string;
+	sourceIdentifier?: string;
+	published: string;
+	lastModified: string;
+	vulnStatus?: string;
+	descriptions: Description[];
+	metrics?: {
+		cvssMetricV31?: CvssMetric[];
+		cvssMetricV30?: CvssMetric[];
+		cvssMetricV2?: CvssMetric[];
+	};
+	weaknesses?: Weakness[];
+	configurations?: Configuration[];
+	references?: Reference[];
+}
+
+interface NvdResponse {
+	vulnerabilities?: Array<{ cve: CveItem }>;
+}
+
+/**
+ * Handle NVD (National Vulnerability Database) CVE URLs
+ */
+export const handleNvd: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("nvd.nist.gov")) return null;
+
+		// Extract CVE ID from /vuln/detail/{CVE-ID}
+		const match = parsed.pathname.match(/\/vuln\/detail\/(CVE-\d{4}-\d+)/i);
+		if (!match) return null;
+
+		const cveId = match[1].toUpperCase();
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from NVD API
+		const apiUrl = `https://services.nvd.nist.gov/rest/json/cves/2.0?cveId=${cveId}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		const data = tryParseJson<NvdResponse>(result.content);
+		if (!data) return null;
+
+		const vuln = data.vulnerabilities?.[0]?.cve;
+		if (!vuln) return null;
+
+		let md = `# ${vuln.id}\n\n`;
+
+		// Status and dates
+		if (vuln.vulnStatus) {
+			md += `**Status:** ${vuln.vulnStatus}\n`;
+		}
+		md += `**Published:** ${formatIsoDate(vuln.published)}`;
+		md += ` · **Modified:** ${formatIsoDate(vuln.lastModified)}\n\n`;
+
+		// Description
+		const desc = vuln.descriptions.find(d => d.lang === "en")?.value;
+		if (desc) {
+			md += `## Description\n\n${desc}\n\n`;
+		}
+
+		// CVSS Scores
+		const cvss31 = vuln.metrics?.cvssMetricV31?.[0];
+		const cvss30 = vuln.metrics?.cvssMetricV30?.[0];
+		const cvss2 = vuln.metrics?.cvssMetricV2?.[0];
+
+		if (cvss31 || cvss30 || cvss2) {
+			md += `## CVSS Scores\n\n`;
+
+			if (cvss31) {
+				const data = cvss31.cvssData as CvssV31;
+				md += `### CVSS 3.1\n\n`;
+				md += `- **Base Score:** ${data.baseScore} (${data.baseSeverity})\n`;
+				md += `- **Vector:** \`${data.vectorString}\`\n`;
+				if (cvss31.exploitabilityScore !== undefined) {
+					md += `- **Exploitability:** ${cvss31.exploitabilityScore}\n`;
+				}
+				if (cvss31.impactScore !== undefined) {
+					md += `- **Impact:** ${cvss31.impactScore}\n`;
+				}
+				md += "\n";
+			}
+
+			if (cvss30 && !cvss31) {
+				const data = cvss30.cvssData as CvssV31;
+				md += `### CVSS 3.0\n\n`;
+				md += `- **Base Score:** ${data.baseScore} (${data.baseSeverity})\n`;
+				md += `- **Vector:** \`${data.vectorString}\`\n`;
+				md += "\n";
+			}
+
+			if (cvss2) {
+				const data = cvss2.cvssData as CvssV2;
+				md += `### CVSS 2.0\n\n`;
+				md += `- **Base Score:** ${data.baseScore}`;
+				if (data.severity) md += ` (${data.severity})`;
+				md += `\n- **Vector:** \`${data.vectorString}\`\n\n`;
+			}
+		}
+
+		// Weaknesses (CWE)
+		const cwes = vuln.weaknesses
+			?.flatMap(w => w.description)
+			.filter(d => d.lang === "en" && d.value !== "NVD-CWE-Other" && d.value !== "NVD-CWE-noinfo");
+
+		if (cwes?.length) {
+			md += `## Weaknesses\n\n`;
+			for (const cwe of cwes) {
+				md += `- ${cwe.value}\n`;
+			}
+			md += "\n";
+		}
+
+		// Affected Products (CPE)
+		const cpes = extractCpes(vuln.configurations);
+		if (cpes.length > 0) {
+			md += `## Affected Products\n\n`;
+			const shown = cpes.slice(0, 20);
+			for (const cpe of shown) {
+				md += `- \`${cpe}\`\n`;
+			}
+			if (cpes.length > 20) {
+				md += `\n*...and ${cpes.length - 20} more*\n`;
+			}
+			md += "\n";
+		}
+
+		// References
+		if (vuln.references?.length) {
+			md += `## References\n\n`;
+			for (const ref of vuln.references.slice(0, 15)) {
+				const tags = ref.tags?.length ? ` (${ref.tags.join(", ")})` : "";
+				md += `- ${ref.url}${tags}\n`;
+			}
+			if (vuln.references.length > 15) {
+				md += `\n*...and ${vuln.references.length - 15} more references*\n`;
+			}
+		}
+
+		return buildResult(md, { url, method: "nvd", fetchedAt, notes: ["Fetched via NVD API"] });
+	} catch {}
+
+	return null;
+};
+
+function extractCpes(configurations?: Configuration[]): string[] {
+	if (!configurations) return [];
+
+	const cpes: string[] = [];
+	for (const config of configurations) {
+		for (const node of config.nodes ?? []) {
+			for (const match of node.cpeMatch ?? []) {
+				if (match.vulnerable && match.criteria) {
+					cpes.push(match.criteria);
+				}
+			}
+		}
+	}
+	return Array.from(new Set(cpes));
+}

package/src/web/scrapers/ollama.ts

@@ -0,0 +1,239 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import { formatBytes } from "../../tools/render-utils";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, decodeHtmlEntities, loadPage } from "./types";
+
+interface OllamaTagDetails {
+	parent_model?: string;
+	format?: string;
+	family?: string;
+	families?: string[] | null;
+	parameter_size?: string;
+	quantization_level?: string;
+}
+
+interface OllamaTagModel {
+	name?: string;
+	model?: string;
+	modified_at?: string;
+	size?: number;
+	digest?: string;
+	details?: OllamaTagDetails;
+}
+
+interface OllamaTagsResponse {
+	models?: OllamaTagModel[];
+}
+
+const VALID_HOSTNAMES = new Set(["ollama.com", "www.ollama.com"]);
+const RESERVED_ROOTS = new Set([
+	"models",
+	"blog",
+	"docs",
+	"download",
+	"cloud",
+	"signin",
+	"signout",
+	"search",
+	"api",
+	"terms",
+	"privacy",
+	"license",
+	"settings",
+]);
+
+function extractMetaDescription(html: string): string | null {
+	const patterns = [
+		/<meta[^>]+name=["']description["'][^>]*content=["']([^"']+)["']/i,
+		/<meta[^>]+property=["']og:description["'][^>]*content=["']([^"']+)["']/i,
+		/<meta[^>]+property=["']twitter:description["'][^>]*content=["']([^"']+)["']/i,
+	];
+
+	for (const pattern of patterns) {
+		const match = html.match(pattern);
+		if (match?.[1]) {
+			return decodeHtmlEntities(match[1].trim());
+		}
+	}
+
+	return null;
+}
+
+function extractParameterSizes(html: string): string[] {
+	const sizes = new Set<string>();
+	const pattern = /x-test-size[^>]*>([^<]+)<\/span>/gi;
+	let match = pattern.exec(html);
+	while (match) {
+		const raw = match[1]?.trim();
+		if (raw) {
+			sizes.add(raw.toUpperCase());
+		}
+		match = pattern.exec(html);
+	}
+
+	return Array.from(sizes);
+}
+
+function extractTagsFromHtml(html: string, baseRef: string): string[] {
+	const tags = new Set<string>();
+	const pattern = /href=["']\/library\/([^"']+)["']/gi;
+	let match = pattern.exec(html);
+	while (match) {
+		const raw = match[1]?.trim();
+		if (raw) {
+			const decoded = decodeHtmlEntities(raw);
+			if (decoded === baseRef || decoded.startsWith(`${baseRef}:`)) {
+				tags.add(decoded);
+			}
+		}
+		match = pattern.exec(html);
+	}
+
+	return Array.from(tags);
+}
+
+function buildModelPath(parts: string[]): string {
+	return parts.map(part => encodeURIComponent(part)).join("/");
+}
+
+function parseOllamaUrl(url: string): { modelRef: string; baseRef: string; pageUrl: string } | null {
+	try {
+		const parsed = new URL(url);
+		if (!VALID_HOSTNAMES.has(parsed.hostname)) return null;
+
+		const parts = parsed.pathname.split("/").filter(Boolean);
+		if (parts.length === 0) return null;
+
+		if (parts[0] === "library" && parts.length >= 2) {
+			const modelRef = decodeURIComponent(parts[1]);
+			const baseRef = modelRef.split(":")[0] ?? modelRef;
+			const pageUrl = `${parsed.origin}/${buildModelPath(["library", baseRef])}`;
+			return { modelRef, baseRef, pageUrl };
+		}
+
+		if (parts.length >= 2 && !RESERVED_ROOTS.has(parts[0])) {
+			const namespace = decodeURIComponent(parts[0]);
+			const model = decodeURIComponent(parts[1]);
+			const modelBase = model.split(":")[0] ?? model;
+			const modelRef = `${namespace}/${model}`;
+			const baseRef = `${namespace}/${modelBase}`;
+			const pageUrl = `${parsed.origin}/${buildModelPath([namespace, modelBase])}`;
+			return { modelRef, baseRef, pageUrl };
+		}
+	} catch {}
+
+	return null;
+}
+
+function sortTags(tags: string[]): string[] {
+	return tags.sort((a, b) => {
+		const aLatest = a.endsWith(":latest");
+		const bLatest = b.endsWith(":latest");
+		if (aLatest && !bLatest) return -1;
+		if (!aLatest && bLatest) return 1;
+		return a.localeCompare(b);
+	});
+}
+
+function formatTagList(tags: string[], maxItems: number): string {
+	const limited = tags.slice(0, maxItems);
+	const formatted = limited.map(tag => `\`${tag}\``).join(", ");
+	if (tags.length > maxItems) {
+		return `${formatted} (and ${tags.length - maxItems} more)`;
+	}
+	return formatted;
+}
+
+function collectParameterSizes(models: OllamaTagModel[], htmlSizes: string[]): string[] {
+	const sizes = new Set<string>();
+	for (const model of models) {
+		const param = model.details?.parameter_size?.trim();
+		if (param) sizes.add(param.toUpperCase());
+	}
+	for (const size of htmlSizes) {
+		sizes.add(size);
+	}
+	return Array.from(sizes);
+}
+
+export const handleOllama: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = parseOllamaUrl(url);
+		if (!parsed) return null;
+
+		const { modelRef, baseRef, pageUrl } = parsed;
+		const fetchedAt = new Date().toISOString();
+
+		const tagsUrl = "https://ollama.com/api/tags";
+		const [tagsResult, pageResult] = await Promise.all([
+			loadPage(tagsUrl, { timeout, signal, headers: { Accept: "application/json" } }),
+			loadPage(pageUrl, { timeout, signal }),
+		]);
+
+		const tagsData = tagsResult.ok ? tryParseJson<OllamaTagsResponse>(tagsResult.content) : null;
+
+		const html = pageResult.ok ? pageResult.content : "";
+		const description = html ? extractMetaDescription(html) : null;
+		const htmlParameterSizes = html ? extractParameterSizes(html) : [];
+		const htmlTags = html ? extractTagsFromHtml(html, baseRef) : [];
+
+		const baseLower = baseRef.toLowerCase();
+		const models = tagsData?.models ?? [];
+		const matchingModels = models.filter(model => {
+			const name = (model.model ?? model.name ?? "").toLowerCase();
+			return name === baseLower || name.startsWith(`${baseLower}:`);
+		});
+
+		const tagRef = modelRef.includes(":") ? modelRef : null;
+		const selectedTag = tagRef ? matchingModels.find(model => (model.model ?? model.name ?? "") === tagRef) : null;
+
+		const availableTagsRaw = matchingModels
+			.map(model => model.model ?? model.name ?? "")
+			.filter(tag => tag.length > 0);
+		const availableTags = sortTags(Array.from(new Set(availableTagsRaw)));
+
+		const fallbackTags = sortTags(Array.from(new Set(htmlTags)));
+		const tagsToUse = availableTags.length > 0 ? availableTags : fallbackTags;
+
+		const parameterSizes = collectParameterSizes(selectedTag ? [selectedTag] : matchingModels, htmlParameterSizes);
+
+		const sizes = matchingModels.map(model => model.size).filter((size): size is number => typeof size === "number");
+		let sizeLine: string | null = null;
+
+		if (selectedTag?.size) {
+			sizeLine = formatBytes(selectedTag.size);
+		} else if (sizes.length > 0) {
+			const minSize = Math.min(...sizes);
+			const maxSize = Math.max(...sizes);
+			sizeLine = minSize === maxSize ? formatBytes(minSize) : `${formatBytes(minSize)} - ${formatBytes(maxSize)}`;
+		}
+
+		let md = `# ${baseRef}\n\n`;
+		if (description) md += `${description}\n\n`;
+
+		md += `**Model:** ${baseRef}\n`;
+		if (tagRef) md += `**Tag:** ${tagRef}\n`;
+		if (parameterSizes.length > 0) md += `**Parameters:** ${parameterSizes.join(", ")}\n`;
+		if (sizeLine) {
+			const label = sizeLine.includes(" - ") ? "Size Range" : "Size";
+			md += `**${label}:** ${sizeLine}\n`;
+		}
+		if (tagsToUse.length > 0) {
+			md += `**Available Tags:** ${formatTagList(tagsToUse, 40)}\n`;
+		}
+
+		return buildResult(md, {
+			url,
+			finalUrl: pageResult.ok ? pageResult.finalUrl : url,
+			method: "ollama",
+			fetchedAt,
+			notes: ["Fetched via Ollama API"],
+		});
+	} catch {}
+
+	return null;
+};