@f5-sales-demo/xcsh 19.51.2

package/src/web/scrapers/rfc.ts

@@ -0,0 +1,201 @@
+import { tryParseJson } from "@f5xc-salesdemos/pi-utils";
+import { buildResult, loadPage, type RenderResult, type SpecialHandler } from "./types";
+
+interface RfcMetadata {
+	doc_id: string;
+	title: string;
+	authors?: Array<{ name: string; affiliation?: string }>;
+	pub_status?: string;
+	current_status?: string;
+	stream?: string;
+	area?: string;
+	wg_acronym?: string;
+	pub_date?: string;
+	page_count?: number;
+	abstract?: string;
+	keywords?: string[];
+	obsoletes?: string[];
+	obsoleted_by?: string[];
+	updates?: string[];
+	updated_by?: string[];
+	see_also?: string[];
+	errata_url?: string;
+}
+
+/**
+ * Extract RFC number from various URL patterns
+ */
+function extractRfcNumber(url: URL): string | null {
+	const { hostname, pathname } = url;
+
+	// https://www.rfc-editor.org/rfc/rfc{number}
+	// https://www.rfc-editor.org/rfc/rfc{number}.html
+	// https://www.rfc-editor.org/rfc/rfc{number}.txt
+	if (hostname === "www.rfc-editor.org" || hostname === "rfc-editor.org") {
+		const match = pathname.match(/\/rfc\/rfc(\d+)(?:\.(?:html|txt|pdf))?$/i);
+		if (match) return match[1];
+	}
+
+	// https://datatracker.ietf.org/doc/rfc{number}/
+	// https://datatracker.ietf.org/doc/html/rfc{number}
+	if (hostname === "datatracker.ietf.org") {
+		const match = pathname.match(/\/doc\/(?:html\/)?rfc(\d+)\/?$/i);
+		if (match) return match[1];
+	}
+
+	// https://tools.ietf.org/html/rfc{number}
+	if (hostname === "tools.ietf.org") {
+		const match = pathname.match(/\/html\/rfc(\d+)$/i);
+		if (match) return match[1];
+	}
+
+	return null;
+}
+
+/**
+ * Clean up RFC plain text - remove page headers/footers and extra formatting
+ */
+function cleanRfcText(text: string): string {
+	const lines = text.split("\n");
+	const cleaned: string[] = [];
+	let skipNext = 0;
+
+	for (let i = 0; i < lines.length; i++) {
+		const line = lines[i];
+
+		// Skip lines we've marked to skip (form feeds and surrounding blank lines)
+		if (skipNext > 0) {
+			skipNext--;
+			continue;
+		}
+
+		// Skip form feed characters and page headers (RFC NNNN ... Month Year pattern)
+		if (line.includes("\f")) {
+			// Skip the form feed line and typically 2-3 following header lines
+			skipNext = 3;
+			continue;
+		}
+
+		// Skip page footer lines (typically just a page number or "[Page N]")
+		if (/^\s*\[Page \d+\]\s*$/.test(line)) {
+			continue;
+		}
+
+		cleaned.push(line);
+	}
+
+	return cleaned.join("\n").replace(/\n{4,}/g, "\n\n\n");
+}
+
+/**
+ * Handle RFC Editor URLs - fetches IETF RFCs
+ */
+export const handleRfc: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const rfcNumber = extractRfcNumber(parsed);
+
+		if (!rfcNumber) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const notes: string[] = [];
+
+		// Fetch metadata JSON and plain text in parallel
+		const metadataUrl = `https://www.rfc-editor.org/rfc/rfc${rfcNumber}.json`;
+		const textUrl = `https://www.rfc-editor.org/rfc/rfc${rfcNumber}.txt`;
+
+		const [metaResult, textResult] = await Promise.all([
+			loadPage(metadataUrl, { timeout: Math.min(timeout, 10), signal }),
+			loadPage(textUrl, { timeout, signal }),
+		]);
+
+		// We need at least the text content
+		if (!textResult.ok) return null;
+
+		let metadata: RfcMetadata | null = null;
+		if (metaResult.ok) {
+			metadata = tryParseJson<RfcMetadata>(metaResult.content);
+			if (metadata) notes.push("Metadata from RFC Editor JSON API");
+		}
+
+		// Build markdown output
+		let md = "";
+
+		if (metadata) {
+			md += `# RFC ${rfcNumber}: ${metadata.title}\n\n`;
+
+			// Authors
+			if (metadata.authors?.length) {
+				const authorList = metadata.authors
+					.map(a => (a.affiliation ? `${a.name} (${a.affiliation})` : a.name))
+					.join(", ");
+				md += `**Authors:** ${authorList}\n`;
+			}
+
+			// Publication info
+			if (metadata.pub_date) md += `**Published:** ${metadata.pub_date}\n`;
+			if (metadata.current_status) md += `**Status:** ${metadata.current_status}\n`;
+			if (metadata.stream) md += `**Stream:** ${metadata.stream}\n`;
+			if (metadata.area) md += `**Area:** ${metadata.area}\n`;
+			if (metadata.wg_acronym) md += `**Working Group:** ${metadata.wg_acronym}\n`;
+			if (metadata.page_count) md += `**Pages:** ${metadata.page_count}\n`;
+
+			// Related RFCs
+			if (metadata.obsoletes?.length) {
+				md += `**Obsoletes:** ${metadata.obsoletes.join(", ")}\n`;
+			}
+			if (metadata.obsoleted_by?.length) {
+				md += `**Obsoleted by:** ${metadata.obsoleted_by.join(", ")}\n`;
+			}
+			if (metadata.updates?.length) {
+				md += `**Updates:** ${metadata.updates.join(", ")}\n`;
+			}
+			if (metadata.updated_by?.length) {
+				md += `**Updated by:** ${metadata.updated_by.join(", ")}\n`;
+			}
+
+			// Keywords
+			if (metadata.keywords?.length) {
+				md += `**Keywords:** ${metadata.keywords.join(", ")}\n`;
+			}
+
+			// Errata
+			if (metadata.errata_url) {
+				md += `**Errata:** ${metadata.errata_url}\n`;
+			}
+
+			md += "\n";
+
+			// Abstract from metadata
+			if (metadata.abstract) {
+				md += `## Abstract\n\n${metadata.abstract}\n\n`;
+			}
+
+			md += "---\n\n";
+		} else {
+			// No metadata, use simple header
+			md += `# RFC ${rfcNumber}\n\n`;
+			notes.push("Metadata not available, showing plain text only");
+		}
+
+		// Add full text content
+		md += "## Full Text\n\n";
+		md += "```\n";
+		md += cleanRfcText(textResult.content);
+		md += "\n```\n";
+
+		return buildResult(md, {
+			url,
+			finalUrl: `https://www.rfc-editor.org/rfc/rfc${rfcNumber}`,
+			method: "rfc",
+			fetchedAt,
+			notes: notes.length ? notes : ["Fetched from RFC Editor"],
+		});
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/rubygems.ts

@@ -0,0 +1,103 @@
+import { tryParseJson } from "@f5xc-salesdemos/pi-utils";
+import { buildResult, formatNumber, loadPage, type RenderResult, type SpecialHandler } from "./types";
+
+interface RubyGemsDependency {
+	name: string;
+	requirements: string;
+}
+
+interface RubyGemsResponse {
+	name: string;
+	version: string;
+	version_created_at?: string;
+	authors?: string;
+	info?: string;
+	licenses?: string[];
+	homepage_uri?: string;
+	source_code_uri?: string;
+	documentation_uri?: string;
+	project_uri?: string;
+	downloads: number;
+	version_downloads?: number;
+	gem_uri?: string;
+	dependencies?: {
+		development?: RubyGemsDependency[];
+		runtime?: RubyGemsDependency[];
+	};
+	metadata?: Record<string, string>;
+}
+
+/**
+ * Handle RubyGems URLs via API
+ */
+export const handleRubyGems: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "rubygems.org" && parsed.hostname !== "www.rubygems.org") return null;
+
+		// Extract gem name from /gems/{name}
+		const match = parsed.pathname.match(/^\/gems\/([^/]+)/);
+		if (!match) return null;
+
+		const gemName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from RubyGems API
+		const apiUrl = `https://rubygems.org/api/v1/gems/${encodeURIComponent(gemName)}.json`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: { Accept: "application/json" },
+		});
+
+		if (!result.ok) return null;
+
+		const gem = tryParseJson<RubyGemsResponse>(result.content);
+		if (!gem) return null;
+
+		let md = `# ${gem.name}\n\n`;
+		if (gem.info) md += `${gem.info}\n\n`;
+
+		// Version and license
+		md += `**Version:** ${gem.version}`;
+		if (gem.licenses?.length) md += ` · **License:** ${gem.licenses.join(", ")}`;
+		md += "\n";
+
+		// Downloads
+		md += `**Total Downloads:** ${formatNumber(gem.downloads)}`;
+		if (gem.version_downloads) md += ` · **Version Downloads:** ${formatNumber(gem.version_downloads)}`;
+		md += "\n\n";
+
+		// Links
+		if (gem.homepage_uri) md += `**Homepage:** ${gem.homepage_uri}\n`;
+		if (gem.source_code_uri) md += `**Source Code:** ${gem.source_code_uri}\n`;
+		if (gem.documentation_uri) md += `**Documentation:** ${gem.documentation_uri}\n`;
+		if (gem.authors) md += `**Authors:** ${gem.authors}\n`;
+
+		// Runtime dependencies
+		const runtimeDeps = gem.dependencies?.runtime;
+		if (runtimeDeps && runtimeDeps.length > 0) {
+			md += `\n## Runtime Dependencies\n\n`;
+			for (const dep of runtimeDeps) {
+				md += `- ${dep.name} ${dep.requirements}\n`;
+			}
+		}
+
+		// Development dependencies
+		const devDeps = gem.dependencies?.development;
+		if (devDeps && devDeps.length > 0) {
+			md += `\n## Development Dependencies\n\n`;
+			for (const dep of devDeps) {
+				md += `- ${dep.name} ${dep.requirements}\n`;
+			}
+		}
+
+		return buildResult(md, { url, method: "rubygems", fetchedAt, notes: ["Fetched via RubyGems API"] });
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/searchcode.ts

@@ -0,0 +1,190 @@
+import { tryParseJson } from "@f5xc-salesdemos/pi-utils";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, formatNumber, loadPage } from "./types";
+
+interface SearchcodeResult {
+	id?: number | string;
+	filename?: string;
+	repo?: string;
+	language?: string;
+	code?: string;
+	lines?: number | string | Array<number | string>;
+	location?: string;
+	url?: string;
+}
+
+interface SearchcodeSearchResponse {
+	query?: string;
+	results?: SearchcodeResult[];
+	total?: number;
+	total_results?: number;
+	nextpage?: number;
+}
+
+const VALID_HOSTS = new Set(["searchcode.com", "www.searchcode.com"]);
+
+function parseLineNumbers(lines: SearchcodeResult["lines"]): number[] | null {
+	if (typeof lines === "number" && Number.isFinite(lines)) return [lines];
+
+	if (typeof lines === "string") {
+		const parts = lines.split(/[,\s]+/).filter(Boolean);
+		const parsed = parts.map(part => Number.parseInt(part, 10)).filter(value => Number.isFinite(value));
+		return parsed.length ? parsed : null;
+	}
+
+	if (Array.isArray(lines)) {
+		const parsed = lines.map(part => Number.parseInt(String(part), 10)).filter(value => Number.isFinite(value));
+		return parsed.length ? parsed : null;
+	}
+
+	return null;
+}
+
+function formatLineNumbers(lines: number[] | null): string | null {
+	if (!lines || lines.length === 0) return null;
+	if (lines.length <= 10) return lines.join(", ");
+	const min = Math.min(...lines);
+	const max = Math.max(...lines);
+	return `${min}-${max} (${lines.length} lines)`;
+}
+
+function formatCodeBlock(
+	code: string | undefined,
+	language: string | undefined,
+	lines: number[] | null,
+): string | null {
+	if (!code) return null;
+
+	const normalized = code.replace(/\r\n/g, "\n").trimEnd();
+	const codeLines = normalized.split("\n");
+	const languageTag = typeof language === "string" ? language.trim().toLowerCase() : "";
+
+	let displayLines = codeLines;
+	if (lines && lines.length === codeLines.length) {
+		displayLines = codeLines.map((line, index) => `${lines[index]}: ${line}`);
+	}
+
+	const fence = languageTag ? languageTag : "";
+	return `\n\n\`\`\`${fence}\n${displayLines.join("\n")}\n\`\`\`\n`;
+}
+
+export const handleSearchcode: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!VALID_HOSTS.has(parsed.hostname)) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const viewMatch = parsed.pathname.match(/^\/codesearch\/view\/([^/?#]+)/);
+		if (viewMatch) {
+			const id = viewMatch[1];
+			const apiUrl = `https://searchcode.com/api/result/${encodeURIComponent(id)}/`;
+			const result = await loadPage(apiUrl, { timeout, signal, headers: { Accept: "application/json" } });
+			if (!result.ok) return null;
+
+			const data = tryParseJson<SearchcodeResult>(result.content);
+			if (!data) return null;
+
+			const filename = data.filename || data.location || `Result ${id}`;
+			const lineNumbers = parseLineNumbers(data.lines);
+			const formattedLines = formatLineNumbers(lineNumbers);
+			const viewUrl = data.url || `https://searchcode.com/codesearch/view/${id}`;
+			const snippetBlock = formatCodeBlock(data.code, data.language, lineNumbers);
+
+			let md = `# ${filename}\n\n`;
+			md += `## Description\n\n`;
+			md += "Code snippet from searchcode.com.\n\n";
+			md += `## Metadata\n\n`;
+			if (data.repo) md += `**Repository:** ${data.repo}\n`;
+			if (data.language) md += `**Language:** ${data.language}\n`;
+			if (data.filename) md += `**File:** ${data.filename}\n`;
+			if (data.location) md += `**Location:** ${data.location}\n`;
+			if (formattedLines) md += `**Lines:** ${formattedLines}\n`;
+			md += `**Result ID:** ${id}\n`;
+			md += `**URL:** ${viewUrl}\n`;
+
+			md += `\n## Snippet`;
+			if (snippetBlock) {
+				md += snippetBlock;
+			} else {
+				md += "\n\n_No snippet available._\n";
+			}
+
+			return buildResult(md, { url, method: "searchcode", fetchedAt, notes: ["Fetched via searchcode API"] });
+		}
+
+		const query = parsed.searchParams.get("q");
+		const isSearchPage =
+			parsed.pathname === "/" || parsed.pathname === "/codesearch" || parsed.pathname === "/codesearch/";
+		if (!query || !isSearchPage) return null;
+
+		const pageRaw = parsed.searchParams.get("p") ?? parsed.searchParams.get("page");
+		const pageNumber = pageRaw ? Number.parseInt(pageRaw, 10) : 0;
+		const page = Number.isFinite(pageNumber) && pageNumber >= 0 ? pageNumber : 0;
+		const apiUrl = `https://searchcode.com/api/codesearch_I/?q=${encodeURIComponent(query)}&p=${page}`;
+		const result = await loadPage(apiUrl, { timeout, signal, headers: { Accept: "application/json" } });
+		if (!result.ok) return null;
+
+		const data = tryParseJson<SearchcodeSearchResponse>(result.content);
+		if (!data) return null;
+
+		const results = Array.isArray(data.results) ? data.results : [];
+		const total =
+			typeof data.total === "number"
+				? data.total
+				: typeof data.total_results === "number"
+					? data.total_results
+					: null;
+
+		let md = `# Searchcode Results\n\n`;
+		md += `## Description\n\n`;
+		md += `Search results for \`${query}\` on searchcode.com.\n\n`;
+		md += `## Metadata\n\n`;
+		md += `**Query:** \`${query}\`\n`;
+		md += `**Page:** ${page}\n`;
+		if (total !== null) md += `**Total Results:** ${formatNumber(total)}\n`;
+		md += `**Result Count:** ${results.length}\n`;
+		if (typeof data.nextpage === "number") md += `**Next Page:** ${data.nextpage}\n`;
+
+		md += `\n## Results\n\n`;
+
+		if (results.length === 0) {
+			md += "_No results found._\n";
+		} else {
+			const maxResults = 10;
+			for (const resultItem of results.slice(0, maxResults)) {
+				const id = resultItem.id !== undefined ? String(resultItem.id) : null;
+				const filename = resultItem.filename || resultItem.location || "Result";
+				const lineNumbers = parseLineNumbers(resultItem.lines);
+				const formattedLines = formatLineNumbers(lineNumbers);
+				const viewUrl = resultItem.url || (id ? `https://searchcode.com/codesearch/view/${id}` : null);
+				const snippetBlock = formatCodeBlock(resultItem.code, resultItem.language, lineNumbers);
+
+				md += `### ${filename}\n\n`;
+				if (resultItem.repo) md += `**Repository:** ${resultItem.repo}\n`;
+				if (resultItem.language) md += `**Language:** ${resultItem.language}\n`;
+				if (resultItem.filename) md += `**File:** ${resultItem.filename}\n`;
+				if (resultItem.location) md += `**Location:** ${resultItem.location}\n`;
+				if (formattedLines) md += `**Lines:** ${formattedLines}\n`;
+				if (viewUrl) md += `**URL:** ${viewUrl}\n`;
+
+				if (snippetBlock) {
+					md += `${snippetBlock}\n`;
+				}
+
+				md += "\n";
+			}
+
+			if (results.length > maxResults) {
+				md += `\n_Only showing first ${maxResults} results._\n`;
+			}
+		}
+
+		return buildResult(md, { url, method: "searchcode", fetchedAt, notes: ["Fetched via searchcode API"] });
+	} catch {}
+
+	return null;
+};