@f5-sales-demo/xcsh 19.51.2

package/src/web/scrapers/fdroid.ts

@@ -0,0 +1,134 @@
+import { tryParseJson } from "@f5xc-salesdemos/pi-utils";
+import type { LocalizedText, RenderResult, SpecialHandler } from "./types";
+import { buildResult, getLocalizedText, loadPage } from "./types";
+
+type FdroidPackage = {
+	packageName?: string;
+	name?: LocalizedText;
+	summary?: LocalizedText;
+	description?: LocalizedText;
+	author?: string | { name?: string; email?: string };
+	authorName?: string;
+	authorEmail?: string;
+	license?: string;
+	categories?: string[];
+	antiFeatures?: string[];
+	sourceCode?: string;
+	packages?: Array<{
+		versionName?: string;
+		versionCode?: number;
+		added?: number;
+		antiFeatures?: string[];
+	}>;
+	suggestedVersionCode?: number;
+	suggestedVersionName?: string;
+};
+
+function normalizeAuthor(data: FdroidPackage): string | undefined {
+	if (data.authorName) return data.authorName;
+	if (typeof data.author === "string") return data.author;
+	if (data.author && typeof data.author !== "string" && typeof data.author.name === "string") return data.author.name;
+	if (data.authorEmail) return data.authorEmail;
+	return undefined;
+}
+
+function normalizeAuthorEmail(data: FdroidPackage): string | undefined {
+	if (data.authorEmail) return data.authorEmail;
+	if (data.author && typeof data.author !== "string" && typeof data.author.email === "string")
+		return data.author.email;
+	return undefined;
+}
+
+function collectAntiFeatures(data: FdroidPackage): string[] {
+	const values = new Set<string>();
+	for (const feature of data.antiFeatures ?? []) values.add(feature);
+	for (const pkg of data.packages ?? []) {
+		for (const feature of pkg.antiFeatures ?? []) values.add(feature);
+	}
+	return Array.from(values);
+}
+
+function resolveSuggestedVersion(data: FdroidPackage): string | undefined {
+	if (data.suggestedVersionName) return data.suggestedVersionName;
+	if (data.suggestedVersionCode) {
+		const match = data.packages?.find(pkg => pkg.versionCode === data.suggestedVersionCode);
+		if (match?.versionName) return match.versionName;
+	}
+	return data.packages?.[0]?.versionName;
+}
+
+/**
+ * Handle F-Droid URLs via API
+ */
+export const handleFdroid: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "f-droid.org" && parsed.hostname !== "www.f-droid.org") return null;
+
+		// Extract package name from /packages/{packageName} or /en/packages/{packageName}
+		const match = parsed.pathname.match(/^\/(?:en\/)?packages\/([^/]+)/);
+		if (!match) return null;
+
+		const packageName = decodeURIComponent(match[1]);
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://f-droid.org/api/v1/packages/${encodeURIComponent(packageName)}`;
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		const data = tryParseJson<FdroidPackage>(result.content);
+		if (!data) return null;
+
+		const displayName = getLocalizedText(data.name) ?? packageName;
+		const summary = getLocalizedText(data.summary);
+		const description = getLocalizedText(data.description);
+		const author = normalizeAuthor(data);
+		const authorEmail = normalizeAuthorEmail(data);
+		const antiFeatures = collectAntiFeatures(data);
+		const latestVersion = resolveSuggestedVersion(data);
+
+		let md = `# ${displayName}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		md += `**Package:** ${packageName}`;
+		if (latestVersion) md += ` · **Latest:** ${latestVersion}`;
+		if (data.license) md += ` · **License:** ${data.license}`;
+		md += "\n";
+
+		if (author) {
+			md += `**Author:** ${author}`;
+			if (authorEmail && authorEmail !== author) md += ` <${authorEmail}>`;
+			md += "\n";
+		}
+
+		if (data.sourceCode) md += `**Source Code:** ${data.sourceCode}\n`;
+		if (data.categories?.length) md += `**Categories:** ${data.categories.join(", ")}\n`;
+		if (antiFeatures.length) md += `**Anti-Features:** ${antiFeatures.join(", ")}\n`;
+
+		if (description) {
+			md += `\n## Description\n\n${description}\n`;
+		}
+
+		if (data.packages?.length) {
+			md += "\n## Version History\n\n";
+			for (const version of data.packages.slice(0, 10)) {
+				const label = version.versionName ?? "unknown";
+				const code = version.versionCode ? ` (${version.versionCode})` : "";
+				md += `- ${label}${code}\n`;
+			}
+		}
+
+		return buildResult(md, { url, method: "fdroid", fetchedAt, notes: ["Fetched via F-Droid API"] });
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/firefox-addons.ts

@@ -0,0 +1,191 @@
+import { tryParseJson } from "@f5xc-salesdemos/pi-utils";
+import type { LocalizedText, RenderResult, SpecialHandler } from "./types";
+import { buildResult, formatNumber, getLocalizedText, htmlToBasicMarkdown, loadPage } from "./types";
+
+type AddonFile = {
+	permissions?: string[];
+	host_permissions?: string[];
+	optional_permissions?: string[];
+	optional_host_permissions?: string[];
+};
+
+type AddonLicense = {
+	name?: LocalizedText;
+	slug?: string;
+	url?: string;
+};
+
+type AddonVersion = {
+	version?: string;
+	license?: AddonLicense;
+	file?: AddonFile;
+};
+
+type AddonHomepage = {
+	url?: LocalizedText;
+	outgoing?: LocalizedText;
+};
+
+type AddonData = {
+	name?: LocalizedText;
+	summary?: LocalizedText;
+	description?: LocalizedText;
+	default_locale?: string;
+	authors?: Array<{ name?: string | null }>;
+	average_daily_users?: number;
+	weekly_downloads?: number;
+	ratings?: { average?: number; count?: number };
+	current_version?: AddonVersion;
+	categories?: string[] | Record<string, string[]>;
+	homepage?: AddonHomepage;
+	url?: string;
+};
+
+function normalizeCategories(categories?: string[] | Record<string, string[]>): string[] {
+	if (!categories) return [];
+	if (Array.isArray(categories)) return categories.filter(Boolean);
+
+	const values: string[] = [];
+	for (const list of Object.values(categories)) {
+		if (Array.isArray(list)) {
+			for (const item of list) {
+				if (item) values.push(item);
+			}
+		}
+	}
+
+	const seen = new Set<string>();
+	return values.filter(item => {
+		if (seen.has(item)) return false;
+		seen.add(item);
+		return true;
+	});
+}
+
+function collectPermissions(file?: AddonFile): string[] {
+	if (!file) return [];
+	const permissions: string[] = [];
+	const seen = new Set<string>();
+
+	const add = (items?: string[]) => {
+		for (const item of items ?? []) {
+			if (!item || seen.has(item)) continue;
+			seen.add(item);
+			permissions.push(item);
+		}
+	};
+
+	add(file.permissions);
+	add(file.host_permissions);
+	add(file.optional_permissions);
+	add(file.optional_host_permissions);
+
+	return permissions;
+}
+
+export const handleFirefoxAddons: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "addons.mozilla.org") return null;
+
+		const segments = parsed.pathname.split("/").filter(Boolean);
+		const addonIndex = segments.indexOf("addon");
+		if (addonIndex === -1) return null;
+
+		const slug = segments[addonIndex + 1] ? decodeURIComponent(segments[addonIndex + 1]) : "";
+		if (!slug) return null;
+
+		const apiUrl = `https://addons.mozilla.org/api/v5/addons/addon/${encodeURIComponent(slug)}/`;
+		const result = await loadPage(apiUrl, { timeout, headers: { Accept: "application/json" }, signal });
+		if (!result.ok) return null;
+
+		const data = tryParseJson<AddonData>(result.content);
+		if (!data) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const defaultLocale = data.default_locale || "en-US";
+
+		const name = getLocalizedText(data.name, defaultLocale) ?? slug;
+		const summary = getLocalizedText(data.summary, defaultLocale);
+		const descriptionRaw = getLocalizedText(data.description, defaultLocale);
+		const description = descriptionRaw ? htmlToBasicMarkdown(descriptionRaw) : undefined;
+
+		const authors = (data.authors ?? [])
+			.map(author => author.name ?? "")
+			.map(author => author.trim())
+			.filter(Boolean);
+
+		const ratingAverage = data.ratings?.average;
+		const ratingCount = data.ratings?.count;
+		const users = data.average_daily_users ?? data.weekly_downloads;
+		const version = data.current_version?.version;
+		const categories = normalizeCategories(data.categories);
+
+		const licenseName =
+			getLocalizedText(data.current_version?.license?.name, defaultLocale) ?? data.current_version?.license?.slug;
+		const licenseUrl = data.current_version?.license?.url;
+
+		const homepage =
+			getLocalizedText(data.homepage?.url, defaultLocale) ??
+			getLocalizedText(data.homepage?.outgoing, defaultLocale);
+
+		const permissions = collectPermissions(data.current_version?.file);
+
+		let md = `# ${name}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		if (authors.length > 0) {
+			md += `**Author${authors.length > 1 ? "s" : ""}:** ${authors.join(", ")}\n`;
+		}
+
+		if (ratingAverage !== undefined) {
+			md += `**Rating:** ${ratingAverage.toFixed(2)}`;
+			if (ratingCount !== undefined) md += ` (${formatNumber(ratingCount)} reviews)`;
+			md += "\n";
+		}
+
+		if (users !== undefined) md += `**Users:** ${formatNumber(users)}\n`;
+		if (version) md += `**Version:** ${version}\n`;
+		if (categories.length > 0) md += `**Categories:** ${categories.join(", ")}\n`;
+
+		if (licenseName && licenseUrl) {
+			md += `**License:** [${licenseName}](${licenseUrl})\n`;
+		} else if (licenseName) {
+			md += `**License:** ${licenseName}\n`;
+		} else if (licenseUrl) {
+			md += `**License:** ${licenseUrl}\n`;
+		}
+
+		if (homepage) md += `**Homepage:** ${homepage}\n`;
+
+		if (description) {
+			md += `\n## Description\n\n${description}\n`;
+		}
+
+		if (permissions.length > 0) {
+			const preview = permissions.slice(0, 40);
+			md += `\n## Permissions (${permissions.length})\n\n`;
+			for (const permission of preview) {
+				md += `- ${permission}\n`;
+			}
+			if (permissions.length > preview.length) {
+				md += `\n*...and ${permissions.length - preview.length} more*\n`;
+			}
+		}
+
+		const finalUrl = data.url ?? result.finalUrl ?? url;
+		return buildResult(md, {
+			url,
+			finalUrl,
+			method: "firefox-addons",
+			fetchedAt,
+			notes: ["Fetched via Firefox Add-ons API"],
+		});
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/flathub.ts

@@ -0,0 +1,223 @@
+import { tryParseJson } from "@f5xc-salesdemos/pi-utils";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, formatIsoDate, formatNumber, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface FlathubScreenshotSize {
+	src?: string;
+	width?: string;
+	height?: string;
+	scale?: string;
+}
+
+interface FlathubScreenshot {
+	caption?: string | null;
+	sizes?: FlathubScreenshotSize[];
+}
+
+interface FlathubRelease {
+	version?: string;
+	timestamp?: string;
+	description?: string | null;
+	url?: string | null;
+	type?: string | null;
+}
+
+interface FlathubAppStream {
+	id?: string;
+	name?: string;
+	summary?: string;
+	description?: string;
+	developer_name?: string;
+	categories?: string[];
+	screenshots?: FlathubScreenshot[];
+	releases?: FlathubRelease[];
+	metadata?: Record<string, unknown>;
+	installs?: number | string;
+	permissions?: unknown;
+}
+
+function extractAppId(pathname: string): string | null {
+	const detailsMatch = pathname.match(/^\/apps\/details\/([^/]+)\/?$/);
+	if (detailsMatch) return decodeURIComponent(detailsMatch[1]);
+
+	const appMatch = pathname.match(/^\/apps\/([^/]+)\/?$/);
+	if (appMatch) return decodeURIComponent(appMatch[1]);
+
+	return null;
+}
+
+function parseNumber(value: unknown): number | null {
+	if (typeof value === "number" && Number.isFinite(value)) return value;
+	if (typeof value === "string") {
+		const cleaned = value.replace(/[^0-9.]/g, "");
+		if (!cleaned) return null;
+		const parsed = Number(cleaned);
+		if (!Number.isNaN(parsed)) return parsed;
+	}
+	return null;
+}
+
+function normalizeStringList(value: unknown): string[] {
+	if (Array.isArray(value)) {
+		return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0);
+	}
+	if (typeof value === "string") {
+		return value
+			.split(/[,;\n]+/)
+			.map(item => item.trim())
+			.filter(Boolean);
+	}
+	return [];
+}
+
+function extractInstalls(app: FlathubAppStream): number | null {
+	const direct = parseNumber(app.installs);
+	if (direct !== null) return direct;
+
+	if (!app.metadata) return null;
+	for (const [key, value] of Object.entries(app.metadata)) {
+		if (!key.toLowerCase().includes("install")) continue;
+		const parsed = parseNumber(value);
+		if (parsed !== null) return parsed;
+	}
+
+	return null;
+}
+
+function extractPermissions(app: FlathubAppStream): string[] {
+	const permissions: string[] = [];
+	permissions.push(...normalizeStringList(app.permissions));
+
+	if (app.metadata) {
+		for (const [key, value] of Object.entries(app.metadata)) {
+			if (!key.toLowerCase().includes("permission")) continue;
+			const list = normalizeStringList(value);
+			if (list.length) {
+				permissions.push(...list);
+				continue;
+			}
+			if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+				permissions.push(`${key}: ${String(value)}`);
+			}
+		}
+	}
+
+	return Array.from(new Set(permissions));
+}
+
+function screenshotArea(size?: FlathubScreenshotSize): number {
+	if (!size) return 0;
+	const width = Number(size.width);
+	const height = Number(size.height);
+	if (!Number.isFinite(width) || !Number.isFinite(height)) return 0;
+	return width * height;
+}
+
+function bestScreenshotUrl(sizes?: FlathubScreenshotSize[]): string | null {
+	if (!sizes || sizes.length === 0) return null;
+
+	let best = sizes[0];
+	let bestArea = screenshotArea(best);
+
+	for (const size of sizes) {
+		const area = screenshotArea(size);
+		if (area > bestArea) {
+			best = size;
+			bestArea = area;
+		}
+	}
+
+	return best.src ?? sizes[0].src ?? null;
+}
+
+export const handleFlathub: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "flathub.org" && parsed.hostname !== "www.flathub.org") return null;
+
+		const appId = extractAppId(parsed.pathname);
+		if (!appId) return null;
+
+		const apiUrl = `https://flathub.org/api/v2/appstream/${encodeURIComponent(appId)}`;
+		const result = await loadPage(apiUrl, { timeout, signal, headers: { Accept: "application/json" } });
+		if (!result.ok) return null;
+
+		const app = tryParseJson<FlathubAppStream>(result.content);
+		if (!app) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const name = app.name ?? app.id ?? appId;
+
+		let md = `# ${name}\n\n`;
+		if (app.summary) md += `${app.summary}\n\n`;
+
+		md += "## Metadata\n\n";
+		md += `**App ID:** ${app.id ?? appId}\n`;
+		if (app.developer_name) md += `**Developer:** ${app.developer_name}\n`;
+
+		const installs = extractInstalls(app);
+		if (installs !== null) md += `**Installs:** ${formatNumber(installs)}\n`;
+
+		if (app.categories?.length) {
+			md += "\n## Categories\n\n";
+			for (const category of app.categories) {
+				md += `- ${category}\n`;
+			}
+		}
+
+		if (app.description) {
+			const description = htmlToBasicMarkdown(app.description);
+			if (description) md += `\n## Description\n\n${description}\n`;
+		}
+
+		const permissions = extractPermissions(app);
+		if (permissions.length) {
+			md += "\n## Permissions\n\n";
+			for (const permission of permissions) {
+				md += `- ${permission}\n`;
+			}
+		}
+
+		if (app.screenshots?.length) {
+			md += "\n## Screenshots\n\n";
+			for (const screenshot of app.screenshots.slice(0, 5)) {
+				const screenshotUrl = bestScreenshotUrl(screenshot.sizes);
+				if (!screenshotUrl) continue;
+				const caption = screenshot.caption ? ` - ${screenshot.caption}` : "";
+				md += `- ${screenshotUrl}${caption}\n`;
+			}
+		}
+
+		if (app.releases?.length) {
+			md += "\n## Releases\n\n";
+			for (const release of app.releases.slice(0, 5)) {
+				const version = release.version ?? "unknown";
+				let line = `- **${version}**`;
+				const date = release.timestamp ? formatIsoDate(Number(release.timestamp) * 1000) : "";
+				if (date) line += ` (${date})`;
+				if (release.type) line += ` · ${release.type}`;
+				if (release.url) line += ` · ${release.url}`;
+				md += `${line}\n`;
+
+				if (release.description) {
+					const releaseDesc = htmlToBasicMarkdown(release.description).replace(/\n+/g, " ").trim();
+					if (releaseDesc) md += `  - ${releaseDesc}\n`;
+				}
+			}
+		}
+
+		return buildResult(md, {
+			url,
+			finalUrl: result.finalUrl,
+			method: "flathub-appstream",
+			fetchedAt,
+			notes: ["Fetched via Flathub Appstream API"],
+		});
+	} catch {}
+
+	return null;
+};

package/src/web/scrapers/github-gist.ts

@@ -0,0 +1,58 @@
+import { fetchGitHubApi } from "./github";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult } from "./types";
+
+/**
+ * Handle GitHub Gist URLs via GitHub API
+ */
+export const handleGitHubGist: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "gist.github.com") return null;
+
+		// Extract gist ID from /username/gistId or just /gistId
+		const parts = parsed.pathname.split("/").filter(Boolean);
+		if (parts.length === 0) return null;
+
+		// Gist ID is always the last path segment (or only segment for anonymous gists)
+		const gistId = parts[parts.length - 1];
+		if (!gistId || !/^[a-f0-9]+$/i.test(gistId)) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch via GitHub API
+		const result = await fetchGitHubApi(`/gists/${gistId}`, timeout, signal);
+		if (!result.ok || !result.data) return null;
+
+		const gist = result.data as {
+			description: string | null;
+			owner?: { login: string };
+			created_at: string;
+			updated_at: string;
+			files: Record<string, { filename: string; language: string | null; size: number; content: string }>;
+			html_url: string;
+		};
+
+		const files = Object.values(gist.files);
+		const owner = gist.owner?.login || "anonymous";
+
+		let md = `# Gist by ${owner}\n\n`;
+		if (gist.description) md += `${gist.description}\n\n`;
+		md += `**Created:** ${gist.created_at} · **Updated:** ${gist.updated_at}\n`;
+		md += `**Files:** ${files.length}\n\n`;
+
+		for (const file of files) {
+			const lang = file.language?.toLowerCase() || "";
+			md += `---\n\n## ${file.filename}\n\n`;
+			md += `\`\`\`${lang}\n${file.content}\n\`\`\`\n\n`;
+		}
+
+		return buildResult(md, { url, method: "github-gist", fetchedAt, notes: ["Fetched via GitHub API"] });
+	} catch {}
+
+	return null;
+};