@f5-sales-demo/xcsh 19.51.2

package/src/web/scrapers/types.ts

@@ -0,0 +1,267 @@
+/**
+ * Shared types and utilities for web-fetch handlers
+ */
+import { ptree } from "@f5xc-salesdemos/pi-utils";
+import { ToolAbortError } from "../../tools/tool-errors";
+
+export { formatNumber } from "@f5xc-salesdemos/pi-utils";
+
+export interface RenderResult {
+	url: string;
+	finalUrl: string;
+	contentType: string;
+	method: string;
+	content: string;
+	fetchedAt: string;
+	truncated: boolean;
+	notes: string[];
+}
+
+export type SpecialHandler = (url: string, timeout: number, signal?: AbortSignal) => Promise<RenderResult | null>;
+
+export const MAX_OUTPUT_CHARS = 500_000;
+export const MAX_BYTES = 50 * 1024 * 1024;
+
+const USER_AGENTS = [
+	"curl/8.0",
+	"Mozilla/5.0 (compatible; TextBot/1.0)",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+];
+
+function isBotBlocked(status: number, content: string): boolean {
+	if (status === 403 || status === 503) {
+		const lower = content.toLowerCase();
+		return (
+			lower.includes("cloudflare") ||
+			lower.includes("captcha") ||
+			lower.includes("challenge") ||
+			lower.includes("blocked") ||
+			lower.includes("access denied") ||
+			lower.includes("bot detection")
+		);
+	}
+	return false;
+}
+
+/**
+ * Truncate and cleanup output
+ */
+export function finalizeOutput(content: string): { content: string; truncated: boolean } {
+	const cleaned = content.replace(/\n{3,}/g, "\n\n").trim();
+	const truncated = cleaned.length > MAX_OUTPUT_CHARS;
+	return {
+		content: cleaned.slice(0, MAX_OUTPUT_CHARS),
+		truncated,
+	};
+}
+
+export interface LoadPageOptions {
+	timeout?: number;
+	headers?: Record<string, string>;
+	method?: string;
+	body?: string;
+	maxBytes?: number;
+	signal?: AbortSignal;
+}
+
+export interface LoadPageResult {
+	content: string;
+	contentType: string;
+	finalUrl: string;
+	ok: boolean;
+	status?: number;
+}
+
+/**
+ * Fetch a page with timeout and size limit
+ */
+export async function loadPage(url: string, options: LoadPageOptions = {}): Promise<LoadPageResult> {
+	const { timeout = 20, headers = {}, maxBytes = MAX_BYTES, signal, method = "GET", body } = options;
+
+	for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) {
+		if (signal?.aborted) {
+			throw new ToolAbortError();
+		}
+
+		const userAgent = USER_AGENTS[attempt];
+		const requestSignal = ptree.combineSignals(signal, timeout * 1000);
+
+		try {
+			const requestInit: RequestInit = {
+				signal: requestSignal,
+				method,
+				headers: {
+					"User-Agent": userAgent,
+					Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+					"Accept-Language": "en-US,en;q=0.5",
+					"Accept-Encoding": "identity", // Cloudflare Markdown-for-Agents returns corrupted bytes when compression is negotiated
+					...headers,
+				},
+				redirect: "follow",
+			};
+
+			if (body !== undefined) {
+				requestInit.body = body;
+			}
+
+			const response = await fetch(url, requestInit);
+
+			const contentType = response.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? "";
+			const finalUrl = response.url;
+
+			const reader = response.body?.getReader();
+			if (!reader) {
+				return { content: "", contentType, finalUrl, ok: false, status: response.status };
+			}
+
+			const chunks: Uint8Array[] = [];
+			let totalSize = 0;
+
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) break;
+
+				chunks.push(value);
+				totalSize += value.length;
+
+				if (totalSize > maxBytes) {
+					reader.cancel();
+					break;
+				}
+			}
+
+			const content = Buffer.concat(chunks).toString("utf-8");
+			if (isBotBlocked(response.status, content) && attempt < USER_AGENTS.length - 1) {
+				continue;
+			}
+
+			if (!response.ok) {
+				return { content, contentType, finalUrl, ok: false, status: response.status };
+			}
+
+			return { content, contentType, finalUrl, ok: true, status: response.status };
+		} catch {
+			if (signal?.aborted) {
+				throw new ToolAbortError();
+			}
+			if (attempt === USER_AGENTS.length - 1) {
+				return { content: "", contentType: "", finalUrl: url, ok: false };
+			}
+		}
+	}
+
+	return { content: "", contentType: "", finalUrl: url, ok: false };
+}
+
+/**
+ * Convert basic HTML to markdown
+ */
+export function htmlToBasicMarkdown(html: string): string {
+	const stripped = html
+		.replace(/<pre[^>]*><code[^>]*>/g, "\n```\n")
+		.replace(/<\/code><\/pre>/g, "\n```\n")
+		.replace(/<code[^>]*>/g, "`")
+		.replace(/<\/code>/g, "`")
+		.replace(/<strong[^>]*>/g, "**")
+		.replace(/<\/strong>/g, "**")
+		.replace(/<b[^>]*>/g, "**")
+		.replace(/<\/b>/g, "**")
+		.replace(/<em[^>]*>/g, "*")
+		.replace(/<\/em>/g, "*")
+		.replace(/<i[^>]*>/g, "*")
+		.replace(/<\/i>/g, "*")
+		.replace(
+			/<a[^>]*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/g,
+			(_, href, text) => `[${text.replace(/<[^>]+>/g, "").trim()}](${href})`,
+		)
+		.replace(/<p[^>]*>/g, "\n\n")
+		.replace(/<\/p>/g, "")
+		.replace(/<br\s*\/?>/g, "\n")
+		.replace(/<li[^>]*>/g, "- ")
+		.replace(/<\/li>/g, "\n")
+		.replace(/<\/?[uo]l[^>]*>/g, "\n")
+		.replace(/<h(\d)[^>]*>/g, (_, n) => `\n${"#".repeat(parseInt(n, 10))} `)
+		.replace(/<\/h\d>/g, "\n")
+		.replace(/<blockquote[^>]*>/g, "\n> ")
+		.replace(/<\/blockquote>/g, "\n")
+		.replace(/<[^>]+>/g, "")
+		.replace(/\n{3,}/g, "\n\n")
+		.trim();
+	return decodeHtmlEntities(stripped);
+}
+
+/**
+ * Build a RenderResult from markdown content. Calls finalizeOutput internally.
+ */
+export function buildResult(
+	md: string,
+	opts: { url: string; finalUrl?: string; method: string; fetchedAt: string; notes?: string[]; contentType?: string },
+): RenderResult {
+	const output = finalizeOutput(md);
+	return {
+		url: opts.url,
+		finalUrl: opts.finalUrl ?? opts.url,
+		contentType: opts.contentType ?? "text/markdown",
+		method: opts.method,
+		content: output.content,
+		fetchedAt: opts.fetchedAt,
+		truncated: output.truncated,
+		notes: opts.notes ?? [],
+	};
+}
+
+/**
+ * Format a date value as YYYY-MM-DD. Returns empty string on invalid input.
+ */
+export function formatIsoDate(value?: string | number | Date): string {
+	if (value == null) return "";
+	if (typeof value === "string") {
+		const datePrefix = value.match(/^\d{4}-\d{2}-\d{2}/);
+		if (datePrefix) return datePrefix[0];
+	}
+	try {
+		return new Date(value).toISOString().split("T")[0];
+	} catch {
+		return "";
+	}
+}
+
+/**
+ * Decode common HTML entities.
+ */
+export function decodeHtmlEntities(text: string): string {
+	return text
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&amp;/g, "&")
+		.replace(/&quot;/g, '"')
+		.replace(/&#0?39;/g, "'")
+		.replace(/&#x27;/g, "'")
+		.replace(/&#x2F;/g, "/")
+		.replace(/&nbsp;/g, " ");
+}
+
+/**
+ * Format seconds into HH:MM:SS or MM:SS.
+ */
+export function formatMediaDuration(totalSeconds: number): string {
+	const hours = Math.floor(totalSeconds / 3600);
+	const minutes = Math.floor((totalSeconds % 3600) / 60);
+	const secs = Math.floor(totalSeconds % 60);
+	if (hours > 0) return `${hours}:${String(minutes).padStart(2, "0")}:${String(secs).padStart(2, "0")}`;
+	return `${minutes}:${String(secs).padStart(2, "0")}`;
+}
+
+/**
+ * Extract localized text, preferring en-US/en.
+ */
+export type LocalizedText = string | Record<string, string | null> | null | undefined;
+
+export function getLocalizedText(value: LocalizedText, defaultLocale?: string): string | undefined {
+	if (value == null) return undefined;
+	if (typeof value === "string") return value;
+	if (defaultLocale && value[defaultLocale]) return value[defaultLocale];
+	return (
+		value["en-US"] ?? value.en_US ?? value.en ?? Object.values(value).find(v => typeof v === "string") ?? undefined
+	);
+}

package/src/web/scrapers/utils.ts

@@ -0,0 +1,109 @@
+import { isRecord, ptree } from "@f5xc-salesdemos/pi-utils";
+
+export { isRecord };
+
+import { ToolAbortError } from "../../tools/tool-errors";
+import { convertBufferWithMarkit } from "../../utils/markit";
+import { MAX_BYTES } from "./types";
+
+export function asRecord(value: unknown): Record<string, unknown> | null {
+	return isRecord(value) ? value : null;
+}
+
+export function asString(value: unknown): string | null {
+	if (typeof value !== "string") return null;
+	const trimmed = value.trim();
+	return trimmed.length > 0 ? trimmed : null;
+}
+
+export function asNumber(value: unknown): number | null {
+	return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+export interface BinaryFetchSuccess {
+	ok: true;
+	buffer: Uint8Array;
+	contentDisposition?: string;
+}
+
+export type BinaryFetchResult = BinaryFetchSuccess | { ok: false; error?: string };
+
+async function readResponseWithLimit(response: Response, maxBytes: number, signal?: AbortSignal): Promise<Uint8Array> {
+	const reader = response.body?.getReader();
+	if (!reader) return new Uint8Array(0);
+
+	const chunks: Buffer[] = [];
+	let totalBytes = 0;
+
+	try {
+		while (true) {
+			if (signal?.aborted) {
+				await reader.cancel();
+				throw new ToolAbortError();
+			}
+			const { done, value } = await reader.read();
+			if (done) break;
+			if (!value || value.byteLength === 0) continue;
+
+			totalBytes += value.byteLength;
+			if (totalBytes > maxBytes) {
+				await reader.cancel();
+				throw new Error(`response exceeds ${maxBytes} bytes`);
+			}
+
+			chunks.push(Buffer.from(value));
+		}
+	} finally {
+		reader.releaseLock();
+	}
+
+	return new Uint8Array(Buffer.concat(chunks, totalBytes));
+}
+
+/**
+ * Fetch binary content from a URL
+ */
+export async function fetchBinary(url: string, timeout: number = 20, signal?: AbortSignal): Promise<BinaryFetchResult> {
+	const requestSignal = ptree.combineSignals(signal, timeout * 1000);
+	try {
+		const response = await fetch(url, {
+			signal: requestSignal,
+			headers: {
+				"User-Agent": "Mozilla/5.0 (compatible; TextBot/1.0)",
+			},
+			redirect: "follow",
+		});
+
+		if (!response.ok) {
+			return { ok: false, error: `HTTP ${response.status}` };
+		}
+
+		const contentDisposition = response.headers.get("content-disposition") || undefined;
+		const contentLength = response.headers.get("content-length");
+		if (contentLength) {
+			const size = Number.parseInt(contentLength, 10);
+			if (Number.isFinite(size) && size > MAX_BYTES) {
+				return { ok: false, error: `content-length ${size} exceeds ${MAX_BYTES}` };
+			}
+		}
+		const buffer = await readResponseWithLimit(response, MAX_BYTES, requestSignal);
+		return { ok: true, buffer, contentDisposition };
+	} catch (err) {
+		if (signal?.aborted) throw new ToolAbortError();
+		if (requestSignal?.aborted) return { ok: false, error: "aborted" };
+		return { ok: false, error: err instanceof Error ? err.message : "Failed to fetch binary" };
+	}
+}
+
+/**
+ * Convert binary content to markdown using markit.
+ */
+export async function convertWithMarkit(
+	buffer: Uint8Array,
+	extension: string,
+	timeout: number = 20,
+	signal?: AbortSignal,
+): Promise<{ content: string; ok: boolean; error?: string }> {
+	const conversionSignal = ptree.combineSignals(signal, timeout * 1000);
+	return convertBufferWithMarkit(buffer, extension, conversionSignal);
+}

package/src/web/scrapers/vimeo.ts

@@ -0,0 +1,133 @@
+import { tryParseJson } from "@f5xc-salesdemos/pi-utils";
+import type { SpecialHandler } from "./types";
+import { buildResult, formatMediaDuration, loadPage } from "./types";
+
+interface VimeoOEmbed {
+	title: string;
+	author_name: string;
+	author_url: string;
+	description?: string;
+	duration: number;
+	thumbnail_url: string;
+	upload_date: string;
+	video_id: number;
+}
+
+interface VimeoVideoConfig {
+	video?: {
+		title?: string;
+		duration?: number;
+		owner?: {
+			name?: string;
+			url?: string;
+		};
+		thumbs?: {
+			base?: string;
+		};
+	};
+	request?: {
+		files?: {
+			progressive?: Array<{
+				quality: string;
+				width: number;
+				height: number;
+				fps: number;
+			}>;
+		};
+	};
+}
+
+/**
+ * Extract video ID from various Vimeo URL formats
+ */
+function extractVideoId(url: string): string | null {
+	try {
+		const parsed = new URL(url);
+
+		// player.vimeo.com/video/{id}
+		if (parsed.hostname === "player.vimeo.com") {
+			const match = parsed.pathname.match(/^\/video\/(\d+)/);
+			return match?.[1] ?? null;
+		}
+
+		// vimeo.com/{id} or vimeo.com/{user}/{id}
+		if (parsed.hostname === "vimeo.com" || parsed.hostname === "www.vimeo.com") {
+			const parts = parsed.pathname.split("/").filter(Boolean);
+			// Last part should be the video ID
+			const lastPart = parts[parts.length - 1];
+			if (lastPart && /^\d+$/.test(lastPart)) {
+				return lastPart;
+			}
+		}
+
+		return null;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Handle Vimeo URLs via oEmbed API
+ */
+export const handleVimeo: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("vimeo.com")) return null;
+
+		const videoId = extractVideoId(url);
+		if (!videoId) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Use canonical URL for oEmbed (handles staffpicks and other URL formats)
+		const canonicalUrl = `https://vimeo.com/${videoId}`;
+		const oembedUrl = `https://vimeo.com/api/oembed.json?url=${encodeURIComponent(canonicalUrl)}`;
+		const oembedResult = await loadPage(oembedUrl, { timeout, signal });
+
+		if (!oembedResult.ok) return null;
+
+		const oembed = tryParseJson<VimeoOEmbed>(oembedResult.content);
+		if (!oembed) return null;
+
+		let md = `# ${oembed.title}\n\n`;
+		md += `**Author:** [${oembed.author_name}](${oembed.author_url})\n`;
+		md += `**Duration:** ${formatMediaDuration(oembed.duration)}\n`;
+
+		if (oembed.upload_date) {
+			md += `**Uploaded:** ${oembed.upload_date}\n`;
+		}
+
+		md += `**Video ID:** ${videoId}\n\n`;
+
+		if (oembed.description) {
+			md += `---\n\n## Description\n\n${oembed.description}\n\n`;
+		}
+
+		md += `---\n\n**Thumbnail:** ${oembed.thumbnail_url}\n`;
+
+		// Try to get additional details from video config
+		try {
+			const configUrl = `https://player.vimeo.com/video/${videoId}/config`;
+			const configResult = await loadPage(configUrl, { timeout: Math.min(timeout, 5), signal });
+
+			if (configResult.ok) {
+				const config = tryParseJson<VimeoVideoConfig>(configResult.content);
+
+				// Add video quality info if available
+				const progressive = config?.request?.files?.progressive;
+				if (progressive && progressive.length > 0) {
+					md += `\n**Available Qualities:**\n`;
+					for (const quality of progressive.slice(0, 5)) {
+						md += `- ${quality.quality}: ${quality.width}x${quality.height} @ ${quality.fps}fps\n`;
+					}
+				}
+			}
+		} catch {
+			// Config fetch is optional - continue without it
+		}
+
+		return buildResult(md, { url, method: "vimeo", fetchedAt, notes: ["Fetched via Vimeo oEmbed API"] });
+	} catch {
+		return null;
+	}
+};

package/src/web/scrapers/vscode-marketplace.ts

@@ -0,0 +1,187 @@
+import { tryParseJson } from "@f5xc-salesdemos/pi-utils";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, formatNumber, loadPage } from "./types";
+
+interface MarketplaceProperty {
+	key?: string;
+	value?: string;
+}
+
+interface MarketplaceVersion {
+	version?: string;
+	properties?: MarketplaceProperty[];
+}
+
+interface MarketplaceStatistic {
+	statisticName?: string;
+	value?: number;
+}
+
+interface MarketplacePublisher {
+	publisherName?: string;
+	displayName?: string;
+}
+
+interface MarketplaceExtension {
+	extensionName?: string;
+	displayName?: string;
+	shortDescription?: string;
+	description?: string;
+	publisher?: MarketplacePublisher;
+	versions?: MarketplaceVersion[];
+	statistics?: MarketplaceStatistic[];
+	categories?: string[];
+	tags?: string[];
+	properties?: MarketplaceProperty[];
+}
+
+interface MarketplaceResponse {
+	results?: Array<{ extensions?: MarketplaceExtension[] }>;
+}
+
+const MARKETPLACE_HOSTS = new Set(["marketplace.visualstudio.com", "www.marketplace.visualstudio.com"]);
+
+function getItemName(parsed: URL): string | null {
+	if (!parsed.pathname.startsWith("/items")) return null;
+	const itemName = parsed.searchParams.get("itemName");
+	if (!itemName) return null;
+	const decoded = decodeURIComponent(itemName);
+	if (!decoded.includes(".")) return null;
+	return decoded;
+}
+
+function toStatMap(stats: MarketplaceStatistic[] | undefined): Map<string, number> {
+	const map = new Map<string, number>();
+	if (!stats) return map;
+	for (const stat of stats) {
+		if (!stat.statisticName || typeof stat.value !== "number") continue;
+		map.set(stat.statisticName.trim().toLowerCase(), stat.value);
+	}
+	return map;
+}
+
+function formatRating(averageRating?: number, ratingCount?: number): string | null {
+	if (averageRating === undefined && ratingCount === undefined) return null;
+	if (averageRating !== undefined) {
+		const formatted = averageRating.toFixed(2).replace(/\.0+$/, "").replace(/\.$/, "");
+		if (ratingCount !== undefined) {
+			return `${formatted} (${formatNumber(ratingCount)} ratings)`;
+		}
+		return formatted;
+	}
+	if (ratingCount !== undefined) {
+		return `${formatNumber(ratingCount)} ratings`;
+	}
+	return null;
+}
+
+function extractRepoLink(properties: MarketplaceProperty[] | undefined): string | null {
+	if (!properties) return null;
+	for (const prop of properties) {
+		const key = prop.key?.trim().toLowerCase();
+		const value = prop.value?.trim();
+		if (!key || !value) continue;
+		if (!value.startsWith("http")) continue;
+		if (key.includes("links.source") || key.includes("repository")) return value;
+	}
+	for (const prop of properties) {
+		const key = prop.key?.trim().toLowerCase();
+		const value = prop.value?.trim();
+		if (!key || !value) continue;
+		if (!value.startsWith("http")) continue;
+		if (key === "source" || key.endsWith(".source")) return value;
+	}
+	return null;
+}
+
+/**
+ * Handle VS Code Marketplace URLs via extension query API
+ */
+export const handleVscodeMarketplace: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!MARKETPLACE_HOSTS.has(parsed.hostname)) return null;
+
+		const itemName = getItemName(parsed);
+		if (!itemName) return null;
+
+		const [publisherFromUrl, ...nameParts] = itemName.split(".");
+		const extensionFromUrl = nameParts.join(".");
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = "https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery";
+		const payload = JSON.stringify({
+			filters: [
+				{
+					criteria: [{ filterType: 7, value: itemName }],
+				},
+			],
+			flags: 950,
+		});
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			method: "POST",
+			body: payload,
+			headers: {
+				"Content-Type": "application/json",
+				Accept: "application/json;api-version=7.2-preview.1",
+			},
+		});
+
+		if (!result.ok) return null;
+
+		const data = tryParseJson<MarketplaceResponse>(result.content);
+		if (!data) return null;
+
+		const extension = data.results?.[0]?.extensions?.[0];
+		if (!extension) return null;
+
+		const extensionName = extension.extensionName ?? extensionFromUrl;
+		const displayName = extension.displayName ?? extensionName ?? itemName;
+		const description = extension.shortDescription ?? extension.description;
+
+		const publisherName = extension.publisher?.publisherName ?? publisherFromUrl;
+		const publisherDisplayName = extension.publisher?.displayName;
+		const publisherLabel =
+			publisherDisplayName && publisherName && publisherDisplayName !== publisherName
+				? `${publisherDisplayName} (${publisherName})`
+				: (publisherDisplayName ?? publisherName);
+
+		const version = extension.versions?.[0]?.version;
+		const statMap = toStatMap(extension.statistics);
+		const installs = statMap.get("install") ?? statMap.get("installs");
+		const averageRating = statMap.get("averagerating");
+		const ratingCount = statMap.get("ratingcount");
+		const ratingLabel = formatRating(averageRating, ratingCount);
+
+		const repoLink = extractRepoLink(extension.versions?.[0]?.properties) ?? extractRepoLink(extension.properties);
+
+		const identifier = publisherName && extensionName ? `${publisherName}.${extensionName}` : itemName;
+
+		let md = `# ${displayName}\n\n`;
+		if (description) md += `${description}\n\n`;
+		md += `**Identifier:** ${identifier}\n`;
+		if (publisherLabel) md += `**Publisher:** ${publisherLabel}\n`;
+		if (version) md += `**Version:** ${version}\n`;
+		if (installs !== undefined) md += `**Installs:** ${formatNumber(installs)}\n`;
+		if (ratingLabel) md += `**Rating:** ${ratingLabel}\n`;
+		if (extension.categories?.length) md += `**Categories:** ${extension.categories.join(", ")}\n`;
+		if (extension.tags?.length) md += `**Tags:** ${extension.tags.join(", ")}\n`;
+		if (repoLink) md += `**Repository:** ${repoLink}\n`;
+
+		return buildResult(md, {
+			url,
+			method: "vscode-marketplace",
+			fetchedAt,
+			notes: ["Fetched via VS Code Marketplace API"],
+		});
+	} catch {}
+
+	return null;
+};