@oh-my-pi/pi-coding-agent 3.30.0 → 3.31.0

package/src/core/tools/{web-fetch-handlers → web-scrapers}/types.ts

@@ -13,9 +13,64 @@ export interface RenderResult {
 	notes: string[];
 }
 
-export type SpecialHandler = (url: string, timeout: number) => Promise<RenderResult | null>;
+export type SpecialHandler = (url: string, timeout: number, signal?: AbortSignal) => Promise<RenderResult | null>;
 
 export const MAX_OUTPUT_CHARS = 500_000;
+const MAX_BYTES = 50 * 1024 * 1024;
+
+const USER_AGENTS = [
+	"curl/8.0",
+	"Mozilla/5.0 (compatible; TextBot/1.0)",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+];
+
+export interface RequestSignal {
+	signal: AbortSignal;
+	cleanup: () => void;
+}
+
+export function createRequestSignal(timeoutMs: number, signal?: AbortSignal): RequestSignal {
+	const controller = new AbortController();
+	let timeoutId: ReturnType<typeof setTimeout> | undefined = setTimeout(() => controller.abort(), timeoutMs);
+	const abortHandler = () => controller.abort();
+
+	if (signal) {
+		if (signal.aborted) {
+			clearTimeout(timeoutId);
+			timeoutId = undefined;
+			controller.abort();
+		} else {
+			signal.addEventListener("abort", abortHandler, { once: true });
+		}
+	}
+
+	const cleanup = () => {
+		if (timeoutId !== undefined) {
+			clearTimeout(timeoutId);
+			timeoutId = undefined;
+		}
+		if (signal) {
+			signal.removeEventListener("abort", abortHandler);
+		}
+	};
+
+	return { signal: controller.signal, cleanup };
+}
+
+function isBotBlocked(status: number, content: string): boolean {
+	if (status === 403 || status === 503) {
+		const lower = content.toLowerCase();
+		return (
+			lower.includes("cloudflare") ||
+			lower.includes("captcha") ||
+			lower.includes("challenge") ||
+			lower.includes("blocked") ||
+			lower.includes("access denied") ||
+			lower.includes("bot detection")
+		);
+	}
+	return false;
+}
 
 /**
  * Truncate and cleanup output
@@ -29,30 +84,41 @@ export function finalizeOutput(content: string): { content: string; truncated: b
 	};
 }
 
+export interface LoadPageOptions {
+	timeout?: number;
+	headers?: Record<string, string>;
+	method?: string;
+	body?: string;
+	maxBytes?: number;
+	signal?: AbortSignal;
+}
+
+export interface LoadPageResult {
+	content: string;
+	contentType: string;
+	finalUrl: string;
+	ok: boolean;
+	status?: number;
+}
+
 /**
  * Fetch a page with timeout and size limit
  */
-export async function loadPage(
-	url: string,
-	options: { timeout?: number; headers?: Record<string, string>; maxBytes?: number } = {},
-): Promise<{ content: string; contentType: string; finalUrl: string; ok: boolean; status?: number }> {
-	const { timeout = 20, headers = {}, maxBytes = 50 * 1024 * 1024 } = options;
+export async function loadPage(url: string, options: LoadPageOptions = {}): Promise<LoadPageResult> {
+	const { timeout = 20, headers = {}, maxBytes = MAX_BYTES, signal, method = "GET", body } = options;
 
-	const userAgents = [
-		"curl/8.0",
-		"Mozilla/5.0 (compatible; TextBot/1.0)",
-		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
-	];
+	for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) {
+		if (signal?.aborted) {
+			return { content: "", contentType: "", finalUrl: url, ok: false };
+		}
 
-	for (let attempt = 0; attempt < userAgents.length; attempt++) {
-		const userAgent = userAgents[attempt];
+		const userAgent = USER_AGENTS[attempt];
+		const { signal: requestSignal, cleanup } = createRequestSignal(timeout * 1000, signal);
 
 		try {
-			const controller = new AbortController();
-			const timeoutId = setTimeout(() => controller.abort(), timeout * 1000);
-
-			const response = await fetch(url, {
-				signal: controller.signal,
+			const requestInit: RequestInit = {
+				signal: requestSignal,
+				method,
 				headers: {
 					"User-Agent": userAgent,
 					Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
@@ -60,9 +126,13 @@ export async function loadPage(
 					...headers,
 				},
 				redirect: "follow",
-			});
+			};
 
-			clearTimeout(timeoutId);
+			if (body !== undefined) {
+				requestInit.body = body;
+			}
+
+			const response = await fetch(url, requestInit);
 
 			const contentType = response.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? "";
 			const finalUrl = response.url;
@@ -91,12 +161,8 @@ export async function loadPage(
 			const decoder = new TextDecoder();
 			const content = decoder.decode(Buffer.concat(chunks));
 
-			// Check if blocked
-			if ((response.status === 403 || response.status === 503) && attempt < userAgents.length - 1) {
-				const lower = content.toLowerCase();
-				if (lower.includes("cloudflare") || lower.includes("captcha") || lower.includes("blocked")) {
-					continue;
-				}
+			if (isBotBlocked(response.status, content) && attempt < USER_AGENTS.length - 1) {
+				continue;
 			}
 
 			if (!response.ok) {
@@ -105,9 +171,14 @@ export async function loadPage(
 
 			return { content, contentType, finalUrl, ok: true, status: response.status };
 		} catch (_err) {
-			if (attempt === userAgents.length - 1) {
+			if (signal?.aborted) {
+				return { content: "", contentType: "", finalUrl: url, ok: false };
+			}
+			if (attempt === USER_AGENTS.length - 1) {
 				return { content: "", contentType: "", finalUrl: url, ok: false };
 			}
+		} finally {
+			cleanup();
 		}
 	}
 

package/src/core/tools/web-scrapers/utils.ts

@@ -0,0 +1,162 @@
+import { tmpdir } from "node:os";
+import * as path from "node:path";
+import { nanoid } from "nanoid";
+import { ensureTool } from "../../../utils/tools-manager";
+import { createRequestSignal } from "./types";
+
+const MAX_BYTES = 50 * 1024 * 1024; // 50MB for binary files
+
+interface ExecResult {
+	stdout: string;
+	stderr: string;
+	ok: boolean;
+	exitCode: number;
+}
+
+type SpawnSyncOptions = NonNullable<Parameters<typeof Bun.spawnSync>[1]>;
+
+function exec(cmd: string, args: string[], options?: { timeout?: number; input?: string | Buffer }): ExecResult {
+	const stdin = (options?.input ?? "ignore") as SpawnSyncOptions["stdin"];
+	const result = Bun.spawnSync([cmd, ...args], {
+		stdin,
+		stdout: "pipe",
+		stderr: "pipe",
+	});
+	return {
+		stdout: result.stdout?.toString() ?? "",
+		stderr: result.stderr?.toString() ?? "",
+		ok: result.exitCode === 0,
+		exitCode: result.exitCode ?? -1,
+	};
+}
+
+export interface ConvertResult {
+	content: string;
+	ok: boolean;
+	error?: string;
+}
+
+export interface BinaryFetchResult {
+	buffer: Buffer;
+	contentType: string;
+	contentDisposition?: string;
+	ok: boolean;
+	status?: number;
+	error?: string;
+}
+
+export async function convertWithMarkitdown(
+	content: Buffer,
+	extensionHint: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<ConvertResult> {
+	if (signal?.aborted) {
+		return { content: "", ok: false, error: "aborted" };
+	}
+
+	const markitdown = await ensureTool("markitdown", true);
+	if (!markitdown) {
+		return { content: "", ok: false, error: "markitdown not available" };
+	}
+
+	// Write to temp file with extension hint
+	const ext = extensionHint || ".bin";
+	const tmpDir = tmpdir();
+	const tmpFile = path.join(tmpDir, `omp-convert-${nanoid()}${ext}`);
+
+	if (content.length > MAX_BYTES) {
+		return { content: "", ok: false, error: `content exceeds ${MAX_BYTES} bytes` };
+	}
+
+	try {
+		await Bun.write(tmpFile, content);
+		const result = exec(markitdown, [tmpFile], { timeout });
+		if (!result.ok) {
+			const stderr = result.stderr.trim();
+			return {
+				content: result.stdout,
+				ok: false,
+				error: stderr.length > 0 ? stderr : `markitdown failed (exit ${result.exitCode})`,
+			};
+		}
+		return { content: result.stdout, ok: true };
+	} finally {
+		try {
+			await Bun.$`rm ${tmpFile}`.quiet();
+		} catch {}
+	}
+}
+
+export async function fetchBinary(url: string, timeout: number, signal?: AbortSignal): Promise<BinaryFetchResult> {
+	if (signal?.aborted) {
+		return { buffer: Buffer.alloc(0), contentType: "", ok: false, error: "aborted" };
+	}
+
+	const { signal: requestSignal, cleanup } = createRequestSignal(timeout * 1000, signal);
+
+	try {
+		const response = await fetch(url, {
+			signal: requestSignal,
+			headers: {
+				"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0",
+			},
+			redirect: "follow",
+		});
+
+		const contentType = response.headers.get("content-type") ?? "";
+		const contentDisposition = response.headers.get("content-disposition") ?? undefined;
+
+		if (!response.ok) {
+			return {
+				buffer: Buffer.alloc(0),
+				contentType,
+				contentDisposition,
+				ok: false,
+				status: response.status,
+				error: `status ${response.status}`,
+			};
+		}
+
+		const contentLength = response.headers.get("content-length");
+		if (contentLength) {
+			const size = Number.parseInt(contentLength, 10);
+			if (Number.isFinite(size) && size > MAX_BYTES) {
+				return {
+					buffer: Buffer.alloc(0),
+					contentType,
+					contentDisposition,
+					ok: false,
+					status: response.status,
+					error: `content-length ${size} exceeds ${MAX_BYTES}`,
+				};
+			}
+		}
+
+		const buffer = Buffer.from(await response.arrayBuffer());
+		if (buffer.length > MAX_BYTES) {
+			return {
+				buffer: Buffer.alloc(0),
+				contentType,
+				contentDisposition,
+				ok: false,
+				status: response.status,
+				error: `response exceeds ${MAX_BYTES} bytes`,
+			};
+		}
+
+		return { buffer, contentType, contentDisposition, ok: true, status: response.status };
+	} catch (err) {
+		if (signal?.aborted) {
+			return { buffer: Buffer.alloc(0), contentType: "", ok: false, error: "aborted" };
+		}
+		return {
+			buffer: Buffer.alloc(0),
+			contentType: "",
+			ok: false,
+			error: `request failed: ${String(err)}`,
+		};
+	} finally {
+		cleanup();
+	}
+}

package/src/core/tools/{web-fetch-handlers → web-scrapers}/vimeo.ts

@@ -79,7 +79,7 @@ function extractVideoId(url: string): string | null {
 /**
  * Handle Vimeo URLs via oEmbed API
  */
-export const handleVimeo: SpecialHandler = async (url: string, timeout: number) => {
+export const handleVimeo: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
 	try {
 		const parsed = new URL(url);
 		if (!parsed.hostname.includes("vimeo.com")) return null;
@@ -92,7 +92,7 @@ export const handleVimeo: SpecialHandler = async (url: string, timeout: number)
 		// Use canonical URL for oEmbed (handles staffpicks and other URL formats)
 		const canonicalUrl = `https://vimeo.com/${videoId}`;
 		const oembedUrl = `https://vimeo.com/api/oembed.json?url=${encodeURIComponent(canonicalUrl)}`;
-		const oembedResult = await loadPage(oembedUrl, { timeout });
+		const oembedResult = await loadPage(oembedUrl, { timeout, signal });
 
 		if (!oembedResult.ok) return null;
 
@@ -117,7 +117,7 @@ export const handleVimeo: SpecialHandler = async (url: string, timeout: number)
 		// Try to get additional details from video config
 		try {
 			const configUrl = `https://player.vimeo.com/video/${videoId}/config`;
-			const configResult = await loadPage(configUrl, { timeout: Math.min(timeout, 5) });
+			const configResult = await loadPage(configUrl, { timeout: Math.min(timeout, 5), signal });
 
 			if (configResult.ok) {
 				const config = JSON.parse(configResult.content) as VimeoVideoConfig;

package/src/core/tools/web-scrapers/vscode-marketplace.ts

@@ -0,0 +1,195 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface MarketplaceProperty {
+	key?: string;
+	value?: string;
+}
+
+interface MarketplaceVersion {
+	version?: string;
+	properties?: MarketplaceProperty[];
+}
+
+interface MarketplaceStatistic {
+	statisticName?: string;
+	value?: number;
+}
+
+interface MarketplacePublisher {
+	publisherName?: string;
+	displayName?: string;
+}
+
+interface MarketplaceExtension {
+	extensionName?: string;
+	displayName?: string;
+	shortDescription?: string;
+	description?: string;
+	publisher?: MarketplacePublisher;
+	versions?: MarketplaceVersion[];
+	statistics?: MarketplaceStatistic[];
+	categories?: string[];
+	tags?: string[];
+	properties?: MarketplaceProperty[];
+}
+
+interface MarketplaceResponse {
+	results?: Array<{ extensions?: MarketplaceExtension[] }>;
+}
+
+const MARKETPLACE_HOSTS = new Set(["marketplace.visualstudio.com", "www.marketplace.visualstudio.com"]);
+
+function getItemName(parsed: URL): string | null {
+	if (!parsed.pathname.startsWith("/items")) return null;
+	const itemName = parsed.searchParams.get("itemName");
+	if (!itemName) return null;
+	const decoded = decodeURIComponent(itemName);
+	if (!decoded.includes(".")) return null;
+	return decoded;
+}
+
+function toStatMap(stats: MarketplaceStatistic[] | undefined): Map<string, number> {
+	const map = new Map<string, number>();
+	if (!stats) return map;
+	for (const stat of stats) {
+		if (!stat.statisticName || typeof stat.value !== "number") continue;
+		map.set(stat.statisticName.trim().toLowerCase(), stat.value);
+	}
+	return map;
+}
+
+function formatRating(averageRating?: number, ratingCount?: number): string | null {
+	if (averageRating === undefined && ratingCount === undefined) return null;
+	if (averageRating !== undefined) {
+		const formatted = averageRating.toFixed(2).replace(/\.0+$/, "").replace(/\.$/, "");
+		if (ratingCount !== undefined) {
+			return `${formatted} (${formatCount(ratingCount)} ratings)`;
+		}
+		return formatted;
+	}
+	if (ratingCount !== undefined) {
+		return `${formatCount(ratingCount)} ratings`;
+	}
+	return null;
+}
+
+function extractRepoLink(properties: MarketplaceProperty[] | undefined): string | null {
+	if (!properties) return null;
+	for (const prop of properties) {
+		const key = prop.key?.trim().toLowerCase();
+		const value = prop.value?.trim();
+		if (!key || !value) continue;
+		if (!value.startsWith("http")) continue;
+		if (key.includes("links.source") || key.includes("repository")) return value;
+	}
+	for (const prop of properties) {
+		const key = prop.key?.trim().toLowerCase();
+		const value = prop.value?.trim();
+		if (!key || !value) continue;
+		if (!value.startsWith("http")) continue;
+		if (key === "source" || key.endsWith(".source")) return value;
+	}
+	return null;
+}
+
+/**
+ * Handle VS Code Marketplace URLs via extension query API
+ */
+export const handleVscodeMarketplace: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!MARKETPLACE_HOSTS.has(parsed.hostname)) return null;
+
+		const itemName = getItemName(parsed);
+		if (!itemName) return null;
+
+		const [publisherFromUrl, ...nameParts] = itemName.split(".");
+		const extensionFromUrl = nameParts.join(".");
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = "https://marketplace.visualstudio.com/_apis/public/gallery/extensionquery";
+		const payload = JSON.stringify({
+			filters: [
+				{
+					criteria: [{ filterType: 7, value: itemName }],
+				},
+			],
+			flags: 950,
+		});
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			method: "POST",
+			body: payload,
+			headers: {
+				"Content-Type": "application/json",
+				Accept: "application/json;api-version=7.2-preview.1",
+			},
+		});
+
+		if (!result.ok) return null;
+
+		let data: MarketplaceResponse;
+		try {
+			data = JSON.parse(result.content) as MarketplaceResponse;
+		} catch {
+			return null;
+		}
+
+		const extension = data.results?.[0]?.extensions?.[0];
+		if (!extension) return null;
+
+		const extensionName = extension.extensionName ?? extensionFromUrl;
+		const displayName = extension.displayName ?? extensionName ?? itemName;
+		const description = extension.shortDescription ?? extension.description;
+
+		const publisherName = extension.publisher?.publisherName ?? publisherFromUrl;
+		const publisherDisplayName = extension.publisher?.displayName;
+		const publisherLabel =
+			publisherDisplayName && publisherName && publisherDisplayName !== publisherName
+				? `${publisherDisplayName} (${publisherName})`
+				: (publisherDisplayName ?? publisherName);
+
+		const version = extension.versions?.[0]?.version;
+		const statMap = toStatMap(extension.statistics);
+		const installs = statMap.get("install") ?? statMap.get("installs");
+		const averageRating = statMap.get("averagerating");
+		const ratingCount = statMap.get("ratingcount");
+		const ratingLabel = formatRating(averageRating, ratingCount);
+
+		const repoLink = extractRepoLink(extension.versions?.[0]?.properties) ?? extractRepoLink(extension.properties);
+
+		const identifier = publisherName && extensionName ? `${publisherName}.${extensionName}` : itemName;
+
+		let md = `# ${displayName}\n\n`;
+		if (description) md += `${description}\n\n`;
+		md += `**Identifier:** ${identifier}\n`;
+		if (publisherLabel) md += `**Publisher:** ${publisherLabel}\n`;
+		if (version) md += `**Version:** ${version}\n`;
+		if (installs !== undefined) md += `**Installs:** ${formatCount(installs)}\n`;
+		if (ratingLabel) md += `**Rating:** ${ratingLabel}\n`;
+		if (extension.categories?.length) md += `**Categories:** ${extension.categories.join(", ")}\n`;
+		if (extension.tags?.length) md += `**Tags:** ${extension.tags.join(", ")}\n`;
+		if (repoLink) md += `**Repository:** ${repoLink}\n`;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "vscode-marketplace",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via VS Code Marketplace API"],
+		};
+	} catch {}
+
+	return null;
+};