@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/tldr.ts

@@ -0,0 +1,51 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+const TLDR_BASE = "https://raw.githubusercontent.com/tldr-pages/tldr/main/pages";
+const PLATFORMS = ["common", "linux", "osx"] as const;
+
+/**
+ * Handle tldr page URLs
+ * - https://tldr.sh/{command}
+ * - https://tldr.ostera.io/{command}
+ */
+export const handleTldr: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "tldr.sh" && parsed.hostname !== "tldr.ostera.io") return null;
+
+		// Extract command from path (e.g., /tar -> tar)
+		const command = parsed.pathname.replace(/^\//, "").replace(/\.md$/, "");
+		if (!command || command.includes("/")) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Try platforms in order: common, linux, osx
+		for (const platform of PLATFORMS) {
+			const rawUrl = `${TLDR_BASE}/${platform}/${command}.md`;
+			const result = await loadPage(rawUrl, { timeout, signal });
+
+			if (result.ok && result.content.trim()) {
+				const output = finalizeOutput(result.content);
+				return {
+					url,
+					finalUrl: rawUrl,
+					contentType: "text/markdown",
+					method: "tldr",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes: [`Fetched from tldr-pages (${platform})`],
+				};
+			}
+		}
+
+		return null;
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/twitter.ts

@@ -0,0 +1,96 @@
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+const NITTER_INSTANCES = [
+	"nitter.privacyredirect.com",
+	"nitter.tiekoetter.com",
+	"nitter.poast.org",
+	"nitter.woodland.cafe",
+];
+
+/**
+ * Handle Twitter/X URLs via Nitter
+ */
+export const handleTwitter: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!["twitter.com", "x.com", "www.twitter.com", "www.x.com"].includes(parsed.hostname)) {
+			return null;
+		}
+
+		const fetchedAt = new Date().toISOString();
+
+		// Try Nitter instances
+		for (const instance of NITTER_INSTANCES) {
+			const nitterUrl = `https://${instance}${parsed.pathname}`;
+			const result = await loadPage(nitterUrl, { timeout: Math.min(timeout, 10), signal });
+
+			if (result.ok && result.content.length > 500) {
+				// Parse the Nitter HTML
+				const doc = parseHtml(result.content);
+
+				// Extract tweet content
+				const tweetContent = doc.querySelector(".tweet-content")?.text?.trim();
+				const fullname = doc.querySelector(".fullname")?.text?.trim();
+				const username = doc.querySelector(".username")?.text?.trim();
+				const date = doc.querySelector(".tweet-date a")?.text?.trim();
+				const stats = doc.querySelector(".tweet-stats")?.text?.trim();
+
+				if (tweetContent) {
+					let md = `# Tweet by ${fullname || "Unknown"} (${username || "@?"})\n\n`;
+					if (date) md += `*${date}*\n\n`;
+					md += `${tweetContent}\n\n`;
+					if (stats) md += `---\n${stats.replace(/\s+/g, " ")}\n`;
+
+					// Check for replies/thread
+					const replies = doc.querySelectorAll(".timeline-item .tweet-content");
+					if (replies.length > 1) {
+						md += `\n---\n\n## Thread/Replies\n\n`;
+						for (const reply of Array.from(replies).slice(1, 10)) {
+							const replyUser = reply.parentNode?.querySelector(".username")?.text?.trim();
+							md += `**${replyUser || "@?"}**: ${reply.text?.trim()}\n\n`;
+						}
+					}
+
+					const output = finalizeOutput(md);
+					return {
+						url,
+						finalUrl: nitterUrl,
+						contentType: "text/markdown",
+						method: "twitter-nitter",
+						content: output.content,
+						fetchedAt,
+						truncated: output.truncated,
+						notes: [`Via Nitter: ${instance}`],
+					};
+				}
+			}
+		}
+	} catch {
+		if (signal?.aborted) {
+			return null;
+		}
+	}
+
+	if (signal?.aborted) {
+		return null;
+	}
+
+	// X.com blocks all bots - return a helpful error instead of falling through
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/plain",
+		method: "twitter-blocked",
+		content:
+			"Twitter/X blocks automated access. Nitter instances were unavailable.\n\nTry:\n- Opening the link in a browser\n- Using a different Nitter instance manually\n- Checking if the tweet is available via an archive service",
+		fetchedAt: new Date().toISOString(),
+		truncated: false,
+		notes: ["X.com blocks bots; Nitter instances unavailable"],
+	};
+};

package/src/core/tools/web-scrapers/types.ts

@@ -0,0 +1,234 @@
+/**
+ * Shared types and utilities for web-fetch handlers
+ */
+
+export interface RenderResult {
+	url: string;
+	finalUrl: string;
+	contentType: string;
+	method: string;
+	content: string;
+	fetchedAt: string;
+	truncated: boolean;
+	notes: string[];
+}
+
+export type SpecialHandler = (url: string, timeout: number, signal?: AbortSignal) => Promise<RenderResult | null>;
+
+export const MAX_OUTPUT_CHARS = 500_000;
+const MAX_BYTES = 50 * 1024 * 1024;
+
+const USER_AGENTS = [
+	"curl/8.0",
+	"Mozilla/5.0 (compatible; TextBot/1.0)",
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+];
+
+export interface RequestSignal {
+	signal: AbortSignal;
+	cleanup: () => void;
+}
+
+export function createRequestSignal(timeoutMs: number, signal?: AbortSignal): RequestSignal {
+	const controller = new AbortController();
+	let timeoutId: ReturnType<typeof setTimeout> | undefined = setTimeout(() => controller.abort(), timeoutMs);
+	const abortHandler = () => controller.abort();
+
+	if (signal) {
+		if (signal.aborted) {
+			clearTimeout(timeoutId);
+			timeoutId = undefined;
+			controller.abort();
+		} else {
+			signal.addEventListener("abort", abortHandler, { once: true });
+		}
+	}
+
+	const cleanup = () => {
+		if (timeoutId !== undefined) {
+			clearTimeout(timeoutId);
+			timeoutId = undefined;
+		}
+		if (signal) {
+			signal.removeEventListener("abort", abortHandler);
+		}
+	};
+
+	return { signal: controller.signal, cleanup };
+}
+
+function isBotBlocked(status: number, content: string): boolean {
+	if (status === 403 || status === 503) {
+		const lower = content.toLowerCase();
+		return (
+			lower.includes("cloudflare") ||
+			lower.includes("captcha") ||
+			lower.includes("challenge") ||
+			lower.includes("blocked") ||
+			lower.includes("access denied") ||
+			lower.includes("bot detection")
+		);
+	}
+	return false;
+}
+
+/**
+ * Truncate and cleanup output
+ */
+export function finalizeOutput(content: string): { content: string; truncated: boolean } {
+	const cleaned = content.replace(/\n{3,}/g, "\n\n").trim();
+	const truncated = cleaned.length > MAX_OUTPUT_CHARS;
+	return {
+		content: cleaned.slice(0, MAX_OUTPUT_CHARS),
+		truncated,
+	};
+}
+
+export interface LoadPageOptions {
+	timeout?: number;
+	headers?: Record<string, string>;
+	method?: string;
+	body?: string;
+	maxBytes?: number;
+	signal?: AbortSignal;
+}
+
+export interface LoadPageResult {
+	content: string;
+	contentType: string;
+	finalUrl: string;
+	ok: boolean;
+	status?: number;
+}
+
+/**
+ * Fetch a page with timeout and size limit
+ */
+export async function loadPage(url: string, options: LoadPageOptions = {}): Promise<LoadPageResult> {
+	const { timeout = 20, headers = {}, maxBytes = MAX_BYTES, signal, method = "GET", body } = options;
+
+	for (let attempt = 0; attempt < USER_AGENTS.length; attempt++) {
+		if (signal?.aborted) {
+			return { content: "", contentType: "", finalUrl: url, ok: false };
+		}
+
+		const userAgent = USER_AGENTS[attempt];
+		const { signal: requestSignal, cleanup } = createRequestSignal(timeout * 1000, signal);
+
+		try {
+			const requestInit: RequestInit = {
+				signal: requestSignal,
+				method,
+				headers: {
+					"User-Agent": userAgent,
+					Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+					"Accept-Language": "en-US,en;q=0.5",
+					...headers,
+				},
+				redirect: "follow",
+			};
+
+			if (body !== undefined) {
+				requestInit.body = body;
+			}
+
+			const response = await fetch(url, requestInit);
+
+			const contentType = response.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? "";
+			const finalUrl = response.url;
+
+			const reader = response.body?.getReader();
+			if (!reader) {
+				return { content: "", contentType, finalUrl, ok: false, status: response.status };
+			}
+
+			const chunks: Uint8Array[] = [];
+			let totalSize = 0;
+
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) break;
+
+				chunks.push(value);
+				totalSize += value.length;
+
+				if (totalSize > maxBytes) {
+					reader.cancel();
+					break;
+				}
+			}
+
+			const decoder = new TextDecoder();
+			const content = decoder.decode(Buffer.concat(chunks));
+
+			if (isBotBlocked(response.status, content) && attempt < USER_AGENTS.length - 1) {
+				continue;
+			}
+
+			if (!response.ok) {
+				return { content, contentType, finalUrl, ok: false, status: response.status };
+			}
+
+			return { content, contentType, finalUrl, ok: true, status: response.status };
+		} catch (_err) {
+			if (signal?.aborted) {
+				return { content: "", contentType: "", finalUrl: url, ok: false };
+			}
+			if (attempt === USER_AGENTS.length - 1) {
+				return { content: "", contentType: "", finalUrl: url, ok: false };
+			}
+		} finally {
+			cleanup();
+		}
+	}
+
+	return { content: "", contentType: "", finalUrl: url, ok: false };
+}
+
+/**
+ * Format large numbers (1000 -> 1K, 1000000 -> 1M)
+ */
+export function formatCount(n: number): string {
+	if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
+	if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
+	return String(n);
+}
+
+/**
+ * Convert basic HTML to markdown
+ */
+export function htmlToBasicMarkdown(html: string): string {
+	return html
+		.replace(/<pre><code[^>]*>/g, "\n```\n")
+		.replace(/<\/code><\/pre>/g, "\n```\n")
+		.replace(/<code>/g, "`")
+		.replace(/<\/code>/g, "`")
+		.replace(/<strong>/g, "**")
+		.replace(/<\/strong>/g, "**")
+		.replace(/<b>/g, "**")
+		.replace(/<\/b>/g, "**")
+		.replace(/<em>/g, "*")
+		.replace(/<\/em>/g, "*")
+		.replace(/<i>/g, "*")
+		.replace(/<\/i>/g, "*")
+		.replace(/<a href="([^"]+)"[^>]*>([^<]+)<\/a>/g, "[$2]($1)")
+		.replace(/<p>/g, "\n\n")
+		.replace(/<\/p>/g, "")
+		.replace(/<br\s*\/?>/g, "\n")
+		.replace(/<li>/g, "- ")
+		.replace(/<\/li>/g, "\n")
+		.replace(/<\/?[uo]l>/g, "\n")
+		.replace(/<h(\d)>/g, (_, n) => `\n${"#".repeat(parseInt(n, 10))} `)
+		.replace(/<\/h\d>/g, "\n")
+		.replace(/<blockquote>/g, "\n> ")
+		.replace(/<\/blockquote>/g, "\n")
+		.replace(/<[^>]+>/g, "")
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&amp;/g, "&")
+		.replace(/&quot;/g, '"')
+		.replace(/&#39;/g, "'")
+		.replace(/&nbsp;/g, " ")
+		.replace(/\n{3,}/g, "\n\n")
+		.trim();
+}

package/src/core/tools/web-scrapers/utils.ts

@@ -0,0 +1,162 @@
+import { tmpdir } from "node:os";
+import * as path from "node:path";
+import { nanoid } from "nanoid";
+import { ensureTool } from "../../../utils/tools-manager";
+import { createRequestSignal } from "./types";
+
+const MAX_BYTES = 50 * 1024 * 1024; // 50MB for binary files
+
+interface ExecResult {
+	stdout: string;
+	stderr: string;
+	ok: boolean;
+	exitCode: number;
+}
+
+type SpawnSyncOptions = NonNullable<Parameters<typeof Bun.spawnSync>[1]>;
+
+function exec(cmd: string, args: string[], options?: { timeout?: number; input?: string | Buffer }): ExecResult {
+	const stdin = (options?.input ?? "ignore") as SpawnSyncOptions["stdin"];
+	const result = Bun.spawnSync([cmd, ...args], {
+		stdin,
+		stdout: "pipe",
+		stderr: "pipe",
+	});
+	return {
+		stdout: result.stdout?.toString() ?? "",
+		stderr: result.stderr?.toString() ?? "",
+		ok: result.exitCode === 0,
+		exitCode: result.exitCode ?? -1,
+	};
+}
+
+export interface ConvertResult {
+	content: string;
+	ok: boolean;
+	error?: string;
+}
+
+export interface BinaryFetchResult {
+	buffer: Buffer;
+	contentType: string;
+	contentDisposition?: string;
+	ok: boolean;
+	status?: number;
+	error?: string;
+}
+
+export async function convertWithMarkitdown(
+	content: Buffer,
+	extensionHint: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<ConvertResult> {
+	if (signal?.aborted) {
+		return { content: "", ok: false, error: "aborted" };
+	}
+
+	const markitdown = await ensureTool("markitdown", true);
+	if (!markitdown) {
+		return { content: "", ok: false, error: "markitdown not available" };
+	}
+
+	// Write to temp file with extension hint
+	const ext = extensionHint || ".bin";
+	const tmpDir = tmpdir();
+	const tmpFile = path.join(tmpDir, `omp-convert-${nanoid()}${ext}`);
+
+	if (content.length > MAX_BYTES) {
+		return { content: "", ok: false, error: `content exceeds ${MAX_BYTES} bytes` };
+	}
+
+	try {
+		await Bun.write(tmpFile, content);
+		const result = exec(markitdown, [tmpFile], { timeout });
+		if (!result.ok) {
+			const stderr = result.stderr.trim();
+			return {
+				content: result.stdout,
+				ok: false,
+				error: stderr.length > 0 ? stderr : `markitdown failed (exit ${result.exitCode})`,
+			};
+		}
+		return { content: result.stdout, ok: true };
+	} finally {
+		try {
+			await Bun.$`rm ${tmpFile}`.quiet();
+		} catch {}
+	}
+}
+
+export async function fetchBinary(url: string, timeout: number, signal?: AbortSignal): Promise<BinaryFetchResult> {
+	if (signal?.aborted) {
+		return { buffer: Buffer.alloc(0), contentType: "", ok: false, error: "aborted" };
+	}
+
+	const { signal: requestSignal, cleanup } = createRequestSignal(timeout * 1000, signal);
+
+	try {
+		const response = await fetch(url, {
+			signal: requestSignal,
+			headers: {
+				"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0",
+			},
+			redirect: "follow",
+		});
+
+		const contentType = response.headers.get("content-type") ?? "";
+		const contentDisposition = response.headers.get("content-disposition") ?? undefined;
+
+		if (!response.ok) {
+			return {
+				buffer: Buffer.alloc(0),
+				contentType,
+				contentDisposition,
+				ok: false,
+				status: response.status,
+				error: `status ${response.status}`,
+			};
+		}
+
+		const contentLength = response.headers.get("content-length");
+		if (contentLength) {
+			const size = Number.parseInt(contentLength, 10);
+			if (Number.isFinite(size) && size > MAX_BYTES) {
+				return {
+					buffer: Buffer.alloc(0),
+					contentType,
+					contentDisposition,
+					ok: false,
+					status: response.status,
+					error: `content-length ${size} exceeds ${MAX_BYTES}`,
+				};
+			}
+		}
+
+		const buffer = Buffer.from(await response.arrayBuffer());
+		if (buffer.length > MAX_BYTES) {
+			return {
+				buffer: Buffer.alloc(0),
+				contentType,
+				contentDisposition,
+				ok: false,
+				status: response.status,
+				error: `response exceeds ${MAX_BYTES} bytes`,
+			};
+		}
+
+		return { buffer, contentType, contentDisposition, ok: true, status: response.status };
+	} catch (err) {
+		if (signal?.aborted) {
+			return { buffer: Buffer.alloc(0), contentType: "", ok: false, error: "aborted" };
+		}
+		return {
+			buffer: Buffer.alloc(0),
+			contentType: "",
+			ok: false,
+			error: `request failed: ${String(err)}`,
+		};
+	} finally {
+		cleanup();
+	}
+}

package/src/core/tools/web-scrapers/vimeo.ts

@@ -0,0 +1,152 @@
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface VimeoOEmbed {
+	title: string;
+	author_name: string;
+	author_url: string;
+	description?: string;
+	duration: number;
+	thumbnail_url: string;
+	upload_date: string;
+	video_id: number;
+}
+
+interface VimeoVideoConfig {
+	video?: {
+		title?: string;
+		duration?: number;
+		owner?: {
+			name?: string;
+			url?: string;
+		};
+		thumbs?: {
+			base?: string;
+		};
+	};
+	request?: {
+		files?: {
+			progressive?: Array<{
+				quality: string;
+				width: number;
+				height: number;
+				fps: number;
+			}>;
+		};
+	};
+}
+
+/**
+ * Format seconds into HH:MM:SS or MM:SS
+ */
+function formatDuration(seconds: number): string {
+	const h = Math.floor(seconds / 3600);
+	const m = Math.floor((seconds % 3600) / 60);
+	const s = seconds % 60;
+	if (h > 0) return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`;
+	return `${m}:${String(s).padStart(2, "0")}`;
+}
+
+/**
+ * Extract video ID from various Vimeo URL formats
+ */
+function extractVideoId(url: string): string | null {
+	try {
+		const parsed = new URL(url);
+
+		// player.vimeo.com/video/{id}
+		if (parsed.hostname === "player.vimeo.com") {
+			const match = parsed.pathname.match(/^\/video\/(\d+)/);
+			return match?.[1] ?? null;
+		}
+
+		// vimeo.com/{id} or vimeo.com/{user}/{id}
+		if (parsed.hostname === "vimeo.com" || parsed.hostname === "www.vimeo.com") {
+			const parts = parsed.pathname.split("/").filter(Boolean);
+			// Last part should be the video ID
+			const lastPart = parts[parts.length - 1];
+			if (lastPart && /^\d+$/.test(lastPart)) {
+				return lastPart;
+			}
+		}
+
+		return null;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Handle Vimeo URLs via oEmbed API
+ */
+export const handleVimeo: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("vimeo.com")) return null;
+
+		const videoId = extractVideoId(url);
+		if (!videoId) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Use canonical URL for oEmbed (handles staffpicks and other URL formats)
+		const canonicalUrl = `https://vimeo.com/${videoId}`;
+		const oembedUrl = `https://vimeo.com/api/oembed.json?url=${encodeURIComponent(canonicalUrl)}`;
+		const oembedResult = await loadPage(oembedUrl, { timeout, signal });
+
+		if (!oembedResult.ok) return null;
+
+		const oembed = JSON.parse(oembedResult.content) as VimeoOEmbed;
+
+		let md = `# ${oembed.title}\n\n`;
+		md += `**Author:** [${oembed.author_name}](${oembed.author_url})\n`;
+		md += `**Duration:** ${formatDuration(oembed.duration)}\n`;
+
+		if (oembed.upload_date) {
+			md += `**Uploaded:** ${oembed.upload_date}\n`;
+		}
+
+		md += `**Video ID:** ${videoId}\n\n`;
+
+		if (oembed.description) {
+			md += `---\n\n## Description\n\n${oembed.description}\n\n`;
+		}
+
+		md += `---\n\n**Thumbnail:** ${oembed.thumbnail_url}\n`;
+
+		// Try to get additional details from video config
+		try {
+			const configUrl = `https://player.vimeo.com/video/${videoId}/config`;
+			const configResult = await loadPage(configUrl, { timeout: Math.min(timeout, 5), signal });
+
+			if (configResult.ok) {
+				const config = JSON.parse(configResult.content) as VimeoVideoConfig;
+
+				// Add video quality info if available
+				const progressive = config.request?.files?.progressive;
+				if (progressive && progressive.length > 0) {
+					md += `\n**Available Qualities:**\n`;
+					for (const quality of progressive.slice(0, 5)) {
+						md += `- ${quality.quality}: ${quality.width}x${quality.height} @ ${quality.fps}fps\n`;
+					}
+				}
+			}
+		} catch {
+			// Config fetch is optional - continue without it
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "vimeo",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Vimeo oEmbed API"],
+		};
+	} catch {
+		return null;
+	}
+};