@oh-my-pi/pi-coding-agent 3.24.0 → 3.30.0

package/src/core/tools/web-fetch-handlers/tldr.ts

@@ -0,0 +1,47 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+const TLDR_BASE = "https://raw.githubusercontent.com/tldr-pages/tldr/main/pages";
+const PLATFORMS = ["common", "linux", "osx"] as const;
+
+/**
+ * Handle tldr page URLs
+ * - https://tldr.sh/{command}
+ * - https://tldr.ostera.io/{command}
+ */
+export const handleTldr: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "tldr.sh" && parsed.hostname !== "tldr.ostera.io") return null;
+
+		// Extract command from path (e.g., /tar -> tar)
+		const command = parsed.pathname.replace(/^\//, "").replace(/\.md$/, "");
+		if (!command || command.includes("/")) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Try platforms in order: common, linux, osx
+		for (const platform of PLATFORMS) {
+			const rawUrl = `${TLDR_BASE}/${platform}/${command}.md`;
+			const result = await loadPage(rawUrl, { timeout });
+
+			if (result.ok && result.content.trim()) {
+				const output = finalizeOutput(result.content);
+				return {
+					url,
+					finalUrl: rawUrl,
+					contentType: "text/markdown",
+					method: "tldr",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes: [`Fetched from tldr-pages (${platform})`],
+				};
+			}
+		}
+
+		return null;
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-fetch-handlers/twitter.ts

@@ -0,0 +1,84 @@
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+const NITTER_INSTANCES = [
+	"nitter.privacyredirect.com",
+	"nitter.tiekoetter.com",
+	"nitter.poast.org",
+	"nitter.woodland.cafe",
+];
+
+/**
+ * Handle Twitter/X URLs via Nitter
+ */
+export const handleTwitter: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!["twitter.com", "x.com", "www.twitter.com", "www.x.com"].includes(parsed.hostname)) {
+			return null;
+		}
+
+		const fetchedAt = new Date().toISOString();
+
+		// Try Nitter instances
+		for (const instance of NITTER_INSTANCES) {
+			const nitterUrl = `https://${instance}${parsed.pathname}`;
+			const result = await loadPage(nitterUrl, { timeout: Math.min(timeout, 10) });
+
+			if (result.ok && result.content.length > 500) {
+				// Parse the Nitter HTML
+				const doc = parseHtml(result.content);
+
+				// Extract tweet content
+				const tweetContent = doc.querySelector(".tweet-content")?.text?.trim();
+				const fullname = doc.querySelector(".fullname")?.text?.trim();
+				const username = doc.querySelector(".username")?.text?.trim();
+				const date = doc.querySelector(".tweet-date a")?.text?.trim();
+				const stats = doc.querySelector(".tweet-stats")?.text?.trim();
+
+				if (tweetContent) {
+					let md = `# Tweet by ${fullname || "Unknown"} (${username || "@?"})\n\n`;
+					if (date) md += `*${date}*\n\n`;
+					md += `${tweetContent}\n\n`;
+					if (stats) md += `---\n${stats.replace(/\s+/g, " ")}\n`;
+
+					// Check for replies/thread
+					const replies = doc.querySelectorAll(".timeline-item .tweet-content");
+					if (replies.length > 1) {
+						md += `\n---\n\n## Thread/Replies\n\n`;
+						for (const reply of Array.from(replies).slice(1, 10)) {
+							const replyUser = reply.parentNode?.querySelector(".username")?.text?.trim();
+							md += `**${replyUser || "@?"}**: ${reply.text?.trim()}\n\n`;
+						}
+					}
+
+					const output = finalizeOutput(md);
+					return {
+						url,
+						finalUrl: nitterUrl,
+						contentType: "text/markdown",
+						method: "twitter-nitter",
+						content: output.content,
+						fetchedAt,
+						truncated: output.truncated,
+						notes: [`Via Nitter: ${instance}`],
+					};
+				}
+			}
+		}
+	} catch {}
+
+	// X.com blocks all bots - return a helpful error instead of falling through
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/plain",
+		method: "twitter-blocked",
+		content:
+			"Twitter/X blocks automated access. Nitter instances were unavailable.\n\nTry:\n- Opening the link in a browser\n- Using a different Nitter instance manually\n- Checking if the tweet is available via an archive service",
+		fetchedAt: new Date().toISOString(),
+		truncated: false,
+		notes: ["X.com blocks bots; Nitter instances unavailable"],
+	};
+};

package/src/core/tools/web-fetch-handlers/types.ts

@@ -0,0 +1,163 @@
+/**
+ * Shared types and utilities for web-fetch handlers
+ */
+
+export interface RenderResult {
+	url: string;
+	finalUrl: string;
+	contentType: string;
+	method: string;
+	content: string;
+	fetchedAt: string;
+	truncated: boolean;
+	notes: string[];
+}
+
+export type SpecialHandler = (url: string, timeout: number) => Promise<RenderResult | null>;
+
+export const MAX_OUTPUT_CHARS = 500_000;
+
+/**
+ * Truncate and cleanup output
+ */
+export function finalizeOutput(content: string): { content: string; truncated: boolean } {
+	const cleaned = content.replace(/\n{3,}/g, "\n\n").trim();
+	const truncated = cleaned.length > MAX_OUTPUT_CHARS;
+	return {
+		content: cleaned.slice(0, MAX_OUTPUT_CHARS),
+		truncated,
+	};
+}
+
+/**
+ * Fetch a page with timeout and size limit
+ */
+export async function loadPage(
+	url: string,
+	options: { timeout?: number; headers?: Record<string, string>; maxBytes?: number } = {},
+): Promise<{ content: string; contentType: string; finalUrl: string; ok: boolean; status?: number }> {
+	const { timeout = 20, headers = {}, maxBytes = 50 * 1024 * 1024 } = options;
+
+	const userAgents = [
+		"curl/8.0",
+		"Mozilla/5.0 (compatible; TextBot/1.0)",
+		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
+	];
+
+	for (let attempt = 0; attempt < userAgents.length; attempt++) {
+		const userAgent = userAgents[attempt];
+
+		try {
+			const controller = new AbortController();
+			const timeoutId = setTimeout(() => controller.abort(), timeout * 1000);
+
+			const response = await fetch(url, {
+				signal: controller.signal,
+				headers: {
+					"User-Agent": userAgent,
+					Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+					"Accept-Language": "en-US,en;q=0.5",
+					...headers,
+				},
+				redirect: "follow",
+			});
+
+			clearTimeout(timeoutId);
+
+			const contentType = response.headers.get("content-type")?.split(";")[0]?.trim().toLowerCase() ?? "";
+			const finalUrl = response.url;
+
+			const reader = response.body?.getReader();
+			if (!reader) {
+				return { content: "", contentType, finalUrl, ok: false, status: response.status };
+			}
+
+			const chunks: Uint8Array[] = [];
+			let totalSize = 0;
+
+			while (true) {
+				const { done, value } = await reader.read();
+				if (done) break;
+
+				chunks.push(value);
+				totalSize += value.length;
+
+				if (totalSize > maxBytes) {
+					reader.cancel();
+					break;
+				}
+			}
+
+			const decoder = new TextDecoder();
+			const content = decoder.decode(Buffer.concat(chunks));
+
+			// Check if blocked
+			if ((response.status === 403 || response.status === 503) && attempt < userAgents.length - 1) {
+				const lower = content.toLowerCase();
+				if (lower.includes("cloudflare") || lower.includes("captcha") || lower.includes("blocked")) {
+					continue;
+				}
+			}
+
+			if (!response.ok) {
+				return { content, contentType, finalUrl, ok: false, status: response.status };
+			}
+
+			return { content, contentType, finalUrl, ok: true, status: response.status };
+		} catch (_err) {
+			if (attempt === userAgents.length - 1) {
+				return { content: "", contentType: "", finalUrl: url, ok: false };
+			}
+		}
+	}
+
+	return { content: "", contentType: "", finalUrl: url, ok: false };
+}
+
+/**
+ * Format large numbers (1000 -> 1K, 1000000 -> 1M)
+ */
+export function formatCount(n: number): string {
+	if (n >= 1_000_000) return `${(n / 1_000_000).toFixed(1)}M`;
+	if (n >= 1_000) return `${(n / 1_000).toFixed(1)}K`;
+	return String(n);
+}
+
+/**
+ * Convert basic HTML to markdown
+ */
+export function htmlToBasicMarkdown(html: string): string {
+	return html
+		.replace(/<pre><code[^>]*>/g, "\n```\n")
+		.replace(/<\/code><\/pre>/g, "\n```\n")
+		.replace(/<code>/g, "`")
+		.replace(/<\/code>/g, "`")
+		.replace(/<strong>/g, "**")
+		.replace(/<\/strong>/g, "**")
+		.replace(/<b>/g, "**")
+		.replace(/<\/b>/g, "**")
+		.replace(/<em>/g, "*")
+		.replace(/<\/em>/g, "*")
+		.replace(/<i>/g, "*")
+		.replace(/<\/i>/g, "*")
+		.replace(/<a href="([^"]+)"[^>]*>([^<]+)<\/a>/g, "[$2]($1)")
+		.replace(/<p>/g, "\n\n")
+		.replace(/<\/p>/g, "")
+		.replace(/<br\s*\/?>/g, "\n")
+		.replace(/<li>/g, "- ")
+		.replace(/<\/li>/g, "\n")
+		.replace(/<\/?[uo]l>/g, "\n")
+		.replace(/<h(\d)>/g, (_, n) => `\n${"#".repeat(parseInt(n, 10))} `)
+		.replace(/<\/h\d>/g, "\n")
+		.replace(/<blockquote>/g, "\n> ")
+		.replace(/<\/blockquote>/g, "\n")
+		.replace(/<[^>]+>/g, "")
+		.replace(/&lt;/g, "<")
+		.replace(/&gt;/g, ">")
+		.replace(/&amp;/g, "&")
+		.replace(/&quot;/g, '"')
+		.replace(/&#39;/g, "'")
+		.replace(/&nbsp;/g, " ")
+		.replace(/\n{3,}/g, "\n\n")
+		.trim();
+}

package/src/core/tools/web-fetch-handlers/utils.ts

@@ -0,0 +1,91 @@
+import { tmpdir } from "node:os";
+import * as path from "node:path";
+import { ensureTool } from "../../../utils/tools-manager";
+
+const MAX_BYTES = 50 * 1024 * 1024; // 50MB for binary files
+
+function exec(
+	cmd: string,
+	args: string[],
+	options?: { timeout?: number; input?: string | Buffer },
+): { stdout: string; stderr: string; ok: boolean } {
+	const result = Bun.spawnSync([cmd, ...args], {
+		stdin: options?.input ? (options.input as any) : "ignore",
+		stdout: "pipe",
+		stderr: "pipe",
+	});
+	return {
+		stdout: result.stdout?.toString() ?? "",
+		stderr: result.stderr?.toString() ?? "",
+		ok: result.exitCode === 0,
+	};
+}
+
+export async function convertWithMarkitdown(
+	content: Buffer,
+	extensionHint: string,
+	timeout: number,
+): Promise<{ content: string; ok: boolean }> {
+	const markitdown = await ensureTool("markitdown", true);
+	if (!markitdown) {
+		return { content: "", ok: false };
+	}
+
+	// Write to temp file with extension hint
+	const ext = extensionHint || ".bin";
+	const tmpDir = tmpdir();
+	const tmpFile = path.join(tmpDir, `omp-convert-${Date.now()}${ext}`);
+
+	try {
+		await Bun.write(tmpFile, content);
+		const result = exec(markitdown, [tmpFile], { timeout });
+		return { content: result.stdout, ok: result.ok };
+	} finally {
+		try {
+			await Bun.$`rm ${tmpFile}`.quiet();
+		} catch {}
+	}
+}
+
+export async function fetchBinary(
+	url: string,
+	timeout: number,
+): Promise<{ buffer: Buffer; contentType: string; contentDisposition?: string; ok: boolean }> {
+	try {
+		const controller = new AbortController();
+		const timeoutId = setTimeout(() => controller.abort(), timeout * 1000);
+
+		const response = await fetch(url, {
+			signal: controller.signal,
+			headers: {
+				"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/131.0.0.0",
+			},
+			redirect: "follow",
+		});
+
+		clearTimeout(timeoutId);
+
+		if (!response.ok) {
+			return { buffer: Buffer.alloc(0), contentType: "", ok: false };
+		}
+
+		const contentType = response.headers.get("content-type") ?? "";
+		const contentDisposition = response.headers.get("content-disposition") ?? undefined;
+		const contentLength = response.headers.get("content-length");
+		if (contentLength) {
+			const size = Number.parseInt(contentLength, 10);
+			if (Number.isFinite(size) && size > MAX_BYTES) {
+				return { buffer: Buffer.alloc(0), contentType, contentDisposition, ok: false };
+			}
+		}
+
+		const buffer = Buffer.from(await response.arrayBuffer());
+		if (buffer.length > MAX_BYTES) {
+			return { buffer: Buffer.alloc(0), contentType, contentDisposition, ok: false };
+		}
+
+		return { buffer, contentType, contentDisposition, ok: true };
+	} catch {
+		return { buffer: Buffer.alloc(0), contentType: "", ok: false };
+	}
+}

package/src/core/tools/web-fetch-handlers/vimeo.ts

@@ -0,0 +1,152 @@
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface VimeoOEmbed {
+	title: string;
+	author_name: string;
+	author_url: string;
+	description?: string;
+	duration: number;
+	thumbnail_url: string;
+	upload_date: string;
+	video_id: number;
+}
+
+interface VimeoVideoConfig {
+	video?: {
+		title?: string;
+		duration?: number;
+		owner?: {
+			name?: string;
+			url?: string;
+		};
+		thumbs?: {
+			base?: string;
+		};
+	};
+	request?: {
+		files?: {
+			progressive?: Array<{
+				quality: string;
+				width: number;
+				height: number;
+				fps: number;
+			}>;
+		};
+	};
+}
+
+/**
+ * Format seconds into HH:MM:SS or MM:SS
+ */
+function formatDuration(seconds: number): string {
+	const h = Math.floor(seconds / 3600);
+	const m = Math.floor((seconds % 3600) / 60);
+	const s = seconds % 60;
+	if (h > 0) return `${h}:${String(m).padStart(2, "0")}:${String(s).padStart(2, "0")}`;
+	return `${m}:${String(s).padStart(2, "0")}`;
+}
+
+/**
+ * Extract video ID from various Vimeo URL formats
+ */
+function extractVideoId(url: string): string | null {
+	try {
+		const parsed = new URL(url);
+
+		// player.vimeo.com/video/{id}
+		if (parsed.hostname === "player.vimeo.com") {
+			const match = parsed.pathname.match(/^\/video\/(\d+)/);
+			return match?.[1] ?? null;
+		}
+
+		// vimeo.com/{id} or vimeo.com/{user}/{id}
+		if (parsed.hostname === "vimeo.com" || parsed.hostname === "www.vimeo.com") {
+			const parts = parsed.pathname.split("/").filter(Boolean);
+			// Last part should be the video ID
+			const lastPart = parts[parts.length - 1];
+			if (lastPart && /^\d+$/.test(lastPart)) {
+				return lastPart;
+			}
+		}
+
+		return null;
+	} catch {
+		return null;
+	}
+}
+
+/**
+ * Handle Vimeo URLs via oEmbed API
+ */
+export const handleVimeo: SpecialHandler = async (url: string, timeout: number) => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("vimeo.com")) return null;
+
+		const videoId = extractVideoId(url);
+		if (!videoId) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Use canonical URL for oEmbed (handles staffpicks and other URL formats)
+		const canonicalUrl = `https://vimeo.com/${videoId}`;
+		const oembedUrl = `https://vimeo.com/api/oembed.json?url=${encodeURIComponent(canonicalUrl)}`;
+		const oembedResult = await loadPage(oembedUrl, { timeout });
+
+		if (!oembedResult.ok) return null;
+
+		const oembed = JSON.parse(oembedResult.content) as VimeoOEmbed;
+
+		let md = `# ${oembed.title}\n\n`;
+		md += `**Author:** [${oembed.author_name}](${oembed.author_url})\n`;
+		md += `**Duration:** ${formatDuration(oembed.duration)}\n`;
+
+		if (oembed.upload_date) {
+			md += `**Uploaded:** ${oembed.upload_date}\n`;
+		}
+
+		md += `**Video ID:** ${videoId}\n\n`;
+
+		if (oembed.description) {
+			md += `---\n\n## Description\n\n${oembed.description}\n\n`;
+		}
+
+		md += `---\n\n**Thumbnail:** ${oembed.thumbnail_url}\n`;
+
+		// Try to get additional details from video config
+		try {
+			const configUrl = `https://player.vimeo.com/video/${videoId}/config`;
+			const configResult = await loadPage(configUrl, { timeout: Math.min(timeout, 5) });
+
+			if (configResult.ok) {
+				const config = JSON.parse(configResult.content) as VimeoVideoConfig;
+
+				// Add video quality info if available
+				const progressive = config.request?.files?.progressive;
+				if (progressive && progressive.length > 0) {
+					md += `\n**Available Qualities:**\n`;
+					for (const quality of progressive.slice(0, 5)) {
+						md += `- ${quality.quality}: ${quality.width}x${quality.height} @ ${quality.fps}fps\n`;
+					}
+				}
+			}
+		} catch {
+			// Config fetch is optional - continue without it
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "vimeo",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Vimeo oEmbed API"],
+		};
+	} catch {
+		return null;
+	}
+};