@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/flathub.ts

@@ -0,0 +1,239 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface FlathubScreenshotSize {
+	src?: string;
+	width?: string;
+	height?: string;
+	scale?: string;
+}
+
+interface FlathubScreenshot {
+	caption?: string | null;
+	sizes?: FlathubScreenshotSize[];
+}
+
+interface FlathubRelease {
+	version?: string;
+	timestamp?: string;
+	description?: string | null;
+	url?: string | null;
+	type?: string | null;
+}
+
+interface FlathubAppStream {
+	id?: string;
+	name?: string;
+	summary?: string;
+	description?: string;
+	developer_name?: string;
+	categories?: string[];
+	screenshots?: FlathubScreenshot[];
+	releases?: FlathubRelease[];
+	metadata?: Record<string, unknown>;
+	installs?: number | string;
+	permissions?: unknown;
+}
+
+function extractAppId(pathname: string): string | null {
+	const detailsMatch = pathname.match(/^\/apps\/details\/([^/]+)\/?$/);
+	if (detailsMatch) return decodeURIComponent(detailsMatch[1]);
+
+	const appMatch = pathname.match(/^\/apps\/([^/]+)\/?$/);
+	if (appMatch) return decodeURIComponent(appMatch[1]);
+
+	return null;
+}
+
+function parseNumber(value: unknown): number | null {
+	if (typeof value === "number" && Number.isFinite(value)) return value;
+	if (typeof value === "string") {
+		const cleaned = value.replace(/[^0-9.]/g, "");
+		if (!cleaned) return null;
+		const parsed = Number(cleaned);
+		if (!Number.isNaN(parsed)) return parsed;
+	}
+	return null;
+}
+
+function normalizeStringList(value: unknown): string[] {
+	if (Array.isArray(value)) {
+		return value.filter((item): item is string => typeof item === "string" && item.trim().length > 0);
+	}
+	if (typeof value === "string") {
+		return value
+			.split(/[,;\n]+/)
+			.map((item) => item.trim())
+			.filter(Boolean);
+	}
+	return [];
+}
+
+function extractInstalls(app: FlathubAppStream): number | null {
+	const direct = parseNumber(app.installs);
+	if (direct !== null) return direct;
+
+	if (!app.metadata) return null;
+	for (const [key, value] of Object.entries(app.metadata)) {
+		if (!key.toLowerCase().includes("install")) continue;
+		const parsed = parseNumber(value);
+		if (parsed !== null) return parsed;
+	}
+
+	return null;
+}
+
+function extractPermissions(app: FlathubAppStream): string[] {
+	const permissions: string[] = [];
+	permissions.push(...normalizeStringList(app.permissions));
+
+	if (app.metadata) {
+		for (const [key, value] of Object.entries(app.metadata)) {
+			if (!key.toLowerCase().includes("permission")) continue;
+			const list = normalizeStringList(value);
+			if (list.length) {
+				permissions.push(...list);
+				continue;
+			}
+			if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
+				permissions.push(`${key}: ${String(value)}`);
+			}
+		}
+	}
+
+	return Array.from(new Set(permissions));
+}
+
+function screenshotArea(size?: FlathubScreenshotSize): number {
+	if (!size) return 0;
+	const width = Number(size.width);
+	const height = Number(size.height);
+	if (!Number.isFinite(width) || !Number.isFinite(height)) return 0;
+	return width * height;
+}
+
+function bestScreenshotUrl(sizes?: FlathubScreenshotSize[]): string | null {
+	if (!sizes || sizes.length === 0) return null;
+
+	let best = sizes[0];
+	let bestArea = screenshotArea(best);
+
+	for (const size of sizes) {
+		const area = screenshotArea(size);
+		if (area > bestArea) {
+			best = size;
+			bestArea = area;
+		}
+	}
+
+	return best.src ?? sizes[0].src ?? null;
+}
+
+function formatReleaseDate(timestamp?: string | null): string | null {
+	if (!timestamp) return null;
+	const seconds = Number(timestamp);
+	if (!Number.isFinite(seconds)) return null;
+	const date = new Date(seconds * 1000);
+	if (Number.isNaN(date.getTime())) return null;
+	return date.toISOString().split("T")[0] ?? null;
+}
+
+export const handleFlathub: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "flathub.org" && parsed.hostname !== "www.flathub.org") return null;
+
+		const appId = extractAppId(parsed.pathname);
+		if (!appId) return null;
+
+		const apiUrl = `https://flathub.org/api/v2/appstream/${encodeURIComponent(appId)}`;
+		const result = await loadPage(apiUrl, { timeout, signal, headers: { Accept: "application/json" } });
+		if (!result.ok) return null;
+
+		let app: FlathubAppStream;
+		try {
+			app = JSON.parse(result.content) as FlathubAppStream;
+		} catch {
+			return null;
+		}
+
+		const fetchedAt = new Date().toISOString();
+		const name = app.name ?? app.id ?? appId;
+
+		let md = `# ${name}\n\n`;
+		if (app.summary) md += `${app.summary}\n\n`;
+
+		md += "## Metadata\n\n";
+		md += `**App ID:** ${app.id ?? appId}\n`;
+		if (app.developer_name) md += `**Developer:** ${app.developer_name}\n`;
+
+		const installs = extractInstalls(app);
+		if (installs !== null) md += `**Installs:** ${formatCount(installs)}\n`;
+
+		if (app.categories?.length) {
+			md += "\n## Categories\n\n";
+			for (const category of app.categories) {
+				md += `- ${category}\n`;
+			}
+		}
+
+		if (app.description) {
+			const description = htmlToBasicMarkdown(app.description);
+			if (description) md += `\n## Description\n\n${description}\n`;
+		}
+
+		const permissions = extractPermissions(app);
+		if (permissions.length) {
+			md += "\n## Permissions\n\n";
+			for (const permission of permissions) {
+				md += `- ${permission}\n`;
+			}
+		}
+
+		if (app.screenshots?.length) {
+			md += "\n## Screenshots\n\n";
+			for (const screenshot of app.screenshots.slice(0, 5)) {
+				const screenshotUrl = bestScreenshotUrl(screenshot.sizes);
+				if (!screenshotUrl) continue;
+				const caption = screenshot.caption ? ` - ${screenshot.caption}` : "";
+				md += `- ${screenshotUrl}${caption}\n`;
+			}
+		}
+
+		if (app.releases?.length) {
+			md += "\n## Releases\n\n";
+			for (const release of app.releases.slice(0, 5)) {
+				const version = release.version ?? "unknown";
+				let line = `- **${version}**`;
+				const date = formatReleaseDate(release.timestamp);
+				if (date) line += ` (${date})`;
+				if (release.type) line += ` · ${release.type}`;
+				if (release.url) line += ` · ${release.url}`;
+				md += `${line}\n`;
+
+				if (release.description) {
+					const releaseDesc = htmlToBasicMarkdown(release.description).replace(/\n+/g, " ").trim();
+					if (releaseDesc) md += `  - ${releaseDesc}\n`;
+				}
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: result.finalUrl,
+			contentType: "text/markdown",
+			method: "flathub-appstream",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Flathub Appstream API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/git-hosting.test.ts

@@ -0,0 +1,272 @@
+import { describe, expect, it } from "bun:test";
+import { handleGitHub } from "./github";
+import { handleGitHubGist } from "./github-gist";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+// =============================================================================
+// GitHub Tests
+// =============================================================================
+
+describe.skipIf(SKIP)("handleGitHub", () => {
+	it("returns null for non-GitHub URLs", async () => {
+		const result = await handleGitHub("https://example.com", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for other git hosting domains", async () => {
+		const result = await handleGitHub("https://gitlab.com/user/repo", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("fetches repository root", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-repo");
+			expect(result.contentType).toBe("text/markdown");
+			expect(result.content).toContain("facebook/react");
+			expect(result.content).toContain("Stars:");
+			expect(result.content).toContain("Forks:");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("fetches another repository", async () => {
+		const result = await handleGitHub("https://github.com/microsoft/typescript", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-repo");
+			// GitHub returns "TypeScript" with capital T
+			expect(result.content).toContain("microsoft/TypeScript");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("fetches file blob", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/blob/main/README.md", 20000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("github-raw");
+		expect(result?.contentType).toBe("text/plain");
+		expect(result?.content.length).toBeGreaterThan(0);
+	});
+
+	it("fetches file blob from specific branch", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/blob/main/package.json", 20000);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("github-raw");
+		expect(result?.content.length).toBeGreaterThan(0);
+	});
+
+	it("fetches directory tree", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/tree/main/packages", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-tree");
+			expect(result.contentType).toBe("text/markdown");
+			expect(result.content).toContain("facebook/react");
+			expect(result.content).toContain("Contents");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("fetches directory tree from root", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/tree/main", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-tree");
+			expect(result.content).toContain("facebook/react");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("fetches issue", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/issues/1", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-issue");
+			expect(result.contentType).toBe("text/markdown");
+			expect(result.content.length).toBeGreaterThan(0);
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("fetches pull request", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/pull/1", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-pr");
+			expect(result.contentType).toBe("text/markdown");
+			expect(result.content.length).toBeGreaterThan(0);
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("fetches issues list", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/issues", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-issues");
+			expect(result.contentType).toBe("text/markdown");
+			expect(result.content.length).toBeGreaterThan(0);
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("handles repository with underscore in name", async () => {
+		const result = await handleGitHub("https://github.com/rust-lang/rust-analyzer", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-repo");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("handles repository with dash in name", async () => {
+		const result = await handleGitHub("https://github.com/vercel/next.js", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-repo");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("returns null for invalid URL structure", async () => {
+		const result = await handleGitHub("https://github.com/", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for single path segment", async () => {
+		const result = await handleGitHub("https://github.com/facebook", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("handles pulls list endpoint", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/pulls", 20000);
+		// Should be handled as pulls list but currently falls back to null
+		// This tests the actual behavior
+		expect(result).toBeDefined();
+	});
+
+	it("fetches file with path containing multiple directories", async () => {
+		const result = await handleGitHub(
+			"https://github.com/facebook/react/blob/main/packages/react/package.json",
+			20000,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("github-raw");
+	});
+
+	it("fetches deeply nested directory", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/tree/main/packages/react/src", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-tree");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("returns null for discussion URLs", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/discussions", 10000);
+		// Discussions not fully implemented, should return null
+		expect(result).toBeDefined();
+	});
+
+	it("handles trailing slash in repository URL", async () => {
+		const result = await handleGitHub("https://github.com/facebook/react/", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-repo");
+		}
+		expect(result).toBeDefined();
+	});
+});
+
+// =============================================================================
+// GitHub Gist Tests
+// =============================================================================
+
+describe.skipIf(SKIP)("handleGitHubGist", () => {
+	it("returns null for non-gist URLs", async () => {
+		const result = await handleGitHubGist("https://example.com", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for github.com URLs", async () => {
+		const result = await handleGitHubGist("https://github.com/user/repo", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for gist.github.com root", async () => {
+		const result = await handleGitHubGist("https://gist.github.com/", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("fetches a public gist with username", async () => {
+		// Using a valid public gist ID (may change but structure should be consistent)
+		const result = await handleGitHubGist("https://gist.github.com/gaearon/edf814aeee85062bc9b9830aeaf27b88", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-gist");
+			expect(result.contentType).toBe("text/markdown");
+			expect(result.content).toContain("Gist by");
+			expect(result.content).toContain("Created:");
+			expect(result.content).toContain("Files:");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("fetches a public gist without username in URL", async () => {
+		// Same gist, accessed via short URL (without username)
+		const result = await handleGitHubGist("https://gist.github.com/edf814aeee85062bc9b9830aeaf27b88", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-gist");
+			expect(result.content).toContain("Gist by");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("returns null for invalid gist ID format", async () => {
+		const result = await handleGitHubGist("https://gist.github.com/invalid-gist-id!", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for non-hexadecimal gist ID", async () => {
+		const result = await handleGitHubGist("https://gist.github.com/notahexstring123", 10000);
+		expect(result).toBeNull();
+	});
+
+	it("handles gist URL with trailing slash", async () => {
+		const result = await handleGitHubGist("https://gist.github.com/gaearon/edf814aeee85062bc9b9830aeaf27b88/", 20000);
+		if (result !== null) {
+			expect(result.method).toBe("github-gist");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("handles gist with revision hash", async () => {
+		const result = await handleGitHubGist(
+			"https://gist.github.com/gaearon/edf814aeee85062bc9b9830aeaf27b88/abc123",
+			20000,
+		);
+		// Should handle revision hash in URL path
+		expect(result).toBeDefined();
+	});
+
+	it("formats gist content as markdown with code blocks", async () => {
+		const result = await handleGitHubGist("https://gist.github.com/gaearon/edf814aeee85062bc9b9830aeaf27b88", 20000);
+		if (result !== null) {
+			expect(result.content).toContain("```");
+			expect(result.content).toContain("---");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("includes file metadata", async () => {
+		const result = await handleGitHubGist("https://gist.github.com/gaearon/edf814aeee85062bc9b9830aeaf27b88", 20000);
+		if (result !== null) {
+			expect(result.content).toContain("Created:");
+			expect(result.content).toContain("Updated:");
+		}
+		expect(result).toBeDefined();
+	});
+
+	it("returns null for nonexistent gist", async () => {
+		const result = await handleGitHubGist("https://gist.github.com/0000000000000000000000000000000000000000", 20000);
+		expect(result).toBeNull();
+	});
+
+	it("handles API rate limiting gracefully", async () => {
+		// This test just ensures no errors are thrown
+		const result = await handleGitHubGist("https://gist.github.com/gaearon/edf814aeee85062bc9b9830aeaf27b88", 5000);
+		expect(result).toBeDefined();
+	});
+});

package/src/core/tools/web-scrapers/github-gist.ts

@@ -0,0 +1,68 @@
+import { fetchGitHubApi } from "./github";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput } from "./types";
+
+/**
+ * Handle GitHub Gist URLs via GitHub API
+ */
+export const handleGitHubGist: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "gist.github.com") return null;
+
+		// Extract gist ID from /username/gistId or just /gistId
+		const parts = parsed.pathname.split("/").filter(Boolean);
+		if (parts.length === 0) return null;
+
+		// Gist ID is always the last path segment (or only segment for anonymous gists)
+		const gistId = parts[parts.length - 1];
+		if (!gistId || !/^[a-f0-9]+$/i.test(gistId)) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch via GitHub API
+		const result = await fetchGitHubApi(`/gists/${gistId}`, timeout, signal);
+		if (!result.ok || !result.data) return null;
+
+		const gist = result.data as {
+			description: string | null;
+			owner?: { login: string };
+			created_at: string;
+			updated_at: string;
+			files: Record<string, { filename: string; language: string | null; size: number; content: string }>;
+			html_url: string;
+		};
+
+		const files = Object.values(gist.files);
+		const owner = gist.owner?.login || "anonymous";
+
+		let md = `# Gist by ${owner}\n\n`;
+		if (gist.description) md += `${gist.description}\n\n`;
+		md += `**Created:** ${gist.created_at} · **Updated:** ${gist.updated_at}\n`;
+		md += `**Files:** ${files.length}\n\n`;
+
+		for (const file of files) {
+			const lang = file.language?.toLowerCase() || "";
+			md += `---\n\n## ${file.filename}\n\n`;
+			md += `\`\`\`${lang}\n${file.content}\n\`\`\`\n\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "github-gist",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via GitHub API"],
+		};
+	} catch {}
+
+	return null;
+};