@oh-my-pi/pi-coding-agent 3.24.0 → 3.30.0

package/src/core/tools/web-fetch-handlers/stackexchange.test.ts

@@ -0,0 +1,120 @@
+import { describe, expect, it } from "bun:test";
+import { handleStackOverflow } from "./stackoverflow";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleStackOverflow", () => {
+	it("returns null for non-SE URLs", async () => {
+		const result = await handleStackOverflow("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for SE site without question path", async () => {
+		const result = await handleStackOverflow("https://stackoverflow.com/tags", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for SE user profile URLs", async () => {
+		const result = await handleStackOverflow("https://stackoverflow.com/users/1", 20);
+		expect(result).toBeNull();
+	});
+
+	// stackoverflow.com - "What is a NullPointerException" (classic, highly voted)
+	it("fetches stackoverflow.com question", async () => {
+		const result = await handleStackOverflow(
+			"https://stackoverflow.com/questions/218384/what-is-a-nullpointerexception-and-how-do-i-fix-it",
+			20,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("stackexchange");
+		expect(result?.content).toContain("NullPointerException");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+		expect(result?.notes?.[0]).toContain("site=stackoverflow");
+	});
+
+	// unix.stackexchange.com - "Why does my shell script choke on whitespace" (classic)
+	it("fetches unix.stackexchange.com question", async () => {
+		const result = await handleStackOverflow(
+			"https://unix.stackexchange.com/questions/131766/why-does-my-shell-script-choke-on-whitespace-or-other-special-characters",
+			20,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("stackexchange");
+		expect(result?.content).toContain("whitespace");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.notes?.[0]).toContain("site=unix");
+	});
+
+	// superuser.com - "What are PATH and other environment variables" (stable)
+	it("fetches superuser.com question", async () => {
+		const result = await handleStackOverflow(
+			"https://superuser.com/questions/284342/what-are-path-and-other-environment-variables-and-how-can-i-set-or-use-them",
+			20,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("stackexchange");
+		expect(result?.content).toContain("PATH");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.notes?.[0]).toContain("site=superuser");
+	});
+
+	// askubuntu.com - "What is the difference between apt and apt-get" (iconic)
+	it("fetches askubuntu.com question", async () => {
+		const result = await handleStackOverflow(
+			"https://askubuntu.com/questions/445384/what-is-the-difference-between-apt-and-apt-get",
+			20,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("stackexchange");
+		expect(result?.content).toContain("apt");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.notes?.[0]).toContain("site=askubuntu");
+	});
+
+	// serverfault.com - "What is a reverse proxy" (stable sysadmin topic)
+	it("fetches serverfault.com question", async () => {
+		const result = await handleStackOverflow(
+			"https://serverfault.com/questions/127021/what-is-the-difference-between-a-proxy-and-a-reverse-proxy",
+			20,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("stackexchange");
+		expect(result?.content).toMatch(/proxy/i);
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.notes?.[0]).toContain("site=serverfault");
+	});
+
+	// Test with www. prefix
+	it("handles www.stackoverflow.com URLs", async () => {
+		const result = await handleStackOverflow(
+			"https://www.stackoverflow.com/questions/218384/what-is-a-nullpointerexception",
+			20,
+		);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("stackexchange");
+	});
+
+	// Verify response structure
+	it("returns complete response structure", async () => {
+		const result = await handleStackOverflow("https://stackoverflow.com/questions/218384", 20);
+		expect(result).not.toBeNull();
+		expect(result).toHaveProperty("url");
+		expect(result).toHaveProperty("finalUrl");
+		expect(result).toHaveProperty("contentType", "text/markdown");
+		expect(result).toHaveProperty("method", "stackexchange");
+		expect(result).toHaveProperty("content");
+		expect(result).toHaveProperty("fetchedAt");
+		expect(result).toHaveProperty("truncated");
+		expect(result).toHaveProperty("notes");
+		// Content should have question structure
+		expect(result?.content).toContain("# ");
+		expect(result?.content).toContain("Score:");
+		expect(result?.content).toContain("Tags:");
+	});
+});

package/src/core/tools/web-fetch-handlers/stackoverflow.ts

@@ -0,0 +1,123 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface SOQuestion {
+	title: string;
+	body: string;
+	score: number;
+	owner: { display_name: string };
+	creation_date: number;
+	tags: string[];
+	answer_count: number;
+	is_answered: boolean;
+}
+
+interface SOAnswer {
+	body: string;
+	score: number;
+	is_accepted: boolean;
+	owner: { display_name: string };
+	creation_date: number;
+}
+
+// Standalone SE network sites (not *.stackexchange.com subdomains)
+const STANDALONE_SE_SITES: Record<string, string> = {
+	"stackoverflow.com": "stackoverflow",
+	"superuser.com": "superuser",
+	"serverfault.com": "serverfault",
+	"askubuntu.com": "askubuntu",
+	"mathoverflow.net": "mathoverflow",
+	"stackapps.com": "stackapps",
+};
+
+/**
+ * Extract the API site parameter from a Stack Exchange hostname
+ * Returns null if not a recognized SE site
+ */
+function getSiteParam(hostname: string): string | null {
+	// Remove www. prefix if present
+	const host = hostname.replace(/^www\./, "");
+
+	// Check standalone sites first
+	if (STANDALONE_SE_SITES[host]) {
+		return STANDALONE_SE_SITES[host];
+	}
+
+	// Handle *.stackexchange.com subdomains (e.g., unix.stackexchange.com → unix)
+	const seMatch = host.match(/^([a-z0-9-]+)\.stackexchange\.com$/);
+	if (seMatch) {
+		return seMatch[1];
+	}
+
+	return null;
+}
+
+/**
+ * Handle Stack Exchange network URLs via API
+ * Supports stackoverflow.com, *.stackexchange.com, superuser.com, serverfault.com, askubuntu.com, etc.
+ */
+export const handleStackOverflow: SpecialHandler = async (
+	url: string,
+	timeout: number,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		const site = getSiteParam(parsed.hostname);
+		if (!site) return null;
+
+		// Extract question ID from URL patterns like /questions/12345/...
+		const match = parsed.pathname.match(/\/questions\/(\d+)/);
+		if (!match) return null;
+
+		const questionId = match[1];
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch question with answers
+		const apiUrl = `https://api.stackexchange.com/2.3/questions/${questionId}?order=desc&sort=votes&site=${site}&filter=withbody`;
+		const qResult = await loadPage(apiUrl, { timeout });
+
+		if (!qResult.ok) return null;
+
+		const qData = JSON.parse(qResult.content) as { items: SOQuestion[] };
+		if (!qData.items?.length) return null;
+
+		const question = qData.items[0];
+
+		let md = `# ${question.title}\n\n`;
+		md += `**Score:** ${question.score} · **Answers:** ${question.answer_count}`;
+		md += question.is_answered ? " (Answered)" : "";
+		md += `\n**Tags:** ${question.tags.join(", ")}\n`;
+		md += `**Asked by:** ${question.owner.display_name} · ${new Date(question.creation_date * 1000).toISOString().split("T")[0]}\n\n`;
+		md += `---\n\n## Question\n\n${htmlToBasicMarkdown(question.body)}\n\n`;
+
+		// Fetch answers
+		const aUrl = `https://api.stackexchange.com/2.3/questions/${questionId}/answers?order=desc&sort=votes&site=${site}&filter=withbody`;
+		const aResult = await loadPage(aUrl, { timeout });
+
+		if (aResult.ok) {
+			const aData = JSON.parse(aResult.content) as { items: SOAnswer[] };
+			if (aData.items?.length) {
+				md += `---\n\n## Answers\n\n`;
+				for (const answer of aData.items.slice(0, 5)) {
+					const accepted = answer.is_accepted ? " (Accepted)" : "";
+					md += `### Score: ${answer.score}${accepted} · by ${answer.owner.display_name}\n\n`;
+					md += `${htmlToBasicMarkdown(answer.body)}\n\n---\n\n`;
+				}
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "stackexchange",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: [`Fetched via Stack Exchange API (site=${site})`],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-fetch-handlers/standards.test.ts

@@ -0,0 +1,122 @@
+import { describe, expect, it } from "bun:test";
+import { handleCheatSh } from "./cheatsh";
+import { handleRfc } from "./rfc";
+import { handleTldr } from "./tldr";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleRfc", () => {
+	it("returns null for non-RFC URLs", async () => {
+		const result = await handleRfc("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for non-matching RFC domains", async () => {
+		const result = await handleRfc("https://www.ietf.org/about/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches RFC 2616 (HTTP/1.1)", async () => {
+		const result = await handleRfc("https://www.rfc-editor.org/rfc/rfc2616", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("rfc");
+		expect(result?.content).toContain("HTTP/1.1");
+		expect(result?.content).toContain("Hypertext Transfer Protocol");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches RFC 2616 via datatracker URL", async () => {
+		const result = await handleRfc("https://datatracker.ietf.org/doc/rfc2616/", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("rfc");
+		expect(result?.content).toContain("HTTP/1.1");
+	});
+
+	it("fetches RFC 2616 via tools.ietf.org URL", async () => {
+		const result = await handleRfc("https://tools.ietf.org/html/rfc2616", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("rfc");
+		expect(result?.content).toContain("HTTP/1.1");
+	});
+
+	it("fetches RFC 793 (TCP)", async () => {
+		const result = await handleRfc("https://www.rfc-editor.org/rfc/rfc793", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("rfc");
+		expect(result?.content).toContain("Transmission Control Protocol");
+	});
+});
+
+describe.skipIf(SKIP)("handleCheatSh", () => {
+	it("returns null for non-cheat.sh URLs", async () => {
+		const result = await handleCheatSh("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for empty topic", async () => {
+		const result = await handleCheatSh("https://cheat.sh/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches curl cheatsheet", async () => {
+		const result = await handleCheatSh("https://cheat.sh/curl", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("cheat.sh");
+		expect(result?.content).toContain("curl");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches tar cheatsheet", async () => {
+		const result = await handleCheatSh("https://cheat.sh/tar", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("cheat.sh");
+		expect(result?.content).toContain("tar");
+	});
+
+	it("fetches cheatsheet via cht.sh alias", async () => {
+		const result = await handleCheatSh("https://cht.sh/curl", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("cheat.sh");
+		expect(result?.content).toContain("curl");
+	});
+});
+
+describe.skipIf(SKIP)("handleTldr", () => {
+	it("returns null for non-tldr URLs", async () => {
+		const result = await handleTldr("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for nested paths", async () => {
+		const result = await handleTldr("https://tldr.sh/nested/path", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches git tldr page", async () => {
+		const result = await handleTldr("https://tldr.sh/git", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("tldr");
+		expect(result?.content).toContain("git");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches curl tldr page", async () => {
+		const result = await handleTldr("https://tldr.sh/curl", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("tldr");
+		expect(result?.content).toContain("curl");
+	});
+
+	it("fetches via tldr.ostera.io alias", async () => {
+		const result = await handleTldr("https://tldr.ostera.io/git", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("tldr");
+		expect(result?.content).toContain("git");
+	});
+});

package/src/core/tools/web-fetch-handlers/terraform.ts

@@ -0,0 +1,296 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface TerraformModule {
+	id: string;
+	namespace: string;
+	name: string;
+	provider: string;
+	version: string;
+	description?: string;
+	source?: string;
+	published_at?: string;
+	downloads: number;
+	verified?: boolean;
+	root?: {
+		inputs?: Array<{
+			name: string;
+			type?: string;
+			description?: string;
+			default?: unknown;
+			required?: boolean;
+		}>;
+		outputs?: Array<{
+			name: string;
+			description?: string;
+		}>;
+		dependencies?: Array<{
+			name: string;
+			source: string;
+			version?: string;
+		}>;
+		resources?: Array<{
+			name: string;
+			type: string;
+		}>;
+	};
+	submodules?: Array<{
+		path: string;
+		name: string;
+	}>;
+}
+
+interface TerraformProvider {
+	id: string;
+	namespace: string;
+	name: string;
+	alias?: string;
+	version: string;
+	description?: string;
+	source?: string;
+	published_at?: string;
+	downloads: number;
+	tier?: string;
+	logo_url?: string;
+	docs?: Array<{
+		id: string;
+		title: string;
+		path: string;
+		slug: string;
+		category: string;
+	}>;
+}
+
+/**
+ * Handle Terraform Registry URLs via API
+ */
+export const handleTerraform: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("registry.terraform.io")) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Match module URL: /modules/{namespace}/{name}/{provider}
+		const moduleMatch = parsed.pathname.match(/^\/modules\/([^/]+)\/([^/]+)\/([^/]+)/);
+		if (moduleMatch) {
+			const [, namespace, name, provider] = moduleMatch;
+			return await handleModuleUrl(url, namespace, name, provider, timeout, fetchedAt);
+		}
+
+		// Match provider URL: /providers/{namespace}/{type}
+		const providerMatch = parsed.pathname.match(/^\/providers\/([^/]+)\/([^/]+)/);
+		if (providerMatch) {
+			const [, namespace, type] = providerMatch;
+			return await handleProviderUrl(url, namespace, type, timeout, fetchedAt);
+		}
+
+		return null;
+	} catch {}
+
+	return null;
+};
+
+async function handleModuleUrl(
+	url: string,
+	namespace: string,
+	name: string,
+	provider: string,
+	timeout: number,
+	fetchedAt: string,
+): Promise<RenderResult | null> {
+	const apiUrl = `https://registry.terraform.io/v1/modules/${namespace}/${name}/${provider}`;
+	const result = await loadPage(apiUrl, {
+		timeout,
+		headers: { Accept: "application/json" },
+	});
+
+	if (!result.ok) return null;
+
+	let mod: TerraformModule;
+	try {
+		mod = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	let md = `# ${mod.namespace}/${mod.name}/${mod.provider}\n\n`;
+
+	if (mod.description) md += `${mod.description}\n\n`;
+
+	// Metadata line
+	md += `**Version:** ${mod.version}`;
+	if (mod.verified) md += " ✓ Verified";
+	md += `\n`;
+	md += `**Downloads:** ${formatCount(mod.downloads)}\n`;
+	if (mod.published_at) {
+		md += `**Published:** ${new Date(mod.published_at).toLocaleDateString()}\n`;
+	}
+	if (mod.source) {
+		md += `**Source:** ${mod.source}\n`;
+	}
+	md += "\n";
+
+	// Usage example
+	md += `## Usage\n\n\`\`\`hcl\nmodule "${mod.name}" {\n  source  = "${mod.namespace}/${mod.name}/${mod.provider}"\n  version = "${mod.version}"\n}\n\`\`\`\n\n`;
+
+	// Inputs
+	const inputs = mod.root?.inputs;
+	if (inputs && inputs.length > 0) {
+		md += `## Inputs (${inputs.length})\n\n`;
+		md += "| Name | Type | Required | Description |\n";
+		md += "|------|------|----------|-------------|\n";
+		for (const input of inputs.slice(0, 30)) {
+			const required = (input.required ?? input.default === undefined) ? "Yes" : "No";
+			const type = input.type ?? "any";
+			const desc = (input.description ?? "").replace(/\|/g, "\\|").replace(/\n/g, " ").slice(0, 80);
+			md += `| ${input.name} | \`${type}\` | ${required} | ${desc} |\n`;
+		}
+		if (inputs.length > 30) {
+			md += `\n*... and ${inputs.length - 30} more inputs*\n`;
+		}
+		md += "\n";
+	}
+
+	// Outputs
+	const outputs = mod.root?.outputs;
+	if (outputs && outputs.length > 0) {
+		md += `## Outputs (${outputs.length})\n\n`;
+		for (const output of outputs.slice(0, 20)) {
+			md += `- **${output.name}**`;
+			if (output.description) md += `: ${output.description.replace(/\n/g, " ").slice(0, 100)}`;
+			md += "\n";
+		}
+		if (outputs.length > 20) {
+			md += `\n*... and ${outputs.length - 20} more outputs*\n`;
+		}
+		md += "\n";
+	}
+
+	// Dependencies
+	const deps = mod.root?.dependencies;
+	if (deps && deps.length > 0) {
+		md += `## Dependencies (${deps.length})\n\n`;
+		for (const dep of deps.slice(0, 15)) {
+			md += `- **${dep.name}**: ${dep.source}`;
+			if (dep.version) md += ` (${dep.version})`;
+			md += "\n";
+		}
+		if (deps.length > 15) {
+			md += `\n*... and ${deps.length - 15} more dependencies*\n`;
+		}
+		md += "\n";
+	}
+
+	// Resources
+	const resources = mod.root?.resources;
+	if (resources && resources.length > 0) {
+		md += `## Resources (${resources.length})\n\n`;
+		for (const res of resources.slice(0, 20)) {
+			md += `- \`${res.type}\` (${res.name})\n`;
+		}
+		if (resources.length > 20) {
+			md += `\n*... and ${resources.length - 20} more resources*\n`;
+		}
+		md += "\n";
+	}
+
+	// Submodules
+	if (mod.submodules && mod.submodules.length > 0) {
+		md += `## Submodules (${mod.submodules.length})\n\n`;
+		for (const sub of mod.submodules.slice(0, 10)) {
+			md += `- **${sub.name}**: \`${sub.path}\`\n`;
+		}
+		if (mod.submodules.length > 10) {
+			md += `\n*... and ${mod.submodules.length - 10} more submodules*\n`;
+		}
+	}
+
+	const output = finalizeOutput(md);
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/markdown",
+		method: "terraform",
+		content: output.content,
+		fetchedAt,
+		truncated: output.truncated,
+		notes: ["Fetched via Terraform Registry API"],
+	};
+}
+
+async function handleProviderUrl(
+	url: string,
+	namespace: string,
+	type: string,
+	timeout: number,
+	fetchedAt: string,
+): Promise<RenderResult | null> {
+	const apiUrl = `https://registry.terraform.io/v1/providers/${namespace}/${type}`;
+	const result = await loadPage(apiUrl, {
+		timeout,
+		headers: { Accept: "application/json" },
+	});
+
+	if (!result.ok) return null;
+
+	let provider: TerraformProvider;
+	try {
+		provider = JSON.parse(result.content);
+	} catch {
+		return null;
+	}
+
+	let md = `# ${provider.namespace}/${provider.name}\n\n`;
+
+	if (provider.description) md += `${provider.description}\n\n`;
+
+	// Metadata
+	md += `**Version:** ${provider.version}\n`;
+	if (provider.tier) md += `**Tier:** ${provider.tier}\n`;
+	md += `**Downloads:** ${formatCount(provider.downloads)}\n`;
+	if (provider.published_at) {
+		md += `**Published:** ${new Date(provider.published_at).toLocaleDateString()}\n`;
+	}
+	if (provider.source) {
+		md += `**Source:** ${provider.source}\n`;
+	}
+	md += "\n";
+
+	// Usage example
+	md += `## Usage\n\n\`\`\`hcl\nterraform {\n  required_providers {\n    ${provider.name} = {\n      source  = "${provider.namespace}/${provider.name}"\n      version = "~> ${provider.version}"\n    }\n  }\n}\n\nprovider "${provider.name}" {\n  # Configuration options\n}\n\`\`\`\n\n`;
+
+	// Documentation summary
+	if (provider.docs && provider.docs.length > 0) {
+		const categories = new Map<string, typeof provider.docs>();
+		for (const doc of provider.docs) {
+			const cat = doc.category || "other";
+			if (!categories.has(cat)) categories.set(cat, []);
+			categories.get(cat)!.push(doc);
+		}
+
+		md += `## Documentation\n\n`;
+		for (const [category, docs] of categories) {
+			md += `### ${category.charAt(0).toUpperCase() + category.slice(1)} (${docs.length})\n\n`;
+			for (const doc of docs.slice(0, 15)) {
+				md += `- [${doc.title}](https://registry.terraform.io/providers/${namespace}/${type}/latest/docs/${doc.category}/${doc.slug})\n`;
+			}
+			if (docs.length > 15) {
+				md += `\n*... and ${docs.length - 15} more*\n`;
+			}
+			md += "\n";
+		}
+	}
+
+	const output = finalizeOutput(md);
+	return {
+		url,
+		finalUrl: url,
+		contentType: "text/markdown",
+		method: "terraform",
+		content: output.content,
+		fetchedAt,
+		truncated: output.truncated,
+		notes: ["Fetched via Terraform Registry API"],
+	};
+}