@oh-my-pi/pi-coding-agent 3.30.0 → 3.31.0

package/src/core/tools/web-scrapers/sourcegraph.ts

@@ -0,0 +1,373 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+const GRAPHQL_ENDPOINT = "https://sourcegraph.com/.api/graphql";
+const GRAPHQL_HEADERS = {
+	Accept: "application/json",
+	"Content-Type": "application/json",
+};
+
+type SourcegraphTarget =
+	| { type: "search"; query: string }
+	| { type: "repo"; repoName: string; rev?: string }
+	| { type: "file"; repoName: string; rev?: string; filePath: string };
+
+interface SourcegraphRepository {
+	name: string;
+	url: string;
+	description?: string | null;
+	defaultBranch?: { name: string } | null;
+}
+
+interface RepoQueryData {
+	repository?: SourcegraphRepository | null;
+}
+
+interface RepoFileQueryData {
+	repository?:
+		| (SourcegraphRepository & {
+				commit?: {
+					blob?: { content?: string | null } | null;
+				} | null;
+		  })
+		| null;
+}
+
+interface SearchQueryData {
+	search?: {
+		results?: {
+			results?: SearchResultItem[] | null;
+			matchCount?: number | null;
+			limitHit?: boolean | null;
+		} | null;
+	} | null;
+}
+
+interface FileMatchResult {
+	__typename: "FileMatch";
+	repository?: { name?: string | null; url?: string | null } | null;
+	file?: { path?: string | null; url?: string | null } | null;
+	lineMatches?: Array<{ preview?: string | null; lineNumber?: number | null }> | null;
+}
+
+interface RepositoryResult {
+	__typename: "Repository";
+	name?: string | null;
+	url?: string | null;
+}
+
+type SearchResultItem = FileMatchResult | RepositoryResult | { __typename: string };
+
+const REPO_QUERY = `query Repo($name: String!) {
+	repository(name: $name) {
+		name
+		url
+		description
+		defaultBranch {
+			name
+		}
+	}
+}`;
+
+const REPO_FILE_QUERY = `query RepoFile($name: String!, $path: String!, $rev: String!) {
+	repository(name: $name) {
+		name
+		url
+		description
+		defaultBranch {
+			name
+		}
+		commit(rev: $rev) {
+			blob(path: $path) {
+				content
+			}
+		}
+	}
+}`;
+
+const SEARCH_QUERY = `query Search($query: String!) {
+	search(query: $query, version: V2) {
+		results {
+			results {
+				__typename
+				... on FileMatch {
+					repository {
+						name
+						url
+					}
+					file {
+						path
+						url
+					}
+					lineMatches {
+						preview
+						lineNumber
+					}
+				}
+				... on Repository {
+					name
+					url
+				}
+			}
+			matchCount
+			limitHit
+		}
+	}
+}`;
+
+function parseSourcegraphUrl(url: string): SourcegraphTarget | null {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "sourcegraph.com" && parsed.hostname !== "www.sourcegraph.com") return null;
+
+		if (parsed.pathname.startsWith("/search")) {
+			const query = parsed.searchParams.get("q")?.trim();
+			if (!query) return null;
+			return { type: "search", query };
+		}
+
+		const parts = parsed.pathname
+			.split("/")
+			.filter(Boolean)
+			.map((part) => decodeURIComponent(part));
+		if (parts.length < 3) return null;
+
+		const hyphenIndex = parts.indexOf("-");
+		const repoParts = hyphenIndex === -1 ? parts : parts.slice(0, hyphenIndex);
+		if (repoParts.length < 3) return null;
+
+		const lastRepoPart = repoParts[repoParts.length - 1];
+		const atIndex = lastRepoPart.indexOf("@");
+		let rev: string | undefined;
+		let repoTail = lastRepoPart;
+		if (atIndex > 0) {
+			repoTail = lastRepoPart.slice(0, atIndex);
+			rev = lastRepoPart.slice(atIndex + 1) || undefined;
+		}
+
+		repoParts[repoParts.length - 1] = repoTail;
+		const repoName = repoParts.join("/");
+
+		if (hyphenIndex !== -1 && parts[hyphenIndex + 1] === "blob") {
+			const filePath = parts.slice(hyphenIndex + 2).join("/");
+			if (!filePath) return null;
+			return { type: "file", repoName, rev, filePath };
+		}
+
+		return { type: "repo", repoName, rev };
+	} catch {
+		return null;
+	}
+}
+
+function safeParseJson<T>(content: string): T | null {
+	try {
+		return JSON.parse(content) as T;
+	} catch {
+		return null;
+	}
+}
+
+async function fetchGraphql<T>(
+	query: string,
+	variables: Record<string, unknown>,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<T | null> {
+	const body = JSON.stringify({ query, variables });
+	const result = await loadPage(GRAPHQL_ENDPOINT, {
+		timeout,
+		headers: GRAPHQL_HEADERS,
+		method: "POST",
+		body,
+		signal,
+	});
+	if (!result.ok) return null;
+
+	const parsed = safeParseJson<{ data?: T; errors?: unknown }>(result.content);
+	if (!parsed?.data) return null;
+	if (Array.isArray(parsed.errors) && parsed.errors.length > 0) return null;
+	return parsed.data;
+}
+
+function isFileMatchResult(result: SearchResultItem): result is FileMatchResult {
+	return result.__typename === "FileMatch";
+}
+
+function isRepositoryResult(result: SearchResultItem): result is RepositoryResult {
+	return result.__typename === "Repository";
+}
+
+function formatRepoMarkdown(repo: SourcegraphRepository): string {
+	let md = `# ${repo.name}\n\n`;
+	if (repo.description) md += `${repo.description}\n\n`;
+	md += `**URL:** ${repo.url}\n`;
+	if (repo.defaultBranch?.name) md += `**Default branch:** ${repo.defaultBranch.name}\n`;
+	return md;
+}
+
+async function renderRepo(
+	repoName: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<{ content: string; ok: boolean }> {
+	const data = await fetchGraphql<RepoQueryData>(REPO_QUERY, { name: repoName }, timeout, signal);
+	if (!data?.repository) return { content: "", ok: false };
+
+	return { content: formatRepoMarkdown(data.repository), ok: true };
+}
+
+async function renderFile(
+	repoName: string,
+	filePath: string,
+	rev: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<{ content: string; ok: boolean }> {
+	const data = await fetchGraphql<RepoFileQueryData>(
+		REPO_FILE_QUERY,
+		{ name: repoName, path: filePath, rev },
+		timeout,
+		signal,
+	);
+	const repo = data?.repository;
+	const content = repo?.commit?.blob?.content ?? null;
+	if (!repo || content === null) return { content: "", ok: false };
+
+	let md = `${formatRepoMarkdown(repo)}\n`;
+	md += `**Path:** ${filePath}\n`;
+	md += `**Revision:** ${rev}\n\n`;
+	md += `---\n\n## File\n\n`;
+	md += "```text\n";
+	md += `${content}\n`;
+	md += "```\n";
+	return { content: md, ok: true };
+}
+
+async function renderSearch(
+	query: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<{ content: string; ok: boolean }> {
+	const data = await fetchGraphql<SearchQueryData>(SEARCH_QUERY, { query }, timeout, signal);
+	const resultsData = data?.search?.results;
+	if (!resultsData) return { content: "", ok: false };
+	const results = resultsData.results ?? [];
+
+	let md = "# Sourcegraph Search\n\n";
+	md += `**Query:** \`${query}\`\n`;
+	if (typeof resultsData?.matchCount === "number") {
+		md += `**Matches:** ${resultsData.matchCount}\n`;
+	}
+	if (typeof resultsData?.limitHit === "boolean") {
+		md += `**Limit hit:** ${resultsData.limitHit ? "yes" : "no"}\n`;
+	}
+	md += "\n";
+
+	if (!results || results.length === 0) {
+		md += "_No results._\n";
+		return { content: md, ok: true };
+	}
+
+	const maxResults = 10;
+	md += "## Results\n\n";
+	for (const result of results.slice(0, maxResults)) {
+		if (isFileMatchResult(result)) {
+			const repoName = result.repository?.name ?? "unknown";
+			const filePath = result.file?.path ?? "unknown";
+			md += `### ${repoName}/${filePath}\n\n`;
+			if (result.repository?.url) md += `**Repository:** ${result.repository.url}\n`;
+			if (result.file?.url) md += `**File:** ${result.file.url}\n`;
+
+			const lineMatches = result.lineMatches ?? [];
+			if (lineMatches.length > 0) {
+				md += "\n```text\n";
+				for (const line of lineMatches.slice(0, 5)) {
+					const preview = (line.preview ?? "").replace(/\n/g, " ").trim();
+					const lineNumber = line.lineNumber ?? 0;
+					md += `L${lineNumber}: ${preview}\n`;
+				}
+				md += "```\n\n";
+			}
+			continue;
+		}
+
+		if (isRepositoryResult(result)) {
+			const name = result.name ?? "unknown";
+			md += `### ${name}\n\n`;
+			if (result.url) md += `**Repository:** ${result.url}\n`;
+			md += "\n";
+		}
+	}
+
+	if (results.length > maxResults) {
+		md += `... and ${results.length - maxResults} more results\n`;
+	}
+
+	return { content: md, ok: true };
+}
+
+export const handleSourcegraph: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const target = parseSourcegraphUrl(url);
+		if (!target) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const notes = ["Fetched via Sourcegraph GraphQL API"];
+
+		switch (target.type) {
+			case "search": {
+				const result = await renderSearch(target.query, timeout, signal);
+				if (!result.ok) return null;
+				const output = finalizeOutput(result.content);
+				return {
+					url,
+					finalUrl: url,
+					contentType: "text/markdown",
+					method: "sourcegraph-search",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes,
+				};
+			}
+			case "file": {
+				const rev = target.rev ?? "HEAD";
+				const result = await renderFile(target.repoName, target.filePath, rev, timeout, signal);
+				if (!result.ok) return null;
+				const output = finalizeOutput(result.content);
+				return {
+					url,
+					finalUrl: url,
+					contentType: "text/markdown",
+					method: "sourcegraph-file",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes,
+				};
+			}
+			case "repo": {
+				const result = await renderRepo(target.repoName, timeout, signal);
+				if (!result.ok) return null;
+				const output = finalizeOutput(result.content);
+				return {
+					url,
+					finalUrl: url,
+					contentType: "text/markdown",
+					method: "sourcegraph-repo",
+					content: output.content,
+					fetchedAt,
+					truncated: output.truncated,
+					notes,
+				};
+			}
+		}
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/spdx.ts

@@ -0,0 +1,121 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, htmlToBasicMarkdown, loadPage } from "./types";
+
+interface SpdxCrossRef {
+	url?: string;
+	isValid?: boolean;
+	isLive?: boolean;
+	match?: string;
+	order?: number;
+}
+
+interface SpdxLicense {
+	licenseId: string;
+	name: string;
+	isOsiApproved?: boolean;
+	isFsfLibre?: boolean;
+	licenseText?: string;
+	licenseTextHtml?: string;
+	seeAlso?: string[];
+	crossRef?: SpdxCrossRef[];
+	comment?: string;
+	licenseComments?: string;
+}
+
+function formatYesNo(value?: boolean): string {
+	if (value === true) return "Yes";
+	if (value === false) return "No";
+	return "Unknown";
+}
+
+function collectCrossReferences(license: SpdxLicense): string[] {
+	const ordered = (license.crossRef ?? [])
+		.filter((ref) => ref.url)
+		.sort((a, b) => (a.order ?? 0) - (b.order ?? 0))
+		.map((ref) => ref.url as string);
+
+	const seeAlso = (license.seeAlso ?? []).filter((url) => url);
+	const combined = [...ordered, ...seeAlso];
+	return combined.filter((url, index) => combined.indexOf(url) === index);
+}
+
+/**
+ * Handle SPDX license URLs via SPDX JSON API
+ */
+export const handleSpdx: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "spdx.org" && parsed.hostname !== "www.spdx.org") return null;
+
+		const match = parsed.pathname.match(/^\/licenses\/([^/]+?)(?:\.html)?\/?$/i);
+		if (!match) return null;
+
+		const licenseId = decodeURIComponent(match[1]);
+		if (!licenseId) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://spdx.org/licenses/${encodeURIComponent(licenseId)}.json`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		let license: SpdxLicense;
+		try {
+			license = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const title = license.name || license.licenseId || licenseId;
+		let md = `# ${title}\n\n`;
+
+		md += `**License ID:** ${license.licenseId ? `\`${license.licenseId}\`` : `\`${licenseId}\``}\n`;
+		md += `**OSI Approved:** ${formatYesNo(license.isOsiApproved)}\n`;
+		md += `**FSF Libre:** ${formatYesNo(license.isFsfLibre)}\n`;
+
+		const description = license.licenseComments ?? license.comment;
+		if (description) {
+			md += `\n## Description\n\n${description}\n`;
+		}
+
+		const crossReferences = collectCrossReferences(license);
+		if (crossReferences.length) {
+			md += `\n## Cross References\n\n`;
+			for (const ref of crossReferences) {
+				md += `- ${ref}\n`;
+			}
+		}
+
+		const licenseText = license.licenseText
+			? license.licenseText
+			: license.licenseTextHtml
+				? htmlToBasicMarkdown(license.licenseTextHtml)
+				: null;
+
+		if (licenseText) {
+			md += `\n## License Text\n\n\`\`\`\n${licenseText}\n\`\`\`\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "spdx-api",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via SPDX license API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/{web-fetch-handlers → web-scrapers}/spotify.ts

@@ -157,7 +157,7 @@ function formatOutput(contentType: string, oEmbed: SpotifyOEmbedResponse, og: Op
 	return sections.join("\n");
 }
 
-export const handleSpotify: SpecialHandler = async (url: string, timeout: number) => {
+export const handleSpotify: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
 	// Check if this is a Spotify URL
 	if (!url.includes("open.spotify.com/")) {
 		return null;
@@ -175,7 +175,7 @@ export const handleSpotify: SpecialHandler = async (url: string, timeout: number
 	// Fetch oEmbed data
 	try {
 		const oEmbedUrl = `https://open.spotify.com/oembed?url=${encodeURIComponent(url)}`;
-		const response = await loadPage(oEmbedUrl, { timeout });
+		const response = await loadPage(oEmbedUrl, { timeout, signal });
 
 		if (response.ok) {
 			oEmbedData = JSON.parse(response.content) as SpotifyOEmbedResponse;
@@ -189,7 +189,7 @@ export const handleSpotify: SpecialHandler = async (url: string, timeout: number
 
 	// Fetch page HTML for Open Graph metadata
 	try {
-		const pageResponse = await loadPage(url, { timeout });
+		const pageResponse = await loadPage(url, { timeout, signal });
 
 		if (pageResponse.ok) {
 			ogData = parseOpenGraph(pageResponse.content);

package/src/core/tools/{web-fetch-handlers → web-scrapers}/stackoverflow.ts

@@ -59,6 +59,7 @@ function getSiteParam(hostname: string): string | null {
 export const handleStackOverflow: SpecialHandler = async (
 	url: string,
 	timeout: number,
+	signal?: AbortSignal,
 ): Promise<RenderResult | null> => {
 	try {
 		const parsed = new URL(url);
@@ -74,7 +75,7 @@ export const handleStackOverflow: SpecialHandler = async (
 
 		// Fetch question with answers
 		const apiUrl = `https://api.stackexchange.com/2.3/questions/${questionId}?order=desc&sort=votes&site=${site}&filter=withbody`;
-		const qResult = await loadPage(apiUrl, { timeout });
+		const qResult = await loadPage(apiUrl, { timeout, signal });
 
 		if (!qResult.ok) return null;
 
@@ -92,7 +93,7 @@ export const handleStackOverflow: SpecialHandler = async (
 
 		// Fetch answers
 		const aUrl = `https://api.stackexchange.com/2.3/questions/${questionId}/answers?order=desc&sort=votes&site=${site}&filter=withbody`;
-		const aResult = await loadPage(aUrl, { timeout });
+		const aResult = await loadPage(aUrl, { timeout, signal });
 
 		if (aResult.ok) {
 			const aData = JSON.parse(aResult.content) as { items: SOAnswer[] };

package/src/core/tools/{web-fetch-handlers → web-scrapers}/terraform.ts

@@ -64,7 +64,11 @@ interface TerraformProvider {
 /**
  * Handle Terraform Registry URLs via API
  */
-export const handleTerraform: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+export const handleTerraform: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
 	try {
 		const parsed = new URL(url);
 		if (!parsed.hostname.includes("registry.terraform.io")) return null;
@@ -75,14 +79,14 @@ export const handleTerraform: SpecialHandler = async (url: string, timeout: numb
 		const moduleMatch = parsed.pathname.match(/^\/modules\/([^/]+)\/([^/]+)\/([^/]+)/);
 		if (moduleMatch) {
 			const [, namespace, name, provider] = moduleMatch;
-			return await handleModuleUrl(url, namespace, name, provider, timeout, fetchedAt);
+			return await handleModuleUrl(url, namespace, name, provider, timeout, signal, fetchedAt);
 		}
 
 		// Match provider URL: /providers/{namespace}/{type}
 		const providerMatch = parsed.pathname.match(/^\/providers\/([^/]+)\/([^/]+)/);
 		if (providerMatch) {
 			const [, namespace, type] = providerMatch;
-			return await handleProviderUrl(url, namespace, type, timeout, fetchedAt);
+			return await handleProviderUrl(url, namespace, type, timeout, signal, fetchedAt);
 		}
 
 		return null;
@@ -97,11 +101,13 @@ async function handleModuleUrl(
 	name: string,
 	provider: string,
 	timeout: number,
+	signal: AbortSignal | undefined,
 	fetchedAt: string,
 ): Promise<RenderResult | null> {
 	const apiUrl = `https://registry.terraform.io/v1/modules/${namespace}/${name}/${provider}`;
 	const result = await loadPage(apiUrl, {
 		timeout,
+		signal,
 		headers: { Accept: "application/json" },
 	});
 
@@ -224,11 +230,13 @@ async function handleProviderUrl(
 	namespace: string,
 	type: string,
 	timeout: number,
+	signal: AbortSignal | undefined,
 	fetchedAt: string,
 ): Promise<RenderResult | null> {
 	const apiUrl = `https://registry.terraform.io/v1/providers/${namespace}/${type}`;
 	const result = await loadPage(apiUrl, {
 		timeout,
+		signal,
 		headers: { Accept: "application/json" },
 	});
 

package/src/core/tools/{web-fetch-handlers → web-scrapers}/tldr.ts

@@ -9,7 +9,11 @@ const PLATFORMS = ["common", "linux", "osx"] as const;
  * - https://tldr.sh/{command}
  * - https://tldr.ostera.io/{command}
  */
-export const handleTldr: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+export const handleTldr: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
 	try {
 		const parsed = new URL(url);
 		if (parsed.hostname !== "tldr.sh" && parsed.hostname !== "tldr.ostera.io") return null;
@@ -23,7 +27,7 @@ export const handleTldr: SpecialHandler = async (url: string, timeout: number):
 		// Try platforms in order: common, linux, osx
 		for (const platform of PLATFORMS) {
 			const rawUrl = `${TLDR_BASE}/${platform}/${command}.md`;
-			const result = await loadPage(rawUrl, { timeout });
+			const result = await loadPage(rawUrl, { timeout, signal });
 
 			if (result.ok && result.content.trim()) {
 				const output = finalizeOutput(result.content);

package/src/core/tools/{web-fetch-handlers → web-scrapers}/twitter.ts

@@ -12,7 +12,11 @@ const NITTER_INSTANCES = [
 /**
  * Handle Twitter/X URLs via Nitter
  */
-export const handleTwitter: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+export const handleTwitter: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
 	try {
 		const parsed = new URL(url);
 		if (!["twitter.com", "x.com", "www.twitter.com", "www.x.com"].includes(parsed.hostname)) {
@@ -24,7 +28,7 @@ export const handleTwitter: SpecialHandler = async (url: string, timeout: number
 		// Try Nitter instances
 		for (const instance of NITTER_INSTANCES) {
 			const nitterUrl = `https://${instance}${parsed.pathname}`;
-			const result = await loadPage(nitterUrl, { timeout: Math.min(timeout, 10) });
+			const result = await loadPage(nitterUrl, { timeout: Math.min(timeout, 10), signal });
 
 			if (result.ok && result.content.length > 500) {
 				// Parse the Nitter HTML
@@ -67,7 +71,15 @@ export const handleTwitter: SpecialHandler = async (url: string, timeout: number
 				}
 			}
 		}
-	} catch {}
+	} catch {
+		if (signal?.aborted) {
+			return null;
+		}
+	}
+
+	if (signal?.aborted) {
+		return null;
+	}
 
 	// X.com blocks all bots - return a helpful error instead of falling through
 	return {