@oh-my-pi/pi-coding-agent 3.30.0 → 3.31.0

package/src/core/tools/web-scrapers/searchcode.ts

@@ -0,0 +1,217 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface SearchcodeResult {
+	id?: number | string;
+	filename?: string;
+	repo?: string;
+	language?: string;
+	code?: string;
+	lines?: number | string | Array<number | string>;
+	location?: string;
+	url?: string;
+}
+
+interface SearchcodeSearchResponse {
+	query?: string;
+	results?: SearchcodeResult[];
+	total?: number;
+	total_results?: number;
+	nextpage?: number;
+}
+
+const VALID_HOSTS = new Set(["searchcode.com", "www.searchcode.com"]);
+
+function parseLineNumbers(lines: SearchcodeResult["lines"]): number[] | null {
+	if (typeof lines === "number" && Number.isFinite(lines)) return [lines];
+
+	if (typeof lines === "string") {
+		const parts = lines.split(/[,\s]+/).filter(Boolean);
+		const parsed = parts.map((part) => Number.parseInt(part, 10)).filter((value) => Number.isFinite(value));
+		return parsed.length ? parsed : null;
+	}
+
+	if (Array.isArray(lines)) {
+		const parsed = lines.map((part) => Number.parseInt(String(part), 10)).filter((value) => Number.isFinite(value));
+		return parsed.length ? parsed : null;
+	}
+
+	return null;
+}
+
+function formatLineNumbers(lines: number[] | null): string | null {
+	if (!lines || lines.length === 0) return null;
+	if (lines.length <= 10) return lines.join(", ");
+	const min = Math.min(...lines);
+	const max = Math.max(...lines);
+	return `${min}-${max} (${lines.length} lines)`;
+}
+
+function formatCodeBlock(
+	code: string | undefined,
+	language: string | undefined,
+	lines: number[] | null,
+): string | null {
+	if (!code) return null;
+
+	const normalized = code.replace(/\r\n/g, "\n").trimEnd();
+	const codeLines = normalized.split("\n");
+	const languageTag = typeof language === "string" ? language.trim().toLowerCase() : "";
+
+	let displayLines = codeLines;
+	if (lines && lines.length === codeLines.length) {
+		displayLines = codeLines.map((line, index) => `${lines[index]}: ${line}`);
+	}
+
+	const fence = languageTag ? languageTag : "";
+	return `\n\n\`\`\`${fence}\n${displayLines.join("\n")}\n\`\`\`\n`;
+}
+
+export const handleSearchcode: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!VALID_HOSTS.has(parsed.hostname)) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const viewMatch = parsed.pathname.match(/^\/codesearch\/view\/([^/?#]+)/);
+		if (viewMatch) {
+			const id = viewMatch[1];
+			const apiUrl = `https://searchcode.com/api/result/${encodeURIComponent(id)}/`;
+			const result = await loadPage(apiUrl, { timeout, signal, headers: { Accept: "application/json" } });
+			if (!result.ok) return null;
+
+			let data: SearchcodeResult;
+			try {
+				data = JSON.parse(result.content) as SearchcodeResult;
+			} catch {
+				return null;
+			}
+
+			const filename = data.filename || data.location || `Result ${id}`;
+			const lineNumbers = parseLineNumbers(data.lines);
+			const formattedLines = formatLineNumbers(lineNumbers);
+			const viewUrl = data.url || `https://searchcode.com/codesearch/view/${id}`;
+			const snippetBlock = formatCodeBlock(data.code, data.language, lineNumbers);
+
+			let md = `# ${filename}\n\n`;
+			md += `## Description\n\n`;
+			md += "Code snippet from searchcode.com.\n\n";
+			md += `## Metadata\n\n`;
+			if (data.repo) md += `**Repository:** ${data.repo}\n`;
+			if (data.language) md += `**Language:** ${data.language}\n`;
+			if (data.filename) md += `**File:** ${data.filename}\n`;
+			if (data.location) md += `**Location:** ${data.location}\n`;
+			if (formattedLines) md += `**Lines:** ${formattedLines}\n`;
+			md += `**Result ID:** ${id}\n`;
+			md += `**URL:** ${viewUrl}\n`;
+
+			md += `\n## Snippet`;
+			if (snippetBlock) {
+				md += snippetBlock;
+			} else {
+				md += "\n\n_No snippet available._\n";
+			}
+
+			const output = finalizeOutput(md);
+			return {
+				url,
+				finalUrl: url,
+				contentType: "text/markdown",
+				method: "searchcode",
+				content: output.content,
+				fetchedAt,
+				truncated: output.truncated,
+				notes: ["Fetched via searchcode API"],
+			};
+		}
+
+		const query = parsed.searchParams.get("q");
+		const isSearchPage =
+			parsed.pathname === "/" || parsed.pathname === "/codesearch" || parsed.pathname === "/codesearch/";
+		if (!query || !isSearchPage) return null;
+
+		const pageRaw = parsed.searchParams.get("p") ?? parsed.searchParams.get("page");
+		const pageNumber = pageRaw ? Number.parseInt(pageRaw, 10) : 0;
+		const page = Number.isFinite(pageNumber) && pageNumber >= 0 ? pageNumber : 0;
+		const apiUrl = `https://searchcode.com/api/codesearch_I/?q=${encodeURIComponent(query)}&p=${page}`;
+		const result = await loadPage(apiUrl, { timeout, signal, headers: { Accept: "application/json" } });
+		if (!result.ok) return null;
+
+		let data: SearchcodeSearchResponse;
+		try {
+			data = JSON.parse(result.content) as SearchcodeSearchResponse;
+		} catch {
+			return null;
+		}
+
+		const results = Array.isArray(data.results) ? data.results : [];
+		const total =
+			typeof data.total === "number"
+				? data.total
+				: typeof data.total_results === "number"
+					? data.total_results
+					: null;
+
+		let md = `# Searchcode Results\n\n`;
+		md += `## Description\n\n`;
+		md += `Search results for \`${query}\` on searchcode.com.\n\n`;
+		md += `## Metadata\n\n`;
+		md += `**Query:** \`${query}\`\n`;
+		md += `**Page:** ${page}\n`;
+		if (total !== null) md += `**Total Results:** ${formatCount(total)}\n`;
+		md += `**Result Count:** ${results.length}\n`;
+		if (typeof data.nextpage === "number") md += `**Next Page:** ${data.nextpage}\n`;
+
+		md += `\n## Results\n\n`;
+
+		if (results.length === 0) {
+			md += "_No results found._\n";
+		} else {
+			const maxResults = 10;
+			for (const resultItem of results.slice(0, maxResults)) {
+				const id = resultItem.id !== undefined ? String(resultItem.id) : null;
+				const filename = resultItem.filename || resultItem.location || "Result";
+				const lineNumbers = parseLineNumbers(resultItem.lines);
+				const formattedLines = formatLineNumbers(lineNumbers);
+				const viewUrl = resultItem.url || (id ? `https://searchcode.com/codesearch/view/${id}` : null);
+				const snippetBlock = formatCodeBlock(resultItem.code, resultItem.language, lineNumbers);
+
+				md += `### ${filename}\n\n`;
+				if (resultItem.repo) md += `**Repository:** ${resultItem.repo}\n`;
+				if (resultItem.language) md += `**Language:** ${resultItem.language}\n`;
+				if (resultItem.filename) md += `**File:** ${resultItem.filename}\n`;
+				if (resultItem.location) md += `**Location:** ${resultItem.location}\n`;
+				if (formattedLines) md += `**Lines:** ${formattedLines}\n`;
+				if (viewUrl) md += `**URL:** ${viewUrl}\n`;
+
+				if (snippetBlock) {
+					md += `${snippetBlock}\n`;
+				}
+
+				md += "\n";
+			}
+
+			if (results.length > maxResults) {
+				md += `\n_Only showing first ${maxResults} results._\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "searchcode",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via searchcode API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/{web-fetch-handlers → web-scrapers}/sec-edgar.ts

@@ -153,7 +153,11 @@ function buildFilingUrl(cik: string, accessionNumber: string, document: string):
 /**
  * Handle SEC EDGAR URLs via data.sec.gov API
  */
-export const handleSecEdgar: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+export const handleSecEdgar: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
 	try {
 		const parsed = new URL(url);
 
@@ -171,6 +175,7 @@ export const handleSecEdgar: SpecialHandler = async (url: string, timeout: numbe
 		const apiUrl = `https://data.sec.gov/submissions/CIK${cik}.json`;
 		const result = await loadPage(apiUrl, {
 			timeout,
+			signal,
 			headers: {
 				"User-Agent": "CodingAgent/1.0 (research tool)",
 				Accept: "application/json",

package/src/core/tools/{web-fetch-handlers → web-scrapers}/semantic-scholar.ts

@@ -43,7 +43,7 @@ function extractPaperId(url: string): string | null {
 	return null;
 }
 
-export const handleSemanticScholar: SpecialHandler = async (url: string, timeout: number) => {
+export const handleSemanticScholar: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
 	if (!url.includes("semanticscholar.org")) return null;
 
 	const paperId = extractPaperId(url);
@@ -77,7 +77,7 @@ export const handleSemanticScholar: SpecialHandler = async (url: string, timeout
 
 	const apiUrl = `https://api.semanticscholar.org/graph/v1/paper/${paperId}?fields=${fields}`;
 
-	const { content, ok, finalUrl } = await loadPage(apiUrl, { timeout });
+	const { content, ok, finalUrl } = await loadPage(apiUrl, { timeout, signal });
 
 	if (!ok || !content) {
 		return {

package/src/core/tools/web-scrapers/snapcraft.ts

@@ -0,0 +1,200 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface SnapcraftPublisher {
+	"display-name"?: string;
+	username?: string;
+	id?: string;
+	validation?: string;
+}
+
+interface SnapcraftChannel {
+	name?: string;
+	track?: string;
+	risk?: string;
+	branch?: string | null;
+	architecture?: string;
+	"released-at"?: string;
+}
+
+interface SnapcraftDownload {
+	size?: number;
+	url?: string;
+	"sha3-384"?: string;
+}
+
+interface SnapcraftChannelMapEntry {
+	channel?: SnapcraftChannel;
+	version?: string;
+	revision?: number | string;
+	download?: SnapcraftDownload;
+	type?: string;
+	"created-at"?: string;
+}
+
+interface SnapcraftSnap {
+	name?: string;
+	title?: string;
+	summary?: string;
+	description?: string;
+	publisher?: SnapcraftPublisher;
+	version?: string;
+	confinement?: string;
+	base?: string;
+	downloads?: number;
+	download?: number;
+}
+
+interface SnapcraftResponse {
+	name?: string;
+	title?: string;
+	summary?: string;
+	description?: string;
+	publisher?: SnapcraftPublisher;
+	version?: string;
+	confinement?: string;
+	base?: string;
+	downloads?: number;
+	download?: number;
+	snap?: SnapcraftSnap;
+	"channel-map"?: SnapcraftChannelMapEntry[];
+}
+
+function formatPublisher(publisher?: SnapcraftPublisher): string | null {
+	if (!publisher) return null;
+	const displayName = publisher["display-name"] ?? publisher.username ?? publisher.id;
+	if (!displayName) return null;
+	if (publisher.username && displayName !== publisher.username) {
+		return `${displayName} (@${publisher.username})`;
+	}
+	return displayName;
+}
+
+function formatChannelName(channel?: SnapcraftChannel): string | null {
+	if (!channel) return null;
+	if (channel.name?.includes("/")) return channel.name;
+	if (channel.track && channel.risk) {
+		const branch = channel.branch ? `/${channel.branch}` : "";
+		return `${channel.track}/${channel.risk}${branch}`;
+	}
+	return channel.name ?? null;
+}
+
+function pickVersionFromChannels(entries: SnapcraftChannelMapEntry[]): string | undefined {
+	const stable = entries.find((entry) => entry.channel?.risk === "stable" && entry.version);
+	if (stable?.version) return stable.version;
+	const first = entries.find((entry) => entry.version);
+	return first?.version;
+}
+
+function extractDownloads(snapInfo: SnapcraftSnap | SnapcraftResponse, data: SnapcraftResponse): number | null {
+	const candidates = [snapInfo.downloads, snapInfo.download, data.downloads, data.download];
+	for (const value of candidates) {
+		if (typeof value === "number" && Number.isFinite(value)) return value;
+	}
+	return null;
+}
+
+export const handleSnapcraft: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "snapcraft.io" && parsed.hostname !== "www.snapcraft.io") return null;
+
+		const installMatch = parsed.pathname.match(/^\/install\/([^/]+)\/?$/);
+		const directMatch = parsed.pathname.match(/^\/([^/]+)\/?$/);
+		if (!installMatch && !directMatch) return null;
+
+		const snapName = decodeURIComponent((installMatch ?? directMatch)![1]);
+		const fetchedAt = new Date().toISOString();
+
+		const apiUrl = `https://api.snapcraft.io/v2/snaps/info/${encodeURIComponent(snapName)}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: {
+				Accept: "application/json",
+				"Snap-Device-Series": "16",
+			},
+		});
+		if (!result.ok) return null;
+
+		let data: SnapcraftResponse;
+		try {
+			data = JSON.parse(result.content) as SnapcraftResponse;
+		} catch {
+			return null;
+		}
+
+		const snapInfo = data.snap ?? data;
+		const name = snapInfo.title ?? snapInfo.name ?? data.name ?? snapName;
+		const summary = snapInfo.summary ?? data.summary;
+		const description = snapInfo.description ?? data.description;
+		const publisher = formatPublisher(snapInfo.publisher ?? data.publisher);
+		const confinement = snapInfo.confinement ?? data.confinement;
+		const base = snapInfo.base ?? data.base;
+
+		const channelMap = data["channel-map"] ?? [];
+		let version = snapInfo.version ?? data.version;
+		if (!version && channelMap.length > 0) {
+			version = pickVersionFromChannels(channelMap);
+		}
+
+		const downloads = extractDownloads(snapInfo, data);
+
+		const channels = new Map<string, { version?: string; architectures: Set<string> }>();
+		for (const entry of channelMap) {
+			const channelName = formatChannelName(entry.channel);
+			if (!channelName) continue;
+			const existing = channels.get(channelName) ?? { architectures: new Set<string>() };
+			if (!existing.version && entry.version) existing.version = entry.version;
+			if (entry.channel?.architecture) existing.architectures.add(entry.channel.architecture);
+			channels.set(channelName, existing);
+		}
+
+		let md = `# ${name}\n\n`;
+		if (summary) md += `${summary}\n\n`;
+
+		md += `**Version:** ${version ?? "unknown"}`;
+		if (confinement) md += ` · **Confinement:** ${confinement}`;
+		if (base) md += ` · **Base:** ${base}`;
+		md += "\n";
+		if (publisher) md += `**Publisher:** ${publisher}\n`;
+		if (downloads !== null) md += `**Downloads:** ${formatCount(downloads)}\n`;
+		md += "\n";
+
+		if (channels.size > 0) {
+			md += "## Channels\n\n";
+			const sortedChannels = Array.from(channels.entries()).sort((a, b) => a[0].localeCompare(b[0]));
+			for (const [channelName, info] of sortedChannels) {
+				const arches = Array.from(info.architectures).sort();
+				const versionSuffix = info.version ? `: ${info.version}` : "";
+				const archSuffix = arches.length > 0 ? ` (${arches.join(", ")})` : "";
+				md += `- ${channelName}${versionSuffix}${archSuffix}\n`;
+			}
+			md += "\n";
+		}
+
+		const descriptionText = description ?? summary;
+		if (descriptionText) {
+			md += `## Description\n\n${descriptionText}\n`;
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "snapcraft",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Snapcraft API"],
+		};
+	} catch {}
+
+	return null;
+};