@wahack/pi-coding-agent 15.11.0

package/src/web/parallel.ts

@@ -0,0 +1,353 @@
+import { type FetchImpl, getEnvApiKey } from "@oh-my-pi/pi-ai";
+import type { AgentStorage } from "../session/agent-storage";
+import { findCredential, withHardTimeout } from "./search/providers/utils";
+
+const PARALLEL_API_URL = "https://api.parallel.ai";
+const PARALLEL_SEARCH_URL = `${PARALLEL_API_URL}/v1beta/search`;
+const PARALLEL_EXTRACT_URL = `${PARALLEL_API_URL}/v1beta/extract`;
+const PARALLEL_BETA_HEADER = "search-extract-2025-10-10";
+
+export interface ParallelUsageItem {
+	name?: string;
+	count?: number;
+}
+
+export interface ParallelSearchSource {
+	title: string;
+	url: string;
+	snippet?: string;
+	publishedDate?: string;
+	excerpts: string[];
+}
+
+export interface ParallelSearchResult {
+	requestId: string;
+	sources: ParallelSearchSource[];
+	warnings: string[];
+	usage: ParallelUsageItem[];
+}
+
+export interface ParallelExtractDocument {
+	url: string;
+	title?: string;
+	publishedDate?: string;
+	excerpts: string[];
+	fullContent?: string;
+}
+
+export interface ParallelExtractErrorEntry {
+	url: string;
+	errorType?: string;
+	httpStatusCode?: number;
+	content?: string;
+}
+
+export interface ParallelExtractResult {
+	requestId: string;
+	results: ParallelExtractDocument[];
+	errors: ParallelExtractErrorEntry[];
+	warnings: string[];
+	usage: ParallelUsageItem[];
+}
+
+export interface ParallelSearchOptions {
+	mode?: "fast" | "research";
+	maxCharsPerResult?: number;
+	signal?: AbortSignal;
+	fetch?: FetchImpl;
+}
+
+export interface ParallelExtractOptions {
+	objective?: string;
+	searchQueries?: string[];
+	excerpts?: boolean;
+	fullContent?: boolean;
+	signal?: AbortSignal;
+	fetch?: FetchImpl;
+}
+
+export class ParallelApiError extends Error {
+	readonly statusCode?: number;
+
+	constructor(message: string, statusCode?: number) {
+		super(message);
+		this.name = "ParallelApiError";
+		this.statusCode = statusCode;
+	}
+}
+
+export function findParallelApiKey(storage: AgentStorage | null | undefined): string | null {
+	return findCredential(storage, getEnvApiKey("parallel"), "parallel");
+}
+
+export function getParallelExtractContent(document: ParallelExtractDocument): string {
+	const excerptContent = document.excerpts
+		.filter(excerpt => excerpt.trim().length > 0)
+		.join("\n\n")
+		.trim();
+	if (excerptContent.length > 0) {
+		return excerptContent;
+	}
+
+	return document.fullContent?.trim() ?? "";
+}
+
+function isObject(value: unknown): value is object {
+	return typeof value === "object" && value !== null;
+}
+
+function getOwnValue(value: object, key: string): unknown {
+	return Object.getOwnPropertyDescriptor(value, key)?.value;
+}
+
+function getString(value: object, key: string): string | undefined {
+	const candidate = getOwnValue(value, key);
+	return typeof candidate === "string" ? candidate : undefined;
+}
+
+function getNumber(value: object, key: string): number | undefined {
+	const candidate = getOwnValue(value, key);
+	return typeof candidate === "number" && Number.isFinite(candidate) ? candidate : undefined;
+}
+
+function getObjectArray(value: object, key: string): object[] {
+	const candidate = getOwnValue(value, key);
+	if (!Array.isArray(candidate)) return [];
+	return candidate.filter(isObject);
+}
+
+function getStringArray(value: object, key: string): string[] {
+	const candidate = getOwnValue(value, key);
+	if (!Array.isArray(candidate)) return [];
+	return candidate.filter(item => typeof item === "string");
+}
+
+function extractParallelErrorMessage(payload: unknown): string | null {
+	if (!isObject(payload)) return null;
+
+	const directMessage = getString(payload, "message") ?? getString(payload, "detail") ?? getString(payload, "error");
+	if (directMessage && directMessage.trim().length > 0) {
+		return directMessage.trim();
+	}
+
+	const errorObject = getOwnValue(payload, "error");
+	if (isObject(errorObject)) {
+		const nestedMessage = getString(errorObject, "message") ?? getString(errorObject, "detail");
+		if (nestedMessage && nestedMessage.trim().length > 0) {
+			return nestedMessage.trim();
+		}
+	}
+
+	return null;
+}
+
+function createParallelApiError(statusCode: number, detail?: string): ParallelApiError {
+	return new ParallelApiError(
+		detail ? `Parallel API error (${statusCode}): ${detail}` : `Parallel API error (${statusCode})`,
+		statusCode,
+	);
+}
+
+function parseParallelErrorResponse(statusCode: number, responseText: string): ParallelApiError {
+	const trimmedResponseText = responseText.trim();
+	if (trimmedResponseText.length === 0) {
+		return createParallelApiError(statusCode);
+	}
+
+	try {
+		const payload: unknown = JSON.parse(trimmedResponseText);
+		return createParallelApiError(statusCode, extractParallelErrorMessage(payload) ?? trimmedResponseText);
+	} catch {
+		return createParallelApiError(statusCode, trimmedResponseText);
+	}
+}
+
+function getAuthHeaders(apiKey: string): {
+	Accept: string;
+	"Content-Type": string;
+	"x-api-key": string;
+	"parallel-beta": string;
+} {
+	return {
+		Accept: "application/json",
+		"Content-Type": "application/json",
+		"x-api-key": apiKey,
+		"parallel-beta": PARALLEL_BETA_HEADER,
+	};
+}
+
+function normalizeSearchMode(mode: ParallelSearchOptions["mode"]): "fast" | "one-shot" {
+	return mode === "research" ? "one-shot" : "fast";
+}
+
+function parseUsageItems(payload: unknown): ParallelUsageItem[] {
+	if (!Array.isArray(payload)) return [];
+
+	const usageItems: ParallelUsageItem[] = [];
+	for (const item of payload) {
+		if (!isObject(item)) continue;
+		usageItems.push({
+			name: getString(item, "name"),
+			count: getNumber(item, "count"),
+		});
+	}
+	return usageItems;
+}
+
+function parseWarnings(payload: unknown): string[] {
+	if (!Array.isArray(payload)) return [];
+
+	const warnings: string[] = [];
+	for (const item of payload) {
+		if (typeof item === "string") {
+			warnings.push(item);
+			continue;
+		}
+		if (!isObject(item)) continue;
+		const message = getString(item, "message") ?? getString(item, "warning");
+		if (message) {
+			warnings.push(message);
+		}
+	}
+	return warnings;
+}
+
+function parseSearchPayload(payload: unknown): ParallelSearchResult {
+	if (!isObject(payload)) {
+		throw new ParallelApiError("Parallel search returned an invalid response payload.");
+	}
+
+	const requestId = getString(payload, "search_id") ?? "";
+	const rawResults = getObjectArray(payload, "results");
+	const sources: ParallelSearchSource[] = [];
+
+	for (const item of rawResults) {
+		const url = getString(item, "url");
+		if (!url) continue;
+
+		const excerpts = getStringArray(item, "excerpts");
+		const snippet = excerpts.length > 0 ? excerpts.join("\n\n") : undefined;
+		sources.push({
+			title: getString(item, "title") ?? url,
+			url,
+			snippet,
+			publishedDate: getString(item, "publish_date"),
+			excerpts,
+		});
+	}
+
+	return {
+		requestId,
+		sources,
+		warnings: parseWarnings(getOwnValue(payload, "warnings")),
+		usage: parseUsageItems(getOwnValue(payload, "usage")),
+	};
+}
+
+function parseExtractPayload(payload: unknown): ParallelExtractResult {
+	if (!isObject(payload)) {
+		throw new ParallelApiError("Parallel extract returned an invalid response payload.");
+	}
+
+	const requestId = getString(payload, "extract_id") ?? "";
+	const resultItems: ParallelExtractDocument[] = [];
+	for (const item of getObjectArray(payload, "results")) {
+		const url = getString(item, "url");
+		if (!url) continue;
+		resultItems.push({
+			url,
+			title: getString(item, "title"),
+			publishedDate: getString(item, "publish_date"),
+			excerpts: getStringArray(item, "excerpts"),
+			fullContent: getString(item, "full_content"),
+		});
+	}
+
+	const errors: ParallelExtractErrorEntry[] = [];
+	for (const item of getObjectArray(payload, "errors")) {
+		const url = getString(item, "url");
+		if (!url) continue;
+		errors.push({
+			url,
+			errorType: getString(item, "error_type"),
+			httpStatusCode: getNumber(item, "http_status_code"),
+			content: getString(item, "content"),
+		});
+	}
+
+	return {
+		requestId,
+		results: resultItems,
+		errors,
+		warnings: parseWarnings(getOwnValue(payload, "warnings")),
+		usage: parseUsageItems(getOwnValue(payload, "usage")),
+	};
+}
+
+export async function searchWithParallel(
+	objective: string,
+	queries: string[],
+	options: ParallelSearchOptions,
+	storage: AgentStorage | null | undefined,
+): Promise<ParallelSearchResult> {
+	const apiKey = findParallelApiKey(storage);
+	if (!apiKey) {
+		throw new ParallelApiError(
+			"Parallel credentials not found. Set PARALLEL_API_KEY or login with 'omp /login parallel'.",
+		);
+	}
+
+	const fetchImpl = options.fetch ?? fetch;
+	const response = await fetchImpl(PARALLEL_SEARCH_URL, {
+		method: "POST",
+		headers: getAuthHeaders(apiKey),
+		body: JSON.stringify({
+			objective,
+			search_queries: queries,
+			mode: normalizeSearchMode(options.mode),
+			excerpts: {
+				max_chars_per_result: options.maxCharsPerResult ?? 10_000,
+			},
+		}),
+		signal: withHardTimeout(options.signal),
+	});
+	if (!response.ok) {
+		throw parseParallelErrorResponse(response.status, await response.text());
+	}
+
+	const payload: unknown = await response.json();
+	return parseSearchPayload(payload);
+}
+
+export async function extractWithParallel(
+	urls: string[],
+	options: ParallelExtractOptions,
+	storage: AgentStorage | null | undefined,
+): Promise<ParallelExtractResult> {
+	const apiKey = findParallelApiKey(storage);
+	if (!apiKey) {
+		throw new ParallelApiError(
+			"Parallel credentials not found. Set PARALLEL_API_KEY or login with 'omp /login parallel'.",
+		);
+	}
+
+	const fetchImpl = options.fetch ?? fetch;
+	const response = await fetchImpl(PARALLEL_EXTRACT_URL, {
+		method: "POST",
+		headers: getAuthHeaders(apiKey),
+		body: JSON.stringify({
+			urls,
+			objective: options.objective,
+			search_queries: options.searchQueries,
+			excerpts: options.excerpts ?? true,
+			full_content: options.fullContent ?? false,
+		}),
+		signal: withHardTimeout(options.signal),
+	});
+	if (!response.ok) {
+		throw parseParallelErrorResponse(response.status, await response.text());
+	}
+
+	const payload: unknown = await response.json();
+	return parseExtractPayload(payload);
+}

package/src/web/scrapers/artifacthub.ts

@@ -0,0 +1,207 @@
+import { tryParseJson } from "@oh-my-pi/pi-utils";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, formatIsoDate, formatNumber, loadPage } from "./types";
+
+interface ArtifactHubMaintainer {
+	name: string;
+	email?: string;
+}
+
+interface ArtifactHubLink {
+	name: string;
+	url: string;
+}
+
+interface ArtifactHubRepository {
+	name: string;
+	display_name?: string;
+	url: string;
+	organization_name?: string;
+	organization_display_name?: string;
+}
+
+interface ArtifactHubPackage {
+	package_id: string;
+	name: string;
+	normalized_name: string;
+	display_name?: string;
+	description?: string;
+	version: string;
+	app_version?: string;
+	license?: string;
+	home_url?: string;
+	readme?: string;
+	install?: string;
+	keywords?: string[];
+	maintainers?: ArtifactHubMaintainer[];
+	links?: ArtifactHubLink[];
+	repository: ArtifactHubRepository;
+	ts: number;
+	created_at: number;
+	stars?: number;
+	official?: boolean;
+	signed?: boolean;
+	security_report_summary?: {
+		low?: number;
+		medium?: number;
+		high?: number;
+		critical?: number;
+	};
+	available_versions?: Array<{ version: string; ts: number }>;
+}
+
+/**
+ * Handle Artifact Hub URLs via API
+ * Supports Helm charts, OLM operators, Falco rules, OPA policies, etc.
+ */
+export const handleArtifactHub: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "artifacthub.io" && parsed.hostname !== "www.artifacthub.io") return null;
+
+		// Extract kind, repo, and package name from /packages/{kind}/{repo}/{name}
+		const match = parsed.pathname.match(/^\/packages\/([^/]+)\/([^/]+)\/([^/]+)/);
+		if (!match) return null;
+
+		const [, kind, repo, name] = match;
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch from Artifact Hub API
+		const apiUrl = `https://artifacthub.io/api/v1/packages/${kind}/${repo}/${name}`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok) return null;
+
+		const pkg = tryParseJson<ArtifactHubPackage>(result.content);
+		if (!pkg) return null;
+
+		const displayName = pkg.display_name || pkg.name;
+		const kindLabel = formatKindLabel(kind);
+
+		let md = `# ${displayName}\n\n`;
+		if (pkg.description) md += `${pkg.description}\n\n`;
+
+		// Basic info line
+		md += `**Type:** ${kindLabel}`;
+		md += ` · **Version:** ${pkg.version}`;
+		if (pkg.app_version) md += ` · **App Version:** ${pkg.app_version}`;
+		if (pkg.license) md += ` · **License:** ${pkg.license}`;
+		md += "\n";
+
+		// Stats and badges
+		const badges: string[] = [];
+		if (pkg.official) badges.push("Official");
+		if (pkg.signed) badges.push("Signed");
+		if (pkg.stars) badges.push(`${formatNumber(pkg.stars)} stars`);
+		if (badges.length > 0) md += `**${badges.join(" · ")}**\n`;
+		md += "\n";
+
+		// Repository info
+		const repoDisplay =
+			pkg.repository.organization_display_name || pkg.repository.display_name || pkg.repository.name;
+		md += `**Repository:** ${repoDisplay}`;
+		if (pkg.repository.url) md += ` ([${pkg.repository.url}](${pkg.repository.url}))`;
+		md += "\n";
+
+		if (pkg.home_url) md += `**Homepage:** ${pkg.home_url}\n`;
+		if (pkg.keywords?.length) md += `**Keywords:** ${pkg.keywords.join(", ")}\n`;
+
+		// Maintainers
+		if (pkg.maintainers?.length) {
+			const maintainerNames = pkg.maintainers.map(m => m.name).join(", ");
+			md += `**Maintainers:** ${maintainerNames}\n`;
+		}
+
+		// Security report summary
+		if (pkg.security_report_summary) {
+			const sec = pkg.security_report_summary;
+			const parts: string[] = [];
+			if (sec.critical) parts.push(`${sec.critical} critical`);
+			if (sec.high) parts.push(`${sec.high} high`);
+			if (sec.medium) parts.push(`${sec.medium} medium`);
+			if (sec.low) parts.push(`${sec.low} low`);
+			if (parts.length > 0) {
+				md += `**Security:** ${parts.join(", ")}\n`;
+			}
+		}
+
+		// Links
+		if (pkg.links?.length) {
+			md += `\n## Links\n\n`;
+			for (const link of pkg.links) {
+				md += `- [${link.name}](${link.url})\n`;
+			}
+		}
+
+		// Install instructions
+		if (pkg.install) {
+			md += `\n## Installation\n\n\`\`\`bash\n${pkg.install.trim()}\n\`\`\`\n`;
+		}
+
+		// Recent versions
+		if (pkg.available_versions?.length) {
+			md += `\n## Recent Versions\n\n`;
+			for (const ver of pkg.available_versions.slice(0, 5)) {
+				const date = formatIsoDate(ver.ts * 1000);
+				md += `- **${ver.version}** (${date})\n`;
+			}
+		}
+
+		// README
+		if (pkg.readme) {
+			md += `\n---\n\n## README\n\n${pkg.readme}\n`;
+		}
+
+		return buildResult(md, {
+			url,
+			method: "artifacthub",
+			fetchedAt,
+			notes: [`Fetched via Artifact Hub API (${kindLabel})`],
+		});
+	} catch {}
+
+	return null;
+};
+
+/**
+ * Convert kind slug to display label
+ */
+function formatKindLabel(kind: string): string {
+	const labels: Record<string, string> = {
+		helm: "Helm Chart",
+		"helm-plugin": "Helm Plugin",
+		falco: "Falco Rules",
+		opa: "OPA Policy",
+		olm: "OLM Operator",
+		tbaction: "Tinkerbell Action",
+		krew: "Krew Plugin",
+		tekton: "Tekton Task",
+		"tekton-pipeline": "Tekton Pipeline",
+		keda: "KEDA Scaler",
+		coredns: "CoreDNS Plugin",
+		keptn: "Keptn Integration",
+		container: "Container Image",
+		kubewarden: "Kubewarden Policy",
+		gatekeeper: "Gatekeeper Policy",
+		kyverno: "Kyverno Policy",
+		"knative-client": "Knative Client Plugin",
+		backstage: "Backstage Plugin",
+		argo: "Argo Template",
+		kubearmor: "KubeArmor Policy",
+		kcl: "KCL Module",
+		headlamp: "Headlamp Plugin",
+		inspektor: "Inspektor Gadget",
+		"meshery-design": "Meshery Design",
+		"opencost-plugin": "OpenCost Plugin",
+		radius: "Radius Recipe",
+	};
+	return labels[kind] || kind.charAt(0).toUpperCase() + kind.slice(1);
+}

package/src/web/scrapers/arxiv.ts

@@ -0,0 +1,83 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, loadPage } from "./types";
+import { convertWithMarkit, fetchBinary } from "./utils";
+
+/**
+ * Handle arXiv URLs via arXiv API
+ */
+export const handleArxiv: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (parsed.hostname !== "arxiv.org") return null;
+
+		// Extract paper ID from various URL formats
+		// /abs/1234.56789, /pdf/1234.56789, /abs/cs/0123456
+		const match = parsed.pathname.match(/\/(abs|pdf)\/(.+?)(?:\.pdf)?$/);
+		if (!match) return null;
+
+		const paperId = match[2];
+		const fetchedAt = new Date().toISOString();
+		const notes: string[] = [];
+
+		// Fetch metadata via arXiv API
+		const apiUrl = `https://export.arxiv.org/api/query?id_list=${paperId}`;
+		const result = await loadPage(apiUrl, { timeout, signal });
+
+		if (!result.ok) return null;
+
+		// Parse the Atom feed response
+		const { parseHTML } = await import("linkedom");
+		const doc = parseHTML(result.content).document;
+		const entry = doc.querySelector("entry");
+
+		if (!entry) return null;
+
+		const title = entry.querySelector("title")?.textContent?.trim()?.replace(/\s+/g, " ");
+		const summary = entry.querySelector("summary")?.textContent?.trim();
+		const authors = Array.from(entry.querySelectorAll("author name") as Iterable<{ textContent: string | null }>)
+			.map(n => n.textContent?.trim())
+			.filter((name): name is string => Boolean(name));
+		const published = entry.querySelector("published")?.textContent?.trim()?.split("T")[0];
+		const categories = Array.from(
+			entry.querySelectorAll("category") as Iterable<{ getAttribute: (name: string) => string | null }>,
+		)
+			.map(c => c.getAttribute("term"))
+			.filter((term): term is string => Boolean(term));
+		const pdfLink = entry.querySelector('link[title="pdf"]')?.getAttribute("href");
+
+		let md = `# ${title || "arXiv Paper"}\n\n`;
+		if (authors.length) md += `**Authors:** ${authors.join(", ")}\n`;
+		if (published) md += `**Published:** ${published}\n`;
+		if (categories.length) md += `**Categories:** ${categories.join(", ")}\n`;
+		md += `**arXiv:** ${paperId}\n\n`;
+		md += `---\n\n## Abstract\n\n${summary || "No abstract available."}\n\n`;
+
+		// If it was a PDF link or we want full content, try to fetch and convert PDF
+		if (match[1] === "pdf" || parsed.pathname.includes(".pdf")) {
+			if (pdfLink) {
+				notes.push("Fetching PDF for full content...");
+				const pdfResult = await fetchBinary(pdfLink, timeout, signal);
+				if (pdfResult.ok) {
+					const converted = await convertWithMarkit(pdfResult.buffer, ".pdf", timeout, signal);
+					if (converted.ok && converted.content.length > 500) {
+						md += `---\n\n## Full Paper\n\n${converted.content}\n`;
+						notes.push("PDF converted via markit");
+					}
+				}
+			}
+		}
+
+		return buildResult(md, {
+			url,
+			method: "arxiv",
+			fetchedAt,
+			notes: notes.length ? notes : ["Fetched via arXiv API"],
+		});
+	} catch {}
+
+	return null;
+};