@oh-my-pi/pi-coding-agent 3.25.0 → 3.31.0

package/src/core/tools/web-scrapers/sec-edgar.ts

@@ -0,0 +1,274 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+interface SecFiling {
+	accessionNumber: string;
+	filingDate: string;
+	reportDate: string;
+	acceptanceDateTime: string;
+	form: string;
+	primaryDocument: string;
+	primaryDocDescription: string;
+}
+
+interface SecCompany {
+	cik: string;
+	entityType: string;
+	sic: string;
+	sicDescription: string;
+	name: string;
+	tickers: string[];
+	exchanges: string[];
+	ein: string;
+	stateOfIncorporation: string;
+	fiscalYearEnd: string;
+	addresses: {
+		business: {
+			street1?: string;
+			street2?: string;
+			city?: string;
+			stateOrCountry?: string;
+			zipCode?: string;
+		};
+		mailing: {
+			street1?: string;
+			street2?: string;
+			city?: string;
+			stateOrCountry?: string;
+			zipCode?: string;
+		};
+	};
+	filings: {
+		recent: {
+			accessionNumber: string[];
+			filingDate: string[];
+			reportDate: string[];
+			acceptanceDateTime: string[];
+			form: string[];
+			primaryDocument: string[];
+			primaryDocDescription: string[];
+		};
+	};
+}
+
+/**
+ * Extract CIK from various SEC EDGAR URL patterns
+ */
+function extractCik(url: URL): string | null {
+	const { hostname, pathname, searchParams } = url;
+
+	// Check hostname
+	if (!hostname.includes("sec.gov")) return null;
+
+	// Pattern: ?CIK=xxx or ?cik=xxx
+	const cikParam = searchParams.get("CIK") || searchParams.get("cik");
+	if (cikParam) {
+		return normalizeCik(cikParam);
+	}
+
+	// Pattern: /cik/XXXXXXXXXX or /cik/XXXXXXXXXX/...
+	const cikPathMatch = pathname.match(/\/cik\/(\d+)/i);
+	if (cikPathMatch) {
+		return normalizeCik(cikPathMatch[1]);
+	}
+
+	// Pattern: /submissions/CIK*.json
+	const submissionsMatch = pathname.match(/\/submissions\/CIK(\d+)\.json/i);
+	if (submissionsMatch) {
+		return normalizeCik(submissionsMatch[1]);
+	}
+
+	// Pattern: /cgi-bin/browse-edgar with company search (no CIK yet)
+	if (pathname.includes("/cgi-bin/browse-edgar") && searchParams.get("company")) {
+		// Company name search - we'd need to search first, skip for now
+		return null;
+	}
+
+	// Pattern: Filing URLs like /Archives/edgar/data/XXXXXXXXXX/...
+	const archivesMatch = pathname.match(/\/Archives\/edgar\/data\/(\d+)/);
+	if (archivesMatch) {
+		return normalizeCik(archivesMatch[1]);
+	}
+
+	return null;
+}
+
+/**
+ * Normalize CIK to 10 digits with leading zeros
+ */
+function normalizeCik(cik: string): string {
+	const cleaned = cik.replace(/\D/g, "");
+	return cleaned.padStart(10, "0");
+}
+
+/**
+ * Format address for display
+ */
+function formatAddress(addr: SecCompany["addresses"]["business"]): string {
+	const parts: string[] = [];
+	if (addr.street1) parts.push(addr.street1);
+	if (addr.street2) parts.push(addr.street2);
+
+	const cityLine: string[] = [];
+	if (addr.city) cityLine.push(addr.city);
+	if (addr.stateOrCountry) cityLine.push(addr.stateOrCountry);
+	if (addr.zipCode) cityLine.push(addr.zipCode);
+	if (cityLine.length) parts.push(cityLine.join(", "));
+
+	return parts.join("\n");
+}
+
+/**
+ * Get recent filings of specific types
+ */
+function getRecentFilings(company: SecCompany, formTypes: string[], limit = 10): SecFiling[] {
+	const { recent } = company.filings;
+	const filings: SecFiling[] = [];
+
+	for (let i = 0; i < recent.form.length && filings.length < limit; i++) {
+		if (formTypes.length === 0 || formTypes.includes(recent.form[i])) {
+			filings.push({
+				accessionNumber: recent.accessionNumber[i],
+				filingDate: recent.filingDate[i],
+				reportDate: recent.reportDate[i],
+				acceptanceDateTime: recent.acceptanceDateTime[i],
+				form: recent.form[i],
+				primaryDocument: recent.primaryDocument[i],
+				primaryDocDescription: recent.primaryDocDescription[i],
+			});
+		}
+	}
+
+	return filings;
+}
+
+/**
+ * Build SEC EDGAR filing URL
+ */
+function buildFilingUrl(cik: string, accessionNumber: string, document: string): string {
+	const accessionNoDashes = accessionNumber.replace(/-/g, "");
+	return `https://www.sec.gov/Archives/edgar/data/${parseInt(cik, 10)}/${accessionNoDashes}/${document}`;
+}
+
+/**
+ * Handle SEC EDGAR URLs via data.sec.gov API
+ */
+export const handleSecEdgar: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+
+		// Check if it's an SEC URL
+		if (!parsed.hostname.includes("sec.gov")) return null;
+
+		// Extract CIK from URL
+		const cik = extractCik(parsed);
+		if (!cik) return null;
+
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch company data from SEC API
+		// SEC requires a proper User-Agent with contact info
+		const apiUrl = `https://data.sec.gov/submissions/CIK${cik}.json`;
+		const result = await loadPage(apiUrl, {
+			timeout,
+			signal,
+			headers: {
+				"User-Agent": "CodingAgent/1.0 (research tool)",
+				Accept: "application/json",
+			},
+		});
+
+		if (!result.ok) return null;
+
+		let company: SecCompany;
+		try {
+			company = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		// Build markdown output
+		let md = `# ${company.name}\n\n`;
+
+		// Basic info
+		md += `**CIK:** ${company.cik}`;
+		if (company.tickers?.length) {
+			md += ` · **Ticker${company.tickers.length > 1 ? "s" : ""}:** ${company.tickers.join(", ")}`;
+		}
+		if (company.exchanges?.length) {
+			md += ` (${company.exchanges.join(", ")})`;
+		}
+		md += "\n";
+
+		if (company.entityType) md += `**Entity Type:** ${company.entityType}\n`;
+		if (company.sic) md += `**SIC:** ${company.sic} - ${company.sicDescription}\n`;
+		if (company.stateOfIncorporation) md += `**State of Incorporation:** ${company.stateOfIncorporation}\n`;
+		if (company.ein) md += `**EIN:** ${company.ein}\n`;
+		if (company.fiscalYearEnd) {
+			const fy = company.fiscalYearEnd;
+			md += `**Fiscal Year End:** ${fy.slice(0, 2)}/${fy.slice(2)}\n`;
+		}
+		md += "\n";
+
+		// Business address
+		if (company.addresses?.business) {
+			const addr = formatAddress(company.addresses.business);
+			if (addr) {
+				md += `## Business Address\n\n${addr}\n\n`;
+			}
+		}
+
+		// Recent key filings (10-K, 10-Q, 8-K)
+		const keyFilings = getRecentFilings(company, ["10-K", "10-K/A", "10-Q", "10-Q/A", "8-K", "8-K/A"], 15);
+		if (keyFilings.length) {
+			md += `## Recent Filings (10-K, 10-Q, 8-K)\n\n`;
+			md += "| Date | Form | Description |\n";
+			md += "|------|------|-------------|\n";
+
+			for (const filing of keyFilings) {
+				const filingUrl = buildFilingUrl(cik, filing.accessionNumber, filing.primaryDocument);
+				const desc = filing.primaryDocDescription || filing.form;
+				md += `| ${filing.filingDate} | [${filing.form}](${filingUrl}) | ${desc} |\n`;
+			}
+			md += "\n";
+		}
+
+		// All recent filings (last 20)
+		const allFilings = getRecentFilings(company, [], 20);
+		if (allFilings.length) {
+			md += `## All Recent Filings\n\n`;
+			md += "| Date | Form | Description |\n";
+			md += "|------|------|-------------|\n";
+
+			for (const filing of allFilings) {
+				const filingUrl = buildFilingUrl(cik, filing.accessionNumber, filing.primaryDocument);
+				const desc = filing.primaryDocDescription || filing.form;
+				md += `| ${filing.filingDate} | [${filing.form}](${filingUrl}) | ${desc} |\n`;
+			}
+			md += "\n";
+		}
+
+		// Links
+		md += `## Links\n\n`;
+		md += `- [SEC EDGAR Filings](https://www.sec.gov/cgi-bin/browse-edgar?action=getcompany&CIK=${cik}&type=&dateb=&owner=include&count=40)\n`;
+		md += `- [Company Search](https://www.sec.gov/cgi-bin/browse-edgar?company=${encodeURIComponent(company.name)}&CIK=&type=&owner=include&count=40&action=getcompany)\n`;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "sec-edgar",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via SEC EDGAR API"],
+		};
+	} catch {}
+
+	return null;
+};

package/src/core/tools/web-scrapers/security.test.ts

@@ -0,0 +1,103 @@
+import { describe, expect, it } from "bun:test";
+import { handleNvd } from "./nvd";
+import { handleOsv } from "./osv";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleNvd", () => {
+	it("returns null for non-NVD URLs", async () => {
+		const result = await handleNvd("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for NVD URLs without CVE detail path", async () => {
+		const result = await handleNvd("https://nvd.nist.gov/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for NVD search URLs", async () => {
+		const result = await handleNvd("https://nvd.nist.gov/vuln/search/results?query=log4j", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches CVE-2021-44228 (Log4Shell)", async () => {
+		const result = await handleNvd("https://nvd.nist.gov/vuln/detail/CVE-2021-44228", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("nvd");
+		expect(result?.content).toContain("CVE-2021-44228");
+		expect(result?.content).toContain("Log4j");
+		expect(result?.content).toContain("CVSS");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches CVE-2014-0160 (Heartbleed)", async () => {
+		const result = await handleNvd("https://nvd.nist.gov/vuln/detail/CVE-2014-0160", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("nvd");
+		expect(result?.content).toContain("CVE-2014-0160");
+		expect(result?.content).toContain("OpenSSL");
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("handles lowercase CVE IDs", async () => {
+		const result = await handleNvd("https://nvd.nist.gov/vuln/detail/cve-2021-44228", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("nvd");
+		expect(result?.content).toContain("CVE-2021-44228");
+	});
+});
+
+describe.skipIf(SKIP)("handleOsv", () => {
+	it("returns null for non-OSV URLs", async () => {
+		const result = await handleOsv("https://example.com", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for OSV homepage", async () => {
+		const result = await handleOsv("https://osv.dev/", 20);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for OSV list URLs", async () => {
+		const result = await handleOsv("https://osv.dev/list", 20);
+		expect(result).toBeNull();
+	});
+
+	it("fetches GHSA-jfh8-c2jp-5v3q (log4j RCE)", async () => {
+		const result = await handleOsv("https://osv.dev/vulnerability/GHSA-jfh8-c2jp-5v3q", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("osv");
+		expect(result?.content).toContain("GHSA-jfh8-c2jp-5v3q");
+		expect(result?.content).toContain("log4j");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.fetchedAt).toBeTruthy();
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches CVE-2021-44228 via OSV", async () => {
+		const result = await handleOsv("https://osv.dev/vulnerability/CVE-2021-44228", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("osv");
+		expect(result?.content).toContain("CVE-2021-44228");
+		expect(result?.truncated).toBeDefined();
+	});
+
+	it("fetches PYSEC vulnerability", async () => {
+		// PYSEC-2021-19 is a well-known pillow vulnerability
+		const result = await handleOsv("https://osv.dev/vulnerability/PYSEC-2021-19", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("osv");
+		expect(result?.content).toContain("PYSEC-2021-19");
+		expect(result?.content).toContain("Affected Packages");
+	});
+
+	it("fetches RUSTSEC vulnerability", async () => {
+		// RUSTSEC-2021-0119 is a well-known actix-web vulnerability
+		const result = await handleOsv("https://osv.dev/vulnerability/RUSTSEC-2021-0119", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("osv");
+		expect(result?.content).toContain("RUSTSEC-2021-0119");
+	});
+});

package/src/core/tools/web-scrapers/semantic-scholar.ts

@@ -0,0 +1,190 @@
+import type { SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+interface SemanticScholarAuthor {
+	name: string;
+	authorId?: string;
+}
+
+interface SemanticScholarPaper {
+	paperId: string;
+	title: string;
+	abstract?: string;
+	authors?: SemanticScholarAuthor[];
+	year?: number;
+	citationCount?: number;
+	referenceCount?: number;
+	fieldsOfStudy?: string[];
+	publicationTypes?: string[];
+	journal?: { name: string; volume?: string; pages?: string };
+	externalIds?: {
+		DOI?: string;
+		ArXiv?: string;
+		PubMed?: string;
+		MAG?: string;
+		CorpusId?: string;
+	};
+	tldr?: { text: string };
+	openAccessPdf?: { url: string };
+}
+
+function extractPaperId(url: string): string | null {
+	const patterns = [
+		/semanticscholar\.org\/paper\/[^/]+\/([a-f0-9]{40})/i,
+		/semanticscholar\.org\/paper\/([a-f0-9]{40})/i,
+		/api\.semanticscholar\.org\/.*\/paper\/([a-f0-9]{40})/i,
+	];
+
+	for (const pattern of patterns) {
+		const match = url.match(pattern);
+		if (match?.[1]) return match[1];
+	}
+
+	return null;
+}
+
+export const handleSemanticScholar: SpecialHandler = async (url: string, timeout: number, signal?: AbortSignal) => {
+	if (!url.includes("semanticscholar.org")) return null;
+
+	const paperId = extractPaperId(url);
+	if (!paperId) {
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/plain",
+			method: "semantic-scholar",
+			content: "Failed to extract paper ID from Semantic Scholar URL",
+			fetchedAt: new Date().toISOString(),
+			truncated: false,
+			notes: ["Invalid URL format"],
+		};
+	}
+
+	const fields = [
+		"title",
+		"abstract",
+		"authors",
+		"year",
+		"citationCount",
+		"referenceCount",
+		"fieldsOfStudy",
+		"publicationTypes",
+		"journal",
+		"externalIds",
+		"tldr",
+		"openAccessPdf",
+	].join(",");
+
+	const apiUrl = `https://api.semanticscholar.org/graph/v1/paper/${paperId}?fields=${fields}`;
+
+	const { content, ok, finalUrl } = await loadPage(apiUrl, { timeout, signal });
+
+	if (!ok || !content) {
+		return {
+			url,
+			finalUrl: apiUrl,
+			contentType: "text/plain",
+			method: "semantic-scholar",
+			content: "Failed to fetch paper from Semantic Scholar API",
+			fetchedAt: new Date().toISOString(),
+			truncated: false,
+			notes: ["API request failed"],
+		};
+	}
+
+	let paper: SemanticScholarPaper;
+	try {
+		paper = JSON.parse(content);
+	} catch {
+		return {
+			url,
+			finalUrl: apiUrl,
+			contentType: "text/plain",
+			method: "semantic-scholar",
+			content: "Failed to parse response from Semantic Scholar API",
+			fetchedAt: new Date().toISOString(),
+			truncated: false,
+			notes: ["JSON parse error"],
+		};
+	}
+
+	const sections: string[] = [];
+
+	sections.push(`# ${paper.title || "Untitled"}`);
+	sections.push("");
+
+	if (paper.authors && paper.authors.length > 0) {
+		const authorList = paper.authors.map((a) => a.name).join(", ");
+		sections.push(`**Authors:** ${authorList}`);
+		sections.push("");
+	}
+
+	const metadata: string[] = [];
+	if (paper.year) metadata.push(`Year: ${paper.year}`);
+	if (paper.journal?.name) metadata.push(`Venue: ${paper.journal.name}`);
+	if (paper.citationCount !== undefined) {
+		metadata.push(`Citations: ${formatCount(paper.citationCount)}`);
+	}
+	if (paper.referenceCount !== undefined) {
+		metadata.push(`References: ${formatCount(paper.referenceCount)}`);
+	}
+	if (metadata.length > 0) {
+		sections.push(metadata.join(" • "));
+		sections.push("");
+	}
+
+	if (paper.fieldsOfStudy && paper.fieldsOfStudy.length > 0) {
+		sections.push(`**Fields:** ${paper.fieldsOfStudy.join(", ")}`);
+		sections.push("");
+	}
+
+	if (paper.tldr?.text) {
+		sections.push("## TL;DR");
+		sections.push("");
+		sections.push(paper.tldr.text);
+		sections.push("");
+	}
+
+	if (paper.abstract) {
+		sections.push("## Abstract");
+		sections.push("");
+		sections.push(paper.abstract);
+		sections.push("");
+	}
+
+	const links: string[] = [];
+	if (paper.openAccessPdf?.url) {
+		links.push(`[PDF](${paper.openAccessPdf.url})`);
+	}
+	if (paper.externalIds?.ArXiv) {
+		links.push(`[arXiv](https://arxiv.org/abs/${paper.externalIds.ArXiv})`);
+	}
+	if (paper.externalIds?.DOI) {
+		links.push(`[DOI](https://doi.org/${paper.externalIds.DOI})`);
+	}
+	if (paper.externalIds?.PubMed) {
+		links.push(`[PubMed](https://pubmed.ncbi.nlm.nih.gov/${paper.externalIds.PubMed}/)`);
+	}
+	links.push(`[Semantic Scholar](https://www.semanticscholar.org/paper/${paper.paperId})`);
+
+	if (links.length > 0) {
+		sections.push("## Links");
+		sections.push("");
+		sections.push(links.join(" • "));
+		sections.push("");
+	}
+
+	const fullContent = sections.join("\n");
+	const { content: finalContent, truncated } = finalizeOutput(fullContent);
+
+	return {
+		url,
+		finalUrl,
+		contentType: "text/markdown",
+		method: "semantic-scholar",
+		content: finalContent,
+		fetchedAt: new Date().toISOString(),
+		truncated,
+		notes: [],
+	};
+};