@oh-my-pi/pi-coding-agent 3.24.0 → 3.30.0

package/src/core/tools/web-fetch-handlers/wikidata.ts

@@ -0,0 +1,349 @@
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, formatCount, loadPage } from "./types";
+
+/**
+ * Common Wikidata property IDs mapped to human-readable names
+ */
+const PROPERTY_LABELS: Record<string, string> = {
+	P31: "Instance of",
+	P279: "Subclass of",
+	P17: "Country",
+	P131: "Located in",
+	P625: "Coordinates",
+	P18: "Image",
+	P154: "Logo",
+	P571: "Founded",
+	P576: "Dissolved",
+	P169: "CEO",
+	P112: "Founded by",
+	P159: "Headquarters",
+	P452: "Industry",
+	P1128: "Employees",
+	P2139: "Revenue",
+	P856: "Website",
+	P21: "Sex/Gender",
+	P27: "Citizenship",
+	P569: "Born",
+	P570: "Died",
+	P19: "Birthplace",
+	P20: "Death place",
+	P106: "Occupation",
+	P108: "Employer",
+	P69: "Educated at",
+	P22: "Father",
+	P25: "Mother",
+	P26: "Spouse",
+	P40: "Child",
+	P166: "Award",
+	P136: "Genre",
+	P495: "Country of origin",
+	P577: "Publication date",
+	P50: "Author",
+	P123: "Publisher",
+	P364: "Original language",
+	P86: "Composer",
+	P57: "Director",
+	P161: "Cast member",
+	P170: "Creator",
+	P178: "Developer",
+	P275: "License",
+	P306: "Operating system",
+	P277: "Programming language",
+	P348: "Version",
+	P1566: "GeoNames ID",
+	P214: "VIAF ID",
+	P227: "GND ID",
+	P213: "ISNI",
+	P496: "ORCID",
+};
+
+interface WikidataEntity {
+	type: string;
+	id: string;
+	labels?: Record<string, { language: string; value: string }>;
+	descriptions?: Record<string, { language: string; value: string }>;
+	aliases?: Record<string, Array<{ language: string; value: string }>>;
+	claims?: Record<string, WikidataClaim[]>;
+	sitelinks?: Record<string, { site: string; title: string }>;
+}
+
+interface WikidataClaim {
+	mainsnak: {
+		snaktype: string;
+		property: string;
+		datavalue?: {
+			type: string;
+			value: WikidataValue;
+		};
+	};
+	rank: string;
+}
+
+type WikidataValue =
+	| string
+	| { "entity-type": string; id: string; "numeric-id": number }
+	| { time: string; precision: number; calendarmodel: string }
+	| { amount: string; unit: string }
+	| { text: string; language: string }
+	| { latitude: number; longitude: number; precision: number };
+
+/**
+ * Handle Wikidata URLs via EntityData API
+ */
+export const handleWikidata: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("wikidata.org")) return null;
+
+		// Extract Q-id from /wiki/Q123 or /entity/Q123
+		const qidMatch = parsed.pathname.match(/\/(?:wiki|entity)\/(Q\d+)/i);
+		if (!qidMatch) return null;
+
+		const qid = qidMatch[1].toUpperCase();
+		const fetchedAt = new Date().toISOString();
+
+		// Fetch entity data from API
+		const apiUrl = `https://www.wikidata.org/wiki/Special:EntityData/${qid}.json`;
+		const result = await loadPage(apiUrl, { timeout });
+
+		if (!result.ok) return null;
+
+		let data: { entities: Record<string, WikidataEntity> };
+		try {
+			data = JSON.parse(result.content);
+		} catch {
+			return null;
+		}
+
+		const entity = data.entities[qid];
+		if (!entity) return null;
+
+		// Get label and description (prefer English)
+		const label = getLocalizedValue(entity.labels, "en") || qid;
+		const description = getLocalizedValue(entity.descriptions, "en");
+		const aliases = getLocalizedAliases(entity.aliases, "en");
+
+		let md = `# ${label} (${qid})\n\n`;
+		if (description) md += `*${description}*\n\n`;
+		if (aliases.length > 0) md += `**Also known as:** ${aliases.join(", ")}\n\n`;
+
+		// Count sitelinks
+		const sitelinkCount = entity.sitelinks ? Object.keys(entity.sitelinks).length : 0;
+		if (sitelinkCount > 0) {
+			md += `**Wikipedia articles:** ${formatCount(sitelinkCount)} languages\n\n`;
+		}
+
+		// Process claims
+		if (entity.claims && Object.keys(entity.claims).length > 0) {
+			md += "## Properties\n\n";
+
+			// Collect entity IDs we need to resolve
+			const entityIdsToResolve = new Set<string>();
+			for (const claims of Object.values(entity.claims)) {
+				for (const claim of claims) {
+					if (claim.mainsnak.datavalue?.type === "wikibase-entityid") {
+						const val = claim.mainsnak.datavalue.value as { id: string };
+						entityIdsToResolve.add(val.id);
+					}
+				}
+			}
+
+			// Fetch labels for referenced entities (limit to 50)
+			const entityLabels = await resolveEntityLabels(Array.from(entityIdsToResolve).slice(0, 50), timeout);
+
+			// Group claims by property
+			const processedProperties: string[] = [];
+			for (const [propId, claims] of Object.entries(entity.claims)) {
+				const propLabel = PROPERTY_LABELS[propId] || propId;
+				const values: string[] = [];
+
+				for (const claim of claims) {
+					if (claim.rank === "deprecated") continue;
+					const value = formatClaimValue(claim, entityLabels);
+					if (value && !values.includes(value)) {
+						values.push(value);
+					}
+				}
+
+				if (values.length > 0) {
+					// Limit values shown per property
+					const displayValues = values.slice(0, 10);
+					const overflow = values.length > 10 ? ` (+${values.length - 10} more)` : "";
+					processedProperties.push(`- **${propLabel}:** ${displayValues.join(", ")}${overflow}`);
+				}
+			}
+
+			// Sort: known properties first, then by property ID
+			processedProperties.sort((a, b) => {
+				const aKnown = Object.values(PROPERTY_LABELS).some((l) => a.includes(`**${l}:**`));
+				const bKnown = Object.values(PROPERTY_LABELS).some((l) => b.includes(`**${l}:**`));
+				if (aKnown && !bKnown) return -1;
+				if (!aKnown && bKnown) return 1;
+				return a.localeCompare(b);
+			});
+
+			// Limit total properties shown
+			const maxProps = 50;
+			md += processedProperties.slice(0, maxProps).join("\n");
+			if (processedProperties.length > maxProps) {
+				md += `\n\n*...and ${processedProperties.length - maxProps} more properties*`;
+			}
+			md += "\n";
+		}
+
+		// Add notable sitelinks
+		if (entity.sitelinks) {
+			const notableSites = ["enwiki", "dewiki", "frwiki", "eswiki", "jawiki", "zhwiki"];
+			const links: string[] = [];
+
+			for (const site of notableSites) {
+				const sitelink = entity.sitelinks[site];
+				if (sitelink) {
+					const lang = site.replace("wiki", "");
+					const wikiUrl = `https://${lang}.wikipedia.org/wiki/${encodeURIComponent(sitelink.title)}`;
+					links.push(`[${lang.toUpperCase()}](${wikiUrl})`);
+				}
+			}
+
+			if (links.length > 0) {
+				md += `\n## Wikipedia Links\n\n${links.join(" · ")}\n`;
+			}
+		}
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "wikidata",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Wikidata EntityData API"],
+		};
+	} catch {}
+
+	return null;
+};
+
+/**
+ * Get localized value with fallback
+ */
+function getLocalizedValue(
+	values: Record<string, { language: string; value: string }> | undefined,
+	preferredLang: string,
+): string | null {
+	if (!values) return null;
+	if (values[preferredLang]) return values[preferredLang].value;
+	// Fallback to any available
+	const first = Object.values(values)[0];
+	return first?.value || null;
+}
+
+/**
+ * Get aliases for a language
+ */
+function getLocalizedAliases(
+	aliases: Record<string, Array<{ language: string; value: string }>> | undefined,
+	preferredLang: string,
+): string[] {
+	if (!aliases) return [];
+	const langAliases = aliases[preferredLang];
+	if (!langAliases) return [];
+	return langAliases.map((a) => a.value);
+}
+
+/**
+ * Resolve entity IDs to their labels via wbgetentities API
+ */
+async function resolveEntityLabels(entityIds: string[], timeout: number): Promise<Record<string, string>> {
+	if (entityIds.length === 0) return {};
+
+	const labels: Record<string, string> = {};
+
+	// Fetch in batches of 50
+	const batchSize = 50;
+	for (let i = 0; i < entityIds.length; i += batchSize) {
+		const batch = entityIds.slice(i, i + batchSize);
+		const apiUrl = `https://www.wikidata.org/w/api.php?action=wbgetentities&ids=${batch.join("|")}&props=labels&languages=en&format=json`;
+
+		try {
+			const result = await loadPage(apiUrl, { timeout: Math.min(timeout, 10) });
+			if (result.ok) {
+				const data = JSON.parse(result.content) as {
+					entities: Record<string, { labels?: Record<string, { value: string }> }>;
+				};
+				for (const [id, entity] of Object.entries(data.entities)) {
+					const label = entity.labels?.en?.value;
+					if (label) labels[id] = label;
+				}
+			}
+		} catch {}
+	}
+
+	return labels;
+}
+
+/**
+ * Format a claim value to human-readable string
+ */
+function formatClaimValue(claim: WikidataClaim, entityLabels: Record<string, string>): string | null {
+	const snak = claim.mainsnak;
+	if (snak.snaktype !== "value" || !snak.datavalue) return null;
+
+	const { type, value } = snak.datavalue;
+
+	switch (type) {
+		case "wikibase-entityid": {
+			const entityVal = value as { id: string };
+			return entityLabels[entityVal.id] || entityVal.id;
+		}
+		case "string":
+			return value as string;
+		case "time": {
+			const timeVal = value as { time: string; precision: number };
+			return formatWikidataTime(timeVal.time, timeVal.precision);
+		}
+		case "quantity": {
+			const qtyVal = value as { amount: string; unit: string };
+			const amount = qtyVal.amount.replace(/^\+/, "");
+			// Extract unit Q-id if present
+			const unitMatch = qtyVal.unit.match(/Q\d+$/);
+			const unit = unitMatch ? entityLabels[unitMatch[0]] || "" : "";
+			return unit ? `${amount} ${unit}` : amount;
+		}
+		case "monolingualtext": {
+			const textVal = value as { text: string; language: string };
+			return textVal.text;
+		}
+		case "globecoordinate": {
+			const coordVal = value as { latitude: number; longitude: number };
+			return `${coordVal.latitude.toFixed(4)}, ${coordVal.longitude.toFixed(4)}`;
+		}
+		default:
+			return null;
+	}
+}
+
+/**
+ * Format Wikidata time value to readable date
+ */
+function formatWikidataTime(time: string, precision: number): string {
+	// Time format: +YYYY-MM-DDT00:00:00Z
+	const match = time.match(/^([+-]?\d+)-(\d{2})-(\d{2})/);
+	if (!match) return time;
+
+	const [, year, month, day] = match;
+	const yearNum = Number.parseInt(year, 10);
+	const absYear = Math.abs(yearNum);
+	const era = yearNum < 0 ? " BCE" : "";
+
+	// Precision: 9=year, 10=month, 11=day
+	if (precision >= 11) {
+		return `${day}/${month}/${absYear}${era}`;
+	}
+	if (precision >= 10) {
+		return `${month}/${absYear}${era}`;
+	}
+	return `${absYear}${era}`;
+}

package/src/core/tools/web-fetch-handlers/wikipedia.test.ts

@@ -0,0 +1,73 @@
+import { describe, expect, it } from "bun:test";
+import { handleWikipedia } from "./wikipedia";
+
+const SKIP = !process.env.WEB_FETCH_INTEGRATION;
+
+describe.skipIf(SKIP)("handleWikipedia", () => {
+	it("returns null for non-Wikipedia URLs", async () => {
+		const result = await handleWikipedia("https://example.com", 10);
+		expect(result).toBeNull();
+	});
+
+	it("returns null for Wikipedia URLs without /wiki/ path", async () => {
+		const result = await handleWikipedia("https://en.wikipedia.org/", 10);
+		expect(result).toBeNull();
+	});
+
+	it("fetches a known article with full metadata", async () => {
+		// "Computer" is a stable, well-established article
+		const result = await handleWikipedia("https://en.wikipedia.org/wiki/Computer", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikipedia");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("Computer");
+		expect(result?.url).toBe("https://en.wikipedia.org/wiki/Computer");
+		expect(result?.finalUrl).toBe("https://en.wikipedia.org/wiki/Computer");
+		expect(result?.truncated).toBe(false);
+		expect(result?.notes).toContain("Fetched via Wikipedia API");
+		expect(result?.fetchedAt).toBeDefined();
+		// Should be a valid ISO timestamp
+		expect(() => new Date(result?.fetchedAt ?? "")).not.toThrow();
+		// The handler should filter out References and External links sections
+		const content = result?.content ?? "";
+		const hasReferencesHeading = /^## References$/m.test(content);
+		const hasExternalLinksHeading = /^## External links$/m.test(content);
+		// At least one of these should be filtered out
+		expect(hasReferencesHeading || hasExternalLinksHeading).toBe(false);
+	});
+
+	it("handles different language wikis", async () => {
+		// German Wikipedia article for "Computer"
+		const result = await handleWikipedia("https://de.wikipedia.org/wiki/Computer", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikipedia");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toContain("Computer");
+	});
+
+	it("handles article with special characters in title", async () => {
+		// Article with special characters: "C++"
+		const result = await handleWikipedia("https://en.wikipedia.org/wiki/C%2B%2B", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikipedia");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toMatch(/C\+\+/);
+	});
+
+	it("handles article with spaces and parentheses in title", async () => {
+		// Artificial intelligence uses underscores for spaces
+		const result = await handleWikipedia("https://en.wikipedia.org/wiki/Artificial_intelligence", 20);
+		expect(result).not.toBeNull();
+		expect(result?.method).toBe("wikipedia");
+		expect(result?.contentType).toBe("text/markdown");
+		expect(result?.content).toMatch(/[Aa]rtificial intelligence/);
+	});
+
+	it("handles non-existent articles gracefully", async () => {
+		const result = await handleWikipedia(
+			"https://en.wikipedia.org/wiki/ThisArticleDefinitelyDoesNotExist123456789",
+			20,
+		);
+		expect(result).toBeNull();
+	});
+});

package/src/core/tools/web-fetch-handlers/wikipedia.ts

@@ -0,0 +1,91 @@
+import { parse as parseHtml } from "node-html-parser";
+import type { RenderResult, SpecialHandler } from "./types";
+import { finalizeOutput, loadPage } from "./types";
+
+/**
+ * Handle Wikipedia URLs via Wikipedia API
+ */
+export const handleWikipedia: SpecialHandler = async (url: string, timeout: number): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		// Match *.wikipedia.org
+		const wikiMatch = parsed.hostname.match(/^(\w+)\.wikipedia\.org$/);
+		if (!wikiMatch) return null;
+
+		const lang = wikiMatch[1];
+		const titleMatch = parsed.pathname.match(/\/wiki\/(.+)/);
+		if (!titleMatch) return null;
+
+		const title = decodeURIComponent(titleMatch[1]);
+		const fetchedAt = new Date().toISOString();
+
+		// Use Wikipedia API to get plain text extract
+		const apiUrl = `https://${lang}.wikipedia.org/api/rest_v1/page/summary/${encodeURIComponent(title)}`;
+		const summaryResult = await loadPage(apiUrl, { timeout });
+
+		let md = "";
+
+		if (summaryResult.ok) {
+			const summary = JSON.parse(summaryResult.content) as {
+				title: string;
+				description?: string;
+				extract: string;
+			};
+			md = `# ${summary.title}\n\n`;
+			if (summary.description) md += `*${summary.description}*\n\n`;
+			md += `${summary.extract}\n\n---\n\n`;
+		}
+
+		// Get full article content via mobile-html or parse API
+		const contentUrl = `https://${lang}.wikipedia.org/api/rest_v1/page/mobile-html/${encodeURIComponent(title)}`;
+		const contentResult = await loadPage(contentUrl, { timeout });
+
+		if (contentResult.ok) {
+			const doc = parseHtml(contentResult.content);
+
+			// Extract main content sections
+			const sections = doc.querySelectorAll("section");
+			for (const section of sections) {
+				const heading = section.querySelector("h2, h3, h4");
+				const headingText = heading?.text?.trim();
+
+				// Skip certain sections
+				if (
+					headingText &&
+					["References", "External links", "See also", "Notes", "Further reading"].includes(headingText)
+				) {
+					continue;
+				}
+
+				if (headingText) {
+					const level = heading?.tagName === "H2" ? "##" : "###";
+					md += `${level} ${headingText}\n\n`;
+				}
+
+				const paragraphs = section.querySelectorAll("p");
+				for (const p of paragraphs) {
+					const text = p.text?.trim();
+					if (text && text.length > 20) {
+						md += `${text}\n\n`;
+					}
+				}
+			}
+		}
+
+		if (!md) return null;
+
+		const output = finalizeOutput(md);
+		return {
+			url,
+			finalUrl: url,
+			contentType: "text/markdown",
+			method: "wikipedia",
+			content: output.content,
+			fetchedAt,
+			truncated: output.truncated,
+			notes: ["Fetched via Wikipedia API"],
+		};
+	} catch {}
+
+	return null;
+};