@f5xc-salesdemos/xcsh 2.2.0 → 14.0.2

package/src/web/scrapers/openlibrary.ts

@@ -0,0 +1,336 @@
+import { tryParseJson } from "@f5xc-salesdemos/pi-utils";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, loadPage } from "./types";
+
+interface OpenLibraryAuthor {
+	name?: string;
+	url?: string;
+}
+
+interface OpenLibrarySubject {
+	name: string;
+	url?: string;
+}
+
+interface OpenLibraryPublisher {
+	name: string;
+}
+
+interface OpenLibraryCover {
+	small?: string;
+	medium?: string;
+	large?: string;
+}
+
+interface OpenLibraryWork {
+	title: string;
+	authors?: Array<{ author: { key: string } }>;
+	description?: string | { value: string };
+	subjects?: string[];
+	subject_places?: string[];
+	subject_times?: string[];
+	covers?: number[];
+	first_publish_date?: string;
+}
+
+interface OpenLibraryEdition {
+	title: string;
+	authors?: Array<{ key: string }>;
+	publishers?: string[];
+	publish_date?: string;
+	number_of_pages?: number;
+	isbn_10?: string[];
+	isbn_13?: string[];
+	covers?: number[];
+	description?: string | { value: string };
+	subjects?: string[];
+	works?: Array<{ key: string }>;
+}
+
+interface OpenLibraryBooksApiResponse {
+	[key: string]: {
+		title: string;
+		authors?: OpenLibraryAuthor[];
+		publishers?: OpenLibraryPublisher[];
+		publish_date?: string;
+		number_of_pages?: number;
+		subjects?: OpenLibrarySubject[];
+		cover?: OpenLibraryCover;
+		url?: string;
+		identifiers?: {
+			isbn_10?: string[];
+			isbn_13?: string[];
+			openlibrary?: string[];
+		};
+	};
+}
+
+/**
+ * Handle Open Library URLs via their API
+ */
+export const handleOpenLibrary: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!parsed.hostname.includes("openlibrary.org")) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const path = parsed.pathname;
+
+		// Match URL patterns
+		const workMatch = path.match(/^\/works\/(OL\d+W)/i);
+		const editionMatch = path.match(/^\/books\/(OL\d+M)/i);
+		const isbnMatch = path.match(/^\/isbn\/(\d{10}|\d{13})/i);
+
+		let md: string | null = null;
+
+		if (workMatch) {
+			md = await fetchWork(workMatch[1], timeout, signal);
+		} else if (editionMatch) {
+			md = await fetchEdition(editionMatch[1], timeout, signal);
+		} else if (isbnMatch) {
+			md = await fetchByIsbn(isbnMatch[1], timeout, signal);
+		}
+
+		if (!md) return null;
+
+		return buildResult(md, { url, method: "openlibrary", fetchedAt, notes: ["Fetched via Open Library API"] });
+	} catch {}
+
+	return null;
+};
+
+async function fetchWork(workId: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
+	const apiUrl = `https://openlibrary.org/works/${workId}.json`;
+	const result = await loadPage(apiUrl, { timeout, signal });
+	if (!result.ok) return null;
+
+	const work = tryParseJson<OpenLibraryWork>(result.content);
+	if (!work) return null;
+
+	let md = `# ${work.title}\n\n`;
+
+	// Fetch author names if we have author keys
+	if (work.authors?.length) {
+		const authorNames = await fetchAuthorNames(
+			work.authors.map(a => a.author.key),
+			timeout,
+			signal,
+		);
+		if (authorNames.length) {
+			md += `**Authors:** ${authorNames.join(", ")}\n`;
+		}
+	}
+
+	if (work.first_publish_date) {
+		md += `**First Published:** ${work.first_publish_date}\n`;
+	}
+
+	if (work.covers?.length) {
+		const coverId = work.covers[0];
+		md += `**Cover:** https://covers.openlibrary.org/b/id/${coverId}-L.jpg\n`;
+	}
+
+	md += `**Open Library:** https://openlibrary.org/works/${workId}\n`;
+	md += "\n";
+
+	const description = extractDescription(work.description);
+	if (description) {
+		md += `## Description\n\n${description}\n\n`;
+	}
+
+	if (work.subjects?.length) {
+		md += `## Subjects\n\n${work.subjects.slice(0, 20).join(", ")}\n`;
+	}
+
+	return md;
+}
+
+async function fetchEdition(editionId: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
+	const apiUrl = `https://openlibrary.org/books/${editionId}.json`;
+	const result = await loadPage(apiUrl, { timeout, signal });
+	if (!result.ok) return null;
+
+	const edition = tryParseJson<OpenLibraryEdition>(result.content);
+	if (!edition) return null;
+
+	let md = `# ${edition.title}\n\n`;
+
+	// Fetch author names
+	if (edition.authors?.length) {
+		const authorNames = await fetchAuthorNames(
+			edition.authors.map(a => a.key),
+			timeout,
+			signal,
+		);
+		if (authorNames.length) {
+			md += `**Authors:** ${authorNames.join(", ")}\n`;
+		}
+	}
+
+	if (edition.publishers?.length) {
+		md += `**Publishers:** ${edition.publishers.join(", ")}\n`;
+	}
+
+	if (edition.publish_date) {
+		md += `**Published:** ${edition.publish_date}\n`;
+	}
+
+	if (edition.number_of_pages) {
+		md += `**Pages:** ${edition.number_of_pages}\n`;
+	}
+
+	const isbns = [...(edition.isbn_13 || []), ...(edition.isbn_10 || [])];
+	if (isbns.length) {
+		md += `**ISBN:** ${isbns[0]}\n`;
+	}
+
+	if (edition.covers?.length) {
+		const coverId = edition.covers[0];
+		md += `**Cover:** https://covers.openlibrary.org/b/id/${coverId}-L.jpg\n`;
+	}
+
+	md += `**Open Library:** https://openlibrary.org/books/${editionId}\n`;
+
+	if (edition.works?.length) {
+		const workKey = edition.works[0].key.replace("/works/", "");
+		md += `**Work:** https://openlibrary.org/works/${workKey}\n`;
+	}
+
+	md += "\n";
+
+	const description = extractDescription(edition.description);
+	if (description) {
+		md += `## Description\n\n${description}\n\n`;
+	}
+
+	if (edition.subjects?.length) {
+		md += `## Subjects\n\n${edition.subjects.slice(0, 20).join(", ")}\n`;
+	}
+
+	return md;
+}
+
+async function fetchByIsbn(isbn: string, timeout: number, signal?: AbortSignal): Promise<string | null> {
+	const apiUrl = `https://openlibrary.org/api/books?bibkeys=ISBN:${isbn}&format=json&jscmd=data`;
+	let result = await loadPage(apiUrl, { timeout, signal });
+	if (!result.ok) {
+		result = await loadPage(apiUrl, { timeout, signal });
+	}
+	if (!result.ok) {
+		return `# Open Library Book\n\n**ISBN:** ${isbn}\n\nBook details are currently unavailable from the Open Library books API.\n`;
+	}
+
+	const data = tryParseJson<OpenLibraryBooksApiResponse>(result.content);
+	if (!data) return null;
+
+	const key = `ISBN:${isbn}`;
+	const book = data[key];
+	if (!book) {
+		// Fallback: search endpoint still returns docs when api/books misses a key.
+		const searchUrl = `https://openlibrary.org/search.json?isbn=${encodeURIComponent(isbn)}&limit=1`;
+		const searchResult = await loadPage(searchUrl, { timeout, signal });
+		if (!searchResult.ok) {
+			return `# Open Library Book\n\n**ISBN:** ${isbn}\n\nBook details are currently unavailable from the Open Library search API.\n`;
+		}
+		const searchData = tryParseJson<{
+			docs?: Array<{
+				title?: string;
+				author_name?: string[];
+				first_publish_year?: number;
+				key?: string;
+			}>;
+		}>(searchResult.content);
+		const doc = searchData?.docs?.[0];
+		if (!doc?.title) {
+			return `# Open Library Book\n\n**ISBN:** ${isbn}\n\nBook details are currently unavailable from Open Library.\n`;
+		}
+
+		let fallbackMd = `# ${doc.title}\n\n`;
+		if (doc.author_name?.length) {
+			fallbackMd += `**Authors:** ${doc.author_name.join(", ")}\n`;
+		}
+		if (doc.first_publish_year) {
+			fallbackMd += `**First Published:** ${doc.first_publish_year}\n`;
+		}
+		fallbackMd += `**ISBN:** ${isbn}\n`;
+		if (doc.key) {
+			fallbackMd += `**Open Library:** https://openlibrary.org${doc.key}\n`;
+		}
+
+		return fallbackMd;
+	}
+
+	let md = `# ${book.title}\n\n`;
+
+	if (book.authors?.length) {
+		md += `**Authors:** ${book.authors.map(a => a.name).join(", ")}\n`;
+	}
+
+	if (book.publishers?.length) {
+		md += `**Publishers:** ${book.publishers.map(p => p.name).join(", ")}\n`;
+	}
+
+	if (book.publish_date) {
+		md += `**Published:** ${book.publish_date}\n`;
+	}
+
+	if (book.number_of_pages) {
+		md += `**Pages:** ${book.number_of_pages}\n`;
+	}
+
+	md += `**ISBN:** ${isbn}\n`;
+
+	if (book.cover?.large || book.cover?.medium) {
+		md += `**Cover:** ${book.cover.large || book.cover.medium}\n`;
+	}
+
+	if (book.url) {
+		md += `**Open Library:** ${book.url}\n`;
+	}
+
+	md += "\n";
+
+	if (book.subjects?.length) {
+		md += `## Subjects\n\n${book.subjects
+			.slice(0, 20)
+			.map(s => s.name)
+			.join(", ")}\n`;
+	}
+
+	return md;
+}
+
+async function fetchAuthorNames(authorKeys: string[], timeout: number, signal?: AbortSignal): Promise<string[]> {
+	const names: string[] = [];
+
+	// Fetch authors in parallel (limit to first 5)
+	const promises = authorKeys.slice(0, 5).map(async key => {
+		const authorKey = key.startsWith("/authors/") ? key : `/authors/${key}`;
+		const apiUrl = `https://openlibrary.org${authorKey}.json`;
+		try {
+			const result = await loadPage(apiUrl, { timeout: Math.min(timeout, 5), signal });
+			if (result.ok) {
+				const author = JSON.parse(result.content) as { name?: string };
+				return author.name || null;
+			}
+		} catch {}
+		return null;
+	});
+
+	const results = await Promise.all(promises);
+	for (const name of results) {
+		if (name) names.push(name);
+	}
+
+	return names;
+}
+
+function extractDescription(desc: string | { value: string } | undefined): string | null {
+	if (!desc) return null;
+	if (typeof desc === "string") return desc;
+	return desc.value || null;
+}

package/src/web/scrapers/orcid.ts

@@ -0,0 +1,286 @@
+/**
+ * ORCID handler for web-fetch
+ */
+
+import { tryParseJson } from "@f5xc-salesdemos/pi-utils";
+import type { RenderResult, SpecialHandler } from "./types";
+import { buildResult, loadPage } from "./types";
+
+const MAX_WORKS = 50;
+const ORCID_PATTERN = /\/(\d{4}-\d{4}-\d{4}-\d{3}[\dXx])(?:\/|$)/;
+
+interface OrcidName {
+	"given-names"?: { value?: string };
+	"family-name"?: { value?: string };
+	"credit-name"?: { value?: string };
+}
+
+interface OrcidBiography {
+	content?: string;
+}
+
+interface OrcidPerson {
+	name?: OrcidName;
+	biography?: OrcidBiography;
+}
+
+interface OrcidSummaryDate {
+	year?: { value?: string };
+	month?: { value?: string };
+	day?: { value?: string };
+}
+
+interface OrcidOrganizationAddress {
+	city?: string;
+	region?: string;
+	country?: string;
+}
+
+interface OrcidOrganization {
+	name?: string;
+	address?: OrcidOrganizationAddress;
+}
+
+interface OrcidAffiliationSummary {
+	organization?: OrcidOrganization;
+	"role-title"?: string;
+	"department-name"?: string;
+	"start-date"?: OrcidSummaryDate;
+	"end-date"?: OrcidSummaryDate;
+}
+
+interface OrcidAffiliationGroupSummary {
+	"employment-summary"?: OrcidAffiliationSummary;
+	"education-summary"?: OrcidAffiliationSummary;
+}
+
+interface OrcidAffiliationGroup {
+	summaries?: OrcidAffiliationGroupSummary[];
+}
+
+interface OrcidAffiliationsContainer {
+	"affiliation-group"?: OrcidAffiliationGroup[];
+	"employment-summary"?: OrcidAffiliationSummary[];
+	"education-summary"?: OrcidAffiliationSummary[];
+}
+
+interface OrcidWorkTitle {
+	title?: { value?: string };
+}
+
+interface OrcidWorkSummary {
+	title?: OrcidWorkTitle;
+}
+
+interface OrcidWorkGroup {
+	"work-summary"?: OrcidWorkSummary[];
+}
+
+interface OrcidWorksContainer {
+	group?: OrcidWorkGroup[];
+}
+
+interface OrcidActivitiesSummary {
+	employments?: OrcidAffiliationsContainer;
+	educations?: OrcidAffiliationsContainer;
+	works?: OrcidWorksContainer;
+}
+
+interface OrcidRecord {
+	"orcid-identifier"?: { path?: string; uri?: string };
+	person?: OrcidPerson;
+	"activities-summary"?: OrcidActivitiesSummary;
+}
+
+function isOrcidHost(hostname: string): boolean {
+	return hostname === "orcid.org" || hostname === "www.orcid.org";
+}
+
+function extractOrcidId(pathname: string): string | null {
+	const match = pathname.match(ORCID_PATTERN);
+	return match?.[1] ?? null;
+}
+
+function formatName(name?: OrcidName): string | null {
+	const credit = name?.["credit-name"]?.value?.trim();
+	if (credit) return credit;
+
+	const given = name?.["given-names"]?.value?.trim();
+	const family = name?.["family-name"]?.value?.trim();
+	if (given && family) return `${given} ${family}`;
+	return given || family || null;
+}
+
+function formatDate(date?: OrcidSummaryDate): string | null {
+	const year = date?.year?.value;
+	if (!year) return null;
+
+	const month = date?.month?.value;
+	const day = date?.day?.value;
+	if (month && day) {
+		return `${year}-${month.padStart(2, "0")}-${day.padStart(2, "0")}`;
+	}
+	if (month) return `${year}-${month.padStart(2, "0")}`;
+	return year;
+}
+
+function collectAffiliations(
+	container: OrcidAffiliationsContainer | undefined,
+	key: "employment-summary" | "education-summary",
+): OrcidAffiliationSummary[] {
+	const summaries: OrcidAffiliationSummary[] = [];
+
+	if (!container) return summaries;
+
+	const direct = container[key];
+	if (direct?.length) summaries.push(...direct);
+
+	const groups = container["affiliation-group"];
+	if (groups?.length) {
+		for (const group of groups) {
+			const groupSummaries = group.summaries || [];
+			for (const summary of groupSummaries) {
+				const entry = summary[key];
+				if (entry) summaries.push(entry);
+			}
+		}
+	}
+
+	return summaries;
+}
+
+function formatAffiliation(summary: OrcidAffiliationSummary): string | null {
+	const organization = summary.organization?.name?.trim();
+	const role = summary["role-title"]?.trim();
+	const department = summary["department-name"]?.trim();
+
+	const address = summary.organization?.address;
+	const locationParts = [address?.city, address?.region, address?.country].filter(Boolean) as string[];
+	const location = locationParts.length > 0 ? locationParts.join(", ") : null;
+
+	const start = formatDate(summary["start-date"]);
+	const end = formatDate(summary["end-date"]);
+	let dates: string | null = null;
+	if (start && end) {
+		dates = `${start} - ${end}`;
+	} else if (start) {
+		dates = `${start} - Present`;
+	} else if (end) {
+		dates = `Until ${end}`;
+	}
+
+	const label = organization || role || department;
+	if (!label) return null;
+
+	const details: string[] = [];
+	if (organization && role) details.push(role);
+	if (!organization && role && department) details.push(department);
+	if (organization && department) details.push(`Dept: ${department}`);
+	if (location) details.push(`Location: ${location}`);
+	if (dates) details.push(`Dates: ${dates}`);
+
+	if (details.length === 0) return label;
+	return `${label} (${details.join("; ")})`;
+}
+
+function collectWorkTitles(container: OrcidWorksContainer | undefined): string[] {
+	const titles: string[] = [];
+	const seen = new Set<string>();
+	const groups = container?.group || [];
+
+	for (const group of groups) {
+		const summaries = group["work-summary"] || [];
+		for (const summary of summaries) {
+			const title = summary.title?.title?.value?.trim();
+			if (!title || seen.has(title)) continue;
+			seen.add(title);
+			titles.push(title);
+			if (titles.length >= MAX_WORKS) return titles;
+		}
+	}
+
+	return titles;
+}
+
+export const handleOrcid: SpecialHandler = async (
+	url: string,
+	timeout: number,
+	signal?: AbortSignal,
+): Promise<RenderResult | null> => {
+	try {
+		const parsed = new URL(url);
+		if (!isOrcidHost(parsed.hostname)) return null;
+
+		const orcid = extractOrcidId(parsed.pathname);
+		if (!orcid) return null;
+
+		const fetchedAt = new Date().toISOString();
+		const apiUrl = `https://pub.orcid.org/v3.0/${orcid}/record`;
+
+		const result = await loadPage(apiUrl, {
+			timeout,
+			headers: { Accept: "application/json" },
+			signal,
+		});
+
+		if (!result.ok || !result.content) return null;
+
+		const record = tryParseJson<OrcidRecord>(result.content);
+		if (!record) return null;
+
+		const personName = formatName(record.person?.name);
+		const biography = record.person?.biography?.content?.trim();
+
+		const activities = record["activities-summary"];
+		const employments = collectAffiliations(activities?.employments, "employment-summary");
+		const educations = collectAffiliations(activities?.educations, "education-summary");
+		const works = collectWorkTitles(activities?.works);
+
+		let md = `# ${personName || "ORCID Profile"}\n\n`;
+		md += `**ORCID:** ${orcid}\n`;
+		md += `**ORCID Profile:** https://orcid.org/${orcid}\n\n`;
+
+		md += "## Biography\n\n";
+		md += biography ? `${biography}\n\n` : "No biography available.\n\n";
+
+		md += "## Affiliations\n\n";
+		let hasAffiliations = false;
+
+		if (employments.length > 0) {
+			hasAffiliations = true;
+			md += "### Employment\n\n";
+			for (const summary of employments) {
+				const line = formatAffiliation(summary);
+				if (line) md += `- ${line}\n`;
+			}
+			md += "\n";
+		}
+
+		if (educations.length > 0) {
+			hasAffiliations = true;
+			md += "### Education\n\n";
+			for (const summary of educations) {
+				const line = formatAffiliation(summary);
+				if (line) md += `- ${line}\n`;
+			}
+			md += "\n";
+		}
+
+		if (!hasAffiliations) {
+			md += "No affiliations available.\n\n";
+		}
+
+		md += "## Works\n\n";
+		if (works.length > 0) {
+			for (const title of works) {
+				md += `- ${title}\n`;
+			}
+		} else {
+			md += "No works available.\n";
+		}
+
+		return buildResult(md, { url, method: "orcid-api", fetchedAt, notes: ["Fetched via ORCID Public API"] });
+	} catch {
+		return null;
+	}
+};