arxiv-api-wrapper 1.1.1 → 2.0.0

package/src/oaiParser.ts

@@ -0,0 +1,264 @@
+/**
+ * Parser for OAI-PMH XML responses from the arXiv OAI endpoint.
+ */
+
+import {
+  type OaiErrorCode,
+  type OaiIdentifyResponse,
+  type OaiMetadataFormat,
+  type OaiMetadataPrefix,
+  type OaiSet,
+  type OaiResumptionToken,
+  type OaiHeader,
+  type OaiRecord,
+  type OaiMetadata,
+  OaiError,
+} from './oaiTypes.js';
+import { XMLParser } from 'fast-xml-parser';
+
+const parser = new XMLParser({
+  ignoreAttributes: false,
+  attributeNamePrefix: '',
+  removeNSPrefix: true,
+  trimValues: true,
+  parseTagValue: false,
+});
+
+const VALID_ERROR_CODES: OaiErrorCode[] = [
+  'badArgument',
+  'badResumptionToken',
+  'badVerb',
+  'cannotDisseminateFormat',
+  'idDoesNotExist',
+  'noMetadataFormats',
+  'noRecordsMatch',
+  'noSetHierarchy',
+];
+
+const VALID_METADATA_PREFIXES: OaiMetadataPrefix[] = ['oai_dc', 'arXiv', 'arXivOld', 'arXivRaw'];
+
+function asArray<T>(x: T | T[] | undefined): T[] {
+  if (x == null) return [];
+  return Array.isArray(x) ? x : [x];
+}
+
+function str(v: unknown): string {
+  if (v == null) return '';
+  return String(v).trim();
+}
+
+function parseErrors(root: Record<string, unknown>): OaiError[] {
+  const errors: OaiError[] = [];
+  const raw = root.error;
+  if (raw == null) return errors;
+  const arr = asArray(raw);
+  for (const e of arr) {
+    const code = (e && typeof e === 'object' && 'code' in e && e.code) as string | undefined;
+    const msg = (e && typeof e === 'object' && '#text' in e ? e['#text'] : e) as string | unknown;
+    const messageText = typeof msg === 'string' ? msg : typeof msg !== 'undefined' ? String(msg) : '';
+    const codeStr = (code ?? 'badArgument') as OaiErrorCode;
+    if (VALID_ERROR_CODES.includes(codeStr)) {
+      errors.push(new OaiError(codeStr, messageText));
+    } else {
+      errors.push(new OaiError('badArgument', messageText || codeStr));
+    }
+  }
+  return errors;
+}
+
+function parseResumptionToken(el: unknown): OaiResumptionToken | undefined {
+  if (el == null || typeof el !== 'object') return undefined;
+  const o = el as Record<string, unknown>;
+  const value = str(o['#text'] ?? o['_'] ?? '');
+  if (!value) return undefined;
+  const token: OaiResumptionToken = { value };
+  if (o.expirationDate != null) token.expirationDate = str(o.expirationDate);
+  if (o.completeListSize != null) token.completeListSize = Number(o.completeListSize);
+  if (o.cursor != null) token.cursor = Number(o.cursor);
+  return token;
+}
+
+function parseHeader(el: unknown): OaiHeader {
+  const o = (el != null && typeof el === 'object' ? el : {}) as Record<string, unknown>;
+  const setSpec = asArray(o.setSpec).map((s) => str(s));
+  const status = o.status != null ? str(o.status) : undefined;
+  return {
+    identifier: str(o.identifier),
+    datestamp: str(o.datestamp),
+    setSpec,
+    ...(status === 'deleted' ? { status: 'deleted' as const } : {}),
+  };
+}
+
+/** Extract metadata as a plain object (first child of metadata is format-specific, e.g. dc or arXiv). */
+function parseMetadata(el: unknown): OaiMetadata | undefined {
+  if (el == null || typeof el !== 'object') return undefined;
+  const o = el as Record<string, unknown>;
+  // metadata has a single child (e.g. dc, arXiv) - flatten one level for convenience
+  const keys = Object.keys(o).filter((k) => !k.startsWith('@') && k !== '#text' && k !== '_');
+  if (keys.length === 0) return undefined;
+  const out: Record<string, unknown> = {};
+  for (const key of keys) {
+    const val = o[key];
+    if (val != null && typeof val === 'object' && !Array.isArray(val)) {
+      out[key] = val;
+    } else {
+      out[key] = val;
+    }
+  }
+  return out as unknown as OaiMetadata;
+}
+
+function parseRecord(el: unknown): OaiRecord {
+  const o = (el != null && typeof el === 'object' ? el : {}) as Record<string, unknown>;
+  const header = parseHeader(o.header);
+  const metadata = o.metadata != null ? parseMetadata(o.metadata) : undefined;
+  const about = o.about != null ? asArray(o.about) : undefined;
+  return { header, ...(metadata != null ? { metadata } : {}), ...(about != null ? { about } : {}) };
+}
+
+function getRoot(xml: string): Record<string, unknown> {
+  const doc = parser.parse(xml) as Record<string, unknown>;
+  const root = doc['OAI-PMH'] ?? doc['OAIPMH'] ?? doc;
+  if (root == null || typeof root !== 'object') {
+    throw new OaiError('badArgument', 'Invalid OAI-PMH response: no root element');
+  }
+  return root as Record<string, unknown>;
+}
+
+/**
+ * Parse OAI-PMH response and throw OaiError if the response contains error elements.
+ * Returns the verb-specific payload (e.g. Identify, ListRecords).
+ */
+export function parseOaiResponse(xml: string): Record<string, unknown> {
+  const root = getRoot(xml);
+  const errors = parseErrors(root);
+  if (errors.length > 0) {
+    const first = errors[0];
+    throw first;
+  }
+  return root;
+}
+
+/**
+ * Parse an Identify response body.
+ */
+export function parseIdentify(xml: string): OaiIdentifyResponse {
+  const root = parseOaiResponse(xml);
+  const id = (root.Identify ?? root.identify) as Record<string, unknown> | undefined;
+  if (!id || typeof id !== 'object') {
+    throw new OaiError('badArgument', 'Invalid Identify response: missing Identify element');
+  }
+  return {
+    repositoryName: str(id.repositoryName),
+    baseURL: str(id.baseURL),
+    protocolVersion: str(id.protocolVersion),
+    adminEmail: asArray(id.adminEmail).map((e) => str(e)),
+    earliestDatestamp: str(id.earliestDatestamp),
+    deletedRecord: (str(id.deletedRecord) || 'no') as 'no' | 'persistent' | 'transient',
+    granularity: (str(id.granularity) || 'YYYY-MM-DD') as 'YYYY-MM-DD' | 'YYYY-MM-DDThh:mm:ssZ',
+    compression: asArray(id.compression).map((c) => str(c)).filter(Boolean),
+    description: id.description != null ? asArray(id.description) : undefined,
+  };
+}
+
+/**
+ * Parse a ListMetadataFormats response body.
+ */
+export function parseListMetadataFormats(xml: string): OaiMetadataFormat[] {
+  const root = parseOaiResponse(xml);
+  const list = root.ListMetadataFormats ?? root.listMetadataFormats;
+  if (!list || typeof list !== 'object') {
+    throw new OaiError('badArgument', 'Invalid ListMetadataFormats response');
+  }
+  const arr = (list as Record<string, unknown>).metadataFormat;
+  const formats = asArray(arr);
+  return formats.map((f: unknown) => {
+    const o = (f && typeof f === 'object' ? f : {}) as Record<string, unknown>;
+    const metadataPrefix = str(o.metadataPrefix);
+    if (!VALID_METADATA_PREFIXES.includes(metadataPrefix as OaiMetadataPrefix)) {
+      throw new OaiError(
+        'cannotDisseminateFormat',
+        `Unsupported metadataPrefix in ListMetadataFormats response: ${metadataPrefix}`
+      );
+    }
+    return {
+      metadataPrefix: metadataPrefix as OaiMetadataPrefix,
+      schema: str(o.schema),
+      metadataNamespace: str(o.metadataNamespace),
+    };
+  });
+}
+
+/**
+ * Parse a ListSets response body.
+ */
+export function parseListSets(xml: string): { sets: OaiSet[]; resumptionToken?: OaiResumptionToken } {
+  const root = parseOaiResponse(xml);
+  const list = root.ListSets ?? root.listSets;
+  if (!list || typeof list !== 'object') {
+    throw new OaiError('badArgument', 'Invalid ListSets response');
+  }
+  const o = list as Record<string, unknown>;
+  const setArr = asArray(o.set);
+  const sets: OaiSet[] = setArr.map((s: unknown) => {
+    const set = (s && typeof s === 'object' ? s : {}) as Record<string, unknown>;
+    return {
+      setSpec: str(set.setSpec),
+      setName: str(set.setName),
+      ...(set.setDescription != null ? { setDescription: set.setDescription } : {}),
+    };
+  });
+  const resumptionToken = parseResumptionToken(o.resumptionToken);
+  return { sets, ...(resumptionToken ? { resumptionToken } : {}) };
+}
+
+/**
+ * Parse a GetRecord response body.
+ */
+export function parseGetRecord(xml: string): OaiRecord {
+  const root = parseOaiResponse(xml);
+  const getRecord = root.GetRecord ?? root.getRecord;
+  if (!getRecord || typeof getRecord !== 'object') {
+    throw new OaiError('badArgument', 'Invalid GetRecord response');
+  }
+  const record = (getRecord as Record<string, unknown>).record;
+  if (!record) throw new OaiError('badArgument', 'Invalid GetRecord response: missing record');
+  return parseRecord(record);
+}
+
+/**
+ * Parse a ListIdentifiers response body.
+ */
+export function parseListIdentifiers(
+  xml: string
+): { headers: OaiHeader[]; resumptionToken?: OaiResumptionToken } {
+  const root = parseOaiResponse(xml);
+  const list = root.ListIdentifiers ?? root.listIdentifiers;
+  if (!list || typeof list !== 'object') {
+    throw new OaiError('badArgument', 'Invalid ListIdentifiers response');
+  }
+  const o = list as Record<string, unknown>;
+  const headerArr = asArray(o.header);
+  const headers = headerArr.map((h: unknown) => parseHeader(h));
+  const resumptionToken = parseResumptionToken(o.resumptionToken);
+  return { headers, ...(resumptionToken ? { resumptionToken } : {}) };
+}
+
+/**
+ * Parse a ListRecords response body.
+ */
+export function parseListRecords(
+  xml: string
+): { records: OaiRecord[]; resumptionToken?: OaiResumptionToken } {
+  const root = parseOaiResponse(xml);
+  const list = root.ListRecords ?? root.listRecords;
+  if (!list || typeof list !== 'object') {
+    throw new OaiError('badArgument', 'Invalid ListRecords response');
+  }
+  const o = list as Record<string, unknown>;
+  const recordArr = asArray(o.record);
+  const records = recordArr.map((r: unknown) => parseRecord(r));
+  const resumptionToken = parseResumptionToken(o.resumptionToken);
+  return { records, ...(resumptionToken ? { resumptionToken } : {}) };
+}

package/src/oaiToArxiv.ts

@@ -0,0 +1,204 @@
+import type { ArxivAuthor, ArxivEntry, ArxivLink, ArxivQueryResult } from './types.js';
+import type { OaiListRecordsResult, OaiMetadata, OaiRecord } from './oaiTypes.js';
+
+const OAI_BASE_URL = 'https://oaipmh.arxiv.org/oai';
+const ARXIV_ABS_BASE = 'https://arxiv.org/abs/';
+const ARXIV_PDF_BASE = 'https://arxiv.org/pdf/';
+
+function normalizeWhitespace(value: string): string {
+  return value.replace(/\s+/g, ' ').trim();
+}
+
+function asArray<T>(value: T | T[] | undefined): T[] {
+  if (value == null) return [];
+  return Array.isArray(value) ? value : [value];
+}
+
+function firstNonEmpty(values: string[]): string {
+  return values.map((v) => normalizeWhitespace(v)).find(Boolean) ?? '';
+}
+
+function toStringArray(value: string | string[] | undefined): string[] {
+  return asArray(value).map((v) => normalizeWhitespace(String(v))).filter(Boolean);
+}
+
+function oaiIdentifierToArxivId(identifier: string): string {
+  return identifier.replace(/^oai:arXiv\.org:/i, '').trim();
+}
+
+function buildDefaultLinks(arxivId: string): ArxivLink[] {
+  if (!arxivId) return [];
+  return [
+    { href: `${ARXIV_ABS_BASE}${arxivId}`, rel: 'alternate', type: 'text/html' },
+    { href: `${ARXIV_PDF_BASE}${arxivId}`, rel: 'related', type: 'application/pdf', title: 'pdf' },
+  ];
+}
+
+function splitCategories(value: string | undefined): string[] {
+  if (!value) return [];
+  return value.trim().split(/\s+/).filter(Boolean);
+}
+
+function parseAuthorsList(value: string | undefined): ArxivAuthor[] {
+  if (!value) return [];
+  return value
+    .split(/\s*,\s*/)
+    .map((name) => normalizeWhitespace(name))
+    .filter(Boolean)
+    .map((name) => ({ name }));
+}
+
+function withVersionSuffix(baseId: string, version: string | undefined): string {
+  if (!version) return baseId;
+  if (/v\d+$/i.test(baseId)) return baseId;
+  return `${baseId}${version}`;
+}
+
+function extractLatestRawVersion(
+  versionValue: { version?: string; date: string; size?: string; source_type?: string } | { version?: string; date: string; size?: string; source_type?: string }[]
+): { version?: string; date: string; size?: string; source_type?: string } {
+  const versions = asArray(versionValue);
+  if (versions.length === 0) return { date: '' };
+  const sorted = [...versions].sort((a, b) => {
+    const va = Number((a.version ?? '').replace(/^v/i, ''));
+    const vb = Number((b.version ?? '').replace(/^v/i, ''));
+    if (Number.isNaN(va) || Number.isNaN(vb)) return 0;
+    return va - vb;
+  });
+  return sorted[sorted.length - 1];
+}
+
+function metadataToEntry(record: OaiRecord, metadata: OaiMetadata): ArxivEntry {
+  const fallbackArxivId = oaiIdentifierToArxivId(record.header.identifier);
+  const fallbackUpdated = record.header.datestamp;
+
+  if ('arXiv' in metadata) {
+    const rawAuthors = asArray(metadata.arXiv.authors?.author);
+    const authors: ArxivAuthor[] = rawAuthors.map((a) => {
+      const name = [a.forenames, a.keyname, a.suffix].filter(Boolean).join(' ').trim();
+      const affiliations = toStringArray(a.affiliation);
+      return { name: name || a.keyname, ...(affiliations[0] ? { affiliation: affiliations[0] } : {}) };
+    });
+    const arxivId = metadata.arXiv.id || fallbackArxivId;
+    const categories = splitCategories(metadata.arXiv.categories);
+    return {
+      id: `${ARXIV_ABS_BASE}${arxivId}`,
+      arxivId,
+      title: normalizeWhitespace(metadata.arXiv.title ?? ''),
+      summary: normalizeWhitespace(metadata.arXiv.abstract ?? ''),
+      published: metadata.arXiv.created ?? fallbackUpdated,
+      updated: metadata.arXiv.updated ?? fallbackUpdated,
+      authors,
+      categories,
+      ...(categories[0] ? { primaryCategory: categories[0] } : {}),
+      links: buildDefaultLinks(arxivId),
+      ...(metadata.arXiv.doi ? { doi: metadata.arXiv.doi } : {}),
+      ...(metadata.arXiv['journal-ref'] ? { journalRef: metadata.arXiv['journal-ref'] } : {}),
+      ...(metadata.arXiv.comments ? { comment: normalizeWhitespace(metadata.arXiv.comments) } : {}),
+    };
+  }
+
+  if ('arXivRaw' in metadata) {
+    const latestVersion = extractLatestRawVersion(metadata.arXivRaw.version);
+    const arxivId = withVersionSuffix(metadata.arXivRaw.id || fallbackArxivId, latestVersion.version);
+    const categories = splitCategories(metadata.arXivRaw.categories);
+    return {
+      id: `${ARXIV_ABS_BASE}${arxivId}`,
+      arxivId,
+      title: normalizeWhitespace(metadata.arXivRaw.title ?? ''),
+      summary: normalizeWhitespace(metadata.arXivRaw.abstract ?? ''),
+      published: asArray(metadata.arXivRaw.version)[0]?.date ?? fallbackUpdated,
+      updated: latestVersion.date || fallbackUpdated,
+      authors: parseAuthorsList(metadata.arXivRaw.authors),
+      categories,
+      ...(categories[0] ? { primaryCategory: categories[0] } : {}),
+      links: buildDefaultLinks(arxivId),
+      ...(metadata.arXivRaw.doi ? { doi: metadata.arXivRaw.doi } : {}),
+      ...(metadata.arXivRaw['journal-ref'] ? { journalRef: metadata.arXivRaw['journal-ref'] } : {}),
+      ...(metadata.arXivRaw.comments ? { comment: normalizeWhitespace(metadata.arXivRaw.comments) } : {}),
+    };
+  }
+
+  if ('arXivOld' in metadata) {
+    const arxivId = metadata.arXivOld.id || fallbackArxivId;
+    const categories = splitCategories(metadata.arXivOld.categories);
+    return {
+      id: `${ARXIV_ABS_BASE}${arxivId}`,
+      arxivId,
+      title: normalizeWhitespace(metadata.arXivOld.title ?? ''),
+      summary: normalizeWhitespace(metadata.arXivOld.abstract ?? ''),
+      published: fallbackUpdated,
+      updated: fallbackUpdated,
+      authors: parseAuthorsList(metadata.arXivOld.authors),
+      categories,
+      ...(categories[0] ? { primaryCategory: categories[0] } : {}),
+      links: buildDefaultLinks(arxivId),
+      ...(metadata.arXivOld.doi ? { doi: metadata.arXivOld.doi } : {}),
+      ...(metadata.arXivOld['journal-ref'] ? { journalRef: metadata.arXivOld['journal-ref'] } : {}),
+      ...(metadata.arXivOld.comments ? { comment: normalizeWhitespace(metadata.arXivOld.comments) } : {}),
+    };
+  }
+
+  const dc = metadata.dc;
+  const identifierValues = toStringArray(dc.identifier);
+  const identifierFromDc = firstNonEmpty(identifierValues);
+  const arxivIdFromDc =
+    oaiIdentifierToArxivId(identifierFromDc).replace(/^https?:\/\/arxiv\.org\/abs\//i, '') ||
+    fallbackArxivId;
+  const creators = toStringArray(dc.creator);
+  const categories = toStringArray(dc.subject);
+  const published = firstNonEmpty(toStringArray(dc.date)) || fallbackUpdated;
+  const summary = firstNonEmpty(toStringArray(dc.description));
+  const title = firstNonEmpty(toStringArray(dc.title));
+
+  return {
+    id: `${ARXIV_ABS_BASE}${arxivIdFromDc}`,
+    arxivId: arxivIdFromDc,
+    title,
+    summary,
+    published,
+    updated: fallbackUpdated,
+    authors: creators.map((name) => ({ name })),
+    categories,
+    ...(categories[0] ? { primaryCategory: categories[0] } : {}),
+    links: buildDefaultLinks(arxivIdFromDc),
+  };
+}
+
+/**
+ * Convert one OAI record to the package's ArxivEntry shape.
+ * Returns null for deleted records or records that do not include metadata.
+ */
+export function oaiRecordToArxivEntry(record: OaiRecord): ArxivEntry | null {
+  if (record.header.status === 'deleted' || record.metadata == null) return null;
+  return metadataToEntry(record, record.metadata);
+}
+
+/** Convert OAI records to ArxivEntry array, skipping deleted/metadata-less records. */
+export function oaiRecordsToArxivEntries(records: OaiRecord[]): ArxivEntry[] {
+  return records
+    .map((record) => oaiRecordToArxivEntry(record))
+    .filter((entry): entry is ArxivEntry => entry != null);
+}
+
+/**
+ * Convert an OAI ListRecords result to the same shape returned by getArxivEntries().
+ * Feed values are synthesized from OAI response data.
+ */
+export function oaiListRecordsToArxivQueryResult(result: OaiListRecordsResult): ArxivQueryResult {
+  const entries = oaiRecordsToArxivEntries(result.records);
+  const latestUpdated = entries.map((e) => e.updated).sort().at(-1) ?? '';
+  const startIndex = Math.max(0, (result.resumptionToken?.cursor ?? entries.length) - entries.length);
+  return {
+    feed: {
+      id: OAI_BASE_URL,
+      updated: latestUpdated,
+      title: 'arXiv OAI converted records',
+      link: OAI_BASE_URL,
+      totalResults: result.resumptionToken?.completeListSize ?? entries.length,
+      startIndex,
+      itemsPerPage: entries.length,
+    },
+    entries,
+  };
+}