arxiv-api-wrapper 1.1.1 → 2.0.1

package/src/oaiTypes.ts

@@ -0,0 +1,260 @@
+/**
+ * OAI-PMH types for the arXiv OAI interface.
+ * @see https://info.arxiv.org/help/oa/index.html#open-archives-initiative-oai
+ * @see https://www.openarchives.org/OAI/openarchivesprotocol.html
+ */
+
+import type { ArxivRateLimitConfig } from './types.js';
+
+/** OAI-PMH error codes. */
+export type OaiErrorCode =
+  | 'badArgument'
+  | 'badResumptionToken'
+  | 'badVerb'
+  | 'cannotDisseminateFormat'
+  | 'idDoesNotExist'
+  | 'noMetadataFormats'
+  | 'noRecordsMatch'
+  | 'noSetHierarchy';
+
+/** Options shared by all OAI request functions. */
+export interface OaiRequestOptions {
+  /** Request timeout in milliseconds (default: 10000). */
+  timeoutMs?: number;
+  /** Number of retry attempts for failed requests (default: 3). */
+  retries?: number;
+  /** Custom User-Agent header for requests. */
+  userAgent?: string;
+  /** Rate limiting configuration. */
+  rateLimit?: ArxivRateLimitConfig;
+}
+
+/** Response to the Identify verb. */
+export interface OaiIdentifyResponse {
+  repositoryName: string;
+  baseURL: string;
+  protocolVersion: string;
+  adminEmail: string[];
+  earliestDatestamp: string;
+  deletedRecord: 'no' | 'persistent' | 'transient';
+  granularity: 'YYYY-MM-DD' | 'YYYY-MM-DDThh:mm:ssZ';
+  compression?: string[];
+  description?: unknown[];
+}
+
+/** A metadata format from ListMetadataFormats. */
+export interface OaiMetadataFormat {
+  metadataPrefix: OaiMetadataPrefix;
+  schema: string;
+  metadataNamespace: string;
+}
+
+/** A set from ListSets (for selective harvesting). */
+export interface OaiSet {
+  setSpec: string;
+  setName: string;
+  setDescription?: unknown;
+}
+
+/** Resumption token for paginated list responses. */
+export interface OaiResumptionToken {
+  /** Opaque token value to pass to the next request. */
+  value: string;
+  /** When the token expires (UTC). */
+  expirationDate?: string;
+  /** Total size of the complete list (may be approximate). */
+  completeListSize?: number;
+  /** Cursor position (number of elements returned so far). */
+  cursor?: number;
+}
+
+/** Record header (identifier, datestamp, setSpecs, optional deleted status). */
+export interface OaiHeader {
+  identifier: string;
+  datestamp: string;
+  setSpec: string[];
+  /** Present and 'deleted' when the record has been withdrawn. */
+  status?: 'deleted';
+}
+
+/** arXiv OAI metadata prefixes supported by the repository. */
+export type OaiMetadataPrefix = 'oai_dc' | 'arXiv' | 'arXivOld' | 'arXivRaw';
+
+type OneOrMany<T> = T | T[];
+
+/** oai_dc metadata (Dublin Core). */
+export interface OaiDcMetadata {
+  dc: {
+    title?: OneOrMany<string>;
+    creator?: OneOrMany<string>;
+    subject?: OneOrMany<string>;
+    description?: OneOrMany<string>;
+    publisher?: OneOrMany<string>;
+    contributor?: OneOrMany<string>;
+    date?: OneOrMany<string>;
+    type?: OneOrMany<string>;
+    format?: OneOrMany<string>;
+    identifier?: OneOrMany<string>;
+    source?: OneOrMany<string>;
+    language?: OneOrMany<string>;
+    relation?: OneOrMany<string>;
+    coverage?: OneOrMany<string>;
+    rights?: OneOrMany<string>;
+  };
+}
+
+/** arXiv author in the arXiv metadata format. */
+export interface OaiArxivAuthor {
+  keyname: string;
+  forenames?: string;
+  suffix?: string;
+  affiliation?: OneOrMany<string>;
+}
+
+/** arXiv metadata (latest-version focused metadata). */
+export interface OaiArxivMetadata {
+  arXiv: {
+    id: string;
+    created?: string;
+    updated?: string;
+    authors?: {
+      author: OneOrMany<OaiArxivAuthor>;
+    };
+    title?: string;
+    'msc-class'?: string;
+    'acm-class'?: string;
+    'report-no'?: string;
+    'journal-ref'?: string;
+    comments?: string;
+    abstract?: string;
+    categories?: string;
+    doi?: string;
+    proxy?: string;
+    license?: string;
+  };
+}
+
+/** arXivOld metadata (legacy arXiv internal format). */
+export interface OaiArxivOldMetadata {
+  arXivOld: {
+    id: string;
+    title?: string;
+    authors?: string;
+    categories?: string;
+    comments?: string;
+    proxy?: string;
+    'report-no'?: string;
+    'msc-class'?: string;
+    'acm-class'?: string;
+    'journal-ref'?: string;
+    doi?: string;
+    abstract?: string;
+    license?: string;
+  };
+}
+
+/** Version entry in arXivRaw metadata. */
+export interface OaiArxivRawVersion {
+  version?: string;
+  date: string;
+  size?: string;
+  source_type?: string;
+}
+
+/** arXivRaw metadata (close to arXiv internal metadata with version history). */
+export interface OaiArxivRawMetadata {
+  arXivRaw: {
+    id: string;
+    submitter: string;
+    version: OneOrMany<OaiArxivRawVersion>;
+    title?: string;
+    authors?: string;
+    categories: string;
+    comments?: string;
+    proxy?: string;
+    'report-no'?: string;
+    'acm-class'?: string;
+    'msc-class'?: string;
+    'journal-ref'?: string;
+    doi?: string;
+    license?: string;
+    abstract?: string;
+  };
+}
+
+/** Mapping from metadataPrefix to metadata payload shape. */
+export interface OaiMetadataByPrefix {
+  oai_dc: OaiDcMetadata;
+  arXiv: OaiArxivMetadata;
+  arXivOld: OaiArxivOldMetadata;
+  arXivRaw: OaiArxivRawMetadata;
+}
+
+/** Metadata part of a record (format-dependent: oai_dc, arXiv, arXivOld, arXivRaw). */
+export type OaiMetadata = OaiMetadataByPrefix[keyof OaiMetadataByPrefix];
+
+/** A single OAI record (header + optional metadata and about). */
+export interface OaiRecord {
+  header: OaiHeader;
+  /** Omitted for deleted records. */
+  metadata?: OaiMetadata;
+  /** Optional about containers (e.g. provenance, rights). */
+  about?: unknown[];
+}
+
+/** Result of ListIdentifiers (headers + optional resumption). */
+export interface OaiListIdentifiersResult {
+  headers: OaiHeader[];
+  resumptionToken?: OaiResumptionToken;
+}
+
+/** Result of ListRecords (records + optional resumption). */
+export interface OaiListRecordsResult {
+  records: OaiRecord[];
+  resumptionToken?: OaiResumptionToken;
+}
+
+/** Result of ListSets (sets + optional resumption). */
+export interface OaiListSetsResult {
+  sets: OaiSet[];
+  resumptionToken?: OaiResumptionToken;
+}
+
+type OaiSelectiveHarvestOptions = {
+  /** Lower bound for datestamp-based selective harvesting (UTC). */
+  from?: string;
+  /** Upper bound for datestamp-based selective harvesting (UTC). */
+  until?: string;
+  /** Set spec for selective harvesting (e.g. cs:cs:AI, physics:hep-th). */
+  set?: string;
+  /** Resumption token must not be provided for an initial selective request. */
+  resumptionToken?: undefined;
+};
+
+type OaiResumptionTokenOnlyOptions = {
+  /** Resumption token from a previous incomplete list response. */
+  resumptionToken: string;
+  /** Selective harvesting parameters are not allowed together with resumptionToken. */
+  from?: never;
+  until?: never;
+  set?: never;
+};
+
+/** Options for ListIdentifiers and ListRecords. */
+export type OaiListOptions = OaiRequestOptions &
+  (OaiSelectiveHarvestOptions | OaiResumptionTokenOnlyOptions);
+
+/** Error thrown when the OAI repository returns an error element. */
+export class OaiError extends Error {
+  readonly code: OaiErrorCode;
+  readonly messageText: string;
+
+  constructor(code: OaiErrorCode, messageText: string = '') {
+    const msg = messageText ? `${code}: ${messageText}` : code;
+    super(msg);
+    this.name = 'OaiError';
+    this.code = code;
+    this.messageText = messageText;
+    Object.setPrototypeOf(this, OaiError.prototype);
+  }
+}

package/tests/oai.integration.test.ts

@@ -0,0 +1,247 @@
+/**
+ * Integration tests for the arXiv OAI-PMH interface (real HTTP calls).
+ * Conservative request size and rate; same pattern as arxivAPI.integration.test.ts.
+ */
+import { describe, it, expect } from 'vitest';
+import {
+  oaiIdentify,
+  oaiListRecords,
+  oaiListRecordsAsyncIterator,
+  oaiListRecordsAll,
+  oaiListIdentifiersAsyncIterator,
+  oaiListIdentifiersAll,
+  oaiListSetsAsyncIterator,
+  oaiListSetsAll,
+} from '../src/oaiClient.js';
+import { OaiError } from '../src/oaiTypes.js';
+
+const OAI_OPTIONS = {
+  timeoutMs: 15000,
+  retries: 2,
+  rateLimit: { tokensPerInterval: 1, intervalMs: 1000 },
+  userAgent: 'arxiv-api-wrapper-tests/1.0',
+};
+
+describe('OAI-PMH integration', () => {
+  it('oaiIdentify returns repository info and protocol version 2.0', async () => {
+    let result;
+    try {
+      result = await oaiIdentify(OAI_OPTIONS);
+    } catch (error) {
+      console.error('oaiIdentify failed:', error);
+      throw new Error(
+        `OAI Identify failed: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+    expect(result.repositoryName).toBeTruthy();
+    expect(result.baseURL).toContain('oaipmh.arxiv.org');
+    expect(result.protocolVersion).toBe('2.0');
+    expect(Array.isArray(result.adminEmail)).toBe(true);
+    expect(result.earliestDatestamp).toBeTruthy();
+  }, 30000);
+
+  it('oaiListRecords returns one page of records with header and metadata', async () => {
+    let result;
+    try {
+      result = await oaiListRecords('oai_dc', {
+        ...OAI_OPTIONS,
+        from: '2024-01-01',
+        until: '2024-01-02',
+      });
+    } catch (error) {
+      console.error('oaiListRecords failed:', error);
+      throw new Error(
+        `OAI ListRecords failed: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+    expect(Array.isArray(result.records)).toBe(true);
+    if (result.records.length > 0) {
+      const rec = result.records[0];
+      expect(rec.header).toBeTruthy();
+      expect(rec.header.identifier).toBeTruthy();
+      expect(rec.header.datestamp).toBeTruthy();
+      expect(rec.metadata).toBeDefined();
+      expect(typeof rec.metadata).toBe('object');
+    }
+    // May or may not have resumptionToken depending on result size
+    if (result.resumptionToken) {
+      expect(result.resumptionToken.value).toBeTruthy();
+    }
+  }, 30000);
+
+  it('oaiListRecords continuation requests work with resumptionToken-only options', async () => {
+    const firstPage = await oaiListRecords('oai_dc', OAI_OPTIONS);
+    expect(firstPage.resumptionToken?.value).toBeTruthy();
+
+    const secondPage = await oaiListRecords('oai_dc', {
+      ...OAI_OPTIONS,
+      resumptionToken: firstPage.resumptionToken!.value,
+    });
+
+    expect(Array.isArray(secondPage.records)).toBe(true);
+  }, 30000);
+
+  it('throws a local OaiError for resumptionToken combined with selective params', async () => {
+    const invalidOptions = {
+      ...OAI_OPTIONS,
+      from: '2024-01-01',
+      resumptionToken: 'fake-token',
+    } as unknown as Parameters<typeof oaiListRecords>[1];
+
+    await expect(
+      oaiListRecords('oai_dc', invalidOptions)
+    ).rejects.toBeInstanceOf(OaiError);
+  });
+
+  it('oaiListRecordsAll returns records across all pages within a small date range', async () => {
+    let result;
+    try {
+      result = await oaiListRecordsAll('oai_dc', {
+        ...OAI_OPTIONS,
+        from: '2024-01-01',
+        until: '2024-01-02',
+        maxRecords: 200,
+      });
+    } catch (error) {
+      console.error('oaiListRecordsAll failed:', error);
+      throw new Error(
+        `OAI ListRecordsAll failed: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+
+    expect(Array.isArray(result.records)).toBe(true);
+    if (result.records.length > 0) {
+      const rec = result.records[0];
+      expect(rec.header).toBeTruthy();
+      expect(rec.header.identifier).toBeTruthy();
+      expect(rec.header.datestamp).toBeTruthy();
+      expect(rec.metadata).toBeDefined();
+      expect(typeof rec.metadata).toBe('object');
+    }
+  }, 30000);
+
+  it('oaiListRecordsAsyncIterator yields records and matches oaiListRecordsAll count for the same cap', async () => {
+    let iteratedRecords: unknown[] = [];
+    let allResult;
+    try {
+      const maxRecords = 25;
+      for await (const record of oaiListRecordsAsyncIterator('oai_dc', {
+        ...OAI_OPTIONS,
+        from: '2024-01-01',
+        until: '2024-01-02',
+        maxRecords,
+      })) {
+        iteratedRecords.push(record);
+      }
+      allResult = await oaiListRecordsAll('oai_dc', {
+        ...OAI_OPTIONS,
+        from: '2024-01-01',
+        until: '2024-01-02',
+        maxRecords,
+      });
+    } catch (error) {
+      console.error('oaiListRecordsAsyncIterator failed:', error);
+      throw new Error(
+        `OAI ListRecordsAsyncIterator failed: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+
+    expect(Array.isArray(iteratedRecords)).toBe(true);
+    expect(iteratedRecords.length).toBeLessThanOrEqual(25);
+    expect(iteratedRecords.length).toBe(allResult.records.length);
+  }, 30000);
+
+  it('oaiListIdentifiersAll returns headers across pages with maxHeaders cap', async () => {
+    let result;
+    try {
+      result = await oaiListIdentifiersAll('oai_dc', {
+        ...OAI_OPTIONS,
+        from: '2024-01-01',
+        until: '2024-01-02',
+        maxHeaders: 50,
+      });
+    } catch (error) {
+      console.error('oaiListIdentifiersAll failed:', error);
+      throw new Error(
+        `OAI ListIdentifiersAll failed: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+    expect(Array.isArray(result.headers)).toBe(true);
+    expect(result.headers.length).toBeLessThanOrEqual(50);
+    if (result.headers.length > 0) {
+      expect(result.headers[0].identifier).toBeTruthy();
+      expect(result.headers[0].datestamp).toBeTruthy();
+    }
+  }, 30000);
+
+  it('oaiListIdentifiersAsyncIterator yields headers and honors maxHeaders', async () => {
+    const headers = [];
+    try {
+      for await (const header of oaiListIdentifiersAsyncIterator('oai_dc', {
+        ...OAI_OPTIONS,
+        from: '2024-01-01',
+        until: '2024-01-02',
+        maxHeaders: 20,
+      })) {
+        headers.push(header);
+      }
+    } catch (error) {
+      console.error('oaiListIdentifiersAsyncIterator failed:', error);
+      throw new Error(
+        `OAI ListIdentifiersAsyncIterator failed: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+
+    expect(Array.isArray(headers)).toBe(true);
+    expect(headers.length).toBeLessThanOrEqual(20);
+    if (headers.length > 0) {
+      expect(headers[0].identifier).toBeTruthy();
+      expect(headers[0].datestamp).toBeTruthy();
+    }
+  }, 30000);
+
+  it('oaiListSetsAll returns sets with maxSets cap', async () => {
+    let result;
+    try {
+      result = await oaiListSetsAll({
+        ...OAI_OPTIONS,
+        maxSets: 20,
+      });
+    } catch (error) {
+      console.error('oaiListSetsAll failed:', error);
+      throw new Error(
+        `OAI ListSetsAll failed: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+    expect(Array.isArray(result.sets)).toBe(true);
+    expect(result.sets.length).toBeLessThanOrEqual(20);
+    if (result.sets.length > 0) {
+      expect(result.sets[0].setSpec).toBeTruthy();
+      expect(result.sets[0].setName).toBeTruthy();
+    }
+  }, 30000);
+
+  it('oaiListSetsAsyncIterator yields sets and honors maxSets', async () => {
+    const sets = [];
+    try {
+      for await (const set of oaiListSetsAsyncIterator({
+        ...OAI_OPTIONS,
+        maxSets: 10,
+      })) {
+        sets.push(set);
+      }
+    } catch (error) {
+      console.error('oaiListSetsAsyncIterator failed:', error);
+      throw new Error(
+        `OAI ListSetsAsyncIterator failed: ${error instanceof Error ? error.message : String(error)}`
+      );
+    }
+
+    expect(Array.isArray(sets)).toBe(true);
+    expect(sets.length).toBeLessThanOrEqual(10);
+    if (sets.length > 0) {
+      expect(sets[0].setSpec).toBeTruthy();
+      expect(sets[0].setName).toBeTruthy();
+    }
+  }, 30000);
+});