arxiv-api-wrapper 2.0.2 → 2.1.1

package/README.md

@@ -92,9 +92,15 @@ for await (const rec of oaiListRecordsAsyncIterator('oai_dc', {
 }
 ```
 
+If you omit `maxRecords` (or `maxHeaders` / `maxSets` on the corresponding iterators), iteration continues until the API is exhausted.
+
 The `oaiListRecordsAll` / `oaiListIdentifiersAll` / `oaiListSetsAll` helpers are convenience wrappers that collect from the corresponding async iterators.
 
-All OAI functions accept optional `timeoutMs`, `retries`, `userAgent`, and `rateLimit` (same as the Atom API). OAI errors (e.g. `idDoesNotExist`, `noRecordsMatch`) are thrown as `OaiError` with a `code` and `messageText`.
+Async iterators keep continuation token metadata in memory while paging. If a token includes an `expirationDate` and that time has passed, iterators fail fast locally with `OaiError` (`code: 'badResumptionToken'`) before attempting another request.
+
+All OAI functions accept optional `timeoutMs`, `retries`, `userAgent`, and `rateLimit` (same as the Atom API). Other OAI errors (e.g. `idDoesNotExist`) are thrown as `OaiError` with a `code` and `messageText`. **`noRecordsMatch`** is treated as “no results”: the wrapper returns an empty list (empty `records` or `headers`) instead of throwing, so you always get a normal result shape from `oaiListRecords` and `oaiListIdentifiers`.
+
+**Differences from OAI-PMH:** The underlying arXiv OAI server returns an error response when a list request matches no records. This wrapper normalises that to an empty list so callers can assume a consistent result type without handling `noRecordsMatch` as an exception.
 
 ## API Reference
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "arxiv-api-wrapper",
-  "version": "2.0.2",
+  "version": "2.1.1",
   "description": "Provides functions wrapping the arXiv API",
   "keywords": [
     "arxiv"
@@ -26,7 +26,7 @@
     "docs:serve": "npx serve docs"
   },
   "dependencies": {
-    "fast-xml-parser": "^5.3.5"
+    "fast-xml-parser": "^5.5.6"
   },
   "devDependencies": {
     "@types/node": "^25.0.0",

package/src/oaiClient.ts

@@ -27,6 +27,7 @@ import type {
   OaiIdentifyResponse,
   OaiMetadataFormat,
   OaiMetadataPrefix,
+  OaiResumptionToken,
   OaiRecord,
   OaiHeader,
   OaiSet,
@@ -116,6 +117,19 @@ function validateUntilDateNotTooLate(until: string | undefined): void {
   }
 }
 
+function validateResumptionTokenNotExpired(resumptionToken: OaiResumptionToken | undefined): void {
+  const expirationDate = resumptionToken?.expirationDate;
+  if (!expirationDate) return;
+  const expiresAtMs = Date.parse(expirationDate);
+  if (Number.isNaN(expiresAtMs)) return;
+  if (Date.now() >= expiresAtMs) {
+    throw new OaiError(
+      'badResumptionToken',
+      `Resumption token expired at ${expirationDate}. Start a new list request without resumptionToken.`
+    );
+  }
+}
+
 /** Build OAI-PMH request URL (exported for unit tests). */
 export function buildOaiUrl(verb: OaiVerb, params: OaiParams): string {
   if (hasResumptionTokenConflicts(params)) {
@@ -261,6 +275,9 @@ export async function oaiGetRecord(
 /**
  * List identifiers (headers only) for selective harvesting (ListIdentifiers verb).
  *
+ * When the repository responds with `noRecordsMatch` (no identifiers match the from/until/set criteria),
+ * this wrapper returns an empty list instead of throwing, unlike the raw OAI-PMH API.
+ *
  * @param metadataPrefix - Required metadata format (e.g. oai_dc, arXiv, arXivRaw).
  * @param listOptions - Optional from, until, set, resumptionToken and request options (timeout, retries, userAgent, rateLimit).
  * @returns Headers and optional resumptionToken for the next page.
@@ -287,13 +304,23 @@ export async function oaiListIdentifiers(
     if (hasValue(until)) params.until = until;
     if (hasValue(set)) params.set = set;
   }
-  const xml = await oaiRequest('ListIdentifiers', params, listOptions);
-  return parseListIdentifiers(xml);
+  try {
+    const xml = await oaiRequest('ListIdentifiers', params, listOptions);
+    return parseListIdentifiers(xml);
+  } catch (e) {
+    if (e instanceof OaiError && e.code === 'noRecordsMatch') {
+      return { headers: [] };
+    }
+    throw e;
+  }
 }
 
 /**
  * List records (full metadata) for selective harvesting (ListRecords verb).
  *
+ * When the repository responds with `noRecordsMatch` (no records match the from/until/set criteria),
+ * this wrapper returns an empty list instead of throwing, unlike the raw OAI-PMH API.
+ *
  * @param metadataPrefix - Required metadata format (e.g. oai_dc, arXiv, arXivRaw).
  * @param listOptions - Optional from, until, set, resumptionToken and request options (timeout, retries, userAgent, rateLimit).
  * @returns Records and optional resumptionToken for the next page.
@@ -320,8 +347,15 @@ export async function oaiListRecords(
     if (hasValue(until)) params.until = until;
     if (hasValue(set)) params.set = set;
   }
-  const xml = await oaiRequest('ListRecords', params, listOptions);
-  return parseListRecords(xml);
+  try {
+    const xml = await oaiRequest('ListRecords', params, listOptions);
+    return parseListRecords(xml);
+  } catch (e) {
+    if (e instanceof OaiError && e.code === 'noRecordsMatch') {
+      return { records: [] };
+    }
+    throw e;
+  }
 }
 
 type OaiListRecordsAllOptions = OaiRequestOptions & {
@@ -345,8 +379,8 @@ type OaiListSetsAllOptions = OaiRequestOptions & {
 /**
  * Iterate records across all pages for a given metadataPrefix and optional selective harvesting options.
  *
- * This helper follows resumption tokens internally and yields records one-by-one until completion or
- * until the optional maxRecords cap is reached.
+ * This helper follows resumption tokens internally and yields records one-by-one until completion.
+ * When maxRecords is provided, it acts as an upper cap; when omitted, no upper cap is applied.
  *
  * @param metadataPrefix - Required metadata format (e.g. oai_dc, arXiv, arXivRaw).
  * @param listOptions - Optional from, until, set, request options (timeout, retries, userAgent, rateLimit) and maxRecords.
@@ -358,12 +392,14 @@ export async function* oaiListRecordsAsyncIterator(
   listOptions?: OaiListRecordsAllOptions
 ): AsyncGenerator<OaiRecord, void, void> {
   let emitted = 0;
-  let resumptionToken: string | undefined;
+  let resumptionToken: OaiResumptionToken | undefined;
   const { maxRecords, from, until, set, ...requestOptions } = listOptions ?? {};
+  const maxEmitted = maxRecords ?? Number.POSITIVE_INFINITY;
 
   do {
-    const pageOptions: OaiListOptions = resumptionToken
-      ? { ...requestOptions, resumptionToken }
+    validateResumptionTokenNotExpired(resumptionToken);
+    const pageOptions: OaiListOptions = resumptionToken?.value
+      ? { ...requestOptions, resumptionToken: resumptionToken.value }
       : { ...requestOptions, ...(from ? { from } : {}), ...(until ? { until } : {}), ...(set ? { set } : {}) };
 
     const page = await oaiListRecords(metadataPrefix, pageOptions);
@@ -371,20 +407,20 @@ export async function* oaiListRecordsAsyncIterator(
     if (records.length === 0) break;
 
     for (const record of records) {
-      if (maxRecords != null && emitted >= maxRecords) return;
+      if (emitted >= maxEmitted) return;
       yield record;
       emitted += 1;
     }
 
-    resumptionToken = page.resumptionToken?.value;
-  } while (resumptionToken);
+    resumptionToken = page.resumptionToken;
+  } while (resumptionToken?.value);
 }
 
 /**
  * Iterate identifiers (headers only) across all pages for a given metadataPrefix and optional selective harvesting options.
  *
- * This helper follows resumption tokens internally and yields headers one-by-one until completion or
- * until the optional maxHeaders cap is reached.
+ * This helper follows resumption tokens internally and yields headers one-by-one until completion.
+ * When maxHeaders is provided, it acts as an upper cap; when omitted, no upper cap is applied.
  *
  * @param metadataPrefix - Required metadata format (e.g. oai_dc, arXiv, arXivRaw).
  * @param listOptions - Optional from, until, set, request options (timeout, retries, userAgent, rateLimit) and maxHeaders.
@@ -396,12 +432,14 @@ export async function* oaiListIdentifiersAsyncIterator(
   listOptions?: OaiListIdentifiersAllOptions
 ): AsyncGenerator<OaiHeader, void, void> {
   let emitted = 0;
-  let resumptionToken: string | undefined;
+  let resumptionToken: OaiResumptionToken | undefined;
   const { maxHeaders, from, until, set, ...requestOptions } = listOptions ?? {};
+  const maxEmitted = maxHeaders ?? Number.POSITIVE_INFINITY;
 
   do {
-    const pageOptions: OaiListOptions = resumptionToken
-      ? { ...requestOptions, resumptionToken }
+    validateResumptionTokenNotExpired(resumptionToken);
+    const pageOptions: OaiListOptions = resumptionToken?.value
+      ? { ...requestOptions, resumptionToken: resumptionToken.value }
       : { ...requestOptions, ...(from ? { from } : {}), ...(until ? { until } : {}), ...(set ? { set } : {}) };
 
     const page = await oaiListIdentifiers(metadataPrefix, pageOptions);
@@ -409,20 +447,20 @@ export async function* oaiListIdentifiersAsyncIterator(
     if (headers.length === 0) break;
 
     for (const header of headers) {
-      if (maxHeaders != null && emitted >= maxHeaders) return;
+      if (emitted >= maxEmitted) return;
       yield header;
       emitted += 1;
     }
 
-    resumptionToken = page.resumptionToken?.value;
-  } while (resumptionToken);
+    resumptionToken = page.resumptionToken;
+  } while (resumptionToken?.value);
 }
 
 /**
  * Iterate sets available for selective harvesting across all pages.
  *
- * This helper follows resumption tokens internally and yields sets one-by-one until completion or
- * until the optional maxSets cap is reached.
+ * This helper follows resumption tokens internally and yields sets one-by-one until completion.
+ * When maxSets is provided, it acts as an upper cap; when omitted, no upper cap is applied.
  *
  * @param options - Optional request configuration (timeout, retries, userAgent, rateLimit) and maxSets.
  * @returns Async iterator yielding sets one-by-one.
@@ -431,28 +469,31 @@ export async function* oaiListSetsAsyncIterator(
   options?: OaiListSetsAllOptions
 ): AsyncGenerator<OaiSet, void, void> {
   let emitted = 0;
-  let resumptionToken: string | undefined;
+  let resumptionToken: OaiResumptionToken | undefined;
   const { maxSets, ...requestOptions } = options ?? {};
+  const maxEmitted = maxSets ?? Number.POSITIVE_INFINITY;
 
   do {
-    const page = await oaiListSets(resumptionToken, requestOptions);
+    validateResumptionTokenNotExpired(resumptionToken);
+    const page = await oaiListSets(resumptionToken?.value, requestOptions);
     const sets = page.sets ?? [];
     if (sets.length === 0) break;
 
     for (const set of sets) {
-      if (maxSets != null && emitted >= maxSets) return;
+      if (emitted >= maxEmitted) return;
       yield set;
       emitted += 1;
     }
 
-    resumptionToken = page.resumptionToken?.value;
-  } while (resumptionToken);
+    resumptionToken = page.resumptionToken;
+  } while (resumptionToken?.value);
 }
 
 /**
  * Fetch all records across all pages for a given metadataPrefix and optional selective harvesting options.
  *
- * This helper collects from oaiListRecordsAsyncIterator until completion or the optional maxRecords cap.
+ * This helper collects from oaiListRecordsAsyncIterator until completion.
+ * When maxRecords is provided, it acts as an upper cap; when omitted, no upper cap is applied.
  *
  * @param metadataPrefix - Required metadata format (e.g. oai_dc, arXiv, arXivRaw).
  * @param listOptions - Optional from, until, set, request options (timeout, retries, userAgent, rateLimit) and maxRecords.
@@ -474,7 +515,8 @@ export async function oaiListRecordsAll(
 /**
  * Fetch all identifiers (headers only) across all pages for a given metadataPrefix and optional selective harvesting options.
  *
- * This helper collects from oaiListIdentifiersAsyncIterator until completion or the optional maxHeaders cap.
+ * This helper collects from oaiListIdentifiersAsyncIterator until completion.
+ * When maxHeaders is provided, it acts as an upper cap; when omitted, no upper cap is applied.
  *
  * @param metadataPrefix - Required metadata format (e.g. oai_dc, arXiv, arXivRaw).
  * @param listOptions - Optional from, until, set, request options (timeout, retries, userAgent, rateLimit) and maxHeaders.
@@ -496,7 +538,8 @@ export async function oaiListIdentifiersAll(
 /**
  * Fetch all sets available for selective harvesting across all pages.
  *
- * This helper collects from oaiListSetsAsyncIterator until completion or the optional maxSets cap.
+ * This helper collects from oaiListSetsAsyncIterator until completion.
+ * When maxSets is provided, it acts as an upper cap; when omitted, no upper cap is applied.
  *
  * @param options - Optional request configuration (timeout, retries, userAgent, rateLimit) and maxSets.
  * @returns All fetched sets as a single array.

package/src/oaiParser.ts

@@ -22,6 +22,9 @@ const parser = new XMLParser({
   removeNSPrefix: true,
   trimValues: true,
   parseTagValue: false,
+  processEntities: {
+    maxTotalExpansions: 0, // 0 disables the limit; arXiv OAI is a trusted source
+  },
 });
 
 const VALID_ERROR_CODES: OaiErrorCode[] = [

package/tests/oai.integration.test.ts

@@ -2,7 +2,7 @@
  * Integration tests for the arXiv OAI-PMH interface (real HTTP calls).
  * Conservative request size and rate; same pattern as arxivAPI.integration.test.ts.
  */
-import { describe, it, expect } from 'vitest';
+import { describe, it, expect, vi, afterEach } from 'vitest';
 import {
   oaiIdentify,
   oaiListRecords,
@@ -22,6 +22,10 @@ const OAI_OPTIONS = {
   userAgent: 'arxiv-api-wrapper-tests/1.0',
 };
 
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
 describe('OAI-PMH integration', () => {
   it('oaiIdentify returns repository info and protocol version 2.0', async () => {
     let result;
@@ -93,6 +97,17 @@ describe('OAI-PMH integration', () => {
     ).rejects.toBeInstanceOf(OaiError);
   });
 
+  it('oaiListRecords returns empty records (no throw) when no records match (noRecordsMatch)', async () => {
+    const result = await oaiListRecords('oai_dc', {
+      ...OAI_OPTIONS,
+      from: '2006-01-01',
+      until: '2006-01-02',
+      set: 'math:math:LO',
+    });
+    expect(result).toEqual({ records: [] });
+    expect(result.records).toHaveLength(0);
+  }, 30000);
+
   it('oaiListRecordsAll returns records across all pages within a small date range', async () => {
     let result;
     try {
@@ -244,4 +259,36 @@ describe('OAI-PMH integration', () => {
       expect(sets[0].setName).toBeTruthy();
     }
   }, 30000);
+
+  it('oaiListRecordsAsyncIterator rejects expired continuation token before another request', async () => {
+    const firstPageXml = `<?xml version="1.0" encoding="UTF-8"?>
+<OAI-PMH xmlns="http://www.openarchives.org/OAI/2.0/">
+  <responseDate>2024-01-15T12:00:00Z</responseDate>
+  <request verb="ListRecords" metadataPrefix="oai_dc">https://oaipmh.arxiv.org/oai</request>
+  <ListRecords>
+    <record>
+      <header>
+        <identifier>oai:arXiv.org:test/integration-1</identifier>
+        <datestamp>2024-01-01</datestamp>
+      </header>
+      <metadata><dc><dc:title>Integration Page 1</dc:title></dc></metadata>
+    </record>
+    <resumptionToken expirationDate="2000-01-01T00:00:00Z">expired-integration-token</resumptionToken>
+  </ListRecords>
+</OAI-PMH>`;
+
+    const fetchMock = vi
+      .spyOn(globalThis, 'fetch')
+      .mockResolvedValue(new Response(firstPageXml, { status: 200 }));
+
+    const iterator = oaiListRecordsAsyncIterator('oai_dc', { retries: 0, timeoutMs: 1000 });
+    const first = await iterator.next();
+    expect(first.done).toBe(false);
+
+    await expect(iterator.next()).rejects.toMatchObject({
+      name: 'OaiError',
+      code: 'badResumptionToken',
+    });
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
 });

package/tests/oai.test.ts

@@ -2,12 +2,13 @@
  * Unit tests for OAI-PMH URL builder and XML parser (no network).
  * Pagination helpers (oaiListRecordsAll, etc.) are covered by integration tests.
  */
-import { describe, it, expect } from 'vitest';
+import { describe, it, expect, vi, afterEach } from 'vitest';
 import {
   buildOaiUrl,
   normalizeOaiIdentifier,
   oaiListIdentifiers,
   oaiListRecords,
+  oaiListRecordsAsyncIterator,
 } from '../src/oaiClient.js';
 import {
   parseIdentify,
@@ -22,6 +23,10 @@ import { OaiError } from '../src/oaiTypes.js';
 
 const OAI_BASE = 'https://oaipmh.arxiv.org/oai';
 
+afterEach(() => {
+  vi.restoreAllMocks();
+});
+
 describe('buildOaiUrl', () => {
   it('includes verb only for Identify', () => {
     const url = buildOaiUrl('Identify', {});
@@ -262,6 +267,38 @@ describe('OAI error handling', () => {
   });
 });
 
+describe('noRecordsMatch returns empty list (wrapper behaviour)', () => {
+  it('oaiListRecords returns { records: [] } when server responds noRecordsMatch', async () => {
+    const noRecordsMatchXml = wrapOaiRoot(`<error code="noRecordsMatch"/>`);
+    vi.spyOn(globalThis, 'fetch').mockResolvedValue(
+      new Response(noRecordsMatchXml, { status: 200 })
+    );
+
+    const result = await oaiListRecords('oai_dc', {
+      from: '2006-01-01',
+      until: '2006-01-02',
+    });
+
+    expect(result).toEqual({ records: [] });
+    expect(result.records).toHaveLength(0);
+  });
+
+  it('oaiListIdentifiers returns { headers: [] } when server responds noRecordsMatch', async () => {
+    const noRecordsMatchXml = wrapOaiRoot(`<error code="noRecordsMatch"/>`);
+    vi.spyOn(globalThis, 'fetch').mockResolvedValue(
+      new Response(noRecordsMatchXml, { status: 200 })
+    );
+
+    const result = await oaiListIdentifiers('oai_dc', {
+      from: '2006-01-01',
+      until: '2006-01-02',
+    });
+
+    expect(result).toEqual({ headers: [] });
+    expect(result.headers).toHaveLength(0);
+  });
+});
+
 describe('resumptionToken validation', () => {
   it('throws a local OaiError when resumptionToken is combined with from in oaiListRecords', async () => {
     const invalidOptions = {
@@ -349,3 +386,122 @@ describe('until date validation', () => {
     expect(url).toContain(`until=${todayUtc}`);
   });
 });
+
+describe('resumptionToken expiration handling in iterators', () => {
+  it('fails fast locally when continuation token is already expired', async () => {
+    const firstPageXml = wrapOaiRoot(`
+  <ListRecords>
+    <record>
+      <header>
+        <identifier>oai:arXiv.org:test/0001</identifier>
+        <datestamp>2024-01-01</datestamp>
+      </header>
+      <metadata><dc><dc:title>Page 1</dc:title></dc></metadata>
+    </record>
+    <resumptionToken expirationDate="2000-01-01T00:00:00Z">expired-token</resumptionToken>
+  </ListRecords>`).replace(
+      '<request verb="Identify">',
+      '<request verb="ListRecords" metadataPrefix="oai_dc">'
+    );
+
+    const fetchMock = vi
+      .spyOn(globalThis, 'fetch')
+      .mockResolvedValue(new Response(firstPageXml, { status: 200 }));
+
+    const iterator = oaiListRecordsAsyncIterator('oai_dc', { retries: 0, timeoutMs: 1000 });
+    const first = await iterator.next();
+    expect(first.done).toBe(false);
+
+    await expect(iterator.next()).rejects.toMatchObject({
+      name: 'OaiError',
+      code: 'badResumptionToken',
+    });
+    expect(fetchMock).toHaveBeenCalledTimes(1);
+  });
+
+  it('continues when continuation token expirationDate is in the future', async () => {
+    const firstPageXml = wrapOaiRoot(`
+  <ListRecords>
+    <record>
+      <header>
+        <identifier>oai:arXiv.org:test/0002</identifier>
+        <datestamp>2024-01-01</datestamp>
+      </header>
+      <metadata><dc><dc:title>Page 1</dc:title></dc></metadata>
+    </record>
+    <resumptionToken expirationDate="2999-01-01T00:00:00Z">live-token</resumptionToken>
+  </ListRecords>`).replace(
+      '<request verb="Identify">',
+      '<request verb="ListRecords" metadataPrefix="oai_dc">'
+    );
+    const secondPageXml = wrapOaiRoot(`
+  <ListRecords>
+    <record>
+      <header>
+        <identifier>oai:arXiv.org:test/0003</identifier>
+        <datestamp>2024-01-02</datestamp>
+      </header>
+      <metadata><dc><dc:title>Page 2</dc:title></dc></metadata>
+    </record>
+  </ListRecords>`).replace(
+      '<request verb="Identify">',
+      '<request verb="ListRecords" metadataPrefix="oai_dc">'
+    );
+
+    const fetchMock = vi
+      .spyOn(globalThis, 'fetch')
+      .mockResolvedValueOnce(new Response(firstPageXml, { status: 200 }))
+      .mockResolvedValueOnce(new Response(secondPageXml, { status: 200 }));
+
+    const records = [];
+    for await (const record of oaiListRecordsAsyncIterator('oai_dc', { retries: 0, timeoutMs: 1000 })) {
+      records.push(record);
+    }
+
+    expect(records).toHaveLength(2);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+  });
+
+  it('preserves previous behavior when expirationDate is omitted', async () => {
+    const firstPageXml = wrapOaiRoot(`
+  <ListRecords>
+    <record>
+      <header>
+        <identifier>oai:arXiv.org:test/0004</identifier>
+        <datestamp>2024-01-01</datestamp>
+      </header>
+      <metadata><dc><dc:title>Page 1</dc:title></dc></metadata>
+    </record>
+    <resumptionToken cursor="1">token-no-expiry</resumptionToken>
+  </ListRecords>`).replace(
+      '<request verb="Identify">',
+      '<request verb="ListRecords" metadataPrefix="oai_dc">'
+    );
+    const secondPageXml = wrapOaiRoot(`
+  <ListRecords>
+    <record>
+      <header>
+        <identifier>oai:arXiv.org:test/0005</identifier>
+        <datestamp>2024-01-02</datestamp>
+      </header>
+      <metadata><dc><dc:title>Page 2</dc:title></dc></metadata>
+    </record>
+  </ListRecords>`).replace(
+      '<request verb="Identify">',
+      '<request verb="ListRecords" metadataPrefix="oai_dc">'
+    );
+
+    const fetchMock = vi
+      .spyOn(globalThis, 'fetch')
+      .mockResolvedValueOnce(new Response(firstPageXml, { status: 200 }))
+      .mockResolvedValueOnce(new Response(secondPageXml, { status: 200 }));
+
+    const records = [];
+    for await (const record of oaiListRecordsAsyncIterator('oai_dc', { retries: 0, timeoutMs: 1000 })) {
+      records.push(record);
+    }
+
+    expect(records).toHaveLength(2);
+    expect(fetchMock).toHaveBeenCalledTimes(2);
+  });
+});