@yedoma-labs/ilyn-pages 0.0.0-stage → 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +232 -2
- package/dist/index.cjs +409 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +108 -0
- package/dist/index.d.ts +108 -0
- package/dist/index.js +404 -0
- package/dist/index.js.map +1 -0
- package/package.json +80 -3
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 yedoma-labs
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,233 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @yedoma-labs/ilyn-pages
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
<picture>
|
|
4
|
+
<source media="(max-width: 640px)" srcset="https://raw.githubusercontent.com/yedoma-labs/assets/main/resized/banner-resized-mobile.png">
|
|
5
|
+
<img src="https://raw.githubusercontent.com/yedoma-labs/assets/main/resized/banner-resized.png" alt="Project Header">
|
|
6
|
+
</picture>
|
|
7
|
+
|
|
8
|
+
[](https://github.com/yedoma-labs/ilyn-pages/actions/workflows/ci.yml)
|
|
9
|
+
[](https://www.npmjs.com/package/@yedoma-labs/ilyn-pages)
|
|
10
|
+
[](https://www.typescriptlang.org/)
|
|
11
|
+
[](https://opensource.org/licenses/MIT)
|
|
12
|
+
|
|
13
|
+
> **ilyn** (*ilyn*) — Yakutian (Sakha) for "(noch zu bestätigen)"
|
|
14
|
+
|
|
15
|
+
Zero-dependency async iteration over every page of a paginated HTTP API, with guards against endless
|
|
16
|
+
cursor loops, skipped or duplicated rows, rate limits in the middle of a run and hanging requests.
|
|
17
|
+
|
|
18
|
+
Zero runtime dependencies. Uses only `fetch`, `URL`, `AbortController` and `Headers`, so it runs on
|
|
19
|
+
Node.js 20+, Bun and Deno.
|
|
20
|
+
|
|
21
|
+
## Why
|
|
22
|
+
|
|
23
|
+
Fetching "all pages" of an API looks trivial and fails in production:
|
|
24
|
+
|
|
25
|
+
- **Offset pagination** (`?offset=200&limit=100`) skips or repeats rows when data changes between two
|
|
26
|
+
requests. Rows inserted before the current offset shift everything down by one, and rows deleted
|
|
27
|
+
shift everything up.
|
|
28
|
+
- **Cursor loops** never end when a server returns the same cursor again.
|
|
29
|
+
- **No page limit** means a misbehaving server can keep a client reading forever.
|
|
30
|
+
- **HTTP 429** in the middle of a loop aborts the whole run unless you handle it per page.
|
|
31
|
+
- **Next links** appear in the body (`{ "next": "..." }`) for some APIs and in the `Link` header
|
|
32
|
+
(RFC 8288) for others. Each needs its own parsing code.
|
|
33
|
+
- `@octokit/plugin-paginate-rest` solves this for GitHub only.
|
|
34
|
+
|
|
35
|
+
`ilyn-pages` is the generic layer for the rest.
|
|
36
|
+
|
|
37
|
+
## Installation
|
|
38
|
+
|
|
39
|
+
```sh
|
|
40
|
+
pnpm add @yedoma-labs/ilyn-pages
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
## Quick start
|
|
44
|
+
|
|
45
|
+
### Next link in the `Link` header
|
|
46
|
+
|
|
47
|
+
```ts
|
|
48
|
+
import { createFetchPager, paginate } from "@yedoma-labs/ilyn-pages";
|
|
49
|
+
|
|
50
|
+
interface Item {
|
|
51
|
+
id: string;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
const pager = createFetchPager<Item>({
|
|
55
|
+
url: "https://api.example.com/items?per_page=100",
|
|
56
|
+
extract: (body) => (body as { items: Item[] }).items,
|
|
57
|
+
// The next URL comes from `Link: <...>; rel="next"` by default.
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
for await (const item of paginate({ fetchPage: pager, maxItems: 10_000 })) {
|
|
61
|
+
console.log(item.id);
|
|
62
|
+
}
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
### Next link in the body
|
|
66
|
+
|
|
67
|
+
```ts
|
|
68
|
+
const pager = createFetchPager<Item>({
|
|
69
|
+
url: "https://api.example.com/items",
|
|
70
|
+
extract: (body) => (body as { data: Item[] }).data,
|
|
71
|
+
nextFromResponse: (_res, body) => (body as { meta: { next: string | null } }).meta.next,
|
|
72
|
+
});
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
Relative next URLs are resolved against the URL of the current request.
|
|
76
|
+
|
|
77
|
+
### Opaque cursors with your own `fetchPage`
|
|
78
|
+
|
|
79
|
+
```ts
|
|
80
|
+
import { paginate, type Page } from "@yedoma-labs/ilyn-pages";
|
|
81
|
+
|
|
82
|
+
const items = paginate({
|
|
83
|
+
fetchPage: async (cursor, signal): Promise<Page<Item>> => {
|
|
84
|
+
const url = new URL("https://api.example.com/items");
|
|
85
|
+
if (cursor !== undefined) url.searchParams.set("cursor", cursor);
|
|
86
|
+
const res = await fetch(url, { signal });
|
|
87
|
+
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
88
|
+
const body = (await res.json()) as { items: Item[]; nextCursor?: string | null };
|
|
89
|
+
return { items: body.items, nextCursor: body.nextCursor };
|
|
90
|
+
},
|
|
91
|
+
});
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
### Handling errors
|
|
95
|
+
|
|
96
|
+
```ts
|
|
97
|
+
import { PaginationError } from "@yedoma-labs/ilyn-pages";
|
|
98
|
+
|
|
99
|
+
try {
|
|
100
|
+
for await (const item of pages) {
|
|
101
|
+
// ...
|
|
102
|
+
}
|
|
103
|
+
} catch (error) {
|
|
104
|
+
if (error instanceof PaginationError) {
|
|
105
|
+
switch (error.code) {
|
|
106
|
+
case "CURSOR_LOOP":
|
|
107
|
+
case "EMPTY_PAGES_LOOP":
|
|
108
|
+
// The server is misbehaving. Report it and stop.
|
|
109
|
+
break;
|
|
110
|
+
case "PAGE_LIMIT":
|
|
111
|
+
// The result is larger than maxPages. Raise the limit or narrow the query.
|
|
112
|
+
break;
|
|
113
|
+
case "RATE_LIMITED":
|
|
114
|
+
case "TIMEOUT":
|
|
115
|
+
// Retry later, the items already yielded are kept by the caller.
|
|
116
|
+
break;
|
|
117
|
+
default:
|
|
118
|
+
throw error;
|
|
119
|
+
}
|
|
120
|
+
console.error(`failed on page ${error.page}: ${error.message}`);
|
|
121
|
+
} else {
|
|
122
|
+
throw error;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
Items that were yielded before an error are not rolled back. Keep what you need as you go.
|
|
128
|
+
|
|
129
|
+
## Guards
|
|
130
|
+
|
|
131
|
+
| Guard | Trigger | Result |
|
|
132
|
+
| --- | --- | --- |
|
|
133
|
+
| Cursor loop | A `nextCursor` was already used in this run | `CURSOR_LOOP` |
|
|
134
|
+
| Empty-page loop | 3 consecutive pages without items that still have a cursor | `EMPTY_PAGES_LOOP` |
|
|
135
|
+
| Page limit | More pages are available after `maxPages` (default 100) | `PAGE_LIMIT` |
|
|
136
|
+
| Item limit | `maxItems` items were yielded | Stops without error |
|
|
137
|
+
| Page timeout | One `fetchPage` call (including `Retry-After` waits) exceeds `timeoutMs` (default 30000) | `TIMEOUT` |
|
|
138
|
+
| Total timeout | The whole iteration exceeds `totalTimeoutMs` (default: none) | `TIMEOUT` |
|
|
139
|
+
| Abort | `signal` is aborted | `ABORTED` |
|
|
140
|
+
| Early exit | `break` in `for await`, or `return()` | Open request signal is aborted |
|
|
141
|
+
| Rate limit | HTTP 429 | Waits for `Retry-After` (seconds or HTTP date, capped at 60 s), retries the page up to 3 times, then `RATE_LIMITED` |
|
|
142
|
+
| Foreign next URL | Next URL has another origin than the start URL | `INVALID_LINK` (unless `allowCrossOriginNext: true`) |
|
|
143
|
+
| Broken `Link` header | Malformed link that mentions `rel="next"` and no valid next link | `INVALID_LINK` |
|
|
144
|
+
|
|
145
|
+
Guard errors are `PaginationError` instances with a `code` and a `page` number (the page whose request or
|
|
146
|
+
response caused the error). Error messages never contain query values or cursors.
|
|
147
|
+
|
|
148
|
+
Other errors pass through unchanged: a non-2xx status from `createFetchPager` is a plain `Error`
|
|
149
|
+
(`Request failed with HTTP 500`), a body that is not JSON is a plain `Error` that names the status,
|
|
150
|
+
and errors from the injected `fetch` are rethrown as they are.
|
|
151
|
+
|
|
152
|
+
## API
|
|
153
|
+
|
|
154
|
+
### `paginate<T>(options): AsyncGenerator<T>`
|
|
155
|
+
|
|
156
|
+
| Option | Default | Description |
|
|
157
|
+
| --- | --- | --- |
|
|
158
|
+
| `fetchPage(cursor, signal)` | required | Loads one page. `cursor` is `undefined` for the first page. |
|
|
159
|
+
| `maxPages` | `100` | Maximum number of pages. A positive integer or `Infinity`. |
|
|
160
|
+
| `maxItems` | `Infinity` | Stops without error after this many items. |
|
|
161
|
+
| `timeoutMs` | `30000` | Timeout per `fetchPage` call. |
|
|
162
|
+
| `totalTimeoutMs` | `Infinity` | Timeout for the whole iteration. |
|
|
163
|
+
| `signal` | none | Aborts the iteration with `ABORTED`. |
|
|
164
|
+
|
|
165
|
+
`fetchPage` returns `{ items: T[]; nextCursor?: string | null }`. `undefined`, `null` and `""` all
|
|
166
|
+
mean that there is no next page.
|
|
167
|
+
|
|
168
|
+
Options are validated when `paginate()` is called and throw a `RangeError` for invalid values.
|
|
169
|
+
Iteration starts with the first `next()`.
|
|
170
|
+
|
|
171
|
+
### `createFetchPager<T>(options): FetchPage<T>`
|
|
172
|
+
|
|
173
|
+
| Option | Default | Description |
|
|
174
|
+
| --- | --- | --- |
|
|
175
|
+
| `url` | required | URL of the first page. |
|
|
176
|
+
| `extract(body)` | required | Returns the items of the parsed JSON body. |
|
|
177
|
+
| `nextFromResponse(res, body)` | `Link` header | Returns the next URL or `null`. |
|
|
178
|
+
| `fetch` | `globalThis.fetch` | Injected fetch implementation. |
|
|
179
|
+
| `allowCrossOriginNext` | `false` | Allows next URLs on another origin. |
|
|
180
|
+
|
|
181
|
+
### `parseLinkHeader(value): Record<string, string>`
|
|
182
|
+
|
|
183
|
+
Parses an RFC 8288 `Link` header. Handles quoted and unquoted `rel`, several relations in one `rel`,
|
|
184
|
+
several links, extra parameters, and URIs that contain commas or semicolons. Relation names are
|
|
185
|
+
lower-cased. The first link wins for duplicate relations. Malformed link values are skipped.
|
|
186
|
+
|
|
187
|
+
### `nextFromLinkHeader(headers): string | undefined`
|
|
188
|
+
|
|
189
|
+
Returns the `rel="next"` URI from a `Headers` object. The URI may be relative.
|
|
190
|
+
|
|
191
|
+
## Offset pagination
|
|
192
|
+
|
|
193
|
+
Offset pagination is not consistent under concurrent writes. Between two requests, rows inserted
|
|
194
|
+
before the offset shift the window and the same row appears on two pages. Rows deleted before the
|
|
195
|
+
offset cause a row to be skipped. The library cannot detect this, because each page is correct for
|
|
196
|
+
the data at the moment it was read.
|
|
197
|
+
|
|
198
|
+
Prefer cursor or keyset pagination. Keyset pagination uses the last sort key, for example
|
|
199
|
+
`?after_id=123`, and keeps working when rows change. If the API only offers offsets, sort by a
|
|
200
|
+
stable unique key, accept that the result is a snapshot per page, and compare the total count before
|
|
201
|
+
and after the run.
|
|
202
|
+
|
|
203
|
+
## Security
|
|
204
|
+
|
|
205
|
+
The next URL comes from the server and is untrusted. By default, `createFetchPager` follows a next
|
|
206
|
+
URL only when its origin matches the origin of the start URL. A next URL on another host, or on
|
|
207
|
+
`http` for an `https` start URL, raises `INVALID_LINK` and is not requested. This stops a compromised
|
|
208
|
+
or misconfigured server from sending the client to a foreign host, where your tokens could leak.
|
|
209
|
+
Enable `allowCrossOriginNext` only if the API documents cross-origin next links.
|
|
210
|
+
|
|
211
|
+
Error messages mask query values (`?token=***`). Credentials and fragments are removed from URLs in
|
|
212
|
+
errors.
|
|
213
|
+
|
|
214
|
+
See [SECURITY.md](SECURITY.md) for the full security model and known limitations.
|
|
215
|
+
|
|
216
|
+
## Runtime support
|
|
217
|
+
|
|
218
|
+
Node.js 20+, Bun and Deno. The package is ESM with a CommonJS build. Tests use an injected `fetch`
|
|
219
|
+
and never touch the network.
|
|
220
|
+
|
|
221
|
+
## Development
|
|
222
|
+
|
|
223
|
+
```sh
|
|
224
|
+
pnpm install
|
|
225
|
+
pnpm lint
|
|
226
|
+
pnpm typecheck
|
|
227
|
+
pnpm test
|
|
228
|
+
pnpm build
|
|
229
|
+
```
|
|
230
|
+
|
|
231
|
+
## License
|
|
232
|
+
|
|
233
|
+
MIT. See [LICENSE](LICENSE).
|
package/dist/index.cjs
ADDED
|
@@ -0,0 +1,409 @@
|
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: "Module" });
|
|
2
|
+
// src/errors.ts
|
|
3
|
+
/**
|
|
4
|
+
* Error raised by the pagination guards.
|
|
5
|
+
*
|
|
6
|
+
* `page` is the 1-based number of the page whose request or response caused the error.
|
|
7
|
+
* Adapters such as `createFetchPager` do not know the page number and use `0`; `paginate()`
|
|
8
|
+
* replaces it with the real number before the error reaches the caller.
|
|
9
|
+
*/
|
|
10
|
+
var PaginationError = class extends Error {
|
|
11
|
+
code;
|
|
12
|
+
page;
|
|
13
|
+
constructor(code, page, message, options) {
|
|
14
|
+
super(message, options);
|
|
15
|
+
this.name = "PaginationError";
|
|
16
|
+
this.code = code;
|
|
17
|
+
this.page = page;
|
|
18
|
+
}
|
|
19
|
+
};
|
|
20
|
+
// src/link-header.ts
|
|
21
|
+
const TOKEN_CHAR = /[!#$%&'*+\-.^_`|~0-9A-Za-z]/;
|
|
22
|
+
const NEXT_MENTION = /\brel\s*=\s*"?[^";,]*\bnext\b/i;
|
|
23
|
+
function isTokenChar(ch) {
|
|
24
|
+
return TOKEN_CHAR.test(ch);
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Tokenizer for RFC 8288 Link headers. URI references are read between `<` and `>`, so commas and
|
|
28
|
+
* semicolons inside a URL are not treated as separators. Quoted parameter values support backslash
|
|
29
|
+
* escapes. Malformed link-values are skipped; the scan resumes at the next `<`.
|
|
30
|
+
*/
|
|
31
|
+
function scanLinkHeader(value) {
|
|
32
|
+
const links = {};
|
|
33
|
+
let malformedNext = false;
|
|
34
|
+
const n = value.length;
|
|
35
|
+
let i = 0;
|
|
36
|
+
const skipSpaces = () => {
|
|
37
|
+
while (i < n && (value.charAt(i) === " " || value.charAt(i) === " ")) i += 1;
|
|
38
|
+
};
|
|
39
|
+
while (i < n) {
|
|
40
|
+
skipSpaces();
|
|
41
|
+
if (i >= n) break;
|
|
42
|
+
if (value.charAt(i) === ",") {
|
|
43
|
+
i += 1;
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
const linkStart = i;
|
|
47
|
+
let target;
|
|
48
|
+
if (value.charAt(i) === "<") {
|
|
49
|
+
const close = value.indexOf(">", i + 1);
|
|
50
|
+
if (close !== -1) {
|
|
51
|
+
target = value.slice(i + 1, close).trim();
|
|
52
|
+
i = close + 1;
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
let ok = target !== void 0;
|
|
56
|
+
const rels = [];
|
|
57
|
+
while (ok) {
|
|
58
|
+
skipSpaces();
|
|
59
|
+
if (i >= n) break;
|
|
60
|
+
const ch = value.charAt(i);
|
|
61
|
+
if (ch === ",") {
|
|
62
|
+
i += 1;
|
|
63
|
+
break;
|
|
64
|
+
}
|
|
65
|
+
if (ch !== ";") {
|
|
66
|
+
ok = false;
|
|
67
|
+
break;
|
|
68
|
+
}
|
|
69
|
+
i += 1;
|
|
70
|
+
skipSpaces();
|
|
71
|
+
const nameStart = i;
|
|
72
|
+
while (i < n && isTokenChar(value.charAt(i))) i += 1;
|
|
73
|
+
const name = value.slice(nameStart, i).toLowerCase();
|
|
74
|
+
if (name === "") {
|
|
75
|
+
ok = false;
|
|
76
|
+
break;
|
|
77
|
+
}
|
|
78
|
+
skipSpaces();
|
|
79
|
+
let paramValue;
|
|
80
|
+
if (value.charAt(i) === "=") {
|
|
81
|
+
i += 1;
|
|
82
|
+
skipSpaces();
|
|
83
|
+
if (value.charAt(i) === "\"") {
|
|
84
|
+
i += 1;
|
|
85
|
+
let quoted = "";
|
|
86
|
+
let closed = false;
|
|
87
|
+
while (i < n) {
|
|
88
|
+
const c = value.charAt(i);
|
|
89
|
+
if (c === "\\" && i + 1 < n) {
|
|
90
|
+
quoted += value.charAt(i + 1);
|
|
91
|
+
i += 2;
|
|
92
|
+
continue;
|
|
93
|
+
}
|
|
94
|
+
i += 1;
|
|
95
|
+
if (c === "\"") {
|
|
96
|
+
closed = true;
|
|
97
|
+
break;
|
|
98
|
+
}
|
|
99
|
+
quoted += c;
|
|
100
|
+
}
|
|
101
|
+
if (!closed) {
|
|
102
|
+
ok = false;
|
|
103
|
+
break;
|
|
104
|
+
}
|
|
105
|
+
paramValue = quoted;
|
|
106
|
+
} else {
|
|
107
|
+
const valueStart = i;
|
|
108
|
+
while (i < n && isTokenChar(value.charAt(i))) i += 1;
|
|
109
|
+
if (i === valueStart) {
|
|
110
|
+
ok = false;
|
|
111
|
+
break;
|
|
112
|
+
}
|
|
113
|
+
paramValue = value.slice(valueStart, i);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (name === "rel" && paramValue !== void 0) rels.push(...paramValue.split(/\s+/).filter((rel) => rel !== ""));
|
|
117
|
+
}
|
|
118
|
+
if (ok && target !== void 0) for (const rel of rels) {
|
|
119
|
+
const key = rel.toLowerCase();
|
|
120
|
+
if (key !== "__proto__" && !Object.hasOwn(links, key)) links[key] = target;
|
|
121
|
+
}
|
|
122
|
+
else {
|
|
123
|
+
malformedNext = malformedNext || NEXT_MENTION.test(value.slice(linkStart));
|
|
124
|
+
const nextLink = value.indexOf("<", linkStart + 1);
|
|
125
|
+
i = nextLink === -1 ? n : nextLink;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
return {
|
|
129
|
+
links,
|
|
130
|
+
malformedNext
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Parses an RFC 8288 `Link` header value into a map from relation type to URI reference.
|
|
135
|
+
*
|
|
136
|
+
* Handles quoted and unquoted `rel` values, several relation types in one `rel`, several links
|
|
137
|
+
* separated by commas, additional parameters and URIs that contain commas or semicolons.
|
|
138
|
+
* Relation names are lower-cased. For duplicate relations the first link wins. Malformed
|
|
139
|
+
* link-values are ignored.
|
|
140
|
+
*/
|
|
141
|
+
function parseLinkHeader(value) {
|
|
142
|
+
return scanLinkHeader(value).links;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Returns the URI reference of the `rel="next"` link from a `Link` header, or `undefined`.
|
|
146
|
+
* The URI may be relative; resolve it against the request URL.
|
|
147
|
+
*
|
|
148
|
+
* Throws a `PaginationError` with code `INVALID_LINK` when a malformed link-value mentions
|
|
149
|
+
* `rel="next"` and no valid next link was found. Otherwise a broken header would look like the
|
|
150
|
+
* last page and silently truncate the data.
|
|
151
|
+
*/
|
|
152
|
+
function nextFromLinkHeader(headers) {
|
|
153
|
+
const raw = headers.get("link");
|
|
154
|
+
if (raw === null) return;
|
|
155
|
+
const { links, malformedNext } = scanLinkHeader(raw);
|
|
156
|
+
if (Object.hasOwn(links, "next")) return links.next;
|
|
157
|
+
if (malformedNext) throw new PaginationError("INVALID_LINK", 0, "Malformed Link header: rel=\"next\" could not be parsed");
|
|
158
|
+
}
|
|
159
|
+
// src/mask-url.ts
|
|
160
|
+
/**
|
|
161
|
+
* Returns a URL that is safe to put into an error message: credentials are removed, query values
|
|
162
|
+
* are replaced with `***` (keys are kept) and fragments are dropped.
|
|
163
|
+
*/
|
|
164
|
+
function maskUrl(input) {
|
|
165
|
+
let url;
|
|
166
|
+
try {
|
|
167
|
+
url = new URL(input);
|
|
168
|
+
} catch {
|
|
169
|
+
return "<invalid URL>";
|
|
170
|
+
}
|
|
171
|
+
const origin = url.origin === "null" ? url.protocol : url.origin;
|
|
172
|
+
const keys = [...new Set(url.searchParams.keys())];
|
|
173
|
+
const query = keys.length > 0 ? `?${keys.map((key) => `${encodeURIComponent(key)}=***`).join("&")}` : "";
|
|
174
|
+
return `${origin}${url.pathname}${query}`;
|
|
175
|
+
}
|
|
176
|
+
// src/fetch-pager.ts
|
|
177
|
+
const MAX_RETRIES = 3;
|
|
178
|
+
const DEFAULT_RETRY_AFTER_MS = 1e3;
|
|
179
|
+
const MAX_RETRY_AFTER_MS = 6e4;
|
|
180
|
+
/**
|
|
181
|
+
* Parses the `Retry-After` value (delay in seconds or HTTP date) into milliseconds, capped at
|
|
182
|
+
* 60 seconds. A missing or unparsable value falls back to one second.
|
|
183
|
+
*/
|
|
184
|
+
function retryAfterMs(header, now) {
|
|
185
|
+
if (header === null) return DEFAULT_RETRY_AFTER_MS;
|
|
186
|
+
const value = header.trim();
|
|
187
|
+
let ms;
|
|
188
|
+
if (/^\d+$/.test(value)) ms = Number(value) * 1e3;
|
|
189
|
+
else {
|
|
190
|
+
const date = Date.parse(value);
|
|
191
|
+
if (Number.isNaN(date)) return DEFAULT_RETRY_AFTER_MS;
|
|
192
|
+
ms = Math.max(0, date - now);
|
|
193
|
+
}
|
|
194
|
+
return Math.min(ms, MAX_RETRY_AFTER_MS);
|
|
195
|
+
}
|
|
196
|
+
/** Waits for `ms`, rejecting with the signal's reason if the signal aborts first. */
|
|
197
|
+
function sleep(ms, signal) {
|
|
198
|
+
return new Promise((resolve, reject) => {
|
|
199
|
+
if (signal.aborted) {
|
|
200
|
+
reject(signal.reason);
|
|
201
|
+
return;
|
|
202
|
+
}
|
|
203
|
+
const onAbort = () => {
|
|
204
|
+
clearTimeout(timer);
|
|
205
|
+
reject(signal.reason);
|
|
206
|
+
};
|
|
207
|
+
const timer = setTimeout(() => {
|
|
208
|
+
signal.removeEventListener("abort", onAbort);
|
|
209
|
+
resolve();
|
|
210
|
+
}, ms);
|
|
211
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
212
|
+
});
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Creates a `fetchPage` function for `paginate()` that follows URL-based cursors.
|
|
216
|
+
*
|
|
217
|
+
* - Next URLs come from the `Link` header (RFC 8288) or from `nextFromResponse`.
|
|
218
|
+
* - Next URLs on a different origin than `url` are rejected with `INVALID_LINK`, unless
|
|
219
|
+
* `allowCrossOriginNext` is set.
|
|
220
|
+
* - HTTP 429 waits for `Retry-After` (seconds or HTTP date, capped at 60 s) and retries the same
|
|
221
|
+
* page up to 3 times, then throws `RATE_LIMITED`. The wait counts towards the page timeout.
|
|
222
|
+
* - Other non-2xx responses throw a plain `Error` that contains only the status code.
|
|
223
|
+
*
|
|
224
|
+
* Errors thrown here have `page: 0`; `paginate()` sets the real page number.
|
|
225
|
+
*/
|
|
226
|
+
function createFetchPager(options) {
|
|
227
|
+
const doFetch = options.fetch ?? globalThis.fetch;
|
|
228
|
+
if (typeof doFetch !== "function") throw new TypeError("No fetch implementation available; pass options.fetch");
|
|
229
|
+
const startUrl = new URL(options.url);
|
|
230
|
+
const startOrigin = startUrl.origin;
|
|
231
|
+
const allowCrossOrigin = options.allowCrossOriginNext === true;
|
|
232
|
+
const extract = options.extract;
|
|
233
|
+
const nextFromResponse = options.nextFromResponse;
|
|
234
|
+
const resolveNext = (raw, base) => {
|
|
235
|
+
if (raw === null || raw === void 0 || raw === "") return null;
|
|
236
|
+
let next;
|
|
237
|
+
try {
|
|
238
|
+
next = new URL(raw, base);
|
|
239
|
+
} catch {
|
|
240
|
+
throw new PaginationError("INVALID_LINK", 0, "Next link is not a valid URL");
|
|
241
|
+
}
|
|
242
|
+
if (!allowCrossOrigin && next.origin !== startOrigin) throw new PaginationError("INVALID_LINK", 0, `Next link points to another origin (${next.origin}) and was not followed. Set allowCrossOriginNext to allow it. Link: ${maskUrl(next.href)}`);
|
|
243
|
+
return next.href;
|
|
244
|
+
};
|
|
245
|
+
return async (cursor, signal) => {
|
|
246
|
+
const target = cursor === void 0 ? startUrl.href : cursor;
|
|
247
|
+
let retries = 0;
|
|
248
|
+
for (;;) {
|
|
249
|
+
const res = await doFetch(target, { signal });
|
|
250
|
+
if (res.status === 429) {
|
|
251
|
+
await res.body?.cancel().catch(() => void 0);
|
|
252
|
+
if (retries >= MAX_RETRIES) throw new PaginationError("RATE_LIMITED", 0, `Still rate limited (HTTP 429) after ${MAX_RETRIES} retries`);
|
|
253
|
+
retries += 1;
|
|
254
|
+
await sleep(retryAfterMs(res.headers.get("retry-after"), Date.now()), signal);
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
if (!res.ok) throw new Error(`Request failed with HTTP ${res.status}`);
|
|
258
|
+
let body;
|
|
259
|
+
try {
|
|
260
|
+
body = await res.json();
|
|
261
|
+
} catch (error) {
|
|
262
|
+
if (error instanceof SyntaxError) throw new Error(`Response of HTTP ${res.status} is not valid JSON`, { cause: error });
|
|
263
|
+
throw error;
|
|
264
|
+
}
|
|
265
|
+
const items = extract(body);
|
|
266
|
+
if (!Array.isArray(items)) throw new TypeError("extract() must return an array of items");
|
|
267
|
+
const raw = nextFromResponse ? nextFromResponse(res, body) : nextFromLinkHeader(res.headers);
|
|
268
|
+
return {
|
|
269
|
+
items,
|
|
270
|
+
nextCursor: resolveNext(raw, target)
|
|
271
|
+
};
|
|
272
|
+
}
|
|
273
|
+
};
|
|
274
|
+
}
|
|
275
|
+
// src/paginate.ts
|
|
276
|
+
const DEFAULT_MAX_PAGES = 100;
|
|
277
|
+
const DEFAULT_TIMEOUT_MS = 3e4;
|
|
278
|
+
const MAX_EMPTY_PAGES = 3;
|
|
279
|
+
function resolveLimits(options) {
|
|
280
|
+
const maxPages = options.maxPages ?? DEFAULT_MAX_PAGES;
|
|
281
|
+
const maxItems = options.maxItems ?? Number.POSITIVE_INFINITY;
|
|
282
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
283
|
+
const totalTimeoutMs = options.totalTimeoutMs ?? Number.POSITIVE_INFINITY;
|
|
284
|
+
if (!(maxPages === Number.POSITIVE_INFINITY || Number.isInteger(maxPages) && maxPages >= 1)) throw new RangeError("maxPages must be a positive integer or Infinity");
|
|
285
|
+
if (!(maxItems === Number.POSITIVE_INFINITY || Number.isInteger(maxItems) && maxItems >= 0)) throw new RangeError("maxItems must be a non-negative integer or Infinity");
|
|
286
|
+
if (!(timeoutMs > 0)) throw new RangeError("timeoutMs must be greater than 0");
|
|
287
|
+
if (!(totalTimeoutMs > 0)) throw new RangeError("totalTimeoutMs must be greater than 0");
|
|
288
|
+
return {
|
|
289
|
+
maxPages,
|
|
290
|
+
maxItems,
|
|
291
|
+
timeoutMs,
|
|
292
|
+
totalTimeoutMs
|
|
293
|
+
};
|
|
294
|
+
}
|
|
295
|
+
/** `undefined`, `null` and `""` mean "no next page". */
|
|
296
|
+
function normalizeCursor(value) {
|
|
297
|
+
return value === void 0 || value === null || value === "" ? void 0 : value;
|
|
298
|
+
}
|
|
299
|
+
/** Adapters without page context throw with page 0; this fills in the real page number. */
|
|
300
|
+
function withPage(error, page) {
|
|
301
|
+
if (error instanceof PaginationError && error.page === 0) return new PaginationError(error.code, page, error.message, { cause: error });
|
|
302
|
+
return error;
|
|
303
|
+
}
|
|
304
|
+
async function* iterate(fetchPage, limits, userSignal) {
|
|
305
|
+
const run = new AbortController();
|
|
306
|
+
const inflight = {};
|
|
307
|
+
const seen = /* @__PURE__ */ new Set();
|
|
308
|
+
let pageNo = 1;
|
|
309
|
+
let cursor;
|
|
310
|
+
let emitted = 0;
|
|
311
|
+
let emptyStreak = 0;
|
|
312
|
+
let totalTimer;
|
|
313
|
+
const onUserAbort = () => {
|
|
314
|
+
run.abort(new PaginationError("ABORTED", pageNo, "Pagination was aborted", { cause: userSignal?.reason }));
|
|
315
|
+
};
|
|
316
|
+
/**
|
|
317
|
+
* Requests one page. The page signal is aborted when the run is aborted (user signal, total
|
|
318
|
+
* timeout or iterator return) and after `timeoutMs`. The promise rejects as soon as the signal
|
|
319
|
+
* fires, even if `fetchPage` ignores the signal.
|
|
320
|
+
*/
|
|
321
|
+
const request = (pageCursor, page) => new Promise((resolve, reject) => {
|
|
322
|
+
if (run.signal.aborted) {
|
|
323
|
+
reject(run.signal.reason);
|
|
324
|
+
return;
|
|
325
|
+
}
|
|
326
|
+
const ctl = new AbortController();
|
|
327
|
+
inflight.ctl = ctl;
|
|
328
|
+
let timer;
|
|
329
|
+
const onParentAbort = () => ctl.abort(run.signal.reason);
|
|
330
|
+
const onPageAbort = () => settle(() => reject(ctl.signal.reason));
|
|
331
|
+
const settle = (done) => {
|
|
332
|
+
clearTimeout(timer);
|
|
333
|
+
run.signal.removeEventListener("abort", onParentAbort);
|
|
334
|
+
ctl.signal.removeEventListener("abort", onPageAbort);
|
|
335
|
+
done();
|
|
336
|
+
};
|
|
337
|
+
run.signal.addEventListener("abort", onParentAbort, { once: true });
|
|
338
|
+
ctl.signal.addEventListener("abort", onPageAbort, { once: true });
|
|
339
|
+
if (Number.isFinite(limits.timeoutMs)) timer = setTimeout(() => {
|
|
340
|
+
ctl.abort(new PaginationError("TIMEOUT", page, `Page ${page} timed out after ${limits.timeoutMs} ms`));
|
|
341
|
+
}, limits.timeoutMs);
|
|
342
|
+
let work;
|
|
343
|
+
try {
|
|
344
|
+
work = Promise.resolve(fetchPage(pageCursor, ctl.signal));
|
|
345
|
+
} catch (error) {
|
|
346
|
+
work = Promise.reject(error);
|
|
347
|
+
}
|
|
348
|
+
work.then((result) => settle(() => resolve(result)), (error) => settle(() => reject(withPage(error, page))));
|
|
349
|
+
});
|
|
350
|
+
if (userSignal?.aborted) onUserAbort();
|
|
351
|
+
else userSignal?.addEventListener("abort", onUserAbort, { once: true });
|
|
352
|
+
if (Number.isFinite(limits.totalTimeoutMs)) totalTimer = setTimeout(() => {
|
|
353
|
+
run.abort(new PaginationError("TIMEOUT", pageNo, `Total timeout of ${limits.totalTimeoutMs} ms exceeded`));
|
|
354
|
+
}, limits.totalTimeoutMs);
|
|
355
|
+
try {
|
|
356
|
+
while (emitted < limits.maxItems) {
|
|
357
|
+
if (run.signal.aborted) throw run.signal.reason;
|
|
358
|
+
const result = await request(cursor, pageNo);
|
|
359
|
+
if (!Array.isArray(result.items)) throw new TypeError("fetchPage must resolve to an object with an items array");
|
|
360
|
+
const next = normalizeCursor(result.nextCursor);
|
|
361
|
+
if (next !== void 0 && seen.has(next)) throw new PaginationError("CURSOR_LOOP", pageNo, `Server returned a cursor that was already used (page ${pageNo}); stopping`);
|
|
362
|
+
if (result.items.length === 0) {
|
|
363
|
+
if (next !== void 0) {
|
|
364
|
+
emptyStreak += 1;
|
|
365
|
+
if (emptyStreak >= MAX_EMPTY_PAGES) throw new PaginationError("EMPTY_PAGES_LOOP", pageNo, `${MAX_EMPTY_PAGES} consecutive empty pages with a next cursor (page ${pageNo}); stopping`);
|
|
366
|
+
}
|
|
367
|
+
} else emptyStreak = 0;
|
|
368
|
+
for (const item of result.items) {
|
|
369
|
+
if (run.signal.aborted) throw run.signal.reason;
|
|
370
|
+
yield item;
|
|
371
|
+
emitted += 1;
|
|
372
|
+
if (emitted >= limits.maxItems) return;
|
|
373
|
+
}
|
|
374
|
+
if (next === void 0) return;
|
|
375
|
+
if (pageNo >= limits.maxPages) throw new PaginationError("PAGE_LIMIT", pageNo, `Page limit of ${limits.maxPages} reached while more pages are available`);
|
|
376
|
+
seen.add(next);
|
|
377
|
+
cursor = next;
|
|
378
|
+
pageNo += 1;
|
|
379
|
+
}
|
|
380
|
+
} finally {
|
|
381
|
+
clearTimeout(totalTimer);
|
|
382
|
+
userSignal?.removeEventListener("abort", onUserAbort);
|
|
383
|
+
inflight.ctl?.abort();
|
|
384
|
+
run.abort();
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
/**
|
|
388
|
+
* Iterates over all items of a paginated API.
|
|
389
|
+
*
|
|
390
|
+
* Guards: repeated cursors (`CURSOR_LOOP`), a page limit (`PAGE_LIMIT`), endless empty pages
|
|
391
|
+
* (`EMPTY_PAGES_LOOP`), per-page and total timeouts (`TIMEOUT`) and an abort signal (`ABORTED`).
|
|
392
|
+
* Reaching `maxItems` stops without error.
|
|
393
|
+
*
|
|
394
|
+
* Breaking out of a `for await` loop calls `return()`, which aborts the signal of any request
|
|
395
|
+
* that is still open.
|
|
396
|
+
*
|
|
397
|
+
* Options are validated when `paginate()` is called. Iteration starts on the first `next()`.
|
|
398
|
+
*/
|
|
399
|
+
function paginate(options) {
|
|
400
|
+
if (typeof options.fetchPage !== "function") throw new TypeError("fetchPage must be a function");
|
|
401
|
+
return iterate(options.fetchPage, resolveLimits(options), options.signal);
|
|
402
|
+
}
|
|
403
|
+
exports.PaginationError = PaginationError;
|
|
404
|
+
exports.createFetchPager = createFetchPager;
|
|
405
|
+
exports.nextFromLinkHeader = nextFromLinkHeader;
|
|
406
|
+
exports.paginate = paginate;
|
|
407
|
+
exports.parseLinkHeader = parseLinkHeader;
|
|
408
|
+
|
|
409
|
+
//# sourceMappingURL=index.cjs.map
|