open-alex-wrapper 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +437 -0
- package/dist/cli.cjs +1880 -0
- package/dist/cli.cjs.map +1 -0
- package/dist/cli.d.cts +31 -0
- package/dist/cli.d.ts +31 -0
- package/dist/cli.js +1848 -0
- package/dist/cli.js.map +1 -0
- package/dist/client-CLRZTPFv.d.cts +1266 -0
- package/dist/client-CLRZTPFv.d.ts +1266 -0
- package/dist/index.cjs +1831 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +105 -0
- package/dist/index.d.ts +105 -0
- package/dist/index.js +1750 -0
- package/dist/index.js.map +1 -0
- package/package.json +66 -0
package/dist/cli.cjs
ADDED
|
@@ -0,0 +1,1880 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
"use strict";
|
|
3
|
+
var __create = Object.create;
|
|
4
|
+
var __defProp = Object.defineProperty;
|
|
5
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
6
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
7
|
+
var __getProtoOf = Object.getPrototypeOf;
|
|
8
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
9
|
+
var __esm = (fn, res) => function __init() {
|
|
10
|
+
return fn && (res = (0, fn[__getOwnPropNames(fn)[0]])(fn = 0)), res;
|
|
11
|
+
};
|
|
12
|
+
var __export = (target, all) => {
|
|
13
|
+
for (var name in all)
|
|
14
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
15
|
+
};
|
|
16
|
+
var __copyProps = (to, from, except, desc) => {
|
|
17
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
18
|
+
for (let key of __getOwnPropNames(from))
|
|
19
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
20
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
21
|
+
}
|
|
22
|
+
return to;
|
|
23
|
+
};
|
|
24
|
+
var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
|
|
25
|
+
// If the importer is in node compatibility mode or this is not an ESM
|
|
26
|
+
// file that has been converted to a CommonJS file using a Babel-
|
|
27
|
+
// compatible transform (i.e. "__esModule" has not been set), then set
|
|
28
|
+
// "default" to the CommonJS "module.exports" for node compatibility.
|
|
29
|
+
isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
|
|
30
|
+
mod
|
|
31
|
+
));
|
|
32
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
33
|
+
|
|
34
|
+
// src/export.ts
|
|
35
|
+
var export_exports = {};
|
|
36
|
+
__export(export_exports, {
|
|
37
|
+
exportToFile: () => exportToFile,
|
|
38
|
+
flatten: () => flatten
|
|
39
|
+
});
|
|
40
|
+
async function exportToFile(source, path, opts = {}) {
|
|
41
|
+
const format = opts.format ?? inferFormat(path);
|
|
42
|
+
const stream = (0, import_node_fs.createWriteStream)(path, { encoding: "utf8" });
|
|
43
|
+
const write = makeWriter(stream);
|
|
44
|
+
let count = 0;
|
|
45
|
+
try {
|
|
46
|
+
if (format === "jsonl") {
|
|
47
|
+
for await (const item of source) {
|
|
48
|
+
await write(JSON.stringify(item) + "\n");
|
|
49
|
+
count++;
|
|
50
|
+
}
|
|
51
|
+
} else if (format === "json") {
|
|
52
|
+
await write("[");
|
|
53
|
+
let first = true;
|
|
54
|
+
for await (const item of source) {
|
|
55
|
+
await write((first ? "" : ",\n") + JSON.stringify(item));
|
|
56
|
+
first = false;
|
|
57
|
+
count++;
|
|
58
|
+
}
|
|
59
|
+
await write("]\n");
|
|
60
|
+
} else {
|
|
61
|
+
let headers = null;
|
|
62
|
+
for await (const item of source) {
|
|
63
|
+
const row = opts.flatten === false ? item : flatten(item);
|
|
64
|
+
if (!headers) {
|
|
65
|
+
headers = Object.keys(row);
|
|
66
|
+
await write(headers.map(csvCell).join(",") + "\n");
|
|
67
|
+
}
|
|
68
|
+
await write(headers.map((h) => csvCell(row[h])).join(",") + "\n");
|
|
69
|
+
count++;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
} finally {
|
|
73
|
+
await endStream(stream);
|
|
74
|
+
}
|
|
75
|
+
return { path, count };
|
|
76
|
+
}
|
|
77
|
+
function makeWriter(stream) {
|
|
78
|
+
return (s) => new Promise((resolve, reject) => {
|
|
79
|
+
stream.write(s, (err) => {
|
|
80
|
+
if (err) reject(err);
|
|
81
|
+
});
|
|
82
|
+
if (stream.writableNeedDrain) stream.once("drain", resolve);
|
|
83
|
+
else resolve();
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
function endStream(stream) {
|
|
87
|
+
return new Promise((resolve, reject) => {
|
|
88
|
+
stream.end((err) => err ? reject(err) : resolve());
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
function inferFormat(path) {
|
|
92
|
+
const p = path.toLowerCase();
|
|
93
|
+
if (p.endsWith(".jsonl") || p.endsWith(".ndjson")) return "jsonl";
|
|
94
|
+
if (p.endsWith(".csv")) return "csv";
|
|
95
|
+
if (p.endsWith(".json")) return "json";
|
|
96
|
+
return "jsonl";
|
|
97
|
+
}
|
|
98
|
+
function flatten(obj, prefix = "", out = {}) {
|
|
99
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
100
|
+
const key = prefix ? `${prefix}.${k}` : k;
|
|
101
|
+
if (v && typeof v === "object" && !Array.isArray(v)) {
|
|
102
|
+
flatten(v, key, out);
|
|
103
|
+
} else {
|
|
104
|
+
out[key] = v;
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
return out;
|
|
108
|
+
}
|
|
109
|
+
function csvCell(value) {
|
|
110
|
+
if (value === null || value === void 0) return "";
|
|
111
|
+
const s = typeof value === "object" ? JSON.stringify(value) : String(value);
|
|
112
|
+
if (/[",\n\r]/.test(s)) return `"${s.replace(/"/g, '""')}"`;
|
|
113
|
+
return s;
|
|
114
|
+
}
|
|
115
|
+
var import_node_fs;
|
|
116
|
+
var init_export = __esm({
|
|
117
|
+
"src/export.ts"() {
|
|
118
|
+
"use strict";
|
|
119
|
+
import_node_fs = require("fs");
|
|
120
|
+
}
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
// src/cli.ts
|
|
124
|
+
var cli_exports = {};
|
|
125
|
+
__export(cli_exports, {
|
|
126
|
+
parseArgs: () => parseArgs,
|
|
127
|
+
runCli: () => runCli
|
|
128
|
+
});
|
|
129
|
+
module.exports = __toCommonJS(cli_exports);
|
|
130
|
+
|
|
131
|
+
// src/cache.ts
|
|
132
|
+
var LRUCache = class {
|
|
133
|
+
constructor(maxEntries = 1e3) {
|
|
134
|
+
this.maxEntries = maxEntries;
|
|
135
|
+
}
|
|
136
|
+
maxEntries;
|
|
137
|
+
map = /* @__PURE__ */ new Map();
|
|
138
|
+
get(key) {
|
|
139
|
+
const entry = this.map.get(key);
|
|
140
|
+
if (!entry) return void 0;
|
|
141
|
+
if (entry.expiresAt !== 0 && entry.expiresAt < Date.now()) {
|
|
142
|
+
this.map.delete(key);
|
|
143
|
+
return void 0;
|
|
144
|
+
}
|
|
145
|
+
this.map.delete(key);
|
|
146
|
+
this.map.set(key, entry);
|
|
147
|
+
return entry.value;
|
|
148
|
+
}
|
|
149
|
+
set(key, value, ttlMs) {
|
|
150
|
+
if (this.map.has(key)) this.map.delete(key);
|
|
151
|
+
this.map.set(key, { value, expiresAt: ttlMs > 0 ? Date.now() + ttlMs : 0 });
|
|
152
|
+
while (this.map.size > this.maxEntries) {
|
|
153
|
+
const oldest = this.map.keys().next().value;
|
|
154
|
+
if (oldest === void 0) break;
|
|
155
|
+
this.map.delete(oldest);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
delete(key) {
|
|
159
|
+
this.map.delete(key);
|
|
160
|
+
}
|
|
161
|
+
clear() {
|
|
162
|
+
this.map.clear();
|
|
163
|
+
}
|
|
164
|
+
get size() {
|
|
165
|
+
return this.map.size;
|
|
166
|
+
}
|
|
167
|
+
};
|
|
168
|
+
|
|
169
|
+
// src/config.ts
|
|
170
|
+
var NOOP_LOGGER = {
|
|
171
|
+
debug: () => {
|
|
172
|
+
},
|
|
173
|
+
info: () => {
|
|
174
|
+
},
|
|
175
|
+
warn: () => {
|
|
176
|
+
},
|
|
177
|
+
error: () => {
|
|
178
|
+
}
|
|
179
|
+
};
|
|
180
|
+
var DEFAULT_RETRY = {
|
|
181
|
+
maxRetries: 5,
|
|
182
|
+
baseDelayMs: 500,
|
|
183
|
+
maxDelayMs: 2e4,
|
|
184
|
+
retryOnStatus: [429, 500, 502, 503, 504]
|
|
185
|
+
};
|
|
186
|
+
var PKG_VERSION = "0.1.0";
|
|
187
|
+
function resolveConfig(config = {}) {
|
|
188
|
+
const env = typeof process !== "undefined" ? process.env : {};
|
|
189
|
+
const apiKey = config.apiKey ?? env.OPENALEX_API_KEY;
|
|
190
|
+
const mailto = config.mailto ?? env.OPENALEX_MAILTO;
|
|
191
|
+
const globalFetch = typeof fetch !== "undefined" ? fetch : void 0;
|
|
192
|
+
const fetchImpl = config.fetch ?? globalFetch;
|
|
193
|
+
if (!fetchImpl) {
|
|
194
|
+
throw new Error(
|
|
195
|
+
"No fetch implementation available. Use Node 18+, or pass `fetch` in the config."
|
|
196
|
+
);
|
|
197
|
+
}
|
|
198
|
+
let cache;
|
|
199
|
+
if (config.cache === true) cache = new LRUCache();
|
|
200
|
+
else if (config.cache && typeof config.cache === "object") cache = config.cache;
|
|
201
|
+
let throttle;
|
|
202
|
+
if (config.throttle === true) throttle = { maxRequestsPerSecond: 10 };
|
|
203
|
+
else if (config.throttle && typeof config.throttle === "object") {
|
|
204
|
+
throttle = {
|
|
205
|
+
maxRequestsPerSecond: config.throttle.maxRequestsPerSecond ?? 10,
|
|
206
|
+
minRemainingUsd: config.throttle.minRemainingUsd
|
|
207
|
+
};
|
|
208
|
+
}
|
|
209
|
+
const ua = [`open-alex-wrapper/${PKG_VERSION}`, config.userAgent, mailto ? `mailto:${mailto}` : void 0].filter(Boolean).join(" ");
|
|
210
|
+
return {
|
|
211
|
+
apiKey,
|
|
212
|
+
mailto,
|
|
213
|
+
baseUrl: (config.baseUrl ?? "https://api.openalex.org").replace(/\/+$/, ""),
|
|
214
|
+
userAgent: ua,
|
|
215
|
+
timeoutMs: config.timeoutMs ?? 6e4,
|
|
216
|
+
retry: { ...DEFAULT_RETRY, ...config.retry },
|
|
217
|
+
concurrency: Math.max(1, config.concurrency ?? 8),
|
|
218
|
+
dailyBudgetUsd: config.dailyBudgetUsd,
|
|
219
|
+
onBudgetWarning: config.onBudgetWarning,
|
|
220
|
+
cache,
|
|
221
|
+
cacheTtlMs: config.cacheTtlMs ?? 3e5,
|
|
222
|
+
throttle,
|
|
223
|
+
fetch: fetchImpl,
|
|
224
|
+
logger: config.logger ?? NOOP_LOGGER
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
// src/errors.ts
|
|
229
|
+
var OpenAlexError = class extends Error {
|
|
230
|
+
status;
|
|
231
|
+
url;
|
|
232
|
+
body;
|
|
233
|
+
constructor(message, ctx = {}) {
|
|
234
|
+
super(message, ctx.cause !== void 0 ? { cause: ctx.cause } : void 0);
|
|
235
|
+
this.name = new.target.name;
|
|
236
|
+
this.status = ctx.status;
|
|
237
|
+
this.url = ctx.url;
|
|
238
|
+
this.body = ctx.body;
|
|
239
|
+
Object.setPrototypeOf(this, new.target.prototype);
|
|
240
|
+
}
|
|
241
|
+
};
|
|
242
|
+
var AuthenticationError = class extends OpenAlexError {
|
|
243
|
+
};
|
|
244
|
+
var NotFoundError = class extends OpenAlexError {
|
|
245
|
+
};
|
|
246
|
+
var ValidationError = class extends OpenAlexError {
|
|
247
|
+
};
|
|
248
|
+
var RateLimitError = class extends OpenAlexError {
|
|
249
|
+
retryAfterMs;
|
|
250
|
+
constructor(message, ctx = {}) {
|
|
251
|
+
super(message, ctx);
|
|
252
|
+
this.retryAfterMs = ctx.retryAfterMs;
|
|
253
|
+
}
|
|
254
|
+
};
|
|
255
|
+
var ServerError = class extends OpenAlexError {
|
|
256
|
+
};
|
|
257
|
+
var TimeoutError = class extends OpenAlexError {
|
|
258
|
+
};
|
|
259
|
+
var NetworkError = class extends OpenAlexError {
|
|
260
|
+
};
|
|
261
|
+
var BudgetExceededError = class extends OpenAlexError {
|
|
262
|
+
spentUsd;
|
|
263
|
+
limitUsd;
|
|
264
|
+
wouldSpendUsd;
|
|
265
|
+
constructor(message, info) {
|
|
266
|
+
super(message);
|
|
267
|
+
this.spentUsd = info.spentUsd;
|
|
268
|
+
this.limitUsd = info.limitUsd;
|
|
269
|
+
this.wouldSpendUsd = info.wouldSpendUsd;
|
|
270
|
+
}
|
|
271
|
+
};
|
|
272
|
+
function errorForStatus(status, message, ctx = {}) {
|
|
273
|
+
const full = { ...ctx, status };
|
|
274
|
+
if (status === 401 || status === 403) return new AuthenticationError(message, full);
|
|
275
|
+
if (status === 404) return new NotFoundError(message, full);
|
|
276
|
+
if (status === 400 || status === 422) return new ValidationError(message, full);
|
|
277
|
+
if (status === 429) return new RateLimitError(message, full);
|
|
278
|
+
if (status >= 500) return new ServerError(message, full);
|
|
279
|
+
return new OpenAlexError(message, full);
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
// src/budget.ts
|
|
283
|
+
var PRICING_USD = {
|
|
284
|
+
get: 0,
|
|
285
|
+
list: 1e-4,
|
|
286
|
+
search: 1e-3,
|
|
287
|
+
download: 0.01,
|
|
288
|
+
autocomplete: 1e-4
|
|
289
|
+
};
|
|
290
|
+
var FREE_DAILY_USD_WITH_KEY = 1;
|
|
291
|
+
var FREE_DAILY_USD_WITHOUT_KEY = 0.1;
|
|
292
|
+
var BASIC_PAGING_MAX_RESULTS = 1e4;
|
|
293
|
+
function estimateExtractionCost(input) {
|
|
294
|
+
const perPage = clampPerPage(input.perPage ?? 200);
|
|
295
|
+
const strategy = input.strategy ?? "cursor";
|
|
296
|
+
const price = PRICING_USD[input.requestType];
|
|
297
|
+
const cappedByBasicPaging = strategy === "basic" && input.count > BASIC_PAGING_MAX_RESULTS;
|
|
298
|
+
const fetchableResults = strategy === "basic" ? Math.min(input.count, BASIC_PAGING_MAX_RESULTS) : input.count;
|
|
299
|
+
const pageCalls = fetchableResults === 0 ? 0 : Math.ceil(fetchableResults / perPage);
|
|
300
|
+
const pagesCostUsd = round6(pageCalls * price);
|
|
301
|
+
const probeCostUsd = round6(price);
|
|
302
|
+
return {
|
|
303
|
+
requestType: input.requestType,
|
|
304
|
+
matchingResults: input.count,
|
|
305
|
+
fetchableResults,
|
|
306
|
+
perPage,
|
|
307
|
+
pageCalls,
|
|
308
|
+
pagesCostUsd,
|
|
309
|
+
probeCostUsd,
|
|
310
|
+
totalCostUsd: round6(pagesCostUsd + probeCostUsd),
|
|
311
|
+
cappedByBasicPaging,
|
|
312
|
+
note: cappedByBasicPaging ? `Basic paging reaches only the first ${BASIC_PAGING_MAX_RESULTS.toLocaleString()} results; use cursor paging (the default) for the full ${input.count.toLocaleString()}.` : void 0
|
|
313
|
+
};
|
|
314
|
+
}
|
|
315
|
+
function clampPerPage(n) {
|
|
316
|
+
if (!Number.isFinite(n)) return 200;
|
|
317
|
+
return Math.max(1, Math.min(200, Math.floor(n)));
|
|
318
|
+
}
|
|
319
|
+
function parseUsageHeaders(headers) {
|
|
320
|
+
const map = normalizeHeaders(headers);
|
|
321
|
+
const raw = {};
|
|
322
|
+
for (const [k, v] of map) {
|
|
323
|
+
if (k.toLowerCase().startsWith("x-ratelimit")) raw[k.toLowerCase()] = v;
|
|
324
|
+
}
|
|
325
|
+
const num = (key) => {
|
|
326
|
+
const v = map.get(key);
|
|
327
|
+
if (v === void 0) return void 0;
|
|
328
|
+
const n = Number(String(v).replace(/[$,\s]/g, ""));
|
|
329
|
+
return Number.isFinite(n) ? n : void 0;
|
|
330
|
+
};
|
|
331
|
+
return {
|
|
332
|
+
costUsd: num("x-ratelimit-cost-usd"),
|
|
333
|
+
costRequiredUsd: num("x-ratelimit-cost-required-usd"),
|
|
334
|
+
remainingUsd: num("x-ratelimit-remaining-usd"),
|
|
335
|
+
limitUsd: num("x-ratelimit-limit-usd"),
|
|
336
|
+
prepaidRemainingUsd: num("x-ratelimit-prepaid-remaining-usd"),
|
|
337
|
+
creditsUsed: num("x-ratelimit-credits-used"),
|
|
338
|
+
creditsRemaining: num("x-ratelimit-remaining"),
|
|
339
|
+
creditsLimit: num("x-ratelimit-limit"),
|
|
340
|
+
resetSeconds: num("x-ratelimit-reset"),
|
|
341
|
+
raw
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
function normalizeHeaders(headers) {
|
|
345
|
+
const out = /* @__PURE__ */ new Map();
|
|
346
|
+
if (!headers) return out;
|
|
347
|
+
if (headers instanceof Map) {
|
|
348
|
+
for (const [k, v] of headers) out.set(k.toLowerCase(), String(v));
|
|
349
|
+
return out;
|
|
350
|
+
}
|
|
351
|
+
if (typeof headers.forEach === "function" && !Array.isArray(headers)) {
|
|
352
|
+
;
|
|
353
|
+
headers.forEach((v, k) => out.set(k.toLowerCase(), v));
|
|
354
|
+
return out;
|
|
355
|
+
}
|
|
356
|
+
for (const [k, v] of Object.entries(headers)) {
|
|
357
|
+
if (v === void 0) continue;
|
|
358
|
+
out.set(k.toLowerCase(), Array.isArray(v) ? v.join(",") : String(v));
|
|
359
|
+
}
|
|
360
|
+
return out;
|
|
361
|
+
}
|
|
362
|
+
function emptyByType() {
|
|
363
|
+
const zero = () => ({ calls: 0, results: 0, estimatedCostUsd: 0 });
|
|
364
|
+
return { get: zero(), list: zero(), search: zero(), download: zero(), autocomplete: zero() };
|
|
365
|
+
}
|
|
366
|
+
var UsageTracker = class {
|
|
367
|
+
constructor(hasApiKey, dailyBudgetUsd) {
|
|
368
|
+
this.hasApiKey = hasApiKey;
|
|
369
|
+
this.dailyBudgetUsd = dailyBudgetUsd;
|
|
370
|
+
}
|
|
371
|
+
hasApiKey;
|
|
372
|
+
dailyBudgetUsd;
|
|
373
|
+
byType = emptyByType();
|
|
374
|
+
estimatedSpentUsd = 0;
|
|
375
|
+
reportedRemainingUsd = null;
|
|
376
|
+
reportedLimitUsd = null;
|
|
377
|
+
prepaidRemainingUsd = null;
|
|
378
|
+
creditsRemaining = null;
|
|
379
|
+
resetSeconds = null;
|
|
380
|
+
lastRequestCostUsd = null;
|
|
381
|
+
lastRawUsageHeaders = {};
|
|
382
|
+
/** The daily allowance in effect: OpenAlex-reported, else soft cap, else free tier. */
|
|
383
|
+
get dailyLimitUsd() {
|
|
384
|
+
if (this.reportedLimitUsd !== null) return this.reportedLimitUsd;
|
|
385
|
+
if (this.dailyBudgetUsd !== void 0) return this.dailyBudgetUsd;
|
|
386
|
+
return this.hasApiKey ? FREE_DAILY_USD_WITH_KEY : FREE_DAILY_USD_WITHOUT_KEY;
|
|
387
|
+
}
|
|
388
|
+
/** Authoritative spend today if headers seen (limit − remaining), else estimate. */
|
|
389
|
+
get spentUsd() {
|
|
390
|
+
if (this.reportedLimitUsd !== null && this.reportedRemainingUsd !== null) {
|
|
391
|
+
return round6(this.reportedLimitUsd - this.reportedRemainingUsd);
|
|
392
|
+
}
|
|
393
|
+
return round6(this.estimatedSpentUsd);
|
|
394
|
+
}
|
|
395
|
+
get remainingUsd() {
|
|
396
|
+
if (this.reportedRemainingUsd !== null) return this.reportedRemainingUsd;
|
|
397
|
+
return round6(Math.max(0, this.dailyLimitUsd - this.spentUsd));
|
|
398
|
+
}
|
|
399
|
+
/** Record one completed request. Called by the HTTP layer. */
|
|
400
|
+
record(type, opts = {}) {
|
|
401
|
+
const bucket = this.byType[type];
|
|
402
|
+
bucket.calls += 1;
|
|
403
|
+
bucket.results += opts.results ?? 0;
|
|
404
|
+
const modelled = PRICING_USD[type];
|
|
405
|
+
bucket.estimatedCostUsd = round6(bucket.estimatedCostUsd + modelled);
|
|
406
|
+
this.estimatedSpentUsd = round6(this.estimatedSpentUsd + modelled);
|
|
407
|
+
if (opts.headers) {
|
|
408
|
+
const p = parseUsageHeaders(opts.headers);
|
|
409
|
+
if (Object.keys(p.raw).length) this.lastRawUsageHeaders = p.raw;
|
|
410
|
+
if (p.remainingUsd !== void 0) this.reportedRemainingUsd = p.remainingUsd;
|
|
411
|
+
if (p.limitUsd !== void 0) this.reportedLimitUsd = p.limitUsd;
|
|
412
|
+
if (p.prepaidRemainingUsd !== void 0) this.prepaidRemainingUsd = p.prepaidRemainingUsd;
|
|
413
|
+
if (p.creditsRemaining !== void 0) this.creditsRemaining = p.creditsRemaining;
|
|
414
|
+
if (p.resetSeconds !== void 0) this.resetSeconds = p.resetSeconds;
|
|
415
|
+
if (p.costUsd !== void 0) this.lastRequestCostUsd = p.costUsd;
|
|
416
|
+
}
|
|
417
|
+
}
|
|
418
|
+
/**
|
|
419
|
+
* Throw {@link BudgetExceededError} if spending `additionalUsd` more would
|
|
420
|
+
* cross the soft cap. Only enforced when `dailyBudgetUsd` was configured.
|
|
421
|
+
*/
|
|
422
|
+
assertWithinBudget(additionalUsd) {
|
|
423
|
+
if (this.dailyBudgetUsd === void 0) return;
|
|
424
|
+
const projected = this.spentUsd + additionalUsd;
|
|
425
|
+
if (projected > this.dailyBudgetUsd) {
|
|
426
|
+
throw new BudgetExceededError(
|
|
427
|
+
`This request would spend ~$${round6(projected).toFixed(
|
|
428
|
+
6
|
|
429
|
+
)}, over the configured daily budget of $${this.dailyBudgetUsd.toFixed(2)}.`,
|
|
430
|
+
{ spentUsd: this.spentUsd, limitUsd: this.dailyBudgetUsd, wouldSpendUsd: round6(additionalUsd) }
|
|
431
|
+
);
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
/** Immutable point-in-time view of usage. */
|
|
435
|
+
snapshot() {
|
|
436
|
+
const reportedSpent = this.reportedLimitUsd !== null && this.reportedRemainingUsd !== null ? round6(this.reportedLimitUsd - this.reportedRemainingUsd) : null;
|
|
437
|
+
return {
|
|
438
|
+
totalCalls: Object.values(this.byType).reduce((a, b) => a + b.calls, 0),
|
|
439
|
+
estimatedSpentUsd: round6(this.estimatedSpentUsd),
|
|
440
|
+
reportedSpentUsd: reportedSpent,
|
|
441
|
+
reportedRemainingUsd: this.reportedRemainingUsd,
|
|
442
|
+
prepaidRemainingUsd: this.prepaidRemainingUsd,
|
|
443
|
+
creditsRemaining: this.creditsRemaining,
|
|
444
|
+
resetSeconds: this.resetSeconds,
|
|
445
|
+
lastRequestCostUsd: this.lastRequestCostUsd,
|
|
446
|
+
dailyLimitUsd: this.dailyLimitUsd,
|
|
447
|
+
spentUsd: this.spentUsd,
|
|
448
|
+
remainingUsd: this.remainingUsd,
|
|
449
|
+
byType: structuredCloneSafe(this.byType),
|
|
450
|
+
hasApiKey: this.hasApiKey
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
/** The last set of raw `x-ratelimit-*` headers OpenAlex sent (for debugging). */
|
|
454
|
+
get lastUsageHeaders() {
|
|
455
|
+
return { ...this.lastRawUsageHeaders };
|
|
456
|
+
}
|
|
457
|
+
/** Human-readable multi-line budget report. */
|
|
458
|
+
report() {
|
|
459
|
+
const s = this.snapshot();
|
|
460
|
+
const lines = [
|
|
461
|
+
"OpenAlex usage report",
|
|
462
|
+
` API key: ${s.hasApiKey ? "yes" : "no (anonymous)"}`,
|
|
463
|
+
` Total API calls: ${s.totalCalls}`,
|
|
464
|
+
` Spend today: $${s.spentUsd.toFixed(6)}${s.reportedSpentUsd !== null ? " (reported by OpenAlex)" : " (estimated)"}`,
|
|
465
|
+
` Daily limit: $${s.dailyLimitUsd.toFixed(4)}`,
|
|
466
|
+
` Remaining: $${s.remainingUsd.toFixed(6)}`
|
|
467
|
+
];
|
|
468
|
+
if (s.prepaidRemainingUsd !== null)
|
|
469
|
+
lines.push(` Prepaid remaining: $${s.prepaidRemainingUsd.toFixed(4)}`);
|
|
470
|
+
if (s.creditsRemaining !== null)
|
|
471
|
+
lines.push(` Credits remaining: ${s.creditsRemaining}`);
|
|
472
|
+
if (s.resetSeconds !== null)
|
|
473
|
+
lines.push(` Budget resets in: ${formatDuration(s.resetSeconds)}`);
|
|
474
|
+
lines.push(" By type:");
|
|
475
|
+
for (const type of Object.keys(s.byType)) {
|
|
476
|
+
const b = s.byType[type];
|
|
477
|
+
if (b.calls === 0) continue;
|
|
478
|
+
lines.push(
|
|
479
|
+
` ${type.padEnd(12)} ${String(b.calls).padStart(6)} calls $${b.estimatedCostUsd.toFixed(6)}`
|
|
480
|
+
);
|
|
481
|
+
}
|
|
482
|
+
return lines.join("\n");
|
|
483
|
+
}
|
|
484
|
+
reset() {
|
|
485
|
+
this.byType = emptyByType();
|
|
486
|
+
this.estimatedSpentUsd = 0;
|
|
487
|
+
this.reportedRemainingUsd = null;
|
|
488
|
+
this.reportedLimitUsd = null;
|
|
489
|
+
this.prepaidRemainingUsd = null;
|
|
490
|
+
this.creditsRemaining = null;
|
|
491
|
+
this.resetSeconds = null;
|
|
492
|
+
this.lastRequestCostUsd = null;
|
|
493
|
+
this.lastRawUsageHeaders = {};
|
|
494
|
+
}
|
|
495
|
+
};
|
|
496
|
+
function formatDuration(seconds) {
|
|
497
|
+
const s = Math.max(0, Math.floor(seconds));
|
|
498
|
+
const h = Math.floor(s / 3600);
|
|
499
|
+
const m = Math.floor(s % 3600 / 60);
|
|
500
|
+
if (h > 0) return `${h}h ${m}m`;
|
|
501
|
+
if (m > 0) return `${m}m ${s % 60}s`;
|
|
502
|
+
return `${s}s`;
|
|
503
|
+
}
|
|
504
|
+
function round6(n) {
|
|
505
|
+
return Math.round(n * 1e6) / 1e6;
|
|
506
|
+
}
|
|
507
|
+
function structuredCloneSafe(v) {
|
|
508
|
+
return JSON.parse(JSON.stringify(v));
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
// src/throttle.ts
|
|
512
|
+
var REAL_CLOCK = {
|
|
513
|
+
now: () => Date.now(),
|
|
514
|
+
sleep: (ms) => new Promise((r) => setTimeout(r, ms))
|
|
515
|
+
};
|
|
516
|
+
var Throttle = class {
|
|
517
|
+
constructor(opts, clock = REAL_CLOCK) {
|
|
518
|
+
this.opts = opts;
|
|
519
|
+
this.clock = clock;
|
|
520
|
+
}
|
|
521
|
+
opts;
|
|
522
|
+
clock;
|
|
523
|
+
nextAllowedAt = 0;
|
|
524
|
+
/**
|
|
525
|
+
* Wait until the next request is allowed to start. Enforces the target rate
|
|
526
|
+
* and multiplies spacing when the remaining budget fraction is low.
|
|
527
|
+
*/
|
|
528
|
+
async gate(state) {
|
|
529
|
+
if (this.opts.minRemainingUsd !== void 0 && state.remainingUsd < this.opts.minRemainingUsd) {
|
|
530
|
+
throw new BudgetExceededError(
|
|
531
|
+
`Auto-throttle floor hit: remaining budget $${state.remainingUsd.toFixed(6)} is below the configured floor $${this.opts.minRemainingUsd.toFixed(6)}.`,
|
|
532
|
+
{
|
|
533
|
+
spentUsd: Math.max(0, state.dailyLimitUsd - state.remainingUsd),
|
|
534
|
+
limitUsd: state.dailyLimitUsd,
|
|
535
|
+
wouldSpendUsd: 0
|
|
536
|
+
}
|
|
537
|
+
);
|
|
538
|
+
}
|
|
539
|
+
let spacing = this.opts.maxRequestsPerSecond > 0 ? 1e3 / this.opts.maxRequestsPerSecond : 0;
|
|
540
|
+
const frac = state.dailyLimitUsd > 0 ? state.remainingUsd / state.dailyLimitUsd : 1;
|
|
541
|
+
if (frac < 0.1) spacing *= 4;
|
|
542
|
+
else if (frac < 0.25) spacing *= 2;
|
|
543
|
+
const now = this.clock.now();
|
|
544
|
+
const wait = Math.max(0, this.nextAllowedAt - now);
|
|
545
|
+
this.nextAllowedAt = Math.max(now, this.nextAllowedAt) + spacing;
|
|
546
|
+
if (wait > 0) await this.clock.sleep(wait);
|
|
547
|
+
}
|
|
548
|
+
};
|
|
549
|
+
|
|
550
|
+
// src/http.ts
|
|
551
|
+
var Semaphore = class {
|
|
552
|
+
constructor(max) {
|
|
553
|
+
this.max = max;
|
|
554
|
+
}
|
|
555
|
+
max;
|
|
556
|
+
active = 0;
|
|
557
|
+
waiters = [];
|
|
558
|
+
async run(fn) {
|
|
559
|
+
await this.acquire();
|
|
560
|
+
try {
|
|
561
|
+
return await fn();
|
|
562
|
+
} finally {
|
|
563
|
+
this.release();
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
acquire() {
|
|
567
|
+
if (this.active < this.max) {
|
|
568
|
+
this.active += 1;
|
|
569
|
+
return Promise.resolve();
|
|
570
|
+
}
|
|
571
|
+
return new Promise((resolve) => this.waiters.push(resolve));
|
|
572
|
+
}
|
|
573
|
+
release() {
|
|
574
|
+
const next = this.waiters.shift();
|
|
575
|
+
if (next) next();
|
|
576
|
+
else this.active -= 1;
|
|
577
|
+
}
|
|
578
|
+
};
|
|
579
|
+
var HttpClient = class {
|
|
580
|
+
constructor(config, usage) {
|
|
581
|
+
this.config = config;
|
|
582
|
+
this.usage = usage;
|
|
583
|
+
this.semaphore = new Semaphore(config.concurrency);
|
|
584
|
+
if (config.throttle) this.throttle = new Throttle(config.throttle);
|
|
585
|
+
}
|
|
586
|
+
config;
|
|
587
|
+
usage;
|
|
588
|
+
semaphore;
|
|
589
|
+
firedWarnings = /* @__PURE__ */ new Set();
|
|
590
|
+
throttle;
|
|
591
|
+
async gateThrottle() {
|
|
592
|
+
if (!this.throttle) return;
|
|
593
|
+
const s = this.usage.snapshot();
|
|
594
|
+
await this.throttle.gate({ remainingUsd: s.remainingUsd, dailyLimitUsd: s.dailyLimitUsd });
|
|
595
|
+
}
|
|
596
|
+
/** Build the full request URL plus a redacted variant (api_key hidden). */
|
|
597
|
+
buildUrl(path, params = {}) {
|
|
598
|
+
const qs = new URLSearchParams();
|
|
599
|
+
for (const [k, v] of Object.entries(params)) {
|
|
600
|
+
if (v !== void 0 && v !== "") qs.set(k, v);
|
|
601
|
+
}
|
|
602
|
+
if (this.config.mailto) qs.set("mailto", this.config.mailto);
|
|
603
|
+
const redactedQs = new URLSearchParams(qs);
|
|
604
|
+
if (this.config.apiKey) {
|
|
605
|
+
qs.set("api_key", this.config.apiKey);
|
|
606
|
+
redactedQs.set("api_key", "REDACTED");
|
|
607
|
+
}
|
|
608
|
+
const base = `${this.config.baseUrl}/${encodeURI(path)}`;
|
|
609
|
+
const q = qs.toString();
|
|
610
|
+
const rq = redactedQs.toString();
|
|
611
|
+
return {
|
|
612
|
+
url: q ? `${base}?${q}` : base,
|
|
613
|
+
redacted: rq ? `${base}?${rq}` : base
|
|
614
|
+
};
|
|
615
|
+
}
|
|
616
|
+
/** Cache key excludes api_key so a shared cache isn't keyed to a secret. */
|
|
617
|
+
cacheKey(path, params) {
|
|
618
|
+
const { redacted } = this.buildUrl(path, params);
|
|
619
|
+
return redacted;
|
|
620
|
+
}
|
|
621
|
+
async request(opts) {
|
|
622
|
+
const params = opts.params ?? {};
|
|
623
|
+
const { url, redacted } = this.buildUrl(opts.path, params);
|
|
624
|
+
const useCache = this.config.cache && !opts.noCache;
|
|
625
|
+
const key = useCache ? this.cacheKey(opts.path, params) : void 0;
|
|
626
|
+
if (useCache && key) {
|
|
627
|
+
const cached = await this.config.cache.get(key);
|
|
628
|
+
if (cached !== void 0) {
|
|
629
|
+
this.config.logger.debug(`cache hit ${redacted}`);
|
|
630
|
+
return {
|
|
631
|
+
data: cached,
|
|
632
|
+
status: 200,
|
|
633
|
+
headers: new Headers(),
|
|
634
|
+
url: redacted,
|
|
635
|
+
cached: true
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
}
|
|
639
|
+
this.usage.assertWithinBudget(PRICING_USD[opts.requestType]);
|
|
640
|
+
await this.gateThrottle();
|
|
641
|
+
const response = await this.semaphore.run(() => this.fetchWithRetry(url, redacted, opts));
|
|
642
|
+
const results = countResults(response.data);
|
|
643
|
+
this.usage.record(opts.requestType, { results, headers: response.headers });
|
|
644
|
+
this.maybeWarnBudget();
|
|
645
|
+
if (useCache && key) {
|
|
646
|
+
await this.config.cache.set(key, response.data, this.config.cacheTtlMs);
|
|
647
|
+
}
|
|
648
|
+
return { ...response, url: redacted };
|
|
649
|
+
}
|
|
650
|
+
async fetchWithRetry(url, redacted, opts) {
|
|
651
|
+
const res = await this.sendWithRetry(url, redacted, { signal: opts.signal });
|
|
652
|
+
const data = await res.json();
|
|
653
|
+
return { data, status: res.status, headers: res.headers, url: redacted, cached: false };
|
|
654
|
+
}
|
|
655
|
+
/** Shared GET-with-retry core: returns the ok `Response` or throws typed. */
|
|
656
|
+
async sendWithRetry(url, redacted, opts) {
|
|
657
|
+
const { retry } = this.config;
|
|
658
|
+
let attempt = 0;
|
|
659
|
+
while (true) {
|
|
660
|
+
const controller = new AbortController();
|
|
661
|
+
const timer = setTimeout(() => controller.abort(), this.config.timeoutMs);
|
|
662
|
+
const signal = opts.signal ? anySignal([opts.signal, controller.signal]) : controller.signal;
|
|
663
|
+
try {
|
|
664
|
+
this.config.logger.debug(`GET ${redacted} (attempt ${attempt + 1})`);
|
|
665
|
+
const res = await this.config.fetch(url, {
|
|
666
|
+
method: "GET",
|
|
667
|
+
headers: { Accept: "application/json, */*", "User-Agent": this.config.userAgent },
|
|
668
|
+
signal
|
|
669
|
+
});
|
|
670
|
+
if (res.ok) return res;
|
|
671
|
+
const body = await safeReadBody(res);
|
|
672
|
+
const retryAfterMs = parseRetryAfter(res.headers.get("retry-after"));
|
|
673
|
+
const err = errorForStatus(res.status, describe(res.status, body, redacted), {
|
|
674
|
+
url: redacted,
|
|
675
|
+
body,
|
|
676
|
+
...res.status === 429 ? { retryAfterMs } : {}
|
|
677
|
+
});
|
|
678
|
+
if (this.shouldRetry(res.status, attempt)) {
|
|
679
|
+
await sleep(this.backoff(attempt, retryAfterMs));
|
|
680
|
+
attempt += 1;
|
|
681
|
+
continue;
|
|
682
|
+
}
|
|
683
|
+
throw err;
|
|
684
|
+
} catch (e) {
|
|
685
|
+
if (e instanceof OpenAlexError) throw e;
|
|
686
|
+
const isAbort = e?.name === "AbortError";
|
|
687
|
+
if (attempt < retry.maxRetries) {
|
|
688
|
+
this.config.logger.warn(`${isAbort ? "timeout" : "network error"}, retrying: ${redacted}`);
|
|
689
|
+
await sleep(this.backoff(attempt));
|
|
690
|
+
attempt += 1;
|
|
691
|
+
continue;
|
|
692
|
+
}
|
|
693
|
+
if (isAbort) {
|
|
694
|
+
throw new TimeoutError(`Request timed out after ${this.config.timeoutMs}ms`, {
|
|
695
|
+
url: redacted,
|
|
696
|
+
cause: e
|
|
697
|
+
});
|
|
698
|
+
}
|
|
699
|
+
throw new NetworkError(`Network request failed: ${e?.message ?? e}`, {
|
|
700
|
+
url: redacted,
|
|
701
|
+
cause: e
|
|
702
|
+
});
|
|
703
|
+
} finally {
|
|
704
|
+
clearTimeout(timer);
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
/** Append api_key + mailto to any absolute URL (for the content host, etc.). */
|
|
709
|
+
authorizeUrl(rawUrl) {
|
|
710
|
+
const u = new URL(rawUrl);
|
|
711
|
+
const r = new URL(rawUrl);
|
|
712
|
+
if (this.config.mailto) {
|
|
713
|
+
u.searchParams.set("mailto", this.config.mailto);
|
|
714
|
+
r.searchParams.set("mailto", this.config.mailto);
|
|
715
|
+
}
|
|
716
|
+
if (this.config.apiKey) {
|
|
717
|
+
u.searchParams.set("api_key", this.config.apiKey);
|
|
718
|
+
r.searchParams.set("api_key", "REDACTED");
|
|
719
|
+
}
|
|
720
|
+
return { url: u.toString(), redacted: r.toString() };
|
|
721
|
+
}
|
|
722
|
+
/**
|
|
723
|
+
* Download a binary resource (e.g. a full-text PDF/XML from the content host).
|
|
724
|
+
* Budget-asserted and usage-recorded like any other request.
|
|
725
|
+
*/
|
|
726
|
+
async download(opts) {
|
|
727
|
+
this.usage.assertWithinBudget(PRICING_USD[opts.requestType]);
|
|
728
|
+
await this.gateThrottle();
|
|
729
|
+
const redacted = opts.redacted ?? opts.url;
|
|
730
|
+
const res = await this.semaphore.run(
|
|
731
|
+
() => this.sendWithRetry(opts.url, redacted, { signal: opts.signal })
|
|
732
|
+
);
|
|
733
|
+
this.usage.record(opts.requestType, { headers: res.headers });
|
|
734
|
+
this.maybeWarnBudget();
|
|
735
|
+
const contentType = res.headers.get("content-type") ?? void 0;
|
|
736
|
+
if (opts.dest) {
|
|
737
|
+
if (!res.body) throw new OpenAlexError("No response body to write", { url: redacted });
|
|
738
|
+
const [{ Readable }, { pipeline }, { createWriteStream: createWriteStream2 }] = await Promise.all([
|
|
739
|
+
import("stream"),
|
|
740
|
+
import("stream/promises"),
|
|
741
|
+
import("fs")
|
|
742
|
+
]);
|
|
743
|
+
await pipeline(Readable.fromWeb(res.body), createWriteStream2(opts.dest));
|
|
744
|
+
return { path: opts.dest, status: res.status, headers: res.headers, contentType };
|
|
745
|
+
}
|
|
746
|
+
const bytes = new Uint8Array(await res.arrayBuffer());
|
|
747
|
+
return { bytes, status: res.status, headers: res.headers, contentType };
|
|
748
|
+
}
|
|
749
|
+
shouldRetry(status, attempt) {
|
|
750
|
+
return attempt < this.config.retry.maxRetries && this.config.retry.retryOnStatus.includes(status);
|
|
751
|
+
}
|
|
752
|
+
backoff(attempt, retryAfterMs) {
|
|
753
|
+
const { baseDelayMs, maxDelayMs } = this.config.retry;
|
|
754
|
+
if (retryAfterMs !== void 0) return Math.min(retryAfterMs, maxDelayMs);
|
|
755
|
+
const exp = Math.min(baseDelayMs * 2 ** attempt, maxDelayMs);
|
|
756
|
+
return Math.round(exp / 2 + Math.random() * (exp / 2));
|
|
757
|
+
}
|
|
758
|
+
maybeWarnBudget() {
|
|
759
|
+
const cb = this.config.onBudgetWarning;
|
|
760
|
+
if (!cb) return;
|
|
761
|
+
const { spentUsd, dailyLimitUsd } = this.usage.snapshot();
|
|
762
|
+
if (!dailyLimitUsd) return;
|
|
763
|
+
const ratio = spentUsd / dailyLimitUsd;
|
|
764
|
+
for (const threshold of [0.8, 1]) {
|
|
765
|
+
if (ratio >= threshold && !this.firedWarnings.has(threshold)) {
|
|
766
|
+
this.firedWarnings.add(threshold);
|
|
767
|
+
cb({ spentUsd, limitUsd: dailyLimitUsd, ratio });
|
|
768
|
+
}
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
};
|
|
772
|
+
function countResults(data) {
|
|
773
|
+
if (data && typeof data === "object" && Array.isArray(data.results)) {
|
|
774
|
+
return data.results.length;
|
|
775
|
+
}
|
|
776
|
+
return 0;
|
|
777
|
+
}
|
|
778
|
+
function describe(status, body, url) {
|
|
779
|
+
const msg = body && typeof body === "object" && "message" in body ? String(body.message) : body && typeof body === "object" && "error" in body ? String(body.error) : void 0;
|
|
780
|
+
return `OpenAlex request failed (${status})${msg ? `: ${msg}` : ""} [${url}]`;
|
|
781
|
+
}
|
|
782
|
+
async function safeReadBody(res) {
|
|
783
|
+
try {
|
|
784
|
+
const text = await res.text();
|
|
785
|
+
try {
|
|
786
|
+
return JSON.parse(text);
|
|
787
|
+
} catch {
|
|
788
|
+
return text;
|
|
789
|
+
}
|
|
790
|
+
} catch {
|
|
791
|
+
return void 0;
|
|
792
|
+
}
|
|
793
|
+
}
|
|
794
|
+
function parseRetryAfter(value) {
|
|
795
|
+
if (!value) return void 0;
|
|
796
|
+
const secs = Number(value);
|
|
797
|
+
if (Number.isFinite(secs)) return Math.max(0, secs * 1e3);
|
|
798
|
+
const date = Date.parse(value);
|
|
799
|
+
if (Number.isFinite(date)) return Math.max(0, date - Date.now());
|
|
800
|
+
return void 0;
|
|
801
|
+
}
|
|
802
|
+
function sleep(ms) {
|
|
803
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
804
|
+
}
|
|
805
|
+
function anySignal(signals) {
|
|
806
|
+
const controller = new AbortController();
|
|
807
|
+
for (const s of signals) {
|
|
808
|
+
if (s.aborted) {
|
|
809
|
+
controller.abort();
|
|
810
|
+
break;
|
|
811
|
+
}
|
|
812
|
+
s.addEventListener("abort", () => controller.abort(), { once: true });
|
|
813
|
+
}
|
|
814
|
+
return controller.signal;
|
|
815
|
+
}
|
|
816
|
+
|
|
817
|
+
// src/pagination.ts
|
|
818
|
+
function resolveStrategy(opts) {
|
|
819
|
+
if (opts.resume) return "cursor";
|
|
820
|
+
if (opts.strategy === "cursor" || opts.strategy === "basic") return opts.strategy;
|
|
821
|
+
return opts.parallel ? "basic" : "cursor";
|
|
822
|
+
}
|
|
823
|
+
async function* streamPages(runner, baseParams, opts = {}) {
|
|
824
|
+
const strategy = resolveStrategy(opts);
|
|
825
|
+
if (strategy === "cursor") {
|
|
826
|
+
yield* cursorPages(runner, baseParams, opts);
|
|
827
|
+
} else if (opts.parallel) {
|
|
828
|
+
yield* parallelBasicPages(runner, baseParams, opts);
|
|
829
|
+
} else {
|
|
830
|
+
yield* sequentialBasicPages(runner, baseParams, opts);
|
|
831
|
+
}
|
|
832
|
+
}
|
|
833
|
+
async function* streamItems(runner, baseParams, opts = {}) {
|
|
834
|
+
let emitted = 0;
|
|
835
|
+
const max = opts.maxResults ?? Infinity;
|
|
836
|
+
for await (const page of streamPages(runner, baseParams, opts)) {
|
|
837
|
+
for (const item of page.results) {
|
|
838
|
+
if (emitted >= max) return;
|
|
839
|
+
yield item;
|
|
840
|
+
emitted += 1;
|
|
841
|
+
}
|
|
842
|
+
if (emitted >= max) return;
|
|
843
|
+
}
|
|
844
|
+
}
|
|
845
|
+
async function* cursorPages(runner, baseParams, opts) {
|
|
846
|
+
const perPage = clampPerPage(opts.perPage ?? 200);
|
|
847
|
+
const fetchOpts = { requestType: opts.requestType, signal: opts.signal };
|
|
848
|
+
const maxResults = opts.maxResults ?? Infinity;
|
|
849
|
+
const maxPages = opts.maxPages ?? Infinity;
|
|
850
|
+
const saved = opts.resume?.load();
|
|
851
|
+
let cursor = saved?.cursor ?? "*";
|
|
852
|
+
let emitted = saved?.emitted ?? 0;
|
|
853
|
+
let pages = 0;
|
|
854
|
+
while (cursor && pages < maxPages && emitted < maxResults) {
|
|
855
|
+
const page = await runner.fetchList(
|
|
856
|
+
{ ...baseParams, "per-page": String(perPage), cursor },
|
|
857
|
+
fetchOpts
|
|
858
|
+
);
|
|
859
|
+
pages += 1;
|
|
860
|
+
emitted += page.results.length;
|
|
861
|
+
if (page.results.length === 0) {
|
|
862
|
+
cursor = null;
|
|
863
|
+
} else {
|
|
864
|
+
cursor = page.meta.next_cursor ?? null;
|
|
865
|
+
}
|
|
866
|
+
opts.resume?.save({ cursor, emitted });
|
|
867
|
+
opts.onProgress?.({ emitted, page: pages, total: page.meta.count ?? null });
|
|
868
|
+
yield page;
|
|
869
|
+
}
|
|
870
|
+
if (cursor === null) opts.resume?.clear();
|
|
871
|
+
}
|
|
872
|
+
async function* sequentialBasicPages(runner, baseParams, opts) {
|
|
873
|
+
const perPage = clampPerPage(opts.perPage ?? 200);
|
|
874
|
+
const fetchOpts = { requestType: opts.requestType, signal: opts.signal };
|
|
875
|
+
const lastAllowedPage = Math.floor(BASIC_PAGING_MAX_RESULTS / perPage);
|
|
876
|
+
const maxPages = Math.min(opts.maxPages ?? Infinity, lastAllowedPage);
|
|
877
|
+
const maxResults = opts.maxResults ?? Infinity;
|
|
878
|
+
let emitted = 0;
|
|
879
|
+
for (let page = 1; page <= maxPages; page++) {
|
|
880
|
+
const result = await runner.fetchList(
|
|
881
|
+
{ ...baseParams, "per-page": String(perPage), page: String(page) },
|
|
882
|
+
fetchOpts
|
|
883
|
+
);
|
|
884
|
+
emitted += result.results.length;
|
|
885
|
+
opts.onProgress?.({ emitted, page, total: result.meta.count ?? null });
|
|
886
|
+
yield result;
|
|
887
|
+
if (result.results.length < perPage || emitted >= maxResults) break;
|
|
888
|
+
}
|
|
889
|
+
}
|
|
890
|
+
async function* parallelBasicPages(runner, baseParams, opts) {
|
|
891
|
+
const perPage = clampPerPage(opts.perPage ?? 200);
|
|
892
|
+
const fetchOpts = { requestType: opts.requestType, signal: opts.signal };
|
|
893
|
+
const lastAllowedPage = Math.floor(BASIC_PAGING_MAX_RESULTS / perPage);
|
|
894
|
+
const first = await runner.fetchList(
|
|
895
|
+
{ ...baseParams, "per-page": String(perPage), page: "1" },
|
|
896
|
+
fetchOpts
|
|
897
|
+
);
|
|
898
|
+
let emitted = first.results.length;
|
|
899
|
+
opts.onProgress?.({ emitted, page: 1, total: first.meta.count ?? null });
|
|
900
|
+
yield first;
|
|
901
|
+
const count = first.meta.count ?? first.results.length;
|
|
902
|
+
if (count > BASIC_PAGING_MAX_RESULTS) {
|
|
903
|
+
runner.logger.warn(
|
|
904
|
+
`parallelBasicPages: ${count.toLocaleString()} results exceed the ${BASIC_PAGING_MAX_RESULTS.toLocaleString()} basic-paging cap; only the first ${BASIC_PAGING_MAX_RESULTS.toLocaleString()} are returned. Use strategy:'cursor' for the full set.`
|
|
905
|
+
);
|
|
906
|
+
}
|
|
907
|
+
const neededPages = Math.ceil(Math.min(count, BASIC_PAGING_MAX_RESULTS) / perPage);
|
|
908
|
+
const totalPages = Math.min(neededPages, lastAllowedPage, opts.maxPages ?? Infinity);
|
|
909
|
+
if (totalPages <= 1 || first.results.length < perPage) return;
|
|
910
|
+
const promises = [];
|
|
911
|
+
for (let page = 2; page <= totalPages; page++) {
|
|
912
|
+
promises.push(
|
|
913
|
+
runner.fetchList({ ...baseParams, "per-page": String(perPage), page: String(page) }, fetchOpts)
|
|
914
|
+
);
|
|
915
|
+
}
|
|
916
|
+
let pageNum = 1;
|
|
917
|
+
for (const p of promises) {
|
|
918
|
+
const page = await p;
|
|
919
|
+
pageNum += 1;
|
|
920
|
+
emitted += page.results.length;
|
|
921
|
+
opts.onProgress?.({ emitted, page: pageNum, total: count });
|
|
922
|
+
yield page;
|
|
923
|
+
if (page.results.length === 0) break;
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
|
|
927
|
+
// src/query.ts
|
|
928
|
+
var PAGINATION_DEFAULT_PER_PAGE = 200;
|
|
929
|
+
var QueryBuilder = class _QueryBuilder {
|
|
930
|
+
constructor(runner, state = { filters: {}, extra: {} }) {
|
|
931
|
+
this.runner = runner;
|
|
932
|
+
this.state = state;
|
|
933
|
+
}
|
|
934
|
+
runner;
|
|
935
|
+
state;
|
|
936
|
+
/** The entity type this query targets (e.g. 'works'). */
|
|
937
|
+
get entityType() {
|
|
938
|
+
return this.runner.entityType;
|
|
939
|
+
}
|
|
940
|
+
clone(patch) {
|
|
941
|
+
return new _QueryBuilder(this.runner, {
|
|
942
|
+
...this.state,
|
|
943
|
+
...patch,
|
|
944
|
+
filters: patch.filters ?? this.state.filters,
|
|
945
|
+
extra: patch.extra ?? this.state.extra
|
|
946
|
+
});
|
|
947
|
+
}
|
|
948
|
+
// --- chainable builders --------------------------------------------------
|
|
949
|
+
/** Merge filters. Array values are OR'd; use `not`/`gt`/`lt` helpers. */
|
|
950
|
+
filter(filters) {
|
|
951
|
+
return this.clone({ filters: { ...this.state.filters, ...filters } });
|
|
952
|
+
}
|
|
953
|
+
/** Full-text relevance search across the entity's default searchable fields. */
|
|
954
|
+
search(term) {
|
|
955
|
+
return this.clone({ search: term });
|
|
956
|
+
}
|
|
957
|
+
/** Search within a specific field, e.g. `.searchField('title', 'crispr')`. */
|
|
958
|
+
searchField(field, term) {
|
|
959
|
+
return this.clone({ filters: { ...this.state.filters, [`${field}.search`]: term } });
|
|
960
|
+
}
|
|
961
|
+
sort(spec) {
|
|
962
|
+
let s;
|
|
963
|
+
if (typeof spec === "string") s = spec;
|
|
964
|
+
else if (Array.isArray(spec)) s = spec.join(",");
|
|
965
|
+
else s = Object.entries(spec).map(([k, v]) => `${k}:${v}`).join(",");
|
|
966
|
+
return this.clone({ sort: s });
|
|
967
|
+
}
|
|
968
|
+
/** Trim the response to specific fields (cheaper, faster payloads). */
|
|
969
|
+
select(fields) {
|
|
970
|
+
const arr = Array.isArray(fields) ? fields : fields.split(",").map((f) => f.trim());
|
|
971
|
+
return this.clone({ select: arr });
|
|
972
|
+
}
|
|
973
|
+
/** Random sample of `n` results (optionally deterministic via `seed`). */
|
|
974
|
+
sample(n, seed) {
|
|
975
|
+
return this.clone({ sampleN: n, seed: seed ?? this.state.seed });
|
|
976
|
+
}
|
|
977
|
+
seed(seed) {
|
|
978
|
+
return this.clone({ seed });
|
|
979
|
+
}
|
|
980
|
+
perPage(n) {
|
|
981
|
+
return this.clone({ perPage: clampPerPage(n) });
|
|
982
|
+
}
|
|
983
|
+
/** Aggregate counts by a field (or two comma-separated fields). */
|
|
984
|
+
groupBy(field) {
|
|
985
|
+
return this.clone({ groupBy: Array.isArray(field) ? field.join(",") : field });
|
|
986
|
+
}
|
|
987
|
+
/** Escape hatch: set any raw query parameter (advanced/semantic search, …). */
|
|
988
|
+
param(key, value) {
|
|
989
|
+
return this.clone({ extra: { ...this.state.extra, [key]: String(value) } });
|
|
990
|
+
}
|
|
991
|
+
// --- introspection -------------------------------------------------------
|
|
992
|
+
/** The OpenAlex query params this builder will send (no paging keys). */
|
|
993
|
+
toParams() {
|
|
994
|
+
const p = { ...this.state.extra };
|
|
995
|
+
const filter = serializeFilters(this.state.filters);
|
|
996
|
+
if (filter) p.filter = filter;
|
|
997
|
+
if (this.state.search) p.search = this.state.search;
|
|
998
|
+
if (this.state.sort) p.sort = this.state.sort;
|
|
999
|
+
if (this.state.select?.length) p.select = this.state.select.join(",");
|
|
1000
|
+
if (this.state.groupBy) p.group_by = this.state.groupBy;
|
|
1001
|
+
if (this.state.sampleN !== void 0) {
|
|
1002
|
+
p.sample = String(this.state.sampleN);
|
|
1003
|
+
if (this.state.seed !== void 0) p.seed = String(this.state.seed);
|
|
1004
|
+
}
|
|
1005
|
+
return p;
|
|
1006
|
+
}
|
|
1007
|
+
get requestType() {
|
|
1008
|
+
if (this.state.search || this.state.extra.search) return "search";
|
|
1009
|
+
for (const k of Object.keys(this.state.filters)) {
|
|
1010
|
+
if (k.endsWith(".search") || k === "default.search") return "search";
|
|
1011
|
+
}
|
|
1012
|
+
return "list";
|
|
1013
|
+
}
|
|
1014
|
+
// --- terminal operations -------------------------------------------------
|
|
1015
|
+
/** Fetch a single page. */
|
|
1016
|
+
async get(opts = {}) {
|
|
1017
|
+
const params = this.toParams();
|
|
1018
|
+
if (opts.perPage !== void 0) params["per-page"] = String(clampPerPage(opts.perPage));
|
|
1019
|
+
else if (this.state.perPage !== void 0) params["per-page"] = String(this.state.perPage);
|
|
1020
|
+
if (opts.page !== void 0) params.page = String(opts.page);
|
|
1021
|
+
return this.runner.fetchList(params, { requestType: this.requestType });
|
|
1022
|
+
}
|
|
1023
|
+
/** Fetch just the first matching entity, or `null`. */
|
|
1024
|
+
async first() {
|
|
1025
|
+
const res = await this.get({ perPage: 1 });
|
|
1026
|
+
return res.results[0] ?? null;
|
|
1027
|
+
}
|
|
1028
|
+
/** Total number of matching results (one cheap probe call). */
|
|
1029
|
+
async count() {
|
|
1030
|
+
const res = await this.runner.fetchList(
|
|
1031
|
+
{ ...this.toParams(), "per-page": "1" },
|
|
1032
|
+
{ requestType: this.requestType }
|
|
1033
|
+
);
|
|
1034
|
+
return res.meta.count;
|
|
1035
|
+
}
|
|
1036
|
+
/** Async-iterate every matching page. */
|
|
1037
|
+
paginate(opts = {}) {
|
|
1038
|
+
return streamPages(this.runner, this.toParams(), this.streamOpts(opts));
|
|
1039
|
+
}
|
|
1040
|
+
/** Async-iterate every matching entity across all pages. */
|
|
1041
|
+
all(opts = {}) {
|
|
1042
|
+
return streamItems(this.runner, this.toParams(), this.streamOpts(opts));
|
|
1043
|
+
}
|
|
1044
|
+
/** Collect all matching entities into an array (respecting `maxResults`). */
|
|
1045
|
+
async toArray(opts = {}) {
|
|
1046
|
+
const out = [];
|
|
1047
|
+
for await (const item of this.all(opts)) out.push(item);
|
|
1048
|
+
return out;
|
|
1049
|
+
}
|
|
1050
|
+
/**
|
|
1051
|
+
* Stream all results using the cost-optimal strategy: one cheap count, then
|
|
1052
|
+
* parallel basic paging when ≤10k results (fastest) or cursor streaming above.
|
|
1053
|
+
*/
|
|
1054
|
+
async *streamOptimized(opts = {}) {
|
|
1055
|
+
const est = await this.estimateCost({ perPage: opts.perPage ?? 200, strategy: "cursor" });
|
|
1056
|
+
const parallel = est.matchingResults <= BASIC_PAGING_MAX_RESULTS;
|
|
1057
|
+
yield* this.all({ ...opts, perPage: opts.perPage ?? 200, parallel });
|
|
1058
|
+
}
|
|
1059
|
+
/** Run a `group_by` aggregation and return the buckets. */
|
|
1060
|
+
async groups(field) {
|
|
1061
|
+
const b = field ? this.groupBy(field) : this;
|
|
1062
|
+
const res = await b.get({ perPage: 200 });
|
|
1063
|
+
return res.group_by ?? [];
|
|
1064
|
+
}
|
|
1065
|
+
/** Estimate the dollar cost of extracting *all* matching results. */
|
|
1066
|
+
async estimateCost(opts = {}) {
|
|
1067
|
+
const count = await this.count();
|
|
1068
|
+
return estimateExtractionCost({
|
|
1069
|
+
count,
|
|
1070
|
+
requestType: this.requestType,
|
|
1071
|
+
perPage: opts.perPage ?? this.state.perPage ?? PAGINATION_DEFAULT_PER_PAGE,
|
|
1072
|
+
strategy: opts.strategy ?? "cursor"
|
|
1073
|
+
});
|
|
1074
|
+
}
|
|
1075
|
+
/** Stream all matching results to a file (JSONL / CSV / JSON). */
|
|
1076
|
+
async export(path, opts = {}) {
|
|
1077
|
+
const { exportToFile: exportToFile2 } = await Promise.resolve().then(() => (init_export(), export_exports));
|
|
1078
|
+
return exportToFile2(this.all(opts), path, opts);
|
|
1079
|
+
}
|
|
1080
|
+
streamOpts(opts) {
|
|
1081
|
+
return {
|
|
1082
|
+
...opts,
|
|
1083
|
+
perPage: opts.perPage ?? this.state.perPage ?? PAGINATION_DEFAULT_PER_PAGE,
|
|
1084
|
+
requestType: opts.requestType ?? this.requestType
|
|
1085
|
+
};
|
|
1086
|
+
}
|
|
1087
|
+
};
|
|
1088
|
+
function serializeScalar(v) {
|
|
1089
|
+
if (typeof v === "boolean") return v ? "true" : "false";
|
|
1090
|
+
return String(v);
|
|
1091
|
+
}
|
|
1092
|
+
function serializeFilterValue(v) {
|
|
1093
|
+
return Array.isArray(v) ? v.map(serializeScalar).join("|") : serializeScalar(v);
|
|
1094
|
+
}
|
|
1095
|
+
function serializeFilters(filters) {
|
|
1096
|
+
const entries = Object.entries(filters);
|
|
1097
|
+
if (entries.length === 0) return void 0;
|
|
1098
|
+
return entries.map(([k, v]) => `${k}:${serializeFilterValue(v)}`).join(",");
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
// src/ids.ts
|
|
1102
|
+
var OPENALEX_ID = /^[WASITCPFKGDL]\d{4,}$/i;
|
|
1103
|
+
var ORCID = /^(\d{4}-){3}\d{3}[\dX]$/i;
|
|
1104
|
+
var ISSN = /^\d{4}-\d{3}[\dX]$/i;
|
|
1105
|
+
var WIKIDATA = /^Q\d+$/i;
|
|
1106
|
+
var DOI = /^10\.\d{4,9}\/\S+$/;
|
|
1107
|
+
var KNOWN_PREFIX = /^(openalex|doi|mag|orcid|ror|issn|issn_l|wikidata|pmid|pmcid):/i;
|
|
1108
|
+
function normalizeId(input) {
|
|
1109
|
+
const id = input.trim();
|
|
1110
|
+
if (!id) return id;
|
|
1111
|
+
if (KNOWN_PREFIX.test(id)) return id;
|
|
1112
|
+
const url = tryUrl(id);
|
|
1113
|
+
if (url) {
|
|
1114
|
+
const host = url.hostname.replace(/^www\./, "");
|
|
1115
|
+
const last = url.pathname.split("/").filter(Boolean).pop() ?? "";
|
|
1116
|
+
if (host.endsWith("openalex.org")) return last;
|
|
1117
|
+
if (host.endsWith("doi.org")) return `doi:${decodeURIComponent(url.pathname.slice(1))}`;
|
|
1118
|
+
if (host.endsWith("orcid.org")) return `orcid:${last}`;
|
|
1119
|
+
if (host.endsWith("ror.org")) return `ror:${last}`;
|
|
1120
|
+
if (host.endsWith("wikidata.org")) return `wikidata:${last}`;
|
|
1121
|
+
if (host.endsWith("ncbi.nlm.nih.gov") && url.pathname.includes("/pmc/"))
|
|
1122
|
+
return `pmcid:${last}`;
|
|
1123
|
+
if (host.endsWith("ncbi.nlm.nih.gov") || host.includes("pubmed"))
|
|
1124
|
+
return `pmid:${last}`;
|
|
1125
|
+
return last || id;
|
|
1126
|
+
}
|
|
1127
|
+
if (OPENALEX_ID.test(id)) return id.toUpperCase().charAt(0) + id.slice(1);
|
|
1128
|
+
if (DOI.test(id)) return `doi:${id}`;
|
|
1129
|
+
if (ORCID.test(id)) return `orcid:${id}`;
|
|
1130
|
+
if (ISSN.test(id)) return `issn:${id}`;
|
|
1131
|
+
if (WIKIDATA.test(id)) return `wikidata:${id.toUpperCase()}`;
|
|
1132
|
+
return id;
|
|
1133
|
+
}
|
|
1134
|
+
function tryUrl(s) {
|
|
1135
|
+
if (!/^https?:\/\//i.test(s)) return null;
|
|
1136
|
+
try {
|
|
1137
|
+
return new URL(s);
|
|
1138
|
+
} catch {
|
|
1139
|
+
return null;
|
|
1140
|
+
}
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
// src/entities.ts
|
|
1144
|
+
var EntityEndpoint = class {
|
|
1145
|
+
constructor(http, entityType) {
|
|
1146
|
+
this.http = http;
|
|
1147
|
+
this.entityType = entityType;
|
|
1148
|
+
}
|
|
1149
|
+
http;
|
|
1150
|
+
entityType;
|
|
1151
|
+
get logger() {
|
|
1152
|
+
return this.http.config.logger;
|
|
1153
|
+
}
|
|
1154
|
+
get defaultConcurrency() {
|
|
1155
|
+
return this.http.config.concurrency;
|
|
1156
|
+
}
|
|
1157
|
+
/** Low-level list fetch used by the query builder & paginators. */
|
|
1158
|
+
async fetchList(params, opts = {}) {
|
|
1159
|
+
const res = await this.http.request({
|
|
1160
|
+
path: this.entityType,
|
|
1161
|
+
params,
|
|
1162
|
+
requestType: opts.requestType ?? "list",
|
|
1163
|
+
noCache: opts.noCache,
|
|
1164
|
+
signal: opts.signal
|
|
1165
|
+
});
|
|
1166
|
+
return res.data;
|
|
1167
|
+
}
|
|
1168
|
+
/** Start a fresh query. */
|
|
1169
|
+
query() {
|
|
1170
|
+
return new QueryBuilder(this);
|
|
1171
|
+
}
|
|
1172
|
+
// --- fluent proxies (each returns a QueryBuilder) ------------------------
|
|
1173
|
+
filter(filters) {
|
|
1174
|
+
return this.query().filter(filters);
|
|
1175
|
+
}
|
|
1176
|
+
search(term) {
|
|
1177
|
+
return this.query().search(term);
|
|
1178
|
+
}
|
|
1179
|
+
searchField(field, term) {
|
|
1180
|
+
return this.query().searchField(field, term);
|
|
1181
|
+
}
|
|
1182
|
+
sort(spec) {
|
|
1183
|
+
return this.query().sort(spec);
|
|
1184
|
+
}
|
|
1185
|
+
select(fields) {
|
|
1186
|
+
return this.query().select(fields);
|
|
1187
|
+
}
|
|
1188
|
+
sample(n, seed) {
|
|
1189
|
+
return this.query().sample(n, seed);
|
|
1190
|
+
}
|
|
1191
|
+
groupBy(field) {
|
|
1192
|
+
return this.query().groupBy(field);
|
|
1193
|
+
}
|
|
1194
|
+
perPage(n) {
|
|
1195
|
+
return this.query().perPage(n);
|
|
1196
|
+
}
|
|
1197
|
+
param(key, value) {
|
|
1198
|
+
return this.query().param(key, value);
|
|
1199
|
+
}
|
|
1200
|
+
// --- terminal proxies over the whole collection --------------------------
|
|
1201
|
+
count() {
|
|
1202
|
+
return this.query().count();
|
|
1203
|
+
}
|
|
1204
|
+
all(opts) {
|
|
1205
|
+
return this.query().all(opts);
|
|
1206
|
+
}
|
|
1207
|
+
paginate(opts) {
|
|
1208
|
+
return this.query().paginate(opts);
|
|
1209
|
+
}
|
|
1210
|
+
toArray(opts) {
|
|
1211
|
+
return this.query().toArray(opts);
|
|
1212
|
+
}
|
|
1213
|
+
groups(field) {
|
|
1214
|
+
return this.query().groups(field);
|
|
1215
|
+
}
|
|
1216
|
+
estimateCost(opts) {
|
|
1217
|
+
return this.query().estimateCost(opts);
|
|
1218
|
+
}
|
|
1219
|
+
export(path, opts) {
|
|
1220
|
+
return this.query().export(path, opts);
|
|
1221
|
+
}
|
|
1222
|
+
// --- single & batch retrieval --------------------------------------------
|
|
1223
|
+
/**
|
|
1224
|
+
* Fetch one entity by any supported ID: OpenAlex ID, DOI, ORCID, ROR, ISSN,
|
|
1225
|
+
* Wikidata QID, PMID, MAG, or a full URL (all normalized automatically).
|
|
1226
|
+
*/
|
|
1227
|
+
async get(id, opts = {}) {
|
|
1228
|
+
const params = {};
|
|
1229
|
+
if (opts.select) params.select = toCsv(opts.select);
|
|
1230
|
+
const res = await this.http.request({
|
|
1231
|
+
path: `${this.entityType}/${normalizeId(id)}`,
|
|
1232
|
+
params,
|
|
1233
|
+
requestType: "get",
|
|
1234
|
+
noCache: opts.noCache
|
|
1235
|
+
});
|
|
1236
|
+
return res.data;
|
|
1237
|
+
}
|
|
1238
|
+
/**
|
|
1239
|
+
* Fetch many entities by OpenAlex ID in as few calls as possible, batching
|
|
1240
|
+
* into `ids.openalex` OR-filters (default 50 per request) run in parallel.
|
|
1241
|
+
* Order is not guaranteed. For mixed external IDs, call {@link get} per ID.
|
|
1242
|
+
*/
|
|
1243
|
+
async getMany(ids, opts = {}) {
|
|
1244
|
+
if (ids.length === 0) return [];
|
|
1245
|
+
const chunkSize = Math.min(Math.max(1, opts.chunkSize ?? 50), 100);
|
|
1246
|
+
const select = opts.select ? toCsv(opts.select) : void 0;
|
|
1247
|
+
const normalized = ids.map((id) => normalizeId(id));
|
|
1248
|
+
const chunks = chunk(normalized, chunkSize);
|
|
1249
|
+
const pages = await Promise.all(
|
|
1250
|
+
chunks.map((c) => {
|
|
1251
|
+
let q = this.query().filter({ "ids.openalex": c });
|
|
1252
|
+
if (select) q = q.select(select);
|
|
1253
|
+
return q.get({ perPage: chunkSize });
|
|
1254
|
+
})
|
|
1255
|
+
);
|
|
1256
|
+
return pages.flatMap((p) => p.results);
|
|
1257
|
+
}
|
|
1258
|
+
/** A single random entity (optionally trimmed with `select`). */
|
|
1259
|
+
async random(opts = {}) {
|
|
1260
|
+
const params = {};
|
|
1261
|
+
if (opts.select) params.select = toCsv(opts.select);
|
|
1262
|
+
const res = await this.http.request({
|
|
1263
|
+
path: `${this.entityType}/random`,
|
|
1264
|
+
params,
|
|
1265
|
+
requestType: "get",
|
|
1266
|
+
noCache: true
|
|
1267
|
+
});
|
|
1268
|
+
return res.data;
|
|
1269
|
+
}
|
|
1270
|
+
/** Typeahead suggestions for this entity type. */
|
|
1271
|
+
async autocomplete(q) {
|
|
1272
|
+
const res = await this.http.request({
|
|
1273
|
+
path: `autocomplete/${this.entityType}`,
|
|
1274
|
+
params: { q },
|
|
1275
|
+
requestType: "autocomplete"
|
|
1276
|
+
});
|
|
1277
|
+
return res.data.results;
|
|
1278
|
+
}
|
|
1279
|
+
};
|
|
1280
|
+
function toCsv(v) {
|
|
1281
|
+
return Array.isArray(v) ? v.join(",") : v;
|
|
1282
|
+
}
|
|
1283
|
+
function chunk(arr, size) {
|
|
1284
|
+
const out = [];
|
|
1285
|
+
for (let i = 0; i < arr.length; i += size) out.push(arr.slice(i, i + size));
|
|
1286
|
+
return out;
|
|
1287
|
+
}
|
|
1288
|
+
|
|
1289
|
+
// src/works.ts
|
|
1290
|
+
var CONTENT_BASE_URL = "https://content.openalex.org";
|
|
1291
|
+
var WorksEndpoint = class extends EntityEndpoint {
|
|
1292
|
+
constructor(http) {
|
|
1293
|
+
super(http, "works");
|
|
1294
|
+
}
|
|
1295
|
+
/** Works that **cite** the given work (incoming citations). */
|
|
1296
|
+
citedBy(workId) {
|
|
1297
|
+
return this.filter({ cites: normalizeId(workId) });
|
|
1298
|
+
}
|
|
1299
|
+
/** Works the given work **cites** — its references (outgoing citations). */
|
|
1300
|
+
references(workId) {
|
|
1301
|
+
return this.filter({ cited_by: normalizeId(workId) });
|
|
1302
|
+
}
|
|
1303
|
+
/** Works OpenAlex considers related to the given work. */
|
|
1304
|
+
related(workId) {
|
|
1305
|
+
return this.filter({ related_to: normalizeId(workId) });
|
|
1306
|
+
}
|
|
1307
|
+
/** Only works that have a downloadable cached PDF. */
|
|
1308
|
+
withPdf() {
|
|
1309
|
+
return this.filter({ "has_content.pdf": true });
|
|
1310
|
+
}
|
|
1311
|
+
/** Only works that have a downloadable TEI-XML (Grobid) parse. */
|
|
1312
|
+
withXml() {
|
|
1313
|
+
return this.filter({ "has_content.grobid_xml": true });
|
|
1314
|
+
}
|
|
1315
|
+
/** The content-host URL for a work's full text (api_key redacted). */
|
|
1316
|
+
fulltextUrl(workId, format = "pdf") {
|
|
1317
|
+
return this.http.authorizeUrl(this.rawFulltextUrl(workId, format)).redacted;
|
|
1318
|
+
}
|
|
1319
|
+
/**
|
|
1320
|
+
* Download a work's cached full text (PDF or TEI-XML). Costs $0.01/call and
|
|
1321
|
+
* is budget-tracked. Pass `dest` to stream to a file, otherwise get bytes.
|
|
1322
|
+
*/
|
|
1323
|
+
async downloadFulltext(workId, opts = {}) {
|
|
1324
|
+
const { url, redacted } = this.http.authorizeUrl(
|
|
1325
|
+
this.rawFulltextUrl(workId, opts.format ?? "pdf")
|
|
1326
|
+
);
|
|
1327
|
+
const res = await this.http.download({
|
|
1328
|
+
url,
|
|
1329
|
+
redacted,
|
|
1330
|
+
requestType: "download",
|
|
1331
|
+
dest: opts.dest,
|
|
1332
|
+
signal: opts.signal
|
|
1333
|
+
});
|
|
1334
|
+
return { bytes: res.bytes, path: res.path, contentType: res.contentType };
|
|
1335
|
+
}
|
|
1336
|
+
rawFulltextUrl(workId, format) {
|
|
1337
|
+
const id = bareWorkId(normalizeId(workId));
|
|
1338
|
+
const ext = format === "xml" ? "grobid-xml" : "pdf";
|
|
1339
|
+
return `${CONTENT_BASE_URL}/works/${id}.${ext}`;
|
|
1340
|
+
}
|
|
1341
|
+
/**
|
|
1342
|
+
* One-hop citation neighbourhood: the work plus its references and citers.
|
|
1343
|
+
* Bounded by `limit` on each side to keep the cost predictable.
|
|
1344
|
+
*/
|
|
1345
|
+
async citationGraph(workId, opts = {}) {
|
|
1346
|
+
const limit = opts.limit ?? 50;
|
|
1347
|
+
const select = opts.select;
|
|
1348
|
+
const [seed, references, citedBy] = await Promise.all([
|
|
1349
|
+
this.get(workId, select ? { select } : {}),
|
|
1350
|
+
this.references(workId).toArray({ maxResults: limit, perPage: Math.min(limit, 200) }),
|
|
1351
|
+
this.citedBy(workId).toArray({ maxResults: limit, perPage: Math.min(limit, 200) })
|
|
1352
|
+
]);
|
|
1353
|
+
return { seed, references, citedBy };
|
|
1354
|
+
}
|
|
1355
|
+
};
|
|
1356
|
+
function bareWorkId(normalized) {
|
|
1357
|
+
const m = /(W\d+)/i.exec(normalized);
|
|
1358
|
+
return m ? m[1].toUpperCase() : normalized;
|
|
1359
|
+
}
|
|
1360
|
+
|
|
1361
|
+
// src/snapshot.ts
|
|
1362
|
+
var SNAPSHOT_BASE_URL = "https://openalex.s3.amazonaws.com";
|
|
1363
|
+
var SNAPSHOT_S3_URI = "s3://openalex";
|
|
1364
|
+
var SnapshotClient = class {
|
|
1365
|
+
baseUrl;
|
|
1366
|
+
format;
|
|
1367
|
+
fetchImpl;
|
|
1368
|
+
constructor(opts = {}) {
|
|
1369
|
+
this.baseUrl = (opts.baseUrl ?? SNAPSHOT_BASE_URL).replace(/\/+$/, "");
|
|
1370
|
+
this.format = opts.format ?? "jsonl";
|
|
1371
|
+
const globalFetch = typeof fetch !== "undefined" ? fetch : void 0;
|
|
1372
|
+
const impl = opts.fetch ?? globalFetch;
|
|
1373
|
+
if (!impl) throw new OpenAlexError("No fetch available for SnapshotClient (Node 18+ or pass `fetch`).");
|
|
1374
|
+
this.fetchImpl = impl;
|
|
1375
|
+
}
|
|
1376
|
+
/** Convert an `s3://openalex/KEY` URL to a public HTTPS URL. */
|
|
1377
|
+
s3ToHttps(s3Url) {
|
|
1378
|
+
const m = /^s3:\/\/([^/]+)\/(.+)$/.exec(s3Url);
|
|
1379
|
+
if (!m) return s3Url;
|
|
1380
|
+
if (m[1] === "openalex") return `${this.baseUrl}/${m[2]}`;
|
|
1381
|
+
return `https://${m[1]}.s3.amazonaws.com/${m[2]}`;
|
|
1382
|
+
}
|
|
1383
|
+
manifestUrl(entity) {
|
|
1384
|
+
return `${this.baseUrl}/data/${this.format}/${entity}/manifest.json`;
|
|
1385
|
+
}
|
|
1386
|
+
combinedManifestUrl() {
|
|
1387
|
+
return `${this.baseUrl}/data/${this.format}/manifest.json`;
|
|
1388
|
+
}
|
|
1389
|
+
/** `aws s3 sync` command to download the whole snapshot or one entity. */
|
|
1390
|
+
awsSyncCommand(entity, dest = "openalex-snapshot") {
|
|
1391
|
+
if (!entity) return `aws s3 sync "${SNAPSHOT_S3_URI}" "${dest}" --no-sign-request`;
|
|
1392
|
+
const key = `data/${this.format}/${entity}`;
|
|
1393
|
+
return `aws s3 sync "${SNAPSHOT_S3_URI}/${key}" "${dest}/${key}" --no-sign-request`;
|
|
1394
|
+
}
|
|
1395
|
+
/** Fetch and normalize an entity's manifest. */
|
|
1396
|
+
async manifest(entity) {
|
|
1397
|
+
const url = this.manifestUrl(entity);
|
|
1398
|
+
const raw = await this.getJson(url);
|
|
1399
|
+
const files = (raw.files ?? []).map((f) => this.toFileEntry(f));
|
|
1400
|
+
return {
|
|
1401
|
+
entity: raw.entity ?? entity,
|
|
1402
|
+
format: raw.format ?? this.format,
|
|
1403
|
+
date: raw.date,
|
|
1404
|
+
recordCount: raw.record_count ?? files.reduce((a, f) => a + f.recordCount, 0),
|
|
1405
|
+
contentLength: raw.content_length ?? files.reduce((a, f) => a + f.contentLength, 0),
|
|
1406
|
+
files
|
|
1407
|
+
};
|
|
1408
|
+
}
|
|
1409
|
+
/** Per-entity totals from the combined manifest (no per-file list). */
|
|
1410
|
+
async totals() {
|
|
1411
|
+
const raw = await this.getJson(this.combinedManifestUrl());
|
|
1412
|
+
return (raw.entities ?? []).map((e) => ({
|
|
1413
|
+
entity: e.entity,
|
|
1414
|
+
recordCount: e.record_count,
|
|
1415
|
+
contentLength: e.content_length
|
|
1416
|
+
}));
|
|
1417
|
+
}
|
|
1418
|
+
/** Totals for a single entity (cheap-ish: reads combined manifest). */
|
|
1419
|
+
async entityTotals(entity) {
|
|
1420
|
+
return (await this.totals()).find((t) => t.entity === entity);
|
|
1421
|
+
}
|
|
1422
|
+
/**
|
|
1423
|
+
* Compare a filtered API extraction against downloading the whole snapshot.
|
|
1424
|
+
* Pass the {@link CostEstimate} from `query.estimateCost()` plus the entity.
|
|
1425
|
+
*/
|
|
1426
|
+
async plan(apiEstimate, entity, opts = {}) {
|
|
1427
|
+
let totals;
|
|
1428
|
+
if (opts.fetchTotals !== false) {
|
|
1429
|
+
try {
|
|
1430
|
+
totals = await this.entityTotals(entity);
|
|
1431
|
+
} catch {
|
|
1432
|
+
totals = void 0;
|
|
1433
|
+
}
|
|
1434
|
+
}
|
|
1435
|
+
const apiCostUsd = apiEstimate.totalCostUsd;
|
|
1436
|
+
const fraction = totals ? apiEstimate.matchingResults / totals.recordCount : void 0;
|
|
1437
|
+
const recommendSnapshot = apiCostUsd >= 1 || fraction !== void 0 && fraction > 0.2;
|
|
1438
|
+
const rationale = recommendSnapshot ? `Your extraction (${apiEstimate.matchingResults.toLocaleString()} results) would cost ~$${apiCostUsd.toFixed(
|
|
1439
|
+
4
|
|
1440
|
+
)} via the API` + (totals ? `, and covers ${((fraction ?? 0) * 100).toFixed(1)}% of all ${entity}. The full ${entity} snapshot (${fmtBytes(
|
|
1441
|
+
totals.contentLength
|
|
1442
|
+
)}, ${totals.recordCount.toLocaleString()} records) is free.` : `. The full ${entity} snapshot is free.`) : `Only ${apiEstimate.matchingResults.toLocaleString()} results (~$${apiCostUsd.toFixed(
|
|
1443
|
+
4
|
|
1444
|
+
)}) \u2014 cheaper/faster to pull the filtered subset via the API than to download the whole ${entity} snapshot.`;
|
|
1445
|
+
return {
|
|
1446
|
+
entity,
|
|
1447
|
+
apiResults: apiEstimate.matchingResults,
|
|
1448
|
+
apiCalls: apiEstimate.pageCalls,
|
|
1449
|
+
apiCostUsd,
|
|
1450
|
+
snapshotCostUsd: 0,
|
|
1451
|
+
snapshotRecords: totals?.recordCount,
|
|
1452
|
+
snapshotBytes: totals?.contentLength,
|
|
1453
|
+
awsCommand: this.awsSyncCommand(entity, opts.dest),
|
|
1454
|
+
recommendation: recommendSnapshot ? "snapshot" : "api",
|
|
1455
|
+
rationale
|
|
1456
|
+
};
|
|
1457
|
+
}
|
|
1458
|
+
/**
|
|
1459
|
+
* Stream entities straight from the public snapshot (gunzip + NDJSON), at $0.
|
|
1460
|
+
* Reads the manifest, then each gzipped part over HTTPS.
|
|
1461
|
+
*/
|
|
1462
|
+
async *stream(entity, opts = {}) {
|
|
1463
|
+
const manifest = await this.manifest(entity);
|
|
1464
|
+
const startPart = opts.startPart ?? 0;
|
|
1465
|
+
const endPart = opts.maxParts !== void 0 ? Math.min(manifest.files.length, startPart + opts.maxParts) : manifest.files.length;
|
|
1466
|
+
const maxRecords = opts.maxRecords ?? Infinity;
|
|
1467
|
+
let emitted = 0;
|
|
1468
|
+
for (let i = startPart; i < endPart; i++) {
|
|
1469
|
+
const file = manifest.files[i];
|
|
1470
|
+
let recordsThisPart = 0;
|
|
1471
|
+
for await (const record of streamGzippedNdjson(this.fetchImpl, file.httpsUrl, opts.signal)) {
|
|
1472
|
+
recordsThisPart++;
|
|
1473
|
+
if (opts.filter && !opts.filter(record)) continue;
|
|
1474
|
+
yield record;
|
|
1475
|
+
emitted++;
|
|
1476
|
+
if (emitted >= maxRecords) {
|
|
1477
|
+
opts.onPart?.({ index: i, total: manifest.files.length, url: file.httpsUrl, records: recordsThisPart });
|
|
1478
|
+
return;
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
opts.onPart?.({ index: i, total: manifest.files.length, url: file.httpsUrl, records: recordsThisPart });
|
|
1482
|
+
}
|
|
1483
|
+
}
|
|
1484
|
+
toFileEntry(f) {
|
|
1485
|
+
return {
|
|
1486
|
+
s3Url: f.url,
|
|
1487
|
+
httpsUrl: this.s3ToHttps(f.url),
|
|
1488
|
+
contentLength: f.meta?.content_length ?? 0,
|
|
1489
|
+
recordCount: f.meta?.record_count ?? 0
|
|
1490
|
+
};
|
|
1491
|
+
}
|
|
1492
|
+
async getJson(url) {
|
|
1493
|
+
const res = await this.fetchImpl(url, { headers: { Accept: "application/json" } });
|
|
1494
|
+
if (!res.ok) {
|
|
1495
|
+
throw new OpenAlexError(`Snapshot request failed (${res.status}) [${url}]`, {
|
|
1496
|
+
status: res.status,
|
|
1497
|
+
url
|
|
1498
|
+
});
|
|
1499
|
+
}
|
|
1500
|
+
return await res.json();
|
|
1501
|
+
}
|
|
1502
|
+
};
|
|
1503
|
+
async function* streamGzippedNdjson(fetchImpl, url, signal) {
|
|
1504
|
+
const res = await fetchImpl(url, signal ? { signal } : {});
|
|
1505
|
+
if (!res.ok || !res.body) {
|
|
1506
|
+
throw new OpenAlexError(`Snapshot part failed (${res.status}) [${url}]`, { status: res.status, url });
|
|
1507
|
+
}
|
|
1508
|
+
const textStream = res.body.pipeThrough(new DecompressionStream("gzip")).pipeThrough(new TextDecoderStream());
|
|
1509
|
+
const reader = textStream.getReader();
|
|
1510
|
+
let buffer = "";
|
|
1511
|
+
try {
|
|
1512
|
+
for (; ; ) {
|
|
1513
|
+
const { done, value } = await reader.read();
|
|
1514
|
+
if (done) break;
|
|
1515
|
+
buffer += value;
|
|
1516
|
+
let nl;
|
|
1517
|
+
while ((nl = buffer.indexOf("\n")) >= 0) {
|
|
1518
|
+
const line = buffer.slice(0, nl);
|
|
1519
|
+
buffer = buffer.slice(nl + 1);
|
|
1520
|
+
if (line.trim()) yield JSON.parse(line);
|
|
1521
|
+
}
|
|
1522
|
+
}
|
|
1523
|
+
if (buffer.trim()) yield JSON.parse(buffer);
|
|
1524
|
+
} finally {
|
|
1525
|
+
reader.releaseLock();
|
|
1526
|
+
}
|
|
1527
|
+
}
|
|
1528
|
+
function fmtBytes(n) {
|
|
1529
|
+
const units = ["B", "KB", "MB", "GB", "TB"];
|
|
1530
|
+
let v = n;
|
|
1531
|
+
let u = 0;
|
|
1532
|
+
while (v >= 1024 && u < units.length - 1) {
|
|
1533
|
+
v /= 1024;
|
|
1534
|
+
u++;
|
|
1535
|
+
}
|
|
1536
|
+
return `${v.toFixed(v >= 10 || u === 0 ? 0 : 1)} ${units[u]}`;
|
|
1537
|
+
}
|
|
1538
|
+
|
|
1539
|
+
// src/planner.ts
|
|
1540
|
+
async function planExtraction(snapshot, query, opts = {}) {
|
|
1541
|
+
const perPage = opts.perPage ?? 200;
|
|
1542
|
+
const est = await query.estimateCost({ perPage, strategy: "cursor" });
|
|
1543
|
+
const count = est.matchingResults;
|
|
1544
|
+
const withinBasic = count <= BASIC_PAGING_MAX_RESULTS;
|
|
1545
|
+
let strategy = withinBasic ? "parallel-basic" : "cursor";
|
|
1546
|
+
let useSnapshot = false;
|
|
1547
|
+
let snapshotAwsCommand;
|
|
1548
|
+
const threshold = opts.snapshotThresholdUsd ?? 1;
|
|
1549
|
+
if (est.totalCostUsd >= threshold) {
|
|
1550
|
+
const plan = await snapshot.plan(est, query.entityType);
|
|
1551
|
+
snapshotAwsCommand = plan.awsCommand;
|
|
1552
|
+
useSnapshot = plan.recommendation === "snapshot";
|
|
1553
|
+
if (useSnapshot) strategy = "snapshot";
|
|
1554
|
+
}
|
|
1555
|
+
const dollars = `$${est.totalCostUsd.toFixed(4)}`;
|
|
1556
|
+
const rationale = useSnapshot ? `~${count.toLocaleString()} results \u2248 ${dollars} via API \u2014 download the free snapshot instead.` : withinBasic ? `${count.toLocaleString()} results fit basic paging (\u226410k): parallel page fetches at per-page ${perPage} are fastest (~${dollars}).` : `${count.toLocaleString()} results exceed 10k: cursor streaming at per-page ${perPage} (~${dollars}).`;
|
|
1557
|
+
return {
|
|
1558
|
+
entity: query.entityType,
|
|
1559
|
+
matchingResults: count,
|
|
1560
|
+
requestType: est.requestType,
|
|
1561
|
+
recommendedPerPage: perPage,
|
|
1562
|
+
recommendedStrategy: strategy,
|
|
1563
|
+
estimate: est,
|
|
1564
|
+
useSnapshot,
|
|
1565
|
+
snapshotAwsCommand,
|
|
1566
|
+
rationale
|
|
1567
|
+
};
|
|
1568
|
+
}
|
|
1569
|
+
|
|
1570
|
+
// src/ngrams.ts
|
|
1571
|
+
async function fetchNgrams(http, workId) {
|
|
1572
|
+
const res = await http.request({
|
|
1573
|
+
path: `works/${normalizeId(workId)}/ngrams`,
|
|
1574
|
+
requestType: "get"
|
|
1575
|
+
});
|
|
1576
|
+
return res.data;
|
|
1577
|
+
}
|
|
1578
|
+
|
|
1579
|
+
// src/client.ts
|
|
1580
|
+
var OpenAlexClient = class {
|
|
1581
|
+
config;
|
|
1582
|
+
http;
|
|
1583
|
+
/** Live spend/budget tracker, updated from every response. */
|
|
1584
|
+
usage;
|
|
1585
|
+
/** Free bulk-data snapshot bridge (manifest, cost comparison, streaming). */
|
|
1586
|
+
snapshot;
|
|
1587
|
+
works;
|
|
1588
|
+
authors;
|
|
1589
|
+
sources;
|
|
1590
|
+
institutions;
|
|
1591
|
+
topics;
|
|
1592
|
+
keywords;
|
|
1593
|
+
publishers;
|
|
1594
|
+
funders;
|
|
1595
|
+
constructor(config = {}) {
|
|
1596
|
+
this.config = resolveConfig(config);
|
|
1597
|
+
this.usage = new UsageTracker(Boolean(this.config.apiKey), this.config.dailyBudgetUsd);
|
|
1598
|
+
this.http = new HttpClient(this.config, this.usage);
|
|
1599
|
+
this.snapshot = new SnapshotClient({ fetch: this.config.fetch });
|
|
1600
|
+
this.works = new WorksEndpoint(this.http);
|
|
1601
|
+
this.authors = new EntityEndpoint(this.http, "authors");
|
|
1602
|
+
this.sources = new EntityEndpoint(this.http, "sources");
|
|
1603
|
+
this.institutions = new EntityEndpoint(this.http, "institutions");
|
|
1604
|
+
this.topics = new EntityEndpoint(this.http, "topics");
|
|
1605
|
+
this.keywords = new EntityEndpoint(this.http, "keywords");
|
|
1606
|
+
this.publishers = new EntityEndpoint(this.http, "publishers");
|
|
1607
|
+
this.funders = new EntityEndpoint(this.http, "funders");
|
|
1608
|
+
}
|
|
1609
|
+
/** Cross-entity typeahead. Optionally scope with `entityType`. */
|
|
1610
|
+
async autocomplete(q, entityType) {
|
|
1611
|
+
const path = entityType ? `autocomplete/${entityType}` : "autocomplete";
|
|
1612
|
+
const res = await this.http.request({
|
|
1613
|
+
path,
|
|
1614
|
+
params: { q },
|
|
1615
|
+
requestType: "autocomplete"
|
|
1616
|
+
});
|
|
1617
|
+
return res.data.results;
|
|
1618
|
+
}
|
|
1619
|
+
/** N-grams for a Work. */
|
|
1620
|
+
ngrams(workId) {
|
|
1621
|
+
return fetchNgrams(this.http, workId);
|
|
1622
|
+
}
|
|
1623
|
+
/**
|
|
1624
|
+
* Decide API vs free snapshot for a query: estimates the API cost, then
|
|
1625
|
+
* compares against downloading the whole entity's snapshot ($0).
|
|
1626
|
+
*/
|
|
1627
|
+
async snapshotPlan(query) {
|
|
1628
|
+
const estimate = await query.estimateCost();
|
|
1629
|
+
return this.snapshot.plan(estimate, query.entityType);
|
|
1630
|
+
}
|
|
1631
|
+
/**
|
|
1632
|
+
* Cost-optimal plan for extracting a query: recommends per-page, cursor-vs-
|
|
1633
|
+
* parallel strategy, and whether the free snapshot beats the API.
|
|
1634
|
+
*/
|
|
1635
|
+
planExtraction(query, opts) {
|
|
1636
|
+
return planExtraction(this.snapshot, query, opts);
|
|
1637
|
+
}
|
|
1638
|
+
/** Escape hatch: issue a raw GET against any path with any params. */
|
|
1639
|
+
async request(path, params = {}, opts = {}) {
|
|
1640
|
+
const res = await this.http.request({
|
|
1641
|
+
path,
|
|
1642
|
+
params,
|
|
1643
|
+
requestType: opts.requestType ?? "list",
|
|
1644
|
+
noCache: opts.noCache
|
|
1645
|
+
});
|
|
1646
|
+
return res.data;
|
|
1647
|
+
}
|
|
1648
|
+
/** Point-in-time usage snapshot. */
|
|
1649
|
+
usageSnapshot() {
|
|
1650
|
+
return this.usage.snapshot();
|
|
1651
|
+
}
|
|
1652
|
+
/** Human-readable spend/budget report. */
|
|
1653
|
+
usageReport() {
|
|
1654
|
+
return this.usage.report();
|
|
1655
|
+
}
|
|
1656
|
+
/** Reset the session usage counters (does not affect OpenAlex's daily total). */
|
|
1657
|
+
resetUsage() {
|
|
1658
|
+
this.usage.reset();
|
|
1659
|
+
}
|
|
1660
|
+
};
|
|
1661
|
+
|
|
1662
|
+
// src/cli.ts
|
|
1663
|
+
var ENTITIES = [
|
|
1664
|
+
"works",
|
|
1665
|
+
"authors",
|
|
1666
|
+
"sources",
|
|
1667
|
+
"institutions",
|
|
1668
|
+
"topics",
|
|
1669
|
+
"keywords",
|
|
1670
|
+
"publishers",
|
|
1671
|
+
"funders"
|
|
1672
|
+
];
|
|
1673
|
+
function parseArgs(argv) {
|
|
1674
|
+
const _ = [];
|
|
1675
|
+
const flags = {};
|
|
1676
|
+
for (let i = 0; i < argv.length; i++) {
|
|
1677
|
+
const a = argv[i];
|
|
1678
|
+
if (a.startsWith("--")) {
|
|
1679
|
+
const eq = a.indexOf("=");
|
|
1680
|
+
if (eq >= 0) {
|
|
1681
|
+
flags[a.slice(2, eq)] = a.slice(eq + 1);
|
|
1682
|
+
} else {
|
|
1683
|
+
const key = a.slice(2);
|
|
1684
|
+
const next = argv[i + 1];
|
|
1685
|
+
if (next === void 0 || next.startsWith("--")) {
|
|
1686
|
+
flags[key] = true;
|
|
1687
|
+
} else {
|
|
1688
|
+
flags[key] = next;
|
|
1689
|
+
i++;
|
|
1690
|
+
}
|
|
1691
|
+
}
|
|
1692
|
+
} else {
|
|
1693
|
+
_.push(a);
|
|
1694
|
+
}
|
|
1695
|
+
}
|
|
1696
|
+
return { _, flags };
|
|
1697
|
+
}
|
|
1698
|
+
async function runCli(argv, deps = {}) {
|
|
1699
|
+
const out = deps.stdout ?? ((s) => process.stdout.write(s + "\n"));
|
|
1700
|
+
const err = deps.stderr ?? ((s) => process.stderr.write(s + "\n"));
|
|
1701
|
+
const { _, flags } = parseArgs(argv);
|
|
1702
|
+
const cmd = _[0];
|
|
1703
|
+
if (!cmd || flags.help || cmd === "help") {
|
|
1704
|
+
out(HELP);
|
|
1705
|
+
return 0;
|
|
1706
|
+
}
|
|
1707
|
+
const client = deps.client ?? new OpenAlexClient({
|
|
1708
|
+
apiKey: str(flags["api-key"]),
|
|
1709
|
+
mailto: str(flags.mailto)
|
|
1710
|
+
});
|
|
1711
|
+
const json = Boolean(flags.json);
|
|
1712
|
+
try {
|
|
1713
|
+
if (cmd === "snapshot") return await snapshotCmd(_, flags, client, out, json);
|
|
1714
|
+
if (cmd === "fulltext") return await fulltextCmd(_, flags, client, out);
|
|
1715
|
+
if (cmd === "get") return await getCmd(_, flags, client, out);
|
|
1716
|
+
if (ENTITIES.includes(cmd)) {
|
|
1717
|
+
return await entityCmd(cmd, flags, client, out, json);
|
|
1718
|
+
}
|
|
1719
|
+
err(`Unknown command: ${cmd}`);
|
|
1720
|
+
out(HELP);
|
|
1721
|
+
return 1;
|
|
1722
|
+
} catch (e) {
|
|
1723
|
+
err(`Error: ${e.message}`);
|
|
1724
|
+
return 1;
|
|
1725
|
+
}
|
|
1726
|
+
}
|
|
1727
|
+
function endpoint(client, entity) {
|
|
1728
|
+
const ep = client[entity];
|
|
1729
|
+
if (!ep) throw new Error(`Unknown entity: ${entity}. One of: ${ENTITIES.join(", ")}`);
|
|
1730
|
+
return ep;
|
|
1731
|
+
}
|
|
1732
|
+
function buildQuery(client, entity, flags) {
|
|
1733
|
+
let q = endpoint(client, entity).query();
|
|
1734
|
+
if (str(flags.filter)) q = q.param("filter", str(flags.filter));
|
|
1735
|
+
if (str(flags.search)) q = q.search(str(flags.search));
|
|
1736
|
+
if (str(flags.select)) q = q.select(str(flags.select));
|
|
1737
|
+
if (str(flags.sort)) q = q.sort(str(flags.sort));
|
|
1738
|
+
if (str(flags["per-page"])) q = q.perPage(Number(flags["per-page"]));
|
|
1739
|
+
return q;
|
|
1740
|
+
}
|
|
1741
|
+
async function entityCmd(entity, flags, client, out, json) {
|
|
1742
|
+
const q = buildQuery(client, entity, flags);
|
|
1743
|
+
const limit = flags.limit !== void 0 ? Number(flags.limit) : void 0;
|
|
1744
|
+
const parallel = Boolean(flags.parallel);
|
|
1745
|
+
if (flags.count) {
|
|
1746
|
+
const n = await q.count();
|
|
1747
|
+
out(json ? JSON.stringify({ count: n }) : `${n}`);
|
|
1748
|
+
return 0;
|
|
1749
|
+
}
|
|
1750
|
+
if (flags.estimate) {
|
|
1751
|
+
const e = await q.estimateCost();
|
|
1752
|
+
out(json ? JSON.stringify(e, null, 2) : formatEstimate(e));
|
|
1753
|
+
return 0;
|
|
1754
|
+
}
|
|
1755
|
+
if (str(flags.export)) {
|
|
1756
|
+
const res = await q.export(str(flags.export), { maxResults: limit, parallel });
|
|
1757
|
+
out(json ? JSON.stringify(res) : `Wrote ${res.count} ${entity} to ${res.path}`);
|
|
1758
|
+
return 0;
|
|
1759
|
+
}
|
|
1760
|
+
const items = await q.toArray({ maxResults: limit ?? 25, parallel });
|
|
1761
|
+
out(JSON.stringify(items, null, json ? 0 : 2));
|
|
1762
|
+
return 0;
|
|
1763
|
+
}
|
|
1764
|
+
async function getCmd(_, flags, client, out) {
|
|
1765
|
+
const entity = _[1];
|
|
1766
|
+
const id = _[2];
|
|
1767
|
+
if (!entity || !id) throw new Error("usage: oaw get <entity> <id> [--select ...]");
|
|
1768
|
+
const ep = endpoint(client, entity);
|
|
1769
|
+
const item = await ep.get(id, str(flags.select) ? { select: str(flags.select) } : {});
|
|
1770
|
+
out(JSON.stringify(item, null, 2));
|
|
1771
|
+
return 0;
|
|
1772
|
+
}
|
|
1773
|
+
async function snapshotCmd(_, flags, client, out, json) {
|
|
1774
|
+
const sub = _[1];
|
|
1775
|
+
if (sub === "totals") {
|
|
1776
|
+
const entity = _[2];
|
|
1777
|
+
if (entity) {
|
|
1778
|
+
const t = await client.snapshot.entityTotals(entity);
|
|
1779
|
+
out(json ? JSON.stringify(t) : formatTotals(t));
|
|
1780
|
+
} else {
|
|
1781
|
+
const ts = await client.snapshot.totals();
|
|
1782
|
+
out(json ? JSON.stringify(ts, null, 2) : ts.map(formatTotals).join("\n"));
|
|
1783
|
+
}
|
|
1784
|
+
return 0;
|
|
1785
|
+
}
|
|
1786
|
+
if (sub === "plan") {
|
|
1787
|
+
const entity = _[2];
|
|
1788
|
+
if (!entity) throw new Error("usage: oaw snapshot plan <entity> [--filter ...]");
|
|
1789
|
+
const plan = await client.snapshotPlan(buildQuery(client, entity, flags));
|
|
1790
|
+
out(json ? JSON.stringify(plan, null, 2) : formatPlan(plan));
|
|
1791
|
+
return 0;
|
|
1792
|
+
}
|
|
1793
|
+
if (sub === "stream") {
|
|
1794
|
+
const entity = _[2];
|
|
1795
|
+
if (!entity) throw new Error("usage: oaw snapshot stream <entity> [--limit N] [--export FILE]");
|
|
1796
|
+
const limit = flags.limit !== void 0 ? Number(flags.limit) : 10;
|
|
1797
|
+
const gen = client.snapshot.stream(entity, { maxRecords: limit });
|
|
1798
|
+
if (str(flags.export)) {
|
|
1799
|
+
const { exportToFile: exportToFile2 } = await Promise.resolve().then(() => (init_export(), export_exports));
|
|
1800
|
+
const res = await exportToFile2(gen, str(flags.export));
|
|
1801
|
+
out(`Wrote ${res.count} ${entity} to ${res.path}`);
|
|
1802
|
+
return 0;
|
|
1803
|
+
}
|
|
1804
|
+
for await (const rec of gen) out(JSON.stringify(rec));
|
|
1805
|
+
return 0;
|
|
1806
|
+
}
|
|
1807
|
+
throw new Error("usage: oaw snapshot <totals|plan|stream> [entity] [flags]");
|
|
1808
|
+
}
|
|
1809
|
+
async function fulltextCmd(_, flags, client, out) {
|
|
1810
|
+
const id = _[1];
|
|
1811
|
+
if (!id) throw new Error("usage: oaw fulltext <workId> [--format pdf|xml] [--out FILE]");
|
|
1812
|
+
const format = flags.format === "xml" ? "xml" : "pdf";
|
|
1813
|
+
const dest = str(flags.out);
|
|
1814
|
+
if (!dest) {
|
|
1815
|
+
out(client.works.fulltextUrl(id, format));
|
|
1816
|
+
return 0;
|
|
1817
|
+
}
|
|
1818
|
+
const res = await client.works.downloadFulltext(id, { format, dest });
|
|
1819
|
+
out(`Saved ${format.toUpperCase()} to ${res.path}`);
|
|
1820
|
+
return 0;
|
|
1821
|
+
}
|
|
1822
|
+
function str(v) {
|
|
1823
|
+
return typeof v === "string" ? v : void 0;
|
|
1824
|
+
}
|
|
1825
|
+
function formatEstimate(e) {
|
|
1826
|
+
return `results: ${e.matchingResults.toLocaleString()}
|
|
1827
|
+
api calls: ${e.pageCalls}
|
|
1828
|
+
est. cost: $${e.totalCostUsd.toFixed(4)}` + (e.cappedByBasicPaging ? " (basic-paging capped at 10k)" : "");
|
|
1829
|
+
}
|
|
1830
|
+
function formatTotals(t) {
|
|
1831
|
+
if (!t) return "(unknown entity)";
|
|
1832
|
+
const gb = (t.contentLength / 1e9).toFixed(1);
|
|
1833
|
+
return `${t.entity.padEnd(14)} ${t.recordCount.toLocaleString().padStart(14)} records ${gb} GB`;
|
|
1834
|
+
}
|
|
1835
|
+
function formatPlan(p) {
|
|
1836
|
+
return `recommendation: ${p.recommendation.toUpperCase()}
|
|
1837
|
+
${p.rationale}
|
|
1838
|
+
api cost: ~$${p.apiCostUsd.toFixed(4)} snapshot: $0
|
|
1839
|
+
bulk download: ${p.awsCommand}`;
|
|
1840
|
+
}
|
|
1841
|
+
var HELP = `oaw: OpenAlex CLI (open-alex-wrapper)
|
|
1842
|
+
|
|
1843
|
+
Usage:
|
|
1844
|
+
oaw <entity> [flags] query an entity (works, authors, sources,
|
|
1845
|
+
institutions, topics, keywords, publishers, funders)
|
|
1846
|
+
oaw get <entity> <id> [--select ..] fetch one entity by any ID/DOI/ORCID/...
|
|
1847
|
+
oaw snapshot totals [entity] snapshot record counts & sizes
|
|
1848
|
+
oaw snapshot plan <entity> [flags] API $ cost vs free snapshot
|
|
1849
|
+
oaw snapshot stream <entity> [flags] stream the free snapshot ($0)
|
|
1850
|
+
oaw fulltext <workId> [flags] download full text (PDF/XML) or print its URL
|
|
1851
|
+
|
|
1852
|
+
Query flags:
|
|
1853
|
+
--filter "k:v,k2:v2" raw OpenAlex filter --search "text"
|
|
1854
|
+
--select id,title trim fields --sort field:desc
|
|
1855
|
+
--per-page N page size (\u2264200) --limit N cap results
|
|
1856
|
+
--parallel fast parallel paging --export FILE write JSONL/CSV/JSON
|
|
1857
|
+
--count just the match count --estimate dollar cost estimate
|
|
1858
|
+
|
|
1859
|
+
Global flags:
|
|
1860
|
+
--api-key KEY (or OPENALEX_API_KEY) --mailto you@x.com --json --help`;
|
|
1861
|
+
function isMain() {
|
|
1862
|
+
const entry = process.argv[1] ?? "";
|
|
1863
|
+
return /(?:^|[\\/])(cli|oaw)(\.(c|m)?js)?$/.test(entry);
|
|
1864
|
+
}
|
|
1865
|
+
if (isMain()) {
|
|
1866
|
+
runCli(process.argv.slice(2)).then(
|
|
1867
|
+
(code) => process.exit(code),
|
|
1868
|
+
(e) => {
|
|
1869
|
+
process.stderr.write(`Fatal: ${e.message}
|
|
1870
|
+
`);
|
|
1871
|
+
process.exit(1);
|
|
1872
|
+
}
|
|
1873
|
+
);
|
|
1874
|
+
}
|
|
1875
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
1876
|
+
0 && (module.exports = {
|
|
1877
|
+
parseArgs,
|
|
1878
|
+
runCli
|
|
1879
|
+
});
|
|
1880
|
+
//# sourceMappingURL=cli.cjs.map
|