cleanedweb 0.0.0-stage → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.js ADDED
@@ -0,0 +1,954 @@
1
+ // src/errors.ts
2
+ var CleanedWebError = class extends Error {
3
+ /** The idempotency key of the run this error belongs to, when there is one. */
4
+ idempotencyKey;
5
+ /** The run this error belongs to, when known. */
6
+ runId;
7
+ constructor(message, options) {
8
+ super(message, options);
9
+ this.name = new.target.name;
10
+ }
11
+ };
12
+ var ConfigurationError = class extends CleanedWebError {
13
+ };
14
+ var CleanedWebConnectionError = class extends CleanedWebError {
15
+ };
16
+ var CleanedWebTimeoutError = class extends CleanedWebConnectionError {
17
+ };
18
+ var CleanedWebAbortError = class extends CleanedWebError {
19
+ constructor(cause) {
20
+ super("The request was aborted by the caller. A run that already started keeps running.", { cause });
21
+ this.name = "AbortError";
22
+ }
23
+ };
24
+ var ResultNotAcceptedError = class extends CleanedWebError {
25
+ reasons;
26
+ constructor(reasons) {
27
+ super(`The run result needs review: ${reasons.join("; ")}`);
28
+ this.reasons = reasons;
29
+ }
30
+ };
31
+ var APIError = class extends CleanedWebError {
32
+ /** HTTP status. */
33
+ status;
34
+ /** The documented error `code` (for example `insufficient_units`), when the body has one. */
35
+ code;
36
+ /** The raw `detail` of the body: text, an object, or a list of validation issues. */
37
+ detail;
38
+ /** The parsed body (or the text, if it was not JSON). Never contains the API key. */
39
+ body;
40
+ /** Seconds to wait before retrying, from `Retry-After` or the body. */
41
+ retryAfter;
42
+ /** Deprecation headers, when the answer carried them. */
43
+ deprecation;
44
+ constructor(init) {
45
+ super(init.message);
46
+ this.status = init.status;
47
+ this.code = init.code;
48
+ this.detail = init.detail;
49
+ this.body = init.body;
50
+ this.retryAfter = init.retryAfter;
51
+ this.deprecation = init.deprecation;
52
+ const runId = init.detail?.run_id;
53
+ if (typeof runId === "string") this.runId = runId;
54
+ }
55
+ };
56
+ var AuthenticationError = class extends APIError {
57
+ };
58
+ var PermissionError = class extends APIError {
59
+ };
60
+ var InsufficientCreditError = class extends APIError {
61
+ /** Units the run needs. */
62
+ required;
63
+ /** Units the balance has. */
64
+ available;
65
+ constructor(init) {
66
+ super(init);
67
+ const detail = init.detail ?? {};
68
+ this.required = detail.required;
69
+ this.available = detail.available;
70
+ }
71
+ };
72
+ var UsageBudgetExceededError = class extends APIError {
73
+ };
74
+ var IdempotencyConflictError = class extends APIError {
75
+ };
76
+ var VersionConflictError = class extends APIError {
77
+ };
78
+ var ResultNotReadyError = class extends APIError {
79
+ };
80
+ var NotFoundError = class extends APIError {
81
+ available;
82
+ constructor(init) {
83
+ super(init);
84
+ const available = init.detail?.available;
85
+ this.available = Array.isArray(available) ? available : void 0;
86
+ }
87
+ };
88
+ var ValidationError = class extends APIError {
89
+ };
90
+ var RateLimitError = class extends APIError {
91
+ };
92
+ var ServerError = class extends APIError {
93
+ };
94
+ var RunFailedError = class extends APIError {
95
+ /** `run_failed` for the failure itself, `failed_run_replayed` when the key was replayed. */
96
+ errorCode;
97
+ /** Whether this answer replays the key. */
98
+ replayed;
99
+ constructor(init) {
100
+ super(init);
101
+ this.errorCode = init.errorCode;
102
+ this.replayed = init.replayed;
103
+ }
104
+ };
105
+
106
+ // src/util.ts
107
+ var KEY_PATTERN = /^[A-Za-z0-9._:-]{1,200}$/;
108
+ function generateIdempotencyKey() {
109
+ const c = globalThis.crypto;
110
+ if (c && typeof c.randomUUID === "function") return c.randomUUID();
111
+ const bytes = new Uint8Array(16);
112
+ if (c && typeof c.getRandomValues === "function") {
113
+ c.getRandomValues(bytes);
114
+ } else {
115
+ for (let i = 0; i < bytes.length; i += 1) bytes[i] = Math.floor(Math.random() * 256);
116
+ }
117
+ bytes[6] = (bytes[6] ?? 0) & 15 | 64;
118
+ bytes[8] = (bytes[8] ?? 0) & 63 | 128;
119
+ const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
120
+ return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
121
+ }
122
+ function assertIdempotencyKey(key) {
123
+ if (typeof key !== "string" || !KEY_PATTERN.test(key)) {
124
+ throw new TypeError(
125
+ 'idempotencyKey must be 1-200 characters: letters, digits, ".", "_", ":" or "-".'
126
+ );
127
+ }
128
+ }
129
+ function parseRetryAfter(value) {
130
+ if (value === null || value === void 0) return void 0;
131
+ const trimmed = value.trim();
132
+ if (trimmed === "") return void 0;
133
+ if (/^\d+(\.\d+)?$/.test(trimmed)) return Number(trimmed);
134
+ const date = Date.parse(trimmed);
135
+ if (Number.isNaN(date)) return void 0;
136
+ return Math.max(0, Math.ceil((date - Date.now()) / 1e3));
137
+ }
138
+ var defaultSleep = (ms, signal) => new Promise((resolve, reject) => {
139
+ if (signal?.aborted) {
140
+ reject(new CleanedWebAbortError(signal.reason));
141
+ return;
142
+ }
143
+ const onAbort = () => {
144
+ clearTimeout(timer);
145
+ reject(new CleanedWebAbortError(signal?.reason));
146
+ };
147
+ const timer = setTimeout(() => {
148
+ signal?.removeEventListener("abort", onAbort);
149
+ resolve();
150
+ }, ms);
151
+ signal?.addEventListener("abort", onAbort, { once: true });
152
+ });
153
+ function throwIfAborted(signal) {
154
+ if (signal?.aborted) throw new CleanedWebAbortError(signal.reason);
155
+ }
156
+ function encodeSegment(value) {
157
+ return encodeURIComponent(value);
158
+ }
159
+ function isRecord(value) {
160
+ return typeof value === "object" && value !== null && !Array.isArray(value);
161
+ }
162
+ function compact(value) {
163
+ return Object.fromEntries(Object.entries(value).filter(([, v]) => v !== void 0));
164
+ }
165
+ function assertIntegerInRange(name, value, min, max) {
166
+ if (value === void 0) return;
167
+ if (!Number.isInteger(value) || value < min || max !== void 0 && value > max) {
168
+ throw new RangeError(
169
+ max === void 0 ? `${name} must be an integer of at least ${min}.` : `${name} must be an integer from ${min} to ${max}.`
170
+ );
171
+ }
172
+ }
173
+
174
+ // src/http.ts
175
+ var RETRYABLE_STATUS = /* @__PURE__ */ new Set([429, 502, 503, 504]);
176
+ var RESULT_NOT_READY = /* @__PURE__ */ new Set(["result_not_available", "result_not_complete", "result_not_running"]);
177
+ var BACKOFF_BASE_MS = 500;
178
+ var BACKOFF_CAP_MS = 8e3;
179
+ function deprecationFrom(headers) {
180
+ const deprecation = headers.get("deprecation");
181
+ const sunset = headers.get("sunset");
182
+ const link = headers.get("link");
183
+ if (deprecation === null && sunset === null) return void 0;
184
+ return { deprecation, sunset, link };
185
+ }
186
+ function stripTrailingSlash(url) {
187
+ return url.replace(/\/+$/, "");
188
+ }
189
+ var HttpClient = class {
190
+ constructor(config) {
191
+ this.config = config;
192
+ }
193
+ config;
194
+ get sleep() {
195
+ return this.config.sleep;
196
+ }
197
+ originFor(target) {
198
+ switch (target) {
199
+ case "run":
200
+ return this.config.baseUrl;
201
+ case "catalog":
202
+ return this.config.catalogUrl;
203
+ default:
204
+ return this.config.siteUrl;
205
+ }
206
+ }
207
+ /**
208
+ * Sends one request, retrying 429/502/503/504 and lost connections with bounded backoff.
209
+ * Every request the SDK sends is safe to repeat: runs carry an idempotency key, the rest are
210
+ * reads or an abort. A failed run (an error with `error_code`) is final and is never retried.
211
+ * Resolves for 2xx; throws a typed error otherwise.
212
+ */
213
+ async request(spec) {
214
+ const { config } = this;
215
+ if (spec.target === "run" && !config.apiKey) {
216
+ throw new ConfigurationError(
217
+ 'No API key. Pass { apiKey } or set CLEANEDWEB_API_KEY (a key with the "run" scope).'
218
+ );
219
+ }
220
+ const url = this.buildUrl(spec);
221
+ const maxRetries = spec.maxRetries ?? config.maxRetries;
222
+ const headers = { Accept: "application/json" };
223
+ if (spec.target === "run" && config.apiKey) headers.Authorization = `Bearer ${config.apiKey}`;
224
+ if (spec.idempotencyKey !== void 0) headers["Idempotency-Key"] = spec.idempotencyKey;
225
+ let payload;
226
+ if (spec.body !== void 0) {
227
+ headers["Content-Type"] = "application/json";
228
+ payload = JSON.stringify(spec.body);
229
+ }
230
+ for (let attempt = 0; ; attempt += 1) {
231
+ throwIfAborted(spec.signal);
232
+ let response;
233
+ try {
234
+ response = await this.send(url, spec, headers, payload);
235
+ } catch (error2) {
236
+ if (error2 instanceof CleanedWebAbortError || error2 instanceof ConfigurationError) throw error2;
237
+ if (attempt < maxRetries) {
238
+ await config.sleep(this.backoff(attempt), spec.signal);
239
+ continue;
240
+ }
241
+ throw error2;
242
+ }
243
+ if (response.status >= 200 && response.status < 300) return response;
244
+ const error = this.toError(response);
245
+ if (attempt < maxRetries && RETRYABLE_STATUS.has(response.status) && !(error instanceof RunFailedError)) {
246
+ const wanted = error.retryAfter === void 0 ? this.backoff(attempt) : error.retryAfter * 1e3;
247
+ if (wanted <= config.maxRetryDelayMs) {
248
+ await config.sleep(wanted, spec.signal);
249
+ continue;
250
+ }
251
+ }
252
+ throw error;
253
+ }
254
+ }
255
+ buildUrl(spec) {
256
+ const query = new URLSearchParams();
257
+ for (const [key, value] of Object.entries(spec.query ?? {})) {
258
+ if (value !== void 0) query.set(key, String(value));
259
+ }
260
+ const qs = query.toString();
261
+ return `${stripTrailingSlash(this.originFor(spec.target))}${spec.path}${qs ? `?${qs}` : ""}`;
262
+ }
263
+ backoff(attempt) {
264
+ const base = Math.min(BACKOFF_BASE_MS * 2 ** attempt, BACKOFF_CAP_MS);
265
+ return Math.round(base / 2 + Math.random() * (base / 2));
266
+ }
267
+ async send(url, spec, headers, payload) {
268
+ const { config } = this;
269
+ const timeoutMs = spec.timeoutMs ?? config.timeoutMs;
270
+ const controller = new AbortController();
271
+ const caller = spec.signal;
272
+ const onCallerAbort = () => controller.abort(caller?.reason);
273
+ caller?.addEventListener("abort", onCallerAbort, { once: true });
274
+ let timedOut = false;
275
+ const timer = setTimeout(() => {
276
+ timedOut = true;
277
+ controller.abort();
278
+ }, timeoutMs);
279
+ try {
280
+ const init = {
281
+ method: spec.method,
282
+ headers,
283
+ // A redirect means the wrong origin: nothing ran. Surface it instead of following it.
284
+ redirect: "manual",
285
+ signal: controller.signal
286
+ };
287
+ if (payload !== void 0) init.body = payload;
288
+ const res = await config.fetch(url, init);
289
+ if (res.status >= 300 && res.status < 400 || res.type === "opaqueredirect") {
290
+ throw new ConfigurationError(
291
+ `The server redirected (HTTP ${res.status}) instead of answering. Nothing ran. Check baseUrl: runs are served by https://api.cleanedweb.com.`
292
+ );
293
+ }
294
+ const text = await res.text();
295
+ return { status: res.status, headers: res.headers, body: this.parseBody(text) };
296
+ } catch (error) {
297
+ if (error instanceof ConfigurationError) throw error;
298
+ if (caller?.aborted) throw new CleanedWebAbortError(caller.reason);
299
+ if (timedOut) {
300
+ throw new CleanedWebTimeoutError(
301
+ `No answer within ${timeoutMs} ms. The run may have started: keep the idempotency key and read the result with it.`,
302
+ { cause: error }
303
+ );
304
+ }
305
+ throw new CleanedWebConnectionError(
306
+ "The connection failed. The run may have started: keep the idempotency key and read the result with it.",
307
+ { cause: error }
308
+ );
309
+ } finally {
310
+ clearTimeout(timer);
311
+ caller?.removeEventListener("abort", onCallerAbort);
312
+ }
313
+ }
314
+ parseBody(text) {
315
+ const key = this.config.apiKey;
316
+ const safe = key && text.includes(key) ? text.split(key).join("[redacted]") : text;
317
+ if (safe === "") return null;
318
+ try {
319
+ return JSON.parse(safe);
320
+ } catch {
321
+ return safe;
322
+ }
323
+ }
324
+ toError(response) {
325
+ const { status, headers, body } = response;
326
+ const detail = isRecord(body) ? body.detail : void 0;
327
+ const detailObject = isRecord(detail) ? detail : void 0;
328
+ const code = typeof detailObject?.code === "string" ? detailObject.code : void 0;
329
+ let message;
330
+ if (typeof detailObject?.message === "string") message = detailObject.message;
331
+ else if (typeof detail === "string" && detail !== "") message = detail;
332
+ else if (Array.isArray(detail)) message = describeIssues(detail);
333
+ else message = `HTTP ${status}`;
334
+ const retryAfter = parseRetryAfter(headers.get("retry-after")) ?? numberOrUndefined(detailObject?.retryAfterSeconds) ?? numberOrUndefined(detailObject?.retry_after);
335
+ const init = {
336
+ status,
337
+ code,
338
+ message: code ? `${code}: ${message}` : message,
339
+ detail,
340
+ body,
341
+ retryAfter,
342
+ deprecation: deprecationFrom(headers)
343
+ };
344
+ const marker = detailObject?.error_code;
345
+ if (marker === "run_failed" || marker === "failed_run_replayed") {
346
+ return new RunFailedError({ ...init, errorCode: marker, replayed: detailObject?.replayed === true });
347
+ }
348
+ if (code === "delivery_identity_conflict") return new IdempotencyConflictError(init);
349
+ if (code === "usage_budget_exceeded") return new UsageBudgetExceededError(init);
350
+ if (code && RESULT_NOT_READY.has(code)) return new ResultNotReadyError(init);
351
+ if (code === "builder_definition_version_conflict" || code === "catalog_version_conflict") {
352
+ return new VersionConflictError(init);
353
+ }
354
+ if (status === 402) return new InsufficientCreditError(init);
355
+ if (status === 401) return new AuthenticationError(init);
356
+ if (status === 403) return new PermissionError(init);
357
+ if (status === 404) return new NotFoundError(init);
358
+ if (status === 429) return new RateLimitError(init);
359
+ if (status === 400 || status === 415 || status === 422) return new ValidationError(init);
360
+ if (status >= 500) return new ServerError(init);
361
+ return new APIError(init);
362
+ }
363
+ };
364
+ function numberOrUndefined(value) {
365
+ return typeof value === "number" && Number.isFinite(value) ? value : void 0;
366
+ }
367
+ function describeIssues(issues) {
368
+ const parts = issues.map((issue) => {
369
+ if (isRecord(issue) && typeof issue.msg === "string") {
370
+ const where = Array.isArray(issue.loc) ? `${issue.loc.join(".")}: ` : "";
371
+ return `${where}${issue.msg}`;
372
+ }
373
+ return typeof issue === "string" ? issue : "invalid value";
374
+ });
375
+ return parts.join("; ") || "invalid request";
376
+ }
377
+
378
+ // src/runner.ts
379
+ var DEFAULT_POLL_SECONDS = 30;
380
+ var MIN_POLL_MS = 1e3;
381
+ function completedFrom(res) {
382
+ const body = isRecord(res.body) ? res.body : {};
383
+ const deprecation = deprecationFrom(res.headers);
384
+ return { ...body, pending: false, ...deprecation ? { deprecation } : {} };
385
+ }
386
+ function pendingFrom(res) {
387
+ const run = isRecord(res.body) ? res.body : {};
388
+ const deprecation = deprecationFrom(res.headers);
389
+ return {
390
+ pending: true,
391
+ run,
392
+ retryAfter: parseRetryAfter(res.headers.get("retry-after")) ?? run.poll_after_seconds,
393
+ ...deprecation ? { deprecation } : {}
394
+ };
395
+ }
396
+ async function settle(first, options) {
397
+ const { http, wait, readResult, signal, onProgress } = options;
398
+ let res = first;
399
+ let runId;
400
+ try {
401
+ for (; ; ) {
402
+ if (res.status !== 202) return completedFrom(res);
403
+ const pending = pendingFrom(res);
404
+ runId = pending.run.run_id ?? runId;
405
+ onProgress?.(pending.run);
406
+ if (!wait) return pending;
407
+ res = await pollOnce(pending.retryAfter);
408
+ }
409
+ } catch (error) {
410
+ if (error instanceof CleanedWebError && runId !== void 0) error.runId ??= runId;
411
+ throw error;
412
+ }
413
+ async function pollOnce(retryAfter) {
414
+ let seconds = retryAfter;
415
+ for (; ; ) {
416
+ throwIfAborted(signal);
417
+ await http.sleep(Math.max(MIN_POLL_MS, (seconds ?? DEFAULT_POLL_SECONDS) * 1e3), signal);
418
+ try {
419
+ return await readResult();
420
+ } catch (error) {
421
+ if (error instanceof ResultNotReadyError) {
422
+ seconds = error.retryAfter;
423
+ continue;
424
+ }
425
+ throw error;
426
+ }
427
+ }
428
+ }
429
+ }
430
+ async function withKey(idempotencyKey, fn) {
431
+ try {
432
+ return await fn();
433
+ } catch (error) {
434
+ if (error instanceof CleanedWebError) error.idempotencyKey ??= idempotencyKey;
435
+ throw error;
436
+ }
437
+ }
438
+ function acceptanceProblems(result, options = {}) {
439
+ const problems = [];
440
+ const summary = result.summary;
441
+ if (!summary) return ["the result has no summary"];
442
+ if (summary.status !== "completed" && summary.status !== "bounded") {
443
+ problems.push(`summary.status is "${String(summary.status)}", not completed or bounded`);
444
+ }
445
+ if (summary.within_scope_passed !== true) problems.push("summary.within_scope_passed is not true");
446
+ const runStatus = result.run?.status;
447
+ if (runStatus !== void 0 && runStatus !== "finished") {
448
+ problems.push(`run.status is "${String(runStatus)}", not finished`);
449
+ }
450
+ if (options.definitionVersion !== void 0 && summary.definition_version !== options.definitionVersion) {
451
+ problems.push("summary.definition_version does not match the contract");
452
+ }
453
+ if (typeof summary.records === "number" && Array.isArray(result.records) && summary.records !== result.records.length) {
454
+ problems.push(`summary.records (${summary.records}) differs from the ${result.records.length} records returned`);
455
+ }
456
+ return problems;
457
+ }
458
+
459
+ // src/catalog.ts
460
+ var PAGE_SIZE = 100;
461
+ var DEFAULT_WAIT_SECONDS = 90;
462
+ var REQUEST_HEADROOM_MS = 3e4;
463
+ function normalizeDomain(value) {
464
+ let host = value.trim().toLowerCase();
465
+ host = host.replace(/^[a-z][a-z0-9+.-]*:\/\//, "");
466
+ host = host.split(/[/?#]/)[0] ?? "";
467
+ host = host.replace(/:\d+$/, "").replace(/^www\./, "");
468
+ return host;
469
+ }
470
+ function matchesSite(api, site) {
471
+ const wanted = normalizeDomain(site);
472
+ if (wanted === "") return true;
473
+ const domain = normalizeDomain(api.site?.domain ?? "");
474
+ if (wanted.includes(".")) {
475
+ return domain === wanted || domain.endsWith(`.${wanted}`) || wanted.endsWith(`.${domain}`);
476
+ }
477
+ const name = (api.site?.name ?? "").toLowerCase();
478
+ return domain.includes(wanted) || name.includes(wanted) || api.name.toLowerCase().includes(wanted);
479
+ }
480
+ function normalizeField(field) {
481
+ return field.trim().toLowerCase();
482
+ }
483
+ var Catalog = class {
484
+ constructor(http) {
485
+ this.http = http;
486
+ }
487
+ http;
488
+ /** One page of the live catalog list. Anonymous. */
489
+ async list(params = {}) {
490
+ assertIntegerInRange("limit", params.limit, 1, 100);
491
+ assertIntegerInRange("offset", params.offset, 0);
492
+ const res = await this.http.request({
493
+ target: "catalog",
494
+ method: "GET",
495
+ path: "/api/catalog/apis",
496
+ query: { limit: params.limit, offset: params.offset, q: params.q, category: params.category, sort: params.sort },
497
+ signal: params.signal
498
+ });
499
+ const items = isRecord(res.body) && Array.isArray(res.body.items) ? res.body.items : [];
500
+ const limit = params.limit ?? 20;
501
+ const offset = params.offset ?? 0;
502
+ return { items, nextOffset: items.length >= limit ? offset + items.length : null };
503
+ }
504
+ /** Every published API, page by page. Offset paging reflects current content, not a snapshot. */
505
+ async *iterate(params = {}) {
506
+ let offset = 0;
507
+ while (offset !== null) {
508
+ const page = await this.list({ ...params, limit: PAGE_SIZE, offset });
509
+ for (const item of page.items) yield item;
510
+ offset = page.nextOffset;
511
+ }
512
+ }
513
+ /** The catalog index on cleanedweb.com (`catalog/index.json`): slugs, sources and key fields. */
514
+ async index(options = {}) {
515
+ const res = await this.http.request({
516
+ target: "site",
517
+ method: "GET",
518
+ path: "/catalog/index.json",
519
+ signal: options.signal
520
+ });
521
+ const body = isRecord(res.body) ? res.body : {};
522
+ return { ...body, apis: Array.isArray(body.apis) ? body.apis : [] };
523
+ }
524
+ /**
525
+ * One API with its endpoints and `active_version`. Takes an `api_id` (`generated_...`) or a
526
+ * listing slug (`github-topics-repositories-api`); a slug is resolved through the catalog index.
527
+ */
528
+ async get(apiIdOrSlug, options = {}) {
529
+ const slugLike = apiIdOrSlug.includes("-") && !apiIdOrSlug.startsWith("generated_");
530
+ if (slugLike) {
531
+ const entry = await this.findSlug(apiIdOrSlug, options.signal);
532
+ if (entry) return this.getById(entry.api_id, options.signal);
533
+ }
534
+ try {
535
+ return await this.getById(apiIdOrSlug, options.signal);
536
+ } catch (error) {
537
+ if (!slugLike && error instanceof NotFoundError) {
538
+ const entry = await this.findSlug(apiIdOrSlug, options.signal).catch(() => void 0);
539
+ if (entry) return this.getById(entry.api_id, options.signal);
540
+ }
541
+ throw error;
542
+ }
543
+ }
544
+ /**
545
+ * The contract of one endpoint: `input_schema`, `output_schema`, the run `path`, and the
546
+ * `definition_version`. `version` reads a catalog major's contract.
547
+ */
548
+ async contract(apiId, endpointId, options = {}) {
549
+ assertIntegerInRange("version", options.version, 1);
550
+ const res = await this.http.request({
551
+ target: "catalog",
552
+ method: "GET",
553
+ path: `/api/catalog/apis/${encodeSegment(apiId)}/endpoints/${encodeSegment(endpointId)}/contract`,
554
+ query: { version: options.version },
555
+ signal: options.signal
556
+ });
557
+ return res.body;
558
+ }
559
+ /**
560
+ * Finds published APIs by site and by the fields you need. This filters on the client: it reads
561
+ * the whole live list, and (for `fields`) the catalog index, whose `key_fields` are the fields a
562
+ * listing highlights. Read the contract before relying on a field it does not list.
563
+ */
564
+ async find(params = {}) {
565
+ const wantFields = (params.fields ?? []).map(normalizeField).filter(Boolean);
566
+ const listParams = compact({ q: params.q, category: params.category, signal: params.signal });
567
+ const indexPromise = this.index({ ...params.signal ? { signal: params.signal } : {} });
568
+ const indexOrNull = wantFields.length > 0 ? indexPromise : indexPromise.catch(() => null);
569
+ const [apis, index] = await Promise.all([this.collect(listParams), indexOrNull]);
570
+ const byId = new Map((index?.apis ?? []).map((entry) => [entry.api_id, entry]));
571
+ const matches = [];
572
+ for (const api of apis) {
573
+ if (params.site !== void 0 && !matchesSite(api, params.site)) continue;
574
+ const entry = byId.get(api.api_id);
575
+ const keyFields = entry?.key_fields?.map(normalizeField);
576
+ if (wantFields.length > 0 && !(keyFields && wantFields.every((f) => keyFields.includes(f)))) continue;
577
+ matches.push({
578
+ ...api,
579
+ ...entry?.slug ? { slug: entry.slug } : {},
580
+ ...entry?.url ? { pageUrl: entry.url } : {},
581
+ ...entry?.contract ? { contractUrl: entry.contract } : {},
582
+ ...entry?.key_fields ? { keyFields: entry.key_fields } : {}
583
+ });
584
+ }
585
+ return matches;
586
+ }
587
+ async run(apiId, endpointId, options = {}) {
588
+ assertIntegerInRange("maxItems", options.maxItems, 1);
589
+ assertIntegerInRange("timeoutSeconds", options.timeoutSeconds, 60, 86400);
590
+ assertIntegerInRange("waitSeconds", options.waitSeconds, 0, 86400);
591
+ assertIntegerInRange("version", options.version, 1);
592
+ const idempotencyKey = options.idempotencyKey ?? generateIdempotencyKey();
593
+ assertIdempotencyKey(idempotencyKey);
594
+ const root = this.root(apiId, endpointId);
595
+ const body = compact({
596
+ input: options.input ?? {},
597
+ max_items: options.maxItems,
598
+ timeout_seconds: options.timeoutSeconds,
599
+ wait_seconds: options.waitSeconds,
600
+ fields: options.fields,
601
+ nulls: options.nulls,
602
+ version: options.version
603
+ });
604
+ return withKey(idempotencyKey, async () => {
605
+ const first = await this.http.request({
606
+ target: "run",
607
+ method: "POST",
608
+ path: `${root}/run`,
609
+ body,
610
+ idempotencyKey,
611
+ signal: options.signal,
612
+ timeoutMs: Math.max(
613
+ this.http.config.timeoutMs,
614
+ (options.waitSeconds ?? DEFAULT_WAIT_SECONDS) * 1e3 + REQUEST_HEADROOM_MS
615
+ )
616
+ });
617
+ const outcome = await settle(first, {
618
+ http: this.http,
619
+ wait: options.wait !== false,
620
+ signal: options.signal,
621
+ onProgress: options.onProgress,
622
+ readResult: () => this.readResult(root, idempotencyKey, options.nulls, options.signal)
623
+ });
624
+ return { ...outcome, idempotencyKey };
625
+ });
626
+ }
627
+ async result(apiId, endpointId, options) {
628
+ const { idempotencyKey } = options;
629
+ assertIdempotencyKey(idempotencyKey);
630
+ const root = this.root(apiId, endpointId);
631
+ return withKey(idempotencyKey, async () => {
632
+ const first = await this.readResult(root, idempotencyKey, options.nulls, options.signal);
633
+ const outcome = await settle(first, {
634
+ http: this.http,
635
+ wait: options.wait !== false,
636
+ signal: options.signal,
637
+ onProgress: options.onProgress,
638
+ readResult: () => this.readResult(root, idempotencyKey, options.nulls, options.signal)
639
+ });
640
+ return { ...outcome, idempotencyKey };
641
+ });
642
+ }
643
+ root(apiId, endpointId) {
644
+ return `/v1/catalog/public/apis/${encodeSegment(apiId)}/endpoints/${encodeSegment(endpointId)}`;
645
+ }
646
+ readResult(root, idempotencyKey, nulls, signal) {
647
+ return this.http.request({
648
+ target: "run",
649
+ method: "GET",
650
+ path: `${root}/result`,
651
+ query: { nulls: nulls === true ? "true" : void 0 },
652
+ idempotencyKey,
653
+ signal
654
+ });
655
+ }
656
+ async getById(apiId, signal) {
657
+ const res = await this.http.request({
658
+ target: "catalog",
659
+ method: "GET",
660
+ path: `/api/catalog/apis/${encodeSegment(apiId)}`,
661
+ signal
662
+ });
663
+ return res.body;
664
+ }
665
+ async findSlug(slug, signal) {
666
+ const index = await this.index(signal ? { signal } : {});
667
+ return index.apis.find((entry) => entry.slug === slug);
668
+ }
669
+ async collect(params) {
670
+ const all = [];
671
+ for await (const api of this.iterate(params)) all.push(api);
672
+ return all;
673
+ }
674
+ };
675
+
676
+ // src/runs.ts
677
+ var DEFAULT_FOLLOW_MS = 2e3;
678
+ var TERMINAL = /* @__PURE__ */ new Set(["finished", "failed", "aborted"]);
679
+ var Runs = class {
680
+ constructor(http) {
681
+ this.http = http;
682
+ }
683
+ http;
684
+ /** The run object: `status`, `stop_reason`, `progress`, `units_charged`. */
685
+ async get(runId, options = {}) {
686
+ const res = await this.http.request({
687
+ target: "run",
688
+ method: "GET",
689
+ path: `/v1/runs/${encodeSegment(runId)}`,
690
+ signal: options.signal
691
+ });
692
+ return res.body;
693
+ }
694
+ /** The caller's runs, newest first. Pass `next_cursor` as `cursor` for the next page. */
695
+ async list(params = {}) {
696
+ assertIntegerInRange("limit", params.limit, 1);
697
+ const res = await this.http.request({
698
+ target: "run",
699
+ method: "GET",
700
+ path: "/v1/runs",
701
+ query: { source_id: params.sourceId, status: params.status, cursor: params.cursor, limit: params.limit },
702
+ signal: params.signal
703
+ });
704
+ const body = isRecord(res.body) ? res.body : {};
705
+ return {
706
+ runs: Array.isArray(body.runs) ? body.runs : [],
707
+ next_cursor: body.next_cursor ?? null
708
+ };
709
+ }
710
+ /** One page of records committed so far. Readable while the run runs. */
711
+ async itemsPage(runId, options = {}) {
712
+ assertIntegerInRange("limit", options.limit, 1);
713
+ const res = await this.http.request({
714
+ target: "run",
715
+ method: "GET",
716
+ path: `/v1/runs/${encodeSegment(runId)}/items`,
717
+ query: {
718
+ cursor: options.cursor ?? 0,
719
+ limit: options.limit,
720
+ nulls: options.nulls === true ? "true" : void 0
721
+ },
722
+ signal: options.signal
723
+ });
724
+ return res.body;
725
+ }
726
+ /** Every page of a run's records, following `next_cursor`. See {@link RunItemsOptions.follow}. */
727
+ async *pages(runId, options = {}) {
728
+ const follow = options.follow !== false;
729
+ const pollMs = options.pollIntervalMs ?? DEFAULT_FOLLOW_MS;
730
+ let cursor = options.cursor ?? 0;
731
+ for (; ; ) {
732
+ throwIfAborted(options.signal);
733
+ const page = await this.itemsPage(runId, { ...options, cursor });
734
+ yield page;
735
+ const next = page.next_cursor;
736
+ const items = Array.isArray(page.items) ? page.items : [];
737
+ if (next === null || next === void 0) return;
738
+ const stalled = items.length === 0 && String(next) === String(cursor);
739
+ if (stalled) {
740
+ if (TERMINAL.has(page.run_status) || !follow) return;
741
+ await this.http.sleep(pollMs, options.signal);
742
+ }
743
+ cursor = next;
744
+ }
745
+ }
746
+ /**
747
+ * Async iterator over a run's records, across cursor pages:
748
+ * `for await (const record of client.runs.items(runId)) { ... }`.
749
+ * While the run is still running it waits for later commits and ends once the run has ended and
750
+ * every record was read (pass `follow: false` to stop at what is committed now). Reading records
751
+ * never collects the website again.
752
+ */
753
+ async *items(runId, options = {}) {
754
+ for await (const page of this.pages(runId, options)) {
755
+ for (const item of page.items ?? []) yield item;
756
+ }
757
+ }
758
+ async result(runId, options = {}) {
759
+ const read = () => this.http.request({
760
+ target: "run",
761
+ method: "GET",
762
+ path: `/v1/runs/${encodeSegment(runId)}/result`,
763
+ query: { nulls: options.nulls === true ? "true" : void 0 },
764
+ signal: options.signal
765
+ });
766
+ try {
767
+ return await settle(await read(), {
768
+ http: this.http,
769
+ wait: options.wait !== false,
770
+ signal: options.signal,
771
+ onProgress: options.onProgress,
772
+ readResult: read
773
+ });
774
+ } catch (error) {
775
+ if (error instanceof Error && "runId" in error) error.runId ??= runId;
776
+ throw error;
777
+ }
778
+ }
779
+ /**
780
+ * Stops a run you no longer need. Answers `202` with the run object while it stops, or `200` if it
781
+ * had already ended. Records committed so far stay readable (and charged).
782
+ */
783
+ async abort(runId, options = {}) {
784
+ const res = await this.http.request({
785
+ target: "run",
786
+ method: "POST",
787
+ path: `/v1/runs/${encodeSegment(runId)}/abort`,
788
+ signal: options.signal
789
+ });
790
+ return res.body;
791
+ }
792
+ };
793
+
794
+ // src/sources.ts
795
+ var DEFAULT_WAIT_SECONDS2 = 90;
796
+ var REQUEST_HEADROOM_MS2 = 3e4;
797
+ var Sources = class {
798
+ constructor(http) {
799
+ this.http = http;
800
+ }
801
+ http;
802
+ async run(sourceId, options = {}) {
803
+ const { idempotencyKey, body, timeoutMs } = this.prepare(options);
804
+ return withKey(idempotencyKey, async () => {
805
+ const first = await this.http.request({
806
+ target: "run",
807
+ method: "POST",
808
+ path: `/v1/sources/${encodeSegment(sourceId)}/run`,
809
+ body,
810
+ idempotencyKey,
811
+ signal: options.signal,
812
+ timeoutMs
813
+ });
814
+ const outcome = await settle(first, {
815
+ http: this.http,
816
+ wait: options.wait !== false,
817
+ signal: options.signal,
818
+ onProgress: options.onProgress,
819
+ readResult: () => this.http.request({
820
+ target: "run",
821
+ method: "GET",
822
+ path: `/v1/runs/${encodeSegment(runIdOf(first))}/result`,
823
+ query: { nulls: options.nulls === true ? "true" : void 0 },
824
+ signal: options.signal
825
+ })
826
+ });
827
+ return { ...outcome, idempotencyKey };
828
+ });
829
+ }
830
+ /**
831
+ * Starts a run without waiting (`POST /v1/sources/{source_id}/runs`; `waitSeconds` defaults to 0).
832
+ * Read records with `runs.items(run.run_id)` and the final body with `runs.result(run.run_id)`.
833
+ */
834
+ async start(sourceId, options = {}) {
835
+ const { idempotencyKey, body, timeoutMs } = this.prepare(options);
836
+ return withKey(idempotencyKey, async () => {
837
+ const res = await this.http.request({
838
+ target: "run",
839
+ method: "POST",
840
+ path: `/v1/sources/${encodeSegment(sourceId)}/runs`,
841
+ body,
842
+ idempotencyKey,
843
+ signal: options.signal,
844
+ timeoutMs
845
+ });
846
+ const retry = res.headers.get("retry-after");
847
+ return {
848
+ run: res.body,
849
+ idempotencyKey,
850
+ replayed: res.headers.get("idempotent-replayed") === "true",
851
+ ...retry !== null && retry !== "" ? { retryAfter: Number(retry) } : {}
852
+ };
853
+ });
854
+ }
855
+ prepare(options) {
856
+ assertIntegerInRange("maxItems", options.maxItems, 1);
857
+ assertIntegerInRange("timeoutSeconds", options.timeoutSeconds, 60, 86400);
858
+ assertIntegerInRange("waitSeconds", options.waitSeconds, 0, 86400);
859
+ if (options.expectedVersion !== void 0 && options.definitionVersion !== void 0) {
860
+ throw new TypeError("Pass either expectedVersion or definitionVersion, not both.");
861
+ }
862
+ const idempotencyKey = options.idempotencyKey ?? generateIdempotencyKey();
863
+ assertIdempotencyKey(idempotencyKey);
864
+ const body = compact({
865
+ expected_version: options.expectedVersion,
866
+ definition_version: options.definitionVersion,
867
+ input: options.input ?? {},
868
+ endpoint: options.endpoint,
869
+ max_items: options.maxItems,
870
+ timeout_seconds: options.timeoutSeconds,
871
+ wait_seconds: options.waitSeconds,
872
+ fields: options.fields,
873
+ nulls: options.nulls
874
+ });
875
+ return {
876
+ idempotencyKey,
877
+ body,
878
+ timeoutMs: Math.max(this.http.config.timeoutMs, (options.waitSeconds ?? DEFAULT_WAIT_SECONDS2) * 1e3 + REQUEST_HEADROOM_MS2)
879
+ };
880
+ }
881
+ };
882
+ function runIdOf(res) {
883
+ const id = isRecord(res.body) ? res.body.run_id : void 0;
884
+ return typeof id === "string" ? id : "";
885
+ }
886
+
887
+ // src/client.ts
888
+ var DEFAULT_BASE_URL = "https://api.cleanedweb.com";
889
+ var DEFAULT_CATALOG_URL = "https://app.cleanedweb.com";
890
+ var DEFAULT_SITE_URL = "https://cleanedweb.com";
891
+ function envApiKey() {
892
+ const proc = globalThis.process;
893
+ const value = proc?.env?.CLEANEDWEB_API_KEY;
894
+ return value && value.trim() !== "" ? value.trim() : void 0;
895
+ }
896
+ function validateOrigin(name, value) {
897
+ let url;
898
+ try {
899
+ url = new URL(value);
900
+ } catch {
901
+ throw new ConfigurationError(`${name} must be an absolute URL, such as ${DEFAULT_BASE_URL}.`);
902
+ }
903
+ if (url.protocol !== "https:" && url.hostname !== "localhost" && url.hostname !== "127.0.0.1") {
904
+ throw new ConfigurationError(`${name} must be an https:// URL, so the API key is not sent in clear text.`);
905
+ }
906
+ return value.replace(/\/+$/, "");
907
+ }
908
+ var CleanedWeb = class {
909
+ /** Browse the public catalog and run published APIs. */
910
+ catalog;
911
+ /** Read and control runs by `run_id`. */
912
+ runs;
913
+ /** Run your own (private) source APIs. */
914
+ sources;
915
+ constructor(options = {}) {
916
+ const apiKey = "apiKey" in options ? options.apiKey : envApiKey();
917
+ const fetchImpl = options.fetch ?? globalThis.fetch;
918
+ if (typeof fetchImpl !== "function") {
919
+ throw new ConfigurationError("No global fetch found. Use Node 18+, or pass { fetch }.");
920
+ }
921
+ const maxRetries = options.maxRetries ?? 2;
922
+ if (!Number.isInteger(maxRetries) || maxRetries < 0) {
923
+ throw new RangeError("maxRetries must be a non-negative integer.");
924
+ }
925
+ const http = new HttpClient({
926
+ apiKey: apiKey && apiKey.trim() !== "" ? apiKey.trim() : void 0,
927
+ baseUrl: validateOrigin("baseUrl", options.baseUrl ?? DEFAULT_BASE_URL),
928
+ catalogUrl: validateOrigin("catalogUrl", options.catalogUrl ?? DEFAULT_CATALOG_URL),
929
+ siteUrl: validateOrigin("siteUrl", options.siteUrl ?? DEFAULT_SITE_URL),
930
+ // Call through globalThis so a bare global fetch keeps its receiver in browsers and workers.
931
+ fetch: options.fetch ?? ((input, init) => fetchImpl.call(globalThis, input, init)),
932
+ maxRetries,
933
+ timeoutMs: options.timeoutMs ?? 12e4,
934
+ maxRetryDelayMs: options.maxRetryDelayMs ?? 3e4,
935
+ sleep: options.sleep ?? defaultSleep
936
+ });
937
+ this.catalog = new Catalog(http);
938
+ this.runs = new Runs(http);
939
+ this.sources = new Sources(http);
940
+ }
941
+ };
942
+
943
+ // src/accept.ts
944
+ function isAccepted(result, options) {
945
+ return acceptanceProblems(result, options).length === 0;
946
+ }
947
+ function assertAccepted(result, options) {
948
+ const problems = acceptanceProblems(result, options);
949
+ if (problems.length > 0) throw new ResultNotAcceptedError(problems);
950
+ }
951
+
952
+ export { APIError, AuthenticationError, Catalog, CleanedWeb, CleanedWebAbortError, CleanedWebConnectionError, CleanedWebError, CleanedWebTimeoutError, ConfigurationError, DEFAULT_BASE_URL, DEFAULT_CATALOG_URL, DEFAULT_SITE_URL, IdempotencyConflictError, InsufficientCreditError, NotFoundError, PermissionError, RateLimitError, ResultNotAcceptedError, ResultNotReadyError, RunFailedError, Runs, ServerError, Sources, UsageBudgetExceededError, ValidationError, VersionConflictError, assertAccepted, generateIdempotencyKey, isAccepted };
953
+ //# sourceMappingURL=index.js.map
954
+ //# sourceMappingURL=index.js.map