cleanedweb 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/LICENSE +21 -0
- package/README.md +223 -2
- package/dist/index.cjs +984 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +775 -0
- package/dist/index.d.ts +775 -0
- package/dist/index.js +954 -0
- package/dist/index.js.map +1 -0
- package/package.json +61 -4
package/dist/index.d.cts
ADDED
|
@@ -0,0 +1,775 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Typed errors. Every error the SDK throws extends {@link CleanedWebError}.
|
|
3
|
+
*
|
|
4
|
+
* Errors never carry request headers, so the API key is not exposed through them.
|
|
5
|
+
* `idempotencyKey` and `runId` are filled in when known, so a caller can recover a run
|
|
6
|
+
* after a failure without having kept the key itself.
|
|
7
|
+
*/
|
|
8
|
+
/** Deprecation headers sent when a pinned catalog major is deprecated or was rerouted. */
|
|
9
|
+
interface DeprecationInfo {
|
|
10
|
+
/** `Deprecation: @<unix seconds>` (RFC 9745 date). */
|
|
11
|
+
deprecation: string | null;
|
|
12
|
+
/** `Sunset: <HTTP date>`, when the major has a sunset date. */
|
|
13
|
+
sunset: string | null;
|
|
14
|
+
/** `Link: <...>; rel="deprecation"`, the listing's changelog. */
|
|
15
|
+
link: string | null;
|
|
16
|
+
}
|
|
17
|
+
declare class CleanedWebError extends Error {
|
|
18
|
+
/** The idempotency key of the run this error belongs to, when there is one. */
|
|
19
|
+
idempotencyKey?: string;
|
|
20
|
+
/** The run this error belongs to, when known. */
|
|
21
|
+
runId?: string;
|
|
22
|
+
constructor(message: string, options?: {
|
|
23
|
+
cause?: unknown;
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
/** The SDK was used with missing or unusable configuration (no API key, wrong origin). */
|
|
27
|
+
declare class ConfigurationError extends CleanedWebError {
|
|
28
|
+
}
|
|
29
|
+
/** No response arrived (connection lost, DNS, TLS). The run may have started: keep the key. */
|
|
30
|
+
declare class CleanedWebConnectionError extends CleanedWebError {
|
|
31
|
+
}
|
|
32
|
+
/** The client-side time limit passed before a response arrived. The run is not stopped. */
|
|
33
|
+
declare class CleanedWebTimeoutError extends CleanedWebConnectionError {
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The caller's `AbortSignal` fired. `name` is `AbortError`. Aborting only stops the SDK from
|
|
37
|
+
* waiting: a run that was already admitted keeps running and is charged. Use `runs.abort` to
|
|
38
|
+
* stop it, and `cause` holds the signal's reason.
|
|
39
|
+
*/
|
|
40
|
+
declare class CleanedWebAbortError extends CleanedWebError {
|
|
41
|
+
constructor(cause?: unknown);
|
|
42
|
+
}
|
|
43
|
+
/** A finished run did not pass the acceptance checks of {@link assertAccepted}. */
|
|
44
|
+
declare class ResultNotAcceptedError extends CleanedWebError {
|
|
45
|
+
readonly reasons: string[];
|
|
46
|
+
constructor(reasons: string[]);
|
|
47
|
+
}
|
|
48
|
+
interface APIErrorInit {
|
|
49
|
+
status: number;
|
|
50
|
+
code?: string | undefined;
|
|
51
|
+
message: string;
|
|
52
|
+
detail?: unknown;
|
|
53
|
+
body?: unknown;
|
|
54
|
+
retryAfter?: number | undefined;
|
|
55
|
+
deprecation?: DeprecationInfo | undefined;
|
|
56
|
+
}
|
|
57
|
+
/** The service answered with an error status. */
|
|
58
|
+
declare class APIError extends CleanedWebError {
|
|
59
|
+
/** HTTP status. */
|
|
60
|
+
readonly status: number;
|
|
61
|
+
/** The documented error `code` (for example `insufficient_units`), when the body has one. */
|
|
62
|
+
readonly code: string | undefined;
|
|
63
|
+
/** The raw `detail` of the body: text, an object, or a list of validation issues. */
|
|
64
|
+
readonly detail: unknown;
|
|
65
|
+
/** The parsed body (or the text, if it was not JSON). Never contains the API key. */
|
|
66
|
+
readonly body: unknown;
|
|
67
|
+
/** Seconds to wait before retrying, from `Retry-After` or the body. */
|
|
68
|
+
readonly retryAfter: number | undefined;
|
|
69
|
+
/** Deprecation headers, when the answer carried them. */
|
|
70
|
+
readonly deprecation: DeprecationInfo | undefined;
|
|
71
|
+
constructor(init: APIErrorInit);
|
|
72
|
+
}
|
|
73
|
+
/** 401: no credential, or an invalid, expired or revoked key. */
|
|
74
|
+
declare class AuthenticationError extends APIError {
|
|
75
|
+
}
|
|
76
|
+
/** 403: the key lacks the `run` scope or is not tied to a workspace account. */
|
|
77
|
+
declare class PermissionError extends APIError {
|
|
78
|
+
}
|
|
79
|
+
/** 402 `insufficient_units`: the balance cannot fund the run. */
|
|
80
|
+
declare class InsufficientCreditError extends APIError {
|
|
81
|
+
/** Units the run needs. */
|
|
82
|
+
readonly required: unknown;
|
|
83
|
+
/** Units the balance has. */
|
|
84
|
+
readonly available: unknown;
|
|
85
|
+
constructor(init: APIErrorInit);
|
|
86
|
+
}
|
|
87
|
+
/** 409 `usage_budget_exceeded`: the workspace's monthly usage budget cannot cover the request. */
|
|
88
|
+
declare class UsageBudgetExceededError extends APIError {
|
|
89
|
+
}
|
|
90
|
+
/** 409 `delivery_identity_conflict`: the idempotency key was already used for another request. */
|
|
91
|
+
declare class IdempotencyConflictError extends APIError {
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* 409 version conflict: the active definition differs from `expectedVersion`
|
|
95
|
+
* (`builder_definition_version_conflict`), or the endpoint's active version changed
|
|
96
|
+
* (`catalog_version_conflict`).
|
|
97
|
+
*/
|
|
98
|
+
declare class VersionConflictError extends APIError {
|
|
99
|
+
}
|
|
100
|
+
/** 409 `result_not_available`, `result_not_complete` or `result_not_running`: retry the same key. */
|
|
101
|
+
declare class ResultNotReadyError extends APIError {
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* 404: API, endpoint, run or delivery not found. For an unknown catalog major
|
|
105
|
+
* (`catalog_version_not_found`) `available` lists the majors that answer.
|
|
106
|
+
*/
|
|
107
|
+
declare class NotFoundError extends APIError {
|
|
108
|
+
readonly available: number[] | undefined;
|
|
109
|
+
constructor(init: APIErrorInit);
|
|
110
|
+
}
|
|
111
|
+
/** 400, 415 or 422: an input, run option or query the service does not accept. Do not retry unchanged. */
|
|
112
|
+
declare class ValidationError extends APIError {
|
|
113
|
+
}
|
|
114
|
+
/** 429. `retryAfter` is the number of seconds to wait, when the service said so. */
|
|
115
|
+
declare class RateLimitError extends APIError {
|
|
116
|
+
}
|
|
117
|
+
/** 5xx without a failed-run marker. The run may not have started; retry with the same key. */
|
|
118
|
+
declare class ServerError extends APIError {
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* The run itself failed (`detail.error_code` is `run_failed` or `failed_run_replayed`). This is final
|
|
122
|
+
* for the idempotency key for 7 days: send a new key to run again. Records saved before the
|
|
123
|
+
* failure stay readable through `runs.items(runId)`.
|
|
124
|
+
*/
|
|
125
|
+
declare class RunFailedError extends APIError {
|
|
126
|
+
/** `run_failed` for the failure itself, `failed_run_replayed` when the key was replayed. */
|
|
127
|
+
readonly errorCode: 'run_failed' | 'failed_run_replayed';
|
|
128
|
+
/** Whether this answer replays the key. */
|
|
129
|
+
readonly replayed: boolean;
|
|
130
|
+
constructor(init: APIErrorInit & {
|
|
131
|
+
errorCode: 'run_failed' | 'failed_run_replayed';
|
|
132
|
+
replayed: boolean;
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/** A fresh idempotency key (a UUID v4). */
|
|
137
|
+
declare function generateIdempotencyKey(): string;
|
|
138
|
+
type Sleep = (ms: number, signal?: AbortSignal) => Promise<void>;
|
|
139
|
+
|
|
140
|
+
type FetchLike = (input: string, init?: RequestInit) => Promise<Response>;
|
|
141
|
+
interface HttpConfig {
|
|
142
|
+
apiKey: string | undefined;
|
|
143
|
+
/** Run origin (runs, results, run routes). */
|
|
144
|
+
baseUrl: string;
|
|
145
|
+
/** Live catalog origin (anonymous reads). */
|
|
146
|
+
catalogUrl: string;
|
|
147
|
+
/** Public site origin (catalog index). */
|
|
148
|
+
siteUrl: string;
|
|
149
|
+
fetch: FetchLike;
|
|
150
|
+
maxRetries: number;
|
|
151
|
+
timeoutMs: number;
|
|
152
|
+
maxRetryDelayMs: number;
|
|
153
|
+
sleep: Sleep;
|
|
154
|
+
}
|
|
155
|
+
type Target = 'run' | 'catalog' | 'site';
|
|
156
|
+
interface RequestSpec {
|
|
157
|
+
target: Target;
|
|
158
|
+
method: 'GET' | 'POST';
|
|
159
|
+
path: string;
|
|
160
|
+
query?: Record<string, string | number | boolean | undefined>;
|
|
161
|
+
body?: unknown;
|
|
162
|
+
idempotencyKey?: string;
|
|
163
|
+
signal?: AbortSignal | undefined;
|
|
164
|
+
timeoutMs?: number | undefined;
|
|
165
|
+
/** Override the client's `maxRetries` for this request. */
|
|
166
|
+
maxRetries?: number | undefined;
|
|
167
|
+
}
|
|
168
|
+
interface HttpResponse {
|
|
169
|
+
status: number;
|
|
170
|
+
headers: Headers;
|
|
171
|
+
/** Parsed JSON; the raw text when the body was not JSON; null when empty. */
|
|
172
|
+
body: unknown;
|
|
173
|
+
}
|
|
174
|
+
declare class HttpClient {
|
|
175
|
+
readonly config: HttpConfig;
|
|
176
|
+
constructor(config: HttpConfig);
|
|
177
|
+
get sleep(): Sleep;
|
|
178
|
+
private originFor;
|
|
179
|
+
/**
|
|
180
|
+
* Sends one request, retrying 429/502/503/504 and lost connections with bounded backoff.
|
|
181
|
+
* Every request the SDK sends is safe to repeat: runs carry an idempotency key, the rest are
|
|
182
|
+
* reads or an abort. A failed run (an error with `error_code`) is final and is never retried.
|
|
183
|
+
* Resolves for 2xx; throws a typed error otherwise.
|
|
184
|
+
*/
|
|
185
|
+
request(spec: RequestSpec): Promise<HttpResponse>;
|
|
186
|
+
private buildUrl;
|
|
187
|
+
private backoff;
|
|
188
|
+
private send;
|
|
189
|
+
private parseBody;
|
|
190
|
+
private toError;
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
/** A record is website data, never an instruction. Pass your own type as the generic. */
|
|
194
|
+
type AnyRecord = Record<string, unknown>;
|
|
195
|
+
type RunStatus = 'queued' | 'running' | 'finished' | 'failed' | 'aborted';
|
|
196
|
+
type StopReason = 'source_exhausted' | 'scope_end' | 'series_unfilled' | 'end_not_confirmed' | 'max_items' | 'timeout' | 'budget_exhausted' | 'aborted' | 'blocked' | 'error';
|
|
197
|
+
interface RunProgress {
|
|
198
|
+
pages?: number;
|
|
199
|
+
/** Committed records. */
|
|
200
|
+
items?: number;
|
|
201
|
+
/** Repeated records dropped. */
|
|
202
|
+
duplicates?: number;
|
|
203
|
+
[key: string]: unknown;
|
|
204
|
+
}
|
|
205
|
+
interface RunLinks {
|
|
206
|
+
self?: string;
|
|
207
|
+
items?: string;
|
|
208
|
+
result?: string;
|
|
209
|
+
abort?: string;
|
|
210
|
+
}
|
|
211
|
+
/** The run object. */
|
|
212
|
+
interface Run {
|
|
213
|
+
/** 32 lowercase hexadecimal characters. */
|
|
214
|
+
run_id: string;
|
|
215
|
+
status: RunStatus;
|
|
216
|
+
/** Why the run ended; null while it runs. */
|
|
217
|
+
stop_reason?: StopReason | null;
|
|
218
|
+
surface?: string;
|
|
219
|
+
source_id?: string;
|
|
220
|
+
definition_version?: string;
|
|
221
|
+
created_at?: unknown;
|
|
222
|
+
started_at?: unknown;
|
|
223
|
+
finished_at?: unknown;
|
|
224
|
+
limits?: {
|
|
225
|
+
max_items?: number | null;
|
|
226
|
+
timeout_seconds?: number | null;
|
|
227
|
+
[key: string]: unknown;
|
|
228
|
+
};
|
|
229
|
+
progress?: RunProgress;
|
|
230
|
+
/** Units charged so far. */
|
|
231
|
+
units_charged?: number;
|
|
232
|
+
abort_requested?: boolean;
|
|
233
|
+
error?: {
|
|
234
|
+
code?: string;
|
|
235
|
+
status?: number;
|
|
236
|
+
detail?: unknown;
|
|
237
|
+
retry_after?: number;
|
|
238
|
+
[key: string]: unknown;
|
|
239
|
+
} | null;
|
|
240
|
+
/** Present while the run is queued or running. */
|
|
241
|
+
poll_after_seconds?: number;
|
|
242
|
+
links?: RunLinks;
|
|
243
|
+
[key: string]: unknown;
|
|
244
|
+
}
|
|
245
|
+
interface RunSummary {
|
|
246
|
+
source_id?: string;
|
|
247
|
+
definition_version?: string;
|
|
248
|
+
/** `bounded` and `partial` mean the source can hold more records than you received. */
|
|
249
|
+
status: 'completed' | 'bounded' | 'partial';
|
|
250
|
+
within_scope_passed?: boolean;
|
|
251
|
+
/** A run does not establish full-site coverage. */
|
|
252
|
+
catalog_complete?: boolean;
|
|
253
|
+
/** Number of records in the run's dataset. */
|
|
254
|
+
records?: number;
|
|
255
|
+
[key: string]: unknown;
|
|
256
|
+
}
|
|
257
|
+
interface Delivery {
|
|
258
|
+
runId: string;
|
|
259
|
+
/** The same identifier as `runId`. */
|
|
260
|
+
chargeReceiptId?: string;
|
|
261
|
+
recordCount?: number;
|
|
262
|
+
}
|
|
263
|
+
/** The body of a run that ended. */
|
|
264
|
+
interface RunResultBody<T = AnyRecord> {
|
|
265
|
+
records: T[];
|
|
266
|
+
summary: RunSummary;
|
|
267
|
+
metadata?: {
|
|
268
|
+
definition_version?: string;
|
|
269
|
+
[key: string]: unknown;
|
|
270
|
+
};
|
|
271
|
+
artifacts?: unknown;
|
|
272
|
+
traceId?: string | null;
|
|
273
|
+
run: Run;
|
|
274
|
+
/** Present for a customer workspace run. */
|
|
275
|
+
delivery?: Delivery;
|
|
276
|
+
/** The catalog major and minor that ran, such as `"2.3"`. */
|
|
277
|
+
version?: string;
|
|
278
|
+
/** Set when a pinned major was sunset or withdrawn and the call ran on the latest major. */
|
|
279
|
+
version_requested?: number;
|
|
280
|
+
rerouted?: string;
|
|
281
|
+
[key: string]: unknown;
|
|
282
|
+
}
|
|
283
|
+
/** A run that ended (read through the run routes, where no idempotency key is involved). */
|
|
284
|
+
interface RunResult<T = AnyRecord> extends RunResultBody<T> {
|
|
285
|
+
pending: false;
|
|
286
|
+
/** Set when the pinned major is deprecated or was rerouted. */
|
|
287
|
+
deprecation?: DeprecationInfo;
|
|
288
|
+
}
|
|
289
|
+
/** A run that is queued or still running. */
|
|
290
|
+
interface RunPending {
|
|
291
|
+
pending: true;
|
|
292
|
+
run: Run;
|
|
293
|
+
/** Seconds the service asked to wait before reading the result. */
|
|
294
|
+
retryAfter?: number | undefined;
|
|
295
|
+
deprecation?: DeprecationInfo;
|
|
296
|
+
}
|
|
297
|
+
/** A run that ended: the body, plus the idempotency key it ran under. */
|
|
298
|
+
interface CompletedRun<T = AnyRecord> extends RunResult<T> {
|
|
299
|
+
/** The idempotency key of this run. Keep it: it reads this run's result again. */
|
|
300
|
+
idempotencyKey: string;
|
|
301
|
+
}
|
|
302
|
+
/** A run that is queued or still running (returned only with `wait: false`). */
|
|
303
|
+
interface PendingRun extends RunPending {
|
|
304
|
+
idempotencyKey: string;
|
|
305
|
+
}
|
|
306
|
+
type RunOutcome<T = AnyRecord> = CompletedRun<T> | PendingRun;
|
|
307
|
+
/** Options shared by every call that starts or reads a run. */
|
|
308
|
+
interface RunCallOptions {
|
|
309
|
+
/** Cancels the SDK's waiting. It does not stop a run that already started: use `runs.abort`. */
|
|
310
|
+
signal?: AbortSignal;
|
|
311
|
+
/** Called with the run object each time the service answers 202 while waiting. */
|
|
312
|
+
onProgress?: (run: Run) => void;
|
|
313
|
+
}
|
|
314
|
+
/** Options that shape a run, as in the request body. */
|
|
315
|
+
interface RunOptions {
|
|
316
|
+
/** Inputs from the API's contract. Unknown fields are rejected. */
|
|
317
|
+
input?: Record<string, unknown>;
|
|
318
|
+
/** Stop after this many de-duplicated records. Start small while you check the record shape. */
|
|
319
|
+
maxItems?: number;
|
|
320
|
+
/** The run's own time budget in seconds (60-86400, default 10800). */
|
|
321
|
+
timeoutSeconds?: number;
|
|
322
|
+
/** How long the first request waits for the run to end before answering 202 (0-86400, default 90). */
|
|
323
|
+
waitSeconds?: number;
|
|
324
|
+
/** Return only these published output fields; required and identity fields are always kept. */
|
|
325
|
+
fields?: string[];
|
|
326
|
+
/** Send every published field, with absent values as `null`. */
|
|
327
|
+
nulls?: boolean;
|
|
328
|
+
/**
|
|
329
|
+
* Save this before you send and reuse it only to recover this same request. Default: a new
|
|
330
|
+
* UUID, returned on the result. 1-200 characters: letters, digits, `.`, `_`, `:`, `-`.
|
|
331
|
+
*/
|
|
332
|
+
idempotencyKey?: string;
|
|
333
|
+
/**
|
|
334
|
+
* `true` (default): keep reading the result after a 202 until the run ends.
|
|
335
|
+
* `false`: return a {@link PendingRun} on 202.
|
|
336
|
+
*/
|
|
337
|
+
wait?: boolean;
|
|
338
|
+
}
|
|
339
|
+
interface CatalogRunOptions extends RunOptions, RunCallOptions {
|
|
340
|
+
/** Pin a catalog major (integer). Leave out for the latest major. */
|
|
341
|
+
version?: number;
|
|
342
|
+
}
|
|
343
|
+
interface SourceRunOptions extends RunOptions, RunCallOptions {
|
|
344
|
+
/** Require the active definition to match this 64-character lowercase SHA-256 version. */
|
|
345
|
+
expectedVersion?: string;
|
|
346
|
+
/** Select an accepted immutable revision. Use instead of `expectedVersion`. */
|
|
347
|
+
definitionVersion?: string;
|
|
348
|
+
/** The endpoint; the supported value is `search`. */
|
|
349
|
+
endpoint?: string;
|
|
350
|
+
}
|
|
351
|
+
interface CatalogResultOptions extends RunCallOptions {
|
|
352
|
+
/** The idempotency key the run was started with. */
|
|
353
|
+
idempotencyKey: string;
|
|
354
|
+
/** Keep reading until the run ends (default `true`). */
|
|
355
|
+
wait?: boolean;
|
|
356
|
+
/** Same view as `nulls: true` on the run. */
|
|
357
|
+
nulls?: boolean;
|
|
358
|
+
}
|
|
359
|
+
interface RunResultOptions extends RunCallOptions {
|
|
360
|
+
/** Keep reading until the run ends (default `true`). */
|
|
361
|
+
wait?: boolean;
|
|
362
|
+
nulls?: boolean;
|
|
363
|
+
}
|
|
364
|
+
interface RunItemsPage<T = AnyRecord> {
|
|
365
|
+
run_id: string;
|
|
366
|
+
items: T[];
|
|
367
|
+
count: number;
|
|
368
|
+
/** Null once the run has ended and every record was read. */
|
|
369
|
+
next_cursor: string | number | null;
|
|
370
|
+
run_status: RunStatus;
|
|
371
|
+
[key: string]: unknown;
|
|
372
|
+
}
|
|
373
|
+
interface RunItemsOptions {
|
|
374
|
+
/** Start position (default 0). */
|
|
375
|
+
cursor?: string | number;
|
|
376
|
+
/** Page size (default 100). */
|
|
377
|
+
limit?: number;
|
|
378
|
+
nulls?: boolean;
|
|
379
|
+
/**
|
|
380
|
+
* While the run is queued or running, keep waiting for records committed later (default
|
|
381
|
+
* `true`). With `false` the iterator ends when the committed records are read.
|
|
382
|
+
*/
|
|
383
|
+
follow?: boolean;
|
|
384
|
+
/** How long to wait between reads while following a running run (default 2000 ms). */
|
|
385
|
+
pollIntervalMs?: number;
|
|
386
|
+
signal?: AbortSignal;
|
|
387
|
+
}
|
|
388
|
+
interface RunListParams {
|
|
389
|
+
sourceId?: string;
|
|
390
|
+
status?: RunStatus;
|
|
391
|
+
cursor?: string | number;
|
|
392
|
+
/** Default 50. */
|
|
393
|
+
limit?: number;
|
|
394
|
+
signal?: AbortSignal;
|
|
395
|
+
}
|
|
396
|
+
interface RunList {
|
|
397
|
+
runs: Run[];
|
|
398
|
+
next_cursor: string | number | null;
|
|
399
|
+
}
|
|
400
|
+
interface CatalogVersionInfo {
|
|
401
|
+
major: number;
|
|
402
|
+
latest_minor: string;
|
|
403
|
+
status: string;
|
|
404
|
+
sunset_at: string | number | null;
|
|
405
|
+
}
|
|
406
|
+
interface CatalogEndpoint {
|
|
407
|
+
endpoint_id: string;
|
|
408
|
+
name: string;
|
|
409
|
+
description?: string;
|
|
410
|
+
/** `available` when the endpoint can be run. */
|
|
411
|
+
status: string;
|
|
412
|
+
acquisition?: string;
|
|
413
|
+
/** The 64-character definition version (only on `catalog.get`). */
|
|
414
|
+
active_version?: string;
|
|
415
|
+
/** The newest minor of the latest major, such as `1.1` (only on `catalog.get`). */
|
|
416
|
+
version?: string;
|
|
417
|
+
versions?: CatalogVersionInfo[];
|
|
418
|
+
changelog?: Array<{
|
|
419
|
+
version: string;
|
|
420
|
+
date: string;
|
|
421
|
+
kind: string;
|
|
422
|
+
changes: unknown[];
|
|
423
|
+
}>;
|
|
424
|
+
}
|
|
425
|
+
interface CatalogSite {
|
|
426
|
+
name: string;
|
|
427
|
+
domain: string;
|
|
428
|
+
description?: string;
|
|
429
|
+
icon_url?: string | null;
|
|
430
|
+
}
|
|
431
|
+
interface CatalogApi {
|
|
432
|
+
api_id: string;
|
|
433
|
+
name: string;
|
|
434
|
+
short_description: string;
|
|
435
|
+
category: string;
|
|
436
|
+
record_types: string[];
|
|
437
|
+
/** Empty means coverage is unspecified, not worldwide. */
|
|
438
|
+
countries: string[];
|
|
439
|
+
/** Empty means coverage is unspecified. */
|
|
440
|
+
languages: string[];
|
|
441
|
+
tags: string[];
|
|
442
|
+
/** Read these before relying on an API for a whole site. */
|
|
443
|
+
limitations: string[];
|
|
444
|
+
site: CatalogSite;
|
|
445
|
+
endpoints: CatalogEndpoint[];
|
|
446
|
+
api_type?: string;
|
|
447
|
+
visibility?: string;
|
|
448
|
+
updated_at?: number;
|
|
449
|
+
acquisition?: string;
|
|
450
|
+
/** `unknown` until a health contract reports evidence. */
|
|
451
|
+
health?: string;
|
|
452
|
+
last_verified_at?: number | null;
|
|
453
|
+
reliability?: {
|
|
454
|
+
window_days?: number;
|
|
455
|
+
runs?: number;
|
|
456
|
+
succeeded?: number;
|
|
457
|
+
last_run_at?: number | null;
|
|
458
|
+
};
|
|
459
|
+
}
|
|
460
|
+
type CatalogSort = 'updated' | 'name';
|
|
461
|
+
interface CatalogListParams {
|
|
462
|
+
/** 1-100 (default 20). */
|
|
463
|
+
limit?: number;
|
|
464
|
+
/** Starts at 0. */
|
|
465
|
+
offset?: number;
|
|
466
|
+
/** Searches text across the whole published inventory. */
|
|
467
|
+
q?: string;
|
|
468
|
+
category?: string;
|
|
469
|
+
sort?: CatalogSort;
|
|
470
|
+
signal?: AbortSignal;
|
|
471
|
+
}
|
|
472
|
+
interface CatalogPage {
|
|
473
|
+
items: CatalogApi[];
|
|
474
|
+
/** The offset of the next page, or null when this page was not full. */
|
|
475
|
+
nextOffset: number | null;
|
|
476
|
+
}
|
|
477
|
+
/** One entry of `https://cleanedweb.com/catalog/index.json`. */
|
|
478
|
+
interface CatalogIndexEntry {
|
|
479
|
+
name: string;
|
|
480
|
+
api_id: string;
|
|
481
|
+
slug: string;
|
|
482
|
+
url: string;
|
|
483
|
+
description?: string;
|
|
484
|
+
source?: {
|
|
485
|
+
name: string;
|
|
486
|
+
domain: string;
|
|
487
|
+
url: string;
|
|
488
|
+
};
|
|
489
|
+
category?: string;
|
|
490
|
+
countries?: string[];
|
|
491
|
+
languages?: string[];
|
|
492
|
+
/** The fields the listing highlights. Not the full output schema. */
|
|
493
|
+
key_fields?: string[];
|
|
494
|
+
updated?: string;
|
|
495
|
+
markdown?: string;
|
|
496
|
+
contract?: string;
|
|
497
|
+
endpoints?: Array<{
|
|
498
|
+
endpoint_id: string;
|
|
499
|
+
name: string;
|
|
500
|
+
major?: number;
|
|
501
|
+
run?: string;
|
|
502
|
+
}>;
|
|
503
|
+
[key: string]: unknown;
|
|
504
|
+
}
|
|
505
|
+
interface CatalogIndex {
|
|
506
|
+
name?: string;
|
|
507
|
+
count?: number;
|
|
508
|
+
updated?: string;
|
|
509
|
+
apis: CatalogIndexEntry[];
|
|
510
|
+
[key: string]: unknown;
|
|
511
|
+
}
|
|
512
|
+
interface CatalogContract {
|
|
513
|
+
endpoint_id: string;
|
|
514
|
+
/** The immutable 64-character version a run uses. */
|
|
515
|
+
definition_version: string;
|
|
516
|
+
method: string;
|
|
517
|
+
/** The run request path, sent to the run origin. */
|
|
518
|
+
path: string;
|
|
519
|
+
input_schema: Record<string, unknown>;
|
|
520
|
+
output_schema: Record<string, unknown>;
|
|
521
|
+
parameters?: unknown[];
|
|
522
|
+
examples?: unknown;
|
|
523
|
+
authentication?: unknown;
|
|
524
|
+
limits?: unknown;
|
|
525
|
+
pagination?: unknown;
|
|
526
|
+
billing?: unknown;
|
|
527
|
+
version?: string;
|
|
528
|
+
versions?: CatalogVersionInfo[];
|
|
529
|
+
changelog?: unknown[];
|
|
530
|
+
[key: string]: unknown;
|
|
531
|
+
}
|
|
532
|
+
interface CatalogFindParams {
|
|
533
|
+
/** A domain (`github.com`), a URL, or part of the site's name. Subdomains match. */
|
|
534
|
+
site?: string;
|
|
535
|
+
/**
|
|
536
|
+
* Output fields you need. Matched against each API's `key_fields` in the catalog index, which
|
|
537
|
+
* lists the fields a listing highlights: an API can carry more fields than it lists.
|
|
538
|
+
*/
|
|
539
|
+
fields?: string[];
|
|
540
|
+
/** Also narrow by the live list's text search. */
|
|
541
|
+
q?: string;
|
|
542
|
+
category?: string;
|
|
543
|
+
signal?: AbortSignal;
|
|
544
|
+
}
|
|
545
|
+
/** A live catalog entry joined with the catalog index. */
|
|
546
|
+
interface CatalogMatch extends CatalogApi {
|
|
547
|
+
/** The listing's slug, when the index has it. */
|
|
548
|
+
slug?: string;
|
|
549
|
+
/** The listing's page on cleanedweb.com. */
|
|
550
|
+
pageUrl?: string;
|
|
551
|
+
/** The listing's `contract.json`. */
|
|
552
|
+
contractUrl?: string;
|
|
553
|
+
/** Fields the listing highlights. */
|
|
554
|
+
keyFields?: string[];
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
/** Browse the public catalog (no API key) and run published catalog APIs. */
|
|
558
|
+
declare class Catalog {
|
|
559
|
+
private readonly http;
|
|
560
|
+
constructor(http: HttpClient);
|
|
561
|
+
/** One page of the live catalog list. Anonymous. */
|
|
562
|
+
list(params?: CatalogListParams): Promise<CatalogPage>;
|
|
563
|
+
/** Every published API, page by page. Offset paging reflects current content, not a snapshot. */
|
|
564
|
+
iterate(params?: Omit<CatalogListParams, 'limit' | 'offset'>): AsyncGenerator<CatalogApi, void, undefined>;
|
|
565
|
+
/** The catalog index on cleanedweb.com (`catalog/index.json`): slugs, sources and key fields. */
|
|
566
|
+
index(options?: {
|
|
567
|
+
signal?: AbortSignal;
|
|
568
|
+
}): Promise<CatalogIndex>;
|
|
569
|
+
/**
|
|
570
|
+
* One API with its endpoints and `active_version`. Takes an `api_id` (`generated_...`) or a
|
|
571
|
+
* listing slug (`github-topics-repositories-api`); a slug is resolved through the catalog index.
|
|
572
|
+
*/
|
|
573
|
+
get(apiIdOrSlug: string, options?: {
|
|
574
|
+
signal?: AbortSignal;
|
|
575
|
+
}): Promise<CatalogApi>;
|
|
576
|
+
/**
|
|
577
|
+
* The contract of one endpoint: `input_schema`, `output_schema`, the run `path`, and the
|
|
578
|
+
* `definition_version`. `version` reads a catalog major's contract.
|
|
579
|
+
*/
|
|
580
|
+
contract(apiId: string, endpointId: string, options?: {
|
|
581
|
+
version?: number;
|
|
582
|
+
signal?: AbortSignal;
|
|
583
|
+
}): Promise<CatalogContract>;
|
|
584
|
+
/**
|
|
585
|
+
* Finds published APIs by site and by the fields you need. This filters on the client: it reads
|
|
586
|
+
* the whole live list, and (for `fields`) the catalog index, whose `key_fields` are the fields a
|
|
587
|
+
* listing highlights. Read the contract before relying on a field it does not list.
|
|
588
|
+
*/
|
|
589
|
+
find(params?: CatalogFindParams): Promise<CatalogMatch[]>;
|
|
590
|
+
/**
|
|
591
|
+
* Runs a catalog endpoint. Charged per record delivered, from your workspace's credit.
|
|
592
|
+
*
|
|
593
|
+
* Sends one `POST`. If the run ends within `waitSeconds` the answer is the result. Otherwise the
|
|
594
|
+
* run continues on the server (`202`): with `wait` (default) this reads `/result` with the same
|
|
595
|
+
* key after each `Retry-After` until the run ends and never resends the POST; with
|
|
596
|
+
* `wait: false` it returns a pending run. The idempotency key (generated unless you pass one) is on
|
|
597
|
+
* the result and on every error thrown, so a lost response can be recovered with
|
|
598
|
+
* {@link Catalog.result}.
|
|
599
|
+
*
|
|
600
|
+
* Check {@link isAccepted} before using the records.
|
|
601
|
+
*/
|
|
602
|
+
run<T = AnyRecord>(apiId: string, endpointId: string, options: CatalogRunOptions & {
|
|
603
|
+
wait: false;
|
|
604
|
+
}): Promise<RunOutcome<T>>;
|
|
605
|
+
run<T = AnyRecord>(apiId: string, endpointId: string, options?: CatalogRunOptions & {
|
|
606
|
+
wait?: true;
|
|
607
|
+
}): Promise<CompletedRun<T>>;
|
|
608
|
+
/**
|
|
609
|
+
* Reads a catalog run's result with the API key and the idempotency key it started with: after a
|
|
610
|
+
* `202`, a lost response, or a crash. It never collects the website again, and works after the
|
|
611
|
+
* API is unpublished or its version changes.
|
|
612
|
+
*/
|
|
613
|
+
result<T = AnyRecord>(apiId: string, endpointId: string, options: CatalogResultOptions & {
|
|
614
|
+
wait: false;
|
|
615
|
+
}): Promise<RunOutcome<T>>;
|
|
616
|
+
result<T = AnyRecord>(apiId: string, endpointId: string, options: CatalogResultOptions & {
|
|
617
|
+
wait?: true;
|
|
618
|
+
}): Promise<CompletedRun<T>>;
|
|
619
|
+
private root;
|
|
620
|
+
private readResult;
|
|
621
|
+
private getById;
|
|
622
|
+
private findSlug;
|
|
623
|
+
private collect;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
/** Read and control runs by `run_id`. Needs a key with the `run` scope (`results` also reads). */
|
|
627
|
+
declare class Runs {
|
|
628
|
+
private readonly http;
|
|
629
|
+
constructor(http: HttpClient);
|
|
630
|
+
/** The run object: `status`, `stop_reason`, `progress`, `units_charged`. */
|
|
631
|
+
get(runId: string, options?: {
|
|
632
|
+
signal?: AbortSignal;
|
|
633
|
+
}): Promise<Run>;
|
|
634
|
+
/** The caller's runs, newest first. Pass `next_cursor` as `cursor` for the next page. */
|
|
635
|
+
list(params?: RunListParams): Promise<RunList>;
|
|
636
|
+
/** One page of records committed so far. Readable while the run runs. */
|
|
637
|
+
itemsPage<T = AnyRecord>(runId: string, options?: Omit<RunItemsOptions, 'follow' | 'pollIntervalMs'>): Promise<RunItemsPage<T>>;
|
|
638
|
+
/** Every page of a run's records, following `next_cursor`. See {@link RunItemsOptions.follow}. */
|
|
639
|
+
pages<T = AnyRecord>(runId: string, options?: RunItemsOptions): AsyncGenerator<RunItemsPage<T>, void, undefined>;
|
|
640
|
+
/**
|
|
641
|
+
* Async iterator over a run's records, across cursor pages:
|
|
642
|
+
* `for await (const record of client.runs.items(runId)) { ... }`.
|
|
643
|
+
* While the run is still running it waits for later commits and ends once the run has ended and
|
|
644
|
+
* every record was read (pass `follow: false` to stop at what is committed now). Reading records
|
|
645
|
+
* never collects the website again.
|
|
646
|
+
*/
|
|
647
|
+
items<T = AnyRecord>(runId: string, options?: RunItemsOptions): AsyncGenerator<T, void, undefined>;
|
|
648
|
+
/**
|
|
649
|
+
* The run's final body: `200` once it ended; with `wait` (default) a `202` keeps reading after
|
|
650
|
+
* `Retry-After` until it does. With `wait: false` it returns a pending run instead.
|
|
651
|
+
*/
|
|
652
|
+
result<T = AnyRecord>(runId: string, options: RunResultOptions & {
|
|
653
|
+
wait: false;
|
|
654
|
+
}): Promise<RunResult<T> | RunPending>;
|
|
655
|
+
result<T = AnyRecord>(runId: string, options?: RunResultOptions & {
|
|
656
|
+
wait?: true;
|
|
657
|
+
}): Promise<RunResult<T>>;
|
|
658
|
+
/**
|
|
659
|
+
* Stops a run you no longer need. Answers `202` with the run object while it stops, or `200` if it
|
|
660
|
+
* had already ended. Records committed so far stay readable (and charged).
|
|
661
|
+
*/
|
|
662
|
+
abort(runId: string, options?: {
|
|
663
|
+
signal?: AbortSignal;
|
|
664
|
+
}): Promise<Run>;
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
/** The run object from `sources.start`. */
|
|
668
|
+
interface StartedRun {
|
|
669
|
+
run: Run;
|
|
670
|
+
idempotencyKey: string;
|
|
671
|
+
/** True when the service replayed an earlier request with the same key (`Idempotent-Replayed`). */
|
|
672
|
+
replayed: boolean;
|
|
673
|
+
/** Seconds the service asked to wait before reading the result. */
|
|
674
|
+
retryAfter?: number | undefined;
|
|
675
|
+
}
|
|
676
|
+
/**
|
|
677
|
+
* Run your own (private) generated source APIs: `POST /v1/sources/{source_id}/run`. The origin
|
|
678
|
+
* comes from the API's connection details: pass it as `baseUrl` to the client. Needs a key with the
|
|
679
|
+
* `run` scope and access to the source.
|
|
680
|
+
*/
|
|
681
|
+
declare class Sources {
|
|
682
|
+
private readonly http;
|
|
683
|
+
constructor(http: HttpClient);
|
|
684
|
+
/**
|
|
685
|
+
* Runs a source and waits for its records (a `202` is followed with `GET /v1/runs/{run_id}/result`,
|
|
686
|
+
* never by resending the POST). Pass `expectedVersion` to fail with a `VersionConflictError`
|
|
687
|
+
* instead of running a different definition than the one you reviewed.
|
|
688
|
+
*/
|
|
689
|
+
run<T = AnyRecord>(sourceId: string, options: SourceRunOptions & {
|
|
690
|
+
wait: false;
|
|
691
|
+
}): Promise<(RunResult<T> | RunPending) & {
|
|
692
|
+
idempotencyKey: string;
|
|
693
|
+
}>;
|
|
694
|
+
run<T = AnyRecord>(sourceId: string, options?: SourceRunOptions & {
|
|
695
|
+
wait?: true;
|
|
696
|
+
}): Promise<RunResult<T> & {
|
|
697
|
+
idempotencyKey: string;
|
|
698
|
+
}>;
|
|
699
|
+
/**
|
|
700
|
+
* Starts a run without waiting (`POST /v1/sources/{source_id}/runs`; `waitSeconds` defaults to 0).
|
|
701
|
+
* Read records with `runs.items(run.run_id)` and the final body with `runs.result(run.run_id)`.
|
|
702
|
+
*/
|
|
703
|
+
start(sourceId: string, options?: Omit<SourceRunOptions, 'wait' | 'onProgress'>): Promise<StartedRun>;
|
|
704
|
+
private prepare;
|
|
705
|
+
}
|
|
706
|
+
|
|
707
|
+
/** The run origin: runs, results and the generic run routes. */
|
|
708
|
+
declare const DEFAULT_BASE_URL = "https://api.cleanedweb.com";
|
|
709
|
+
/** The live catalog (anonymous reads). */
|
|
710
|
+
declare const DEFAULT_CATALOG_URL = "https://app.cleanedweb.com";
|
|
711
|
+
/** The public site (catalog index). */
|
|
712
|
+
declare const DEFAULT_SITE_URL = "https://cleanedweb.com";
|
|
713
|
+
interface CleanedWebOptions {
|
|
714
|
+
/**
|
|
715
|
+
* An API key with the `run` scope. Defaults to `process.env.CLEANEDWEB_API_KEY`. Keep it on the
|
|
716
|
+
* server: never bundle it into browser code. Catalog reads need no key.
|
|
717
|
+
*/
|
|
718
|
+
apiKey?: string | undefined;
|
|
719
|
+
/**
|
|
720
|
+
* The run origin, without a path (default `https://api.cleanedweb.com`). For a private source
|
|
721
|
+
* API, use the origin from its connection details. The API key is sent only to this origin.
|
|
722
|
+
*/
|
|
723
|
+
baseUrl?: string;
|
|
724
|
+
/** The live catalog origin (default `https://app.cleanedweb.com`). Never receives the key. */
|
|
725
|
+
catalogUrl?: string;
|
|
726
|
+
/** The public site origin for the catalog index (default `https://cleanedweb.com`). Never receives the key. */
|
|
727
|
+
siteUrl?: string;
|
|
728
|
+
/** A fetch implementation (default: the global `fetch`). */
|
|
729
|
+
fetch?: FetchLike;
|
|
730
|
+
/**
|
|
731
|
+
* How often a request is repeated after a lost connection, `429`, `502`, `503` or `504`
|
|
732
|
+
* (default 2; 0 turns retries off). Retries keep the idempotency key and wait `Retry-After`. A
|
|
733
|
+
* failed run is never retried.
|
|
734
|
+
*/
|
|
735
|
+
maxRetries?: number;
|
|
736
|
+
/** Per-request client time limit in ms (default 120000). A run request waits longer when `waitSeconds` asks for it. */
|
|
737
|
+
timeoutMs?: number;
|
|
738
|
+
/** A `Retry-After` longer than this many ms is thrown instead of waited for (default 30000). */
|
|
739
|
+
maxRetryDelayMs?: number;
|
|
740
|
+
/** Replaces the timer used between retries and polls. For tests. */
|
|
741
|
+
sleep?: Sleep;
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* The CleanedWeb client.
|
|
745
|
+
*
|
|
746
|
+
* ```ts
|
|
747
|
+
* const cw = new CleanedWeb(); // reads CLEANEDWEB_API_KEY
|
|
748
|
+
* const result = await cw.catalog.run(apiId, endpointId, { input: {}, maxItems: 25 });
|
|
749
|
+
* ```
|
|
750
|
+
*/
|
|
751
|
+
declare class CleanedWeb {
|
|
752
|
+
/** Browse the public catalog and run published APIs. */
|
|
753
|
+
readonly catalog: Catalog;
|
|
754
|
+
/** Read and control runs by `run_id`. */
|
|
755
|
+
readonly runs: Runs;
|
|
756
|
+
/** Run your own (private) source APIs. */
|
|
757
|
+
readonly sources: Sources;
|
|
758
|
+
constructor(options?: CleanedWebOptions);
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
interface AcceptOptions {
|
|
762
|
+
/** The contract's `definition_version`: also require `summary.definition_version` to match. */
|
|
763
|
+
definitionVersion?: string;
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
/**
|
|
767
|
+
* Whether a finished run passes the documented acceptance checks: `summary.status` is `completed`
|
|
768
|
+
* or `bounded`, `summary.within_scope_passed` is true, the run finished (and, with `definitionVersion`, the
|
|
769
|
+
* summary names that version). `bounded` and `partial` mean the source can hold more records.
|
|
770
|
+
*/
|
|
771
|
+
declare function isAccepted(result: RunResultBody<unknown>, options?: AcceptOptions): boolean;
|
|
772
|
+
/** Throws a {@link ResultNotAcceptedError} naming each failed check. */
|
|
773
|
+
declare function assertAccepted(result: RunResultBody<unknown>, options?: AcceptOptions): void;
|
|
774
|
+
|
|
775
|
+
export { APIError, type APIErrorInit, type AcceptOptions, type AnyRecord, AuthenticationError, Catalog, type CatalogApi, type CatalogContract, type CatalogEndpoint, type CatalogFindParams, type CatalogIndex, type CatalogIndexEntry, type CatalogListParams, type CatalogMatch, type CatalogPage, type CatalogResultOptions, type CatalogRunOptions, type CatalogSite, type CatalogSort, type CatalogVersionInfo, CleanedWeb, CleanedWebAbortError, CleanedWebConnectionError, CleanedWebError, type CleanedWebOptions, CleanedWebTimeoutError, type CompletedRun, ConfigurationError, DEFAULT_BASE_URL, DEFAULT_CATALOG_URL, DEFAULT_SITE_URL, type Delivery, type DeprecationInfo, type FetchLike, IdempotencyConflictError, InsufficientCreditError, NotFoundError, type PendingRun, PermissionError, RateLimitError, ResultNotAcceptedError, ResultNotReadyError, type Run, type RunCallOptions, RunFailedError, type RunItemsOptions, type RunItemsPage, type RunLinks, type RunList, type RunListParams, type RunOptions, type RunOutcome, type RunPending, type RunProgress, type RunResult, type RunResultBody, type RunResultOptions, type RunStatus, type RunSummary, Runs, ServerError, type Sleep, type SourceRunOptions, Sources, type StartedRun, type StopReason, UsageBudgetExceededError, ValidationError, VersionConflictError, assertAccepted, generateIdempotencyKey, isAccepted };
|