@crawlbrulee/sdk 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -76
- package/dist/index.cjs +778 -617
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +654 -668
- package/dist/index.d.ts +654 -668
- package/dist/index.js +777 -616
- package/dist/index.js.map +1 -1
- package/package.json +6 -6
package/dist/index.d.cts
CHANGED
|
@@ -1,30 +1,31 @@
|
|
|
1
|
+
//#region src/http.d.ts
|
|
1
2
|
/** HTTP methods used by the SDK. */
|
|
2
3
|
type HttpMethod = 'GET' | 'POST';
|
|
3
4
|
/** Options the SDK accepts at construction time for the HTTP layer. */
|
|
4
5
|
interface HttpClientOptions {
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
6
|
+
/** API key sent as `Authorization: Bearer <key>`. */
|
|
7
|
+
apiKey: string;
|
|
8
|
+
/**
|
|
9
|
+
* Override the base URL. Trailing slashes are stripped. Falls back to
|
|
10
|
+
* {@link CwblInstrumentation.getBaseUrl} (which resolves to the production
|
|
11
|
+
* host) when unset.
|
|
12
|
+
*/
|
|
13
|
+
baseUrl?: string;
|
|
14
|
+
/**
|
|
15
|
+
* Per-request timeout in milliseconds. Pass `0` (or omit) to disable the
|
|
16
|
+
* timeout entirely.
|
|
17
|
+
*/
|
|
18
|
+
timeoutMs?: number;
|
|
18
19
|
}
|
|
19
20
|
/** Per-call overrides accepted on every resource method. */
|
|
20
21
|
interface RequestOptions {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
22
|
+
/** Abort the request when this signal fires. Composable with the timeout. */
|
|
23
|
+
signal?: AbortSignal;
|
|
24
|
+
/**
|
|
25
|
+
* Override the constructor-level `timeoutMs` for this call. Pass `0` to
|
|
26
|
+
* disable the timeout for this call.
|
|
27
|
+
*/
|
|
28
|
+
timeoutMs?: number;
|
|
28
29
|
}
|
|
29
30
|
/**
|
|
30
31
|
* Minimal `fetch`-based HTTP layer used by {@link Crawlbrulee}. Handles:
|
|
@@ -43,27 +44,28 @@ interface RequestOptions {
|
|
|
43
44
|
* module.
|
|
44
45
|
*/
|
|
45
46
|
declare class HttpClient {
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
47
|
+
readonly baseUrl: string;
|
|
48
|
+
private readonly apiKey;
|
|
49
|
+
private readonly fetch;
|
|
50
|
+
private readonly timeoutMs;
|
|
51
|
+
constructor(options: HttpClientOptions);
|
|
52
|
+
/** Send a `GET` request and parse the response as `T`. */
|
|
53
|
+
get<T>(path: string, options?: RequestOptions): Promise<T>;
|
|
54
|
+
/** Send a `POST` request with a JSON body and parse the response as `T`. */
|
|
55
|
+
post<T>(path: string, body: unknown, options?: RequestOptions): Promise<T>;
|
|
56
|
+
private send;
|
|
57
|
+
private buildUrl;
|
|
58
|
+
private buildHeaders;
|
|
59
|
+
/**
|
|
60
|
+
* Build a single `AbortSignal` that fires when either the caller-supplied
|
|
61
|
+
* signal aborts OR the per-request timeout elapses. The returned `cleanup`
|
|
62
|
+
* callback MUST be invoked on every exit path so we don't leak timers or
|
|
63
|
+
* dead listeners on long-lived caller signals.
|
|
64
|
+
*/
|
|
65
|
+
private composeSignal;
|
|
65
66
|
}
|
|
66
|
-
|
|
67
|
+
//#endregion
|
|
68
|
+
//#region src/types/common.d.ts
|
|
67
69
|
/**
|
|
68
70
|
* Shared primitive types used across crawlbrulee request and response shapes.
|
|
69
71
|
*/
|
|
@@ -74,32 +76,30 @@ declare class HttpClient {
|
|
|
74
76
|
* - `advanced` — residential proxy, higher success rate on protected sites.
|
|
75
77
|
* - `auto` — start at the basic tier and escalate to advanced on failure;
|
|
76
78
|
* billed at the delivered tier. This is the default when `proxy` is omitted.
|
|
77
|
-
* - `none` — skip the proxy entirely. Rejected in production; available on
|
|
78
|
-
* staging only as a debug/perf-test toggle.
|
|
79
79
|
*/
|
|
80
|
-
type ProxyTier = 'basic' | 'advanced' | 'auto'
|
|
80
|
+
type ProxyTier = 'basic' | 'advanced' | 'auto';
|
|
81
81
|
/**
|
|
82
82
|
* Proxy tier the server actually used to route a fetch, as reported back in
|
|
83
83
|
* {@link Usage.proxy}. Unlike the request-side {@link ProxyTier}, this never
|
|
84
84
|
* includes `auto` — when a request asks for `auto`, the server resolves it to a
|
|
85
85
|
* concrete tier and echoes the resolved value here.
|
|
86
86
|
*/
|
|
87
|
-
type ResolvedProxyTier = '
|
|
87
|
+
type ResolvedProxyTier = 'basic' | 'advanced';
|
|
88
88
|
/**
|
|
89
89
|
* Usage accounting for a single billable operation, returned on the response
|
|
90
90
|
* envelope of scrape, map, and async-status (when terminal). All crawlbrulee
|
|
91
91
|
* responses report this on `response_meta.usage`.
|
|
92
92
|
*/
|
|
93
93
|
interface Usage {
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
94
|
+
/**
|
|
95
|
+
* Credits charged for this operation. `0` on a cache hit (no fresh fetch was
|
|
96
|
+
* performed).
|
|
97
|
+
*/
|
|
98
|
+
credits: number;
|
|
99
|
+
/** The proxy tier the server resolved and used (never `auto`). */
|
|
100
|
+
proxy: ResolvedProxyTier;
|
|
101
|
+
/** Whether the result was served from cache (no fresh fetch). */
|
|
102
|
+
cache_hit: boolean;
|
|
103
103
|
}
|
|
104
104
|
/**
|
|
105
105
|
* Response envelope `response_meta` carried by scrape responses and async-status
|
|
@@ -107,8 +107,8 @@ interface Usage {
|
|
|
107
107
|
* responses extend this shape with `pagination` + `truncation`.
|
|
108
108
|
*/
|
|
109
109
|
interface ResponseMeta {
|
|
110
|
-
|
|
111
|
-
|
|
110
|
+
/** Usage accounting for the operation. */
|
|
111
|
+
usage: Usage;
|
|
112
112
|
}
|
|
113
113
|
/** Screenshot capture mode: visible viewport or the full scrollable page. */
|
|
114
114
|
type ScreenshotType = 'viewport' | 'full_page';
|
|
@@ -116,23 +116,23 @@ type ScreenshotType = 'viewport' | 'full_page';
|
|
|
116
116
|
type ScreenshotDeviceMode = 'desktop' | 'mobile';
|
|
117
117
|
/** Pre-capture cleanup options applied to the page before the screenshot. */
|
|
118
118
|
interface ScreenshotCleanup {
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
119
|
+
/**
|
|
120
|
+
* Remove ads, cookie banners, and popups before capturing. Defaults to
|
|
121
|
+
* `true` server-side.
|
|
122
|
+
*/
|
|
123
|
+
ads_and_popups?: boolean;
|
|
124
124
|
}
|
|
125
125
|
/** A `wait` action: pause for `ms` milliseconds before the next step. */
|
|
126
126
|
interface ScreenshotWaitAction {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
127
|
+
type: 'wait';
|
|
128
|
+
/** Milliseconds to wait. Must be a non-negative integer. */
|
|
129
|
+
ms: number;
|
|
130
130
|
}
|
|
131
131
|
/** A `scroll` action: scroll the page by `pixels` (positive = down). */
|
|
132
132
|
interface ScreenshotScrollAction {
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
133
|
+
type: 'scroll';
|
|
134
|
+
/** Pixels to scroll. Positive scrolls down, negative scrolls up. */
|
|
135
|
+
pixels: number;
|
|
136
136
|
}
|
|
137
137
|
/**
|
|
138
138
|
* Actions performed before the screenshot is taken. The server caps the total
|
|
@@ -145,37 +145,37 @@ type ScreenshotBeforeAction = ScreenshotWaitAction | ScreenshotScrollAction;
|
|
|
145
145
|
* supported — it cuts the screenshot into horizontal tiles of `height` px.
|
|
146
146
|
*/
|
|
147
147
|
interface ScreenshotSliceAction {
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
148
|
+
type: 'slice';
|
|
149
|
+
/** Tile height in pixels. Minimum 500. */
|
|
150
|
+
height: number;
|
|
151
151
|
}
|
|
152
152
|
type ScreenshotAfterAction = ScreenshotSliceAction;
|
|
153
153
|
/** Custom browser viewport dimensions used during a screenshot capture. */
|
|
154
154
|
interface ScreenshotViewport {
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
155
|
+
/** Viewport width in pixels. Integer in `[16, 10000]`; out-of-range values are rejected with a 400. */
|
|
156
|
+
width: number;
|
|
157
|
+
/** Viewport height in pixels. Integer in `[16, 10000]`; out-of-range values are rejected with a 400. */
|
|
158
|
+
height: number;
|
|
159
|
+
/**
|
|
160
|
+
* Device pixel ratio (e.g. 2 for retina). Fractional values are allowed;
|
|
161
|
+
* must be in `[1, 4]`. Defaults to 1 server-side.
|
|
162
|
+
*/
|
|
163
|
+
device_scale_factor?: number;
|
|
164
164
|
}
|
|
165
165
|
/** Screenshot capture configuration. Pass this on `extract.screenshot`. */
|
|
166
166
|
interface ScreenshotRequest {
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
167
|
+
/** Capture mode: `viewport` (visible only) or `full_page`. */
|
|
168
|
+
type: ScreenshotType;
|
|
169
|
+
/** Custom viewport. If omitted, the `device_mode` defaults are used. */
|
|
170
|
+
viewport?: ScreenshotViewport;
|
|
171
|
+
/** Emulate desktop or mobile. Defaults to `desktop`. */
|
|
172
|
+
device_mode?: ScreenshotDeviceMode;
|
|
173
|
+
/** Page cleanup applied before capture. */
|
|
174
|
+
cleanup?: ScreenshotCleanup;
|
|
175
|
+
/** Pre-capture actions (waits and scrolls). Maximum 5 entries. */
|
|
176
|
+
actions_before?: ScreenshotBeforeAction[];
|
|
177
|
+
/** Post-capture actions (e.g. slice into tiles). Maximum 1 entry. */
|
|
178
|
+
actions_after?: ScreenshotAfterAction[];
|
|
179
179
|
}
|
|
180
180
|
/**
|
|
181
181
|
* Machine-readable error names returned by the crawlbrulee API. Stable
|
|
@@ -186,117 +186,118 @@ type ApiErrorName = 'usage_allocation_error' | 'request_timeout' | 'invalid_url'
|
|
|
186
186
|
type UsageAllocationReason = 'credit_limit' | 'concurrency_limit' | 'overage_hard_cap' | 'duplicate_reservation' | 'internal_error';
|
|
187
187
|
/** Snapshot of the org's current usage at the moment the error was raised. */
|
|
188
188
|
interface UsageLimitDetails {
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
189
|
+
/** Current credit usage in the billing period. */
|
|
190
|
+
current_usage?: number;
|
|
191
|
+
/** Credits currently reserved by in-flight jobs. */
|
|
192
|
+
current_reserved?: number;
|
|
193
|
+
/** Maximum credits allowed in the billing period. */
|
|
194
|
+
max_credits?: number;
|
|
195
|
+
/** Number of currently running concurrent jobs. */
|
|
196
|
+
current_concurrent?: number;
|
|
197
|
+
/** Maximum concurrent jobs allowed. */
|
|
198
|
+
max_concurrent?: number;
|
|
199
199
|
}
|
|
200
200
|
/** Discriminated detail for `error_name = usage_allocation_error`. */
|
|
201
201
|
interface UsageAllocationErrorDetails {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
202
|
+
error_name: 'usage_allocation_error';
|
|
203
|
+
reason: UsageAllocationReason;
|
|
204
|
+
details?: UsageLimitDetails;
|
|
205
205
|
}
|
|
206
206
|
/** Discriminated detail for `error_name = too_many_requests`. */
|
|
207
207
|
interface RateLimitErrorDetails {
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
208
|
+
error_name: 'too_many_requests';
|
|
209
|
+
/** Suggested wait time before retrying. */
|
|
210
|
+
retry_after_ms?: number;
|
|
211
|
+
/** Which rate limit was exceeded (e.g. `org`, `ip`). */
|
|
212
|
+
limited_by?: string;
|
|
213
213
|
}
|
|
214
214
|
/** Union of all known `details` payloads on an `ApiErrorResponse`. */
|
|
215
215
|
type ApiErrorDetails = UsageAllocationErrorDetails | RateLimitErrorDetails;
|
|
216
216
|
/** Standard JSON error shape returned for any non-2xx response. */
|
|
217
217
|
interface ApiErrorResponse {
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
218
|
+
/** Machine-readable identifier — safe to switch on. */
|
|
219
|
+
name: ApiErrorName;
|
|
220
|
+
/** Human-readable error message. */
|
|
221
|
+
message: string;
|
|
222
|
+
/** Error-specific structured detail; only present for some `name`s. */
|
|
223
|
+
details?: ApiErrorDetails;
|
|
224
224
|
}
|
|
225
|
-
|
|
225
|
+
//#endregion
|
|
226
|
+
//#region src/types/scrape.d.ts
|
|
226
227
|
/**
|
|
227
228
|
* Which content formats to extract from the scraped page. Every field is
|
|
228
229
|
* optional; the server defaults are noted on each field. The default request
|
|
229
230
|
* extracts `{ metadata: true, cleaned_html: true }`.
|
|
230
231
|
*/
|
|
231
232
|
interface ScrapeExtract {
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
233
|
+
/** Extract page metadata (title, description, OG/Twitter tags, etc.). Default `true`. */
|
|
234
|
+
metadata?: boolean;
|
|
235
|
+
/** Extract cleaned HTML (main content only). Default `true`. */
|
|
236
|
+
cleaned_html?: boolean;
|
|
237
|
+
/** Extract the page as clean Markdown. Default `false`. */
|
|
238
|
+
markdown?: boolean;
|
|
239
|
+
/** Return the raw, unprocessed HTML. Default `false`. */
|
|
240
|
+
raw_html?: boolean;
|
|
241
|
+
/** Extract all links found on the page. Default `false`. */
|
|
242
|
+
links?: boolean;
|
|
243
|
+
/**
|
|
244
|
+
* Extract all inline images found on the page. Default `false`. Image URLs
|
|
245
|
+
* preserve their query string, and document-relative `src`s are resolved
|
|
246
|
+
* against the full page URL (browser parity) — same rules as `links`.
|
|
247
|
+
*/
|
|
248
|
+
images?: boolean;
|
|
249
|
+
/** Capture a screenshot. Omit to skip; set to a `ScreenshotRequest` to enable. */
|
|
250
|
+
screenshot?: ScreenshotRequest;
|
|
250
251
|
}
|
|
251
252
|
/** Cache settings for a scrape request. */
|
|
252
253
|
interface ScrapeCache {
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
254
|
+
/**
|
|
255
|
+
* Maximum cache age. Either a number of seconds (non-negative integer) or
|
|
256
|
+
* an ISO-8601 datetime cutoff — cached entries older than this are skipped.
|
|
257
|
+
* Defaults to 2 days when omitted.
|
|
258
|
+
*/
|
|
259
|
+
max_age?: number | string;
|
|
260
|
+
/**
|
|
261
|
+
* Treat URLs with different query parameters as the same cache entry.
|
|
262
|
+
* Defaults to `false`.
|
|
263
|
+
*/
|
|
264
|
+
ignore_query_params?: boolean;
|
|
264
265
|
}
|
|
265
266
|
/** Optional locale + country emulation for the scrape. */
|
|
266
267
|
interface ScrapeLocation {
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
268
|
+
/**
|
|
269
|
+
* BCP-47 locale (e.g. `en-US`, `de-DE`, `pt-BR`). Sent as `Accept-Language`
|
|
270
|
+
* and reflected in `navigator.language` when JS rendering is requested.
|
|
271
|
+
*/
|
|
272
|
+
locale?: string;
|
|
273
|
+
/**
|
|
274
|
+
* ISO 3166-1 alpha-2 country code (e.g. `US`, `DE`, `BR`). Drives the
|
|
275
|
+
* emulated browser timezone. Case-insensitive.
|
|
276
|
+
*/
|
|
277
|
+
country?: string;
|
|
277
278
|
}
|
|
278
279
|
/** Request body for `POST /api/scrape` (and `POST /api/scrape/async`). */
|
|
279
280
|
interface ScrapeRequest {
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
281
|
+
/** The URL to scrape. */
|
|
282
|
+
url: string;
|
|
283
|
+
/** Which content formats to extract. Defaults to `metadata + cleaned_html`. */
|
|
284
|
+
extract?: ScrapeExtract;
|
|
285
|
+
/** Cache settings for this request. */
|
|
286
|
+
cache?: ScrapeCache;
|
|
287
|
+
/**
|
|
288
|
+
* Use a headless browser to render JavaScript before scraping. Adds latency
|
|
289
|
+
* and credits — only enable when the page requires it. Default `false`.
|
|
290
|
+
*/
|
|
291
|
+
require_js?: boolean;
|
|
292
|
+
/** CSS selectors to strip from the extracted content. */
|
|
293
|
+
exclude_selectors?: string[];
|
|
294
|
+
/**
|
|
295
|
+
* Proxy tier to use for fetching. Defaults to `auto` (tries the basic tier
|
|
296
|
+
* first, escalates to advanced on failure; billed at the delivered tier).
|
|
297
|
+
*/
|
|
298
|
+
proxy?: ProxyTier;
|
|
299
|
+
/** Optional locale + country emulation. */
|
|
300
|
+
location?: ScrapeLocation;
|
|
300
301
|
}
|
|
301
302
|
/**
|
|
302
303
|
* Per-job completion webhook, attached when submitting an ASYNC scrape via
|
|
@@ -308,20 +309,20 @@ interface ScrapeRequest {
|
|
|
308
309
|
* (Account → Webhooks) — there is no per-request secret.
|
|
309
310
|
*/
|
|
310
311
|
interface AsyncScrapeWebhook {
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
312
|
+
/**
|
|
313
|
+
* Endpoint that receives a single signed `POST` when the job reaches a
|
|
314
|
+
* terminal state. Must be an `http`/`https` URL (HTTPS is required in
|
|
315
|
+
* production) of at most 2048 characters. The body is a
|
|
316
|
+
* `scrape.complete` envelope signed with your organization webhook secret
|
|
317
|
+
* on the `X-Cwbl-Signature` header — verify it with `verifyWebhookSignature`.
|
|
318
|
+
*/
|
|
319
|
+
url: string;
|
|
320
|
+
/**
|
|
321
|
+
* Opaque correlation object echoed verbatim in the webhook payload's
|
|
322
|
+
* `data.metadata`. Must serialize to at most 2048 bytes (UTF-8 JSON). Use it
|
|
323
|
+
* to route deliveries without keeping your own `job_id` mapping.
|
|
324
|
+
*/
|
|
325
|
+
metadata?: Record<string, unknown>;
|
|
325
326
|
}
|
|
326
327
|
/**
|
|
327
328
|
* Request body for `POST /api/scrape/async`: a {@link ScrapeRequest} plus an
|
|
@@ -329,384 +330,365 @@ interface AsyncScrapeWebhook {
|
|
|
329
330
|
* is async-only and is not accepted by the synchronous `scrape()` endpoint.
|
|
330
331
|
*/
|
|
331
332
|
interface AsyncScrapeRequest extends ScrapeRequest {
|
|
332
|
-
|
|
333
|
-
|
|
333
|
+
/** Optional completion webhook delivered when this job finishes. */
|
|
334
|
+
webhook?: AsyncScrapeWebhook;
|
|
334
335
|
}
|
|
335
336
|
/** Viewport metadata returned alongside a captured screenshot. */
|
|
336
337
|
interface ScreenshotViewportInfo {
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
338
|
+
width: number;
|
|
339
|
+
height: number;
|
|
340
|
+
device_scale_factor: number;
|
|
340
341
|
}
|
|
341
342
|
/** Image-level properties of a captured screenshot (or tile). */
|
|
342
343
|
interface ScreenshotProperties {
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
344
|
+
/** File name of the image asset. */
|
|
345
|
+
file_name: string;
|
|
346
|
+
/** MIME type (e.g. `image/png`). */
|
|
347
|
+
mime: string;
|
|
348
|
+
/** Image width in pixels. */
|
|
349
|
+
width: number;
|
|
350
|
+
/** Image height in pixels. */
|
|
351
|
+
height: number;
|
|
352
|
+
/** Viewport dimensions used during capture. */
|
|
353
|
+
viewport: ScreenshotViewportInfo;
|
|
353
354
|
}
|
|
354
355
|
/** One horizontal tile of a sliced full-page screenshot. */
|
|
355
356
|
interface ScreenshotSlice {
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
357
|
+
/** 0-based row index of this slice. */
|
|
358
|
+
row_nr: number;
|
|
359
|
+
/** Signed URL to download this slice image. */
|
|
360
|
+
url: string;
|
|
361
|
+
type: 'slice';
|
|
362
|
+
/** Image-level properties of this slice. */
|
|
363
|
+
properties: ScreenshotProperties;
|
|
363
364
|
}
|
|
364
365
|
/** Result block returned when a screenshot was requested. */
|
|
365
366
|
interface ScreenshotResult {
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
367
|
+
/** Signed URL to download the full screenshot image. */
|
|
368
|
+
url: string;
|
|
369
|
+
/** Capture mode that was used. */
|
|
370
|
+
type: ScreenshotType;
|
|
371
|
+
/** Image-level properties of the full screenshot. */
|
|
372
|
+
properties: ScreenshotProperties;
|
|
373
|
+
/** Tile slices, present only when the `slice` `actions_after` was requested. */
|
|
374
|
+
slices?: ScreenshotSlice[];
|
|
374
375
|
}
|
|
375
376
|
/** A single inline image discovered on the page. */
|
|
376
377
|
interface PageInlineImage {
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
378
|
+
/**
|
|
379
|
+
* Absolute URL of the image, query string preserved. Document-relative
|
|
380
|
+
* `src`s are resolved against the full page URL (browser parity).
|
|
381
|
+
*/
|
|
382
|
+
url: string;
|
|
383
|
+
/** Alt text of the image, or `null` if not set. */
|
|
384
|
+
alt: string | null;
|
|
384
385
|
}
|
|
385
386
|
/** A single link discovered on the page. */
|
|
386
387
|
interface PageLink {
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
388
|
+
/** Anchor text of the link. */
|
|
389
|
+
text: string;
|
|
390
|
+
/** The link URL as it appears on the page (absolute or relative). */
|
|
391
|
+
href: string;
|
|
392
|
+
/** Whether the link points to the same domain as the scraped page. */
|
|
393
|
+
internal: boolean;
|
|
393
394
|
}
|
|
394
395
|
/** Structured page metadata extracted from `<head>`. */
|
|
395
396
|
interface ScrapeMetadata {
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
397
|
+
title?: string;
|
|
398
|
+
description?: string;
|
|
399
|
+
keywords?: string[];
|
|
400
|
+
canonical?: string;
|
|
401
|
+
og_url?: string;
|
|
402
|
+
og_title?: string;
|
|
403
|
+
og_description?: string;
|
|
404
|
+
og_type?: string;
|
|
405
|
+
og_site_name?: string;
|
|
406
|
+
og_locale?: string;
|
|
407
|
+
og_locale_alternate?: string[];
|
|
408
|
+
og_image?: string;
|
|
409
|
+
author?: string;
|
|
410
|
+
date_modified?: string;
|
|
411
|
+
date_published?: string;
|
|
412
|
+
twitter_site?: string;
|
|
413
|
+
twitter_card?: string;
|
|
414
|
+
twitter_description?: string;
|
|
415
|
+
twitter_title?: string;
|
|
416
|
+
twitter_image?: string;
|
|
417
|
+
robots?: string;
|
|
418
|
+
favicon_url?: string | null;
|
|
418
419
|
}
|
|
419
420
|
/** Successful response from `POST /api/scrape` and `GET /api/scrape/result/:jobId`. */
|
|
420
421
|
interface ScrapeResponse {
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
422
|
+
/** The URL that was actually scraped (after any redirects). */
|
|
423
|
+
url: string;
|
|
424
|
+
/** `Content-Type` header returned by the origin. */
|
|
425
|
+
content_type?: string;
|
|
426
|
+
/**
|
|
427
|
+
* Extract fields that were requested but aren't supported for this
|
|
428
|
+
* content type (e.g. asking for `markdown` of a PDF).
|
|
429
|
+
*/
|
|
430
|
+
unsupported_fields?: string[];
|
|
431
|
+
/** Page content converted to clean Markdown (when `extract.markdown`). */
|
|
432
|
+
markdown?: string;
|
|
433
|
+
/** Cleaned HTML of the main page content (when `extract.cleaned_html`). */
|
|
434
|
+
cleaned_html?: string;
|
|
435
|
+
/** Raw, unprocessed HTML (when `extract.raw_html`). */
|
|
436
|
+
raw_html?: string;
|
|
437
|
+
/** Inline images discovered on the page (when `extract.images`). */
|
|
438
|
+
images?: PageInlineImage[];
|
|
439
|
+
/** Links discovered on the page (when `extract.links`). */
|
|
440
|
+
links?: PageLink[];
|
|
441
|
+
/**
|
|
442
|
+
* Captured screenshot (when `extract.screenshot`). In rare cases a screenshot
|
|
443
|
+
* can't be captured; when that happens the rest of your requested outputs are
|
|
444
|
+
* still returned and this field is simply left out (so it reads back as
|
|
445
|
+
* `undefined`). Guard with `page.screenshot?.url`.
|
|
446
|
+
*/
|
|
447
|
+
screenshot?: ScreenshotResult;
|
|
448
|
+
/** Extracted page metadata (when `extract.metadata`, on by default). */
|
|
449
|
+
metadata?: ScrapeMetadata;
|
|
450
|
+
/**
|
|
451
|
+
* Non-error notices about the scrape (e.g. `screenshot_truncated` when a
|
|
452
|
+
* long page exceeded the scrolling-screenshot height cap). Stable codes —
|
|
453
|
+
* safe to switch on. Currently surfaced only on fresh scrapes; cache hits
|
|
454
|
+
* omit warnings.
|
|
455
|
+
*/
|
|
456
|
+
warnings?: string[];
|
|
457
|
+
/**
|
|
458
|
+
* Response envelope metadata. Carries `usage` (credits charged, resolved
|
|
459
|
+
* proxy tier, and whether the result was a cache hit).
|
|
460
|
+
*/
|
|
461
|
+
response_meta: ResponseMeta;
|
|
461
462
|
}
|
|
462
|
-
|
|
463
|
+
//#endregion
|
|
464
|
+
//#region src/types/map.d.ts
|
|
463
465
|
/** Filter which link types appear in the map result. */
|
|
464
466
|
interface MapTypes {
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
467
|
+
/** Include internal links (same domain). Default `true`. */
|
|
468
|
+
internal?: boolean;
|
|
469
|
+
/** Include links to subdomains of the target. Default `true`. */
|
|
470
|
+
internal_subdomains?: boolean;
|
|
471
|
+
/** Include external links (different domains). Default `true`. */
|
|
472
|
+
external?: boolean;
|
|
471
473
|
}
|
|
472
474
|
/** Cache settings for a map request. */
|
|
473
475
|
interface MapCache {
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
476
|
+
/**
|
|
477
|
+
* Maximum cache age — either seconds or an ISO-8601 datetime cutoff.
|
|
478
|
+
* Defaults to 7 days when omitted.
|
|
479
|
+
*/
|
|
480
|
+
max_age?: number | string;
|
|
479
481
|
}
|
|
480
482
|
/** Country-only egress emulation (map requests have no locale knob). */
|
|
481
483
|
interface MapLocation {
|
|
482
|
-
|
|
483
|
-
|
|
484
|
+
/** ISO 3166-1 alpha-2 country code (e.g. `US`). Case-insensitive. */
|
|
485
|
+
country?: string;
|
|
484
486
|
}
|
|
485
487
|
/** Request body for `POST /api/map`. */
|
|
486
488
|
interface MapRequest {
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
489
|
+
/** The website URL to map. */
|
|
490
|
+
url: string;
|
|
491
|
+
/**
|
|
492
|
+
* Proxy tier to use for fetching. Defaults to `auto` (tries the basic tier
|
|
493
|
+
* first, escalates to advanced on failure; billed at the delivered tier).
|
|
494
|
+
*/
|
|
495
|
+
proxy?: ProxyTier;
|
|
496
|
+
/** Only use sitemap.xml — skip homepage link extraction. Default `false`. */
|
|
497
|
+
sitemap_only?: boolean;
|
|
498
|
+
/** Filter which link types to include. */
|
|
499
|
+
types?: MapTypes;
|
|
500
|
+
/** Cache settings for this request. */
|
|
501
|
+
cache?: MapCache;
|
|
502
|
+
/**
|
|
503
|
+
* Maximum number of URLs to store in the map. Must be in `(0, 100 000]`.
|
|
504
|
+
* Defaults to 100 000.
|
|
505
|
+
*/
|
|
506
|
+
max_urls?: number;
|
|
507
|
+
/** 1-based page number for paginated results. Defaults to 1. */
|
|
508
|
+
page?: number;
|
|
509
|
+
/**
|
|
510
|
+
* Number of URLs per page. Must be in `(0, 10 000]`. Defaults to 10 000.
|
|
511
|
+
*/
|
|
512
|
+
limit?: number;
|
|
513
|
+
/** Optional country emulation. */
|
|
514
|
+
location?: MapLocation;
|
|
513
515
|
}
|
|
514
516
|
/** Single discovered URL in a map result. */
|
|
515
517
|
interface MapLinkItem {
|
|
516
|
-
|
|
517
|
-
|
|
518
|
+
/** The discovered URL. */
|
|
519
|
+
url: string;
|
|
518
520
|
}
|
|
519
521
|
/** Pagination details on a map response. */
|
|
520
522
|
interface MapPagination {
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
523
|
+
page: number;
|
|
524
|
+
limit: number;
|
|
525
|
+
/** Total number of URLs in the stored map. */
|
|
526
|
+
total: number;
|
|
527
|
+
total_pages: number;
|
|
528
|
+
has_more: boolean;
|
|
527
529
|
}
|
|
528
530
|
/** Information about whether the stored or returned map was truncated. */
|
|
529
531
|
interface MapTruncation {
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
532
|
+
/** Whether the stored map was capped by `max_urls`. */
|
|
533
|
+
storage_capped: boolean;
|
|
534
|
+
/** Whether the response was capped by pagination. */
|
|
535
|
+
response_capped: boolean;
|
|
536
|
+
/** Total URLs found before the `max_urls` cap was applied. */
|
|
537
|
+
total_before_max_urls: number;
|
|
538
|
+
/** Total URLs detected during discovery before the storage cap was applied. */
|
|
539
|
+
total_detected_before_storage_cap: number;
|
|
538
540
|
}
|
|
539
541
|
/**
|
|
540
542
|
* Response envelope for a map result: the shared {@link ResponseMeta} (`usage`)
|
|
541
543
|
* plus the map-specific `pagination` and `truncation`.
|
|
542
544
|
*/
|
|
543
545
|
interface MapResponseMeta extends ResponseMeta {
|
|
544
|
-
|
|
545
|
-
|
|
546
|
+
pagination: MapPagination;
|
|
547
|
+
truncation: MapTruncation;
|
|
546
548
|
}
|
|
547
549
|
/** Success response from `POST /api/map`. */
|
|
548
550
|
interface MapResponse {
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
551
|
+
/** The current page of discovered URLs. */
|
|
552
|
+
links: MapLinkItem[];
|
|
553
|
+
/** Usage, pagination, and truncation metadata for the result set. */
|
|
554
|
+
response_meta: MapResponseMeta;
|
|
553
555
|
}
|
|
554
|
-
|
|
556
|
+
//#endregion
|
|
557
|
+
//#region src/types/async.d.ts
|
|
555
558
|
/** Job lifecycle states for an async scrape. */
|
|
556
559
|
type AsyncJobStatus = 'pending' | 'running' | 'done' | 'failed';
|
|
557
560
|
/** Response body of `POST /api/scrape/async`. */
|
|
558
561
|
interface AsyncScrapeResponse {
|
|
559
|
-
|
|
560
|
-
|
|
562
|
+
/** Job identifier — pass it to `getScrapeStatus` / `getScrapeResult`. */
|
|
563
|
+
job_id: string;
|
|
561
564
|
}
|
|
562
565
|
/** Response body of `GET /api/scrape/status/:jobId`. */
|
|
563
566
|
interface AsyncJobStatusResponse {
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
567
|
+
/** The job identifier. */
|
|
568
|
+
job_id: string;
|
|
569
|
+
/** Current state of the job. */
|
|
570
|
+
status: AsyncJobStatus;
|
|
571
|
+
/** ISO-8601 UTC timestamp when the job was created. */
|
|
572
|
+
created_at: string;
|
|
573
|
+
/** Error message if the job ended in `failed`. */
|
|
574
|
+
error?: string;
|
|
575
|
+
/**
|
|
576
|
+
* Response envelope metadata — present only when the job has reached the
|
|
577
|
+
* terminal `done` state. Carries `usage` (credits charged, resolved proxy
|
|
578
|
+
* tier, and whether the result was a cache hit).
|
|
579
|
+
*/
|
|
580
|
+
response_meta?: ResponseMeta;
|
|
578
581
|
}
|
|
579
|
-
|
|
582
|
+
//#endregion
|
|
583
|
+
//#region src/types/account.d.ts
|
|
580
584
|
/** Response from `GET /api/usage`. Current billing-cycle snapshot. */
|
|
581
585
|
interface UsageResponse {
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
586
|
+
/**
|
|
587
|
+
* Total credits available for the current billing cycle
|
|
588
|
+
* (plan base + purchased + gifted).
|
|
589
|
+
*/
|
|
590
|
+
total_credits: number;
|
|
591
|
+
/**
|
|
592
|
+
* Credits spent so far in the current billing cycle. May exceed
|
|
593
|
+
* `total_credits` on plans that allow overages.
|
|
594
|
+
*/
|
|
595
|
+
used_credits: number;
|
|
596
|
+
/**
|
|
597
|
+
* Remaining credits, `max(0, total_credits - used_credits)`. Clamped to 0
|
|
598
|
+
* while in overage.
|
|
599
|
+
*/
|
|
600
|
+
available_credits: number;
|
|
601
|
+
/**
|
|
602
|
+
* Percentage of `total_credits` used in the current cycle, rounded to one
|
|
603
|
+
* decimal. Not capped — values above 100 indicate overage.
|
|
604
|
+
*/
|
|
605
|
+
used_quota_percent: number;
|
|
606
|
+
/**
|
|
607
|
+
* Maximum number of concurrent jobs allowed for the org
|
|
608
|
+
* (plan base + purchased + gifted extras).
|
|
609
|
+
*/
|
|
610
|
+
max_concurrency: number;
|
|
611
|
+
/**
|
|
612
|
+
* ISO-8601 UTC timestamp when the current billing cycle ends and
|
|
613
|
+
* `used_credits` resets to 0.
|
|
614
|
+
*/
|
|
615
|
+
usage_reset: string;
|
|
612
616
|
}
|
|
613
617
|
/** Response from `GET /api/whoami`. Identifies the calling API token. */
|
|
614
618
|
interface WhoamiResponse {
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
619
|
+
/** Display name of the organization that owns the token. */
|
|
620
|
+
organization_name: string;
|
|
621
|
+
/** User-assigned name of the API token. */
|
|
622
|
+
token_name: string;
|
|
623
|
+
/** Truncated preview of the API token (e.g. `cwbl_…xyz`). Safe to display. */
|
|
624
|
+
token_preview: string;
|
|
621
625
|
}
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
* Async scrape completion webhooks.
|
|
625
|
-
*
|
|
626
|
-
* When you submit a job with {@link Crawlbrulee.scrapeAsync} the API can deliver
|
|
627
|
-
* a `scrape.complete` webhook to your configured endpoint once the job reaches a
|
|
628
|
-
* terminal state. The envelope on the wire is exactly:
|
|
629
|
-
*
|
|
630
|
-
* ```json
|
|
631
|
-
* {
|
|
632
|
-
* "event_id": "evt_…",
|
|
633
|
-
* "timestamp": "2026-06-13T12:00:00.000Z",
|
|
634
|
-
* "event": "scrape.complete",
|
|
635
|
-
* "data": {
|
|
636
|
-
* "job_id": "job_…",
|
|
637
|
-
* "status": "success",
|
|
638
|
-
* "url": "https://…",
|
|
639
|
-
* "completed_at": "…",
|
|
640
|
-
* "metadata": { "tenant": "acme" },
|
|
641
|
-
* "response_meta": { "usage": { "credits": 1, "proxy": "basic", "cache_hit": false } }
|
|
642
|
-
* }
|
|
643
|
-
* }
|
|
644
|
-
* ```
|
|
645
|
-
*/
|
|
646
|
-
|
|
626
|
+
//#endregion
|
|
627
|
+
//#region src/types/webhooks.d.ts
|
|
647
628
|
/** Terminal status carried by a {@link ScrapeCompleteWebhook}. */
|
|
648
629
|
type ScrapeWebhookStatus = 'success' | 'failed' | 'cancelled';
|
|
649
630
|
/** `data` block of a {@link ScrapeCompleteWebhook}. */
|
|
650
631
|
interface ScrapeCompleteWebhookData {
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
632
|
+
/** The async job identifier — pass it to `getScrapeResult`. */
|
|
633
|
+
job_id: string;
|
|
634
|
+
/** Terminal state of the job. */
|
|
635
|
+
status: ScrapeWebhookStatus;
|
|
636
|
+
/** The URL that was scraped. */
|
|
637
|
+
url: string;
|
|
638
|
+
/** ISO-8601 UTC timestamp when the job reached its terminal state. */
|
|
639
|
+
completed_at: string;
|
|
640
|
+
/** Failure message — present only when `status === 'failed'`. */
|
|
641
|
+
error?: string;
|
|
642
|
+
/**
|
|
643
|
+
* Correlation data echoed back from the original scrape request's
|
|
644
|
+
* `webhook.metadata`, if any.
|
|
645
|
+
*/
|
|
646
|
+
metadata?: Record<string, unknown>;
|
|
647
|
+
/**
|
|
648
|
+
* Response envelope metadata — `usage` (credits charged, resolved proxy tier,
|
|
649
|
+
* and whether the result was a cache hit). Present only on `status: 'success'`
|
|
650
|
+
* deliveries; omitted for `failed` / `cancelled` (no usage was charged).
|
|
651
|
+
*/
|
|
652
|
+
response_meta?: ResponseMeta;
|
|
672
653
|
}
|
|
673
654
|
/**
|
|
674
655
|
* Webhook envelope delivered when an async scrape job completes. The `event`
|
|
675
656
|
* discriminator is always the literal `'scrape.complete'`.
|
|
676
657
|
*/
|
|
677
658
|
interface ScrapeCompleteWebhook {
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
659
|
+
/** Unique event identifier — also delivered in the `X-Cwbl-Event-Id` header. */
|
|
660
|
+
event_id: string;
|
|
661
|
+
/** ISO-8601 UTC timestamp when the event was emitted. */
|
|
662
|
+
timestamp: string;
|
|
663
|
+
/** Event type discriminator. */
|
|
664
|
+
event: 'scrape.complete';
|
|
665
|
+
/** Event payload. */
|
|
666
|
+
data: ScrapeCompleteWebhookData;
|
|
686
667
|
}
|
|
687
|
-
|
|
668
|
+
//#endregion
|
|
669
|
+
//#region src/client.d.ts
|
|
688
670
|
/** Options accepted by the {@link Crawlbrulee} constructor. */
|
|
689
671
|
interface CrawlbruleeOptions {
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
672
|
+
/**
|
|
673
|
+
* API key sent as `Authorization: Bearer <key>`. Required — to read from the
|
|
674
|
+
* environment instead, use {@link Crawlbrulee.fromEnv}. Leading and trailing
|
|
675
|
+
* whitespace is stripped; an empty / whitespace-only value is rejected.
|
|
676
|
+
*/
|
|
677
|
+
apiKey: string;
|
|
678
|
+
/**
|
|
679
|
+
* Override the base URL the SDK targets. Defaults to the production host
|
|
680
|
+
* ({@link DEFAULT_BASE_URL}). Intended for local development and staging
|
|
681
|
+
* (e.g. `https://api.staging.crawlbrulee.com`) — production callers should
|
|
682
|
+
* leave it unset. Trailing slashes are stripped.
|
|
683
|
+
*/
|
|
684
|
+
baseUrl?: string;
|
|
685
|
+
/**
|
|
686
|
+
* Per-request timeout in milliseconds. Defaults to `0` (no timeout). Set to a
|
|
687
|
+
* positive number to abort slow requests; a per-call `timeoutMs` override
|
|
688
|
+
* takes precedence. The timeout covers the WHOLE request, including the
|
|
689
|
+
* response body read.
|
|
690
|
+
*/
|
|
691
|
+
timeoutMs?: number;
|
|
710
692
|
}
|
|
711
693
|
/**
|
|
712
694
|
* Options accepted by {@link Crawlbrulee.waitForScrape}.
|
|
@@ -717,13 +699,13 @@ interface CrawlbruleeOptions {
|
|
|
717
699
|
* the client with `timeoutMs` set.
|
|
718
700
|
*/
|
|
719
701
|
interface WaitForScrapeOptions extends Omit<RequestOptions, 'timeoutMs'> {
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
702
|
+
/** Time between status polls in milliseconds. Default `2000`. */
|
|
703
|
+
intervalMs?: number;
|
|
704
|
+
/**
|
|
705
|
+
* Maximum total time to wait before giving up, in milliseconds. Default
|
|
706
|
+
* `300_000` (5 minutes). Pass `0` to wait indefinitely.
|
|
707
|
+
*/
|
|
708
|
+
timeoutMs?: number;
|
|
727
709
|
}
|
|
728
710
|
/**
|
|
729
711
|
* Official client for the crawlbrulee API.
|
|
@@ -744,102 +726,103 @@ interface WaitForScrapeOptions extends Omit<RequestOptions, 'timeoutMs'> {
|
|
|
744
726
|
* ```
|
|
745
727
|
*/
|
|
746
728
|
declare class Crawlbrulee {
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
729
|
+
/** Resolved base URL — trailing slash already stripped. */
|
|
730
|
+
readonly baseUrl: string;
|
|
731
|
+
/** Underlying HTTP layer. Exposed for advanced use cases (custom endpoints). */
|
|
732
|
+
readonly http: HttpClient;
|
|
733
|
+
constructor(options: CrawlbruleeOptions);
|
|
734
|
+
/**
|
|
735
|
+
* Build a {@link Crawlbrulee} reading the API key from
|
|
736
|
+
* `process.env.CRAWLBRULEE_API_KEY`. Throws if the variable is unset, empty,
|
|
737
|
+
* or whitespace.
|
|
738
|
+
*
|
|
739
|
+
* Any other constructor option can be passed via `overrides`.
|
|
740
|
+
*
|
|
741
|
+
* @example
|
|
742
|
+
* ```ts
|
|
743
|
+
* const crawlbrulee = Crawlbrulee.fromEnv()
|
|
744
|
+
* const crawlbrulee = Crawlbrulee.fromEnv({ timeoutMs: 30_000 })
|
|
745
|
+
* ```
|
|
746
|
+
*/
|
|
747
|
+
static fromEnv(overrides?: Omit<CrawlbruleeOptions, 'apiKey'>): Crawlbrulee;
|
|
748
|
+
/**
|
|
749
|
+
* Scrape a URL synchronously and return the extracted content.
|
|
750
|
+
*
|
|
751
|
+
* The request blocks until the scrape is finished. For long-running jobs
|
|
752
|
+
* (heavy JS rendering, screenshots of long pages) prefer
|
|
753
|
+
* {@link Crawlbrulee.scrapeAsync} so the connection isn't held open.
|
|
754
|
+
*
|
|
755
|
+
* @param request — body for `POST /api/scrape`.
|
|
756
|
+
* @param options — per-call timeout and abort signal.
|
|
757
|
+
*/
|
|
758
|
+
scrape(request: ScrapeRequest, options?: RequestOptions): Promise<ScrapeResponse>;
|
|
759
|
+
/**
|
|
760
|
+
* Submit an asynchronous scrape job and return its `job_id`. Poll the job
|
|
761
|
+
* with {@link Crawlbrulee.getScrapeStatus} or wait for completion with
|
|
762
|
+
* {@link Crawlbrulee.waitForScrape}.
|
|
763
|
+
*
|
|
764
|
+
* Pass an optional `webhook` to have the API deliver a signed
|
|
765
|
+
* `scrape.complete` `POST` to your endpoint when the job finishes (see
|
|
766
|
+
* {@link AsyncScrapeWebhook}). This field is async-only.
|
|
767
|
+
*/
|
|
768
|
+
scrapeAsync(request: AsyncScrapeRequest, options?: RequestOptions): Promise<AsyncScrapeResponse>;
|
|
769
|
+
/** Look up the current status of an async scrape job. */
|
|
770
|
+
getScrapeStatus(jobId: string, options?: RequestOptions): Promise<AsyncJobStatusResponse>;
|
|
771
|
+
/**
|
|
772
|
+
* Fetch the result of a completed async scrape job. Throws if the job is
|
|
773
|
+
* still pending/running — call {@link Crawlbrulee.getScrapeStatus}
|
|
774
|
+
* first, or use {@link Crawlbrulee.waitForScrape} to poll-then-fetch.
|
|
775
|
+
*/
|
|
776
|
+
getScrapeResult(jobId: string, options?: RequestOptions): Promise<ScrapeResponse>;
|
|
777
|
+
/**
|
|
778
|
+
* Fetch the scrape result referenced by a `scrape.complete` webhook body.
|
|
779
|
+
*
|
|
780
|
+
* Always verify the webhook signature with `verifyWebhookSignature` before
|
|
781
|
+
* acting on it; this method trusts the parsed body it is handed.
|
|
782
|
+
*
|
|
783
|
+
* Behavior by `data.status`:
|
|
784
|
+
* - `success` — delegates to {@link Crawlbrulee.getScrapeResult} for the
|
|
785
|
+
* webhook's `job_id` and returns the parsed result.
|
|
786
|
+
* - `failed` — throws a {@link CrawlbruleeError} carrying `data.error`
|
|
787
|
+
* (`errorName: 'job_failed'`); there is no result to fetch.
|
|
788
|
+
* - `cancelled` — throws a {@link CrawlbruleeError}
|
|
789
|
+
* (`errorName: 'client_closed_request'`).
|
|
790
|
+
*
|
|
791
|
+
* A non-`scrape.complete` envelope throws a {@link CrawlbruleeError}
|
|
792
|
+
* defensively. Any HTTP error from the underlying fetch propagates as the
|
|
793
|
+
* usual typed `CrawlbruleeError` subclass.
|
|
794
|
+
*/
|
|
795
|
+
fetchScrapeResultFromWebhook(webhook: ScrapeCompleteWebhook, options?: RequestOptions): Promise<ScrapeResponse>;
|
|
796
|
+
/**
|
|
797
|
+
* Poll an async scrape job until it reaches a terminal state, then return
|
|
798
|
+
* the scrape result.
|
|
799
|
+
*
|
|
800
|
+
* Throws a {@link CrawlbruleeError} when:
|
|
801
|
+
* - the job ends in `failed` (`errorName: 'job_failed'`),
|
|
802
|
+
* - the server reports an unexpected status (`errorName: 'job_failed'`),
|
|
803
|
+
* - the overall wait exceeds `timeoutMs` (`errorName: 'request_timeout'`),
|
|
804
|
+
* - the caller's `signal` aborts (`errorName: 'client_closed_request'`).
|
|
805
|
+
*/
|
|
806
|
+
waitForScrape(jobId: string, options?: WaitForScrapeOptions): Promise<ScrapeResponse>;
|
|
807
|
+
/**
|
|
808
|
+
* Build (or return a cached) site link-map for a domain. Combines sitemap
|
|
809
|
+
* discovery with the freshest cached homepage scrape when available.
|
|
810
|
+
*/
|
|
811
|
+
map(request: MapRequest, options?: RequestOptions): Promise<MapResponse>;
|
|
812
|
+
/**
|
|
813
|
+
* Return the current billing-cycle usage: total/used/available credits,
|
|
814
|
+
* used quota percentage, max concurrency, and when the cycle resets.
|
|
815
|
+
*/
|
|
816
|
+
usage(options?: RequestOptions): Promise<UsageResponse>;
|
|
817
|
+
/**
|
|
818
|
+
* Return the organization name and identifying details of the API token
|
|
819
|
+
* used to authenticate this request. Useful for confirming which key is in
|
|
820
|
+
* use before performing destructive operations.
|
|
821
|
+
*/
|
|
822
|
+
whoami(options?: RequestOptions): Promise<WhoamiResponse>;
|
|
841
823
|
}
|
|
842
|
-
|
|
824
|
+
//#endregion
|
|
825
|
+
//#region src/errors.d.ts
|
|
843
826
|
/**
|
|
844
827
|
* Base error class for every failure raised by the SDK.
|
|
845
828
|
*
|
|
@@ -858,29 +841,29 @@ declare class Crawlbrulee {
|
|
|
858
841
|
* {@link isCrawlbruleeError} helper.
|
|
859
842
|
*/
|
|
860
843
|
declare class CrawlbruleeError extends Error {
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
844
|
+
/** HTTP status code; `0` for transport-level failures with no response. */
|
|
845
|
+
readonly status: number;
|
|
846
|
+
/** The `name` field from the API error body, or `null` for transport errors. */
|
|
847
|
+
readonly errorName: ApiErrorName | null;
|
|
848
|
+
/** Structured detail block from the API error body, if any. */
|
|
849
|
+
readonly details?: ApiErrorDetails;
|
|
850
|
+
/** The original parsed error body, when one was received. */
|
|
851
|
+
readonly response?: ApiErrorResponse;
|
|
852
|
+
constructor(message: string, options: {
|
|
853
|
+
status: number;
|
|
854
|
+
errorName: ApiErrorName | null;
|
|
855
|
+
details?: ApiErrorDetails;
|
|
856
|
+
response?: ApiErrorResponse;
|
|
857
|
+
cause?: unknown;
|
|
858
|
+
});
|
|
876
859
|
}
|
|
877
860
|
/** Raised for 401 / 403 responses (missing, invalid, or unauthorized API key). */
|
|
878
861
|
declare class AuthenticationError extends CrawlbruleeError {
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
862
|
+
constructor(message: string, options: {
|
|
863
|
+
status: number;
|
|
864
|
+
errorName: ApiErrorName;
|
|
865
|
+
response?: ApiErrorResponse;
|
|
866
|
+
});
|
|
884
867
|
}
|
|
885
868
|
/**
|
|
886
869
|
* Raised for HTTP 429 responses. When the server included a `retry_after_ms`
|
|
@@ -892,16 +875,16 @@ declare class AuthenticationError extends CrawlbruleeError {
|
|
|
892
875
|
* available on `response`.
|
|
893
876
|
*/
|
|
894
877
|
declare class RateLimitError extends CrawlbruleeError {
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
|
|
899
|
-
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
878
|
+
readonly errorName: 'too_many_requests';
|
|
879
|
+
/** Suggested delay (ms) before retrying, when the server provided one. */
|
|
880
|
+
readonly retryAfterMs?: number;
|
|
881
|
+
/** Which rate limit was tripped (e.g. `org`, `ip`), when provided. */
|
|
882
|
+
readonly limitedBy?: string;
|
|
883
|
+
constructor(message: string, options: {
|
|
884
|
+
status: number;
|
|
885
|
+
details?: RateLimitErrorDetails;
|
|
886
|
+
response?: ApiErrorResponse;
|
|
887
|
+
});
|
|
905
888
|
}
|
|
906
889
|
/**
|
|
907
890
|
* Raised when the API rejects a request because the org's plan limits would
|
|
@@ -910,32 +893,32 @@ declare class RateLimitError extends CrawlbruleeError {
|
|
|
910
893
|
* `errorName` is always the literal `'usage_allocation_error'`.
|
|
911
894
|
*/
|
|
912
895
|
declare class UsageAllocationError extends CrawlbruleeError {
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
896
|
+
readonly errorName: 'usage_allocation_error';
|
|
897
|
+
/** Specific reason the allocation was denied. */
|
|
898
|
+
readonly reason: UsageAllocationErrorDetails['reason'];
|
|
899
|
+
/** Current usage / limit snapshot at the time of the rejection. */
|
|
900
|
+
readonly usage?: UsageAllocationErrorDetails['details'];
|
|
901
|
+
constructor(message: string, options: {
|
|
902
|
+
status: number;
|
|
903
|
+
details: UsageAllocationErrorDetails;
|
|
904
|
+
response?: ApiErrorResponse;
|
|
905
|
+
});
|
|
923
906
|
}
|
|
924
907
|
/** Raised for 4xx responses caused by an invalid request shape or arguments. */
|
|
925
908
|
declare class ValidationError extends CrawlbruleeError {
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
909
|
+
constructor(message: string, options: {
|
|
910
|
+
status: number;
|
|
911
|
+
errorName: ApiErrorName;
|
|
912
|
+
response?: ApiErrorResponse;
|
|
913
|
+
});
|
|
931
914
|
}
|
|
932
915
|
/** Raised for 404 responses (e.g. unknown async job ID). */
|
|
933
916
|
declare class NotFoundError extends CrawlbruleeError {
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
917
|
+
constructor(message: string, options: {
|
|
918
|
+
status: number;
|
|
919
|
+
errorName: ApiErrorName;
|
|
920
|
+
response?: ApiErrorResponse;
|
|
921
|
+
});
|
|
939
922
|
}
|
|
940
923
|
/**
|
|
941
924
|
* Raised when a request cannot be sent or no structured response is parsed.
|
|
@@ -946,15 +929,16 @@ declare class NotFoundError extends CrawlbruleeError {
|
|
|
946
929
|
* - `null` — generic transport failure (network error, non-JSON body, etc.).
|
|
947
930
|
*/
|
|
948
931
|
declare class TransportError extends CrawlbruleeError {
|
|
949
|
-
|
|
950
|
-
|
|
951
|
-
|
|
952
|
-
|
|
953
|
-
|
|
932
|
+
constructor(message: string, options?: {
|
|
933
|
+
status?: number;
|
|
934
|
+
errorName?: 'request_timeout' | 'client_closed_request' | null;
|
|
935
|
+
cause?: unknown;
|
|
936
|
+
});
|
|
954
937
|
}
|
|
955
938
|
/** Narrow `unknown` to the SDK's base error type. */
|
|
956
939
|
declare function isCrawlbruleeError(err: unknown): err is CrawlbruleeError;
|
|
957
|
-
|
|
940
|
+
//#endregion
|
|
941
|
+
//#region src/config.d.ts
|
|
958
942
|
/**
|
|
959
943
|
* Production base URL of the crawlbrulee API. Used by default when the caller
|
|
960
944
|
* doesn't pass a `baseUrl` to {@link Crawlbrulee}. Local development and
|
|
@@ -965,7 +949,8 @@ declare const DEFAULT_BASE_URL = "https://api.crawlbrulee.com";
|
|
|
965
949
|
declare const DEFAULT_REQUEST_TIMEOUT_MS = 0;
|
|
966
950
|
/** Environment variable read by `Crawlbrulee.fromEnv()` to source the API key. */
|
|
967
951
|
declare const ENV_API_KEY = "CRAWLBRULEE_API_KEY";
|
|
968
|
-
|
|
952
|
+
//#endregion
|
|
953
|
+
//#region src/webhooks.d.ts
|
|
969
954
|
/**
|
|
970
955
|
* Verification for async scrape completion webhooks.
|
|
971
956
|
*
|
|
@@ -1001,33 +986,33 @@ type WebhookSignatureSource = 'primary' | 'rotated';
|
|
|
1001
986
|
type WebhookVerificationFailureReason = 'missing_signature' | 'malformed_signature' | 'timestamp_out_of_tolerance' | 'signature_mismatch';
|
|
1002
987
|
/** Result of {@link verifyWebhookSignature}. Verification failure is returned, not thrown. */
|
|
1003
988
|
type WebhookVerificationResult = {
|
|
1004
|
-
|
|
1005
|
-
|
|
989
|
+
verified: true;
|
|
990
|
+
signedWith: WebhookSignatureSource;
|
|
1006
991
|
} | {
|
|
1007
|
-
|
|
1008
|
-
|
|
992
|
+
verified: false;
|
|
993
|
+
reason: WebhookVerificationFailureReason;
|
|
1009
994
|
};
|
|
1010
995
|
/** Options for {@link verifyWebhookSignature}. */
|
|
1011
996
|
interface VerifyWebhookSignatureOptions {
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
|
|
997
|
+
/**
|
|
998
|
+
* The raw request body, exactly as received. Pass the bytes/string the server
|
|
999
|
+
* signed — do NOT re-serialize parsed JSON, or the signature will not match.
|
|
1000
|
+
*/
|
|
1001
|
+
payload: string | Uint8Array;
|
|
1002
|
+
/**
|
|
1003
|
+
* The request headers. Accepts a fetch `Headers` instance or a plain object
|
|
1004
|
+
* (Express/Node give lowercased keys, values possibly arrays). Lookup is
|
|
1005
|
+
* case-insensitive.
|
|
1006
|
+
*/
|
|
1007
|
+
headers: Headers | Record<string, string | string[] | undefined>;
|
|
1008
|
+
/** The current signing secret (`whsec_…`). */
|
|
1009
|
+
secret: string;
|
|
1010
|
+
/**
|
|
1011
|
+
* Replay-protection window in seconds. Defaults to
|
|
1012
|
+
* {@link DEFAULT_WEBHOOK_TOLERANCE_SECONDS} (300). Pass `0` (or any falsy
|
|
1013
|
+
* value) to disable the timestamp check entirely.
|
|
1014
|
+
*/
|
|
1015
|
+
toleranceSeconds?: number;
|
|
1031
1016
|
}
|
|
1032
1017
|
/**
|
|
1033
1018
|
* Verify a crawlbrulee webhook signature against the primary and rotated
|
|
@@ -1057,5 +1042,6 @@ interface VerifyWebhookSignatureOptions {
|
|
|
1057
1042
|
* ```
|
|
1058
1043
|
*/
|
|
1059
1044
|
declare function verifyWebhookSignature(options: VerifyWebhookSignatureOptions): Promise<WebhookVerificationResult>;
|
|
1060
|
-
|
|
1061
|
-
export {
|
|
1045
|
+
//#endregion
|
|
1046
|
+
export { ApiErrorDetails, ApiErrorName, ApiErrorResponse, AsyncJobStatus, AsyncJobStatusResponse, AsyncScrapeRequest, AsyncScrapeResponse, AsyncScrapeWebhook, AuthenticationError, Crawlbrulee, CrawlbruleeError, type CrawlbruleeOptions, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_WEBHOOK_TOLERANCE_SECONDS, ENV_API_KEY, type HttpMethod, MapCache, MapLinkItem, MapLocation, MapPagination, MapRequest, MapResponse, MapResponseMeta, MapTruncation, MapTypes, NotFoundError, PageInlineImage, PageLink, ProxyTier, RateLimitError, RateLimitErrorDetails, type RequestOptions, ResolvedProxyTier, ResponseMeta, ScrapeCache, ScrapeCompleteWebhook, ScrapeCompleteWebhookData, ScrapeExtract, ScrapeLocation, ScrapeMetadata, ScrapeRequest, ScrapeResponse, ScrapeWebhookStatus, ScreenshotAfterAction, ScreenshotBeforeAction, ScreenshotCleanup, ScreenshotDeviceMode, ScreenshotProperties, ScreenshotRequest, ScreenshotResult, ScreenshotScrollAction, ScreenshotSlice, ScreenshotSliceAction, ScreenshotType, ScreenshotViewport, ScreenshotViewportInfo, ScreenshotWaitAction, TransportError, Usage, UsageAllocationError, UsageAllocationErrorDetails, UsageAllocationReason, UsageLimitDetails, UsageResponse, ValidationError, type VerifyWebhookSignatureOptions, WEBHOOK_EVENT_ID_HEADER, WEBHOOK_SIGNATURE_HEADER, WEBHOOK_SIGNATURE_ROTATED_HEADER, type WaitForScrapeOptions, type WebhookSignatureSource, type WebhookVerificationFailureReason, type WebhookVerificationResult, WhoamiResponse, isCrawlbruleeError, verifyWebhookSignature };
|
|
1047
|
+
//# sourceMappingURL=index.d.cts.map
|