@crawlbrulee/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +219 -0
- package/dist/index.cjs +538 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +781 -0
- package/dist/index.d.ts +781 -0
- package/dist/index.js +525 -0
- package/dist/index.js.map +1 -0
- package/package.json +71 -0
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,781 @@
|
|
|
1
|
+
/** HTTP methods used by the SDK. */
|
|
2
|
+
type HttpMethod = 'GET' | 'POST';
|
|
3
|
+
/** Options the SDK accepts at construction time for the HTTP layer. */
|
|
4
|
+
interface HttpClientOptions {
|
|
5
|
+
/** Base URL of the API (trailing slash is stripped). */
|
|
6
|
+
baseUrl: string;
|
|
7
|
+
/** API key sent as `Authorization: Bearer <key>`. */
|
|
8
|
+
apiKey: string;
|
|
9
|
+
/**
|
|
10
|
+
* Per-request timeout in milliseconds. Pass `0` (or omit) to disable the
|
|
11
|
+
* timeout entirely.
|
|
12
|
+
*/
|
|
13
|
+
timeoutMs?: number;
|
|
14
|
+
}
|
|
15
|
+
/** Per-call overrides accepted on every resource method. */
|
|
16
|
+
interface RequestOptions {
|
|
17
|
+
/** Abort the request when this signal fires. Composable with the timeout. */
|
|
18
|
+
signal?: AbortSignal;
|
|
19
|
+
/**
|
|
20
|
+
* Override the constructor-level `timeoutMs` for this call. Pass `0` to
|
|
21
|
+
* disable the timeout for this call.
|
|
22
|
+
*/
|
|
23
|
+
timeoutMs?: number;
|
|
24
|
+
}
|
|
25
|
+
/**
|
|
26
|
+
* Minimal `fetch`-based HTTP layer used by {@link Crawlbrulee}. Handles:
|
|
27
|
+
*
|
|
28
|
+
* - URL composition (joining `baseUrl` and path safely).
|
|
29
|
+
* - JSON serialization and parsing.
|
|
30
|
+
* - The `Authorization: Bearer …` header.
|
|
31
|
+
* - Composing the caller's `AbortSignal` with an internal timeout signal. The
|
|
32
|
+
* timeout covers the WHOLE request, including the response body read — not
|
|
33
|
+
* just the time-to-headers.
|
|
34
|
+
* - Mapping non-2xx responses to typed `CrawlbruleeError` subclasses via
|
|
35
|
+
* {@link createApiError}.
|
|
36
|
+
*
|
|
37
|
+
* The `fetch` implementation is sourced from {@link CwblInstrumentation} at
|
|
38
|
+
* construction time so tests can stub the module.
|
|
39
|
+
*/
|
|
40
|
+
declare class HttpClient {
|
|
41
|
+
private readonly baseUrl;
|
|
42
|
+
private readonly apiKey;
|
|
43
|
+
private readonly fetch;
|
|
44
|
+
private readonly timeoutMs;
|
|
45
|
+
constructor(options: HttpClientOptions);
|
|
46
|
+
/** Send a `GET` request and parse the response as `T`. */
|
|
47
|
+
get<T>(path: string, options?: RequestOptions): Promise<T>;
|
|
48
|
+
/** Send a `POST` request with a JSON body and parse the response as `T`. */
|
|
49
|
+
post<T>(path: string, body: unknown, options?: RequestOptions): Promise<T>;
|
|
50
|
+
private send;
|
|
51
|
+
private buildUrl;
|
|
52
|
+
private buildHeaders;
|
|
53
|
+
/**
|
|
54
|
+
* Build a single `AbortSignal` that fires when either the caller-supplied
|
|
55
|
+
* signal aborts OR the per-request timeout elapses. The returned `cleanup`
|
|
56
|
+
* callback MUST be invoked on every exit path so we don't leak timers or
|
|
57
|
+
* dead listeners on long-lived caller signals.
|
|
58
|
+
*/
|
|
59
|
+
private composeSignal;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Shared primitive types used across crawlbrulee request and response shapes.
|
|
64
|
+
*/
|
|
65
|
+
/**
|
|
66
|
+
* Proxy tier used to route the fetch.
|
|
67
|
+
*
|
|
68
|
+
* - `basic` — datacenter proxy, lowest cost (default).
|
|
69
|
+
* - `advanced` — residential proxy, higher success rate on protected sites.
|
|
70
|
+
* - `auto` — let crawlbrulee pick the right tier per target.
|
|
71
|
+
* - `none` — skip the proxy entirely. Rejected in production; available on
|
|
72
|
+
* staging only as a debug/perf-test toggle.
|
|
73
|
+
*/
|
|
74
|
+
type ProxyTier = 'basic' | 'advanced' | 'auto' | 'none';
|
|
75
|
+
/** Screenshot capture mode: visible viewport or the full scrollable page. */
|
|
76
|
+
type ScreenshotType = 'viewport' | 'full_page';
|
|
77
|
+
/** Emulated device class for the viewport (drives default width/height). */
|
|
78
|
+
type ScreenshotDeviceMode = 'desktop' | 'mobile';
|
|
79
|
+
/** Pre-capture cleanup options applied to the page before the screenshot. */
|
|
80
|
+
interface ScreenshotCleanup {
|
|
81
|
+
/**
|
|
82
|
+
* Remove ads, cookie banners, and popups before capturing. Defaults to
|
|
83
|
+
* `true` server-side.
|
|
84
|
+
*/
|
|
85
|
+
ads_and_popups?: boolean;
|
|
86
|
+
}
|
|
87
|
+
/** A `wait` action: pause for `ms` milliseconds before the next step. */
|
|
88
|
+
interface ScreenshotWaitAction {
|
|
89
|
+
type: 'wait';
|
|
90
|
+
/** Milliseconds to wait. Must be a non-negative integer. */
|
|
91
|
+
ms: number;
|
|
92
|
+
}
|
|
93
|
+
/** A `scroll` action: scroll the page by `pixels` (positive = down). */
|
|
94
|
+
interface ScreenshotScrollAction {
|
|
95
|
+
type: 'scroll';
|
|
96
|
+
/** Pixels to scroll. Positive scrolls down, negative scrolls up. */
|
|
97
|
+
pixels: number;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Actions performed before the screenshot is taken. The server caps the total
|
|
101
|
+
* wait time (~20 s) and total absolute scroll distance (~50 000 px) across
|
|
102
|
+
* the array; at most 5 actions are accepted.
|
|
103
|
+
*/
|
|
104
|
+
type ScreenshotBeforeAction = ScreenshotWaitAction | ScreenshotScrollAction;
|
|
105
|
+
/**
|
|
106
|
+
* Action performed after the screenshot is taken. Currently only `slice` is
|
|
107
|
+
* supported — it cuts the screenshot into horizontal tiles of `height` px.
|
|
108
|
+
*/
|
|
109
|
+
interface ScreenshotSliceAction {
|
|
110
|
+
type: 'slice';
|
|
111
|
+
/** Tile height in pixels. Minimum 500. */
|
|
112
|
+
height: number;
|
|
113
|
+
}
|
|
114
|
+
type ScreenshotAfterAction = ScreenshotSliceAction;
|
|
115
|
+
/** Custom browser viewport dimensions used during a screenshot capture. */
|
|
116
|
+
interface ScreenshotViewport {
|
|
117
|
+
/** Viewport width in pixels. */
|
|
118
|
+
width: number;
|
|
119
|
+
/** Viewport height in pixels. */
|
|
120
|
+
height: number;
|
|
121
|
+
/** Device pixel ratio (e.g. 2 for retina). Defaults to 1 server-side. */
|
|
122
|
+
device_scale_factor?: number;
|
|
123
|
+
}
|
|
124
|
+
/** Screenshot capture configuration. Pass this on `extract.screenshot`. */
|
|
125
|
+
interface ScreenshotRequest {
|
|
126
|
+
/** Capture mode: `viewport` (visible only) or `full_page`. */
|
|
127
|
+
type: ScreenshotType;
|
|
128
|
+
/** Custom viewport. If omitted, the `device_mode` defaults are used. */
|
|
129
|
+
viewport?: ScreenshotViewport;
|
|
130
|
+
/** Emulate desktop or mobile. Defaults to `desktop`. */
|
|
131
|
+
device_mode?: ScreenshotDeviceMode;
|
|
132
|
+
/** Page cleanup applied before capture. */
|
|
133
|
+
cleanup?: ScreenshotCleanup;
|
|
134
|
+
/** Pre-capture actions (waits and scrolls). Maximum 5 entries. */
|
|
135
|
+
actions_before?: ScreenshotBeforeAction[];
|
|
136
|
+
/** Post-capture actions (e.g. slice into tiles). Maximum 1 entry. */
|
|
137
|
+
actions_after?: ScreenshotAfterAction[];
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Machine-readable error names returned by the crawlbrulee API. Stable
|
|
141
|
+
* identifiers — clients can switch on them.
|
|
142
|
+
*/
|
|
143
|
+
type ApiErrorName = 'usage_allocation_error' | 'request_timeout' | 'invalid_url' | 'url_too_long' | 'client_closed_request' | 'reset_password_token_expired' | 'user_not_found' | 'unsupported_url_schema' | 'url_credentials_not_supported' | 'blocked_url' | 'scrape_error' | 'job_failed' | 'incorrect_login_method_used' | 'not_found' | 'invalid_credentials' | 'resource_already_exists' | 'access_denied' | 'internal_server_error' | 'too_many_requests' | 'unsupported_content' | 'validation_error' | 'antibot_blocked';
|
|
144
|
+
/** Reason a usage allocation was denied (when `error_name = usage_allocation_error`). */
|
|
145
|
+
type UsageAllocationReason = 'credit_limit' | 'concurrency_limit' | 'overage_hard_cap' | 'duplicate_reservation' | 'internal_error';
|
|
146
|
+
/** Snapshot of the org's current usage at the moment the error was raised. */
|
|
147
|
+
interface UsageLimitDetails {
|
|
148
|
+
/** Current credit usage in the billing period. */
|
|
149
|
+
current_usage?: number;
|
|
150
|
+
/** Credits currently reserved by in-flight jobs. */
|
|
151
|
+
current_reserved?: number;
|
|
152
|
+
/** Maximum credits allowed in the billing period. */
|
|
153
|
+
max_credits?: number;
|
|
154
|
+
/** Number of currently running concurrent jobs. */
|
|
155
|
+
current_concurrent?: number;
|
|
156
|
+
/** Maximum concurrent jobs allowed. */
|
|
157
|
+
max_concurrent?: number;
|
|
158
|
+
}
|
|
159
|
+
/** Discriminated detail for `error_name = usage_allocation_error`. */
|
|
160
|
+
interface UsageAllocationErrorDetails {
|
|
161
|
+
error_name: 'usage_allocation_error';
|
|
162
|
+
reason: UsageAllocationReason;
|
|
163
|
+
details?: UsageLimitDetails;
|
|
164
|
+
}
|
|
165
|
+
/** Discriminated detail for `error_name = too_many_requests`. */
|
|
166
|
+
interface RateLimitErrorDetails {
|
|
167
|
+
error_name: 'too_many_requests';
|
|
168
|
+
/** Suggested wait time before retrying. */
|
|
169
|
+
retry_after_ms?: number;
|
|
170
|
+
/** Which rate limit was exceeded (e.g. `org`, `ip`). */
|
|
171
|
+
limited_by?: string;
|
|
172
|
+
}
|
|
173
|
+
/** Union of all known `details` payloads on an `ApiErrorResponse`. */
|
|
174
|
+
type ApiErrorDetails = UsageAllocationErrorDetails | RateLimitErrorDetails;
|
|
175
|
+
/** Standard JSON error shape returned for any non-2xx response. */
|
|
176
|
+
interface ApiErrorResponse {
|
|
177
|
+
/** Machine-readable identifier — safe to switch on. */
|
|
178
|
+
name: ApiErrorName;
|
|
179
|
+
/** Human-readable error message. */
|
|
180
|
+
message: string;
|
|
181
|
+
/** Error-specific structured detail; only present for some `name`s. */
|
|
182
|
+
details?: ApiErrorDetails;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/**
|
|
186
|
+
* Which content formats to extract from the scraped page. Every field is
|
|
187
|
+
* optional; the server defaults are noted on each field. The default request
|
|
188
|
+
* extracts `{ metadata: true, cleaned_html: true }`.
|
|
189
|
+
*/
|
|
190
|
+
interface ScrapeExtract {
|
|
191
|
+
/** Extract page metadata (title, description, OG/Twitter tags, etc.). Default `true`. */
|
|
192
|
+
metadata?: boolean;
|
|
193
|
+
/** Extract cleaned HTML (main content only). Default `true`. */
|
|
194
|
+
cleaned_html?: boolean;
|
|
195
|
+
/** Extract the page as clean Markdown. Default `false`. */
|
|
196
|
+
markdown?: boolean;
|
|
197
|
+
/** Return the raw, unprocessed HTML. Default `false`. */
|
|
198
|
+
raw_html?: boolean;
|
|
199
|
+
/** Extract all links found on the page. Default `false`. */
|
|
200
|
+
links?: boolean;
|
|
201
|
+
/** Extract all inline images found on the page. Default `false`. */
|
|
202
|
+
images?: boolean;
|
|
203
|
+
/** Capture a screenshot. Omit to skip; set to a `ScreenshotRequest` to enable. */
|
|
204
|
+
screenshot?: ScreenshotRequest;
|
|
205
|
+
}
|
|
206
|
+
/** Cache settings for a scrape request. */
|
|
207
|
+
interface ScrapeCache {
|
|
208
|
+
/**
|
|
209
|
+
* Maximum cache age. Either a number of seconds (non-negative integer) or
|
|
210
|
+
* an ISO-8601 datetime cutoff — cached entries older than this are skipped.
|
|
211
|
+
* Defaults to 2 days when omitted.
|
|
212
|
+
*/
|
|
213
|
+
max_age?: number | string;
|
|
214
|
+
/**
|
|
215
|
+
* Treat URLs with different query parameters as the same cache entry.
|
|
216
|
+
* Defaults to `false`.
|
|
217
|
+
*/
|
|
218
|
+
ignore_query_params?: boolean;
|
|
219
|
+
}
|
|
220
|
+
/** Optional locale + country emulation for the scrape. */
|
|
221
|
+
interface ScrapeLocation {
|
|
222
|
+
/**
|
|
223
|
+
* BCP-47 locale (e.g. `en-US`, `de-DE`, `pt-BR`). Sent as `Accept-Language`
|
|
224
|
+
* and reflected in `navigator.language` when JS rendering is requested.
|
|
225
|
+
*/
|
|
226
|
+
locale?: string;
|
|
227
|
+
/**
|
|
228
|
+
* ISO 3166-1 alpha-2 country code (e.g. `US`, `DE`, `BR`). Drives the
|
|
229
|
+
* emulated browser timezone. Case-insensitive.
|
|
230
|
+
*/
|
|
231
|
+
country?: string;
|
|
232
|
+
}
|
|
233
|
+
/** Request body for `POST /api/scrape` (and `POST /api/scrape/async`). */
|
|
234
|
+
interface ScrapeRequest {
|
|
235
|
+
/** The URL to scrape. */
|
|
236
|
+
url: string;
|
|
237
|
+
/** Which content formats to extract. Defaults to `metadata + cleaned_html`. */
|
|
238
|
+
extract?: ScrapeExtract;
|
|
239
|
+
/** Cache settings for this request. */
|
|
240
|
+
cache?: ScrapeCache;
|
|
241
|
+
/**
|
|
242
|
+
* Use a headless browser to render JavaScript before scraping. Adds latency
|
|
243
|
+
* and credits — only enable when the page requires it. Default `false`.
|
|
244
|
+
*/
|
|
245
|
+
require_js?: boolean;
|
|
246
|
+
/** CSS selectors to strip from the extracted content. */
|
|
247
|
+
exclude_selectors?: string[];
|
|
248
|
+
/** Proxy tier to use for fetching. Defaults to `basic`. */
|
|
249
|
+
proxy?: ProxyTier;
|
|
250
|
+
/** Optional locale + country emulation. */
|
|
251
|
+
location?: ScrapeLocation;
|
|
252
|
+
}
|
|
253
|
+
/** Viewport metadata returned alongside a captured screenshot. */
|
|
254
|
+
interface ScreenshotViewportInfo {
|
|
255
|
+
width: number;
|
|
256
|
+
height: number;
|
|
257
|
+
device_scale_factor: number;
|
|
258
|
+
}
|
|
259
|
+
/** Image-level properties of a captured screenshot (or tile). */
|
|
260
|
+
interface ScreenshotProperties {
|
|
261
|
+
/** File name of the image asset. */
|
|
262
|
+
file_name: string;
|
|
263
|
+
/** MIME type (e.g. `image/png`). */
|
|
264
|
+
mime: string;
|
|
265
|
+
/** Image width in pixels. */
|
|
266
|
+
width: number;
|
|
267
|
+
/** Image height in pixels. */
|
|
268
|
+
height: number;
|
|
269
|
+
/** Viewport dimensions used during capture. */
|
|
270
|
+
viewport: ScreenshotViewportInfo;
|
|
271
|
+
}
|
|
272
|
+
/** One horizontal tile of a sliced full-page screenshot. */
|
|
273
|
+
interface ScreenshotSlice {
|
|
274
|
+
/** 0-based row index of this slice. */
|
|
275
|
+
row_nr: number;
|
|
276
|
+
/** Signed URL to download this slice image. */
|
|
277
|
+
url: string;
|
|
278
|
+
type: 'slice';
|
|
279
|
+
/** Image-level properties of this slice. */
|
|
280
|
+
properties: ScreenshotProperties;
|
|
281
|
+
}
|
|
282
|
+
/** Result block returned when a screenshot was requested. */
|
|
283
|
+
interface ScreenshotResult {
|
|
284
|
+
/** Signed URL to download the full screenshot image. */
|
|
285
|
+
url: string;
|
|
286
|
+
/** Capture mode that was used. */
|
|
287
|
+
type: ScreenshotType;
|
|
288
|
+
/** Image-level properties of the full screenshot. */
|
|
289
|
+
properties: ScreenshotProperties;
|
|
290
|
+
/** Tile slices, present only when the `slice` `actions_after` was requested. */
|
|
291
|
+
slices?: ScreenshotSlice[];
|
|
292
|
+
}
|
|
293
|
+
/** A single inline image discovered on the page. */
|
|
294
|
+
interface PageInlineImage {
|
|
295
|
+
/** Absolute URL of the image. */
|
|
296
|
+
url: string;
|
|
297
|
+
/** Alt text of the image, or `null` if not set. */
|
|
298
|
+
alt: string | null;
|
|
299
|
+
}
|
|
300
|
+
/** A single link discovered on the page. */
|
|
301
|
+
interface PageLink {
|
|
302
|
+
/** Anchor text of the link. */
|
|
303
|
+
text: string;
|
|
304
|
+
/** The link URL as it appears on the page (absolute or relative). */
|
|
305
|
+
href: string;
|
|
306
|
+
/** Whether the link points to the same domain as the scraped page. */
|
|
307
|
+
internal: boolean;
|
|
308
|
+
}
|
|
309
|
+
/** Structured page metadata extracted from `<head>`. */
|
|
310
|
+
interface ScrapeMetadata {
|
|
311
|
+
title?: string;
|
|
312
|
+
description?: string;
|
|
313
|
+
keywords?: string[];
|
|
314
|
+
canonical?: string;
|
|
315
|
+
og_url?: string;
|
|
316
|
+
og_title?: string;
|
|
317
|
+
og_description?: string;
|
|
318
|
+
og_type?: string;
|
|
319
|
+
og_site_name?: string;
|
|
320
|
+
og_locale?: string;
|
|
321
|
+
og_locale_alternate?: string[];
|
|
322
|
+
og_image?: string;
|
|
323
|
+
author?: string;
|
|
324
|
+
date_modified?: string;
|
|
325
|
+
date_published?: string;
|
|
326
|
+
twitter_site?: string;
|
|
327
|
+
twitter_card?: string;
|
|
328
|
+
twitter_description?: string;
|
|
329
|
+
twitter_title?: string;
|
|
330
|
+
twitter_image?: string;
|
|
331
|
+
robots?: string;
|
|
332
|
+
favicon_url?: string | null;
|
|
333
|
+
}
|
|
334
|
+
/** Successful response from `POST /api/scrape` and `GET /api/scrape/result/:jobId`. */
|
|
335
|
+
interface ScrapeResponse {
|
|
336
|
+
/** The URL that was actually scraped (after any redirects). */
|
|
337
|
+
url: string;
|
|
338
|
+
/** `Content-Type` header returned by the origin. */
|
|
339
|
+
content_type?: string;
|
|
340
|
+
/**
|
|
341
|
+
* Extract fields that were requested but aren't supported for this
|
|
342
|
+
* content type (e.g. asking for `markdown` of a PDF).
|
|
343
|
+
*/
|
|
344
|
+
unsupported_fields?: string[];
|
|
345
|
+
/** Page content converted to clean Markdown (when `extract.markdown`). */
|
|
346
|
+
markdown?: string;
|
|
347
|
+
/** Cleaned HTML of the main page content (when `extract.cleaned_html`). */
|
|
348
|
+
cleaned_html?: string;
|
|
349
|
+
/** Raw, unprocessed HTML (when `extract.raw_html`). */
|
|
350
|
+
raw_html?: string;
|
|
351
|
+
/** Inline images discovered on the page (when `extract.images`). */
|
|
352
|
+
images?: PageInlineImage[];
|
|
353
|
+
/** Links discovered on the page (when `extract.links`). */
|
|
354
|
+
links?: PageLink[];
|
|
355
|
+
/** Captured screenshot (when `extract.screenshot`). */
|
|
356
|
+
screenshot?: ScreenshotResult;
|
|
357
|
+
/** Extracted page metadata (when `extract.metadata`, on by default). */
|
|
358
|
+
metadata?: ScrapeMetadata;
|
|
359
|
+
/**
|
|
360
|
+
* Non-error notices about the scrape (e.g. `screenshot_truncated` when a
|
|
361
|
+
* long page exceeded the scrolling-screenshot height cap). Stable codes —
|
|
362
|
+
* safe to switch on. Currently surfaced only on fresh scrapes; cache hits
|
|
363
|
+
* omit warnings.
|
|
364
|
+
*/
|
|
365
|
+
warnings?: string[];
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
/** Filter which link types appear in the map result. */
|
|
369
|
+
interface MapTypes {
|
|
370
|
+
/** Include internal links (same domain). Default `true`. */
|
|
371
|
+
internal?: boolean;
|
|
372
|
+
/** Include links to subdomains of the target. Default `true`. */
|
|
373
|
+
internal_subdomains?: boolean;
|
|
374
|
+
/** Include external links (different domains). Default `true`. */
|
|
375
|
+
external?: boolean;
|
|
376
|
+
}
|
|
377
|
+
/** Cache settings for a map request. */
|
|
378
|
+
interface MapCache {
|
|
379
|
+
/**
|
|
380
|
+
* Maximum cache age — either seconds or an ISO-8601 datetime cutoff.
|
|
381
|
+
* Defaults to 7 days when omitted.
|
|
382
|
+
*/
|
|
383
|
+
max_age?: number | string;
|
|
384
|
+
}
|
|
385
|
+
/** Country-only egress emulation (map requests have no locale knob). */
|
|
386
|
+
interface MapLocation {
|
|
387
|
+
/** ISO 3166-1 alpha-2 country code (e.g. `US`). Case-insensitive. */
|
|
388
|
+
country?: string;
|
|
389
|
+
}
|
|
390
|
+
/** Request body for `POST /api/map`. */
|
|
391
|
+
interface MapRequest {
|
|
392
|
+
/** The website URL to map. */
|
|
393
|
+
url: string;
|
|
394
|
+
/** Proxy tier to use for fetching. Defaults to `basic`. */
|
|
395
|
+
proxy?: ProxyTier;
|
|
396
|
+
/** Only use sitemap.xml — skip homepage link extraction. Default `false`. */
|
|
397
|
+
sitemap_only?: boolean;
|
|
398
|
+
/** Filter which link types to include. */
|
|
399
|
+
types?: MapTypes;
|
|
400
|
+
/** Cache settings for this request. */
|
|
401
|
+
cache?: MapCache;
|
|
402
|
+
/**
|
|
403
|
+
* Maximum number of URLs to store in the map. Must be in `(0, 100 000]`.
|
|
404
|
+
* Defaults to 100 000.
|
|
405
|
+
*/
|
|
406
|
+
max_urls?: number;
|
|
407
|
+
/** 1-based page number for paginated results. Defaults to 1. */
|
|
408
|
+
page?: number;
|
|
409
|
+
/**
|
|
410
|
+
* Number of URLs per page. Must be in `(0, 10 000]`. Defaults to 10 000.
|
|
411
|
+
*/
|
|
412
|
+
limit?: number;
|
|
413
|
+
/** Optional country emulation. */
|
|
414
|
+
location?: MapLocation;
|
|
415
|
+
}
|
|
416
|
+
/** Single discovered URL in a map result. */
|
|
417
|
+
interface MapLinkItem {
|
|
418
|
+
/** The discovered URL. */
|
|
419
|
+
url: string;
|
|
420
|
+
}
|
|
421
|
+
/** Pagination details on a map response. */
|
|
422
|
+
interface MapPagination {
|
|
423
|
+
page: number;
|
|
424
|
+
limit: number;
|
|
425
|
+
/** Total number of URLs in the stored map. */
|
|
426
|
+
total: number;
|
|
427
|
+
total_pages: number;
|
|
428
|
+
has_more: boolean;
|
|
429
|
+
}
|
|
430
|
+
/** Information about whether the stored or returned map was truncated. */
|
|
431
|
+
interface MapTruncation {
|
|
432
|
+
/** Whether the stored map was capped by `max_urls`. */
|
|
433
|
+
storage_capped: boolean;
|
|
434
|
+
/** Whether the response was capped by pagination. */
|
|
435
|
+
response_capped: boolean;
|
|
436
|
+
/** Total URLs found before the `max_urls` cap was applied. */
|
|
437
|
+
total_before_max_urls: number;
|
|
438
|
+
/** Total URLs detected during discovery before the storage cap was applied. */
|
|
439
|
+
total_detected_before_storage_cap: number;
|
|
440
|
+
}
|
|
441
|
+
/** Success response from `POST /api/map`. */
|
|
442
|
+
interface MapResponse {
|
|
443
|
+
/** The current page of discovered URLs. */
|
|
444
|
+
links: MapLinkItem[];
|
|
445
|
+
/** Pagination + truncation metadata for the result set. */
|
|
446
|
+
meta: {
|
|
447
|
+
pagination: MapPagination;
|
|
448
|
+
truncation: MapTruncation;
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** Job lifecycle states for an async scrape. */
|
|
453
|
+
type AsyncJobStatus = 'pending' | 'running' | 'done' | 'failed';
|
|
454
|
+
/** Response body of `POST /api/scrape/async`. */
|
|
455
|
+
interface AsyncScrapeResponse {
|
|
456
|
+
/** Job identifier — pass it to `getScrapeStatus` / `getScrapeResult`. */
|
|
457
|
+
job_id: string;
|
|
458
|
+
}
|
|
459
|
+
/**
|
|
460
|
+
* Response body of `GET /api/scrape/status/:jobId`.
|
|
461
|
+
*
|
|
462
|
+
* Note: this response uses camelCase field names (`jobId`, `createdAt`) while
|
|
463
|
+
* most other crawlbrulee responses use snake_case (e.g. `job_id` on
|
|
464
|
+
* {@link AsyncScrapeResponse}, `total_credits` on `UsageResponse`). The SDK
|
|
465
|
+
* mirrors the wire format faithfully — if the inconsistency trips you up,
|
|
466
|
+
* destructure with explicit names.
|
|
467
|
+
*/
|
|
468
|
+
interface AsyncJobStatusResponse {
|
|
469
|
+
/** The job identifier. */
|
|
470
|
+
jobId: string;
|
|
471
|
+
/** Current state of the job. */
|
|
472
|
+
status: AsyncJobStatus;
|
|
473
|
+
/** ISO-8601 UTC timestamp when the job was created. */
|
|
474
|
+
createdAt: string;
|
|
475
|
+
/** Error message if the job ended in `failed`. */
|
|
476
|
+
error?: string;
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
/** Response from `GET /api/usage`. Current billing-cycle snapshot. */
|
|
480
|
+
interface UsageResponse {
|
|
481
|
+
/**
|
|
482
|
+
* Total credits available for the current billing cycle
|
|
483
|
+
* (plan base + purchased + gifted).
|
|
484
|
+
*/
|
|
485
|
+
total_credits: number;
|
|
486
|
+
/**
|
|
487
|
+
* Credits spent so far in the current billing cycle. May exceed
|
|
488
|
+
* `total_credits` on plans that allow overages.
|
|
489
|
+
*/
|
|
490
|
+
used_credits: number;
|
|
491
|
+
/**
|
|
492
|
+
* Remaining credits, `max(0, total_credits - used_credits)`. Clamped to 0
|
|
493
|
+
* while in overage.
|
|
494
|
+
*/
|
|
495
|
+
available_credits: number;
|
|
496
|
+
/**
|
|
497
|
+
* Percentage of `total_credits` used in the current cycle, rounded to one
|
|
498
|
+
* decimal. Not capped — values above 100 indicate overage.
|
|
499
|
+
*/
|
|
500
|
+
used_quota_percent: number;
|
|
501
|
+
/**
|
|
502
|
+
* Maximum number of concurrent jobs allowed for the org
|
|
503
|
+
* (plan base + purchased + gifted extras).
|
|
504
|
+
*/
|
|
505
|
+
max_concurrency: number;
|
|
506
|
+
/**
|
|
507
|
+
* ISO-8601 UTC timestamp when the current billing cycle ends and
|
|
508
|
+
* `used_credits` resets to 0.
|
|
509
|
+
*/
|
|
510
|
+
usage_reset: string;
|
|
511
|
+
}
|
|
512
|
+
/** Response from `GET /api/whoami`. Identifies the calling API token. */
|
|
513
|
+
interface WhoamiResponse {
|
|
514
|
+
/** Display name of the organization that owns the token. */
|
|
515
|
+
organization_name: string;
|
|
516
|
+
/** User-assigned name of the API token. */
|
|
517
|
+
token_name: string;
|
|
518
|
+
/** Truncated preview of the API token (e.g. `cble_…xyz`). Safe to display. */
|
|
519
|
+
token_preview: string;
|
|
520
|
+
}
|
|
521
|
+
|
|
522
|
+
/** Options accepted by the {@link Crawlbrulee} constructor. */
|
|
523
|
+
interface CrawlbruleeOptions {
|
|
524
|
+
/**
|
|
525
|
+
* API key sent as `Authorization: Bearer <key>`. Required — to read from the
|
|
526
|
+
* environment instead, use {@link Crawlbrulee.fromEnv}. Leading and trailing
|
|
527
|
+
* whitespace is stripped; an empty / whitespace-only value is rejected.
|
|
528
|
+
*/
|
|
529
|
+
apiKey: string;
|
|
530
|
+
/**
|
|
531
|
+
* @internal
|
|
532
|
+
* Override the base URL. Reserved for local development and tests — production
|
|
533
|
+
* always uses the burned-in {@link DEFAULT_BASE_URL}. Trailing slashes are
|
|
534
|
+
* stripped.
|
|
535
|
+
*/
|
|
536
|
+
baseUrl?: string;
|
|
537
|
+
/**
|
|
538
|
+
* Per-request timeout in milliseconds. Defaults to `0` (no timeout). Set to a
|
|
539
|
+
* positive number to abort slow requests; a per-call `timeoutMs` override
|
|
540
|
+
* takes precedence. The timeout covers the WHOLE request, including the
|
|
541
|
+
* response body read.
|
|
542
|
+
*/
|
|
543
|
+
timeoutMs?: number;
|
|
544
|
+
}
|
|
545
|
+
/**
|
|
546
|
+
* Options accepted by {@link Crawlbrulee.waitForScrape}.
|
|
547
|
+
*
|
|
548
|
+
* Note: `timeoutMs` here is the OVERALL wait budget across all polls — not the
|
|
549
|
+
* per-HTTP-request timeout. The per-poll HTTP timeout is whatever the client
|
|
550
|
+
* was constructed with; if you want to bound each individual poll, construct
|
|
551
|
+
* the client with `timeoutMs` set.
|
|
552
|
+
*/
|
|
553
|
+
interface WaitForScrapeOptions extends Omit<RequestOptions, 'timeoutMs'> {
|
|
554
|
+
/** Time between status polls in milliseconds. Default `2000`. */
|
|
555
|
+
intervalMs?: number;
|
|
556
|
+
/**
|
|
557
|
+
* Maximum total time to wait before giving up, in milliseconds. Default
|
|
558
|
+
* `300_000` (5 minutes). Pass `0` to wait indefinitely.
|
|
559
|
+
*/
|
|
560
|
+
timeoutMs?: number;
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* Official client for the crawlbrulee API.
|
|
564
|
+
*
|
|
565
|
+
* @example
|
|
566
|
+
* ```ts
|
|
567
|
+
* import { Crawlbrulee } from '@crawlbrulee/sdk'
|
|
568
|
+
*
|
|
569
|
+
* const crawlbrulee = new Crawlbrulee({ apiKey: 'cble_…' })
|
|
570
|
+
* // or read CRAWLBRULEE_API_KEY from the environment:
|
|
571
|
+
* const crawlbrulee = Crawlbrulee.fromEnv()
|
|
572
|
+
*
|
|
573
|
+
* const page = await crawlbrulee.scrape({
|
|
574
|
+
* url: 'https://example.com',
|
|
575
|
+
* extract: { markdown: true, links: true },
|
|
576
|
+
* })
|
|
577
|
+
* console.log(page.markdown)
|
|
578
|
+
* ```
|
|
579
|
+
*/
|
|
580
|
+
declare class Crawlbrulee {
|
|
581
|
+
/** Resolved base URL — trailing slash already stripped. */
|
|
582
|
+
readonly baseUrl: string;
|
|
583
|
+
/** Underlying HTTP layer. Exposed for advanced use cases (custom endpoints). */
|
|
584
|
+
readonly http: HttpClient;
|
|
585
|
+
constructor(options: CrawlbruleeOptions);
|
|
586
|
+
/**
|
|
587
|
+
* Build a {@link Crawlbrulee} reading the API key from
|
|
588
|
+
* `process.env.CRAWLBRULEE_API_KEY`. Throws if the variable is unset, empty,
|
|
589
|
+
* or whitespace.
|
|
590
|
+
*
|
|
591
|
+
* Any other constructor option can be passed via `overrides`.
|
|
592
|
+
*
|
|
593
|
+
* @example
|
|
594
|
+
* ```ts
|
|
595
|
+
* const crawlbrulee = Crawlbrulee.fromEnv()
|
|
596
|
+
* const crawlbrulee = Crawlbrulee.fromEnv({ timeoutMs: 30_000 })
|
|
597
|
+
* ```
|
|
598
|
+
*/
|
|
599
|
+
static fromEnv(overrides?: Omit<CrawlbruleeOptions, 'apiKey'>): Crawlbrulee;
|
|
600
|
+
/**
|
|
601
|
+
* Scrape a URL synchronously and return the extracted content.
|
|
602
|
+
*
|
|
603
|
+
* The request blocks until the scrape is finished. For long-running jobs
|
|
604
|
+
* (heavy JS rendering, screenshots of long pages) prefer
|
|
605
|
+
* {@link Crawlbrulee.scrapeAsync} so the connection isn't held open.
|
|
606
|
+
*
|
|
607
|
+
* @param request — body for `POST /api/scrape`.
|
|
608
|
+
* @param options — per-call timeout and abort signal.
|
|
609
|
+
*/
|
|
610
|
+
scrape(request: ScrapeRequest, options?: RequestOptions): Promise<ScrapeResponse>;
|
|
611
|
+
/**
|
|
612
|
+
* Submit an asynchronous scrape job and return its `job_id`. Poll the job
|
|
613
|
+
* with {@link Crawlbrulee.getScrapeStatus} or wait for completion with
|
|
614
|
+
* {@link Crawlbrulee.waitForScrape}.
|
|
615
|
+
*/
|
|
616
|
+
scrapeAsync(request: ScrapeRequest, options?: RequestOptions): Promise<AsyncScrapeResponse>;
|
|
617
|
+
/** Look up the current status of an async scrape job. */
|
|
618
|
+
getScrapeStatus(jobId: string, options?: RequestOptions): Promise<AsyncJobStatusResponse>;
|
|
619
|
+
/**
|
|
620
|
+
* Fetch the result of a completed async scrape job. Throws if the job is
|
|
621
|
+
* still pending/running — call {@link Crawlbrulee.getScrapeStatus}
|
|
622
|
+
* first, or use {@link Crawlbrulee.waitForScrape} to poll-then-fetch.
|
|
623
|
+
*/
|
|
624
|
+
getScrapeResult(jobId: string, options?: RequestOptions): Promise<ScrapeResponse>;
|
|
625
|
+
/**
|
|
626
|
+
* Poll an async scrape job until it reaches a terminal state, then return
|
|
627
|
+
* the scrape result.
|
|
628
|
+
*
|
|
629
|
+
* Throws a {@link CrawlbruleeError} when:
|
|
630
|
+
* - the job ends in `failed` (`errorName: 'job_failed'`),
|
|
631
|
+
* - the server reports an unexpected status (`errorName: 'job_failed'`),
|
|
632
|
+
* - the overall wait exceeds `timeoutMs` (`errorName: 'request_timeout'`),
|
|
633
|
+
* - the caller's `signal` aborts (`errorName: 'client_closed_request'`).
|
|
634
|
+
*/
|
|
635
|
+
waitForScrape(jobId: string, options?: WaitForScrapeOptions): Promise<ScrapeResponse>;
|
|
636
|
+
/**
|
|
637
|
+
* Build (or return a cached) site link-map for a domain. Combines sitemap
|
|
638
|
+
* discovery with the freshest cached homepage scrape when available.
|
|
639
|
+
*/
|
|
640
|
+
map(request: MapRequest, options?: RequestOptions): Promise<MapResponse>;
|
|
641
|
+
/**
|
|
642
|
+
* Return the current billing-cycle usage: total/used/available credits,
|
|
643
|
+
* used quota percentage, max concurrency, and when the cycle resets.
|
|
644
|
+
*/
|
|
645
|
+
usage(options?: RequestOptions): Promise<UsageResponse>;
|
|
646
|
+
/**
|
|
647
|
+
* Return the organization name and identifying details of the API token
|
|
648
|
+
* used to authenticate this request. Useful for confirming which key is in
|
|
649
|
+
* use before performing destructive operations.
|
|
650
|
+
*/
|
|
651
|
+
whoami(options?: RequestOptions): Promise<WhoamiResponse>;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
/**
|
|
655
|
+
* Base error class for every failure raised by the SDK.
|
|
656
|
+
*
|
|
657
|
+
* Two kinds of failures end up here:
|
|
658
|
+
*
|
|
659
|
+
* 1. **API errors** — the server returned a non-2xx response with a well-formed
|
|
660
|
+
* JSON body. In that case `status`, `errorName` and (sometimes) `details`
|
|
661
|
+
* are populated.
|
|
662
|
+
* 2. **Transport errors** — the request never produced a structured response
|
|
663
|
+
* (network failure, abort, timeout, non-JSON body, etc.). In that case
|
|
664
|
+
* `status` may be `0` and `errorName` is one of the synthetic transport
|
|
665
|
+
* names (`request_timeout`, `client_closed_request`) or `null`.
|
|
666
|
+
*
|
|
667
|
+
* Typed subclasses are exported for the most common cases. To branch on more
|
|
668
|
+
* specific server-side errors, switch on `err.errorName` or use the
|
|
669
|
+
* {@link isCrawlbruleeError} helper.
|
|
670
|
+
*/
|
|
671
|
+
declare class CrawlbruleeError extends Error {
|
|
672
|
+
/** HTTP status code; `0` for transport-level failures with no response. */
|
|
673
|
+
readonly status: number;
|
|
674
|
+
/** The `name` field from the API error body, or `null` for transport errors. */
|
|
675
|
+
readonly errorName: ApiErrorName | null;
|
|
676
|
+
/** Structured detail block from the API error body, if any. */
|
|
677
|
+
readonly details?: ApiErrorDetails;
|
|
678
|
+
/** The original parsed error body, when one was received. */
|
|
679
|
+
readonly response?: ApiErrorResponse;
|
|
680
|
+
constructor(message: string, options: {
|
|
681
|
+
status: number;
|
|
682
|
+
errorName: ApiErrorName | null;
|
|
683
|
+
details?: ApiErrorDetails;
|
|
684
|
+
response?: ApiErrorResponse;
|
|
685
|
+
cause?: unknown;
|
|
686
|
+
});
|
|
687
|
+
}
|
|
688
|
+
/** Raised for 401 / 403 responses (missing, invalid, or unauthorized API key). */
|
|
689
|
+
declare class AuthenticationError extends CrawlbruleeError {
|
|
690
|
+
constructor(message: string, options: {
|
|
691
|
+
status: number;
|
|
692
|
+
errorName: ApiErrorName;
|
|
693
|
+
response?: ApiErrorResponse;
|
|
694
|
+
});
|
|
695
|
+
}
|
|
696
|
+
/**
|
|
697
|
+
* Raised for HTTP 429 responses. When the server included a `retry_after_ms`
|
|
698
|
+
* hint in `details` it is surfaced directly on the instance.
|
|
699
|
+
*
|
|
700
|
+
* `errorName` is always the literal `'too_many_requests'` — the SDK normalizes
|
|
701
|
+
* this even when the server returns a 429 with a different `name` field
|
|
702
|
+
* (e.g. a CDN coalescing upstream rate limiting). The original body is still
|
|
703
|
+
* available on `response`.
|
|
704
|
+
*/
|
|
705
|
+
declare class RateLimitError extends CrawlbruleeError {
|
|
706
|
+
readonly errorName: 'too_many_requests';
|
|
707
|
+
/** Suggested delay (ms) before retrying, when the server provided one. */
|
|
708
|
+
readonly retryAfterMs?: number;
|
|
709
|
+
/** Which rate limit was tripped (e.g. `org`, `ip`), when provided. */
|
|
710
|
+
readonly limitedBy?: string;
|
|
711
|
+
constructor(message: string, options: {
|
|
712
|
+
status: number;
|
|
713
|
+
details?: RateLimitErrorDetails;
|
|
714
|
+
response?: ApiErrorResponse;
|
|
715
|
+
});
|
|
716
|
+
}
|
|
717
|
+
/**
|
|
718
|
+
* Raised when the API rejects a request because the org's plan limits would
|
|
719
|
+
* be exceeded (credit limit, concurrency cap, overage hard cap, etc.).
|
|
720
|
+
*
|
|
721
|
+
* `errorName` is always the literal `'usage_allocation_error'`.
|
|
722
|
+
*/
|
|
723
|
+
declare class UsageAllocationError extends CrawlbruleeError {
|
|
724
|
+
readonly errorName: 'usage_allocation_error';
|
|
725
|
+
/** Specific reason the allocation was denied. */
|
|
726
|
+
readonly reason: UsageAllocationErrorDetails['reason'];
|
|
727
|
+
/** Current usage / limit snapshot at the time of the rejection. */
|
|
728
|
+
readonly usage?: UsageAllocationErrorDetails['details'];
|
|
729
|
+
constructor(message: string, options: {
|
|
730
|
+
status: number;
|
|
731
|
+
details: UsageAllocationErrorDetails;
|
|
732
|
+
response?: ApiErrorResponse;
|
|
733
|
+
});
|
|
734
|
+
}
|
|
735
|
+
/** Raised for 4xx responses caused by an invalid request shape or arguments. */
|
|
736
|
+
declare class ValidationError extends CrawlbruleeError {
|
|
737
|
+
constructor(message: string, options: {
|
|
738
|
+
status: number;
|
|
739
|
+
errorName: ApiErrorName;
|
|
740
|
+
response?: ApiErrorResponse;
|
|
741
|
+
});
|
|
742
|
+
}
|
|
743
|
+
/** Raised for 404 responses (e.g. unknown async job ID). */
|
|
744
|
+
declare class NotFoundError extends CrawlbruleeError {
|
|
745
|
+
constructor(message: string, options: {
|
|
746
|
+
status: number;
|
|
747
|
+
errorName: ApiErrorName;
|
|
748
|
+
response?: ApiErrorResponse;
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
/**
|
|
752
|
+
* Raised when a request cannot be sent or no structured response is parsed.
|
|
753
|
+
*
|
|
754
|
+
* The `errorName` discriminates the cause:
|
|
755
|
+
* - `'request_timeout'` — the per-request timeout fired.
|
|
756
|
+
* - `'client_closed_request'` — the caller's `AbortSignal` fired.
|
|
757
|
+
* - `null` — generic transport failure (network error, non-JSON body, etc.).
|
|
758
|
+
*/
|
|
759
|
+
declare class TransportError extends CrawlbruleeError {
|
|
760
|
+
constructor(message: string, options?: {
|
|
761
|
+
status?: number;
|
|
762
|
+
errorName?: 'request_timeout' | 'client_closed_request' | null;
|
|
763
|
+
cause?: unknown;
|
|
764
|
+
});
|
|
765
|
+
}
|
|
766
|
+
/** Narrow `unknown` to the SDK's base error type. */
|
|
767
|
+
declare function isCrawlbruleeError(err: unknown): err is CrawlbruleeError;
|
|
768
|
+
|
|
769
|
+
/**
|
|
770
|
+
* Production base URL of the crawlbrulee API. Burned in at build time —
|
|
771
|
+
* customers always hit production. The base URL is intentionally not
|
|
772
|
+
* configurable via env var; tests and local development override it through
|
|
773
|
+
* the `baseUrl` constructor option (marked `@internal`).
|
|
774
|
+
*/
|
|
775
|
+
declare const DEFAULT_BASE_URL = "https://api.crawlbrulee.com";
|
|
776
|
+
/** Default request timeout (60 s) when the caller doesn't specify one. */
|
|
777
|
+
declare const DEFAULT_REQUEST_TIMEOUT_MS = 60000;
|
|
778
|
+
/** Environment variable read by `Crawlbrulee.fromEnv()` to source the API key. */
|
|
779
|
+
declare const ENV_API_KEY = "CRAWLBRULEE_API_KEY";
|
|
780
|
+
|
|
781
|
+
export { type ApiErrorDetails, type ApiErrorName, type ApiErrorResponse, type AsyncJobStatus, type AsyncJobStatusResponse, type AsyncScrapeResponse, AuthenticationError, Crawlbrulee, CrawlbruleeError, type CrawlbruleeOptions, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, ENV_API_KEY, type HttpMethod, type MapCache, type MapLinkItem, type MapLocation, type MapPagination, type MapRequest, type MapResponse, type MapTruncation, type MapTypes, NotFoundError, type PageInlineImage, type PageLink, type ProxyTier, RateLimitError, type RateLimitErrorDetails, type RequestOptions, type ScrapeCache, type ScrapeExtract, type ScrapeLocation, type ScrapeMetadata, type ScrapeRequest, type ScrapeResponse, type ScreenshotAfterAction, type ScreenshotBeforeAction, type ScreenshotCleanup, type ScreenshotDeviceMode, type ScreenshotProperties, type ScreenshotRequest, type ScreenshotResult, type ScreenshotScrollAction, type ScreenshotSlice, type ScreenshotSliceAction, type ScreenshotType, type ScreenshotViewport, type ScreenshotViewportInfo, type ScreenshotWaitAction, TransportError, UsageAllocationError, type UsageAllocationErrorDetails, type UsageAllocationReason, type UsageLimitDetails, type UsageResponse, ValidationError, type WaitForScrapeOptions, type WhoamiResponse, isCrawlbruleeError };
|