@crawlbrulee/sdk 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +661 -0
- package/README.md +219 -0
- package/dist/index.cjs +538 -0
- package/dist/index.cjs.map +1 -0
- package/dist/index.d.cts +781 -0
- package/dist/index.d.ts +781 -0
- package/dist/index.js +525 -0
- package/dist/index.js.map +1 -0
- package/package.json +71 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,525 @@
|
|
|
1
|
+
// src/config.ts
|
|
2
|
+
var DEFAULT_BASE_URL = "https://api.crawlbrulee.com";
|
|
3
|
+
var DEFAULT_REQUEST_TIMEOUT_MS = 6e4;
|
|
4
|
+
var ENV_API_KEY = "CRAWLBRULEE_API_KEY";
|
|
5
|
+
var USER_AGENT = "@crawlbrulee/sdk/0.1.0 (node)";
|
|
6
|
+
|
|
7
|
+
// src/errors.ts
|
|
8
|
+
var CrawlbruleeError = class extends Error {
|
|
9
|
+
/** HTTP status code; `0` for transport-level failures with no response. */
|
|
10
|
+
status;
|
|
11
|
+
/** The `name` field from the API error body, or `null` for transport errors. */
|
|
12
|
+
errorName;
|
|
13
|
+
/** Structured detail block from the API error body, if any. */
|
|
14
|
+
details;
|
|
15
|
+
/** The original parsed error body, when one was received. */
|
|
16
|
+
response;
|
|
17
|
+
constructor(message, options) {
|
|
18
|
+
super(message, options.cause !== void 0 ? { cause: options.cause } : void 0);
|
|
19
|
+
this.name = "CrawlbruleeError";
|
|
20
|
+
this.status = options.status;
|
|
21
|
+
this.errorName = options.errorName;
|
|
22
|
+
this.details = options.details;
|
|
23
|
+
this.response = options.response;
|
|
24
|
+
}
|
|
25
|
+
};
|
|
26
|
+
var AuthenticationError = class extends CrawlbruleeError {
|
|
27
|
+
constructor(message, options) {
|
|
28
|
+
super(message, options);
|
|
29
|
+
this.name = "AuthenticationError";
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
var RateLimitError = class extends CrawlbruleeError {
|
|
33
|
+
errorName;
|
|
34
|
+
/** Suggested delay (ms) before retrying, when the server provided one. */
|
|
35
|
+
retryAfterMs;
|
|
36
|
+
/** Which rate limit was tripped (e.g. `org`, `ip`), when provided. */
|
|
37
|
+
limitedBy;
|
|
38
|
+
constructor(message, options) {
|
|
39
|
+
super(message, { ...options, errorName: "too_many_requests", details: options.details });
|
|
40
|
+
this.name = "RateLimitError";
|
|
41
|
+
this.errorName = "too_many_requests";
|
|
42
|
+
this.retryAfterMs = options.details?.retry_after_ms;
|
|
43
|
+
this.limitedBy = options.details?.limited_by;
|
|
44
|
+
}
|
|
45
|
+
};
|
|
46
|
+
var UsageAllocationError = class extends CrawlbruleeError {
|
|
47
|
+
errorName;
|
|
48
|
+
/** Specific reason the allocation was denied. */
|
|
49
|
+
reason;
|
|
50
|
+
/** Current usage / limit snapshot at the time of the rejection. */
|
|
51
|
+
usage;
|
|
52
|
+
constructor(message, options) {
|
|
53
|
+
super(message, { ...options, errorName: "usage_allocation_error" });
|
|
54
|
+
this.name = "UsageAllocationError";
|
|
55
|
+
this.errorName = "usage_allocation_error";
|
|
56
|
+
this.reason = options.details.reason;
|
|
57
|
+
this.usage = options.details.details;
|
|
58
|
+
}
|
|
59
|
+
};
|
|
60
|
+
var ValidationError = class extends CrawlbruleeError {
|
|
61
|
+
constructor(message, options) {
|
|
62
|
+
super(message, options);
|
|
63
|
+
this.name = "ValidationError";
|
|
64
|
+
}
|
|
65
|
+
};
|
|
66
|
+
var NotFoundError = class extends CrawlbruleeError {
|
|
67
|
+
constructor(message, options) {
|
|
68
|
+
super(message, options);
|
|
69
|
+
this.name = "NotFoundError";
|
|
70
|
+
}
|
|
71
|
+
};
|
|
72
|
+
var TransportError = class extends CrawlbruleeError {
|
|
73
|
+
constructor(message, options = {}) {
|
|
74
|
+
super(message, {
|
|
75
|
+
status: options.status ?? 0,
|
|
76
|
+
errorName: options.errorName ?? null,
|
|
77
|
+
cause: options.cause
|
|
78
|
+
});
|
|
79
|
+
this.name = "TransportError";
|
|
80
|
+
}
|
|
81
|
+
};
|
|
82
|
+
function isCrawlbruleeError(err) {
|
|
83
|
+
return err instanceof CrawlbruleeError;
|
|
84
|
+
}
|
|
85
|
+
function createApiError(body, status) {
|
|
86
|
+
const { name, message, details } = body;
|
|
87
|
+
const response = body;
|
|
88
|
+
switch (name) {
|
|
89
|
+
case "too_many_requests":
|
|
90
|
+
return new RateLimitError(message, {
|
|
91
|
+
status,
|
|
92
|
+
details: details?.error_name === "too_many_requests" ? details : void 0,
|
|
93
|
+
response
|
|
94
|
+
});
|
|
95
|
+
case "usage_allocation_error": {
|
|
96
|
+
const usageDetails = details?.error_name === "usage_allocation_error" ? details : { error_name: "usage_allocation_error", reason: "internal_error" };
|
|
97
|
+
return new UsageAllocationError(message, { status, details: usageDetails, response });
|
|
98
|
+
}
|
|
99
|
+
case "invalid_credentials":
|
|
100
|
+
case "access_denied":
|
|
101
|
+
return new AuthenticationError(message, { status, errorName: name, response });
|
|
102
|
+
case "not_found":
|
|
103
|
+
return new NotFoundError(message, { status, errorName: name, response });
|
|
104
|
+
case "validation_error":
|
|
105
|
+
case "invalid_url":
|
|
106
|
+
case "url_too_long":
|
|
107
|
+
case "unsupported_url_schema":
|
|
108
|
+
case "url_credentials_not_supported":
|
|
109
|
+
case "blocked_url":
|
|
110
|
+
case "unsupported_content":
|
|
111
|
+
return new ValidationError(message, { status, errorName: name, response });
|
|
112
|
+
}
|
|
113
|
+
if (status === 429) {
|
|
114
|
+
return new RateLimitError(message, { status, response });
|
|
115
|
+
}
|
|
116
|
+
if (status === 401 || status === 403) {
|
|
117
|
+
return new AuthenticationError(message, { status, errorName: name, response });
|
|
118
|
+
}
|
|
119
|
+
if (status === 404) {
|
|
120
|
+
return new NotFoundError(message, { status, errorName: name, response });
|
|
121
|
+
}
|
|
122
|
+
return new CrawlbruleeError(message, { status, errorName: name, details, response });
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// src/instrumentation.ts
|
|
126
|
+
var CwblInstrumentation = {
|
|
127
|
+
/**
|
|
128
|
+
* Resolve the `fetch` implementation the SDK should use. Throws a
|
|
129
|
+
* {@link CrawlbruleeError} if the runtime does not expose a global `fetch`.
|
|
130
|
+
*/
|
|
131
|
+
getFetch() {
|
|
132
|
+
const g = globalThis;
|
|
133
|
+
if (typeof g.fetch !== "function") {
|
|
134
|
+
throw new CrawlbruleeError(
|
|
135
|
+
"No global fetch is available in this runtime. crawlbrulee requires Node.js 20+, Bun, Deno, or a modern browser/edge runtime.",
|
|
136
|
+
{ status: 0, errorName: null }
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
return g.fetch.bind(globalThis);
|
|
140
|
+
}
|
|
141
|
+
};
|
|
142
|
+
|
|
143
|
+
// src/http.ts
|
|
144
|
+
var HttpClient = class {
|
|
145
|
+
baseUrl;
|
|
146
|
+
apiKey;
|
|
147
|
+
fetch;
|
|
148
|
+
timeoutMs;
|
|
149
|
+
constructor(options) {
|
|
150
|
+
this.baseUrl = stripTrailingSlash(options.baseUrl);
|
|
151
|
+
this.apiKey = options.apiKey;
|
|
152
|
+
this.fetch = CwblInstrumentation.getFetch();
|
|
153
|
+
this.timeoutMs = options.timeoutMs ?? 0;
|
|
154
|
+
}
|
|
155
|
+
/** Send a `GET` request and parse the response as `T`. */
|
|
156
|
+
get(path, options) {
|
|
157
|
+
return this.send({ method: "GET", path, ...options });
|
|
158
|
+
}
|
|
159
|
+
/** Send a `POST` request with a JSON body and parse the response as `T`. */
|
|
160
|
+
post(path, body, options) {
|
|
161
|
+
return this.send({ method: "POST", path, body, ...options });
|
|
162
|
+
}
|
|
163
|
+
async send(args) {
|
|
164
|
+
const url = this.buildUrl(args.path);
|
|
165
|
+
const headers = this.buildHeaders(args);
|
|
166
|
+
const body = args.body === void 0 ? void 0 : JSON.stringify(args.body);
|
|
167
|
+
const composed = this.composeSignal(args.signal, args.timeoutMs);
|
|
168
|
+
try {
|
|
169
|
+
let res;
|
|
170
|
+
try {
|
|
171
|
+
res = await this.fetch(url, {
|
|
172
|
+
method: args.method,
|
|
173
|
+
headers,
|
|
174
|
+
body,
|
|
175
|
+
signal: composed.signal
|
|
176
|
+
});
|
|
177
|
+
} catch (cause) {
|
|
178
|
+
throw abortOrNetworkError(cause, composed.timedOut(), args.timeoutMs ?? this.timeoutMs);
|
|
179
|
+
}
|
|
180
|
+
let text;
|
|
181
|
+
try {
|
|
182
|
+
text = await res.text();
|
|
183
|
+
} catch (cause) {
|
|
184
|
+
if (isAbortError(cause)) {
|
|
185
|
+
throw abortOrNetworkError(cause, composed.timedOut(), args.timeoutMs ?? this.timeoutMs);
|
|
186
|
+
}
|
|
187
|
+
throw new TransportError(`Failed to read response body (status ${res.status}).`, {
|
|
188
|
+
status: res.status,
|
|
189
|
+
cause
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
const parsed = parseJsonOrThrow(text, res.status);
|
|
193
|
+
if (!res.ok) throw toApiError(parsed, res.status, text);
|
|
194
|
+
return parsed;
|
|
195
|
+
} finally {
|
|
196
|
+
composed.cleanup();
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
buildUrl(path) {
|
|
200
|
+
if (!path.startsWith("/")) {
|
|
201
|
+
throw new TypeError(`crawlbrulee SDK: path must start with '/' (received '${path}')`);
|
|
202
|
+
}
|
|
203
|
+
return `${this.baseUrl}${path}`;
|
|
204
|
+
}
|
|
205
|
+
buildHeaders(args) {
|
|
206
|
+
const headers = {
|
|
207
|
+
accept: "application/json",
|
|
208
|
+
"user-agent": USER_AGENT,
|
|
209
|
+
authorization: `Bearer ${this.apiKey}`
|
|
210
|
+
};
|
|
211
|
+
if (args.body !== void 0) headers["content-type"] = "application/json";
|
|
212
|
+
return headers;
|
|
213
|
+
}
|
|
214
|
+
/**
|
|
215
|
+
* Build a single `AbortSignal` that fires when either the caller-supplied
|
|
216
|
+
* signal aborts OR the per-request timeout elapses. The returned `cleanup`
|
|
217
|
+
* callback MUST be invoked on every exit path so we don't leak timers or
|
|
218
|
+
* dead listeners on long-lived caller signals.
|
|
219
|
+
*/
|
|
220
|
+
composeSignal(callerSignal, overrideTimeoutMs) {
|
|
221
|
+
const timeoutMs = overrideTimeoutMs ?? this.timeoutMs;
|
|
222
|
+
const hasTimeout = Number.isFinite(timeoutMs) && timeoutMs > 0;
|
|
223
|
+
if (!hasTimeout && !callerSignal) {
|
|
224
|
+
return { signal: void 0, timedOut: () => false, cleanup: () => {
|
|
225
|
+
} };
|
|
226
|
+
}
|
|
227
|
+
if (!hasTimeout) {
|
|
228
|
+
return { signal: callerSignal, timedOut: () => false, cleanup: () => {
|
|
229
|
+
} };
|
|
230
|
+
}
|
|
231
|
+
const controller = new AbortController();
|
|
232
|
+
let didTimeout = false;
|
|
233
|
+
const timer = setTimeout(() => {
|
|
234
|
+
didTimeout = true;
|
|
235
|
+
controller.abort(new Error("request_timeout"));
|
|
236
|
+
}, timeoutMs);
|
|
237
|
+
let onCallerAbort;
|
|
238
|
+
if (callerSignal) {
|
|
239
|
+
if (callerSignal.aborted) {
|
|
240
|
+
clearTimeout(timer);
|
|
241
|
+
controller.abort(callerSignal.reason);
|
|
242
|
+
} else {
|
|
243
|
+
onCallerAbort = () => {
|
|
244
|
+
clearTimeout(timer);
|
|
245
|
+
controller.abort(callerSignal.reason);
|
|
246
|
+
};
|
|
247
|
+
callerSignal.addEventListener("abort", onCallerAbort, { once: true });
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
const cleanup = () => {
|
|
251
|
+
clearTimeout(timer);
|
|
252
|
+
if (onCallerAbort && callerSignal) {
|
|
253
|
+
callerSignal.removeEventListener("abort", onCallerAbort);
|
|
254
|
+
}
|
|
255
|
+
};
|
|
256
|
+
return { signal: controller.signal, timedOut: () => didTimeout, cleanup };
|
|
257
|
+
}
|
|
258
|
+
};
|
|
259
|
+
function stripTrailingSlash(url) {
|
|
260
|
+
return url.replace(/\/+$/, "");
|
|
261
|
+
}
|
|
262
|
+
function isAbortError(err) {
|
|
263
|
+
return err instanceof Error && err.name === "AbortError";
|
|
264
|
+
}
|
|
265
|
+
function abortOrNetworkError(cause, timedOut, timeoutMs) {
|
|
266
|
+
if (isAbortError(cause)) {
|
|
267
|
+
if (timedOut) {
|
|
268
|
+
return new TransportError(`Request timed out after ${timeoutMs}ms.`, {
|
|
269
|
+
errorName: "request_timeout",
|
|
270
|
+
cause
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
return new TransportError("Request aborted by caller.", {
|
|
274
|
+
errorName: "client_closed_request",
|
|
275
|
+
cause
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
return new TransportError(formatNetworkErrorMessage(cause), { cause });
|
|
279
|
+
}
|
|
280
|
+
function formatNetworkErrorMessage(cause) {
|
|
281
|
+
if (cause instanceof Error) {
|
|
282
|
+
return `Network error: ${cause.message}`;
|
|
283
|
+
}
|
|
284
|
+
return "Network error: unknown failure while sending the request.";
|
|
285
|
+
}
|
|
286
|
+
function parseJsonOrThrow(text, status) {
|
|
287
|
+
if (text === "") return {};
|
|
288
|
+
try {
|
|
289
|
+
return JSON.parse(text);
|
|
290
|
+
} catch (cause) {
|
|
291
|
+
const preview = text.length > 200 ? `${text.slice(0, 200)}\u2026` : text;
|
|
292
|
+
throw new TransportError(`Unexpected non-JSON response (status ${status}): ${preview}`, {
|
|
293
|
+
status,
|
|
294
|
+
cause
|
|
295
|
+
});
|
|
296
|
+
}
|
|
297
|
+
}
|
|
298
|
+
function toApiError(parsed, status, rawText) {
|
|
299
|
+
if (isApiErrorResponse(parsed)) {
|
|
300
|
+
return createApiError(parsed, status);
|
|
301
|
+
}
|
|
302
|
+
const preview = rawText.length > 200 ? `${rawText.slice(0, 200)}\u2026` : rawText;
|
|
303
|
+
return new TransportError(`HTTP ${status}: ${preview || "(empty body)"}`, { status });
|
|
304
|
+
}
|
|
305
|
+
function isApiErrorResponse(value) {
|
|
306
|
+
if (value === null || typeof value !== "object") return false;
|
|
307
|
+
const v = value;
|
|
308
|
+
return typeof v.name === "string" && typeof v.message === "string";
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// src/client.ts
|
|
312
|
+
var Crawlbrulee = class _Crawlbrulee {
|
|
313
|
+
/** Resolved base URL — trailing slash already stripped. */
|
|
314
|
+
baseUrl;
|
|
315
|
+
/** Underlying HTTP layer. Exposed for advanced use cases (custom endpoints). */
|
|
316
|
+
http;
|
|
317
|
+
constructor(options) {
|
|
318
|
+
const apiKey = options.apiKey?.trim();
|
|
319
|
+
if (!apiKey) {
|
|
320
|
+
throw new CrawlbruleeError(
|
|
321
|
+
`Missing API key. Pass { apiKey } to Crawlbrulee or call Crawlbrulee.fromEnv() to read ${ENV_API_KEY}.`,
|
|
322
|
+
{ status: 0, errorName: null }
|
|
323
|
+
);
|
|
324
|
+
}
|
|
325
|
+
const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, "");
|
|
326
|
+
this.baseUrl = baseUrl;
|
|
327
|
+
this.http = new HttpClient({
|
|
328
|
+
baseUrl,
|
|
329
|
+
apiKey,
|
|
330
|
+
timeoutMs: options.timeoutMs
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
/**
|
|
334
|
+
* Build a {@link Crawlbrulee} reading the API key from
|
|
335
|
+
* `process.env.CRAWLBRULEE_API_KEY`. Throws if the variable is unset, empty,
|
|
336
|
+
* or whitespace.
|
|
337
|
+
*
|
|
338
|
+
* Any other constructor option can be passed via `overrides`.
|
|
339
|
+
*
|
|
340
|
+
* @example
|
|
341
|
+
* ```ts
|
|
342
|
+
* const crawlbrulee = Crawlbrulee.fromEnv()
|
|
343
|
+
* const crawlbrulee = Crawlbrulee.fromEnv({ timeoutMs: 30_000 })
|
|
344
|
+
* ```
|
|
345
|
+
*/
|
|
346
|
+
static fromEnv(overrides = {}) {
|
|
347
|
+
const apiKey = readEnv(ENV_API_KEY);
|
|
348
|
+
if (!apiKey) {
|
|
349
|
+
throw new CrawlbruleeError(
|
|
350
|
+
`${ENV_API_KEY} is not set. Export it in your shell, or pass apiKey to new Crawlbrulee({ apiKey }).`,
|
|
351
|
+
{ status: 0, errorName: null }
|
|
352
|
+
);
|
|
353
|
+
}
|
|
354
|
+
return new _Crawlbrulee({ ...overrides, apiKey });
|
|
355
|
+
}
|
|
356
|
+
// ------------------------------------------------------------------
|
|
357
|
+
// Scraping
|
|
358
|
+
// ------------------------------------------------------------------
|
|
359
|
+
/**
|
|
360
|
+
* Scrape a URL synchronously and return the extracted content.
|
|
361
|
+
*
|
|
362
|
+
* The request blocks until the scrape is finished. For long-running jobs
|
|
363
|
+
* (heavy JS rendering, screenshots of long pages) prefer
|
|
364
|
+
* {@link Crawlbrulee.scrapeAsync} so the connection isn't held open.
|
|
365
|
+
*
|
|
366
|
+
* @param request — body for `POST /api/scrape`.
|
|
367
|
+
* @param options — per-call timeout and abort signal.
|
|
368
|
+
*/
|
|
369
|
+
scrape(request, options) {
|
|
370
|
+
return this.http.post("/api/scrape", request, options);
|
|
371
|
+
}
|
|
372
|
+
/**
|
|
373
|
+
* Submit an asynchronous scrape job and return its `job_id`. Poll the job
|
|
374
|
+
* with {@link Crawlbrulee.getScrapeStatus} or wait for completion with
|
|
375
|
+
* {@link Crawlbrulee.waitForScrape}.
|
|
376
|
+
*/
|
|
377
|
+
scrapeAsync(request, options) {
|
|
378
|
+
return this.http.post("/api/scrape/async", request, options);
|
|
379
|
+
}
|
|
380
|
+
/** Look up the current status of an async scrape job. */
|
|
381
|
+
getScrapeStatus(jobId, options) {
|
|
382
|
+
assertNonEmptyJobId(jobId);
|
|
383
|
+
return this.http.get(
|
|
384
|
+
`/api/scrape/status/${encodeURIComponent(jobId)}`,
|
|
385
|
+
options
|
|
386
|
+
);
|
|
387
|
+
}
|
|
388
|
+
/**
|
|
389
|
+
* Fetch the result of a completed async scrape job. Throws if the job is
|
|
390
|
+
* still pending/running — call {@link Crawlbrulee.getScrapeStatus}
|
|
391
|
+
* first, or use {@link Crawlbrulee.waitForScrape} to poll-then-fetch.
|
|
392
|
+
*/
|
|
393
|
+
getScrapeResult(jobId, options) {
|
|
394
|
+
assertNonEmptyJobId(jobId);
|
|
395
|
+
return this.http.get(`/api/scrape/result/${encodeURIComponent(jobId)}`, options);
|
|
396
|
+
}
|
|
397
|
+
/**
|
|
398
|
+
* Poll an async scrape job until it reaches a terminal state, then return
|
|
399
|
+
* the scrape result.
|
|
400
|
+
*
|
|
401
|
+
* Throws a {@link CrawlbruleeError} when:
|
|
402
|
+
* - the job ends in `failed` (`errorName: 'job_failed'`),
|
|
403
|
+
* - the server reports an unexpected status (`errorName: 'job_failed'`),
|
|
404
|
+
* - the overall wait exceeds `timeoutMs` (`errorName: 'request_timeout'`),
|
|
405
|
+
* - the caller's `signal` aborts (`errorName: 'client_closed_request'`).
|
|
406
|
+
*/
|
|
407
|
+
async waitForScrape(jobId, options = {}) {
|
|
408
|
+
assertNonEmptyJobId(jobId);
|
|
409
|
+
const intervalMs = options.intervalMs ?? 2e3;
|
|
410
|
+
const timeoutMs = options.timeoutMs ?? 3e5;
|
|
411
|
+
const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : Number.POSITIVE_INFINITY;
|
|
412
|
+
while (true) {
|
|
413
|
+
throwIfAborted(options.signal);
|
|
414
|
+
if (Date.now() >= deadline) {
|
|
415
|
+
throw new CrawlbruleeError(
|
|
416
|
+
`Timed out after ${timeoutMs}ms waiting for async scrape job ${jobId}.`,
|
|
417
|
+
{ status: 0, errorName: "request_timeout" }
|
|
418
|
+
);
|
|
419
|
+
}
|
|
420
|
+
const status = await this.getScrapeStatus(jobId, { signal: options.signal });
|
|
421
|
+
switch (status.status) {
|
|
422
|
+
case "done":
|
|
423
|
+
return this.getScrapeResult(jobId, { signal: options.signal });
|
|
424
|
+
case "failed":
|
|
425
|
+
throw new CrawlbruleeError(status.error ?? `Async scrape job ${jobId} failed.`, {
|
|
426
|
+
status: 0,
|
|
427
|
+
errorName: "job_failed"
|
|
428
|
+
});
|
|
429
|
+
case "pending":
|
|
430
|
+
case "running":
|
|
431
|
+
break;
|
|
432
|
+
default:
|
|
433
|
+
throw new CrawlbruleeError(
|
|
434
|
+
`Async scrape job ${jobId} returned unexpected status '${String(status.status)}'.`,
|
|
435
|
+
{ status: 0, errorName: "job_failed" }
|
|
436
|
+
);
|
|
437
|
+
}
|
|
438
|
+
await sleep(intervalMs, options.signal);
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
// ------------------------------------------------------------------
|
|
442
|
+
// Mapping
|
|
443
|
+
// ------------------------------------------------------------------
|
|
444
|
+
/**
|
|
445
|
+
* Build (or return a cached) site link-map for a domain. Combines sitemap
|
|
446
|
+
* discovery with the freshest cached homepage scrape when available.
|
|
447
|
+
*/
|
|
448
|
+
map(request, options) {
|
|
449
|
+
return this.http.post("/api/map", request, options);
|
|
450
|
+
}
|
|
451
|
+
// ------------------------------------------------------------------
|
|
452
|
+
// Account
|
|
453
|
+
// ------------------------------------------------------------------
|
|
454
|
+
/**
|
|
455
|
+
* Return the current billing-cycle usage: total/used/available credits,
|
|
456
|
+
* used quota percentage, max concurrency, and when the cycle resets.
|
|
457
|
+
*/
|
|
458
|
+
usage(options) {
|
|
459
|
+
return this.http.get("/api/usage", options);
|
|
460
|
+
}
|
|
461
|
+
/**
|
|
462
|
+
* Return the organization name and identifying details of the API token
|
|
463
|
+
* used to authenticate this request. Useful for confirming which key is in
|
|
464
|
+
* use before performing destructive operations.
|
|
465
|
+
*/
|
|
466
|
+
whoami(options) {
|
|
467
|
+
return this.http.get("/api/whoami", options);
|
|
468
|
+
}
|
|
469
|
+
};
|
|
470
|
+
function readEnv(name) {
|
|
471
|
+
try {
|
|
472
|
+
if (typeof process === "undefined" || !process.env) return void 0;
|
|
473
|
+
const v = process.env[name];
|
|
474
|
+
return typeof v === "string" && v.trim().length > 0 ? v.trim() : void 0;
|
|
475
|
+
} catch {
|
|
476
|
+
return void 0;
|
|
477
|
+
}
|
|
478
|
+
}
|
|
479
|
+
function assertNonEmptyJobId(jobId) {
|
|
480
|
+
if (typeof jobId !== "string" || jobId.trim().length === 0) {
|
|
481
|
+
throw new CrawlbruleeError("jobId must be a non-empty string.", {
|
|
482
|
+
status: 0,
|
|
483
|
+
errorName: null
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
function throwIfAborted(signal) {
|
|
488
|
+
if (signal?.aborted) {
|
|
489
|
+
throw new CrawlbruleeError("Request aborted by caller.", {
|
|
490
|
+
status: 0,
|
|
491
|
+
errorName: "client_closed_request",
|
|
492
|
+
cause: signal.reason
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
}
|
|
496
|
+
function sleep(ms, signal) {
|
|
497
|
+
return new Promise((resolve, reject) => {
|
|
498
|
+
const onAbort = () => {
|
|
499
|
+
clearTimeout(timer);
|
|
500
|
+
reject(
|
|
501
|
+
new CrawlbruleeError("Request aborted by caller.", {
|
|
502
|
+
status: 0,
|
|
503
|
+
errorName: "client_closed_request",
|
|
504
|
+
cause: signal?.reason
|
|
505
|
+
})
|
|
506
|
+
);
|
|
507
|
+
};
|
|
508
|
+
const timer = setTimeout(() => {
|
|
509
|
+
signal?.removeEventListener("abort", onAbort);
|
|
510
|
+
resolve();
|
|
511
|
+
}, ms);
|
|
512
|
+
if (signal) {
|
|
513
|
+
if (signal.aborted) {
|
|
514
|
+
clearTimeout(timer);
|
|
515
|
+
onAbort();
|
|
516
|
+
return;
|
|
517
|
+
}
|
|
518
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
519
|
+
}
|
|
520
|
+
});
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
export { AuthenticationError, Crawlbrulee, CrawlbruleeError, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, ENV_API_KEY, NotFoundError, RateLimitError, TransportError, UsageAllocationError, ValidationError, isCrawlbruleeError };
|
|
524
|
+
//# sourceMappingURL=index.js.map
|
|
525
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/config.ts","../src/errors.ts","../src/instrumentation.ts","../src/http.ts","../src/client.ts"],"names":[],"mappings":";AAMO,IAAM,gBAAA,GAAmB;AAGzB,IAAM,0BAAA,GAA6B;AAGnC,IAAM,WAAA,GAAc;AAGpB,IAAM,UAAA,GAAa,+BAAA;;;ACUnB,IAAM,gBAAA,GAAN,cAA+B,KAAA,CAAM;AAAA;AAAA,EAEjC,MAAA;AAAA;AAAA,EAEA,SAAA;AAAA;AAAA,EAEA,OAAA;AAAA;AAAA,EAEA,QAAA;AAAA,EAET,WAAA,CACE,SACA,OAAA,EAOA;AACA,IAAA,KAAA,CAAM,OAAA,EAAS,QAAQ,KAAA,KAAU,MAAA,GAAY,EAAE,KAAA,EAAO,OAAA,CAAQ,KAAA,EAAM,GAAI,MAAS,CAAA;AACjF,IAAA,IAAA,CAAK,IAAA,GAAO,kBAAA;AACZ,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,YAAY,OAAA,CAAQ,SAAA;AACzB,IAAA,IAAA,CAAK,UAAU,OAAA,CAAQ,OAAA;AACvB,IAAA,IAAA,CAAK,WAAW,OAAA,CAAQ,QAAA;AAAA,EAC1B;AACF;AAGO,IAAM,mBAAA,GAAN,cAAkC,gBAAA,CAAiB;AAAA,EACxD,WAAA,CACE,SACA,OAAA,EACA;AACA,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,qBAAA;AAAA,EACd;AACF;AAWO,IAAM,cAAA,GAAN,cAA6B,gBAAA,CAAiB;AAAA,EACjC,SAAA;AAAA;AAAA,EAET,YAAA;AAAA;AAAA,EAEA,SAAA;AAAA,EAET,WAAA,CACE,SACA,OAAA,EAKA;AACA,IAAA,KAAA,CAAM,OAAA,EAAS,EAAE,GAAG,OAAA,EAAS,WAAW,mBAAA,EAAqB,OAAA,EAAS,OAAA,CAAQ,OAAA,EAAS,CAAA;AACvF,IAAA,IAAA,CAAK,IAAA,GAAO,gBAAA;AACZ,IAAA,IAAA,CAAK,SAAA,GAAY,mBAAA;AACjB,IAAA,IAAA,CAAK,YAAA,GAAe,QAAQ,OAAA,EAAS,cAAA;AACrC,IAAA,IAAA,CAAK,SAAA,GAAY,QAAQ,OAAA,EAAS,UAAA;AAAA,EACpC;AACF;AAQO,IAAM,oBAAA,GAAN,cAAmC,gBAAA,CAAiB;AAAA,EACvC,SAAA;AAAA;AAAA,EAET,MAAA;AAAA;AAAA,EAEA,KAAA;AAAA,EAET,WAAA,CACE,SACA,OAAA,EAKA;AACA,IAAA,KAAA,CAAM,SAAS,EAAE,GAAG,OAAA,EAAS,SAAA,EAAW,0BAA0B,CAAA;AAClE,IAAA,IAAA,CAAK,IAAA,GAAO,sBAAA;AACZ,IAAA,IAAA,CAAK,SAAA,GAAY,wBAAA;AACjB,IAAA,IAAA,CAAK,MAAA,GAAS,QAAQ,OAAA,CAAQ,MAAA;AAC9B,IAAA,IAAA,CAAK,KAAA,GAAQ,QAAQ,OAAA,CAAQ,OAAA;AAAA,EAC/B;AACF;AAGO,IAAM,eAAA,GAAN,cAA8B,gBAAA,CAAiB;AAAA,EACpD,WAAA,CACE,SACA,OAAA,EACA;AACA,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,iBAAA;AAAA,EACd;AACF;AAGO,IAAM,aAAA,GAAN,cAA4B,gBAAA,CAAiB;AAAA,EAClD,WAAA,CACE,SACA,OAAA,EACA;AACA,IAAA,KAAA,CAAM,SAAS,OAAO,CAAA;AACtB,IAAA,IAAA,CAAK,IAAA,GAAO,eAAA;AAAA,EACd;AACF;AAUO,IAAM,cAAA,GAAN,cAA6B,gBAAA,CAAiB;AAAA,EACnD,WAAA,CACE,OAAA,EACA,OAAA,GAII,EAAC,EACL;AACA,IAAA,KAAA,CAAM,OAAA,EAAS;AAAA,MACb,MAAA,EAAQ,QAAQ,MAAA,IAAU,CAAA;AAAA,MAC1B,SAAA,EAAW,QAAQ,SAAA,IAAa,IAAA;AAAA,MAChC,OAAO,OAAA,CAAQ;AAAA,KAChB,CAAA;AACD,IAAA,IAAA,CAAK,IAAA,GAAO,gBAAA;AAAA,EACd;AACF;AAGO,SAAS,mBAAmB,GAAA,EAAuC;AACxE,EAAA,OAAO,GAAA,YAAe,gBAAA;AACxB;AAYO,SAAS,cAAA,CAAe,MAAwB,MAAA,EAAkC;AACvF,EAAA,MAAM,EAAE,IAAA,EAAM,OAAA,EAAS,OAAA,EAAQ,GAAI,IAAA;AACnC,EAAA,MAAM,QAAA,GAAW,IAAA;AAEjB,EAAA,QAAQ,IAAA;AAAM,IACZ,KAAK,mBAAA;AACH,MAAA,OAAO,IAAI,eAAe,OAAA,EAAS;AAAA,QACjC,MAAA;AAAA,QACA,OAAA,EAAS,OAAA,EAAS,UAAA,KAAe,mBAAA,GAAsB,OAAA,GAAU,MAAA;AAAA,QACjE;AAAA,OACD,CAAA;AAAA,IAEH,KAAK,wBAAA,EAA0B;AAG7B,MAAA,MAAM,YAAA,GACJ,SAAS,UAAA,KAAe,wBAAA,GACpB,UACA,EAAE,UAAA,EAAY,wBAAA,EAA0B,MAAA,EAAQ,gBAAA,EAAiB;AACvE,MAAA,OAAO,IAAI,qBAAqB,OAAA,EAAS,EAAE,QAAQ,OAAA,EAAS,YAAA,EAAc,UAAU,CAAA;AAAA,IACtF;AAAA,IAEA,KAAK,qBAAA;AAAA,IACL,KAAK,eAAA;AACH,MAAA,OAAO,IAAI,oBAAoB,OAAA,EAAS,EAAE,QAAQ,SAAA,EAAW,IAAA,EAAM,UAAU,CAAA;AAAA,IAE/E,KAAK,WAAA;AACH,MAAA,OAAO,IAAI,cAAc,OAAA,EAAS,EAAE,QAAQ,SAAA,EAAW,IAAA,EAAM,UAAU,CAAA;AAAA,IAEzE,KAAK,kBAAA;AAAA,IACL,KAAK,aAAA;AAAA,IACL,KAAK,cAAA;AAAA,IACL,KAAK,wBAAA;AAAA,IACL,KAAK,+BAAA;AAAA,IACL,KAAK,aAAA;AAAA,IACL,KAAK,qBAAA;AACH,MAAA,OAAO,IAAI,gBAAgB,OAAA,EAAS,EAAE,QAAQ,SAAA,EAAW,IAAA,EAAM,UAAU,CAAA;AAAA;AAM7E,EAAA,IAAI,WAAW,GAAA,EAAK;AAClB,IAAA,OAAO,IAAI,cAAA,CAAe,OAAA,EAAS,EAAE,MAAA,EAAQ,UAAU,CAAA;AAAA,EACzD;AACA,EAAA,IAAI,MAAA,KAAW,GAAA,IAAO,MAAA,KAAW,GAAA,EAAK;AACpC,IAAA,OAAO,IAAI,oBAAoB,OAAA,EAAS,EAAE,QAAQ,SAAA,EAAW,IAAA,EAAM,UAAU,CAAA;AAAA,EAC/E;AACA,EAAA,IAAI,WAAW,GAAA,EAAK;AAClB,IAAA,OAAO,IAAI,cAAc,OAAA,EAAS,EAAE,QAAQ,SAAA,EAAW,IAAA,EAAM,UAAU,CAAA;AAAA,EACzE;AAEA,EAAA,OAAO,IAAI,iBAAiB,OAAA,EAAS,EAAE,QAAQ,SAAA,EAAW,IAAA,EAAM,OAAA,EAAS,QAAA,EAAU,CAAA;AACrF;;;ACpOO,IAAM,mBAAA,GAAsB;AAAA;AAAA;AAAA;AAAA;AAAA,EAKjC,QAAA,GAAsB;AACpB,IAAA,MAAM,CAAA,GAAI,UAAA;AACV,IAAA,IAAI,OAAO,CAAA,CAAE,KAAA,KAAU,UAAA,EAAY;AACjC,MAAA,MAAM,IAAI,gBAAA;AAAA,QACR,8HAAA;AAAA,QACA,EAAE,MAAA,EAAQ,CAAA,EAAG,SAAA,EAAW,IAAA;AAAK,OAC/B;AAAA,IACF;AACA,IAAA,OAAO,CAAA,CAAE,KAAA,CAAM,IAAA,CAAK,UAAU,CAAA;AAAA,EAChC;AACF,CAAA;;;ACgCO,IAAM,aAAN,MAAiB;AAAA,EACL,OAAA;AAAA,EACA,MAAA;AAAA,EACA,KAAA;AAAA,EACA,SAAA;AAAA,EAEjB,YAAY,OAAA,EAA4B;AACtC,IAAA,IAAA,CAAK,OAAA,GAAU,kBAAA,CAAmB,OAAA,CAAQ,OAAO,CAAA;AACjD,IAAA,IAAA,CAAK,SAAS,OAAA,CAAQ,MAAA;AACtB,IAAA,IAAA,CAAK,KAAA,GAAQ,oBAAoB,QAAA,EAAS;AAC1C,IAAA,IAAA,CAAK,SAAA,GAAY,QAAQ,SAAA,IAAa,CAAA;AAAA,EACxC;AAAA;AAAA,EAGA,GAAA,CAAO,MAAc,OAAA,EAAsC;AACzD,IAAA,OAAO,IAAA,CAAK,KAAQ,EAAE,MAAA,EAAQ,OAAO,IAAA,EAAM,GAAG,SAAS,CAAA;AAAA,EACzD;AAAA;AAAA,EAGA,IAAA,CAAQ,IAAA,EAAc,IAAA,EAAe,OAAA,EAAsC;AACzE,IAAA,OAAO,IAAA,CAAK,KAAQ,EAAE,MAAA,EAAQ,QAAQ,IAAA,EAAM,IAAA,EAAM,GAAG,OAAA,EAAS,CAAA;AAAA,EAChE;AAAA,EAEA,MAAc,KAAQ,IAAA,EAA4B;AAChD,IAAA,MAAM,GAAA,GAAM,IAAA,CAAK,QAAA,CAAS,IAAA,CAAK,IAAI,CAAA;AACnC,IAAA,MAAM,OAAA,GAAU,IAAA,CAAK,YAAA,CAAa,IAAI,CAAA;AACtC,IAAA,MAAM,IAAA,GAAO,KAAK,IAAA,KAAS,MAAA,GAAY,SAAY,IAAA,CAAK,SAAA,CAAU,KAAK,IAAI,CAAA;AAC3E,IAAA,MAAM,WAAW,IAAA,CAAK,aAAA,CAAc,IAAA,CAAK,MAAA,EAAQ,KAAK,SAAS,CAAA;AAE/D,IAAA,IAAI;AACF,MAAA,IAAI,GAAA;AACJ,MAAA,IAAI;AACF,QAAA,GAAA,GAAM,MAAM,IAAA,CAAK,KAAA,CAAM,GAAA,EAAK;AAAA,UAC1B,QAAQ,IAAA,CAAK,MAAA;AAAA,UACb,OAAA;AAAA,UACA,IAAA;AAAA,UACA,QAAQ,QAAA,CAAS;AAAA,SAClB,CAAA;AAAA,MACH,SAAS,KAAA,EAAgB;AACvB,QAAA,MAAM,mBAAA,CAAoB,OAAO,QAAA,CAAS,QAAA,IAAY,IAAA,CAAK,SAAA,IAAa,KAAK,SAAS,CAAA;AAAA,MACxF;AAEA,MAAA,IAAI,IAAA;AACJ,MAAA,IAAI;AACF,QAAA,IAAA,GAAO,MAAM,IAAI,IAAA,EAAK;AAAA,MACxB,SAAS,KAAA,EAAgB;AACvB,QAAA,IAAI,YAAA,CAAa,KAAK,CAAA,EAAG;AACvB,UAAA,MAAM,mBAAA,CAAoB,OAAO,QAAA,CAAS,QAAA,IAAY,IAAA,CAAK,SAAA,IAAa,KAAK,SAAS,CAAA;AAAA,QACxF;AACA,QAAA,MAAM,IAAI,cAAA,CAAe,CAAA,qCAAA,EAAwC,GAAA,CAAI,MAAM,CAAA,EAAA,CAAA,EAAM;AAAA,UAC/E,QAAQ,GAAA,CAAI,MAAA;AAAA,UACZ;AAAA,SACD,CAAA;AAAA,MACH;AAEA,MAAA,MAAM,MAAA,GAAS,gBAAA,CAAiB,IAAA,EAAM,GAAA,CAAI,MAAM,CAAA;AAChD,MAAA,IAAI,CAAC,IAAI,EAAA,EAAI,MAAM,WAAW,MAAA,EAAQ,GAAA,CAAI,QAAQ,IAAI,CAAA;AACtD,MAAA,OAAO,MAAA;AAAA,IACT,CAAA,SAAE;AACA,MAAA,QAAA,CAAS,OAAA,EAAQ;AAAA,IACnB;AAAA,EACF;AAAA,EAEQ,SAAS,IAAA,EAAsB;AACrC,IAAA,IAAI,CAAC,IAAA,CAAK,UAAA,CAAW,GAAG,CAAA,EAAG;AACzB,MAAA,MAAM,IAAI,SAAA,CAAU,CAAA,qDAAA,EAAwD,IAAI,CAAA,EAAA,CAAI,CAAA;AAAA,IACtF;AACA,IAAA,OAAO,CAAA,EAAG,IAAA,CAAK,OAAO,CAAA,EAAG,IAAI,CAAA,CAAA;AAAA,EAC/B;AAAA,EAEQ,aAAa,IAAA,EAAwC;AAC3D,IAAA,MAAM,OAAA,GAAkC;AAAA,MACtC,MAAA,EAAQ,kBAAA;AAAA,MACR,YAAA,EAAc,UAAA;AAAA,MACd,aAAA,EAAe,CAAA,OAAA,EAAU,IAAA,CAAK,MAAM,CAAA;AAAA,KACtC;AACA,IAAA,IAAI,IAAA,CAAK,IAAA,KAAS,MAAA,EAAW,OAAA,CAAQ,cAAc,CAAA,GAAI,kBAAA;AACvD,IAAA,OAAO,OAAA;AAAA,EACT;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAQQ,aAAA,CACN,cACA,iBAAA,EACgB;AAChB,IAAA,MAAM,SAAA,GAAY,qBAAqB,IAAA,CAAK,SAAA;AAC5C,IAAA,MAAM,UAAA,GAAa,MAAA,CAAO,QAAA,CAAS,SAAS,KAAK,SAAA,GAAY,CAAA;AAE7D,IAAA,IAAI,CAAC,UAAA,IAAc,CAAC,YAAA,EAAc;AAChC,MAAA,OAAO,EAAE,MAAA,EAAQ,MAAA,EAAW,UAAU,MAAM,KAAA,EAAO,SAAS,MAAM;AAAA,MAAC,CAAA,EAAE;AAAA,IACvE;AAEA,IAAA,IAAI,CAAC,UAAA,EAAY;AACf,MAAA,OAAO,EAAE,MAAA,EAAQ,YAAA,EAAc,UAAU,MAAM,KAAA,EAAO,SAAS,MAAM;AAAA,MAAC,CAAA,EAAE;AAAA,IAC1E;AAEA,IAAA,MAAM,UAAA,GAAa,IAAI,eAAA,EAAgB;AACvC,IAAA,IAAI,UAAA,GAAa,KAAA;AACjB,IAAA,MAAM,KAAA,GAAQ,WAAW,MAAM;AAC7B,MAAA,UAAA,GAAa,IAAA;AACb,MAAA,UAAA,CAAW,KAAA,CAAM,IAAI,KAAA,CAAM,iBAAiB,CAAC,CAAA;AAAA,IAC/C,GAAG,SAAS,CAAA;AAEZ,IAAA,IAAI,aAAA;AACJ,IAAA,IAAI,YAAA,EAAc;AAChB,MAAA,IAAI,aAAa,OAAA,EAAS;AACxB,QAAA,YAAA,CAAa,KAAK,CAAA;AAClB,QAAA,UAAA,CAAW,KAAA,CAAM,aAAa,MAAM,CAAA;AAAA,MACtC,CAAA,MAAO;AACL,QAAA,aAAA,GAAgB,MAAM;AACpB,UAAA,YAAA,CAAa,KAAK,CAAA;AAClB,UAAA,UAAA,CAAW,KAAA,CAAM,aAAa,MAAM,CAAA;AAAA,QACtC,CAAA;AACA,QAAA,YAAA,CAAa,iBAAiB,OAAA,EAAS,aAAA,EAAe,EAAE,IAAA,EAAM,MAAM,CAAA;AAAA,MACtE;AAAA,IACF;AAEA,IAAA,MAAM,UAAU,MAAM;AACpB,MAAA,YAAA,CAAa,KAAK,CAAA;AAClB,MAAA,IAAI,iBAAiB,YAAA,EAAc;AACjC,QAAA,YAAA,CAAa,mBAAA,CAAoB,SAAS,aAAa,CAAA;AAAA,MACzD;AAAA,IACF,CAAA;AAEA,IAAA,OAAO,EAAE,MAAA,EAAQ,UAAA,CAAW,QAAQ,QAAA,EAAU,MAAM,YAAY,OAAA,EAAQ;AAAA,EAC1E;AACF,CAAA;AAEA,SAAS,mBAAmB,GAAA,EAAqB;AAC/C,EAAA,OAAO,GAAA,CAAI,OAAA,CAAQ,MAAA,EAAQ,EAAE,CAAA;AAC/B;AAEA,SAAS,aAAa,GAAA,EAAuB;AAC3C,EAAA,OAAO,GAAA,YAAe,KAAA,IAAS,GAAA,CAAI,IAAA,KAAS,YAAA;AAC9C;AAEA,SAAS,mBAAA,CAAoB,KAAA,EAAgB,QAAA,EAAmB,SAAA,EAAmC;AACjG,EAAA,IAAI,YAAA,CAAa,KAAK,CAAA,EAAG;AACvB,IAAA,IAAI,QAAA,EAAU;AACZ,MAAA,OAAO,IAAI,cAAA,CAAe,CAAA,wBAAA,EAA2B,SAAS,CAAA,GAAA,CAAA,EAAO;AAAA,QACnE,SAAA,EAAW,iBAAA;AAAA,QACX;AAAA,OACD,CAAA;AAAA,IACH;AACA,IAAA,OAAO,IAAI,eAAe,4BAAA,EAA8B;AAAA,MACtD,SAAA,EAAW,uBAAA;AAAA,MACX;AAAA,KACD,CAAA;AAAA,EACH;AACA,EAAA,OAAO,IAAI,cAAA,CAAe,yBAAA,CAA0B,KAAK,CAAA,EAAG,EAAE,OAAO,CAAA;AACvE;AAEA,SAAS,0BAA0B,KAAA,EAAwB;AACzD,EAAA,IAAI,iBAAiB,KAAA,EAAO;AAC1B,IAAA,OAAO,CAAA,eAAA,EAAkB,MAAM,OAAO,CAAA,CAAA;AAAA,EACxC;AACA,EAAA,OAAO,2DAAA;AACT;AAEA,SAAS,gBAAA,CAAiB,MAAc,MAAA,EAAyB;AAC/D,EAAA,IAAI,IAAA,KAAS,EAAA,EAAI,OAAO,EAAC;AACzB,EAAA,IAAI;AACF,IAAA,OAAO,IAAA,CAAK,MAAM,IAAI,CAAA;AAAA,EACxB,SAAS,KAAA,EAAgB;AACvB,IAAA,MAAM,OAAA,GAAU,IAAA,CAAK,MAAA,GAAS,GAAA,GAAM,CAAA,EAAG,KAAK,KAAA,CAAM,CAAA,EAAG,GAAG,CAAC,CAAA,MAAA,CAAA,GAAM,IAAA;AAC/D,IAAA,MAAM,IAAI,cAAA,CAAe,CAAA,qCAAA,EAAwC,MAAM,CAAA,GAAA,EAAM,OAAO,CAAA,CAAA,EAAI;AAAA,MACtF,MAAA;AAAA,MACA;AAAA,KACD,CAAA;AAAA,EACH;AACF;AAEA,SAAS,UAAA,CAAW,MAAA,EAAiB,MAAA,EAAgB,OAAA,EAAmC;AACtF,EAAA,IAAI,kBAAA,CAAmB,MAAM,CAAA,EAAG;AAC9B,IAAA,OAAO,cAAA,CAAe,QAAQ,MAAM,CAAA;AAAA,EACtC;AACA,EAAA,MAAM,OAAA,GAAU,OAAA,CAAQ,MAAA,GAAS,GAAA,GAAM,CAAA,EAAG,QAAQ,KAAA,CAAM,CAAA,EAAG,GAAG,CAAC,CAAA,MAAA,CAAA,GAAM,OAAA;AACrE,EAAA,OAAO,IAAI,cAAA,CAAe,CAAA,KAAA,EAAQ,MAAM,CAAA,EAAA,EAAK,WAAW,cAAc,CAAA,CAAA,EAAI,EAAE,MAAA,EAAQ,CAAA;AACtF;AAEA,SAAS,mBAAmB,KAAA,EAA2C;AACrE,EAAA,IAAI,KAAA,KAAU,IAAA,IAAQ,OAAO,KAAA,KAAU,UAAU,OAAO,KAAA;AACxD,EAAA,MAAM,CAAA,GAAI,KAAA;AACV,EAAA,OAAO,OAAO,CAAA,CAAE,IAAA,KAAS,QAAA,IAAY,OAAO,EAAE,OAAA,KAAY,QAAA;AAC5D;;;AChLO,IAAM,WAAA,GAAN,MAAM,YAAA,CAAY;AAAA;AAAA,EAEd,OAAA;AAAA;AAAA,EAEA,IAAA;AAAA,EAET,YAAY,OAAA,EAA6B;AACvC,IAAA,MAAM,MAAA,GAAS,OAAA,CAAQ,MAAA,EAAQ,IAAA,EAAK;AACpC,IAAA,IAAI,CAAC,MAAA,EAAQ;AACX,MAAA,MAAM,IAAI,gBAAA;AAAA,QACR,yFAAyF,WAAW,CAAA,CAAA,CAAA;AAAA,QACpG,EAAE,MAAA,EAAQ,CAAA,EAAG,SAAA,EAAW,IAAA;AAAK,OAC/B;AAAA,IACF;AACA,IAAA,MAAM,WAAW,OAAA,CAAQ,OAAA,IAAW,gBAAA,EAAkB,OAAA,CAAQ,QAAQ,EAAE,CAAA;AAExE,IAAA,IAAA,CAAK,OAAA,GAAU,OAAA;AACf,IAAA,IAAA,CAAK,IAAA,GAAO,IAAI,UAAA,CAAW;AAAA,MACzB,OAAA;AAAA,MACA,MAAA;AAAA,MACA,WAAW,OAAA,CAAQ;AAAA,KACpB,CAAA;AAAA,EACH;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAeA,OAAO,OAAA,CAAQ,SAAA,GAAgD,EAAC,EAAgB;AAC9E,IAAA,MAAM,MAAA,GAAS,QAAQ,WAAW,CAAA;AAClC,IAAA,IAAI,CAAC,MAAA,EAAQ;AACX,MAAA,MAAM,IAAI,gBAAA;AAAA,QACR,GAAG,WAAW,CAAA,oFAAA,CAAA;AAAA,QACd,EAAE,MAAA,EAAQ,CAAA,EAAG,SAAA,EAAW,IAAA;AAAK,OAC/B;AAAA,IACF;AACA,IAAA,OAAO,IAAI,YAAA,CAAY,EAAE,GAAG,SAAA,EAAW,QAAQ,CAAA;AAAA,EACjD;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAgBA,MAAA,CAAO,SAAwB,OAAA,EAAmD;AAChF,IAAA,OAAO,IAAA,CAAK,IAAA,CAAK,IAAA,CAAqB,aAAA,EAAe,SAAS,OAAO,CAAA;AAAA,EACvE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,WAAA,CAAY,SAAwB,OAAA,EAAwD;AAC1F,IAAA,OAAO,IAAA,CAAK,IAAA,CAAK,IAAA,CAA0B,mBAAA,EAAqB,SAAS,OAAO,CAAA;AAAA,EAClF;AAAA;AAAA,EAGA,eAAA,CAAgB,OAAe,OAAA,EAA2D;AACxF,IAAA,mBAAA,CAAoB,KAAK,CAAA;AACzB,IAAA,OAAO,KAAK,IAAA,CAAK,GAAA;AAAA,MACf,CAAA,mBAAA,EAAsB,kBAAA,CAAmB,KAAK,CAAC,CAAA,CAAA;AAAA,MAC/C;AAAA,KACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,eAAA,CAAgB,OAAe,OAAA,EAAmD;AAChF,IAAA,mBAAA,CAAoB,KAAK,CAAA;AACzB,IAAA,OAAO,IAAA,CAAK,KAAK,GAAA,CAAoB,CAAA,mBAAA,EAAsB,mBAAmB,KAAK,CAAC,IAAI,OAAO,CAAA;AAAA,EACjG;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAYA,MAAM,aAAA,CAAc,KAAA,EAAe,OAAA,GAAgC,EAAC,EAA4B;AAC9F,IAAA,mBAAA,CAAoB,KAAK,CAAA;AACzB,IAAA,MAAM,UAAA,GAAa,QAAQ,UAAA,IAAc,GAAA;AACzC,IAAA,MAAM,SAAA,GAAY,QAAQ,SAAA,IAAa,GAAA;AACvC,IAAA,MAAM,WAAW,SAAA,GAAY,CAAA,GAAI,KAAK,GAAA,EAAI,GAAI,YAAY,MAAA,CAAO,iBAAA;AAEjE,IAAA,OAAO,IAAA,EAAM;AACX,MAAA,cAAA,CAAe,QAAQ,MAAM,CAAA;AAC7B,MAAA,IAAI,IAAA,CAAK,GAAA,EAAI,IAAK,QAAA,EAAU;AAC1B,QAAA,MAAM,IAAI,gBAAA;AAAA,UACR,CAAA,gBAAA,EAAmB,SAAS,CAAA,gCAAA,EAAmC,KAAK,CAAA,CAAA,CAAA;AAAA,UACpE,EAAE,MAAA,EAAQ,CAAA,EAAG,SAAA,EAAW,iBAAA;AAAkB,SAC5C;AAAA,MACF;AAEA,MAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,eAAA,CAAgB,OAAO,EAAE,MAAA,EAAQ,OAAA,CAAQ,MAAA,EAAQ,CAAA;AAE3E,MAAA,QAAQ,OAAO,MAAA;AAAQ,QACrB,KAAK,MAAA;AACH,UAAA,OAAO,KAAK,eAAA,CAAgB,KAAA,EAAO,EAAE,MAAA,EAAQ,OAAA,CAAQ,QAAQ,CAAA;AAAA,QAE/D,KAAK,QAAA;AACH,UAAA,MAAM,IAAI,gBAAA,CAAiB,MAAA,CAAO,KAAA,IAAS,CAAA,iBAAA,EAAoB,KAAK,CAAA,QAAA,CAAA,EAAY;AAAA,YAC9E,MAAA,EAAQ,CAAA;AAAA,YACR,SAAA,EAAW;AAAA,WACZ,CAAA;AAAA,QAEH,KAAK,SAAA;AAAA,QACL,KAAK,SAAA;AACH,UAAA;AAAA,QAEF;AACE,UAAA,MAAM,IAAI,gBAAA;AAAA,YACR,oBAAoB,KAAK,CAAA,6BAAA,EAAgC,MAAA,CAAO,MAAA,CAAO,MAAM,CAAC,CAAA,EAAA,CAAA;AAAA,YAC9E,EAAE,MAAA,EAAQ,CAAA,EAAG,SAAA,EAAW,YAAA;AAAa,WACvC;AAAA;AAGJ,MAAA,MAAM,KAAA,CAAM,UAAA,EAAY,OAAA,CAAQ,MAAM,CAAA;AAAA,IACxC;AAAA,EACF;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,GAAA,CAAI,SAAqB,OAAA,EAAgD;AACvE,IAAA,OAAO,IAAA,CAAK,IAAA,CAAK,IAAA,CAAkB,UAAA,EAAY,SAAS,OAAO,CAAA;AAAA,EACjE;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAUA,MAAM,OAAA,EAAkD;AACtD,IAAA,OAAO,IAAA,CAAK,IAAA,CAAK,GAAA,CAAmB,YAAA,EAAc,OAAO,CAAA;AAAA,EAC3D;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,EAOA,OAAO,OAAA,EAAmD;AACxD,IAAA,OAAO,IAAA,CAAK,IAAA,CAAK,GAAA,CAAoB,aAAA,EAAe,OAAO,CAAA;AAAA,EAC7D;AACF;AAOA,SAAS,QAAQ,IAAA,EAAkC;AACjD,EAAA,IAAI;AACF,IAAA,IAAI,OAAO,OAAA,KAAY,WAAA,IAAe,CAAC,OAAA,CAAQ,KAAK,OAAO,KAAA,CAAA;AAC3D,IAAA,MAAM,CAAA,GAAI,OAAA,CAAQ,GAAA,CAAI,IAAI,CAAA;AAC1B,IAAA,OAAO,OAAO,CAAA,KAAM,QAAA,IAAY,CAAA,CAAE,IAAA,GAAO,MAAA,GAAS,CAAA,GAAI,CAAA,CAAE,IAAA,EAAK,GAAI,KAAA,CAAA;AAAA,EACnE,CAAA,CAAA,MAAQ;AACN,IAAA,OAAO,MAAA;AAAA,EACT;AACF;AAEA,SAAS,oBAAoB,KAAA,EAAqB;AAChD,EAAA,IAAI,OAAO,KAAA,KAAU,QAAA,IAAY,MAAM,IAAA,EAAK,CAAE,WAAW,CAAA,EAAG;AAC1D,IAAA,MAAM,IAAI,iBAAiB,mCAAA,EAAqC;AAAA,MAC9D,MAAA,EAAQ,CAAA;AAAA,MACR,SAAA,EAAW;AAAA,KACZ,CAAA;AAAA,EACH;AACF;AAEA,SAAS,eAAe,MAAA,EAAuC;AAC7D,EAAA,IAAI,QAAQ,OAAA,EAAS;AACnB,IAAA,MAAM,IAAI,iBAAiB,4BAAA,EAA8B;AAAA,MACvD,MAAA,EAAQ,CAAA;AAAA,MACR,SAAA,EAAW,uBAAA;AAAA,MACX,OAAO,MAAA,CAAO;AAAA,KACf,CAAA;AAAA,EACH;AACF;AAEA,SAAS,KAAA,CAAM,IAAY,MAAA,EAAgD;AACzE,EAAA,OAAO,IAAI,OAAA,CAAQ,CAAC,OAAA,EAAS,MAAA,KAAW;AACtC,IAAA,MAAM,UAAU,MAAM;AACpB,MAAA,YAAA,CAAa,KAAK,CAAA;AAClB,MAAA,MAAA;AAAA,QACE,IAAI,iBAAiB,4BAAA,EAA8B;AAAA,UACjD,MAAA,EAAQ,CAAA;AAAA,UACR,SAAA,EAAW,uBAAA;AAAA,UACX,OAAO,MAAA,EAAQ;AAAA,SAChB;AAAA,OACH;AAAA,IACF,CAAA;AACA,IAAA,MAAM,KAAA,GAAQ,WAAW,MAAM;AAC7B,MAAA,MAAA,EAAQ,mBAAA,CAAoB,SAAS,OAAO,CAAA;AAC5C,MAAA,OAAA,EAAQ;AAAA,IACV,GAAG,EAAE,CAAA;AACL,IAAA,IAAI,MAAA,EAAQ;AACV,MAAA,IAAI,OAAO,OAAA,EAAS;AAClB,QAAA,YAAA,CAAa,KAAK,CAAA;AAClB,QAAA,OAAA,EAAQ;AACR,QAAA;AAAA,MACF;AACA,MAAA,MAAA,CAAO,iBAAiB,OAAA,EAAS,OAAA,EAAS,EAAE,IAAA,EAAM,MAAM,CAAA;AAAA,IAC1D;AAAA,EACF,CAAC,CAAA;AACH","file":"index.js","sourcesContent":["/**\n * Production base URL of the crawlbrulee API. Burned in at build time —\n * customers always hit production. The base URL is intentionally not\n * configurable via env var; tests and local development override it through\n * the `baseUrl` constructor option (marked `@internal`).\n */\nexport const DEFAULT_BASE_URL = 'https://api.crawlbrulee.com'\n\n/** Default request timeout (60 s) when the caller doesn't specify one. */\nexport const DEFAULT_REQUEST_TIMEOUT_MS = 60_000\n\n/** Environment variable read by `Crawlbrulee.fromEnv()` to source the API key. */\nexport const ENV_API_KEY = 'CRAWLBRULEE_API_KEY'\n\n/** Identifies the SDK in the `User-Agent` header. Kept in one place for easy bumping. */\nexport const USER_AGENT = '@crawlbrulee/sdk/0.1.0 (node)'\n","import type {\n ApiErrorDetails,\n ApiErrorName,\n ApiErrorResponse,\n RateLimitErrorDetails,\n UsageAllocationErrorDetails,\n} from './types/common.js'\n\n/**\n * Base error class for every failure raised by the SDK.\n *\n * Two kinds of failures end up here:\n *\n * 1. **API errors** — the server returned a non-2xx response with a well-formed\n * JSON body. In that case `status`, `errorName` and (sometimes) `details`\n * are populated.\n * 2. **Transport errors** — the request never produced a structured response\n * (network failure, abort, timeout, non-JSON body, etc.). In that case\n * `status` may be `0` and `errorName` is one of the synthetic transport\n * names (`request_timeout`, `client_closed_request`) or `null`.\n *\n * Typed subclasses are exported for the most common cases. To branch on more\n * specific server-side errors, switch on `err.errorName` or use the\n * {@link isCrawlbruleeError} helper.\n */\nexport class CrawlbruleeError extends Error {\n /** HTTP status code; `0` for transport-level failures with no response. */\n readonly status: number\n /** The `name` field from the API error body, or `null` for transport errors. */\n readonly errorName: ApiErrorName | null\n /** Structured detail block from the API error body, if any. */\n readonly details?: ApiErrorDetails\n /** The original parsed error body, when one was received. */\n readonly response?: ApiErrorResponse\n\n constructor(\n message: string,\n options: {\n status: number\n errorName: ApiErrorName | null\n details?: ApiErrorDetails\n response?: ApiErrorResponse\n cause?: unknown\n }\n ) {\n super(message, options.cause !== undefined ? { cause: options.cause } : undefined)\n this.name = 'CrawlbruleeError'\n this.status = options.status\n this.errorName = options.errorName\n this.details = options.details\n this.response = options.response\n }\n}\n\n/** Raised for 401 / 403 responses (missing, invalid, or unauthorized API key). */\nexport class AuthenticationError extends CrawlbruleeError {\n constructor(\n message: string,\n options: { status: number; errorName: ApiErrorName; response?: ApiErrorResponse }\n ) {\n super(message, options)\n this.name = 'AuthenticationError'\n }\n}\n\n/**\n * Raised for HTTP 429 responses. When the server included a `retry_after_ms`\n * hint in `details` it is surfaced directly on the instance.\n *\n * `errorName` is always the literal `'too_many_requests'` — the SDK normalizes\n * this even when the server returns a 429 with a different `name` field\n * (e.g. a CDN coalescing upstream rate limiting). The original body is still\n * available on `response`.\n */\nexport class RateLimitError extends CrawlbruleeError {\n override readonly errorName: 'too_many_requests'\n /** Suggested delay (ms) before retrying, when the server provided one. */\n readonly retryAfterMs?: number\n /** Which rate limit was tripped (e.g. `org`, `ip`), when provided. */\n readonly limitedBy?: string\n\n constructor(\n message: string,\n options: {\n status: number\n details?: RateLimitErrorDetails\n response?: ApiErrorResponse\n }\n ) {\n super(message, { ...options, errorName: 'too_many_requests', details: options.details })\n this.name = 'RateLimitError'\n this.errorName = 'too_many_requests'\n this.retryAfterMs = options.details?.retry_after_ms\n this.limitedBy = options.details?.limited_by\n }\n}\n\n/**\n * Raised when the API rejects a request because the org's plan limits would\n * be exceeded (credit limit, concurrency cap, overage hard cap, etc.).\n *\n * `errorName` is always the literal `'usage_allocation_error'`.\n */\nexport class UsageAllocationError extends CrawlbruleeError {\n override readonly errorName: 'usage_allocation_error'\n /** Specific reason the allocation was denied. */\n readonly reason: UsageAllocationErrorDetails['reason']\n /** Current usage / limit snapshot at the time of the rejection. */\n readonly usage?: UsageAllocationErrorDetails['details']\n\n constructor(\n message: string,\n options: {\n status: number\n details: UsageAllocationErrorDetails\n response?: ApiErrorResponse\n }\n ) {\n super(message, { ...options, errorName: 'usage_allocation_error' })\n this.name = 'UsageAllocationError'\n this.errorName = 'usage_allocation_error'\n this.reason = options.details.reason\n this.usage = options.details.details\n }\n}\n\n/** Raised for 4xx responses caused by an invalid request shape or arguments. */\nexport class ValidationError extends CrawlbruleeError {\n constructor(\n message: string,\n options: { status: number; errorName: ApiErrorName; response?: ApiErrorResponse }\n ) {\n super(message, options)\n this.name = 'ValidationError'\n }\n}\n\n/** Raised for 404 responses (e.g. unknown async job ID). */\nexport class NotFoundError extends CrawlbruleeError {\n constructor(\n message: string,\n options: { status: number; errorName: ApiErrorName; response?: ApiErrorResponse }\n ) {\n super(message, options)\n this.name = 'NotFoundError'\n }\n}\n\n/**\n * Raised when a request cannot be sent or no structured response is parsed.\n *\n * The `errorName` discriminates the cause:\n * - `'request_timeout'` — the per-request timeout fired.\n * - `'client_closed_request'` — the caller's `AbortSignal` fired.\n * - `null` — generic transport failure (network error, non-JSON body, etc.).\n */\nexport class TransportError extends CrawlbruleeError {\n constructor(\n message: string,\n options: {\n status?: number\n errorName?: 'request_timeout' | 'client_closed_request' | null\n cause?: unknown\n } = {}\n ) {\n super(message, {\n status: options.status ?? 0,\n errorName: options.errorName ?? null,\n cause: options.cause,\n })\n this.name = 'TransportError'\n }\n}\n\n/** Narrow `unknown` to the SDK's base error type. */\nexport function isCrawlbruleeError(err: unknown): err is CrawlbruleeError {\n return err instanceof CrawlbruleeError\n}\n\n/**\n * Map an API error body + HTTP status to the most specific error class.\n *\n * Dispatch is **name-first**: the body's `name` field is the most reliable\n * signal of what went wrong. Status code is used only as a fallback when the\n * name is unrecognized (e.g. a CDN-synthesized error). This avoids\n * miscategorizing things like a 403 with `name: 'not_found'` as an auth error.\n *\n * Internal — used by the HTTP layer.\n */\nexport function createApiError(body: ApiErrorResponse, status: number): CrawlbruleeError {\n const { name, message, details } = body\n const response = body\n\n switch (name) {\n case 'too_many_requests':\n return new RateLimitError(message, {\n status,\n details: details?.error_name === 'too_many_requests' ? details : undefined,\n response,\n })\n\n case 'usage_allocation_error': {\n // Without a structured details block we still want a typed error — fall\n // back to a synthetic `internal_error` reason so callers can branch.\n const usageDetails: UsageAllocationErrorDetails =\n details?.error_name === 'usage_allocation_error'\n ? details\n : { error_name: 'usage_allocation_error', reason: 'internal_error' }\n return new UsageAllocationError(message, { status, details: usageDetails, response })\n }\n\n case 'invalid_credentials':\n case 'access_denied':\n return new AuthenticationError(message, { status, errorName: name, response })\n\n case 'not_found':\n return new NotFoundError(message, { status, errorName: name, response })\n\n case 'validation_error':\n case 'invalid_url':\n case 'url_too_long':\n case 'unsupported_url_schema':\n case 'url_credentials_not_supported':\n case 'blocked_url':\n case 'unsupported_content':\n return new ValidationError(message, { status, errorName: name, response })\n }\n\n // Name was not specific enough — fall back to status-based heuristics, but\n // never override what the name said. A 429 with an unrecognized name still\n // promotes to RateLimitError (the class invariant normalizes errorName).\n if (status === 429) {\n return new RateLimitError(message, { status, response })\n }\n if (status === 401 || status === 403) {\n return new AuthenticationError(message, { status, errorName: name, response })\n }\n if (status === 404) {\n return new NotFoundError(message, { status, errorName: name, response })\n }\n\n return new CrawlbruleeError(message, { status, errorName: name, details, response })\n}\n","import { CrawlbruleeError } from './errors.js'\n\n/** Function shape compatible with the global `fetch`. */\nexport type FetchLike = typeof fetch\n\n/**\n * Centralized factory for low-level dependencies the SDK injects into its HTTP\n * layer. Production code resolves `getFetch()` to the runtime's global `fetch`;\n * tests stub this module to return a mock implementation.\n *\n * This is internal — it is not exported from the package's public entry. Tests\n * import it from `src/instrumentation.js` directly and use `vi.spyOn` to\n * substitute behavior.\n */\nexport const CwblInstrumentation = {\n /**\n * Resolve the `fetch` implementation the SDK should use. Throws a\n * {@link CrawlbruleeError} if the runtime does not expose a global `fetch`.\n */\n getFetch(): FetchLike {\n const g = globalThis as { fetch?: FetchLike }\n if (typeof g.fetch !== 'function') {\n throw new CrawlbruleeError(\n 'No global fetch is available in this runtime. crawlbrulee requires Node.js 20+, Bun, Deno, or a modern browser/edge runtime.',\n { status: 0, errorName: null }\n )\n }\n return g.fetch.bind(globalThis)\n },\n}\n","import { USER_AGENT } from './config.js'\nimport { TransportError, createApiError, type CrawlbruleeError } from './errors.js'\nimport { CwblInstrumentation, type FetchLike } from './instrumentation.js'\nimport type { ApiErrorResponse } from './types/common.js'\n\n/** HTTP methods used by the SDK. */\nexport type HttpMethod = 'GET' | 'POST'\n\n/** Options the SDK accepts at construction time for the HTTP layer. */\nexport interface HttpClientOptions {\n /** Base URL of the API (trailing slash is stripped). */\n baseUrl: string\n /** API key sent as `Authorization: Bearer <key>`. */\n apiKey: string\n /**\n * Per-request timeout in milliseconds. Pass `0` (or omit) to disable the\n * timeout entirely.\n */\n timeoutMs?: number\n}\n\n/** Per-call overrides accepted on every resource method. */\nexport interface RequestOptions {\n /** Abort the request when this signal fires. Composable with the timeout. */\n signal?: AbortSignal\n /**\n * Override the constructor-level `timeoutMs` for this call. Pass `0` to\n * disable the timeout for this call.\n */\n timeoutMs?: number\n}\n\ninterface SendArgs extends RequestOptions {\n method: HttpMethod\n path: string\n body?: unknown\n}\n\ninterface ComposedSignal {\n signal: AbortSignal | undefined\n /** Returns `true` if the abort was triggered by the per-request timeout. */\n timedOut: () => boolean\n /** Releases the timer and any listeners attached to the caller's signal. */\n cleanup: () => void\n}\n\n/**\n * Minimal `fetch`-based HTTP layer used by {@link Crawlbrulee}. Handles:\n *\n * - URL composition (joining `baseUrl` and path safely).\n * - JSON serialization and parsing.\n * - The `Authorization: Bearer …` header.\n * - Composing the caller's `AbortSignal` with an internal timeout signal. The\n * timeout covers the WHOLE request, including the response body read — not\n * just the time-to-headers.\n * - Mapping non-2xx responses to typed `CrawlbruleeError` subclasses via\n * {@link createApiError}.\n *\n * The `fetch` implementation is sourced from {@link CwblInstrumentation} at\n * construction time so tests can stub the module.\n */\nexport class HttpClient {\n private readonly baseUrl: string\n private readonly apiKey: string\n private readonly fetch: FetchLike\n private readonly timeoutMs: number\n\n constructor(options: HttpClientOptions) {\n this.baseUrl = stripTrailingSlash(options.baseUrl)\n this.apiKey = options.apiKey\n this.fetch = CwblInstrumentation.getFetch()\n this.timeoutMs = options.timeoutMs ?? 0\n }\n\n /** Send a `GET` request and parse the response as `T`. */\n get<T>(path: string, options?: RequestOptions): Promise<T> {\n return this.send<T>({ method: 'GET', path, ...options })\n }\n\n /** Send a `POST` request with a JSON body and parse the response as `T`. */\n post<T>(path: string, body: unknown, options?: RequestOptions): Promise<T> {\n return this.send<T>({ method: 'POST', path, body, ...options })\n }\n\n private async send<T>(args: SendArgs): Promise<T> {\n const url = this.buildUrl(args.path)\n const headers = this.buildHeaders(args)\n const body = args.body === undefined ? undefined : JSON.stringify(args.body)\n const composed = this.composeSignal(args.signal, args.timeoutMs)\n\n try {\n let res: Response\n try {\n res = await this.fetch(url, {\n method: args.method,\n headers,\n body,\n signal: composed.signal,\n })\n } catch (cause: unknown) {\n throw abortOrNetworkError(cause, composed.timedOut(), args.timeoutMs ?? this.timeoutMs)\n }\n\n let text: string\n try {\n text = await res.text()\n } catch (cause: unknown) {\n if (isAbortError(cause)) {\n throw abortOrNetworkError(cause, composed.timedOut(), args.timeoutMs ?? this.timeoutMs)\n }\n throw new TransportError(`Failed to read response body (status ${res.status}).`, {\n status: res.status,\n cause,\n })\n }\n\n const parsed = parseJsonOrThrow(text, res.status)\n if (!res.ok) throw toApiError(parsed, res.status, text)\n return parsed as T\n } finally {\n composed.cleanup()\n }\n }\n\n private buildUrl(path: string): string {\n if (!path.startsWith('/')) {\n throw new TypeError(`crawlbrulee SDK: path must start with '/' (received '${path}')`)\n }\n return `${this.baseUrl}${path}`\n }\n\n private buildHeaders(args: SendArgs): Record<string, string> {\n const headers: Record<string, string> = {\n accept: 'application/json',\n 'user-agent': USER_AGENT,\n authorization: `Bearer ${this.apiKey}`,\n }\n if (args.body !== undefined) headers['content-type'] = 'application/json'\n return headers\n }\n\n /**\n * Build a single `AbortSignal` that fires when either the caller-supplied\n * signal aborts OR the per-request timeout elapses. The returned `cleanup`\n * callback MUST be invoked on every exit path so we don't leak timers or\n * dead listeners on long-lived caller signals.\n */\n private composeSignal(\n callerSignal: AbortSignal | undefined,\n overrideTimeoutMs: number | undefined\n ): ComposedSignal {\n const timeoutMs = overrideTimeoutMs ?? this.timeoutMs\n const hasTimeout = Number.isFinite(timeoutMs) && timeoutMs > 0\n\n if (!hasTimeout && !callerSignal) {\n return { signal: undefined, timedOut: () => false, cleanup: () => {} }\n }\n\n if (!hasTimeout) {\n return { signal: callerSignal, timedOut: () => false, cleanup: () => {} }\n }\n\n const controller = new AbortController()\n let didTimeout = false\n const timer = setTimeout(() => {\n didTimeout = true\n controller.abort(new Error('request_timeout'))\n }, timeoutMs)\n\n let onCallerAbort: (() => void) | undefined\n if (callerSignal) {\n if (callerSignal.aborted) {\n clearTimeout(timer)\n controller.abort(callerSignal.reason)\n } else {\n onCallerAbort = () => {\n clearTimeout(timer)\n controller.abort(callerSignal.reason)\n }\n callerSignal.addEventListener('abort', onCallerAbort, { once: true })\n }\n }\n\n const cleanup = () => {\n clearTimeout(timer)\n if (onCallerAbort && callerSignal) {\n callerSignal.removeEventListener('abort', onCallerAbort)\n }\n }\n\n return { signal: controller.signal, timedOut: () => didTimeout, cleanup }\n }\n}\n\nfunction stripTrailingSlash(url: string): string {\n return url.replace(/\\/+$/, '')\n}\n\nfunction isAbortError(err: unknown): boolean {\n return err instanceof Error && err.name === 'AbortError'\n}\n\nfunction abortOrNetworkError(cause: unknown, timedOut: boolean, timeoutMs: number): TransportError {\n if (isAbortError(cause)) {\n if (timedOut) {\n return new TransportError(`Request timed out after ${timeoutMs}ms.`, {\n errorName: 'request_timeout',\n cause,\n })\n }\n return new TransportError('Request aborted by caller.', {\n errorName: 'client_closed_request',\n cause,\n })\n }\n return new TransportError(formatNetworkErrorMessage(cause), { cause })\n}\n\nfunction formatNetworkErrorMessage(cause: unknown): string {\n if (cause instanceof Error) {\n return `Network error: ${cause.message}`\n }\n return 'Network error: unknown failure while sending the request.'\n}\n\nfunction parseJsonOrThrow(text: string, status: number): unknown {\n if (text === '') return {}\n try {\n return JSON.parse(text)\n } catch (cause: unknown) {\n const preview = text.length > 200 ? `${text.slice(0, 200)}…` : text\n throw new TransportError(`Unexpected non-JSON response (status ${status}): ${preview}`, {\n status,\n cause,\n })\n }\n}\n\nfunction toApiError(parsed: unknown, status: number, rawText: string): CrawlbruleeError {\n if (isApiErrorResponse(parsed)) {\n return createApiError(parsed, status)\n }\n const preview = rawText.length > 200 ? `${rawText.slice(0, 200)}…` : rawText\n return new TransportError(`HTTP ${status}: ${preview || '(empty body)'}`, { status })\n}\n\nfunction isApiErrorResponse(value: unknown): value is ApiErrorResponse {\n if (value === null || typeof value !== 'object') return false\n const v = value as Record<string, unknown>\n return typeof v.name === 'string' && typeof v.message === 'string'\n}\n","import { DEFAULT_BASE_URL, ENV_API_KEY } from './config.js'\nimport { CrawlbruleeError } from './errors.js'\nimport { HttpClient, type RequestOptions } from './http.js'\nimport type {\n AsyncJobStatusResponse,\n AsyncScrapeResponse,\n MapRequest,\n MapResponse,\n ScrapeRequest,\n ScrapeResponse,\n UsageResponse,\n WhoamiResponse,\n} from './types/index.js'\n\n/** Options accepted by the {@link Crawlbrulee} constructor. */\nexport interface CrawlbruleeOptions {\n /**\n * API key sent as `Authorization: Bearer <key>`. Required — to read from the\n * environment instead, use {@link Crawlbrulee.fromEnv}. Leading and trailing\n * whitespace is stripped; an empty / whitespace-only value is rejected.\n */\n apiKey: string\n /**\n * @internal\n * Override the base URL. Reserved for local development and tests — production\n * always uses the burned-in {@link DEFAULT_BASE_URL}. Trailing slashes are\n * stripped.\n */\n baseUrl?: string\n /**\n * Per-request timeout in milliseconds. Defaults to `0` (no timeout). Set to a\n * positive number to abort slow requests; a per-call `timeoutMs` override\n * takes precedence. The timeout covers the WHOLE request, including the\n * response body read.\n */\n timeoutMs?: number\n}\n\n/**\n * Options accepted by {@link Crawlbrulee.waitForScrape}.\n *\n * Note: `timeoutMs` here is the OVERALL wait budget across all polls — not the\n * per-HTTP-request timeout. The per-poll HTTP timeout is whatever the client\n * was constructed with; if you want to bound each individual poll, construct\n * the client with `timeoutMs` set.\n */\nexport interface WaitForScrapeOptions extends Omit<RequestOptions, 'timeoutMs'> {\n /** Time between status polls in milliseconds. Default `2000`. */\n intervalMs?: number\n /**\n * Maximum total time to wait before giving up, in milliseconds. Default\n * `300_000` (5 minutes). Pass `0` to wait indefinitely.\n */\n timeoutMs?: number\n}\n\n/**\n * Official client for the crawlbrulee API.\n *\n * @example\n * ```ts\n * import { Crawlbrulee } from '@crawlbrulee/sdk'\n *\n * const crawlbrulee = new Crawlbrulee({ apiKey: 'cble_…' })\n * // or read CRAWLBRULEE_API_KEY from the environment:\n * const crawlbrulee = Crawlbrulee.fromEnv()\n *\n * const page = await crawlbrulee.scrape({\n * url: 'https://example.com',\n * extract: { markdown: true, links: true },\n * })\n * console.log(page.markdown)\n * ```\n */\nexport class Crawlbrulee {\n /** Resolved base URL — trailing slash already stripped. */\n readonly baseUrl: string\n /** Underlying HTTP layer. Exposed for advanced use cases (custom endpoints). */\n readonly http: HttpClient\n\n constructor(options: CrawlbruleeOptions) {\n const apiKey = options.apiKey?.trim()\n if (!apiKey) {\n throw new CrawlbruleeError(\n `Missing API key. Pass { apiKey } to Crawlbrulee or call Crawlbrulee.fromEnv() to read ${ENV_API_KEY}.`,\n { status: 0, errorName: null }\n )\n }\n const baseUrl = (options.baseUrl ?? DEFAULT_BASE_URL).replace(/\\/+$/, '')\n\n this.baseUrl = baseUrl\n this.http = new HttpClient({\n baseUrl,\n apiKey,\n timeoutMs: options.timeoutMs,\n })\n }\n\n /**\n * Build a {@link Crawlbrulee} reading the API key from\n * `process.env.CRAWLBRULEE_API_KEY`. Throws if the variable is unset, empty,\n * or whitespace.\n *\n * Any other constructor option can be passed via `overrides`.\n *\n * @example\n * ```ts\n * const crawlbrulee = Crawlbrulee.fromEnv()\n * const crawlbrulee = Crawlbrulee.fromEnv({ timeoutMs: 30_000 })\n * ```\n */\n static fromEnv(overrides: Omit<CrawlbruleeOptions, 'apiKey'> = {}): Crawlbrulee {\n const apiKey = readEnv(ENV_API_KEY)\n if (!apiKey) {\n throw new CrawlbruleeError(\n `${ENV_API_KEY} is not set. Export it in your shell, or pass apiKey to new Crawlbrulee({ apiKey }).`,\n { status: 0, errorName: null }\n )\n }\n return new Crawlbrulee({ ...overrides, apiKey })\n }\n\n // ------------------------------------------------------------------\n // Scraping\n // ------------------------------------------------------------------\n\n /**\n * Scrape a URL synchronously and return the extracted content.\n *\n * The request blocks until the scrape is finished. For long-running jobs\n * (heavy JS rendering, screenshots of long pages) prefer\n * {@link Crawlbrulee.scrapeAsync} so the connection isn't held open.\n *\n * @param request — body for `POST /api/scrape`.\n * @param options — per-call timeout and abort signal.\n */\n scrape(request: ScrapeRequest, options?: RequestOptions): Promise<ScrapeResponse> {\n return this.http.post<ScrapeResponse>('/api/scrape', request, options)\n }\n\n /**\n * Submit an asynchronous scrape job and return its `job_id`. Poll the job\n * with {@link Crawlbrulee.getScrapeStatus} or wait for completion with\n * {@link Crawlbrulee.waitForScrape}.\n */\n scrapeAsync(request: ScrapeRequest, options?: RequestOptions): Promise<AsyncScrapeResponse> {\n return this.http.post<AsyncScrapeResponse>('/api/scrape/async', request, options)\n }\n\n /** Look up the current status of an async scrape job. */\n getScrapeStatus(jobId: string, options?: RequestOptions): Promise<AsyncJobStatusResponse> {\n assertNonEmptyJobId(jobId)\n return this.http.get<AsyncJobStatusResponse>(\n `/api/scrape/status/${encodeURIComponent(jobId)}`,\n options\n )\n }\n\n /**\n * Fetch the result of a completed async scrape job. Throws if the job is\n * still pending/running — call {@link Crawlbrulee.getScrapeStatus}\n * first, or use {@link Crawlbrulee.waitForScrape} to poll-then-fetch.\n */\n getScrapeResult(jobId: string, options?: RequestOptions): Promise<ScrapeResponse> {\n assertNonEmptyJobId(jobId)\n return this.http.get<ScrapeResponse>(`/api/scrape/result/${encodeURIComponent(jobId)}`, options)\n }\n\n /**\n * Poll an async scrape job until it reaches a terminal state, then return\n * the scrape result.\n *\n * Throws a {@link CrawlbruleeError} when:\n * - the job ends in `failed` (`errorName: 'job_failed'`),\n * - the server reports an unexpected status (`errorName: 'job_failed'`),\n * - the overall wait exceeds `timeoutMs` (`errorName: 'request_timeout'`),\n * - the caller's `signal` aborts (`errorName: 'client_closed_request'`).\n */\n async waitForScrape(jobId: string, options: WaitForScrapeOptions = {}): Promise<ScrapeResponse> {\n assertNonEmptyJobId(jobId)\n const intervalMs = options.intervalMs ?? 2000\n const timeoutMs = options.timeoutMs ?? 300_000\n const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : Number.POSITIVE_INFINITY\n\n while (true) {\n throwIfAborted(options.signal)\n if (Date.now() >= deadline) {\n throw new CrawlbruleeError(\n `Timed out after ${timeoutMs}ms waiting for async scrape job ${jobId}.`,\n { status: 0, errorName: 'request_timeout' }\n )\n }\n\n const status = await this.getScrapeStatus(jobId, { signal: options.signal })\n\n switch (status.status) {\n case 'done':\n return this.getScrapeResult(jobId, { signal: options.signal })\n\n case 'failed':\n throw new CrawlbruleeError(status.error ?? `Async scrape job ${jobId} failed.`, {\n status: 0,\n errorName: 'job_failed',\n })\n\n case 'pending':\n case 'running':\n break\n\n default:\n throw new CrawlbruleeError(\n `Async scrape job ${jobId} returned unexpected status '${String(status.status)}'.`,\n { status: 0, errorName: 'job_failed' }\n )\n }\n\n await sleep(intervalMs, options.signal)\n }\n }\n\n // ------------------------------------------------------------------\n // Mapping\n // ------------------------------------------------------------------\n\n /**\n * Build (or return a cached) site link-map for a domain. Combines sitemap\n * discovery with the freshest cached homepage scrape when available.\n */\n map(request: MapRequest, options?: RequestOptions): Promise<MapResponse> {\n return this.http.post<MapResponse>('/api/map', request, options)\n }\n\n // ------------------------------------------------------------------\n // Account\n // ------------------------------------------------------------------\n\n /**\n * Return the current billing-cycle usage: total/used/available credits,\n * used quota percentage, max concurrency, and when the cycle resets.\n */\n usage(options?: RequestOptions): Promise<UsageResponse> {\n return this.http.get<UsageResponse>('/api/usage', options)\n }\n\n /**\n * Return the organization name and identifying details of the API token\n * used to authenticate this request. Useful for confirming which key is in\n * use before performing destructive operations.\n */\n whoami(options?: RequestOptions): Promise<WhoamiResponse> {\n return this.http.get<WhoamiResponse>('/api/whoami', options)\n }\n}\n\n/**\n * Defensive read of `process.env[name]`. Guards both the absence of `process`\n * (browser / edge runtimes) and Deno's permission throw on env access without\n * `--allow-env`.\n */\nfunction readEnv(name: string): string | undefined {\n try {\n if (typeof process === 'undefined' || !process.env) return undefined\n const v = process.env[name]\n return typeof v === 'string' && v.trim().length > 0 ? v.trim() : undefined\n } catch {\n return undefined\n }\n}\n\nfunction assertNonEmptyJobId(jobId: string): void {\n if (typeof jobId !== 'string' || jobId.trim().length === 0) {\n throw new CrawlbruleeError('jobId must be a non-empty string.', {\n status: 0,\n errorName: null,\n })\n }\n}\n\nfunction throwIfAborted(signal: AbortSignal | undefined): void {\n if (signal?.aborted) {\n throw new CrawlbruleeError('Request aborted by caller.', {\n status: 0,\n errorName: 'client_closed_request',\n cause: signal.reason,\n })\n }\n}\n\nfunction sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {\n return new Promise((resolve, reject) => {\n const onAbort = () => {\n clearTimeout(timer)\n reject(\n new CrawlbruleeError('Request aborted by caller.', {\n status: 0,\n errorName: 'client_closed_request',\n cause: signal?.reason,\n })\n )\n }\n const timer = setTimeout(() => {\n signal?.removeEventListener('abort', onAbort)\n resolve()\n }, ms)\n if (signal) {\n if (signal.aborted) {\n clearTimeout(timer)\n onAbort()\n return\n }\n signal.addEventListener('abort', onAbort, { once: true })\n }\n })\n}\n"]}
|