@crawlbrulee/sdk 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -76
- package/dist/index.cjs +778 -617
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +654 -668
- package/dist/index.d.ts +654 -668
- package/dist/index.js +777 -616
- package/dist/index.js.map +1 -1
- package/package.json +6 -6
package/dist/index.js
CHANGED
|
@@ -1,688 +1,849 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
1
|
+
//#region src/config.ts
|
|
2
|
+
/**
|
|
3
|
+
* Production base URL of the crawlbrulee API. Used by default when the caller
|
|
4
|
+
* doesn't pass a `baseUrl` to {@link Crawlbrulee}. Local development and
|
|
5
|
+
* staging callers point at their own host via that option.
|
|
6
|
+
*/
|
|
7
|
+
const DEFAULT_BASE_URL = "https://api.crawlbrulee.com";
|
|
8
|
+
/** Default request timeout when the caller doesn't specify one (0 disables the timeout). */
|
|
9
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 0;
|
|
10
|
+
/** Environment variable read by `Crawlbrulee.fromEnv()` to source the API key. */
|
|
11
|
+
const ENV_API_KEY = "CRAWLBRULEE_API_KEY";
|
|
12
|
+
/** Identifies the SDK in the `User-Agent` header. Kept in one place for easy bumping. */
|
|
13
|
+
const USER_AGENT = "@crawlbrulee/sdk/0.7.0 (node)";
|
|
6
14
|
|
|
7
|
-
|
|
15
|
+
//#endregion
|
|
16
|
+
//#region src/errors.ts
|
|
17
|
+
/**
|
|
18
|
+
* Base error class for every failure raised by the SDK.
|
|
19
|
+
*
|
|
20
|
+
* Two kinds of failures end up here:
|
|
21
|
+
*
|
|
22
|
+
* 1. **API errors** — the server returned a non-2xx response with a well-formed
|
|
23
|
+
* JSON body. In that case `status`, `errorName` and (sometimes) `details`
|
|
24
|
+
* are populated.
|
|
25
|
+
* 2. **Transport errors** — the request never produced a structured response
|
|
26
|
+
* (network failure, abort, timeout, non-JSON body, etc.). In that case
|
|
27
|
+
* `status` may be `0` and `errorName` is one of the synthetic transport
|
|
28
|
+
* names (`request_timeout`, `client_closed_request`) or `null`.
|
|
29
|
+
*
|
|
30
|
+
* Typed subclasses are exported for the most common cases. To branch on more
|
|
31
|
+
* specific server-side errors, switch on `err.errorName` or use the
|
|
32
|
+
* {@link isCrawlbruleeError} helper.
|
|
33
|
+
*/
|
|
8
34
|
var CrawlbruleeError = class extends Error {
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
35
|
+
/** HTTP status code; `0` for transport-level failures with no response. */
|
|
36
|
+
status;
|
|
37
|
+
/** The `name` field from the API error body, or `null` for transport errors. */
|
|
38
|
+
errorName;
|
|
39
|
+
/** Structured detail block from the API error body, if any. */
|
|
40
|
+
details;
|
|
41
|
+
/** The original parsed error body, when one was received. */
|
|
42
|
+
response;
|
|
43
|
+
constructor(message, options) {
|
|
44
|
+
super(message, options.cause !== void 0 ? { cause: options.cause } : void 0);
|
|
45
|
+
this.name = "CrawlbruleeError";
|
|
46
|
+
this.status = options.status;
|
|
47
|
+
this.errorName = options.errorName;
|
|
48
|
+
this.details = options.details;
|
|
49
|
+
this.response = options.response;
|
|
50
|
+
}
|
|
25
51
|
};
|
|
52
|
+
/** Raised for 401 / 403 responses (missing, invalid, or unauthorized API key). */
|
|
26
53
|
var AuthenticationError = class extends CrawlbruleeError {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
54
|
+
constructor(message, options) {
|
|
55
|
+
super(message, options);
|
|
56
|
+
this.name = "AuthenticationError";
|
|
57
|
+
}
|
|
31
58
|
};
|
|
59
|
+
/**
|
|
60
|
+
* Raised for HTTP 429 responses. When the server included a `retry_after_ms`
|
|
61
|
+
* hint in `details` it is surfaced directly on the instance.
|
|
62
|
+
*
|
|
63
|
+
* `errorName` is always the literal `'too_many_requests'` — the SDK normalizes
|
|
64
|
+
* this even when the server returns a 429 with a different `name` field
|
|
65
|
+
* (e.g. a CDN coalescing upstream rate limiting). The original body is still
|
|
66
|
+
* available on `response`.
|
|
67
|
+
*/
|
|
32
68
|
var RateLimitError = class extends CrawlbruleeError {
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
69
|
+
errorName;
|
|
70
|
+
/** Suggested delay (ms) before retrying, when the server provided one. */
|
|
71
|
+
retryAfterMs;
|
|
72
|
+
/** Which rate limit was tripped (e.g. `org`, `ip`), when provided. */
|
|
73
|
+
limitedBy;
|
|
74
|
+
constructor(message, options) {
|
|
75
|
+
super(message, {
|
|
76
|
+
...options,
|
|
77
|
+
errorName: "too_many_requests",
|
|
78
|
+
details: options.details
|
|
79
|
+
});
|
|
80
|
+
this.name = "RateLimitError";
|
|
81
|
+
this.errorName = "too_many_requests";
|
|
82
|
+
this.retryAfterMs = options.details?.retry_after_ms;
|
|
83
|
+
this.limitedBy = options.details?.limited_by;
|
|
84
|
+
}
|
|
45
85
|
};
|
|
86
|
+
/**
|
|
87
|
+
* Raised when the API rejects a request because the org's plan limits would
|
|
88
|
+
* be exceeded (credit limit, concurrency cap, overage hard cap, etc.).
|
|
89
|
+
*
|
|
90
|
+
* `errorName` is always the literal `'usage_allocation_error'`.
|
|
91
|
+
*/
|
|
46
92
|
var UsageAllocationError = class extends CrawlbruleeError {
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
93
|
+
errorName;
|
|
94
|
+
/** Specific reason the allocation was denied. */
|
|
95
|
+
reason;
|
|
96
|
+
/** Current usage / limit snapshot at the time of the rejection. */
|
|
97
|
+
usage;
|
|
98
|
+
constructor(message, options) {
|
|
99
|
+
super(message, {
|
|
100
|
+
...options,
|
|
101
|
+
errorName: "usage_allocation_error"
|
|
102
|
+
});
|
|
103
|
+
this.name = "UsageAllocationError";
|
|
104
|
+
this.errorName = "usage_allocation_error";
|
|
105
|
+
this.reason = options.details.reason;
|
|
106
|
+
this.usage = options.details.details;
|
|
107
|
+
}
|
|
59
108
|
};
|
|
109
|
+
/** Raised for 4xx responses caused by an invalid request shape or arguments. */
|
|
60
110
|
var ValidationError = class extends CrawlbruleeError {
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
111
|
+
constructor(message, options) {
|
|
112
|
+
super(message, options);
|
|
113
|
+
this.name = "ValidationError";
|
|
114
|
+
}
|
|
65
115
|
};
|
|
116
|
+
/** Raised for 404 responses (e.g. unknown async job ID). */
|
|
66
117
|
var NotFoundError = class extends CrawlbruleeError {
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
118
|
+
constructor(message, options) {
|
|
119
|
+
super(message, options);
|
|
120
|
+
this.name = "NotFoundError";
|
|
121
|
+
}
|
|
71
122
|
};
|
|
123
|
+
/**
|
|
124
|
+
* Raised when a request cannot be sent or no structured response is parsed.
|
|
125
|
+
*
|
|
126
|
+
* The `errorName` discriminates the cause:
|
|
127
|
+
* - `'request_timeout'` — the per-request timeout fired.
|
|
128
|
+
* - `'client_closed_request'` — the caller's `AbortSignal` fired.
|
|
129
|
+
* - `null` — generic transport failure (network error, non-JSON body, etc.).
|
|
130
|
+
*/
|
|
72
131
|
var TransportError = class extends CrawlbruleeError {
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
132
|
+
constructor(message, options = {}) {
|
|
133
|
+
super(message, {
|
|
134
|
+
status: options.status ?? 0,
|
|
135
|
+
errorName: options.errorName ?? null,
|
|
136
|
+
cause: options.cause
|
|
137
|
+
});
|
|
138
|
+
this.name = "TransportError";
|
|
139
|
+
}
|
|
81
140
|
};
|
|
141
|
+
/** Narrow `unknown` to the SDK's base error type. */
|
|
82
142
|
function isCrawlbruleeError(err) {
|
|
83
|
-
|
|
143
|
+
return err instanceof CrawlbruleeError;
|
|
84
144
|
}
|
|
145
|
+
/**
|
|
146
|
+
* Map an API error body + HTTP status to the most specific error class.
|
|
147
|
+
*
|
|
148
|
+
* Dispatch is **name-first**: the body's `name` field is the most reliable
|
|
149
|
+
* signal of what went wrong. Status code is used only as a fallback when the
|
|
150
|
+
* name is unrecognized (e.g. a CDN-synthesized error). This avoids
|
|
151
|
+
* miscategorizing things like a 403 with `name: 'not_found'` as an auth error.
|
|
152
|
+
*
|
|
153
|
+
* Internal — used by the HTTP layer.
|
|
154
|
+
*/
|
|
85
155
|
function createApiError(body, status) {
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
156
|
+
const { name, message, details } = body;
|
|
157
|
+
const response = body;
|
|
158
|
+
switch (name) {
|
|
159
|
+
case "too_many_requests": return new RateLimitError(message, {
|
|
160
|
+
status,
|
|
161
|
+
details: details?.error_name === "too_many_requests" ? details : void 0,
|
|
162
|
+
response
|
|
163
|
+
});
|
|
164
|
+
case "usage_allocation_error": {
|
|
165
|
+
const usageDetails = details?.error_name === "usage_allocation_error" ? details : {
|
|
166
|
+
error_name: "usage_allocation_error",
|
|
167
|
+
reason: "internal_error"
|
|
168
|
+
};
|
|
169
|
+
return new UsageAllocationError(message, {
|
|
170
|
+
status,
|
|
171
|
+
details: usageDetails,
|
|
172
|
+
response
|
|
173
|
+
});
|
|
174
|
+
}
|
|
175
|
+
case "invalid_credentials":
|
|
176
|
+
case "access_denied": return new AuthenticationError(message, {
|
|
177
|
+
status,
|
|
178
|
+
errorName: name,
|
|
179
|
+
response
|
|
180
|
+
});
|
|
181
|
+
case "not_found": return new NotFoundError(message, {
|
|
182
|
+
status,
|
|
183
|
+
errorName: name,
|
|
184
|
+
response
|
|
185
|
+
});
|
|
186
|
+
case "validation_error":
|
|
187
|
+
case "invalid_url":
|
|
188
|
+
case "url_too_long":
|
|
189
|
+
case "unsupported_url_schema":
|
|
190
|
+
case "url_credentials_not_supported":
|
|
191
|
+
case "blocked_url":
|
|
192
|
+
case "unsupported_content": return new ValidationError(message, {
|
|
193
|
+
status,
|
|
194
|
+
errorName: name,
|
|
195
|
+
response
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
if (status === 429) return new RateLimitError(message, {
|
|
199
|
+
status,
|
|
200
|
+
response
|
|
201
|
+
});
|
|
202
|
+
if (status === 401 || status === 403) return new AuthenticationError(message, {
|
|
203
|
+
status,
|
|
204
|
+
errorName: name,
|
|
205
|
+
response
|
|
206
|
+
});
|
|
207
|
+
if (status === 404) return new NotFoundError(message, {
|
|
208
|
+
status,
|
|
209
|
+
errorName: name,
|
|
210
|
+
response
|
|
211
|
+
});
|
|
212
|
+
return new CrawlbruleeError(message, {
|
|
213
|
+
status,
|
|
214
|
+
errorName: name,
|
|
215
|
+
details,
|
|
216
|
+
response
|
|
217
|
+
});
|
|
123
218
|
}
|
|
124
219
|
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
220
|
+
//#endregion
|
|
221
|
+
//#region src/instrumentation.ts
|
|
222
|
+
/**
|
|
223
|
+
* Centralized factory for the low-level dependencies the SDK injects into its
|
|
224
|
+
* HTTP layer. Production code resolves these to the runtime's global `fetch`
|
|
225
|
+
* and the burned-in production base URL; tests stub this module to swap in
|
|
226
|
+
* mocks and alternate hosts.
|
|
227
|
+
*
|
|
228
|
+
* This is internal — it is not exported from the package's public entry. Tests
|
|
229
|
+
* import it from `src/instrumentation.js` directly and use `vi.spyOn` to
|
|
230
|
+
* substitute behavior.
|
|
231
|
+
*/
|
|
232
|
+
const CwblInstrumentation = {
|
|
233
|
+
/**
|
|
234
|
+
* Resolve the `fetch` implementation the SDK should use. Throws a
|
|
235
|
+
* {@link CrawlbruleeError} if the runtime does not expose a global `fetch`.
|
|
236
|
+
*/
|
|
237
|
+
getFetch() {
|
|
238
|
+
const g = globalThis;
|
|
239
|
+
if (typeof g.fetch !== "function") throw new CrawlbruleeError("No global fetch is available in this runtime. crawlbrulee requires Node.js 22+, Bun, Deno, or a modern browser/edge runtime.", {
|
|
240
|
+
status: 0,
|
|
241
|
+
errorName: null
|
|
242
|
+
});
|
|
243
|
+
return g.fetch.bind(globalThis);
|
|
244
|
+
},
|
|
245
|
+
/**
|
|
246
|
+
* Resolve the base URL the SDK should target. Returns the production host by
|
|
247
|
+
* default; tests stub this to point at a mock origin.
|
|
248
|
+
*/
|
|
249
|
+
getBaseUrl() {
|
|
250
|
+
return DEFAULT_BASE_URL;
|
|
251
|
+
}
|
|
148
252
|
};
|
|
149
253
|
|
|
150
|
-
|
|
254
|
+
//#endregion
|
|
255
|
+
//#region src/http.ts
|
|
256
|
+
/**
|
|
257
|
+
* Minimal `fetch`-based HTTP layer used by {@link Crawlbrulee}. Handles:
|
|
258
|
+
*
|
|
259
|
+
* - URL composition (joining `baseUrl` and path safely).
|
|
260
|
+
* - JSON serialization and parsing.
|
|
261
|
+
* - The `Authorization: Bearer …` header.
|
|
262
|
+
* - Composing the caller's `AbortSignal` with an internal timeout signal. The
|
|
263
|
+
* timeout covers the WHOLE request, including the response body read — not
|
|
264
|
+
* just the time-to-headers.
|
|
265
|
+
* - Mapping non-2xx responses to typed `CrawlbruleeError` subclasses via
|
|
266
|
+
* {@link createApiError}.
|
|
267
|
+
*
|
|
268
|
+
* The base URL and `fetch` implementation are sourced from
|
|
269
|
+
* {@link CwblInstrumentation} at construction time so tests can stub the
|
|
270
|
+
* module.
|
|
271
|
+
*/
|
|
151
272
|
var HttpClient = class {
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
273
|
+
baseUrl;
|
|
274
|
+
apiKey;
|
|
275
|
+
fetch;
|
|
276
|
+
timeoutMs;
|
|
277
|
+
constructor(options) {
|
|
278
|
+
this.baseUrl = stripTrailingSlash(options.baseUrl ?? CwblInstrumentation.getBaseUrl());
|
|
279
|
+
this.apiKey = options.apiKey;
|
|
280
|
+
this.fetch = CwblInstrumentation.getFetch();
|
|
281
|
+
this.timeoutMs = options.timeoutMs ?? 0;
|
|
282
|
+
}
|
|
283
|
+
/** Send a `GET` request and parse the response as `T`. */
|
|
284
|
+
get(path, options) {
|
|
285
|
+
return this.send({
|
|
286
|
+
method: "GET",
|
|
287
|
+
path,
|
|
288
|
+
...options
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
/** Send a `POST` request with a JSON body and parse the response as `T`. */
|
|
292
|
+
post(path, body, options) {
|
|
293
|
+
return this.send({
|
|
294
|
+
method: "POST",
|
|
295
|
+
path,
|
|
296
|
+
body,
|
|
297
|
+
...options
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
async send(args) {
|
|
301
|
+
const url = this.buildUrl(args.path);
|
|
302
|
+
const headers = this.buildHeaders(args);
|
|
303
|
+
const body = args.body === void 0 ? void 0 : JSON.stringify(args.body);
|
|
304
|
+
const composed = this.composeSignal(args.signal, args.timeoutMs);
|
|
305
|
+
try {
|
|
306
|
+
let res;
|
|
307
|
+
try {
|
|
308
|
+
res = await this.fetch(url, {
|
|
309
|
+
method: args.method,
|
|
310
|
+
headers,
|
|
311
|
+
body,
|
|
312
|
+
signal: composed.signal
|
|
313
|
+
});
|
|
314
|
+
} catch (cause) {
|
|
315
|
+
throw abortOrNetworkError(cause, composed.timedOut(), args.timeoutMs ?? this.timeoutMs);
|
|
316
|
+
}
|
|
317
|
+
let text;
|
|
318
|
+
try {
|
|
319
|
+
text = await res.text();
|
|
320
|
+
} catch (cause) {
|
|
321
|
+
if (isAbortError(cause)) throw abortOrNetworkError(cause, composed.timedOut(), args.timeoutMs ?? this.timeoutMs);
|
|
322
|
+
throw new TransportError(`Failed to read response body (status ${res.status}).`, {
|
|
323
|
+
status: res.status,
|
|
324
|
+
cause
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
const parsed = parseJsonOrThrow(text, res.status);
|
|
328
|
+
if (!res.ok) throw toApiError(parsed, res.status, text);
|
|
329
|
+
return parsed;
|
|
330
|
+
} finally {
|
|
331
|
+
composed.cleanup();
|
|
332
|
+
}
|
|
333
|
+
}
|
|
334
|
+
buildUrl(path) {
|
|
335
|
+
if (!path.startsWith("/")) throw new TypeError(`crawlbrulee SDK: path must start with '/' (received '${path}')`);
|
|
336
|
+
return `${this.baseUrl}${path}`;
|
|
337
|
+
}
|
|
338
|
+
buildHeaders(args) {
|
|
339
|
+
const headers = {
|
|
340
|
+
accept: "application/json",
|
|
341
|
+
"user-agent": USER_AGENT,
|
|
342
|
+
authorization: `Bearer ${this.apiKey}`
|
|
343
|
+
};
|
|
344
|
+
if (args.body !== void 0) headers["content-type"] = "application/json";
|
|
345
|
+
return headers;
|
|
346
|
+
}
|
|
347
|
+
/**
|
|
348
|
+
* Build a single `AbortSignal` that fires when either the caller-supplied
|
|
349
|
+
* signal aborts OR the per-request timeout elapses. The returned `cleanup`
|
|
350
|
+
* callback MUST be invoked on every exit path so we don't leak timers or
|
|
351
|
+
* dead listeners on long-lived caller signals.
|
|
352
|
+
*/
|
|
353
|
+
composeSignal(callerSignal, overrideTimeoutMs) {
|
|
354
|
+
const timeoutMs = overrideTimeoutMs ?? this.timeoutMs;
|
|
355
|
+
const hasTimeout = Number.isFinite(timeoutMs) && timeoutMs > 0;
|
|
356
|
+
if (!hasTimeout && !callerSignal) return {
|
|
357
|
+
signal: void 0,
|
|
358
|
+
timedOut: () => false,
|
|
359
|
+
cleanup: () => {}
|
|
360
|
+
};
|
|
361
|
+
if (!hasTimeout) return {
|
|
362
|
+
signal: callerSignal,
|
|
363
|
+
timedOut: () => false,
|
|
364
|
+
cleanup: () => {}
|
|
365
|
+
};
|
|
366
|
+
const controller = new AbortController();
|
|
367
|
+
let didTimeout = false;
|
|
368
|
+
const timer = setTimeout(() => {
|
|
369
|
+
didTimeout = true;
|
|
370
|
+
controller.abort(/* @__PURE__ */ new Error("request_timeout"));
|
|
371
|
+
}, timeoutMs);
|
|
372
|
+
let onCallerAbort;
|
|
373
|
+
if (callerSignal) if (callerSignal.aborted) {
|
|
374
|
+
clearTimeout(timer);
|
|
375
|
+
controller.abort(callerSignal.reason);
|
|
376
|
+
} else {
|
|
377
|
+
onCallerAbort = () => {
|
|
378
|
+
clearTimeout(timer);
|
|
379
|
+
controller.abort(callerSignal.reason);
|
|
380
|
+
};
|
|
381
|
+
callerSignal.addEventListener("abort", onCallerAbort, { once: true });
|
|
382
|
+
}
|
|
383
|
+
const cleanup = () => {
|
|
384
|
+
clearTimeout(timer);
|
|
385
|
+
if (onCallerAbort && callerSignal) callerSignal.removeEventListener("abort", onCallerAbort);
|
|
386
|
+
};
|
|
387
|
+
return {
|
|
388
|
+
signal: controller.signal,
|
|
389
|
+
timedOut: () => didTimeout,
|
|
390
|
+
cleanup
|
|
391
|
+
};
|
|
392
|
+
}
|
|
265
393
|
};
|
|
266
394
|
function stripTrailingSlash(url) {
|
|
267
|
-
|
|
395
|
+
return url.replace(/\/+$/, "");
|
|
268
396
|
}
|
|
269
397
|
function isAbortError(err) {
|
|
270
|
-
|
|
398
|
+
return err instanceof Error && err.name === "AbortError";
|
|
271
399
|
}
|
|
272
400
|
function abortOrNetworkError(cause, timedOut, timeoutMs) {
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
}
|
|
285
|
-
return new TransportError(formatNetworkErrorMessage(cause), { cause });
|
|
401
|
+
if (isAbortError(cause)) {
|
|
402
|
+
if (timedOut) return new TransportError(`Request timed out after ${timeoutMs}ms.`, {
|
|
403
|
+
errorName: "request_timeout",
|
|
404
|
+
cause
|
|
405
|
+
});
|
|
406
|
+
return new TransportError("Request aborted by caller.", {
|
|
407
|
+
errorName: "client_closed_request",
|
|
408
|
+
cause
|
|
409
|
+
});
|
|
410
|
+
}
|
|
411
|
+
return new TransportError(formatNetworkErrorMessage(cause), { cause });
|
|
286
412
|
}
|
|
287
413
|
function formatNetworkErrorMessage(cause) {
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
}
|
|
291
|
-
return "Network error: unknown failure while sending the request.";
|
|
414
|
+
if (cause instanceof Error) return `Network error: ${cause.message}`;
|
|
415
|
+
return "Network error: unknown failure while sending the request.";
|
|
292
416
|
}
|
|
293
417
|
function parseJsonOrThrow(text, status) {
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
}
|
|
418
|
+
if (text === "") return {};
|
|
419
|
+
try {
|
|
420
|
+
return JSON.parse(text);
|
|
421
|
+
} catch (cause) {
|
|
422
|
+
throw new TransportError(`Unexpected non-JSON response (status ${status}): ${text.length > 200 ? `${text.slice(0, 200)}…` : text}`, {
|
|
423
|
+
status,
|
|
424
|
+
cause
|
|
425
|
+
});
|
|
426
|
+
}
|
|
304
427
|
}
|
|
305
428
|
function toApiError(parsed, status, rawText) {
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
}
|
|
309
|
-
const preview = rawText.length > 200 ? `${rawText.slice(0, 200)}\u2026` : rawText;
|
|
310
|
-
return new TransportError(`HTTP ${status}: ${preview || "(empty body)"}`, { status });
|
|
429
|
+
if (isApiErrorResponse(parsed)) return createApiError(parsed, status);
|
|
430
|
+
return new TransportError(`HTTP ${status}: ${(rawText.length > 200 ? `${rawText.slice(0, 200)}…` : rawText) || "(empty body)"}`, { status });
|
|
311
431
|
}
|
|
312
432
|
function isApiErrorResponse(value) {
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
433
|
+
if (value === null || typeof value !== "object") return false;
|
|
434
|
+
const v = value;
|
|
435
|
+
return typeof v.name === "string" && typeof v.message === "string";
|
|
316
436
|
}
|
|
317
437
|
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
return this.http.get("/api/whoami", options);
|
|
520
|
-
}
|
|
438
|
+
//#endregion
|
|
439
|
+
//#region src/client.ts
|
|
440
|
+
/**
|
|
441
|
+
* Official client for the crawlbrulee API.
|
|
442
|
+
*
|
|
443
|
+
* @example
|
|
444
|
+
* ```ts
|
|
445
|
+
* import { Crawlbrulee } from '@crawlbrulee/sdk'
|
|
446
|
+
*
|
|
447
|
+
* const crawlbrulee = new Crawlbrulee({ apiKey: 'cwbl_…' })
|
|
448
|
+
* // or read CRAWLBRULEE_API_KEY from the environment:
|
|
449
|
+
* const crawlbrulee = Crawlbrulee.fromEnv()
|
|
450
|
+
*
|
|
451
|
+
* const page = await crawlbrulee.scrape({
|
|
452
|
+
* url: 'https://example.com',
|
|
453
|
+
* extract: { markdown: true, links: true },
|
|
454
|
+
* })
|
|
455
|
+
* console.log(page.markdown)
|
|
456
|
+
* ```
|
|
457
|
+
*/
|
|
458
|
+
var Crawlbrulee = class Crawlbrulee {
|
|
459
|
+
/** Resolved base URL — trailing slash already stripped. */
|
|
460
|
+
baseUrl;
|
|
461
|
+
/** Underlying HTTP layer. Exposed for advanced use cases (custom endpoints). */
|
|
462
|
+
http;
|
|
463
|
+
constructor(options) {
|
|
464
|
+
const apiKey = options.apiKey?.trim();
|
|
465
|
+
if (!apiKey) throw new CrawlbruleeError(`Missing API key. Pass { apiKey } to Crawlbrulee or call Crawlbrulee.fromEnv() to read ${ENV_API_KEY}.`, {
|
|
466
|
+
status: 0,
|
|
467
|
+
errorName: null
|
|
468
|
+
});
|
|
469
|
+
this.http = new HttpClient({
|
|
470
|
+
apiKey,
|
|
471
|
+
baseUrl: options.baseUrl,
|
|
472
|
+
timeoutMs: options.timeoutMs
|
|
473
|
+
});
|
|
474
|
+
this.baseUrl = this.http.baseUrl;
|
|
475
|
+
}
|
|
476
|
+
/**
|
|
477
|
+
* Build a {@link Crawlbrulee} reading the API key from
|
|
478
|
+
* `process.env.CRAWLBRULEE_API_KEY`. Throws if the variable is unset, empty,
|
|
479
|
+
* or whitespace.
|
|
480
|
+
*
|
|
481
|
+
* Any other constructor option can be passed via `overrides`.
|
|
482
|
+
*
|
|
483
|
+
* @example
|
|
484
|
+
* ```ts
|
|
485
|
+
* const crawlbrulee = Crawlbrulee.fromEnv()
|
|
486
|
+
* const crawlbrulee = Crawlbrulee.fromEnv({ timeoutMs: 30_000 })
|
|
487
|
+
* ```
|
|
488
|
+
*/
|
|
489
|
+
static fromEnv(overrides = {}) {
|
|
490
|
+
const apiKey = readEnv(ENV_API_KEY);
|
|
491
|
+
if (!apiKey) throw new CrawlbruleeError(`${ENV_API_KEY} is not set. Export it in your shell, or pass apiKey to new Crawlbrulee({ apiKey }).`, {
|
|
492
|
+
status: 0,
|
|
493
|
+
errorName: null
|
|
494
|
+
});
|
|
495
|
+
return new Crawlbrulee({
|
|
496
|
+
...overrides,
|
|
497
|
+
apiKey
|
|
498
|
+
});
|
|
499
|
+
}
|
|
500
|
+
/**
|
|
501
|
+
* Scrape a URL synchronously and return the extracted content.
|
|
502
|
+
*
|
|
503
|
+
* The request blocks until the scrape is finished. For long-running jobs
|
|
504
|
+
* (heavy JS rendering, screenshots of long pages) prefer
|
|
505
|
+
* {@link Crawlbrulee.scrapeAsync} so the connection isn't held open.
|
|
506
|
+
*
|
|
507
|
+
* @param request — body for `POST /api/scrape`.
|
|
508
|
+
* @param options — per-call timeout and abort signal.
|
|
509
|
+
*/
|
|
510
|
+
scrape(request, options) {
|
|
511
|
+
return this.http.post("/api/scrape", request, options);
|
|
512
|
+
}
|
|
513
|
+
/**
|
|
514
|
+
* Submit an asynchronous scrape job and return its `job_id`. Poll the job
|
|
515
|
+
* with {@link Crawlbrulee.getScrapeStatus} or wait for completion with
|
|
516
|
+
* {@link Crawlbrulee.waitForScrape}.
|
|
517
|
+
*
|
|
518
|
+
* Pass an optional `webhook` to have the API deliver a signed
|
|
519
|
+
* `scrape.complete` `POST` to your endpoint when the job finishes (see
|
|
520
|
+
* {@link AsyncScrapeWebhook}). This field is async-only.
|
|
521
|
+
*/
|
|
522
|
+
scrapeAsync(request, options) {
|
|
523
|
+
return this.http.post("/api/scrape/async", request, options);
|
|
524
|
+
}
|
|
525
|
+
/** Look up the current status of an async scrape job. */
|
|
526
|
+
getScrapeStatus(jobId, options) {
|
|
527
|
+
assertNonEmptyJobId(jobId);
|
|
528
|
+
return this.http.get(`/api/scrape/status/${encodeURIComponent(jobId)}`, options);
|
|
529
|
+
}
|
|
530
|
+
/**
|
|
531
|
+
* Fetch the result of a completed async scrape job. Throws if the job is
|
|
532
|
+
* still pending/running — call {@link Crawlbrulee.getScrapeStatus}
|
|
533
|
+
* first, or use {@link Crawlbrulee.waitForScrape} to poll-then-fetch.
|
|
534
|
+
*/
|
|
535
|
+
getScrapeResult(jobId, options) {
|
|
536
|
+
assertNonEmptyJobId(jobId);
|
|
537
|
+
return this.http.get(`/api/scrape/result/${encodeURIComponent(jobId)}`, options);
|
|
538
|
+
}
|
|
539
|
+
/**
|
|
540
|
+
* Fetch the scrape result referenced by a `scrape.complete` webhook body.
|
|
541
|
+
*
|
|
542
|
+
* Always verify the webhook signature with `verifyWebhookSignature` before
|
|
543
|
+
* acting on it; this method trusts the parsed body it is handed.
|
|
544
|
+
*
|
|
545
|
+
* Behavior by `data.status`:
|
|
546
|
+
* - `success` — delegates to {@link Crawlbrulee.getScrapeResult} for the
|
|
547
|
+
* webhook's `job_id` and returns the parsed result.
|
|
548
|
+
* - `failed` — throws a {@link CrawlbruleeError} carrying `data.error`
|
|
549
|
+
* (`errorName: 'job_failed'`); there is no result to fetch.
|
|
550
|
+
* - `cancelled` — throws a {@link CrawlbruleeError}
|
|
551
|
+
* (`errorName: 'client_closed_request'`).
|
|
552
|
+
*
|
|
553
|
+
* A non-`scrape.complete` envelope throws a {@link CrawlbruleeError}
|
|
554
|
+
* defensively. Any HTTP error from the underlying fetch propagates as the
|
|
555
|
+
* usual typed `CrawlbruleeError` subclass.
|
|
556
|
+
*/
|
|
557
|
+
async fetchScrapeResultFromWebhook(webhook, options) {
|
|
558
|
+
if (webhook?.event !== "scrape.complete") throw new CrawlbruleeError(`Expected a 'scrape.complete' webhook but received '${String(webhook?.event)}'.`, {
|
|
559
|
+
status: 0,
|
|
560
|
+
errorName: "validation_error"
|
|
561
|
+
});
|
|
562
|
+
const { job_id: jobId, status, error } = webhook.data;
|
|
563
|
+
switch (status) {
|
|
564
|
+
case "success": return this.getScrapeResult(jobId, options);
|
|
565
|
+
case "failed": throw new CrawlbruleeError(error ?? `Async scrape job ${jobId} failed.`, {
|
|
566
|
+
status: 0,
|
|
567
|
+
errorName: "job_failed"
|
|
568
|
+
});
|
|
569
|
+
case "cancelled": throw new CrawlbruleeError(`Async scrape job ${jobId} was cancelled.`, {
|
|
570
|
+
status: 0,
|
|
571
|
+
errorName: "client_closed_request"
|
|
572
|
+
});
|
|
573
|
+
default: throw new CrawlbruleeError(`Async scrape webhook for job ${jobId} carried an unexpected status '${String(status)}'.`, {
|
|
574
|
+
status: 0,
|
|
575
|
+
errorName: "validation_error"
|
|
576
|
+
});
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* Poll an async scrape job until it reaches a terminal state, then return
|
|
581
|
+
* the scrape result.
|
|
582
|
+
*
|
|
583
|
+
* Throws a {@link CrawlbruleeError} when:
|
|
584
|
+
* - the job ends in `failed` (`errorName: 'job_failed'`),
|
|
585
|
+
* - the server reports an unexpected status (`errorName: 'job_failed'`),
|
|
586
|
+
* - the overall wait exceeds `timeoutMs` (`errorName: 'request_timeout'`),
|
|
587
|
+
* - the caller's `signal` aborts (`errorName: 'client_closed_request'`).
|
|
588
|
+
*/
|
|
589
|
+
async waitForScrape(jobId, options = {}) {
|
|
590
|
+
assertNonEmptyJobId(jobId);
|
|
591
|
+
const intervalMs = options.intervalMs ?? 2e3;
|
|
592
|
+
const timeoutMs = options.timeoutMs ?? 3e5;
|
|
593
|
+
const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : Number.POSITIVE_INFINITY;
|
|
594
|
+
while (true) {
|
|
595
|
+
throwIfAborted(options.signal);
|
|
596
|
+
if (Date.now() >= deadline) throw new CrawlbruleeError(`Timed out after ${timeoutMs}ms waiting for async scrape job ${jobId}.`, {
|
|
597
|
+
status: 0,
|
|
598
|
+
errorName: "request_timeout"
|
|
599
|
+
});
|
|
600
|
+
const status = await this.getScrapeStatus(jobId, { signal: options.signal });
|
|
601
|
+
switch (status.status) {
|
|
602
|
+
case "done": return this.getScrapeResult(jobId, { signal: options.signal });
|
|
603
|
+
case "failed": throw new CrawlbruleeError(status.error ?? `Async scrape job ${jobId} failed.`, {
|
|
604
|
+
status: 0,
|
|
605
|
+
errorName: "job_failed"
|
|
606
|
+
});
|
|
607
|
+
case "pending":
|
|
608
|
+
case "running": break;
|
|
609
|
+
default: throw new CrawlbruleeError(`Async scrape job ${jobId} returned unexpected status '${String(status.status)}'.`, {
|
|
610
|
+
status: 0,
|
|
611
|
+
errorName: "job_failed"
|
|
612
|
+
});
|
|
613
|
+
}
|
|
614
|
+
await sleep(intervalMs, options.signal);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
/**
|
|
618
|
+
* Build (or return a cached) site link-map for a domain. Combines sitemap
|
|
619
|
+
* discovery with the freshest cached homepage scrape when available.
|
|
620
|
+
*/
|
|
621
|
+
map(request, options) {
|
|
622
|
+
return this.http.post("/api/map", request, options);
|
|
623
|
+
}
|
|
624
|
+
/**
|
|
625
|
+
* Return the current billing-cycle usage: total/used/available credits,
|
|
626
|
+
* used quota percentage, max concurrency, and when the cycle resets.
|
|
627
|
+
*/
|
|
628
|
+
usage(options) {
|
|
629
|
+
return this.http.get("/api/usage", options);
|
|
630
|
+
}
|
|
631
|
+
/**
|
|
632
|
+
* Return the organization name and identifying details of the API token
|
|
633
|
+
* used to authenticate this request. Useful for confirming which key is in
|
|
634
|
+
* use before performing destructive operations.
|
|
635
|
+
*/
|
|
636
|
+
whoami(options) {
|
|
637
|
+
return this.http.get("/api/whoami", options);
|
|
638
|
+
}
|
|
521
639
|
};
|
|
640
|
+
/**
|
|
641
|
+
* Defensive read of `process.env[name]`. Guards both the absence of `process`
|
|
642
|
+
* (browser / edge runtimes) and Deno's permission throw on env access without
|
|
643
|
+
* `--allow-env`.
|
|
644
|
+
*/
|
|
522
645
|
function readEnv(name) {
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
646
|
+
try {
|
|
647
|
+
if (typeof process === "undefined" || !process.env) return void 0;
|
|
648
|
+
const v = process.env[name];
|
|
649
|
+
return typeof v === "string" && v.trim().length > 0 ? v.trim() : void 0;
|
|
650
|
+
} catch {
|
|
651
|
+
return;
|
|
652
|
+
}
|
|
530
653
|
}
|
|
531
654
|
function assertNonEmptyJobId(jobId) {
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
});
|
|
537
|
-
}
|
|
655
|
+
if (typeof jobId !== "string" || jobId.trim().length === 0) throw new CrawlbruleeError("jobId must be a non-empty string.", {
|
|
656
|
+
status: 0,
|
|
657
|
+
errorName: null
|
|
658
|
+
});
|
|
538
659
|
}
|
|
539
660
|
function throwIfAborted(signal) {
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
});
|
|
546
|
-
}
|
|
661
|
+
if (signal?.aborted) throw new CrawlbruleeError("Request aborted by caller.", {
|
|
662
|
+
status: 0,
|
|
663
|
+
errorName: "client_closed_request",
|
|
664
|
+
cause: signal.reason
|
|
665
|
+
});
|
|
547
666
|
}
|
|
548
667
|
function sleep(ms, signal) {
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
}
|
|
572
|
-
});
|
|
668
|
+
return new Promise((resolve, reject) => {
|
|
669
|
+
const onAbort = () => {
|
|
670
|
+
clearTimeout(timer);
|
|
671
|
+
reject(new CrawlbruleeError("Request aborted by caller.", {
|
|
672
|
+
status: 0,
|
|
673
|
+
errorName: "client_closed_request",
|
|
674
|
+
cause: signal?.reason
|
|
675
|
+
}));
|
|
676
|
+
};
|
|
677
|
+
const timer = setTimeout(() => {
|
|
678
|
+
signal?.removeEventListener("abort", onAbort);
|
|
679
|
+
resolve();
|
|
680
|
+
}, ms);
|
|
681
|
+
if (signal) {
|
|
682
|
+
if (signal.aborted) {
|
|
683
|
+
clearTimeout(timer);
|
|
684
|
+
onAbort();
|
|
685
|
+
return;
|
|
686
|
+
}
|
|
687
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
688
|
+
}
|
|
689
|
+
});
|
|
573
690
|
}
|
|
574
691
|
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
692
|
+
//#endregion
|
|
693
|
+
//#region src/webhooks.ts
|
|
694
|
+
/**
|
|
695
|
+
* Verification for async scrape completion webhooks.
|
|
696
|
+
*
|
|
697
|
+
* {@link verifyWebhookSignature} validates the signature crawlbrulee attaches to
|
|
698
|
+
* every webhook delivery. It is a standalone, network-free helper built on Web
|
|
699
|
+
* Crypto (`globalThis.crypto.subtle`) so it runs unchanged on Node.js 22+,
|
|
700
|
+
* browsers, Bun, Deno, and edge runtimes — it never touches `node:crypto`.
|
|
701
|
+
*/
|
|
702
|
+
/** HTTP header carrying the primary webhook signature (always present). */
|
|
703
|
+
const WEBHOOK_SIGNATURE_HEADER = "X-Cwbl-Signature";
|
|
704
|
+
/**
|
|
705
|
+
* HTTP header carrying a signature produced with the previous signing secret.
|
|
706
|
+
* Present only during a signing-secret rotation grace window.
|
|
707
|
+
*/
|
|
708
|
+
const WEBHOOK_SIGNATURE_ROTATED_HEADER = "X-Cwbl-Signature-Rotated";
|
|
709
|
+
/** HTTP header carrying the unique event id, useful for delivery de-duplication. */
|
|
710
|
+
const WEBHOOK_EVENT_ID_HEADER = "X-Cwbl-Event-Id";
|
|
711
|
+
/** Default replay-protection window (seconds) applied to the signed timestamp. */
|
|
712
|
+
const DEFAULT_WEBHOOK_TOLERANCE_SECONDS = 300;
|
|
713
|
+
const SIGNATURE_FORMAT = /^t=(\d+),v1=([0-9a-f]{64})$/;
|
|
714
|
+
/**
|
|
715
|
+
* Verify a crawlbrulee webhook signature against the primary and rotated
|
|
716
|
+
* headers.
|
|
717
|
+
*
|
|
718
|
+
* The signing scheme matches the backend:
|
|
719
|
+
* - the signed payload is `` `${t}.${rawBody}` `` where `t` is the unix-seconds
|
|
720
|
+
* integer from the header and `rawBody` is the raw request body,
|
|
721
|
+
* - the signature is `HMAC-SHA256(secret, signedPayload)` as lowercase hex,
|
|
722
|
+
* - the header value is `t=<unix_seconds>,v1=<64_hex>`.
|
|
723
|
+
*
|
|
724
|
+
* The supplied `secret` is tried against the primary header first, then the
|
|
725
|
+
* rotated header (which the API emits during a signing-secret rotation grace
|
|
726
|
+
* window). Whichever matches wins, and the result reports which header it was.
|
|
727
|
+
*
|
|
728
|
+
* This NEVER throws on a verification failure — failures are normal control
|
|
729
|
+
* flow and are returned as `{ verified: false, reason }`.
|
|
730
|
+
*
|
|
731
|
+
* @example
|
|
732
|
+
* ```ts
|
|
733
|
+
* const result = await verifyWebhookSignature({
|
|
734
|
+
* payload: rawBody,
|
|
735
|
+
* headers: req.headers,
|
|
736
|
+
* secret: process.env.CRAWLBRULEE_WEBHOOK_SECRET!,
|
|
737
|
+
* })
|
|
738
|
+
* if (!result.verified) return res.status(400).end()
|
|
739
|
+
* ```
|
|
740
|
+
*/
|
|
581
741
|
async function verifyWebhookSignature(options) {
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
742
|
+
const { payload, headers, secret } = options;
|
|
743
|
+
const toleranceSeconds = options.toleranceSeconds ?? 300;
|
|
744
|
+
const primaryHeader = getHeader(headers, WEBHOOK_SIGNATURE_HEADER);
|
|
745
|
+
const rotatedHeader = getHeader(headers, WEBHOOK_SIGNATURE_ROTATED_HEADER);
|
|
746
|
+
if (primaryHeader === void 0 && rotatedHeader === void 0) return {
|
|
747
|
+
verified: false,
|
|
748
|
+
reason: "missing_signature"
|
|
749
|
+
};
|
|
750
|
+
const nowSeconds = Math.floor(Date.now() / 1e3);
|
|
751
|
+
const body = toBytes(payload);
|
|
752
|
+
const key = await importHmacKey(secret);
|
|
753
|
+
let failure = "malformed_signature";
|
|
754
|
+
for (const source of ["primary", "rotated"]) {
|
|
755
|
+
const raw = source === "primary" ? primaryHeader : rotatedHeader;
|
|
756
|
+
if (raw === void 0) continue;
|
|
757
|
+
const parsed = parseSignatureHeader(raw);
|
|
758
|
+
if (!parsed) continue;
|
|
759
|
+
if (toleranceSeconds && Math.abs(nowSeconds - parsed.timestamp) > toleranceSeconds) {
|
|
760
|
+
failure = mostSpecificFailure(failure, "timestamp_out_of_tolerance");
|
|
761
|
+
continue;
|
|
762
|
+
}
|
|
763
|
+
if (constantTimeEqualHex(await computeSignatureHex(key, parsed.timestamp, body), parsed.signature)) return {
|
|
764
|
+
verified: true,
|
|
765
|
+
signedWith: source
|
|
766
|
+
};
|
|
767
|
+
failure = mostSpecificFailure(failure, "signature_mismatch");
|
|
768
|
+
}
|
|
769
|
+
return {
|
|
770
|
+
verified: false,
|
|
771
|
+
reason: failure
|
|
772
|
+
};
|
|
611
773
|
}
|
|
774
|
+
/**
|
|
775
|
+
* Rank verification failures so the returned reason reflects the most
|
|
776
|
+
* actionable problem encountered across the two headers.
|
|
777
|
+
*/
|
|
612
778
|
function mostSpecificFailure(current, candidate) {
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
779
|
+
const rank = {
|
|
780
|
+
missing_signature: 0,
|
|
781
|
+
malformed_signature: 1,
|
|
782
|
+
timestamp_out_of_tolerance: 2,
|
|
783
|
+
signature_mismatch: 3
|
|
784
|
+
};
|
|
785
|
+
return rank[candidate] > rank[current] ? candidate : current;
|
|
620
786
|
}
|
|
787
|
+
/** Case-insensitive header lookup over `Headers` or a plain object. */
|
|
621
788
|
function getHeader(headers, name) {
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
return value ?? void 0;
|
|
631
|
-
}
|
|
632
|
-
return void 0;
|
|
789
|
+
if (typeof Headers !== "undefined" && headers instanceof Headers) return headers.get(name) ?? void 0;
|
|
790
|
+
const target = name.toLowerCase();
|
|
791
|
+
for (const key of Object.keys(headers)) {
|
|
792
|
+
if (key.toLowerCase() !== target) continue;
|
|
793
|
+
const value = headers[key];
|
|
794
|
+
if (Array.isArray(value)) return value[0];
|
|
795
|
+
return value ?? void 0;
|
|
796
|
+
}
|
|
633
797
|
}
|
|
634
798
|
function parseSignatureHeader(value) {
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
799
|
+
const match = SIGNATURE_FORMAT.exec(value.trim());
|
|
800
|
+
if (!match) return null;
|
|
801
|
+
const timestamp = Number(match[1]);
|
|
802
|
+
if (!Number.isSafeInteger(timestamp)) return null;
|
|
803
|
+
return {
|
|
804
|
+
timestamp,
|
|
805
|
+
signature: match[2]
|
|
806
|
+
};
|
|
640
807
|
}
|
|
641
808
|
function toBytes(payload) {
|
|
642
|
-
|
|
809
|
+
return typeof payload === "string" ? new TextEncoder().encode(payload) : payload;
|
|
643
810
|
}
|
|
644
811
|
function importHmacKey(secret) {
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
false,
|
|
650
|
-
["sign"]
|
|
651
|
-
);
|
|
812
|
+
return getSubtle().importKey("raw", new TextEncoder().encode(secret), {
|
|
813
|
+
name: "HMAC",
|
|
814
|
+
hash: "SHA-256"
|
|
815
|
+
}, false, ["sign"]);
|
|
652
816
|
}
|
|
653
817
|
async function computeSignatureHex(key, timestamp, body) {
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
818
|
+
const prefix = new TextEncoder().encode(`${timestamp}.`);
|
|
819
|
+
const message = new Uint8Array(prefix.length + body.length);
|
|
820
|
+
message.set(prefix, 0);
|
|
821
|
+
message.set(body, prefix.length);
|
|
822
|
+
const digest = await getSubtle().sign("HMAC", key, message);
|
|
823
|
+
return toHex(new Uint8Array(digest));
|
|
660
824
|
}
|
|
661
825
|
function toHex(bytes) {
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
}
|
|
666
|
-
return hex;
|
|
826
|
+
let hex = "";
|
|
827
|
+
for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
|
|
828
|
+
return hex;
|
|
667
829
|
}
|
|
830
|
+
/**
|
|
831
|
+
* Length-checked, constant-time comparison of two lowercase hex strings. Folds
|
|
832
|
+
* every byte into an accumulator with XOR — never early-returns on the first
|
|
833
|
+
* mismatch — so timing does not leak how much of the signature matched.
|
|
834
|
+
*/
|
|
668
835
|
function constantTimeEqualHex(a, b) {
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
}
|
|
674
|
-
return diff === 0;
|
|
836
|
+
if (a.length !== b.length) return false;
|
|
837
|
+
let diff = 0;
|
|
838
|
+
for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
839
|
+
return diff === 0;
|
|
675
840
|
}
|
|
676
841
|
function getSubtle() {
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
"Web Crypto (globalThis.crypto.subtle) is not available in this runtime. crawlbrulee webhook verification requires Node.js 20+, Bun, Deno, or a modern browser/edge runtime."
|
|
681
|
-
);
|
|
682
|
-
}
|
|
683
|
-
return subtle;
|
|
842
|
+
const subtle = globalThis.crypto?.subtle;
|
|
843
|
+
if (!subtle) throw new Error("Web Crypto (globalThis.crypto.subtle) is not available in this runtime. crawlbrulee webhook verification requires Node.js 22+, Bun, Deno, or a modern browser/edge runtime.");
|
|
844
|
+
return subtle;
|
|
684
845
|
}
|
|
685
846
|
|
|
847
|
+
//#endregion
|
|
686
848
|
export { AuthenticationError, Crawlbrulee, CrawlbruleeError, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_WEBHOOK_TOLERANCE_SECONDS, ENV_API_KEY, NotFoundError, RateLimitError, TransportError, UsageAllocationError, ValidationError, WEBHOOK_EVENT_ID_HEADER, WEBHOOK_SIGNATURE_HEADER, WEBHOOK_SIGNATURE_ROTATED_HEADER, isCrawlbruleeError, verifyWebhookSignature };
|
|
687
|
-
//# sourceMappingURL=index.js.map
|
|
688
849
|
//# sourceMappingURL=index.js.map
|