@crawlbrulee/sdk 0.6.0 → 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +156 -76
- package/dist/index.cjs +778 -617
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +654 -668
- package/dist/index.d.ts +654 -668
- package/dist/index.js +777 -616
- package/dist/index.js.map +1 -1
- package/package.json +6 -6
package/dist/index.cjs
CHANGED
|
@@ -1,690 +1,852 @@
|
|
|
1
|
-
'
|
|
1
|
+
Object.defineProperty(exports, Symbol.toStringTag, { value: 'Module' });
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
3
|
+
//#region src/config.ts
|
|
4
|
+
/**
|
|
5
|
+
* Production base URL of the crawlbrulee API. Used by default when the caller
|
|
6
|
+
* doesn't pass a `baseUrl` to {@link Crawlbrulee}. Local development and
|
|
7
|
+
* staging callers point at their own host via that option.
|
|
8
|
+
*/
|
|
9
|
+
const DEFAULT_BASE_URL = "https://api.crawlbrulee.com";
|
|
10
|
+
/** Default request timeout when the caller doesn't specify one (0 disables the timeout). */
|
|
11
|
+
const DEFAULT_REQUEST_TIMEOUT_MS = 0;
|
|
12
|
+
/** Environment variable read by `Crawlbrulee.fromEnv()` to source the API key. */
|
|
13
|
+
const ENV_API_KEY = "CRAWLBRULEE_API_KEY";
|
|
14
|
+
/** Identifies the SDK in the `User-Agent` header. Kept in one place for easy bumping. */
|
|
15
|
+
const USER_AGENT = "@crawlbrulee/sdk/0.7.0 (node)";
|
|
8
16
|
|
|
9
|
-
|
|
17
|
+
//#endregion
|
|
18
|
+
//#region src/errors.ts
|
|
19
|
+
/**
|
|
20
|
+
* Base error class for every failure raised by the SDK.
|
|
21
|
+
*
|
|
22
|
+
* Two kinds of failures end up here:
|
|
23
|
+
*
|
|
24
|
+
* 1. **API errors** — the server returned a non-2xx response with a well-formed
|
|
25
|
+
* JSON body. In that case `status`, `errorName` and (sometimes) `details`
|
|
26
|
+
* are populated.
|
|
27
|
+
* 2. **Transport errors** — the request never produced a structured response
|
|
28
|
+
* (network failure, abort, timeout, non-JSON body, etc.). In that case
|
|
29
|
+
* `status` may be `0` and `errorName` is one of the synthetic transport
|
|
30
|
+
* names (`request_timeout`, `client_closed_request`) or `null`.
|
|
31
|
+
*
|
|
32
|
+
* Typed subclasses are exported for the most common cases. To branch on more
|
|
33
|
+
* specific server-side errors, switch on `err.errorName` or use the
|
|
34
|
+
* {@link isCrawlbruleeError} helper.
|
|
35
|
+
*/
|
|
10
36
|
var CrawlbruleeError = class extends Error {
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
37
|
+
/** HTTP status code; `0` for transport-level failures with no response. */
|
|
38
|
+
status;
|
|
39
|
+
/** The `name` field from the API error body, or `null` for transport errors. */
|
|
40
|
+
errorName;
|
|
41
|
+
/** Structured detail block from the API error body, if any. */
|
|
42
|
+
details;
|
|
43
|
+
/** The original parsed error body, when one was received. */
|
|
44
|
+
response;
|
|
45
|
+
constructor(message, options) {
|
|
46
|
+
super(message, options.cause !== void 0 ? { cause: options.cause } : void 0);
|
|
47
|
+
this.name = "CrawlbruleeError";
|
|
48
|
+
this.status = options.status;
|
|
49
|
+
this.errorName = options.errorName;
|
|
50
|
+
this.details = options.details;
|
|
51
|
+
this.response = options.response;
|
|
52
|
+
}
|
|
27
53
|
};
|
|
54
|
+
/** Raised for 401 / 403 responses (missing, invalid, or unauthorized API key). */
|
|
28
55
|
var AuthenticationError = class extends CrawlbruleeError {
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
56
|
+
constructor(message, options) {
|
|
57
|
+
super(message, options);
|
|
58
|
+
this.name = "AuthenticationError";
|
|
59
|
+
}
|
|
33
60
|
};
|
|
61
|
+
/**
|
|
62
|
+
* Raised for HTTP 429 responses. When the server included a `retry_after_ms`
|
|
63
|
+
* hint in `details` it is surfaced directly on the instance.
|
|
64
|
+
*
|
|
65
|
+
* `errorName` is always the literal `'too_many_requests'` — the SDK normalizes
|
|
66
|
+
* this even when the server returns a 429 with a different `name` field
|
|
67
|
+
* (e.g. a CDN coalescing upstream rate limiting). The original body is still
|
|
68
|
+
* available on `response`.
|
|
69
|
+
*/
|
|
34
70
|
var RateLimitError = class extends CrawlbruleeError {
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
71
|
+
errorName;
|
|
72
|
+
/** Suggested delay (ms) before retrying, when the server provided one. */
|
|
73
|
+
retryAfterMs;
|
|
74
|
+
/** Which rate limit was tripped (e.g. `org`, `ip`), when provided. */
|
|
75
|
+
limitedBy;
|
|
76
|
+
constructor(message, options) {
|
|
77
|
+
super(message, {
|
|
78
|
+
...options,
|
|
79
|
+
errorName: "too_many_requests",
|
|
80
|
+
details: options.details
|
|
81
|
+
});
|
|
82
|
+
this.name = "RateLimitError";
|
|
83
|
+
this.errorName = "too_many_requests";
|
|
84
|
+
this.retryAfterMs = options.details?.retry_after_ms;
|
|
85
|
+
this.limitedBy = options.details?.limited_by;
|
|
86
|
+
}
|
|
47
87
|
};
|
|
88
|
+
/**
|
|
89
|
+
* Raised when the API rejects a request because the org's plan limits would
|
|
90
|
+
* be exceeded (credit limit, concurrency cap, overage hard cap, etc.).
|
|
91
|
+
*
|
|
92
|
+
* `errorName` is always the literal `'usage_allocation_error'`.
|
|
93
|
+
*/
|
|
48
94
|
var UsageAllocationError = class extends CrawlbruleeError {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
95
|
+
errorName;
|
|
96
|
+
/** Specific reason the allocation was denied. */
|
|
97
|
+
reason;
|
|
98
|
+
/** Current usage / limit snapshot at the time of the rejection. */
|
|
99
|
+
usage;
|
|
100
|
+
constructor(message, options) {
|
|
101
|
+
super(message, {
|
|
102
|
+
...options,
|
|
103
|
+
errorName: "usage_allocation_error"
|
|
104
|
+
});
|
|
105
|
+
this.name = "UsageAllocationError";
|
|
106
|
+
this.errorName = "usage_allocation_error";
|
|
107
|
+
this.reason = options.details.reason;
|
|
108
|
+
this.usage = options.details.details;
|
|
109
|
+
}
|
|
61
110
|
};
|
|
111
|
+
/** Raised for 4xx responses caused by an invalid request shape or arguments. */
|
|
62
112
|
var ValidationError = class extends CrawlbruleeError {
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
113
|
+
constructor(message, options) {
|
|
114
|
+
super(message, options);
|
|
115
|
+
this.name = "ValidationError";
|
|
116
|
+
}
|
|
67
117
|
};
|
|
118
|
+
/** Raised for 404 responses (e.g. unknown async job ID). */
|
|
68
119
|
var NotFoundError = class extends CrawlbruleeError {
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
120
|
+
constructor(message, options) {
|
|
121
|
+
super(message, options);
|
|
122
|
+
this.name = "NotFoundError";
|
|
123
|
+
}
|
|
73
124
|
};
|
|
125
|
+
/**
|
|
126
|
+
* Raised when a request cannot be sent or no structured response is parsed.
|
|
127
|
+
*
|
|
128
|
+
* The `errorName` discriminates the cause:
|
|
129
|
+
* - `'request_timeout'` — the per-request timeout fired.
|
|
130
|
+
* - `'client_closed_request'` — the caller's `AbortSignal` fired.
|
|
131
|
+
* - `null` — generic transport failure (network error, non-JSON body, etc.).
|
|
132
|
+
*/
|
|
74
133
|
var TransportError = class extends CrawlbruleeError {
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
134
|
+
constructor(message, options = {}) {
|
|
135
|
+
super(message, {
|
|
136
|
+
status: options.status ?? 0,
|
|
137
|
+
errorName: options.errorName ?? null,
|
|
138
|
+
cause: options.cause
|
|
139
|
+
});
|
|
140
|
+
this.name = "TransportError";
|
|
141
|
+
}
|
|
83
142
|
};
|
|
143
|
+
/** Narrow `unknown` to the SDK's base error type. */
|
|
84
144
|
function isCrawlbruleeError(err) {
|
|
85
|
-
|
|
145
|
+
return err instanceof CrawlbruleeError;
|
|
86
146
|
}
|
|
147
|
+
/**
|
|
148
|
+
* Map an API error body + HTTP status to the most specific error class.
|
|
149
|
+
*
|
|
150
|
+
* Dispatch is **name-first**: the body's `name` field is the most reliable
|
|
151
|
+
* signal of what went wrong. Status code is used only as a fallback when the
|
|
152
|
+
* name is unrecognized (e.g. a CDN-synthesized error). This avoids
|
|
153
|
+
* miscategorizing things like a 403 with `name: 'not_found'` as an auth error.
|
|
154
|
+
*
|
|
155
|
+
* Internal — used by the HTTP layer.
|
|
156
|
+
*/
|
|
87
157
|
function createApiError(body, status) {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
158
|
+
const { name, message, details } = body;
|
|
159
|
+
const response = body;
|
|
160
|
+
switch (name) {
|
|
161
|
+
case "too_many_requests": return new RateLimitError(message, {
|
|
162
|
+
status,
|
|
163
|
+
details: details?.error_name === "too_many_requests" ? details : void 0,
|
|
164
|
+
response
|
|
165
|
+
});
|
|
166
|
+
case "usage_allocation_error": {
|
|
167
|
+
const usageDetails = details?.error_name === "usage_allocation_error" ? details : {
|
|
168
|
+
error_name: "usage_allocation_error",
|
|
169
|
+
reason: "internal_error"
|
|
170
|
+
};
|
|
171
|
+
return new UsageAllocationError(message, {
|
|
172
|
+
status,
|
|
173
|
+
details: usageDetails,
|
|
174
|
+
response
|
|
175
|
+
});
|
|
176
|
+
}
|
|
177
|
+
case "invalid_credentials":
|
|
178
|
+
case "access_denied": return new AuthenticationError(message, {
|
|
179
|
+
status,
|
|
180
|
+
errorName: name,
|
|
181
|
+
response
|
|
182
|
+
});
|
|
183
|
+
case "not_found": return new NotFoundError(message, {
|
|
184
|
+
status,
|
|
185
|
+
errorName: name,
|
|
186
|
+
response
|
|
187
|
+
});
|
|
188
|
+
case "validation_error":
|
|
189
|
+
case "invalid_url":
|
|
190
|
+
case "url_too_long":
|
|
191
|
+
case "unsupported_url_schema":
|
|
192
|
+
case "url_credentials_not_supported":
|
|
193
|
+
case "blocked_url":
|
|
194
|
+
case "unsupported_content": return new ValidationError(message, {
|
|
195
|
+
status,
|
|
196
|
+
errorName: name,
|
|
197
|
+
response
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
if (status === 429) return new RateLimitError(message, {
|
|
201
|
+
status,
|
|
202
|
+
response
|
|
203
|
+
});
|
|
204
|
+
if (status === 401 || status === 403) return new AuthenticationError(message, {
|
|
205
|
+
status,
|
|
206
|
+
errorName: name,
|
|
207
|
+
response
|
|
208
|
+
});
|
|
209
|
+
if (status === 404) return new NotFoundError(message, {
|
|
210
|
+
status,
|
|
211
|
+
errorName: name,
|
|
212
|
+
response
|
|
213
|
+
});
|
|
214
|
+
return new CrawlbruleeError(message, {
|
|
215
|
+
status,
|
|
216
|
+
errorName: name,
|
|
217
|
+
details,
|
|
218
|
+
response
|
|
219
|
+
});
|
|
125
220
|
}
|
|
126
221
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
222
|
+
//#endregion
|
|
223
|
+
//#region src/instrumentation.ts
|
|
224
|
+
/**
|
|
225
|
+
* Centralized factory for the low-level dependencies the SDK injects into its
|
|
226
|
+
* HTTP layer. Production code resolves these to the runtime's global `fetch`
|
|
227
|
+
* and the burned-in production base URL; tests stub this module to swap in
|
|
228
|
+
* mocks and alternate hosts.
|
|
229
|
+
*
|
|
230
|
+
* This is internal — it is not exported from the package's public entry. Tests
|
|
231
|
+
* import it from `src/instrumentation.js` directly and use `vi.spyOn` to
|
|
232
|
+
* substitute behavior.
|
|
233
|
+
*/
|
|
234
|
+
const CwblInstrumentation = {
|
|
235
|
+
/**
|
|
236
|
+
* Resolve the `fetch` implementation the SDK should use. Throws a
|
|
237
|
+
* {@link CrawlbruleeError} if the runtime does not expose a global `fetch`.
|
|
238
|
+
*/
|
|
239
|
+
getFetch() {
|
|
240
|
+
const g = globalThis;
|
|
241
|
+
if (typeof g.fetch !== "function") throw new CrawlbruleeError("No global fetch is available in this runtime. crawlbrulee requires Node.js 22+, Bun, Deno, or a modern browser/edge runtime.", {
|
|
242
|
+
status: 0,
|
|
243
|
+
errorName: null
|
|
244
|
+
});
|
|
245
|
+
return g.fetch.bind(globalThis);
|
|
246
|
+
},
|
|
247
|
+
/**
|
|
248
|
+
* Resolve the base URL the SDK should target. Returns the production host by
|
|
249
|
+
* default; tests stub this to point at a mock origin.
|
|
250
|
+
*/
|
|
251
|
+
getBaseUrl() {
|
|
252
|
+
return DEFAULT_BASE_URL;
|
|
253
|
+
}
|
|
150
254
|
};
|
|
151
255
|
|
|
152
|
-
|
|
256
|
+
//#endregion
|
|
257
|
+
//#region src/http.ts
|
|
258
|
+
/**
|
|
259
|
+
* Minimal `fetch`-based HTTP layer used by {@link Crawlbrulee}. Handles:
|
|
260
|
+
*
|
|
261
|
+
* - URL composition (joining `baseUrl` and path safely).
|
|
262
|
+
* - JSON serialization and parsing.
|
|
263
|
+
* - The `Authorization: Bearer …` header.
|
|
264
|
+
* - Composing the caller's `AbortSignal` with an internal timeout signal. The
|
|
265
|
+
* timeout covers the WHOLE request, including the response body read — not
|
|
266
|
+
* just the time-to-headers.
|
|
267
|
+
* - Mapping non-2xx responses to typed `CrawlbruleeError` subclasses via
|
|
268
|
+
* {@link createApiError}.
|
|
269
|
+
*
|
|
270
|
+
* The base URL and `fetch` implementation are sourced from
|
|
271
|
+
* {@link CwblInstrumentation} at construction time so tests can stub the
|
|
272
|
+
* module.
|
|
273
|
+
*/
|
|
153
274
|
var HttpClient = class {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
275
|
+
baseUrl;
|
|
276
|
+
apiKey;
|
|
277
|
+
fetch;
|
|
278
|
+
timeoutMs;
|
|
279
|
+
constructor(options) {
|
|
280
|
+
this.baseUrl = stripTrailingSlash(options.baseUrl ?? CwblInstrumentation.getBaseUrl());
|
|
281
|
+
this.apiKey = options.apiKey;
|
|
282
|
+
this.fetch = CwblInstrumentation.getFetch();
|
|
283
|
+
this.timeoutMs = options.timeoutMs ?? 0;
|
|
284
|
+
}
|
|
285
|
+
/** Send a `GET` request and parse the response as `T`. */
|
|
286
|
+
get(path, options) {
|
|
287
|
+
return this.send({
|
|
288
|
+
method: "GET",
|
|
289
|
+
path,
|
|
290
|
+
...options
|
|
291
|
+
});
|
|
292
|
+
}
|
|
293
|
+
/** Send a `POST` request with a JSON body and parse the response as `T`. */
|
|
294
|
+
post(path, body, options) {
|
|
295
|
+
return this.send({
|
|
296
|
+
method: "POST",
|
|
297
|
+
path,
|
|
298
|
+
body,
|
|
299
|
+
...options
|
|
300
|
+
});
|
|
301
|
+
}
|
|
302
|
+
async send(args) {
|
|
303
|
+
const url = this.buildUrl(args.path);
|
|
304
|
+
const headers = this.buildHeaders(args);
|
|
305
|
+
const body = args.body === void 0 ? void 0 : JSON.stringify(args.body);
|
|
306
|
+
const composed = this.composeSignal(args.signal, args.timeoutMs);
|
|
307
|
+
try {
|
|
308
|
+
let res;
|
|
309
|
+
try {
|
|
310
|
+
res = await this.fetch(url, {
|
|
311
|
+
method: args.method,
|
|
312
|
+
headers,
|
|
313
|
+
body,
|
|
314
|
+
signal: composed.signal
|
|
315
|
+
});
|
|
316
|
+
} catch (cause) {
|
|
317
|
+
throw abortOrNetworkError(cause, composed.timedOut(), args.timeoutMs ?? this.timeoutMs);
|
|
318
|
+
}
|
|
319
|
+
let text;
|
|
320
|
+
try {
|
|
321
|
+
text = await res.text();
|
|
322
|
+
} catch (cause) {
|
|
323
|
+
if (isAbortError(cause)) throw abortOrNetworkError(cause, composed.timedOut(), args.timeoutMs ?? this.timeoutMs);
|
|
324
|
+
throw new TransportError(`Failed to read response body (status ${res.status}).`, {
|
|
325
|
+
status: res.status,
|
|
326
|
+
cause
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
const parsed = parseJsonOrThrow(text, res.status);
|
|
330
|
+
if (!res.ok) throw toApiError(parsed, res.status, text);
|
|
331
|
+
return parsed;
|
|
332
|
+
} finally {
|
|
333
|
+
composed.cleanup();
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
buildUrl(path) {
|
|
337
|
+
if (!path.startsWith("/")) throw new TypeError(`crawlbrulee SDK: path must start with '/' (received '${path}')`);
|
|
338
|
+
return `${this.baseUrl}${path}`;
|
|
339
|
+
}
|
|
340
|
+
buildHeaders(args) {
|
|
341
|
+
const headers = {
|
|
342
|
+
accept: "application/json",
|
|
343
|
+
"user-agent": USER_AGENT,
|
|
344
|
+
authorization: `Bearer ${this.apiKey}`
|
|
345
|
+
};
|
|
346
|
+
if (args.body !== void 0) headers["content-type"] = "application/json";
|
|
347
|
+
return headers;
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* Build a single `AbortSignal` that fires when either the caller-supplied
|
|
351
|
+
* signal aborts OR the per-request timeout elapses. The returned `cleanup`
|
|
352
|
+
* callback MUST be invoked on every exit path so we don't leak timers or
|
|
353
|
+
* dead listeners on long-lived caller signals.
|
|
354
|
+
*/
|
|
355
|
+
composeSignal(callerSignal, overrideTimeoutMs) {
|
|
356
|
+
const timeoutMs = overrideTimeoutMs ?? this.timeoutMs;
|
|
357
|
+
const hasTimeout = Number.isFinite(timeoutMs) && timeoutMs > 0;
|
|
358
|
+
if (!hasTimeout && !callerSignal) return {
|
|
359
|
+
signal: void 0,
|
|
360
|
+
timedOut: () => false,
|
|
361
|
+
cleanup: () => {}
|
|
362
|
+
};
|
|
363
|
+
if (!hasTimeout) return {
|
|
364
|
+
signal: callerSignal,
|
|
365
|
+
timedOut: () => false,
|
|
366
|
+
cleanup: () => {}
|
|
367
|
+
};
|
|
368
|
+
const controller = new AbortController();
|
|
369
|
+
let didTimeout = false;
|
|
370
|
+
const timer = setTimeout(() => {
|
|
371
|
+
didTimeout = true;
|
|
372
|
+
controller.abort(/* @__PURE__ */ new Error("request_timeout"));
|
|
373
|
+
}, timeoutMs);
|
|
374
|
+
let onCallerAbort;
|
|
375
|
+
if (callerSignal) if (callerSignal.aborted) {
|
|
376
|
+
clearTimeout(timer);
|
|
377
|
+
controller.abort(callerSignal.reason);
|
|
378
|
+
} else {
|
|
379
|
+
onCallerAbort = () => {
|
|
380
|
+
clearTimeout(timer);
|
|
381
|
+
controller.abort(callerSignal.reason);
|
|
382
|
+
};
|
|
383
|
+
callerSignal.addEventListener("abort", onCallerAbort, { once: true });
|
|
384
|
+
}
|
|
385
|
+
const cleanup = () => {
|
|
386
|
+
clearTimeout(timer);
|
|
387
|
+
if (onCallerAbort && callerSignal) callerSignal.removeEventListener("abort", onCallerAbort);
|
|
388
|
+
};
|
|
389
|
+
return {
|
|
390
|
+
signal: controller.signal,
|
|
391
|
+
timedOut: () => didTimeout,
|
|
392
|
+
cleanup
|
|
393
|
+
};
|
|
394
|
+
}
|
|
267
395
|
};
|
|
268
396
|
function stripTrailingSlash(url) {
|
|
269
|
-
|
|
397
|
+
return url.replace(/\/+$/, "");
|
|
270
398
|
}
|
|
271
399
|
function isAbortError(err) {
|
|
272
|
-
|
|
400
|
+
return err instanceof Error && err.name === "AbortError";
|
|
273
401
|
}
|
|
274
402
|
function abortOrNetworkError(cause, timedOut, timeoutMs) {
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
}
|
|
287
|
-
return new TransportError(formatNetworkErrorMessage(cause), { cause });
|
|
403
|
+
if (isAbortError(cause)) {
|
|
404
|
+
if (timedOut) return new TransportError(`Request timed out after ${timeoutMs}ms.`, {
|
|
405
|
+
errorName: "request_timeout",
|
|
406
|
+
cause
|
|
407
|
+
});
|
|
408
|
+
return new TransportError("Request aborted by caller.", {
|
|
409
|
+
errorName: "client_closed_request",
|
|
410
|
+
cause
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
return new TransportError(formatNetworkErrorMessage(cause), { cause });
|
|
288
414
|
}
|
|
289
415
|
function formatNetworkErrorMessage(cause) {
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
}
|
|
293
|
-
return "Network error: unknown failure while sending the request.";
|
|
416
|
+
if (cause instanceof Error) return `Network error: ${cause.message}`;
|
|
417
|
+
return "Network error: unknown failure while sending the request.";
|
|
294
418
|
}
|
|
295
419
|
function parseJsonOrThrow(text, status) {
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
}
|
|
420
|
+
if (text === "") return {};
|
|
421
|
+
try {
|
|
422
|
+
return JSON.parse(text);
|
|
423
|
+
} catch (cause) {
|
|
424
|
+
throw new TransportError(`Unexpected non-JSON response (status ${status}): ${text.length > 200 ? `${text.slice(0, 200)}…` : text}`, {
|
|
425
|
+
status,
|
|
426
|
+
cause
|
|
427
|
+
});
|
|
428
|
+
}
|
|
306
429
|
}
|
|
307
430
|
function toApiError(parsed, status, rawText) {
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
}
|
|
311
|
-
const preview = rawText.length > 200 ? `${rawText.slice(0, 200)}\u2026` : rawText;
|
|
312
|
-
return new TransportError(`HTTP ${status}: ${preview || "(empty body)"}`, { status });
|
|
431
|
+
if (isApiErrorResponse(parsed)) return createApiError(parsed, status);
|
|
432
|
+
return new TransportError(`HTTP ${status}: ${(rawText.length > 200 ? `${rawText.slice(0, 200)}…` : rawText) || "(empty body)"}`, { status });
|
|
313
433
|
}
|
|
314
434
|
function isApiErrorResponse(value) {
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
435
|
+
if (value === null || typeof value !== "object") return false;
|
|
436
|
+
const v = value;
|
|
437
|
+
return typeof v.name === "string" && typeof v.message === "string";
|
|
318
438
|
}
|
|
319
439
|
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
return this.http.get("/api/whoami", options);
|
|
522
|
-
}
|
|
440
|
+
//#endregion
|
|
441
|
+
//#region src/client.ts
|
|
442
|
+
/**
|
|
443
|
+
* Official client for the crawlbrulee API.
|
|
444
|
+
*
|
|
445
|
+
* @example
|
|
446
|
+
* ```ts
|
|
447
|
+
* import { Crawlbrulee } from '@crawlbrulee/sdk'
|
|
448
|
+
*
|
|
449
|
+
* const crawlbrulee = new Crawlbrulee({ apiKey: 'cwbl_…' })
|
|
450
|
+
* // or read CRAWLBRULEE_API_KEY from the environment:
|
|
451
|
+
* const crawlbrulee = Crawlbrulee.fromEnv()
|
|
452
|
+
*
|
|
453
|
+
* const page = await crawlbrulee.scrape({
|
|
454
|
+
* url: 'https://example.com',
|
|
455
|
+
* extract: { markdown: true, links: true },
|
|
456
|
+
* })
|
|
457
|
+
* console.log(page.markdown)
|
|
458
|
+
* ```
|
|
459
|
+
*/
|
|
460
|
+
var Crawlbrulee = class Crawlbrulee {
|
|
461
|
+
/** Resolved base URL — trailing slash already stripped. */
|
|
462
|
+
baseUrl;
|
|
463
|
+
/** Underlying HTTP layer. Exposed for advanced use cases (custom endpoints). */
|
|
464
|
+
http;
|
|
465
|
+
constructor(options) {
|
|
466
|
+
const apiKey = options.apiKey?.trim();
|
|
467
|
+
if (!apiKey) throw new CrawlbruleeError(`Missing API key. Pass { apiKey } to Crawlbrulee or call Crawlbrulee.fromEnv() to read ${ENV_API_KEY}.`, {
|
|
468
|
+
status: 0,
|
|
469
|
+
errorName: null
|
|
470
|
+
});
|
|
471
|
+
this.http = new HttpClient({
|
|
472
|
+
apiKey,
|
|
473
|
+
baseUrl: options.baseUrl,
|
|
474
|
+
timeoutMs: options.timeoutMs
|
|
475
|
+
});
|
|
476
|
+
this.baseUrl = this.http.baseUrl;
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* Build a {@link Crawlbrulee} reading the API key from
|
|
480
|
+
* `process.env.CRAWLBRULEE_API_KEY`. Throws if the variable is unset, empty,
|
|
481
|
+
* or whitespace.
|
|
482
|
+
*
|
|
483
|
+
* Any other constructor option can be passed via `overrides`.
|
|
484
|
+
*
|
|
485
|
+
* @example
|
|
486
|
+
* ```ts
|
|
487
|
+
* const crawlbrulee = Crawlbrulee.fromEnv()
|
|
488
|
+
* const crawlbrulee = Crawlbrulee.fromEnv({ timeoutMs: 30_000 })
|
|
489
|
+
* ```
|
|
490
|
+
*/
|
|
491
|
+
static fromEnv(overrides = {}) {
|
|
492
|
+
const apiKey = readEnv(ENV_API_KEY);
|
|
493
|
+
if (!apiKey) throw new CrawlbruleeError(`${ENV_API_KEY} is not set. Export it in your shell, or pass apiKey to new Crawlbrulee({ apiKey }).`, {
|
|
494
|
+
status: 0,
|
|
495
|
+
errorName: null
|
|
496
|
+
});
|
|
497
|
+
return new Crawlbrulee({
|
|
498
|
+
...overrides,
|
|
499
|
+
apiKey
|
|
500
|
+
});
|
|
501
|
+
}
|
|
502
|
+
/**
|
|
503
|
+
* Scrape a URL synchronously and return the extracted content.
|
|
504
|
+
*
|
|
505
|
+
* The request blocks until the scrape is finished. For long-running jobs
|
|
506
|
+
* (heavy JS rendering, screenshots of long pages) prefer
|
|
507
|
+
* {@link Crawlbrulee.scrapeAsync} so the connection isn't held open.
|
|
508
|
+
*
|
|
509
|
+
* @param request — body for `POST /api/scrape`.
|
|
510
|
+
* @param options — per-call timeout and abort signal.
|
|
511
|
+
*/
|
|
512
|
+
scrape(request, options) {
|
|
513
|
+
return this.http.post("/api/scrape", request, options);
|
|
514
|
+
}
|
|
515
|
+
/**
|
|
516
|
+
* Submit an asynchronous scrape job and return its `job_id`. Poll the job
|
|
517
|
+
* with {@link Crawlbrulee.getScrapeStatus} or wait for completion with
|
|
518
|
+
* {@link Crawlbrulee.waitForScrape}.
|
|
519
|
+
*
|
|
520
|
+
* Pass an optional `webhook` to have the API deliver a signed
|
|
521
|
+
* `scrape.complete` `POST` to your endpoint when the job finishes (see
|
|
522
|
+
* {@link AsyncScrapeWebhook}). This field is async-only.
|
|
523
|
+
*/
|
|
524
|
+
scrapeAsync(request, options) {
|
|
525
|
+
return this.http.post("/api/scrape/async", request, options);
|
|
526
|
+
}
|
|
527
|
+
/** Look up the current status of an async scrape job. */
|
|
528
|
+
getScrapeStatus(jobId, options) {
|
|
529
|
+
assertNonEmptyJobId(jobId);
|
|
530
|
+
return this.http.get(`/api/scrape/status/${encodeURIComponent(jobId)}`, options);
|
|
531
|
+
}
|
|
532
|
+
/**
|
|
533
|
+
* Fetch the result of a completed async scrape job. Throws if the job is
|
|
534
|
+
* still pending/running — call {@link Crawlbrulee.getScrapeStatus}
|
|
535
|
+
* first, or use {@link Crawlbrulee.waitForScrape} to poll-then-fetch.
|
|
536
|
+
*/
|
|
537
|
+
getScrapeResult(jobId, options) {
|
|
538
|
+
assertNonEmptyJobId(jobId);
|
|
539
|
+
return this.http.get(`/api/scrape/result/${encodeURIComponent(jobId)}`, options);
|
|
540
|
+
}
|
|
541
|
+
/**
|
|
542
|
+
* Fetch the scrape result referenced by a `scrape.complete` webhook body.
|
|
543
|
+
*
|
|
544
|
+
* Always verify the webhook signature with `verifyWebhookSignature` before
|
|
545
|
+
* acting on it; this method trusts the parsed body it is handed.
|
|
546
|
+
*
|
|
547
|
+
* Behavior by `data.status`:
|
|
548
|
+
* - `success` — delegates to {@link Crawlbrulee.getScrapeResult} for the
|
|
549
|
+
* webhook's `job_id` and returns the parsed result.
|
|
550
|
+
* - `failed` — throws a {@link CrawlbruleeError} carrying `data.error`
|
|
551
|
+
* (`errorName: 'job_failed'`); there is no result to fetch.
|
|
552
|
+
* - `cancelled` — throws a {@link CrawlbruleeError}
|
|
553
|
+
* (`errorName: 'client_closed_request'`).
|
|
554
|
+
*
|
|
555
|
+
* A non-`scrape.complete` envelope throws a {@link CrawlbruleeError}
|
|
556
|
+
* defensively. Any HTTP error from the underlying fetch propagates as the
|
|
557
|
+
* usual typed `CrawlbruleeError` subclass.
|
|
558
|
+
*/
|
|
559
|
+
async fetchScrapeResultFromWebhook(webhook, options) {
|
|
560
|
+
if (webhook?.event !== "scrape.complete") throw new CrawlbruleeError(`Expected a 'scrape.complete' webhook but received '${String(webhook?.event)}'.`, {
|
|
561
|
+
status: 0,
|
|
562
|
+
errorName: "validation_error"
|
|
563
|
+
});
|
|
564
|
+
const { job_id: jobId, status, error } = webhook.data;
|
|
565
|
+
switch (status) {
|
|
566
|
+
case "success": return this.getScrapeResult(jobId, options);
|
|
567
|
+
case "failed": throw new CrawlbruleeError(error ?? `Async scrape job ${jobId} failed.`, {
|
|
568
|
+
status: 0,
|
|
569
|
+
errorName: "job_failed"
|
|
570
|
+
});
|
|
571
|
+
case "cancelled": throw new CrawlbruleeError(`Async scrape job ${jobId} was cancelled.`, {
|
|
572
|
+
status: 0,
|
|
573
|
+
errorName: "client_closed_request"
|
|
574
|
+
});
|
|
575
|
+
default: throw new CrawlbruleeError(`Async scrape webhook for job ${jobId} carried an unexpected status '${String(status)}'.`, {
|
|
576
|
+
status: 0,
|
|
577
|
+
errorName: "validation_error"
|
|
578
|
+
});
|
|
579
|
+
}
|
|
580
|
+
}
|
|
581
|
+
/**
|
|
582
|
+
* Poll an async scrape job until it reaches a terminal state, then return
|
|
583
|
+
* the scrape result.
|
|
584
|
+
*
|
|
585
|
+
* Throws a {@link CrawlbruleeError} when:
|
|
586
|
+
* - the job ends in `failed` (`errorName: 'job_failed'`),
|
|
587
|
+
* - the server reports an unexpected status (`errorName: 'job_failed'`),
|
|
588
|
+
* - the overall wait exceeds `timeoutMs` (`errorName: 'request_timeout'`),
|
|
589
|
+
* - the caller's `signal` aborts (`errorName: 'client_closed_request'`).
|
|
590
|
+
*/
|
|
591
|
+
async waitForScrape(jobId, options = {}) {
|
|
592
|
+
assertNonEmptyJobId(jobId);
|
|
593
|
+
const intervalMs = options.intervalMs ?? 2e3;
|
|
594
|
+
const timeoutMs = options.timeoutMs ?? 3e5;
|
|
595
|
+
const deadline = timeoutMs > 0 ? Date.now() + timeoutMs : Number.POSITIVE_INFINITY;
|
|
596
|
+
while (true) {
|
|
597
|
+
throwIfAborted(options.signal);
|
|
598
|
+
if (Date.now() >= deadline) throw new CrawlbruleeError(`Timed out after ${timeoutMs}ms waiting for async scrape job ${jobId}.`, {
|
|
599
|
+
status: 0,
|
|
600
|
+
errorName: "request_timeout"
|
|
601
|
+
});
|
|
602
|
+
const status = await this.getScrapeStatus(jobId, { signal: options.signal });
|
|
603
|
+
switch (status.status) {
|
|
604
|
+
case "done": return this.getScrapeResult(jobId, { signal: options.signal });
|
|
605
|
+
case "failed": throw new CrawlbruleeError(status.error ?? `Async scrape job ${jobId} failed.`, {
|
|
606
|
+
status: 0,
|
|
607
|
+
errorName: "job_failed"
|
|
608
|
+
});
|
|
609
|
+
case "pending":
|
|
610
|
+
case "running": break;
|
|
611
|
+
default: throw new CrawlbruleeError(`Async scrape job ${jobId} returned unexpected status '${String(status.status)}'.`, {
|
|
612
|
+
status: 0,
|
|
613
|
+
errorName: "job_failed"
|
|
614
|
+
});
|
|
615
|
+
}
|
|
616
|
+
await sleep(intervalMs, options.signal);
|
|
617
|
+
}
|
|
618
|
+
}
|
|
619
|
+
/**
|
|
620
|
+
* Build (or return a cached) site link-map for a domain. Combines sitemap
|
|
621
|
+
* discovery with the freshest cached homepage scrape when available.
|
|
622
|
+
*/
|
|
623
|
+
map(request, options) {
|
|
624
|
+
return this.http.post("/api/map", request, options);
|
|
625
|
+
}
|
|
626
|
+
/**
|
|
627
|
+
* Return the current billing-cycle usage: total/used/available credits,
|
|
628
|
+
* used quota percentage, max concurrency, and when the cycle resets.
|
|
629
|
+
*/
|
|
630
|
+
usage(options) {
|
|
631
|
+
return this.http.get("/api/usage", options);
|
|
632
|
+
}
|
|
633
|
+
/**
|
|
634
|
+
* Return the organization name and identifying details of the API token
|
|
635
|
+
* used to authenticate this request. Useful for confirming which key is in
|
|
636
|
+
* use before performing destructive operations.
|
|
637
|
+
*/
|
|
638
|
+
whoami(options) {
|
|
639
|
+
return this.http.get("/api/whoami", options);
|
|
640
|
+
}
|
|
523
641
|
};
|
|
642
|
+
/**
|
|
643
|
+
* Defensive read of `process.env[name]`. Guards both the absence of `process`
|
|
644
|
+
* (browser / edge runtimes) and Deno's permission throw on env access without
|
|
645
|
+
* `--allow-env`.
|
|
646
|
+
*/
|
|
524
647
|
function readEnv(name) {
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
648
|
+
try {
|
|
649
|
+
if (typeof process === "undefined" || !process.env) return void 0;
|
|
650
|
+
const v = process.env[name];
|
|
651
|
+
return typeof v === "string" && v.trim().length > 0 ? v.trim() : void 0;
|
|
652
|
+
} catch {
|
|
653
|
+
return;
|
|
654
|
+
}
|
|
532
655
|
}
|
|
533
656
|
function assertNonEmptyJobId(jobId) {
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
});
|
|
539
|
-
}
|
|
657
|
+
if (typeof jobId !== "string" || jobId.trim().length === 0) throw new CrawlbruleeError("jobId must be a non-empty string.", {
|
|
658
|
+
status: 0,
|
|
659
|
+
errorName: null
|
|
660
|
+
});
|
|
540
661
|
}
|
|
541
662
|
function throwIfAborted(signal) {
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
});
|
|
548
|
-
}
|
|
663
|
+
if (signal?.aborted) throw new CrawlbruleeError("Request aborted by caller.", {
|
|
664
|
+
status: 0,
|
|
665
|
+
errorName: "client_closed_request",
|
|
666
|
+
cause: signal.reason
|
|
667
|
+
});
|
|
549
668
|
}
|
|
550
669
|
function sleep(ms, signal) {
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
}
|
|
574
|
-
});
|
|
670
|
+
return new Promise((resolve, reject) => {
|
|
671
|
+
const onAbort = () => {
|
|
672
|
+
clearTimeout(timer);
|
|
673
|
+
reject(new CrawlbruleeError("Request aborted by caller.", {
|
|
674
|
+
status: 0,
|
|
675
|
+
errorName: "client_closed_request",
|
|
676
|
+
cause: signal?.reason
|
|
677
|
+
}));
|
|
678
|
+
};
|
|
679
|
+
const timer = setTimeout(() => {
|
|
680
|
+
signal?.removeEventListener("abort", onAbort);
|
|
681
|
+
resolve();
|
|
682
|
+
}, ms);
|
|
683
|
+
if (signal) {
|
|
684
|
+
if (signal.aborted) {
|
|
685
|
+
clearTimeout(timer);
|
|
686
|
+
onAbort();
|
|
687
|
+
return;
|
|
688
|
+
}
|
|
689
|
+
signal.addEventListener("abort", onAbort, { once: true });
|
|
690
|
+
}
|
|
691
|
+
});
|
|
575
692
|
}
|
|
576
693
|
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
694
|
+
//#endregion
|
|
695
|
+
//#region src/webhooks.ts
|
|
696
|
+
/**
|
|
697
|
+
* Verification for async scrape completion webhooks.
|
|
698
|
+
*
|
|
699
|
+
* {@link verifyWebhookSignature} validates the signature crawlbrulee attaches to
|
|
700
|
+
* every webhook delivery. It is a standalone, network-free helper built on Web
|
|
701
|
+
* Crypto (`globalThis.crypto.subtle`) so it runs unchanged on Node.js 22+,
|
|
702
|
+
* browsers, Bun, Deno, and edge runtimes — it never touches `node:crypto`.
|
|
703
|
+
*/
|
|
704
|
+
/** HTTP header carrying the primary webhook signature (always present). */
|
|
705
|
+
const WEBHOOK_SIGNATURE_HEADER = "X-Cwbl-Signature";
|
|
706
|
+
/**
|
|
707
|
+
* HTTP header carrying a signature produced with the previous signing secret.
|
|
708
|
+
* Present only during a signing-secret rotation grace window.
|
|
709
|
+
*/
|
|
710
|
+
const WEBHOOK_SIGNATURE_ROTATED_HEADER = "X-Cwbl-Signature-Rotated";
|
|
711
|
+
/** HTTP header carrying the unique event id, useful for delivery de-duplication. */
|
|
712
|
+
const WEBHOOK_EVENT_ID_HEADER = "X-Cwbl-Event-Id";
|
|
713
|
+
/** Default replay-protection window (seconds) applied to the signed timestamp. */
|
|
714
|
+
const DEFAULT_WEBHOOK_TOLERANCE_SECONDS = 300;
|
|
715
|
+
const SIGNATURE_FORMAT = /^t=(\d+),v1=([0-9a-f]{64})$/;
|
|
716
|
+
/**
|
|
717
|
+
* Verify a crawlbrulee webhook signature against the primary and rotated
|
|
718
|
+
* headers.
|
|
719
|
+
*
|
|
720
|
+
* The signing scheme matches the backend:
|
|
721
|
+
* - the signed payload is `` `${t}.${rawBody}` `` where `t` is the unix-seconds
|
|
722
|
+
* integer from the header and `rawBody` is the raw request body,
|
|
723
|
+
* - the signature is `HMAC-SHA256(secret, signedPayload)` as lowercase hex,
|
|
724
|
+
* - the header value is `t=<unix_seconds>,v1=<64_hex>`.
|
|
725
|
+
*
|
|
726
|
+
* The supplied `secret` is tried against the primary header first, then the
|
|
727
|
+
* rotated header (which the API emits during a signing-secret rotation grace
|
|
728
|
+
* window). Whichever matches wins, and the result reports which header it was.
|
|
729
|
+
*
|
|
730
|
+
* This NEVER throws on a verification failure — failures are normal control
|
|
731
|
+
* flow and are returned as `{ verified: false, reason }`.
|
|
732
|
+
*
|
|
733
|
+
* @example
|
|
734
|
+
* ```ts
|
|
735
|
+
* const result = await verifyWebhookSignature({
|
|
736
|
+
* payload: rawBody,
|
|
737
|
+
* headers: req.headers,
|
|
738
|
+
* secret: process.env.CRAWLBRULEE_WEBHOOK_SECRET!,
|
|
739
|
+
* })
|
|
740
|
+
* if (!result.verified) return res.status(400).end()
|
|
741
|
+
* ```
|
|
742
|
+
*/
|
|
583
743
|
async function verifyWebhookSignature(options) {
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
744
|
+
const { payload, headers, secret } = options;
|
|
745
|
+
const toleranceSeconds = options.toleranceSeconds ?? 300;
|
|
746
|
+
const primaryHeader = getHeader(headers, WEBHOOK_SIGNATURE_HEADER);
|
|
747
|
+
const rotatedHeader = getHeader(headers, WEBHOOK_SIGNATURE_ROTATED_HEADER);
|
|
748
|
+
if (primaryHeader === void 0 && rotatedHeader === void 0) return {
|
|
749
|
+
verified: false,
|
|
750
|
+
reason: "missing_signature"
|
|
751
|
+
};
|
|
752
|
+
const nowSeconds = Math.floor(Date.now() / 1e3);
|
|
753
|
+
const body = toBytes(payload);
|
|
754
|
+
const key = await importHmacKey(secret);
|
|
755
|
+
let failure = "malformed_signature";
|
|
756
|
+
for (const source of ["primary", "rotated"]) {
|
|
757
|
+
const raw = source === "primary" ? primaryHeader : rotatedHeader;
|
|
758
|
+
if (raw === void 0) continue;
|
|
759
|
+
const parsed = parseSignatureHeader(raw);
|
|
760
|
+
if (!parsed) continue;
|
|
761
|
+
if (toleranceSeconds && Math.abs(nowSeconds - parsed.timestamp) > toleranceSeconds) {
|
|
762
|
+
failure = mostSpecificFailure(failure, "timestamp_out_of_tolerance");
|
|
763
|
+
continue;
|
|
764
|
+
}
|
|
765
|
+
if (constantTimeEqualHex(await computeSignatureHex(key, parsed.timestamp, body), parsed.signature)) return {
|
|
766
|
+
verified: true,
|
|
767
|
+
signedWith: source
|
|
768
|
+
};
|
|
769
|
+
failure = mostSpecificFailure(failure, "signature_mismatch");
|
|
770
|
+
}
|
|
771
|
+
return {
|
|
772
|
+
verified: false,
|
|
773
|
+
reason: failure
|
|
774
|
+
};
|
|
613
775
|
}
|
|
776
|
+
/**
|
|
777
|
+
* Rank verification failures so the returned reason reflects the most
|
|
778
|
+
* actionable problem encountered across the two headers.
|
|
779
|
+
*/
|
|
614
780
|
function mostSpecificFailure(current, candidate) {
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
781
|
+
const rank = {
|
|
782
|
+
missing_signature: 0,
|
|
783
|
+
malformed_signature: 1,
|
|
784
|
+
timestamp_out_of_tolerance: 2,
|
|
785
|
+
signature_mismatch: 3
|
|
786
|
+
};
|
|
787
|
+
return rank[candidate] > rank[current] ? candidate : current;
|
|
622
788
|
}
|
|
789
|
+
/** Case-insensitive header lookup over `Headers` or a plain object. */
|
|
623
790
|
function getHeader(headers, name) {
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
return value ?? void 0;
|
|
633
|
-
}
|
|
634
|
-
return void 0;
|
|
791
|
+
if (typeof Headers !== "undefined" && headers instanceof Headers) return headers.get(name) ?? void 0;
|
|
792
|
+
const target = name.toLowerCase();
|
|
793
|
+
for (const key of Object.keys(headers)) {
|
|
794
|
+
if (key.toLowerCase() !== target) continue;
|
|
795
|
+
const value = headers[key];
|
|
796
|
+
if (Array.isArray(value)) return value[0];
|
|
797
|
+
return value ?? void 0;
|
|
798
|
+
}
|
|
635
799
|
}
|
|
636
800
|
function parseSignatureHeader(value) {
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
801
|
+
const match = SIGNATURE_FORMAT.exec(value.trim());
|
|
802
|
+
if (!match) return null;
|
|
803
|
+
const timestamp = Number(match[1]);
|
|
804
|
+
if (!Number.isSafeInteger(timestamp)) return null;
|
|
805
|
+
return {
|
|
806
|
+
timestamp,
|
|
807
|
+
signature: match[2]
|
|
808
|
+
};
|
|
642
809
|
}
|
|
643
810
|
function toBytes(payload) {
|
|
644
|
-
|
|
811
|
+
return typeof payload === "string" ? new TextEncoder().encode(payload) : payload;
|
|
645
812
|
}
|
|
646
813
|
function importHmacKey(secret) {
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
false,
|
|
652
|
-
["sign"]
|
|
653
|
-
);
|
|
814
|
+
return getSubtle().importKey("raw", new TextEncoder().encode(secret), {
|
|
815
|
+
name: "HMAC",
|
|
816
|
+
hash: "SHA-256"
|
|
817
|
+
}, false, ["sign"]);
|
|
654
818
|
}
|
|
655
819
|
async function computeSignatureHex(key, timestamp, body) {
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
820
|
+
const prefix = new TextEncoder().encode(`${timestamp}.`);
|
|
821
|
+
const message = new Uint8Array(prefix.length + body.length);
|
|
822
|
+
message.set(prefix, 0);
|
|
823
|
+
message.set(body, prefix.length);
|
|
824
|
+
const digest = await getSubtle().sign("HMAC", key, message);
|
|
825
|
+
return toHex(new Uint8Array(digest));
|
|
662
826
|
}
|
|
663
827
|
function toHex(bytes) {
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
}
|
|
668
|
-
return hex;
|
|
828
|
+
let hex = "";
|
|
829
|
+
for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
|
|
830
|
+
return hex;
|
|
669
831
|
}
|
|
832
|
+
/**
|
|
833
|
+
* Length-checked, constant-time comparison of two lowercase hex strings. Folds
|
|
834
|
+
* every byte into an accumulator with XOR — never early-returns on the first
|
|
835
|
+
* mismatch — so timing does not leak how much of the signature matched.
|
|
836
|
+
*/
|
|
670
837
|
function constantTimeEqualHex(a, b) {
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
}
|
|
676
|
-
return diff === 0;
|
|
838
|
+
if (a.length !== b.length) return false;
|
|
839
|
+
let diff = 0;
|
|
840
|
+
for (let i = 0; i < a.length; i++) diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
841
|
+
return diff === 0;
|
|
677
842
|
}
|
|
678
843
|
function getSubtle() {
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
"Web Crypto (globalThis.crypto.subtle) is not available in this runtime. crawlbrulee webhook verification requires Node.js 20+, Bun, Deno, or a modern browser/edge runtime."
|
|
683
|
-
);
|
|
684
|
-
}
|
|
685
|
-
return subtle;
|
|
844
|
+
const subtle = globalThis.crypto?.subtle;
|
|
845
|
+
if (!subtle) throw new Error("Web Crypto (globalThis.crypto.subtle) is not available in this runtime. crawlbrulee webhook verification requires Node.js 22+, Bun, Deno, or a modern browser/edge runtime.");
|
|
846
|
+
return subtle;
|
|
686
847
|
}
|
|
687
848
|
|
|
849
|
+
//#endregion
|
|
688
850
|
exports.AuthenticationError = AuthenticationError;
|
|
689
851
|
exports.Crawlbrulee = Crawlbrulee;
|
|
690
852
|
exports.CrawlbruleeError = CrawlbruleeError;
|
|
@@ -702,5 +864,4 @@ exports.WEBHOOK_SIGNATURE_HEADER = WEBHOOK_SIGNATURE_HEADER;
|
|
|
702
864
|
exports.WEBHOOK_SIGNATURE_ROTATED_HEADER = WEBHOOK_SIGNATURE_ROTATED_HEADER;
|
|
703
865
|
exports.isCrawlbruleeError = isCrawlbruleeError;
|
|
704
866
|
exports.verifyWebhookSignature = verifyWebhookSignature;
|
|
705
|
-
//# sourceMappingURL=index.cjs.map
|
|
706
867
|
//# sourceMappingURL=index.cjs.map
|