@crawlbrulee/sdk 0.1.5 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +79 -3
- package/dist/index.cjs +166 -0
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +199 -2
- package/dist/index.d.ts +199 -2
- package/dist/index.js +162 -1
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -255,6 +255,40 @@ interface ScrapeRequest {
|
|
|
255
255
|
/** Optional locale + country emulation. */
|
|
256
256
|
location?: ScrapeLocation;
|
|
257
257
|
}
|
|
258
|
+
/**
|
|
259
|
+
* Per-job completion webhook, attached when submitting an ASYNC scrape via
|
|
260
|
+
* {@link Crawlbrulee.scrapeAsync}. Async-only: the synchronous `scrape()`
|
|
261
|
+
* response IS the notification, so {@link ScrapeRequest} deliberately omits
|
|
262
|
+
* this field and the sync `/api/scrape` endpoint rejects it.
|
|
263
|
+
*
|
|
264
|
+
* Configure the signing secret used for deliveries in the dashboard
|
|
265
|
+
* (Account → Webhooks) — there is no per-request secret.
|
|
266
|
+
*/
|
|
267
|
+
interface AsyncScrapeWebhook {
|
|
268
|
+
/**
|
|
269
|
+
* Endpoint that receives a single signed `POST` when the job reaches a
|
|
270
|
+
* terminal state. Must be an `http`/`https` URL (HTTPS is required in
|
|
271
|
+
* production) of at most 2048 characters. The body is a
|
|
272
|
+
* `scrape.complete` envelope signed with your organization webhook secret
|
|
273
|
+
* on the `X-Cwbl-Signature` header — verify it with `verifyWebhookSignature`.
|
|
274
|
+
*/
|
|
275
|
+
url: string;
|
|
276
|
+
/**
|
|
277
|
+
* Opaque correlation object echoed verbatim in the webhook payload's
|
|
278
|
+
* `data.metadata`. Must serialize to at most 2048 bytes (UTF-8 JSON). Use it
|
|
279
|
+
* to route deliveries without keeping your own `job_id` mapping.
|
|
280
|
+
*/
|
|
281
|
+
metadata?: Record<string, unknown>;
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Request body for `POST /api/scrape/async`: a {@link ScrapeRequest} plus an
|
|
285
|
+
* optional per-job completion {@link AsyncScrapeWebhook}. The `webhook` field
|
|
286
|
+
* is async-only and is not accepted by the synchronous `scrape()` endpoint.
|
|
287
|
+
*/
|
|
288
|
+
interface AsyncScrapeRequest extends ScrapeRequest {
|
|
289
|
+
/** Optional completion webhook delivered when this job finishes. */
|
|
290
|
+
webhook?: AsyncScrapeWebhook;
|
|
291
|
+
}
|
|
258
292
|
/** Viewport metadata returned alongside a captured screenshot. */
|
|
259
293
|
interface ScreenshotViewportInfo {
|
|
260
294
|
width: number;
|
|
@@ -524,6 +558,54 @@ interface WhoamiResponse {
|
|
|
524
558
|
token_preview: string;
|
|
525
559
|
}
|
|
526
560
|
|
|
561
|
+
/**
|
|
562
|
+
* Async scrape completion webhooks.
|
|
563
|
+
*
|
|
564
|
+
* When you submit a job with {@link Crawlbrulee.scrapeAsync} the API can deliver
|
|
565
|
+
* a `scrape.complete` webhook to your configured endpoint once the job reaches a
|
|
566
|
+
* terminal state. The envelope on the wire is exactly:
|
|
567
|
+
*
|
|
568
|
+
* ```json
|
|
569
|
+
* {
|
|
570
|
+
* "event_id": "evt_…",
|
|
571
|
+
* "timestamp": "2026-06-13T12:00:00.000Z",
|
|
572
|
+
* "event": "scrape.complete",
|
|
573
|
+
* "data": { "job_id": "job_…", "status": "success", "url": "https://…", "completed_at": "…" }
|
|
574
|
+
* }
|
|
575
|
+
* ```
|
|
576
|
+
*/
|
|
577
|
+
/** Terminal status carried by a {@link ScrapeCompleteWebhook}. */
|
|
578
|
+
type ScrapeWebhookStatus = 'success' | 'failed' | 'cancelled';
|
|
579
|
+
/** `data` block of a {@link ScrapeCompleteWebhook}. */
|
|
580
|
+
interface ScrapeCompleteWebhookData {
|
|
581
|
+
/** The async job identifier — pass it to `getScrapeResult`. */
|
|
582
|
+
job_id: string;
|
|
583
|
+
/** Terminal state of the job. */
|
|
584
|
+
status: ScrapeWebhookStatus;
|
|
585
|
+
/** The URL that was scraped. */
|
|
586
|
+
url: string;
|
|
587
|
+
/** ISO-8601 UTC timestamp when the job reached its terminal state. */
|
|
588
|
+
completed_at: string;
|
|
589
|
+
/** Failure message — present only when `status === 'failed'`. */
|
|
590
|
+
error?: string;
|
|
591
|
+
/** Correlation data echoed back from the original scrape request, if any. */
|
|
592
|
+
metadata?: Record<string, unknown>;
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* Webhook envelope delivered when an async scrape job completes. The `event`
|
|
596
|
+
* discriminator is always the literal `'scrape.complete'`.
|
|
597
|
+
*/
|
|
598
|
+
interface ScrapeCompleteWebhook {
|
|
599
|
+
/** Unique event identifier — also delivered in the `X-Cwbl-Event-Id` header. */
|
|
600
|
+
event_id: string;
|
|
601
|
+
/** ISO-8601 UTC timestamp when the event was emitted. */
|
|
602
|
+
timestamp: string;
|
|
603
|
+
/** Event type discriminator. */
|
|
604
|
+
event: 'scrape.complete';
|
|
605
|
+
/** Event payload. */
|
|
606
|
+
data: ScrapeCompleteWebhookData;
|
|
607
|
+
}
|
|
608
|
+
|
|
527
609
|
/** Options accepted by the {@link Crawlbrulee} constructor. */
|
|
528
610
|
interface CrawlbruleeOptions {
|
|
529
611
|
/**
|
|
@@ -617,8 +699,12 @@ declare class Crawlbrulee {
|
|
|
617
699
|
* Submit an asynchronous scrape job and return its `job_id`. Poll the job
|
|
618
700
|
* with {@link Crawlbrulee.getScrapeStatus} or wait for completion with
|
|
619
701
|
* {@link Crawlbrulee.waitForScrape}.
|
|
702
|
+
*
|
|
703
|
+
* Pass an optional `webhook` to have the API deliver a signed
|
|
704
|
+
* `scrape.complete` `POST` to your endpoint when the job finishes (see
|
|
705
|
+
* {@link AsyncScrapeWebhook}). This field is async-only.
|
|
620
706
|
*/
|
|
621
|
-
scrapeAsync(request:
|
|
707
|
+
scrapeAsync(request: AsyncScrapeRequest, options?: RequestOptions): Promise<AsyncScrapeResponse>;
|
|
622
708
|
/** Look up the current status of an async scrape job. */
|
|
623
709
|
getScrapeStatus(jobId: string, options?: RequestOptions): Promise<AsyncJobStatusResponse>;
|
|
624
710
|
/**
|
|
@@ -627,6 +713,25 @@ declare class Crawlbrulee {
|
|
|
627
713
|
* first, or use {@link Crawlbrulee.waitForScrape} to poll-then-fetch.
|
|
628
714
|
*/
|
|
629
715
|
getScrapeResult(jobId: string, options?: RequestOptions): Promise<ScrapeResponse>;
|
|
716
|
+
/**
|
|
717
|
+
* Fetch the scrape result referenced by a `scrape.complete` webhook body.
|
|
718
|
+
*
|
|
719
|
+
* Always verify the webhook signature with `verifyWebhookSignature` before
|
|
720
|
+
* acting on it; this method trusts the parsed body it is handed.
|
|
721
|
+
*
|
|
722
|
+
* Behavior by `data.status`:
|
|
723
|
+
* - `success` — delegates to {@link Crawlbrulee.getScrapeResult} for the
|
|
724
|
+
* webhook's `job_id` and returns the parsed result.
|
|
725
|
+
* - `failed` — throws a {@link CrawlbruleeError} carrying `data.error`
|
|
726
|
+
* (`errorName: 'job_failed'`); there is no result to fetch.
|
|
727
|
+
* - `cancelled` — throws a {@link CrawlbruleeError}
|
|
728
|
+
* (`errorName: 'client_closed_request'`).
|
|
729
|
+
*
|
|
730
|
+
* A non-`scrape.complete` envelope throws a {@link CrawlbruleeError}
|
|
731
|
+
* defensively. Any HTTP error from the underlying fetch propagates as the
|
|
732
|
+
* usual typed `CrawlbruleeError` subclass.
|
|
733
|
+
*/
|
|
734
|
+
fetchScrapeResultFromWebhook(webhook: ScrapeCompleteWebhook, options?: RequestOptions): Promise<ScrapeResponse>;
|
|
630
735
|
/**
|
|
631
736
|
* Poll an async scrape job until it reaches a terminal state, then return
|
|
632
737
|
* the scrape result.
|
|
@@ -782,4 +887,96 @@ declare const DEFAULT_REQUEST_TIMEOUT_MS = 60000;
|
|
|
782
887
|
/** Environment variable read by `Crawlbrulee.fromEnv()` to source the API key. */
|
|
783
888
|
declare const ENV_API_KEY = "CRAWLBRULEE_API_KEY";
|
|
784
889
|
|
|
785
|
-
|
|
890
|
+
/**
|
|
891
|
+
* Verification for async scrape completion webhooks.
|
|
892
|
+
*
|
|
893
|
+
* {@link verifyWebhookSignature} validates the signature crawlbrulee attaches to
|
|
894
|
+
* every webhook delivery. It is a standalone, network-free helper built on Web
|
|
895
|
+
* Crypto (`globalThis.crypto.subtle`) so it runs unchanged on Node.js 22+,
|
|
896
|
+
* browsers, Bun, Deno, and edge runtimes — it never touches `node:crypto`.
|
|
897
|
+
*/
|
|
898
|
+
/** HTTP header carrying the primary webhook signature (always present). */
|
|
899
|
+
declare const WEBHOOK_SIGNATURE_HEADER = "X-Cwbl-Signature";
|
|
900
|
+
/**
|
|
901
|
+
* HTTP header carrying a signature produced with the previous signing secret.
|
|
902
|
+
* Present only during a signing-secret rotation grace window.
|
|
903
|
+
*/
|
|
904
|
+
declare const WEBHOOK_SIGNATURE_ROTATED_HEADER = "X-Cwbl-Signature-Rotated";
|
|
905
|
+
/** HTTP header carrying the unique event id, useful for delivery de-duplication. */
|
|
906
|
+
declare const WEBHOOK_EVENT_ID_HEADER = "X-Cwbl-Event-Id";
|
|
907
|
+
/** Default replay-protection window (seconds) applied to the signed timestamp. */
|
|
908
|
+
declare const DEFAULT_WEBHOOK_TOLERANCE_SECONDS = 300;
|
|
909
|
+
/** Which signature header satisfied verification. */
|
|
910
|
+
type WebhookSignatureSource = 'primary' | 'rotated';
|
|
911
|
+
/**
|
|
912
|
+
* Why a webhook signature failed to verify.
|
|
913
|
+
*
|
|
914
|
+
* - `missing_signature` — neither the primary nor the rotated header was present.
|
|
915
|
+
* - `malformed_signature` — a header was present but not in the expected
|
|
916
|
+
* `t=<unix_seconds>,v1=<64_hex>` format.
|
|
917
|
+
* - `timestamp_out_of_tolerance` — the signed timestamp drifted further from now
|
|
918
|
+
* than `toleranceSeconds` allows (replay protection).
|
|
919
|
+
* - `signature_mismatch` — a well-formed, in-tolerance signature did not match
|
|
920
|
+
* the one computed from the payload and secret.
|
|
921
|
+
*/
|
|
922
|
+
type WebhookVerificationFailureReason = 'missing_signature' | 'malformed_signature' | 'timestamp_out_of_tolerance' | 'signature_mismatch';
|
|
923
|
+
/** Result of {@link verifyWebhookSignature}. Verification failure is returned, not thrown. */
|
|
924
|
+
type WebhookVerificationResult = {
|
|
925
|
+
verified: true;
|
|
926
|
+
signedWith: WebhookSignatureSource;
|
|
927
|
+
} | {
|
|
928
|
+
verified: false;
|
|
929
|
+
reason: WebhookVerificationFailureReason;
|
|
930
|
+
};
|
|
931
|
+
/** Options for {@link verifyWebhookSignature}. */
|
|
932
|
+
interface VerifyWebhookSignatureOptions {
|
|
933
|
+
/**
|
|
934
|
+
* The raw request body, exactly as received. Pass the bytes/string the server
|
|
935
|
+
* signed — do NOT re-serialize parsed JSON, or the signature will not match.
|
|
936
|
+
*/
|
|
937
|
+
payload: string | Uint8Array;
|
|
938
|
+
/**
|
|
939
|
+
* The request headers. Accepts a fetch `Headers` instance or a plain object
|
|
940
|
+
* (Express/Node give lowercased keys, values possibly arrays). Lookup is
|
|
941
|
+
* case-insensitive.
|
|
942
|
+
*/
|
|
943
|
+
headers: Headers | Record<string, string | string[] | undefined>;
|
|
944
|
+
/** The current signing secret (`whsec_…`). */
|
|
945
|
+
secret: string;
|
|
946
|
+
/**
|
|
947
|
+
* Replay-protection window in seconds. Defaults to
|
|
948
|
+
* {@link DEFAULT_WEBHOOK_TOLERANCE_SECONDS} (300). Pass `0` (or any falsy
|
|
949
|
+
* value) to disable the timestamp check entirely.
|
|
950
|
+
*/
|
|
951
|
+
toleranceSeconds?: number;
|
|
952
|
+
}
|
|
953
|
+
/**
|
|
954
|
+
* Verify a crawlbrulee webhook signature against the primary and rotated
|
|
955
|
+
* headers.
|
|
956
|
+
*
|
|
957
|
+
* The signing scheme matches the backend:
|
|
958
|
+
* - the signed payload is `` `${t}.${rawBody}` `` where `t` is the unix-seconds
|
|
959
|
+
* integer from the header and `rawBody` is the raw request body,
|
|
960
|
+
* - the signature is `HMAC-SHA256(secret, signedPayload)` as lowercase hex,
|
|
961
|
+
* - the header value is `t=<unix_seconds>,v1=<64_hex>`.
|
|
962
|
+
*
|
|
963
|
+
* The supplied `secret` is tried against the primary header first, then the
|
|
964
|
+
* rotated header (which the API emits during a signing-secret rotation grace
|
|
965
|
+
* window). Whichever matches wins, and the result reports which header it was.
|
|
966
|
+
*
|
|
967
|
+
* This NEVER throws on a verification failure — failures are normal control
|
|
968
|
+
* flow and are returned as `{ verified: false, reason }`.
|
|
969
|
+
*
|
|
970
|
+
* @example
|
|
971
|
+
* ```ts
|
|
972
|
+
* const result = await verifyWebhookSignature({
|
|
973
|
+
* payload: rawBody,
|
|
974
|
+
* headers: req.headers,
|
|
975
|
+
* secret: process.env.CRAWLBRULEE_WEBHOOK_SECRET!,
|
|
976
|
+
* })
|
|
977
|
+
* if (!result.verified) return res.status(400).end()
|
|
978
|
+
* ```
|
|
979
|
+
*/
|
|
980
|
+
declare function verifyWebhookSignature(options: VerifyWebhookSignatureOptions): Promise<WebhookVerificationResult>;
|
|
981
|
+
|
|
982
|
+
export { type ApiErrorDetails, type ApiErrorName, type ApiErrorResponse, type AsyncJobStatus, type AsyncJobStatusResponse, type AsyncScrapeRequest, type AsyncScrapeResponse, type AsyncScrapeWebhook, AuthenticationError, Crawlbrulee, CrawlbruleeError, type CrawlbruleeOptions, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_WEBHOOK_TOLERANCE_SECONDS, ENV_API_KEY, type HttpMethod, type MapCache, type MapLinkItem, type MapLocation, type MapPagination, type MapRequest, type MapResponse, type MapTruncation, type MapTypes, NotFoundError, type PageInlineImage, type PageLink, type ProxyTier, RateLimitError, type RateLimitErrorDetails, type RequestOptions, type ScrapeCache, type ScrapeCompleteWebhook, type ScrapeCompleteWebhookData, type ScrapeExtract, type ScrapeLocation, type ScrapeMetadata, type ScrapeRequest, type ScrapeResponse, type ScrapeWebhookStatus, type ScreenshotAfterAction, type ScreenshotBeforeAction, type ScreenshotCleanup, type ScreenshotDeviceMode, type ScreenshotProperties, type ScreenshotRequest, type ScreenshotResult, type ScreenshotScrollAction, type ScreenshotSlice, type ScreenshotSliceAction, type ScreenshotType, type ScreenshotViewport, type ScreenshotViewportInfo, type ScreenshotWaitAction, TransportError, UsageAllocationError, type UsageAllocationErrorDetails, type UsageAllocationReason, type UsageLimitDetails, type UsageResponse, ValidationError, type VerifyWebhookSignatureOptions, WEBHOOK_EVENT_ID_HEADER, WEBHOOK_SIGNATURE_HEADER, WEBHOOK_SIGNATURE_ROTATED_HEADER, type WaitForScrapeOptions, type WebhookSignatureSource, type WebhookVerificationFailureReason, type WebhookVerificationResult, type WhoamiResponse, isCrawlbruleeError, verifyWebhookSignature };
|
package/dist/index.d.ts
CHANGED
|
@@ -255,6 +255,40 @@ interface ScrapeRequest {
|
|
|
255
255
|
/** Optional locale + country emulation. */
|
|
256
256
|
location?: ScrapeLocation;
|
|
257
257
|
}
|
|
258
|
+
/**
|
|
259
|
+
* Per-job completion webhook, attached when submitting an ASYNC scrape via
|
|
260
|
+
* {@link Crawlbrulee.scrapeAsync}. Async-only: the synchronous `scrape()`
|
|
261
|
+
* response IS the notification, so {@link ScrapeRequest} deliberately omits
|
|
262
|
+
* this field and the sync `/api/scrape` endpoint rejects it.
|
|
263
|
+
*
|
|
264
|
+
* Configure the signing secret used for deliveries in the dashboard
|
|
265
|
+
* (Account → Webhooks) — there is no per-request secret.
|
|
266
|
+
*/
|
|
267
|
+
interface AsyncScrapeWebhook {
|
|
268
|
+
/**
|
|
269
|
+
* Endpoint that receives a single signed `POST` when the job reaches a
|
|
270
|
+
* terminal state. Must be an `http`/`https` URL (HTTPS is required in
|
|
271
|
+
* production) of at most 2048 characters. The body is a
|
|
272
|
+
* `scrape.complete` envelope signed with your organization webhook secret
|
|
273
|
+
* on the `X-Cwbl-Signature` header — verify it with `verifyWebhookSignature`.
|
|
274
|
+
*/
|
|
275
|
+
url: string;
|
|
276
|
+
/**
|
|
277
|
+
* Opaque correlation object echoed verbatim in the webhook payload's
|
|
278
|
+
* `data.metadata`. Must serialize to at most 2048 bytes (UTF-8 JSON). Use it
|
|
279
|
+
* to route deliveries without keeping your own `job_id` mapping.
|
|
280
|
+
*/
|
|
281
|
+
metadata?: Record<string, unknown>;
|
|
282
|
+
}
|
|
283
|
+
/**
|
|
284
|
+
* Request body for `POST /api/scrape/async`: a {@link ScrapeRequest} plus an
|
|
285
|
+
* optional per-job completion {@link AsyncScrapeWebhook}. The `webhook` field
|
|
286
|
+
* is async-only and is not accepted by the synchronous `scrape()` endpoint.
|
|
287
|
+
*/
|
|
288
|
+
interface AsyncScrapeRequest extends ScrapeRequest {
|
|
289
|
+
/** Optional completion webhook delivered when this job finishes. */
|
|
290
|
+
webhook?: AsyncScrapeWebhook;
|
|
291
|
+
}
|
|
258
292
|
/** Viewport metadata returned alongside a captured screenshot. */
|
|
259
293
|
interface ScreenshotViewportInfo {
|
|
260
294
|
width: number;
|
|
@@ -524,6 +558,54 @@ interface WhoamiResponse {
|
|
|
524
558
|
token_preview: string;
|
|
525
559
|
}
|
|
526
560
|
|
|
561
|
+
/**
|
|
562
|
+
* Async scrape completion webhooks.
|
|
563
|
+
*
|
|
564
|
+
* When you submit a job with {@link Crawlbrulee.scrapeAsync} the API can deliver
|
|
565
|
+
* a `scrape.complete` webhook to your configured endpoint once the job reaches a
|
|
566
|
+
* terminal state. The envelope on the wire is exactly:
|
|
567
|
+
*
|
|
568
|
+
* ```json
|
|
569
|
+
* {
|
|
570
|
+
* "event_id": "evt_…",
|
|
571
|
+
* "timestamp": "2026-06-13T12:00:00.000Z",
|
|
572
|
+
* "event": "scrape.complete",
|
|
573
|
+
* "data": { "job_id": "job_…", "status": "success", "url": "https://…", "completed_at": "…" }
|
|
574
|
+
* }
|
|
575
|
+
* ```
|
|
576
|
+
*/
|
|
577
|
+
/** Terminal status carried by a {@link ScrapeCompleteWebhook}. */
|
|
578
|
+
type ScrapeWebhookStatus = 'success' | 'failed' | 'cancelled';
|
|
579
|
+
/** `data` block of a {@link ScrapeCompleteWebhook}. */
|
|
580
|
+
interface ScrapeCompleteWebhookData {
|
|
581
|
+
/** The async job identifier — pass it to `getScrapeResult`. */
|
|
582
|
+
job_id: string;
|
|
583
|
+
/** Terminal state of the job. */
|
|
584
|
+
status: ScrapeWebhookStatus;
|
|
585
|
+
/** The URL that was scraped. */
|
|
586
|
+
url: string;
|
|
587
|
+
/** ISO-8601 UTC timestamp when the job reached its terminal state. */
|
|
588
|
+
completed_at: string;
|
|
589
|
+
/** Failure message — present only when `status === 'failed'`. */
|
|
590
|
+
error?: string;
|
|
591
|
+
/** Correlation data echoed back from the original scrape request, if any. */
|
|
592
|
+
metadata?: Record<string, unknown>;
|
|
593
|
+
}
|
|
594
|
+
/**
|
|
595
|
+
* Webhook envelope delivered when an async scrape job completes. The `event`
|
|
596
|
+
* discriminator is always the literal `'scrape.complete'`.
|
|
597
|
+
*/
|
|
598
|
+
interface ScrapeCompleteWebhook {
|
|
599
|
+
/** Unique event identifier — also delivered in the `X-Cwbl-Event-Id` header. */
|
|
600
|
+
event_id: string;
|
|
601
|
+
/** ISO-8601 UTC timestamp when the event was emitted. */
|
|
602
|
+
timestamp: string;
|
|
603
|
+
/** Event type discriminator. */
|
|
604
|
+
event: 'scrape.complete';
|
|
605
|
+
/** Event payload. */
|
|
606
|
+
data: ScrapeCompleteWebhookData;
|
|
607
|
+
}
|
|
608
|
+
|
|
527
609
|
/** Options accepted by the {@link Crawlbrulee} constructor. */
|
|
528
610
|
interface CrawlbruleeOptions {
|
|
529
611
|
/**
|
|
@@ -617,8 +699,12 @@ declare class Crawlbrulee {
|
|
|
617
699
|
* Submit an asynchronous scrape job and return its `job_id`. Poll the job
|
|
618
700
|
* with {@link Crawlbrulee.getScrapeStatus} or wait for completion with
|
|
619
701
|
* {@link Crawlbrulee.waitForScrape}.
|
|
702
|
+
*
|
|
703
|
+
* Pass an optional `webhook` to have the API deliver a signed
|
|
704
|
+
* `scrape.complete` `POST` to your endpoint when the job finishes (see
|
|
705
|
+
* {@link AsyncScrapeWebhook}). This field is async-only.
|
|
620
706
|
*/
|
|
621
|
-
scrapeAsync(request:
|
|
707
|
+
scrapeAsync(request: AsyncScrapeRequest, options?: RequestOptions): Promise<AsyncScrapeResponse>;
|
|
622
708
|
/** Look up the current status of an async scrape job. */
|
|
623
709
|
getScrapeStatus(jobId: string, options?: RequestOptions): Promise<AsyncJobStatusResponse>;
|
|
624
710
|
/**
|
|
@@ -627,6 +713,25 @@ declare class Crawlbrulee {
|
|
|
627
713
|
* first, or use {@link Crawlbrulee.waitForScrape} to poll-then-fetch.
|
|
628
714
|
*/
|
|
629
715
|
getScrapeResult(jobId: string, options?: RequestOptions): Promise<ScrapeResponse>;
|
|
716
|
+
/**
|
|
717
|
+
* Fetch the scrape result referenced by a `scrape.complete` webhook body.
|
|
718
|
+
*
|
|
719
|
+
* Always verify the webhook signature with `verifyWebhookSignature` before
|
|
720
|
+
* acting on it; this method trusts the parsed body it is handed.
|
|
721
|
+
*
|
|
722
|
+
* Behavior by `data.status`:
|
|
723
|
+
* - `success` — delegates to {@link Crawlbrulee.getScrapeResult} for the
|
|
724
|
+
* webhook's `job_id` and returns the parsed result.
|
|
725
|
+
* - `failed` — throws a {@link CrawlbruleeError} carrying `data.error`
|
|
726
|
+
* (`errorName: 'job_failed'`); there is no result to fetch.
|
|
727
|
+
* - `cancelled` — throws a {@link CrawlbruleeError}
|
|
728
|
+
* (`errorName: 'client_closed_request'`).
|
|
729
|
+
*
|
|
730
|
+
* A non-`scrape.complete` envelope throws a {@link CrawlbruleeError}
|
|
731
|
+
* defensively. Any HTTP error from the underlying fetch propagates as the
|
|
732
|
+
* usual typed `CrawlbruleeError` subclass.
|
|
733
|
+
*/
|
|
734
|
+
fetchScrapeResultFromWebhook(webhook: ScrapeCompleteWebhook, options?: RequestOptions): Promise<ScrapeResponse>;
|
|
630
735
|
/**
|
|
631
736
|
* Poll an async scrape job until it reaches a terminal state, then return
|
|
632
737
|
* the scrape result.
|
|
@@ -782,4 +887,96 @@ declare const DEFAULT_REQUEST_TIMEOUT_MS = 60000;
|
|
|
782
887
|
/** Environment variable read by `Crawlbrulee.fromEnv()` to source the API key. */
|
|
783
888
|
declare const ENV_API_KEY = "CRAWLBRULEE_API_KEY";
|
|
784
889
|
|
|
785
|
-
|
|
890
|
+
/**
|
|
891
|
+
* Verification for async scrape completion webhooks.
|
|
892
|
+
*
|
|
893
|
+
* {@link verifyWebhookSignature} validates the signature crawlbrulee attaches to
|
|
894
|
+
* every webhook delivery. It is a standalone, network-free helper built on Web
|
|
895
|
+
* Crypto (`globalThis.crypto.subtle`) so it runs unchanged on Node.js 22+,
|
|
896
|
+
* browsers, Bun, Deno, and edge runtimes — it never touches `node:crypto`.
|
|
897
|
+
*/
|
|
898
|
+
/** HTTP header carrying the primary webhook signature (always present). */
|
|
899
|
+
declare const WEBHOOK_SIGNATURE_HEADER = "X-Cwbl-Signature";
|
|
900
|
+
/**
|
|
901
|
+
* HTTP header carrying a signature produced with the previous signing secret.
|
|
902
|
+
* Present only during a signing-secret rotation grace window.
|
|
903
|
+
*/
|
|
904
|
+
declare const WEBHOOK_SIGNATURE_ROTATED_HEADER = "X-Cwbl-Signature-Rotated";
|
|
905
|
+
/** HTTP header carrying the unique event id, useful for delivery de-duplication. */
|
|
906
|
+
declare const WEBHOOK_EVENT_ID_HEADER = "X-Cwbl-Event-Id";
|
|
907
|
+
/** Default replay-protection window (seconds) applied to the signed timestamp. */
|
|
908
|
+
declare const DEFAULT_WEBHOOK_TOLERANCE_SECONDS = 300;
|
|
909
|
+
/** Which signature header satisfied verification. */
|
|
910
|
+
type WebhookSignatureSource = 'primary' | 'rotated';
|
|
911
|
+
/**
|
|
912
|
+
* Why a webhook signature failed to verify.
|
|
913
|
+
*
|
|
914
|
+
* - `missing_signature` — neither the primary nor the rotated header was present.
|
|
915
|
+
* - `malformed_signature` — a header was present but not in the expected
|
|
916
|
+
* `t=<unix_seconds>,v1=<64_hex>` format.
|
|
917
|
+
* - `timestamp_out_of_tolerance` — the signed timestamp drifted further from now
|
|
918
|
+
* than `toleranceSeconds` allows (replay protection).
|
|
919
|
+
* - `signature_mismatch` — a well-formed, in-tolerance signature did not match
|
|
920
|
+
* the one computed from the payload and secret.
|
|
921
|
+
*/
|
|
922
|
+
type WebhookVerificationFailureReason = 'missing_signature' | 'malformed_signature' | 'timestamp_out_of_tolerance' | 'signature_mismatch';
|
|
923
|
+
/** Result of {@link verifyWebhookSignature}. Verification failure is returned, not thrown. */
|
|
924
|
+
type WebhookVerificationResult = {
|
|
925
|
+
verified: true;
|
|
926
|
+
signedWith: WebhookSignatureSource;
|
|
927
|
+
} | {
|
|
928
|
+
verified: false;
|
|
929
|
+
reason: WebhookVerificationFailureReason;
|
|
930
|
+
};
|
|
931
|
+
/** Options for {@link verifyWebhookSignature}. */
|
|
932
|
+
interface VerifyWebhookSignatureOptions {
|
|
933
|
+
/**
|
|
934
|
+
* The raw request body, exactly as received. Pass the bytes/string the server
|
|
935
|
+
* signed — do NOT re-serialize parsed JSON, or the signature will not match.
|
|
936
|
+
*/
|
|
937
|
+
payload: string | Uint8Array;
|
|
938
|
+
/**
|
|
939
|
+
* The request headers. Accepts a fetch `Headers` instance or a plain object
|
|
940
|
+
* (Express/Node give lowercased keys, values possibly arrays). Lookup is
|
|
941
|
+
* case-insensitive.
|
|
942
|
+
*/
|
|
943
|
+
headers: Headers | Record<string, string | string[] | undefined>;
|
|
944
|
+
/** The current signing secret (`whsec_…`). */
|
|
945
|
+
secret: string;
|
|
946
|
+
/**
|
|
947
|
+
* Replay-protection window in seconds. Defaults to
|
|
948
|
+
* {@link DEFAULT_WEBHOOK_TOLERANCE_SECONDS} (300). Pass `0` (or any falsy
|
|
949
|
+
* value) to disable the timestamp check entirely.
|
|
950
|
+
*/
|
|
951
|
+
toleranceSeconds?: number;
|
|
952
|
+
}
|
|
953
|
+
/**
|
|
954
|
+
* Verify a crawlbrulee webhook signature against the primary and rotated
|
|
955
|
+
* headers.
|
|
956
|
+
*
|
|
957
|
+
* The signing scheme matches the backend:
|
|
958
|
+
* - the signed payload is `` `${t}.${rawBody}` `` where `t` is the unix-seconds
|
|
959
|
+
* integer from the header and `rawBody` is the raw request body,
|
|
960
|
+
* - the signature is `HMAC-SHA256(secret, signedPayload)` as lowercase hex,
|
|
961
|
+
* - the header value is `t=<unix_seconds>,v1=<64_hex>`.
|
|
962
|
+
*
|
|
963
|
+
* The supplied `secret` is tried against the primary header first, then the
|
|
964
|
+
* rotated header (which the API emits during a signing-secret rotation grace
|
|
965
|
+
* window). Whichever matches wins, and the result reports which header it was.
|
|
966
|
+
*
|
|
967
|
+
* This NEVER throws on a verification failure — failures are normal control
|
|
968
|
+
* flow and are returned as `{ verified: false, reason }`.
|
|
969
|
+
*
|
|
970
|
+
* @example
|
|
971
|
+
* ```ts
|
|
972
|
+
* const result = await verifyWebhookSignature({
|
|
973
|
+
* payload: rawBody,
|
|
974
|
+
* headers: req.headers,
|
|
975
|
+
* secret: process.env.CRAWLBRULEE_WEBHOOK_SECRET!,
|
|
976
|
+
* })
|
|
977
|
+
* if (!result.verified) return res.status(400).end()
|
|
978
|
+
* ```
|
|
979
|
+
*/
|
|
980
|
+
declare function verifyWebhookSignature(options: VerifyWebhookSignatureOptions): Promise<WebhookVerificationResult>;
|
|
981
|
+
|
|
982
|
+
export { type ApiErrorDetails, type ApiErrorName, type ApiErrorResponse, type AsyncJobStatus, type AsyncJobStatusResponse, type AsyncScrapeRequest, type AsyncScrapeResponse, type AsyncScrapeWebhook, AuthenticationError, Crawlbrulee, CrawlbruleeError, type CrawlbruleeOptions, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_WEBHOOK_TOLERANCE_SECONDS, ENV_API_KEY, type HttpMethod, type MapCache, type MapLinkItem, type MapLocation, type MapPagination, type MapRequest, type MapResponse, type MapTruncation, type MapTypes, NotFoundError, type PageInlineImage, type PageLink, type ProxyTier, RateLimitError, type RateLimitErrorDetails, type RequestOptions, type ScrapeCache, type ScrapeCompleteWebhook, type ScrapeCompleteWebhookData, type ScrapeExtract, type ScrapeLocation, type ScrapeMetadata, type ScrapeRequest, type ScrapeResponse, type ScrapeWebhookStatus, type ScreenshotAfterAction, type ScreenshotBeforeAction, type ScreenshotCleanup, type ScreenshotDeviceMode, type ScreenshotProperties, type ScreenshotRequest, type ScreenshotResult, type ScreenshotScrollAction, type ScreenshotSlice, type ScreenshotSliceAction, type ScreenshotType, type ScreenshotViewport, type ScreenshotViewportInfo, type ScreenshotWaitAction, TransportError, UsageAllocationError, type UsageAllocationErrorDetails, type UsageAllocationReason, type UsageLimitDetails, type UsageResponse, ValidationError, type VerifyWebhookSignatureOptions, WEBHOOK_EVENT_ID_HEADER, WEBHOOK_SIGNATURE_HEADER, WEBHOOK_SIGNATURE_ROTATED_HEADER, type WaitForScrapeOptions, type WebhookSignatureSource, type WebhookVerificationFailureReason, type WebhookVerificationResult, type WhoamiResponse, isCrawlbruleeError, verifyWebhookSignature };
|
package/dist/index.js
CHANGED
|
@@ -375,6 +375,10 @@ var Crawlbrulee = class _Crawlbrulee {
|
|
|
375
375
|
* Submit an asynchronous scrape job and return its `job_id`. Poll the job
|
|
376
376
|
* with {@link Crawlbrulee.getScrapeStatus} or wait for completion with
|
|
377
377
|
* {@link Crawlbrulee.waitForScrape}.
|
|
378
|
+
*
|
|
379
|
+
* Pass an optional `webhook` to have the API deliver a signed
|
|
380
|
+
* `scrape.complete` `POST` to your endpoint when the job finishes (see
|
|
381
|
+
* {@link AsyncScrapeWebhook}). This field is async-only.
|
|
378
382
|
*/
|
|
379
383
|
scrapeAsync(request, options) {
|
|
380
384
|
return this.http.post("/api/scrape/async", request, options);
|
|
@@ -396,6 +400,52 @@ var Crawlbrulee = class _Crawlbrulee {
|
|
|
396
400
|
assertNonEmptyJobId(jobId);
|
|
397
401
|
return this.http.get(`/api/scrape/result/${encodeURIComponent(jobId)}`, options);
|
|
398
402
|
}
|
|
403
|
+
/**
|
|
404
|
+
* Fetch the scrape result referenced by a `scrape.complete` webhook body.
|
|
405
|
+
*
|
|
406
|
+
* Always verify the webhook signature with `verifyWebhookSignature` before
|
|
407
|
+
* acting on it; this method trusts the parsed body it is handed.
|
|
408
|
+
*
|
|
409
|
+
* Behavior by `data.status`:
|
|
410
|
+
* - `success` — delegates to {@link Crawlbrulee.getScrapeResult} for the
|
|
411
|
+
* webhook's `job_id` and returns the parsed result.
|
|
412
|
+
* - `failed` — throws a {@link CrawlbruleeError} carrying `data.error`
|
|
413
|
+
* (`errorName: 'job_failed'`); there is no result to fetch.
|
|
414
|
+
* - `cancelled` — throws a {@link CrawlbruleeError}
|
|
415
|
+
* (`errorName: 'client_closed_request'`).
|
|
416
|
+
*
|
|
417
|
+
* A non-`scrape.complete` envelope throws a {@link CrawlbruleeError}
|
|
418
|
+
* defensively. Any HTTP error from the underlying fetch propagates as the
|
|
419
|
+
* usual typed `CrawlbruleeError` subclass.
|
|
420
|
+
*/
|
|
421
|
+
async fetchScrapeResultFromWebhook(webhook, options) {
|
|
422
|
+
if (webhook?.event !== "scrape.complete") {
|
|
423
|
+
throw new CrawlbruleeError(
|
|
424
|
+
`Expected a 'scrape.complete' webhook but received '${String(webhook?.event)}'.`,
|
|
425
|
+
{ status: 0, errorName: "validation_error" }
|
|
426
|
+
);
|
|
427
|
+
}
|
|
428
|
+
const { job_id: jobId, status, error } = webhook.data;
|
|
429
|
+
switch (status) {
|
|
430
|
+
case "success":
|
|
431
|
+
return this.getScrapeResult(jobId, options);
|
|
432
|
+
case "failed":
|
|
433
|
+
throw new CrawlbruleeError(error ?? `Async scrape job ${jobId} failed.`, {
|
|
434
|
+
status: 0,
|
|
435
|
+
errorName: "job_failed"
|
|
436
|
+
});
|
|
437
|
+
case "cancelled":
|
|
438
|
+
throw new CrawlbruleeError(`Async scrape job ${jobId} was cancelled.`, {
|
|
439
|
+
status: 0,
|
|
440
|
+
errorName: "client_closed_request"
|
|
441
|
+
});
|
|
442
|
+
default:
|
|
443
|
+
throw new CrawlbruleeError(
|
|
444
|
+
`Async scrape webhook for job ${jobId} carried an unexpected status '${String(status)}'.`,
|
|
445
|
+
{ status: 0, errorName: "validation_error" }
|
|
446
|
+
);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
399
449
|
/**
|
|
400
450
|
* Poll an async scrape job until it reaches a terminal state, then return
|
|
401
451
|
* the scrape result.
|
|
@@ -522,6 +572,117 @@ function sleep(ms, signal) {
|
|
|
522
572
|
});
|
|
523
573
|
}
|
|
524
574
|
|
|
525
|
-
|
|
575
|
+
// src/webhooks.ts
|
|
576
|
+
var WEBHOOK_SIGNATURE_HEADER = "X-Cwbl-Signature";
|
|
577
|
+
var WEBHOOK_SIGNATURE_ROTATED_HEADER = "X-Cwbl-Signature-Rotated";
|
|
578
|
+
var WEBHOOK_EVENT_ID_HEADER = "X-Cwbl-Event-Id";
|
|
579
|
+
var DEFAULT_WEBHOOK_TOLERANCE_SECONDS = 300;
|
|
580
|
+
var SIGNATURE_FORMAT = /^t=(\d+),v1=([0-9a-f]{64})$/;
|
|
581
|
+
async function verifyWebhookSignature(options) {
|
|
582
|
+
const { payload, headers, secret } = options;
|
|
583
|
+
const toleranceSeconds = options.toleranceSeconds ?? DEFAULT_WEBHOOK_TOLERANCE_SECONDS;
|
|
584
|
+
const primaryHeader = getHeader(headers, WEBHOOK_SIGNATURE_HEADER);
|
|
585
|
+
const rotatedHeader = getHeader(headers, WEBHOOK_SIGNATURE_ROTATED_HEADER);
|
|
586
|
+
if (primaryHeader === void 0 && rotatedHeader === void 0) {
|
|
587
|
+
return { verified: false, reason: "missing_signature" };
|
|
588
|
+
}
|
|
589
|
+
const nowSeconds = Math.floor(Date.now() / 1e3);
|
|
590
|
+
const body = toBytes(payload);
|
|
591
|
+
const key = await importHmacKey(secret);
|
|
592
|
+
let failure = "malformed_signature";
|
|
593
|
+
for (const source of ["primary", "rotated"]) {
|
|
594
|
+
const raw = source === "primary" ? primaryHeader : rotatedHeader;
|
|
595
|
+
if (raw === void 0) continue;
|
|
596
|
+
const parsed = parseSignatureHeader(raw);
|
|
597
|
+
if (!parsed) {
|
|
598
|
+
continue;
|
|
599
|
+
}
|
|
600
|
+
if (toleranceSeconds && Math.abs(nowSeconds - parsed.timestamp) > toleranceSeconds) {
|
|
601
|
+
failure = mostSpecificFailure(failure, "timestamp_out_of_tolerance");
|
|
602
|
+
continue;
|
|
603
|
+
}
|
|
604
|
+
const expected = await computeSignatureHex(key, parsed.timestamp, body);
|
|
605
|
+
if (constantTimeEqualHex(expected, parsed.signature)) {
|
|
606
|
+
return { verified: true, signedWith: source };
|
|
607
|
+
}
|
|
608
|
+
failure = mostSpecificFailure(failure, "signature_mismatch");
|
|
609
|
+
}
|
|
610
|
+
return { verified: false, reason: failure };
|
|
611
|
+
}
|
|
612
|
+
function mostSpecificFailure(current, candidate) {
|
|
613
|
+
const rank = {
|
|
614
|
+
missing_signature: 0,
|
|
615
|
+
malformed_signature: 1,
|
|
616
|
+
timestamp_out_of_tolerance: 2,
|
|
617
|
+
signature_mismatch: 3
|
|
618
|
+
};
|
|
619
|
+
return rank[candidate] > rank[current] ? candidate : current;
|
|
620
|
+
}
|
|
621
|
+
function getHeader(headers, name) {
|
|
622
|
+
if (typeof Headers !== "undefined" && headers instanceof Headers) {
|
|
623
|
+
return headers.get(name) ?? void 0;
|
|
624
|
+
}
|
|
625
|
+
const target = name.toLowerCase();
|
|
626
|
+
for (const key of Object.keys(headers)) {
|
|
627
|
+
if (key.toLowerCase() !== target) continue;
|
|
628
|
+
const value = headers[key];
|
|
629
|
+
if (Array.isArray(value)) return value[0];
|
|
630
|
+
return value ?? void 0;
|
|
631
|
+
}
|
|
632
|
+
return void 0;
|
|
633
|
+
}
|
|
634
|
+
function parseSignatureHeader(value) {
|
|
635
|
+
const match = SIGNATURE_FORMAT.exec(value.trim());
|
|
636
|
+
if (!match) return null;
|
|
637
|
+
const timestamp = Number(match[1]);
|
|
638
|
+
if (!Number.isSafeInteger(timestamp)) return null;
|
|
639
|
+
return { timestamp, signature: match[2] };
|
|
640
|
+
}
|
|
641
|
+
function toBytes(payload) {
|
|
642
|
+
return typeof payload === "string" ? new TextEncoder().encode(payload) : payload;
|
|
643
|
+
}
|
|
644
|
+
function importHmacKey(secret) {
|
|
645
|
+
return getSubtle().importKey(
|
|
646
|
+
"raw",
|
|
647
|
+
new TextEncoder().encode(secret),
|
|
648
|
+
{ name: "HMAC", hash: "SHA-256" },
|
|
649
|
+
false,
|
|
650
|
+
["sign"]
|
|
651
|
+
);
|
|
652
|
+
}
|
|
653
|
+
async function computeSignatureHex(key, timestamp, body) {
|
|
654
|
+
const prefix = new TextEncoder().encode(`${timestamp}.`);
|
|
655
|
+
const message = new Uint8Array(prefix.length + body.length);
|
|
656
|
+
message.set(prefix, 0);
|
|
657
|
+
message.set(body, prefix.length);
|
|
658
|
+
const digest = await getSubtle().sign("HMAC", key, message);
|
|
659
|
+
return toHex(new Uint8Array(digest));
|
|
660
|
+
}
|
|
661
|
+
function toHex(bytes) {
|
|
662
|
+
let hex = "";
|
|
663
|
+
for (const byte of bytes) {
|
|
664
|
+
hex += byte.toString(16).padStart(2, "0");
|
|
665
|
+
}
|
|
666
|
+
return hex;
|
|
667
|
+
}
|
|
668
|
+
function constantTimeEqualHex(a, b) {
|
|
669
|
+
if (a.length !== b.length) return false;
|
|
670
|
+
let diff = 0;
|
|
671
|
+
for (let i = 0; i < a.length; i++) {
|
|
672
|
+
diff |= a.charCodeAt(i) ^ b.charCodeAt(i);
|
|
673
|
+
}
|
|
674
|
+
return diff === 0;
|
|
675
|
+
}
|
|
676
|
+
function getSubtle() {
|
|
677
|
+
const subtle = globalThis.crypto?.subtle;
|
|
678
|
+
if (!subtle) {
|
|
679
|
+
throw new Error(
|
|
680
|
+
"Web Crypto (globalThis.crypto.subtle) is not available in this runtime. crawlbrulee webhook verification requires Node.js 20+, Bun, Deno, or a modern browser/edge runtime."
|
|
681
|
+
);
|
|
682
|
+
}
|
|
683
|
+
return subtle;
|
|
684
|
+
}
|
|
685
|
+
|
|
686
|
+
export { AuthenticationError, Crawlbrulee, CrawlbruleeError, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_WEBHOOK_TOLERANCE_SECONDS, ENV_API_KEY, NotFoundError, RateLimitError, TransportError, UsageAllocationError, ValidationError, WEBHOOK_EVENT_ID_HEADER, WEBHOOK_SIGNATURE_HEADER, WEBHOOK_SIGNATURE_ROTATED_HEADER, isCrawlbruleeError, verifyWebhookSignature };
|
|
526
687
|
//# sourceMappingURL=index.js.map
|
|
527
688
|
//# sourceMappingURL=index.js.map
|