@crawlbrulee/sdk 0.13.0 → 0.14.0

package/dist/index.d.cts

@@ -129,14 +129,6 @@ interface ResponseMeta {
 type ScreenshotType = 'viewport' | 'full_page';
 /** Emulated device class for the viewport (drives default width/height). */
 type ScreenshotDeviceMode = 'desktop' | 'mobile';
-/** Pre-capture cleanup options applied to the page before the screenshot. */
-interface ScreenshotCleanup {
-  /**
-   * Remove ads, cookie banners, and popups before capturing. Defaults to
-   * `true` server-side.
-   */
-  ads_and_popups?: boolean;
-}
 /** A `wait` action: pause for `ms` milliseconds before the next step. */
 interface ScreenshotWaitAction {
   type: 'wait';
@@ -185,8 +177,6 @@ interface ScreenshotRequest {
   viewport?: ScreenshotViewport;
   /** Emulate desktop or mobile. Defaults to `desktop`. */
   device_mode?: ScreenshotDeviceMode;
-  /** Page cleanup applied before capture. */
-  cleanup?: ScreenshotCleanup;
   /** Pre-capture actions (waits and scrolls). Maximum 5 entries. */
   actions_before?: ScreenshotBeforeAction[];
   /** Post-capture actions (e.g. slice into tiles). Maximum 1 entry. */
@@ -196,7 +186,7 @@ interface ScreenshotRequest {
  * Machine-readable error names returned by the crawlbrulee API. Stable
  * identifiers — clients can switch on them.
  */
-type ApiErrorName = 'usage_allocation_error' | 'request_timeout' | 'invalid_url' | 'url_too_long' | 'client_closed_request' | 'reset_password_token_expired' | 'user_not_found' | 'unsupported_url_schema' | 'url_credentials_not_supported' | 'blocked_url' | 'scrape_error' | 'job_failed' | 'incorrect_login_method_used' | 'not_found' | 'invalid_credentials' | 'resource_already_exists' | 'access_denied' | 'internal_server_error' | 'service_unavailable' | 'too_many_requests' | 'unsupported_content' | 'unsupported_screenshot_output' | 'validation_error' | 'antibot_blocked';
+type ApiErrorName = 'usage_allocation_error' | 'request_timeout' | 'invalid_url' | 'url_too_long' | 'client_closed_request' | 'reset_password_token_expired' | 'user_not_found' | 'unsupported_url_schema' | 'url_credentials_not_supported' | 'blocked_url' | 'scrape_error' | 'job_failed' | 'incorrect_login_method_used' | 'not_found' | 'invalid_credentials' | 'resource_already_exists' | 'access_denied' | 'internal_server_error' | 'service_unavailable' | 'too_many_requests' | 'unsupported_content' | 'unsupported_screenshot_output' | 'validation_error' | 'antibot_blocked' | 'too_many_redirects' | 'page_too_large';
 /** Reason a usage allocation was denied (when `error_name = usage_allocation_error`). */
 type UsageAllocationReason = 'credit_limit' | 'concurrency_limit' | 'duplicate_reservation' | 'internal_error';
 /** Snapshot of the org's current usage at the moment the error was raised. */
@@ -243,6 +233,10 @@ interface ApiErrorResponse {
  * Which content formats to extract from the scraped page. Every field is
  * optional; the server defaults are noted on each field. The default request
  * extracts `{ metadata: true, cleaned_html: true }`.
+ *
+ * `extract` only selects what is returned — it is not part of the cache key.
+ * Every scrape stores the whole page, so adding a field (e.g. `raw_html`) to an
+ * otherwise identical request is still a 0-credit cache hit.
  */
 interface ScrapeExtract {
   /** Extract page metadata (title, description, OG/Twitter tags, etc.). Default `true`. */
@@ -274,7 +268,37 @@ interface ScrapeExtract {
   /** Capture a screenshot. Omit to skip; set to a `ScreenshotRequest` to enable. */
   screenshot?: ScreenshotRequest;
 }
-/** Cache settings for a scrape request. */
+/**
+ * What is removed from the page before any output is built.
+ *
+ * Applies to `markdown`, `cleaned_html`, `links` and `images` on every engine,
+ * and to the screenshot. It never applies to `raw_html` — that is always the
+ * page as it arrived, before anything was removed.
+ */
+interface ScrapeCleanup {
+  /**
+   * Remove ads, cookie banners, consent dialogs and chat widgets. Defaults to
+   * `true` server-side. Set it to `false` to capture the page as-is, or to get
+   * past a site that refuses to serve content to an ad-blocking client.
+   */
+  ads_and_popups?: boolean;
+  /**
+   * CSS selectors whose elements are removed before anything is captured. Use
+   * it for a banner or widget `ads_and_popups` does not recognise.
+   *
+   * At most 100 selectors, each at most 500 characters. Sending any selector
+   * here makes the request skip the cache, so it always costs a live fetch.
+   */
+  exclude_selectors?: string[];
+}
+/**
+ * Cache settings for a scrape request. A cached result is keyed on the url
+ * (after tracking-param cleanup), the screenshot setup (type, viewport, device
+ * mode), `cleanup.ads_and_popups` and `location.locale`. `extract` is not part
+ * of the key. `require_js: true` only matches browser-rendered results.
+ * `cleanup.exclude_selectors` and a non-zero `actions_before` wait or scroll
+ * disable the cache for that request.
+ */
 interface ScrapeCache {
   /**
    * Maximum cache age. Either a number of seconds (non-negative integer) or
@@ -314,8 +338,14 @@ interface ScrapeRequest {
    * and credits — only enable when the page requires it. Default `false`.
    */
   require_js?: boolean;
-  /** CSS selectors to strip from the extracted content. */
-  exclude_selectors?: string[];
+  /**
+   * What is removed from the page before any output is built. Shapes
+   * `markdown`, `cleaned_html`, `links`, `images` and the screenshot.
+   *
+   * Never applies to `raw_html`, which is always the page before anything was
+   * removed.
+   */
+  cleanup?: ScrapeCleanup;
   /**
    * Proxy tier to use for fetching. Defaults to `auto` (tries the basic tier
    * first, escalates to advanced on failure; billed at the delivered tier).
@@ -542,7 +572,13 @@ interface ScrapeResponse {
 //#region src/types/map.d.ts
 /** Filter which link types appear in the map result. */
 interface MapTypes {
-  /** Include internal links (same domain). Default `true`. */
+  /**
+   * Include internal links (same domain; `www.` and the bare domain count as
+   * the same site). Default `true`.
+   *
+   * Note this is only about *classifying* a link — the URLs that come back keep
+   * the host exactly as the site publishes it. See {@link MapLinkItem.url}.
+   */
   internal?: boolean;
   /** Include links to subdomains of the target. Default `true`. */
   internal_subdomains?: boolean;
@@ -578,14 +614,20 @@ interface MapRequest {
   /** Cache settings for this request. */
   cache?: MapCache;
   /**
-   * Maximum number of URLs to store in the map. Must be in `(0, 100 000]`.
-   * Defaults to 100 000.
+   * Maximum number of URLs to discover and store in the map. Must be in
+   * `(0, 100 000]`. Defaults to 5 000.
+   *
+   * Sitemap discovery stops as soon as this many URLs have been found, so a
+   * smaller value is a cheaper and faster crawl, not just a shorter answer. A
+   * map that stopped this way comes back with exactly `max_urls` links and
+   * {@link MapTruncation.discovery_cap_reason} set to `max_urls` — ask again
+   * with a higher `max_urls` to get more.
    */
   max_urls?: number;
   /** 1-based page number for paginated results. Defaults to 1. */
   page?: number;
   /**
-   * Number of URLs per page. Must be in `(0, 10 000]`. Defaults to 10 000.
+   * Number of URLs per page. Must be in `(0, 10 000]`. Defaults to 5 000.
    */
   limit?: number;
   /** Optional country emulation. */
@@ -593,7 +635,12 @@ interface MapRequest {
 }
 /** Single discovered URL in a map result. */
 interface MapLinkItem {
-  /** The discovered URL. */
+  /**
+   * The discovered URL, normalised the same way `/scrape` normalises the `url`
+   * it returns: the host is kept as the site publishes it (a `www.` host stays
+   * `www.`), the query string is sorted with known tracking parameters
+   * stripped, and a trailing slash is dropped.
+   */
   url: string;
 }
 /** Pagination details on a map response. */
@@ -605,16 +652,50 @@ interface MapPagination {
   total_pages: number;
   has_more: boolean;
 }
+/**
+ * Which limit stopped sitemap discovery first, or `null` when nothing stopped
+ * it.
+ *
+ * - `max_urls` — your own {@link MapRequest.max_urls} was reached. This is the
+ *   only reason you can do something about: ask again with a higher one.
+ * - `time` — discovery ran out of its time budget.
+ * - `file_budget` — the site has more sitemap files than one request reads.
+ * - `depth` — the site's sitemap indexes nest too deeply.
+ * - `file_size` — a sitemap file was too large to read.
+ */
+type MapDiscoveryCapReason = 'max_urls' | 'time' | 'file_budget' | 'depth' | 'file_size';
 /** Information about whether the stored or returned map was truncated. */
 interface MapTruncation {
-  /** Whether the stored map was capped by `max_urls`. */
+  /** Whether the stored map hit the 100 000-URL storage cap. */
   storage_capped: boolean;
-  /** Whether the response was capped by pagination. */
+  /**
+   * Whether more links were eligible than `max_urls`, so the list was trimmed.
+   * Discovery itself stops at `max_urls`, so this is normally `true` only when
+   * home-page links pushed the total past it. A map that ran into the
+   * `max_urls` limit during discovery reports `response_capped: false` and
+   * signals the stop through {@link discovery_cap_reason} instead.
+   */
   response_capped: boolean;
   /** Total URLs found before the `max_urls` cap was applied. */
   total_before_max_urls: number;
   /** Total URLs detected during discovery before the storage cap was applied. */
   total_detected_before_storage_cap: number;
+  /**
+   * Whether sitemap discovery stopped before it had read every sitemap file it
+   * found. When `true`, the site has more pages than this map lists.
+   */
+  discovery_capped: boolean;
+  /**
+   * How many sitemap files were skipped or only partly read during discovery,
+   * because a file was too large, could not be fetched, or a discovery limit
+   * was reached. Integer `>= 0`.
+   */
+  sitemaps_skipped: number;
+  /**
+   * Which limit stopped sitemap discovery first, or `null` when nothing did.
+   * Only `max_urls` is something you can change from the request.
+   */
+  discovery_cap_reason: MapDiscoveryCapReason | null;
 }
 /** Billing engine reported by map: fresh discovery or a cached result. */
 type MapBillingEngine = 'text' | 'cache';
@@ -638,7 +719,14 @@ interface MapResponseMeta {
 }
 /** Success response from `POST /api/map`. */
 interface MapResponse {
-  /** The current page of discovered URLs. */
+  /**
+   * The current page of discovered URLs.
+   *
+   * Ordering is stable: link type first, then home-page links before
+   * sitemap-only links, then shallower paths before deeper ones, then
+   * alphabetical. Where a URL came from drives that ordering server-side and is
+   * deliberately not returned.
+   */
   links: MapLinkItem[];
   /** Usage, pagination, and truncation metadata for the result set. */
   response_meta: MapResponseMeta;
@@ -947,7 +1035,14 @@ declare class CrawlbruleeError extends Error {
     cause?: unknown;
   });
 }
-/** Raised for 401 / 403 responses (missing, invalid, or unauthorized API key). */
+/**
+ * Raised when the API rejects your credentials — a missing, invalid, or
+ * unauthorized API key (`invalid_credentials`, `access_denied`).
+ *
+ * Not every 403 is a key problem: a 403 carrying `antibot_blocked` is the
+ * *target site* blocking us and raises {@link AntibotBlockedError} instead.
+ * Only an unrecognized 403 name falls back to this class.
+ */
 declare class AuthenticationError extends CrawlbruleeError {
   constructor(message: string, options: {
     status: number;
@@ -955,6 +1050,46 @@ declare class AuthenticationError extends CrawlbruleeError {
     response?: ApiErrorResponse;
   });
 }
+/**
+ * Raised when the target site's anti-bot protection blocked the request
+ * (HTTP 403, `antibot_blocked`). Not an API-key problem — retrying the same
+ * tier rarely helps; try a higher proxy tier or skip the site.
+ */
+declare class AntibotBlockedError extends CrawlbruleeError {
+  constructor(message: string, options: {
+    status: number;
+    errorName: ApiErrorName;
+    response?: ApiErrorResponse;
+  });
+}
+/**
+ * Raised when the target site redirected the request in a loop, or through
+ * more hops than the API follows (HTTP 422, `too_many_redirects`). Like
+ * {@link AntibotBlockedError} this is the target's doing — not a key problem
+ * and not a bad request — so it is neither an `AuthenticationError` nor a
+ * `ValidationError`. Retrying rarely helps. Returned by both `scrape` and `map`.
+ */
+declare class TooManyRedirectsError extends CrawlbruleeError {
+  constructor(message: string, options: {
+    status: number;
+    errorName: ApiErrorName;
+    response?: ApiErrorResponse;
+  });
+}
+/**
+ * Raised when the page's HTML was too large to process (HTTP 422,
+ * `page_too_large`). Like {@link TooManyRedirectsError} this is about the page,
+ * not your request — so it is neither an `AuthenticationError` nor a
+ * `ValidationError`. It is terminal: the same URL fails the same way, so do not
+ * retry it. Returned by `scrape`.
+ */
+declare class PageTooLargeError extends CrawlbruleeError {
+  constructor(message: string, options: {
+    status: number;
+    errorName: ApiErrorName;
+    response?: ApiErrorResponse;
+  });
+}
 /**
  * Raised for HTTP 429 responses. When the server included a `retry_after_ms`
  * hint in `details` it is surfaced directly on the instance.
@@ -1149,5 +1284,5 @@ interface VerifyWebhookSignatureOptions {
  */
 declare function verifyWebhookSignature(options: VerifyWebhookSignatureOptions): Promise<WebhookVerificationResult>;
 //#endregion
-export { ApiErrorDetails, ApiErrorName, ApiErrorResponse, AsyncJobStatus, AsyncJobStatusResponse, AsyncScrapeRequest, AsyncScrapeResponse, AsyncScrapeWebhook, AuthenticationError, BillingEngine, Crawlbrulee, CrawlbruleeError, type CrawlbruleeOptions, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_WEBHOOK_TOLERANCE_SECONDS, ENV_API_KEY, type HttpMethod, MapBillingEngine, MapCache, MapLinkItem, MapLocation, MapPagination, MapRequest, MapResponse, MapResponseMeta, MapTruncation, MapTypes, MapUsage, NotFoundError, PageInlineImage, PageLink, ProxyTier, RateLimitError, RateLimitErrorDetails, type RequestOptions, ResolvedProxyTier, ResponseMeta, ScrapeCache, ScrapeCompleteWebhook, ScrapeCompleteWebhookData, ScrapeExtract, ScrapeLocation, ScrapeMetadata, ScrapeRequest, ScrapeResponse, ScrapeWarningCode, ScrapeWebhookStatus, ScreenshotAfterAction, ScreenshotBeforeAction, ScreenshotCleanup, ScreenshotDeviceMode, ScreenshotProperties, ScreenshotRequest, ScreenshotResult, ScreenshotScrollAction, ScreenshotSlice, ScreenshotSliceAction, ScreenshotType, ScreenshotViewport, ScreenshotViewportInfo, ScreenshotWaitAction, ServiceUnavailableError, TransportError, Usage, UsageAllocationError, UsageAllocationErrorDetails, UsageAllocationReason, UsageLimitDetails, UsageResponse, ValidationError, type VerifyWebhookSignatureOptions, WEBHOOK_EVENT_ID_HEADER, WEBHOOK_SIGNATURE_HEADER, WEBHOOK_SIGNATURE_ROTATED_HEADER, type WaitForScrapeOptions, type WebhookSignatureSource, type WebhookVerificationFailureReason, type WebhookVerificationResult, WhoamiResponse, isCrawlbruleeError, verifyWebhookSignature };
+export { AntibotBlockedError, ApiErrorDetails, ApiErrorName, ApiErrorResponse, AsyncJobStatus, AsyncJobStatusResponse, AsyncScrapeRequest, AsyncScrapeResponse, AsyncScrapeWebhook, AuthenticationError, BillingEngine, Crawlbrulee, CrawlbruleeError, type CrawlbruleeOptions, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_WEBHOOK_TOLERANCE_SECONDS, ENV_API_KEY, type HttpMethod, MapBillingEngine, MapCache, MapDiscoveryCapReason, MapLinkItem, MapLocation, MapPagination, MapRequest, MapResponse, MapResponseMeta, MapTruncation, MapTypes, MapUsage, NotFoundError, PageInlineImage, PageLink, PageTooLargeError, ProxyTier, RateLimitError, RateLimitErrorDetails, type RequestOptions, ResolvedProxyTier, ResponseMeta, ScrapeCache, ScrapeCleanup, ScrapeCompleteWebhook, ScrapeCompleteWebhookData, ScrapeExtract, ScrapeLocation, ScrapeMetadata, ScrapeRequest, ScrapeResponse, ScrapeWarningCode, ScrapeWebhookStatus, ScreenshotAfterAction, ScreenshotBeforeAction, ScreenshotDeviceMode, ScreenshotProperties, ScreenshotRequest, ScreenshotResult, ScreenshotScrollAction, ScreenshotSlice, ScreenshotSliceAction, ScreenshotType, ScreenshotViewport, ScreenshotViewportInfo, ScreenshotWaitAction, ServiceUnavailableError, TooManyRedirectsError, TransportError, Usage, UsageAllocationError, UsageAllocationErrorDetails, UsageAllocationReason, UsageLimitDetails, UsageResponse, ValidationError, type VerifyWebhookSignatureOptions, WEBHOOK_EVENT_ID_HEADER, WEBHOOK_SIGNATURE_HEADER, WEBHOOK_SIGNATURE_ROTATED_HEADER, type WaitForScrapeOptions, type WebhookSignatureSource, type WebhookVerificationFailureReason, type WebhookVerificationResult, WhoamiResponse, isCrawlbruleeError, verifyWebhookSignature };
 //# sourceMappingURL=index.d.cts.map

package/dist/index.d.ts

@@ -129,14 +129,6 @@ interface ResponseMeta {
 type ScreenshotType = 'viewport' | 'full_page';
 /** Emulated device class for the viewport (drives default width/height). */
 type ScreenshotDeviceMode = 'desktop' | 'mobile';
-/** Pre-capture cleanup options applied to the page before the screenshot. */
-interface ScreenshotCleanup {
-  /**
-   * Remove ads, cookie banners, and popups before capturing. Defaults to
-   * `true` server-side.
-   */
-  ads_and_popups?: boolean;
-}
 /** A `wait` action: pause for `ms` milliseconds before the next step. */
 interface ScreenshotWaitAction {
   type: 'wait';
@@ -185,8 +177,6 @@ interface ScreenshotRequest {
   viewport?: ScreenshotViewport;
   /** Emulate desktop or mobile. Defaults to `desktop`. */
   device_mode?: ScreenshotDeviceMode;
-  /** Page cleanup applied before capture. */
-  cleanup?: ScreenshotCleanup;
   /** Pre-capture actions (waits and scrolls). Maximum 5 entries. */
   actions_before?: ScreenshotBeforeAction[];
   /** Post-capture actions (e.g. slice into tiles). Maximum 1 entry. */
@@ -196,7 +186,7 @@ interface ScreenshotRequest {
  * Machine-readable error names returned by the crawlbrulee API. Stable
  * identifiers — clients can switch on them.
  */
-type ApiErrorName = 'usage_allocation_error' | 'request_timeout' | 'invalid_url' | 'url_too_long' | 'client_closed_request' | 'reset_password_token_expired' | 'user_not_found' | 'unsupported_url_schema' | 'url_credentials_not_supported' | 'blocked_url' | 'scrape_error' | 'job_failed' | 'incorrect_login_method_used' | 'not_found' | 'invalid_credentials' | 'resource_already_exists' | 'access_denied' | 'internal_server_error' | 'service_unavailable' | 'too_many_requests' | 'unsupported_content' | 'unsupported_screenshot_output' | 'validation_error' | 'antibot_blocked';
+type ApiErrorName = 'usage_allocation_error' | 'request_timeout' | 'invalid_url' | 'url_too_long' | 'client_closed_request' | 'reset_password_token_expired' | 'user_not_found' | 'unsupported_url_schema' | 'url_credentials_not_supported' | 'blocked_url' | 'scrape_error' | 'job_failed' | 'incorrect_login_method_used' | 'not_found' | 'invalid_credentials' | 'resource_already_exists' | 'access_denied' | 'internal_server_error' | 'service_unavailable' | 'too_many_requests' | 'unsupported_content' | 'unsupported_screenshot_output' | 'validation_error' | 'antibot_blocked' | 'too_many_redirects' | 'page_too_large';
 /** Reason a usage allocation was denied (when `error_name = usage_allocation_error`). */
 type UsageAllocationReason = 'credit_limit' | 'concurrency_limit' | 'duplicate_reservation' | 'internal_error';
 /** Snapshot of the org's current usage at the moment the error was raised. */
@@ -243,6 +233,10 @@ interface ApiErrorResponse {
  * Which content formats to extract from the scraped page. Every field is
  * optional; the server defaults are noted on each field. The default request
  * extracts `{ metadata: true, cleaned_html: true }`.
+ *
+ * `extract` only selects what is returned — it is not part of the cache key.
+ * Every scrape stores the whole page, so adding a field (e.g. `raw_html`) to an
+ * otherwise identical request is still a 0-credit cache hit.
  */
 interface ScrapeExtract {
   /** Extract page metadata (title, description, OG/Twitter tags, etc.). Default `true`. */
@@ -274,7 +268,37 @@ interface ScrapeExtract {
   /** Capture a screenshot. Omit to skip; set to a `ScreenshotRequest` to enable. */
   screenshot?: ScreenshotRequest;
 }
-/** Cache settings for a scrape request. */
+/**
+ * What is removed from the page before any output is built.
+ *
+ * Applies to `markdown`, `cleaned_html`, `links` and `images` on every engine,
+ * and to the screenshot. It never applies to `raw_html` — that is always the
+ * page as it arrived, before anything was removed.
+ */
+interface ScrapeCleanup {
+  /**
+   * Remove ads, cookie banners, consent dialogs and chat widgets. Defaults to
+   * `true` server-side. Set it to `false` to capture the page as-is, or to get
+   * past a site that refuses to serve content to an ad-blocking client.
+   */
+  ads_and_popups?: boolean;
+  /**
+   * CSS selectors whose elements are removed before anything is captured. Use
+   * it for a banner or widget `ads_and_popups` does not recognise.
+   *
+   * At most 100 selectors, each at most 500 characters. Sending any selector
+   * here makes the request skip the cache, so it always costs a live fetch.
+   */
+  exclude_selectors?: string[];
+}
+/**
+ * Cache settings for a scrape request. A cached result is keyed on the url
+ * (after tracking-param cleanup), the screenshot setup (type, viewport, device
+ * mode), `cleanup.ads_and_popups` and `location.locale`. `extract` is not part
+ * of the key. `require_js: true` only matches browser-rendered results.
+ * `cleanup.exclude_selectors` and a non-zero `actions_before` wait or scroll
+ * disable the cache for that request.
+ */
 interface ScrapeCache {
   /**
    * Maximum cache age. Either a number of seconds (non-negative integer) or
@@ -314,8 +338,14 @@ interface ScrapeRequest {
    * and credits — only enable when the page requires it. Default `false`.
    */
   require_js?: boolean;
-  /** CSS selectors to strip from the extracted content. */
-  exclude_selectors?: string[];
+  /**
+   * What is removed from the page before any output is built. Shapes
+   * `markdown`, `cleaned_html`, `links`, `images` and the screenshot.
+   *
+   * Never applies to `raw_html`, which is always the page before anything was
+   * removed.
+   */
+  cleanup?: ScrapeCleanup;
   /**
    * Proxy tier to use for fetching. Defaults to `auto` (tries the basic tier
    * first, escalates to advanced on failure; billed at the delivered tier).
@@ -542,7 +572,13 @@ interface ScrapeResponse {
 //#region src/types/map.d.ts
 /** Filter which link types appear in the map result. */
 interface MapTypes {
-  /** Include internal links (same domain). Default `true`. */
+  /**
+   * Include internal links (same domain; `www.` and the bare domain count as
+   * the same site). Default `true`.
+   *
+   * Note this is only about *classifying* a link — the URLs that come back keep
+   * the host exactly as the site publishes it. See {@link MapLinkItem.url}.
+   */
   internal?: boolean;
   /** Include links to subdomains of the target. Default `true`. */
   internal_subdomains?: boolean;
@@ -578,14 +614,20 @@ interface MapRequest {
   /** Cache settings for this request. */
   cache?: MapCache;
   /**
-   * Maximum number of URLs to store in the map. Must be in `(0, 100 000]`.
-   * Defaults to 100 000.
+   * Maximum number of URLs to discover and store in the map. Must be in
+   * `(0, 100 000]`. Defaults to 5 000.
+   *
+   * Sitemap discovery stops as soon as this many URLs have been found, so a
+   * smaller value is a cheaper and faster crawl, not just a shorter answer. A
+   * map that stopped this way comes back with exactly `max_urls` links and
+   * {@link MapTruncation.discovery_cap_reason} set to `max_urls` — ask again
+   * with a higher `max_urls` to get more.
    */
   max_urls?: number;
   /** 1-based page number for paginated results. Defaults to 1. */
   page?: number;
   /**
-   * Number of URLs per page. Must be in `(0, 10 000]`. Defaults to 10 000.
+   * Number of URLs per page. Must be in `(0, 10 000]`. Defaults to 5 000.
    */
   limit?: number;
   /** Optional country emulation. */
@@ -593,7 +635,12 @@ interface MapRequest {
 }
 /** Single discovered URL in a map result. */
 interface MapLinkItem {
-  /** The discovered URL. */
+  /**
+   * The discovered URL, normalised the same way `/scrape` normalises the `url`
+   * it returns: the host is kept as the site publishes it (a `www.` host stays
+   * `www.`), the query string is sorted with known tracking parameters
+   * stripped, and a trailing slash is dropped.
+   */
   url: string;
 }
 /** Pagination details on a map response. */
@@ -605,16 +652,50 @@ interface MapPagination {
   total_pages: number;
   has_more: boolean;
 }
+/**
+ * Which limit stopped sitemap discovery first, or `null` when nothing stopped
+ * it.
+ *
+ * - `max_urls` — your own {@link MapRequest.max_urls} was reached. This is the
+ *   only reason you can do something about: ask again with a higher one.
+ * - `time` — discovery ran out of its time budget.
+ * - `file_budget` — the site has more sitemap files than one request reads.
+ * - `depth` — the site's sitemap indexes nest too deeply.
+ * - `file_size` — a sitemap file was too large to read.
+ */
+type MapDiscoveryCapReason = 'max_urls' | 'time' | 'file_budget' | 'depth' | 'file_size';
 /** Information about whether the stored or returned map was truncated. */
 interface MapTruncation {
-  /** Whether the stored map was capped by `max_urls`. */
+  /** Whether the stored map hit the 100 000-URL storage cap. */
   storage_capped: boolean;
-  /** Whether the response was capped by pagination. */
+  /**
+   * Whether more links were eligible than `max_urls`, so the list was trimmed.
+   * Discovery itself stops at `max_urls`, so this is normally `true` only when
+   * home-page links pushed the total past it. A map that ran into the
+   * `max_urls` limit during discovery reports `response_capped: false` and
+   * signals the stop through {@link discovery_cap_reason} instead.
+   */
   response_capped: boolean;
   /** Total URLs found before the `max_urls` cap was applied. */
   total_before_max_urls: number;
   /** Total URLs detected during discovery before the storage cap was applied. */
   total_detected_before_storage_cap: number;
+  /**
+   * Whether sitemap discovery stopped before it had read every sitemap file it
+   * found. When `true`, the site has more pages than this map lists.
+   */
+  discovery_capped: boolean;
+  /**
+   * How many sitemap files were skipped or only partly read during discovery,
+   * because a file was too large, could not be fetched, or a discovery limit
+   * was reached. Integer `>= 0`.
+   */
+  sitemaps_skipped: number;
+  /**
+   * Which limit stopped sitemap discovery first, or `null` when nothing did.
+   * Only `max_urls` is something you can change from the request.
+   */
+  discovery_cap_reason: MapDiscoveryCapReason | null;
 }
 /** Billing engine reported by map: fresh discovery or a cached result. */
 type MapBillingEngine = 'text' | 'cache';
@@ -638,7 +719,14 @@ interface MapResponseMeta {
 }
 /** Success response from `POST /api/map`. */
 interface MapResponse {
-  /** The current page of discovered URLs. */
+  /**
+   * The current page of discovered URLs.
+   *
+   * Ordering is stable: link type first, then home-page links before
+   * sitemap-only links, then shallower paths before deeper ones, then
+   * alphabetical. Where a URL came from drives that ordering server-side and is
+   * deliberately not returned.
+   */
   links: MapLinkItem[];
   /** Usage, pagination, and truncation metadata for the result set. */
   response_meta: MapResponseMeta;
@@ -947,7 +1035,14 @@ declare class CrawlbruleeError extends Error {
     cause?: unknown;
   });
 }
-/** Raised for 401 / 403 responses (missing, invalid, or unauthorized API key). */
+/**
+ * Raised when the API rejects your credentials — a missing, invalid, or
+ * unauthorized API key (`invalid_credentials`, `access_denied`).
+ *
+ * Not every 403 is a key problem: a 403 carrying `antibot_blocked` is the
+ * *target site* blocking us and raises {@link AntibotBlockedError} instead.
+ * Only an unrecognized 403 name falls back to this class.
+ */
 declare class AuthenticationError extends CrawlbruleeError {
   constructor(message: string, options: {
     status: number;
@@ -955,6 +1050,46 @@ declare class AuthenticationError extends CrawlbruleeError {
     response?: ApiErrorResponse;
   });
 }
+/**
+ * Raised when the target site's anti-bot protection blocked the request
+ * (HTTP 403, `antibot_blocked`). Not an API-key problem — retrying the same
+ * tier rarely helps; try a higher proxy tier or skip the site.
+ */
+declare class AntibotBlockedError extends CrawlbruleeError {
+  constructor(message: string, options: {
+    status: number;
+    errorName: ApiErrorName;
+    response?: ApiErrorResponse;
+  });
+}
+/**
+ * Raised when the target site redirected the request in a loop, or through
+ * more hops than the API follows (HTTP 422, `too_many_redirects`). Like
+ * {@link AntibotBlockedError} this is the target's doing — not a key problem
+ * and not a bad request — so it is neither an `AuthenticationError` nor a
+ * `ValidationError`. Retrying rarely helps. Returned by both `scrape` and `map`.
+ */
+declare class TooManyRedirectsError extends CrawlbruleeError {
+  constructor(message: string, options: {
+    status: number;
+    errorName: ApiErrorName;
+    response?: ApiErrorResponse;
+  });
+}
+/**
+ * Raised when the page's HTML was too large to process (HTTP 422,
+ * `page_too_large`). Like {@link TooManyRedirectsError} this is about the page,
+ * not your request — so it is neither an `AuthenticationError` nor a
+ * `ValidationError`. It is terminal: the same URL fails the same way, so do not
+ * retry it. Returned by `scrape`.
+ */
+declare class PageTooLargeError extends CrawlbruleeError {
+  constructor(message: string, options: {
+    status: number;
+    errorName: ApiErrorName;
+    response?: ApiErrorResponse;
+  });
+}
 /**
  * Raised for HTTP 429 responses. When the server included a `retry_after_ms`
  * hint in `details` it is surfaced directly on the instance.
@@ -1149,5 +1284,5 @@ interface VerifyWebhookSignatureOptions {
  */
 declare function verifyWebhookSignature(options: VerifyWebhookSignatureOptions): Promise<WebhookVerificationResult>;
 //#endregion
-export { ApiErrorDetails, ApiErrorName, ApiErrorResponse, AsyncJobStatus, AsyncJobStatusResponse, AsyncScrapeRequest, AsyncScrapeResponse, AsyncScrapeWebhook, AuthenticationError, BillingEngine, Crawlbrulee, CrawlbruleeError, type CrawlbruleeOptions, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_WEBHOOK_TOLERANCE_SECONDS, ENV_API_KEY, type HttpMethod, MapBillingEngine, MapCache, MapLinkItem, MapLocation, MapPagination, MapRequest, MapResponse, MapResponseMeta, MapTruncation, MapTypes, MapUsage, NotFoundError, PageInlineImage, PageLink, ProxyTier, RateLimitError, RateLimitErrorDetails, type RequestOptions, ResolvedProxyTier, ResponseMeta, ScrapeCache, ScrapeCompleteWebhook, ScrapeCompleteWebhookData, ScrapeExtract, ScrapeLocation, ScrapeMetadata, ScrapeRequest, ScrapeResponse, ScrapeWarningCode, ScrapeWebhookStatus, ScreenshotAfterAction, ScreenshotBeforeAction, ScreenshotCleanup, ScreenshotDeviceMode, ScreenshotProperties, ScreenshotRequest, ScreenshotResult, ScreenshotScrollAction, ScreenshotSlice, ScreenshotSliceAction, ScreenshotType, ScreenshotViewport, ScreenshotViewportInfo, ScreenshotWaitAction, ServiceUnavailableError, TransportError, Usage, UsageAllocationError, UsageAllocationErrorDetails, UsageAllocationReason, UsageLimitDetails, UsageResponse, ValidationError, type VerifyWebhookSignatureOptions, WEBHOOK_EVENT_ID_HEADER, WEBHOOK_SIGNATURE_HEADER, WEBHOOK_SIGNATURE_ROTATED_HEADER, type WaitForScrapeOptions, type WebhookSignatureSource, type WebhookVerificationFailureReason, type WebhookVerificationResult, WhoamiResponse, isCrawlbruleeError, verifyWebhookSignature };
+export { AntibotBlockedError, ApiErrorDetails, ApiErrorName, ApiErrorResponse, AsyncJobStatus, AsyncJobStatusResponse, AsyncScrapeRequest, AsyncScrapeResponse, AsyncScrapeWebhook, AuthenticationError, BillingEngine, Crawlbrulee, CrawlbruleeError, type CrawlbruleeOptions, DEFAULT_BASE_URL, DEFAULT_REQUEST_TIMEOUT_MS, DEFAULT_WEBHOOK_TOLERANCE_SECONDS, ENV_API_KEY, type HttpMethod, MapBillingEngine, MapCache, MapDiscoveryCapReason, MapLinkItem, MapLocation, MapPagination, MapRequest, MapResponse, MapResponseMeta, MapTruncation, MapTypes, MapUsage, NotFoundError, PageInlineImage, PageLink, PageTooLargeError, ProxyTier, RateLimitError, RateLimitErrorDetails, type RequestOptions, ResolvedProxyTier, ResponseMeta, ScrapeCache, ScrapeCleanup, ScrapeCompleteWebhook, ScrapeCompleteWebhookData, ScrapeExtract, ScrapeLocation, ScrapeMetadata, ScrapeRequest, ScrapeResponse, ScrapeWarningCode, ScrapeWebhookStatus, ScreenshotAfterAction, ScreenshotBeforeAction, ScreenshotDeviceMode, ScreenshotProperties, ScreenshotRequest, ScreenshotResult, ScreenshotScrollAction, ScreenshotSlice, ScreenshotSliceAction, ScreenshotType, ScreenshotViewport, ScreenshotViewportInfo, ScreenshotWaitAction, ServiceUnavailableError, TooManyRedirectsError, TransportError, Usage, UsageAllocationError, UsageAllocationErrorDetails, UsageAllocationReason, UsageLimitDetails, UsageResponse, ValidationError, type VerifyWebhookSignatureOptions, WEBHOOK_EVENT_ID_HEADER, WEBHOOK_SIGNATURE_HEADER, WEBHOOK_SIGNATURE_ROTATED_HEADER, type WaitForScrapeOptions, type WebhookSignatureSource, type WebhookVerificationFailureReason, type WebhookVerificationResult, WhoamiResponse, isCrawlbruleeError, verifyWebhookSignature };
 //# sourceMappingURL=index.d.ts.map