@xberg-io/crawlberg 1.7.2 → 1.9.0

package/index.d.ts

@@ -211,6 +211,25 @@ export interface BrowserConfig {
   endpoint?: string
   /** Timeout for browser page load and rendering (in milliseconds when serialized). */
   timeout?: number
+  /**
+   * Overall deadline for a single browser fetch, covering browser launch (or
+   * page acquisition from a shared pool), page setup, navigation, rendering,
+   * and screenshot capture. Must exceed `timeout` to leave room for launch
+   * and setup overhead; a fetch that has not returned within this deadline
+   * fails with a timeout error (in milliseconds when serialized).
+   *
+   * Shutdown/teardown is governed separately by `shutdown_timeout` and is
+   * not counted against this deadline: an already-computed result is
+   * delivered to the caller without waiting for the browser process to
+   * exit.
+   */
+  overallTimeout?: number
+  /**
+   * How long to wait for the browser process to close and exit cleanly
+   * during teardown before the process is forcibly killed (in milliseconds
+   * when serialized).
+   */
+  shutdownTimeout?: number
   /** Wait strategy after browser navigation. */
   wait?: BrowserWait | 'network_idle' | 'selector' | 'fixed'
   /** CSS selector to wait for when `wait` is `Selector`. */
@@ -252,6 +271,28 @@ export interface BrowserConfig {
    * Default: true. When false, each request gets a fresh Page.
    */
   sessionAffinity?: boolean
+  /**
+   * Chrome or Chromium executable to launch. When set, crawlberg launches only this
+   * binary, and a path that is missing or not executable is an error that names the
+   * path; crawlberg never falls back to a different Chrome. When unset, crawlberg uses
+   * the `CHROME` environment variable, then searches the machine for an installed Chrome,
+   * Chromium or Edge. Chromiumoxide backend only: ignored, with a warning, when `endpoint`
+   * is set, with the native backend, and by scrapes and crawls that use a shared browser pool.
+   */
+  chromePath?: string
+  /**
+   * Extra Chrome command-line flags, each written as `--flag` or `--flag=value`, for example
+   * `--user-agent=...`. A flag here replaces a crawlberg default flag of the same name
+   * (`--lang=fr` replaces crawlberg's `--lang=en_US`). Rejected: an entry that does not
+   * start with `--`, a flag name with an uppercase letter (Chrome flag names are lowercase),
+   * a flag named twice, and `--headless`, `--remote-debugging-port` and `--user-data-dir`,
+   * which crawlberg sets itself to run Chrome. Set this only from trusted configuration,
+   * like `proxy`: flags such as `--proxy-server` and `--host-resolver-rules` send Chrome's
+   * traffic around the `ssrf` policy. Chromiumoxide backend only: ignored, with a warning,
+   * when `endpoint` is set, with the native backend, and by scrapes and crawls that use a
+   * shared browser pool.
+   */
+  chromeArgs?: Array<string>
 }
 
 export declare function browserConfigDefault(): BrowserConfig
@@ -388,6 +429,16 @@ export interface ContentConfig {
   wrapWidth?: number
   /** Include document structure tree in output. Default: `true`. */
   includeDocumentStructure?: boolean
+  /**
+   * Prepend a YAML frontmatter block (`title`, `description`, etc., extracted from
+   * `<head>`) to the markdown output. Default: `true`.
+   *
+   * This only controls the frontmatter text inside `markdown.content`. Crawlberg
+   * never reads `<head>` metadata back out of the converter's result -- `PageMetadata`
+   * is populated independently by `crate.html.metadata.extract_metadata` from the
+   * parsed DOM, so turning this off does not lose any metadata field.
+   */
+  extractMetadata?: boolean
 }
 
 export declare function contentConfigDefault(): ContentConfig
@@ -445,12 +496,21 @@ export interface CrawlConfig {
   bm25Query?: string
   /** Minimum BM25 score a page must reach to be kept. Defaults to `0.0`. */
   bm25Threshold?: number
-  /** Whether to respect robots.txt directives. */
+  /**
+   * Whether to respect robots.txt directives. A crawl that respects them also honours
+   * the page's own robots instructions: it does not follow the links of a page marked
+   * `nofollow` by its robots meta tag or an `X-Robots-Tag` header. A link marked
+   * `rel="nofollow"` is a hint, not a robots directive, and is still followed. A `noindex`
+   * page is still crawled and its links followed; the page result marks it with
+   * `noindex_detected`.
+   */
   respectRobotsTxt?: boolean
   /**
    * When true, HTTP-level error responses (404 NotFound, 403 Forbidden, WAF blocks)
    * are surfaced as `ScrapeResult` records with the matching `status_code` rather
-   * than raised as `CrawlError`. Default `false` preserves the historical
+   * than raised as `CrawlError`. A WAF block reports the status of the refused
+   * response when it is a 4xx or 5xx, and 403 when the block page came with a
+   * 2xx status. Default `false` preserves the historical
    * throw-on-error contract for direct fetches. Independently of this flag,
    * 404s reached at the end of a redirect chain are *always* surfaced softly —
    * the user opted into redirect-following, so receiving a 404 there is part of
@@ -459,15 +519,69 @@ export interface CrawlConfig {
   softHttpErrors?: boolean
   /** Custom user-agent string. */
   userAgent?: string
-  /** Whether to restrict crawling to the same domain. */
+  /**
+   * Whether to confine *document* links (`.pdf`, `.docx`, `.zip`, ...) to the seed domain.
+   *
+   * Page links are always confined to the seed host, widened to its subdomains by
+   * `Self.allow_subdomains`; this flag does not loosen that. It applies only to document
+   * links, which are classified by file extension before their host is considered and so
+   * are followed cross-host by default -- the usual case being documents served from a CDN
+   * or object store. Set this to `true` to require documents to live on the seed domain too.
+   */
   stayOnDomain?: boolean
-  /** Whether to allow subdomains when `stay_on_domain` is true. */
+  /**
+   * Whether subdomains of the seed host are in scope.
+   *
+   * Applies to page links unconditionally, and to document links when
+   * `Self.stay_on_domain` is set.
+   */
   allowSubdomains?: boolean
   /** Regex patterns for paths to include during crawling. */
   includePaths?: Array<string>
   /** Regex patterns for paths to exclude during crawling. */
   excludePaths?: Array<string>
-  /** Custom HTTP headers to send with each request. */
+  /**
+   * Whether `include_paths`/`exclude_paths` match against `path?query` instead of just
+   * `path`. Defaults to `false`, matching path only: a pattern anchored with `$` (e.g.
+   * `/feed/?$`) changes meaning once the query joins the matched text, so this must stay
+   * opt-in rather than silently changing what an existing config matches. Has no effect
+   * when `Self.path_patterns_match_url` is `true`.
+   */
+  pathPatternsMatchQuery?: boolean
+  /**
+   * Whether `include_paths`/`exclude_paths` match against the full URL,
+   * `scheme://host[:port]/path?query`, so a pattern can scope by host. Defaults to `false`.
+   * When `true` it takes precedence over `Self.path_patterns_match_query`: the query is
+   * part of the full URL whatever that flag says. The matched text never contains a
+   * `user:password@`, a fragment or a default port, and the host is in punycode
+   * (`bücher.de` is matched as `xn--bcher-kva.de`).
+   */
+  pathPatternsMatchUrl?: boolean
+  /**
+   * Whether the crawl-dedup key includes the (sorted) query string. Defaults to `false`,
+   * matching historical behavior: `/item?id=1` and `/item?id=2` are treated as one page and
+   * only the first is fetched. `true` keeps the query, sorted, in the key, so each distinct
+   * query is fetched once.
+   */
+  dedupIncludeQuery?: boolean
+  /**
+   * Whether to strip `tracking_params` from a discovered URL before it is deduplicated,
+   * fetched, and reported. Defaults to `false`, so no tracking parameters are stripped
+   * unless explicitly enabled.
+   */
+  stripTrackingParams?: boolean
+  /**
+   * Query parameter name patterns to strip when `strip_tracking_params` is `true`. A
+   * pattern ending in `*` matches by prefix (`utm_*` matches `utm_source`, `utm_campaign`,
+   * ...); any other pattern matches the parameter name exactly. Defaults to
+   * `["utm_*", "fbclid", "gclid", "ref"]`, applied only once `strip_tracking_params` is
+   * enabled.
+   */
+  trackingParams?: Array<string>
+  /**
+   * Custom HTTP headers to send with each request to the seed URL's host. A request to another host
+   * does not carry them.
+   */
   customHeaders?: Record<string, string>
   /** Timeout for individual HTTP requests (in milliseconds when serialized). */
   requestTimeout?: number
@@ -478,10 +592,27 @@ export interface CrawlConfig {
   rateLimitMs?: number
   /** Maximum number of redirects to follow. */
   maxRedirects?: number
-  /** Number of retry attempts for failed requests. */
+  /** Number of retry attempts for failed requests. Bounded by `MAX_RETRY_COUNT`. */
   retryCount?: number
-  /** HTTP status codes that should trigger a retry. */
+  /**
+   * HTTP status codes that should trigger a retry. When empty, every rate limit, server
+   * error, bad gateway and timeout is retried. When set, only a failure whose status is
+   * listed is retried, so a timeout without a response is not.
+   */
   retryCodes?: Array<number>
+  /**
+   * Initial delay, in milliseconds, before the first retry. Doubled on each
+   * subsequent attempt (capped at `retry_max_delay_ms`). Defaults to 100ms.
+   */
+  retryInitialDelayMs?: number
+  /** Upper bound, in milliseconds, on the exponential retry backoff. Defaults to 60s. */
+  retryMaxDelayMs?: number
+  /**
+   * Fraction of the per-domain rate-limit delay to randomly jitter by, in `[0.0, 1.0]`.
+   * `0.0` (the default) applies no jitter and preserves the previous fixed-interval
+   * behaviour; `0.1` jitters the delay by up to ±10%.
+   */
+  rateLimitJitterRatio?: number
   /** Whether to enable cookie handling. */
   cookiesEnabled?: boolean
   /** Authentication configuration. */
@@ -678,6 +809,29 @@ export interface CrawlPageResult {
   downloadedDocument?: JsDownloadedDocument
   /** Whether the browser fallback was used to fetch this page. */
   browserUsed?: boolean
+  /**
+   * The URL this page's content was actually fetched from, after following any HTTP,
+   * `Refresh` header, or `<meta http-equiv="refresh">` redirect `url` pointed at.
+   * Equal to `url` when the fetch did not redirect.
+   */
+  finalUrl?: string
+  /** Redirect hops taken to reach `final_url` from `url`. */
+  redirectCount?: number
+  /**
+   * Whether the page asked not to be indexed, by its robots meta tag or `X-Robots-Tag`
+   * header. The page is still crawled and its links still followed.
+   */
+  noindexDetected?: boolean
+  /**
+   * Whether the page asked that its links not be followed, by its robots meta tag or
+   * `X-Robots-Tag` header. When the crawl respects robots, its links are not followed.
+   */
+  nofollowDetected?: boolean
+  /**
+   * URLs of the requests the page sent in browser mode that the SSRF policy refused, without
+   * their credentials, each listed once. The page is kept; only the refused requests failed.
+   */
+  ssrfRefusedUrls?: Array<string>
 }
 
 /** The result of a multi-page crawl operation. */
@@ -943,6 +1097,11 @@ export interface InteractionResult {
    * callers that never request a screenshot do not pay the encoding cost.
    */
   screenshotBase64?: string
+  /**
+   * URLs of the requests the page sent during the session that the SSRF policy refused,
+   * including during the extra wait, without their credentials, each listed once.
+   */
+  ssrfRefusedUrls?: Array<string>
 }
 
 /** A JSON-LD structured data entry found on a page. */
@@ -1182,7 +1341,7 @@ export interface ScrapeResult {
   noindexDetected?: boolean
   /** Whether a nofollow directive was detected. */
   nofollowDetected?: boolean
-  /** The X-Robots-Tag header value, if present. */
+  /** The X-Robots-Tag header values, joined with `, ` when the response sent more than one. */
   xRobotsTag?: string
   /** Whether the content is a PDF. */
   isPdf?: boolean
@@ -1221,6 +1380,11 @@ export interface ScrapeResult {
    * populated when `BrowserBackend.Native` was used for this request.
    */
   browser?: BrowserExtras
+  /**
+   * URLs of the requests the page sent in browser mode that the SSRF policy refused, without
+   * their credentials, each listed once. The page is kept; only the refused requests failed.
+   */
+  ssrfRefusedUrls?: Array<string>
 }
 
 /** Direction for a scroll action. */

package/index.js

@@ -77,8 +77,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-android-arm64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-android-arm64/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -93,8 +93,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-android-arm-eabi')
         const bindingPackageVersion = require('@xberg-io/crawlberg-android-arm-eabi/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -114,8 +114,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-win32-x64-gnu')
         const bindingPackageVersion = require('@xberg-io/crawlberg-win32-x64-gnu/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -130,8 +130,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-win32-x64-msvc')
         const bindingPackageVersion = require('@xberg-io/crawlberg-win32-x64-msvc/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -147,8 +147,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-win32-ia32-msvc')
         const bindingPackageVersion = require('@xberg-io/crawlberg-win32-ia32-msvc/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -163,8 +163,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-win32-arm64-msvc')
         const bindingPackageVersion = require('@xberg-io/crawlberg-win32-arm64-msvc/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -182,8 +182,8 @@ function requireNative() {
     try {
       const binding = require('@xberg-io/crawlberg-darwin-universal')
       const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-universal/package.json').version
-      if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-        throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+      if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+        throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
       }
       return binding
     } catch (e) {
@@ -198,8 +198,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-darwin-x64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-x64/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -214,8 +214,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-darwin-arm64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-arm64/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -234,8 +234,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-freebsd-x64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-x64/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -250,8 +250,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-freebsd-arm64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-arm64/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -271,8 +271,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-x64-musl')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-x64-musl/package.json').version
-          if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -287,8 +287,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-x64-gnu')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-x64-gnu/package.json').version
-          if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -305,8 +305,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-arm64-musl')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm64-musl/package.json').version
-          if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -321,8 +321,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-arm64-gnu')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm64-gnu/package.json').version
-          if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -339,8 +339,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-arm-musleabihf')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm-musleabihf/package.json').version
-          if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -355,8 +355,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-arm-gnueabihf')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm-gnueabihf/package.json').version
-          if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -373,8 +373,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-loong64-musl')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-loong64-musl/package.json').version
-          if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -389,8 +389,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-loong64-gnu')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-loong64-gnu/package.json').version
-          if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -407,8 +407,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-riscv64-musl')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-riscv64-musl/package.json').version
-          if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -423,8 +423,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-riscv64-gnu')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-riscv64-gnu/package.json').version
-          if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -440,8 +440,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-linux-ppc64-gnu')
         const bindingPackageVersion = require('@xberg-io/crawlberg-linux-ppc64-gnu/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -456,8 +456,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-linux-s390x-gnu')
         const bindingPackageVersion = require('@xberg-io/crawlberg-linux-s390x-gnu/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -476,8 +476,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-openharmony-arm64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-arm64/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -492,8 +492,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-openharmony-x64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-x64/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -508,8 +508,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-openharmony-arm')
         const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-arm/package.json').version
-        if (bindingPackageVersion !== '1.7.2' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.9.0' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -648,8 +648,8 @@ if (!nativeBinding || forceWasi) {
       if (!candidateFailed) {
         if (process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
           const bindingPackageVersion = require('@xberg-io/crawlberg-wasm32-wasi/package.json').version
-          if (bindingPackageVersion !== '1.7.2') {
-            throw new Error(`WASI binding package version mismatch, expected 1.7.2 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.9.0') {
+            throw new Error(`WASI binding package version mismatch, expected 1.9.0 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
         }
         wasiBinding = require('@xberg-io/crawlberg-wasm32-wasi')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xberg-io/crawlberg",
-  "version": "1.7.2",
+  "version": "1.9.0",
   "description": "High-performance web crawling engine",
   "license": "MIT",
   "repository": {
@@ -22,12 +22,12 @@
     "*.node"
   ],
   "optionalDependencies": {
-    "@xberg-io/crawlberg-darwin-arm64": "1.7.2",
-    "@xberg-io/crawlberg-darwin-x64": "1.7.2",
-    "@xberg-io/crawlberg-linux-arm64-gnu": "1.7.2",
-    "@xberg-io/crawlberg-linux-x64-gnu": "1.7.2",
-    "@xberg-io/crawlberg-win32-arm64-msvc": "1.7.2",
-    "@xberg-io/crawlberg-win32-x64-msvc": "1.7.2"
+    "@xberg-io/crawlberg-darwin-arm64": "1.9.0",
+    "@xberg-io/crawlberg-darwin-x64": "1.9.0",
+    "@xberg-io/crawlberg-linux-arm64-gnu": "1.9.0",
+    "@xberg-io/crawlberg-linux-x64-gnu": "1.9.0",
+    "@xberg-io/crawlberg-win32-arm64-msvc": "1.9.0",
+    "@xberg-io/crawlberg-win32-x64-msvc": "1.9.0"
   },
   "napi": {
     "packageName": "@xberg-io/crawlberg",