@xberg-io/crawlberg 1.1.4 → 1.2.1

package/index.d.ts

@@ -357,7 +357,7 @@ export interface ContentConfig {
   removeForms?: boolean
   /**
    * HTML tag names to strip (render children only, remove the tag wrapper).
-   * Default: `["noscript"]`.
+   * Default: `[]`.
    */
   stripTags?: Array<string>
   /** HTML tag names to preserve as raw HTML in output. */
@@ -369,6 +369,12 @@ export interface ContentConfig {
    * excluded elements and all descendants are dropped. Supports CSS selectors:
    * `.class`, `#id`, `[attribute]`, compound selectors.
    *
+   * Default: `["noscript"]`. `<noscript>` fallback content (no-JS notices,
+   * tracking pixels, GTM iframes) is meant for browsers with JavaScript
+   * disabled, not for a markdown reader, and `strip_tags` cannot drop it —
+   * on `preprocessing_preset: "standard"` (crawlberg's only path) it only
+   * removes the wrapper and still renders the children. ~keep
+   *
    * Example: `[".cookie-banner", "#ad-container", "[role='complementary']"]`
    */
   excludeSelectors?: Array<string>
@@ -407,6 +413,13 @@ export interface CrawlConfig {
   maxDepth?: number
   /** Maximum number of pages to crawl. */
   maxPages?: number
+  /**
+   * Maximum links enqueued from a single page. Defaults to 10000.
+   *
+   * Bounds the work one hostile or pathological page can create; links past the
+   * cap are dropped and a warning is logged.
+   */
+  maxLinksPerPage?: number
   /** Maximum number of concurrent requests. */
   maxConcurrent?: number
   /** Whether to respect robots.txt directives. */
@@ -472,7 +485,17 @@ export interface CrawlConfig {
   proxy?: ProxyConfig
   /** List of user-agent strings for rotation. If non-empty, overrides `user_agent`. */
   userAgents?: Array<string>
-  /** Whether to capture a screenshot when using the browser. */
+  /**
+   * Whether to capture a screenshot when using the browser.
+   *
+   * Only supported by `scrape()` with `BrowserBackend.Chromiumoxide` and
+   * `BrowserMode.Always` or `Stealth`. A screenshot is 100–500 KB of PNG per page,
+   * so `crawl()` does not carry screenshots in `CrawlPageResult`/`CrawlResult` at
+   * all — a multi-thousand-page crawl holding one per page in memory is not a safe
+   * default. Setting this with any other configuration (a different backend,
+   * `BrowserMode.Auto`/`Never`, or during `crawl()`) has no effect and logs a
+   * warning rather than silently doing nothing.
+   */
   captureScreenshot?: boolean
   /**
    * Re-enqueue discovered `LinkType.Document` URLs into the crawl frontier so
@@ -487,15 +510,42 @@ export interface CrawlConfig {
    * outer `max_depth` and (if set) `document_url_depth` permit it.
    */
   documentUrlDepth?: number
-  /** Whether to download non-HTML documents (PDF, DOCX, images, code, etc.) instead of skipping them. */
+  /**
+   * Whether to download non-HTML documents (PDF, DOCX, images, code, etc.) instead of skipping them.
+   * Defaults to `true` — unlike `download_assets` and `capture_screenshot`, which default to `false`.
+   */
   downloadDocuments?: boolean
   /** Maximum size in bytes for document downloads. Defaults to 50 MB. */
   documentMaxSize?: number
   /** Allowlist of MIME types to download. If empty, uses built-in defaults. */
   documentMimeTypes?: Array<string>
+  /**
+   * Directory to stream downloaded document bytes into instead of holding them in
+   * memory on `DownloadedDocument.content`. When set, `content` is left empty and
+   * `DownloadedDocument.content_path` is populated with `<dir>/<content_hash>.<ext>`.
+   * `None` (default) preserves today's in-memory-only behavior. Has no effect on
+   * wasm32, which has no filesystem — use `document_content_encoding` there instead.
+   */
+  documentOutputDir?: string
+  /**
+   * Opt-in encoding that duplicates `DownloadedDocument.content` into a serializable
+   * field for language bindings that need the bytes in-memory (`content` itself is
+   * `alef(skip)`ed). `None` (default) means no encoding is produced. Independent of
+   * `document_output_dir` — set both to get a file on disk and an in-memory copy.
+   */
+  documentContentEncoding?: JsDocumentContentEncoding
   /** Path to write WARC output. If `None`, WARC output is disabled. */
   warcOutput?: string
-  /** Named browser profile for persistent sessions (cookies, localStorage). */
+  /**
+   * Named browser profile for persistent sessions (cookies, localStorage).
+   *
+   * Chromiumoxide backend only. The native backend runs an in-process JavaScript
+   * engine with no Chrome process and therefore no profile directory, so this is
+   * ignored there and logs a warning. It is also ignored — with a warning — when a
+   * shared browser pool is in use (the pool launches before any per-crawl config
+   * exists) or when connecting to an external CDP endpoint whose process crawlberg
+   * does not own.
+   */
   browserProfile?: string
   /** Whether to save changes back to the browser profile on exit. */
   saveBrowserProfile?: boolean
@@ -503,11 +553,38 @@ export interface CrawlConfig {
    * SSRF policy for outbound network requests. Default: deny private networks,
    * allow http/https only, max 5 redirects.
    *
-   * Phase 1: `deny_private` and `max_redirects` are exposed to all language
-   * bindings. `allowlist` is skipped (see `SsrfPolicy` fields) and will be
-   * added in a follow-up when `HostMatcher`'s tagged-enum FFI form is decided.
+   * `deny_private`, `allowlist` and `max_redirects` are exposed to all language
+   * bindings. `scheme_allowlist` stays Rust-only — see `SsrfPolicy`.
+   *
+   * **wasm32 (including Node.js): `deny_private` does not stop hostname-based
+   * requests.** There is no DNS resolution on this target, so only a literal IP host is
+   * checked against the policy — a domain name is always permitted, regardless of
+   * `deny_private`. Under Node, where `fetch` enforces no CORS, this means a service
+   * embedding the wasm binding can be driven to internal hosts by domain name even with
+   * `deny_private = true`. Enforce egress restrictions at the network layer for that
+   * deployment target; do not rely on this field. See `crawlberg.net.validate_url`.
    */
   ssrf?: JsSsrfPolicy
+  /**
+   * Pins `SsrfPolicy.deny_private` to a caller-chosen value, bypassing the
+   * `CRAWLBERG_ALLOW_PRIVATE_NETWORK` operator override entirely for this config.
+   *
+   * `ssrf.deny_private` is a plain, always-serialized `bool`: several alef-generated
+   * bindings construct `SsrfPolicy.default()` (hardcoding `deny_private: true`)
+   * whenever their caller never touches SSRF settings at all, so `true` on that field
+   * alone cannot distinguish "the caller wants private networks denied" from "the
+   * binding's own structural default landed on `true`". The environment variable
+   * exists precisely to resolve that ambiguity in the common case by treating any
+   * `true` as inconclusive and deferring to the operator.
+   *
+   * Set this field when that default-deferral is wrong for your call — e.g. a test
+   * that must prove `deny_private: true` still denies even while the operator has set
+   * `CRAWLBERG_ALLOW_PRIVATE_NETWORK` suite-wide for every other call. `None` (default)
+   * preserves today's behavior: the environment variable may still flip
+   * `ssrf.deny_private` to `false`. `Some(value)` pins `ssrf.deny_private` to `value`
+   * and the environment variable is not consulted for this config.
+   */
+  ssrfDenyPrivateExplicit?: boolean
 }
 
 export declare function crawlConfigDefault(): CrawlConfig
@@ -616,6 +693,19 @@ export interface CrawlStreamRequest {
  */
 export declare function createEngine(config?: CrawlConfig | undefined | null): CrawlEngineHandle
 
+/**
+ * Opt-in encoding applied to a downloaded document's bytes for callers who need the
+ * content available in a serializable field rather than reading it from disk.
+ *
+ * `None` (the `CrawlConfig.document_content_encoding` default) produces neither — unlike
+ * screenshots, base64-encoding a document by default would duplicate an already
+ * up-to-`document_max_size` buffer (50 MB default) in memory per document.
+ */
+export declare const enum DocumentContentEncoding {
+  /** Populate `DownloadedDocument.content_base64` with a base64-encoded copy. */
+  Base64 = 'base_64'
+}
+
 /** A downloaded asset from a page. */
 export interface DownloadedAsset {
   /** The original URL of the asset. */
@@ -652,6 +742,21 @@ export interface DownloadedDocument {
   contentHash?: string
   /** Selected response headers. */
   headers?: Record<string, string>
+  /**
+   * True when `content` (or the file at `content_path`) was truncated to
+   * `document_max_size`; `size` still reports the original, untruncated length.
+   */
+  truncated?: boolean
+  /**
+   * Filesystem path the document was streamed to when `document_output_dir` was
+   * set. `content` is empty in memory when this is populated.
+   */
+  contentPath?: string
+  /**
+   * Base64-encoded copy of `content`, populated only when
+   * `document_content_encoding` was set to `Base64`.
+   */
+  contentBase64?: string
 }
 
 /** Metadata about an LLM extraction pass. */
@@ -717,6 +822,28 @@ export interface HeadingInfo {
   text?: string
 }
 
+/**
+ * Hostname/IP allowlist matcher for SSRF policy.
+ *
+ * Serializes as an internally-tagged object so each variant is distinguishable on the
+ * wire and round-trips losslessly:
+ *
+ * ```json
+ * {"type": "exact",  "value": "api.example.com"}
+ * {"type": "suffix", "value": ".example.com"}
+ * {"type": "cidr",   "value": "10.0.0.0/8"}
+ * ```
+ *
+ * A bare JSON string is still accepted on deserialization and resolves to `Exact`,
+ * preserving configs written against the previous untagged representation.
+ *
+ * `Exact`: HostMatcher.Exact
+ */
+export interface HostMatcher {
+  type: string
+  value?: string
+}
+
 /** An hreflang alternate link entry. */
 export interface HreflangEntry {
   /** The language code (e.g., "en", "fr", "x-default"). */
@@ -762,6 +889,13 @@ export interface InteractionResult {
   finalHtml?: string
   /** Final page URL (may have changed due to navigation). */
   finalUrl?: string
+  /**
+   * Base64-encoded PNG screenshot taken after all actions.
+   *
+   * Populated only when a `PageAction.Screenshot` action actually ran, so
+   * callers that never request a screenshot do not pay the encoding cost.
+   */
+  screenshotBase64?: string
 }
 
 /** A JSON-LD structured data entry found on a page. */
@@ -1025,6 +1159,14 @@ export interface ScrapeResult {
   extractedData?: any
   /** Metadata about the LLM extraction pass (cost, tokens, model). */
   extractionMeta?: ExtractionMeta
+  /**
+   * Base64-encoded PNG screenshot of the page.
+   *
+   * Populated only when `CrawlConfig.capture_screenshot` was enabled for this
+   * request, so callers that never requested a screenshot do not pay the
+   * encoding cost.
+   */
+  screenshotBase64?: string
   /** Downloaded non-HTML document (PDF, DOCX, image, code, etc.). */
   downloadedDocument?: DownloadedDocument
   /**
@@ -1058,6 +1200,24 @@ export interface SitemapUrl {
 export interface SsrfPolicy {
   /** If true, reject URLs that resolve to private/metadata IP ranges. */
   denyPrivate?: boolean
+  /**
+   * Hostnames and IP ranges permitted regardless of `deny_private`.
+   *
+   * The allowlist is an *override* of `deny_private`, not an intersection with it.
+   * Precedence, in order:
+   *
+   * 1. `deny_private == false` permits everything; the allowlist is not consulted.
+   * 2. A hostname matching an `Exact` or `Suffix` entry is permitted immediately,
+   *    *before* DNS resolution — so the deny-list is never applied to it. This trusts
+   *    the host string: a name that resolves into private space is still permitted.
+   * 3. A literal or resolved IP inside a `Cidr` entry is permitted even though it is
+   *    in the default deny-list.
+   * 4. Otherwise the default deny-list decides.
+   *
+   * An empty allowlist therefore denies nothing by itself — it simply leaves
+   * `deny_private` and the deny-list in sole control.
+   */
+  allowlist?: Array<JsHostMatcher>
   /** Maximum number of HTTP redirects to follow during validation. */
   maxRedirects?: number
 }
@@ -1075,5 +1235,13 @@ export declare function ssrfPolicyDefault(): SsrfPolicy
  * - Outbound requests in a browser go through the fetch API, which enforces its own network policies.
  * - Rust-side SSRF checking is unenforceable and redundant in a wasm32 context.
  * - For testing and localhost access, the host's network sandbox is the enforcing boundary.
+ *
+ * **Node.js caveat:** `deny_private` (whatever its value) has no effect on hostname-based
+ * requests under `wasm32`. There is no DNS resolution on this target, so `validate_url`
+ * only ever checks a literal IP host; a domain name falls straight through to `Ok(())`. In a
+ * browser this is covered by same-origin/CORS. Node's `fetch` enforces no CORS, so a Node
+ * service embedding this wasm module can be driven to internal hosts by domain name even
+ * though `deny_private = true`. Do not rely on this policy to stop that in Node — enforce
+ * egress restrictions (network policy, firewall, proxy allowlist) outside the process.
  */
 export declare function ssrfPolicyFromEnv(): SsrfPolicy

package/index.js

@@ -77,8 +77,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-android-arm64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-android-arm64/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -93,8 +93,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-android-arm-eabi')
         const bindingPackageVersion = require('@xberg-io/crawlberg-android-arm-eabi/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -114,8 +114,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-win32-x64-gnu')
         const bindingPackageVersion = require('@xberg-io/crawlberg-win32-x64-gnu/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -130,8 +130,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-win32-x64-msvc')
         const bindingPackageVersion = require('@xberg-io/crawlberg-win32-x64-msvc/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -147,8 +147,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-win32-ia32-msvc')
         const bindingPackageVersion = require('@xberg-io/crawlberg-win32-ia32-msvc/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -163,8 +163,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-win32-arm64-msvc')
         const bindingPackageVersion = require('@xberg-io/crawlberg-win32-arm64-msvc/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -182,8 +182,8 @@ function requireNative() {
     try {
       const binding = require('@xberg-io/crawlberg-darwin-universal')
       const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-universal/package.json').version
-      if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-        throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+      if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+        throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
       }
       return binding
     } catch (e) {
@@ -198,8 +198,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-darwin-x64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-x64/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -214,8 +214,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-darwin-arm64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-darwin-arm64/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -234,8 +234,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-freebsd-x64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-x64/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -250,8 +250,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-freebsd-arm64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-freebsd-arm64/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -271,8 +271,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-x64-musl')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-x64-musl/package.json').version
-          if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -287,8 +287,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-x64-gnu')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-x64-gnu/package.json').version
-          if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -305,8 +305,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-arm64-musl')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm64-musl/package.json').version
-          if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -321,8 +321,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-arm64-gnu')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm64-gnu/package.json').version
-          if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -339,8 +339,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-arm-musleabihf')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm-musleabihf/package.json').version
-          if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -355,8 +355,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-arm-gnueabihf')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-arm-gnueabihf/package.json').version
-          if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -373,8 +373,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-loong64-musl')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-loong64-musl/package.json').version
-          if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -389,8 +389,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-loong64-gnu')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-loong64-gnu/package.json').version
-          if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -407,8 +407,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-riscv64-musl')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-riscv64-musl/package.json').version
-          if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -423,8 +423,8 @@ function requireNative() {
         try {
           const binding = require('@xberg-io/crawlberg-linux-riscv64-gnu')
           const bindingPackageVersion = require('@xberg-io/crawlberg-linux-riscv64-gnu/package.json').version
-          if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-            throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+          if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+            throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
           }
           return binding
         } catch (e) {
@@ -440,8 +440,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-linux-ppc64-gnu')
         const bindingPackageVersion = require('@xberg-io/crawlberg-linux-ppc64-gnu/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -456,8 +456,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-linux-s390x-gnu')
         const bindingPackageVersion = require('@xberg-io/crawlberg-linux-s390x-gnu/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -476,8 +476,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-openharmony-arm64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-arm64/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -492,8 +492,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-openharmony-x64')
         const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-x64/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -508,8 +508,8 @@ function requireNative() {
       try {
         const binding = require('@xberg-io/crawlberg-openharmony-arm')
         const bindingPackageVersion = require('@xberg-io/crawlberg-openharmony-arm/package.json').version
-        if (bindingPackageVersion !== '1.1.4' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
-          throw new Error(`Native binding package version mismatch, expected 1.1.4 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
+        if (bindingPackageVersion !== '1.2.1' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') {
+          throw new Error(`Native binding package version mismatch, expected 1.2.1 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`)
         }
         return binding
       } catch (e) {
@@ -608,6 +608,8 @@ module.exports.crawl = nativeBinding.crawl
 module.exports.crawlConfigDefault = nativeBinding.crawlConfigDefault
 module.exports.crawlStream = nativeBinding.crawlStream
 module.exports.createEngine = nativeBinding.createEngine
+module.exports.DocumentContentEncoding = nativeBinding.DocumentContentEncoding
+module.exports.JsDocumentContentEncoding = nativeBinding.JsDocumentContentEncoding
 module.exports.FeedType = nativeBinding.FeedType
 module.exports.JsFeedType = nativeBinding.JsFeedType
 module.exports.generateCitations = nativeBinding.generateCitations

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xberg-io/crawlberg",
-  "version": "1.1.4",
+  "version": "1.2.1",
   "description": "High-performance web crawling engine",
   "license": "MIT",
   "repository": {
@@ -22,14 +22,14 @@
     "*.node"
   ],
   "optionalDependencies": {
-    "@xberg-io/crawlberg-darwin-arm64": "1.1.4",
-    "@xberg-io/crawlberg-darwin-x64": "1.1.4",
-    "@xberg-io/crawlberg-linux-arm64-gnu": "1.1.4",
-    "@xberg-io/crawlberg-linux-arm64-musl": "1.1.4",
-    "@xberg-io/crawlberg-linux-x64-gnu": "1.1.4",
-    "@xberg-io/crawlberg-linux-x64-musl": "1.1.4",
-    "@xberg-io/crawlberg-win32-arm64-msvc": "1.1.4",
-    "@xberg-io/crawlberg-win32-x64-msvc": "1.1.4"
+    "@xberg-io/crawlberg-darwin-arm64": "1.2.1",
+    "@xberg-io/crawlberg-darwin-x64": "1.2.1",
+    "@xberg-io/crawlberg-linux-arm64-gnu": "1.2.1",
+    "@xberg-io/crawlberg-linux-arm64-musl": "1.2.1",
+    "@xberg-io/crawlberg-linux-x64-gnu": "1.2.1",
+    "@xberg-io/crawlberg-linux-x64-musl": "1.2.1",
+    "@xberg-io/crawlberg-win32-arm64-msvc": "1.2.1",
+    "@xberg-io/crawlberg-win32-x64-msvc": "1.2.1"
   },
   "napi": {
     "packageName": "@xberg-io/crawlberg",