crawlberg 1.11.1 → 1.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 409b1edc648234b832f62ed311e71d5be8e76a27c2483c613124da7075739946
-  data.tar.gz: bc13e5d4e5ac46d7840cfcd1121a59f7c784191e2b3f97102d4a55324ad1cc1d
+  metadata.gz: ccfb786cd775f2189206d95ae30218971296b115ec48e1a64c3fff68ce602c75
+  data.tar.gz: c02f1627a22312d32df5362159fb2a54b5d9ee4737c15754443dedb971325628
 SHA512:
-  metadata.gz: cffaf1ec4a66183ff3f625f5bb37a6eea1615f7439bc9e7c3eb1cc25378ff69ddd609e1d5c7e2b2f6c910fccc7aa08bf4628ebff775c161d34b289a2e8755c21
-  data.tar.gz: ef778845f998c6a338acdd2eb820b110cb29a93054a7f295adbdd7cf4dd9b60156482e5b5265d77390722dc52693a1e9904eb1874c236cbe3bc41c7f62a56dac
+  metadata.gz: 7a8d0a19cfd37e7a879f5f4e15af84156e56cf062c0afe545b550d12f6210aec14bab5af9fceb583f1a93de6d5db69d8fd519fba668226f4fa1b248f126addad
+  data.tar.gz: 2c3e3287fbd7eec2e6f1ea9222e5ad8867b39b3de75075c115fc68baa1972edd933a0b33c93fa7d27c035dbf13524465ce4eec6045220c2efcbb69bbe6aad3f2

data/ext/crawlberg_rb/native/Cargo.lock

@@ -63,9 +63,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470"
 
 [[package]]
 name = "arc-swap"
-version = "1.9.2"
+version = "1.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c049c0be4daef0b145cb3555416b3b8ef5b7888a38aea1a3a155801fe7b0810b"
+checksum = "380962b3d93b99642e50b9febf72c071e97e8b32d86eae9cf4ef582bb960ae6e"
 dependencies = [
  "rustversion",
 ]
@@ -87,9 +87,9 @@ dependencies = [
 
 [[package]]
 name = "async-compression"
-version = "0.4.50"
+version = "0.4.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee19bd99b43e3691acbad4e840420a4881cea6c0b66a208125a824f8fd53f5a1"
+checksum = "9d5755640120b24961f8231e05892e4a4f7e0d04e8aeca161508c9c397946970"
 dependencies = [
  "compression-codecs",
  "compression-core",
@@ -160,12 +160,6 @@ dependencies = [
  "pkg-config",
 ]
 
-[[package]]
-name = "base64"
-version = "0.22.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
-
 [[package]]
 name = "base64"
 version = "0.23.1"
@@ -280,9 +274,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
 
 [[package]]
 name = "cc"
-version = "1.8.0"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6651c9ed80effdc7db0ff72512157f901af5e3549e341e24b1dd4887d836d838"
+checksum = "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -413,9 +407,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
 
 [[package]]
 name = "combine"
-version = "4.6.8"
+version = "4.6.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e"
+checksum = "4356eba6976435498d05a311ee9fc486da3bc81c200d3f0d7dd9703b41913f92"
 dependencies = [
  "bytes",
  "memchr",
@@ -423,9 +417,9 @@ dependencies = [
 
 [[package]]
 name = "compression-codecs"
-version = "0.4.45"
+version = "0.4.46"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "98fc98460ba0ad5317075d3632b8dfc45d0be8c4a49347c2a38272019717614a"
+checksum = "cae36683101b2899f48bd31f8db9839437356188e3b1ff23096e73dab09ba8fe"
 dependencies = [
  "brotli",
  "compression-core",
@@ -435,9 +429,9 @@ dependencies = [
 
 [[package]]
 name = "compression-core"
-version = "0.4.33"
+version = "0.4.34"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e8ccc4ea9f6acc32d102c0f6d471d11d913ad15f20c04de743374861fa1d414"
+checksum = "2ebdda89edc534bf8d5dd2857562633b6495557e5846552ade994b99ed13c36a"
 
 [[package]]
 name = "const-oid"
@@ -543,9 +537,9 @@ dependencies = [
 
 [[package]]
 name = "crawlberg"
-version = "1.11.1"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c441d340160d808d8d494be20e91e87c37227ad09881cf70b7e32d6eabf2c94a"
+checksum = "816cd63cd218f1072b402230d946d4805923b1d53a2faa39d5ce71d58d3f022a"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -553,7 +547,7 @@ dependencies = [
  "astral-tl",
  "async-trait",
  "async-tungstenite",
- "base64 0.23.1",
+ "base64",
  "blake3",
  "bytes",
  "chardetng",
@@ -613,12 +607,12 @@ dependencies = [
 
 [[package]]
 name = "crawlberg-chromiumoxide"
-version = "0.9.3"
+version = "0.9.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8add9abf085c5f95d7dfbde84b67e6b6f2fb95a0ca288c98a4e8ceae169d2d3f"
+checksum = "937d12e969f7090c02cb64dff52375f3e26e669cbdda7bc49ce364fb8ce7bb68"
 dependencies = [
  "async-tungstenite",
- "base64 0.22.1",
+ "base64",
  "chromiumoxide_cdp",
  "chromiumoxide_types",
  "dunce",
@@ -641,7 +635,7 @@ dependencies = [
 
 [[package]]
 name = "crawlberg-rb"
-version = "1.11.1"
+version = "1.12.0"
 dependencies = [
  "crawlberg",
  "futures",
@@ -654,15 +648,15 @@ dependencies = [
 
 [[package]]
 name = "crawlberg-robots"
-version = "1.11.1"
+version = "1.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "07c5d34335e3703ed5d12bffa68b67e33063347a219cae698bf939f9fb41a62f"
+checksum = "a7f393d7f0a1f96526de81142b88a733b92b9ca8ad3fc35b1c23c90818b50029"
 
 [[package]]
 name = "crc32fast"
-version = "1.5.2"
+version = "1.5.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78"
+checksum = "0ba106e844045ddc02c20768ac4b0b7cb45f2cc2ecbd833ac9f5dacb183c2055"
 dependencies = [
  "cfg-if",
 ]
@@ -851,9 +845,9 @@ dependencies = [
 
 [[package]]
 name = "find-msvc-tools"
-version = "0.1.14"
+version = "0.1.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aedcfb3409746eddb02b9e19ebda1c3394f759a152e48ee875a0844d1b955484"
+checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48"
 
 [[package]]
 name = "flate2"
@@ -1092,13 +1086,13 @@ checksum = "491194d601b4fa9fac3c9571e014070a0ad1bc3392d1bc10419695196a8b3cc3"
 
 [[package]]
 name = "html-to-markdown-rs"
-version = "3.18.1"
+version = "3.18.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "36135ae2471a090d086837db2f4794dca3bf8ca0ba288b799a0bb85dd3f0acf0"
+checksum = "456da57af20dc98ba1ec1ab70079f977ec29bf62b713ee1acaa71799a275adb4"
 dependencies = [
  "ahash",
  "astral-tl",
- "base64 0.23.1",
+ "base64",
  "bitflags",
  "html-escape",
  "html5ever",
@@ -1222,7 +1216,7 @@ version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ddc03d96684f9226b8a787cdb71488417b53ab5ea8fdb1dac946cb9431cc8bff"
 dependencies = [
- "base64 0.23.1",
+ "base64",
  "bytes",
  "futures-channel",
  "futures-util",
@@ -1573,9 +1567,9 @@ checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb"
 
 [[package]]
 name = "libc"
-version = "0.2.189"
+version = "0.2.191"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
+checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a"
 
 [[package]]
 name = "libloading"
@@ -2000,9 +1994,9 @@ dependencies = [
 
 [[package]]
 name = "psl"
-version = "2.1.241"
+version = "2.1.244"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c7319b480e204d67e759cad7886178335d26324c7c8573c126cc9dfa7350d064"
+checksum = "bc3ac0c25b0c40554a755e56dfe268855c53f391bbb7922a1c1fe59b73b163a9"
 dependencies = [
  "psl-types",
 ]
@@ -2232,7 +2226,7 @@ version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "16a1cfa75cc186dd73d5818e510e042e40927bccc9c236b061cea97e1eb08029"
 dependencies = [
- "base64 0.23.1",
+ "base64",
  "bytes",
  "cookie",
  "cookie_store",
@@ -2373,9 +2367,9 @@ checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f"
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.15"
+version = "0.103.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
+checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281"
 dependencies = [
  "aws-lc-rs",
  "ring",
@@ -2400,9 +2394,9 @@ dependencies = [
 
 [[package]]
 name = "schannel"
-version = "0.1.29"
+version = "0.1.30"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
+checksum = "9cb23c424d8ee841bb4d813677ca237e2a610067d594fec130f87d6093495ab5"
 dependencies = [
  "windows-sys 0.61.2",
 ]
@@ -2843,9 +2837,9 @@ checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee"
 
 [[package]]
 name = "tokio"
-version = "1.53.3"
+version = "1.54.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ce3335fa71841cda333a58d7615b03901380ecf09d59b3296d21f8bbac0dde4e"
+checksum = "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3"
 dependencies = [
  "bytes",
  "libc",
@@ -3225,9 +3219,9 @@ dependencies = [
 
 [[package]]
 name = "webpki-root-certs"
-version = "1.0.9"
+version = "1.0.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b96554aa2acc8ccdb7e1c9a58a7a68dd5d13bccc69cd124cb09406db612a1c9b"
+checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb"
 dependencies = [
  "rustls-pki-types",
 ]

data/ext/crawlberg_rb/native/Cargo.toml

@@ -1,11 +1,11 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:7fb2df2846ff405ffd7dba3601d74e88dbb7578bb5175c54cc2e8e10229e6ddd
+# alef:hash:5c5de05901161d333a4343e3a35f6b8e0967bdc7fbaa657e0ccb4f328fc2701b
 # To regenerate: alef generate
 # To verify freshness: alef verify
 
 [package]
 name = "crawlberg-rb"
-version = "1.11.1"
+version = "1.12.0"
 edition = "2024"
 authors = ["Na'aman Hirschfeld <naaman@xberg.io>", "Tobias Perlstein <tobias@xberg.io>"]
 license = "MIT"
@@ -23,7 +23,7 @@ path = "../src/lib.rs"
 crate-type = ["cdylib"]
 
 [dependencies]
-crawlberg = { version = "1.11.1", features = ["interact", "browser-chromiumoxide"] }
+crawlberg = { version = "1.12.0", features = ["interact", "browser-chromiumoxide"] }
 futures = "0.3"
 magnus = "0.9"
 rb-sys = ">=0.9.130, <0.10"

data/ext/crawlberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:2ec1fdc93f72b06b0d045fd1f6b6288fad256babb9754b0e89759ebc3fd4b1f8
+// alef:hash:582ed106e4be9129fdfae3b203c9f9ccb338efae1989ad0fdde9d99890d81bd7
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -843,6 +843,8 @@ pub struct BrowserConfig {
     session_affinity: bool,
     chrome_path: Option<String>,
     chrome_args: Vec<String>,
+    scroll_containers: bool,
+    dom_quiet_window: u64,
 }
 
 unsafe impl IntoValueFromNative for BrowserConfig {}
@@ -892,6 +894,8 @@ impl std::fmt::Debug for BrowserConfig {
         debug.field("session_affinity", &self.session_affinity);
         debug.field("chrome_path", &self.chrome_path);
         debug.field("chrome_args", &"<redacted>");
+        debug.field("scroll_containers", &self.scroll_containers);
+        debug.field("dom_quiet_window", &self.dom_quiet_window);
         debug.finish()
     }
 }
@@ -1070,6 +1074,24 @@ impl BrowserConfig {
                 })?,
                 None => Default::default(),
             },
+            scroll_containers: match kwargs.get(ruby.to_symbol("scroll_containers")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `scroll_containers`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            dom_quiet_window: match kwargs.get(ruby.to_symbol("dom_quiet_window")) {
+                Some(v) => u64::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `dom_quiet_window`: {}", e),
+                    )
+                })?,
+                None => 150,
+            },
         })
     }
 
@@ -1140,6 +1162,14 @@ impl BrowserConfig {
     fn chrome_args(&self) -> Vec<String> {
         self.chrome_args.clone()
     }
+
+    fn scroll_containers(&self) -> bool {
+        self.scroll_containers
+    }
+
+    fn dom_quiet_window(&self) -> u64 {
+        self.dom_quiet_window.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize)]
@@ -1936,6 +1966,7 @@ pub struct CrawlConfig {
     save_browser_profile: bool,
     ssrf: SsrfPolicy,
     ssrf_deny_private_explicit: Option<bool>,
+    danger_accept_invalid_certs: bool,
 }
 
 unsafe impl IntoValueFromNative for CrawlConfig {}
@@ -2023,6 +2054,7 @@ impl std::fmt::Debug for CrawlConfig {
         debug.field("save_browser_profile", &self.save_browser_profile);
         debug.field("ssrf", &self.ssrf);
         debug.field("ssrf_deny_private_explicit", &self.ssrf_deny_private_explicit);
+        debug.field("danger_accept_invalid_certs", &self.danger_accept_invalid_certs);
         debug.finish()
     }
 }
@@ -2549,6 +2581,15 @@ impl CrawlConfig {
                 })?),
                 None => None,
             },
+            danger_accept_invalid_certs: match kwargs.get(ruby.to_symbol("danger_accept_invalid_certs")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `danger_accept_invalid_certs`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
         })
     }
 
@@ -2772,6 +2813,10 @@ impl CrawlConfig {
         self.ssrf_deny_private_explicit
     }
 
+    fn danger_accept_invalid_certs(&self) -> bool {
+        self.danger_accept_invalid_certs
+    }
+
     fn validate(&self) -> Result<(), Error> {
         #[allow(clippy::needless_update)]
         let core_self = crawlberg::CrawlConfig {
@@ -2890,6 +2935,8 @@ impl CrawlConfig {
 
             ssrf_deny_private_explicit: self.ssrf_deny_private_explicit,
 
+            danger_accept_invalid_certs: self.danger_accept_invalid_certs,
+
             ..Default::default()
         };
         let result = core_self.validate().map_err(|e| {
@@ -3454,7 +3501,7 @@ impl CrawlPageResult {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::CrawlResult")]
 pub struct CrawlResult {
@@ -3466,6 +3513,7 @@ pub struct CrawlResult {
     cookies: Vec<CookieInfo>,
     stayed_on_domain: bool,
     browser_used: bool,
+    waf_block: Option<WafBlockDetails>,
 }
 
 unsafe impl IntoValueFromNative for CrawlResult {}
@@ -3495,6 +3543,14 @@ impl magnus::TryConvert for CrawlResult {
 }
 
 unsafe impl TryConvertOwned for CrawlResult {}
+impl<'de> serde::Deserialize<'de> for CrawlResult {
+    fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        <crawlberg::CrawlResult as serde::Deserialize>::deserialize(deserializer).map(Into::into)
+    }
+}
 
 impl Default for CrawlResult {
     fn default() -> Self {
@@ -3581,6 +3637,15 @@ impl CrawlResult {
                 })?,
                 None => Default::default(),
             },
+            waf_block: match kwargs.get(ruby.to_symbol("waf_block")).filter(|v| !v.is_nil()) {
+                Some(v) => Some(WafBlockDetails::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `waf_block`: {}", e),
+                    )
+                })?),
+                None => None,
+            },
         })
     }
 
@@ -3616,6 +3681,10 @@ impl CrawlResult {
         self.browser_used
     }
 
+    fn waf_block(&self) -> Option<WafBlockDetails> {
+        self.waf_block.clone()
+    }
+
     fn unique_normalized_urls(&self) -> usize {
         #[allow(clippy::needless_update)]
         let core_self = crawlberg::CrawlResult {
@@ -3635,6 +3704,8 @@ impl CrawlResult {
 
             browser_used: self.browser_used,
 
+            waf_block: self.waf_block.clone().map(Into::into),
+
             ..Default::default()
         };
         core_self.unique_normalized_urls()
@@ -6845,6 +6916,99 @@ impl SsrfPolicy {
     }
 }
 
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(default)]
+#[magnus::wrap(class = "Crawlberg::WafBlockDetails")]
+pub struct WafBlockDetails {
+    vendor: String,
+    status: Option<u16>,
+    body_prefix: Option<String>,
+}
+
+unsafe impl IntoValueFromNative for WafBlockDetails {}
+
+impl magnus::TryConvert for WafBlockDetails {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        if let Ok(r) = <&WafBlockDetails as magnus::TryConvert>::try_convert(val) {
+            return Ok(r.clone());
+        }
+        let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
+            s
+        } else {
+            val.funcall::<_, _, String>("to_json", ()).map_err(|e| {
+                magnus::Error::new(
+                    unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                    format!("no implicit conversion into WafBlockDetails: {}", e),
+                )
+            })?
+        };
+        serde_json::from_str::<WafBlockDetails>(&json_str).map_err(|e| {
+            magnus::Error::new(
+                unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                format!("failed to deserialize WafBlockDetails: {}", e),
+            )
+        })
+    }
+}
+
+unsafe impl TryConvertOwned for WafBlockDetails {}
+
+impl Default for WafBlockDetails {
+    fn default() -> Self {
+        crawlberg::WafBlockDetails::default().into()
+    }
+}
+
+impl WafBlockDetails {
+    fn new(args: &[magnus::Value]) -> Result<Self, magnus::Error> {
+        let ruby = unsafe { magnus::Ruby::get_unchecked() };
+        let args = magnus::scan_args::scan_args::<(), (Option<magnus::RHash>,), (), (), (), ()>(args)?;
+        let (kwargs_opt,) = args.optional;
+        let kwargs = kwargs_opt.unwrap_or_else(|| ruby.hash_new());
+        Ok(Self {
+            vendor: match kwargs.get(ruby.to_symbol("vendor")) {
+                Some(v) => String::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `vendor`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
+            status: match kwargs.get(ruby.to_symbol("status")).filter(|v| !v.is_nil()) {
+                Some(v) => Some(u16::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `status`: {}", e),
+                    )
+                })?),
+                None => None,
+            },
+            body_prefix: match kwargs.get(ruby.to_symbol("body_prefix")).filter(|v| !v.is_nil()) {
+                Some(v) => Some(String::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `body_prefix`: {}", e),
+                    )
+                })?),
+                None => None,
+            },
+        })
+    }
+
+    fn vendor(&self) -> String {
+        self.vendor.clone()
+    }
+
+    fn status(&self) -> Option<u16> {
+        self.status
+    }
+
+    fn body_prefix(&self) -> Option<String> {
+        self.body_prefix.clone()
+    }
+}
+
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum AssetCategory {
@@ -8136,6 +8300,8 @@ impl From<BrowserConfig> for crawlberg::BrowserConfig {
             session_affinity: val.session_affinity,
             chrome_path: val.chrome_path.map(Into::into),
             chrome_args: val.chrome_args.into_iter().collect(),
+            scroll_containers: val.scroll_containers,
+            dom_quiet_window: std::time::Duration::from_millis(val.dom_quiet_window),
             ..Default::default()
         }
     }
@@ -8162,6 +8328,8 @@ impl From<crawlberg::BrowserConfig> for BrowserConfig {
             session_affinity: val.session_affinity,
             chrome_path: val.chrome_path.map(|p| p.to_string_lossy().to_string()),
             chrome_args: val.chrome_args.into_iter().collect(),
+            scroll_containers: val.scroll_containers,
+            dom_quiet_window: val.dom_quiet_window.as_millis() as u64,
         }
     }
 }
@@ -8379,6 +8547,7 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
             save_browser_profile: val.save_browser_profile,
             ssrf: val.ssrf.into(),
             ssrf_deny_private_explicit: val.ssrf_deny_private_explicit,
+            danger_accept_invalid_certs: val.danger_accept_invalid_certs,
             ..Default::default()
         }
     }
@@ -8447,6 +8616,7 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
             save_browser_profile: val.save_browser_profile,
             ssrf: val.ssrf.into(),
             ssrf_deny_private_explicit: val.ssrf_deny_private_explicit,
+            danger_accept_invalid_certs: val.danger_accept_invalid_certs,
         }
     }
 }
@@ -8536,6 +8706,7 @@ impl From<CrawlResult> for crawlberg::CrawlResult {
             cookies: val.cookies.into_iter().map(Into::into).collect(),
             stayed_on_domain: val.stayed_on_domain,
             browser_used: val.browser_used,
+            waf_block: val.waf_block.map(Into::into),
             ..Default::default()
         }
     }
@@ -8553,6 +8724,7 @@ impl From<crawlberg::CrawlResult> for CrawlResult {
             cookies: val.cookies.into_iter().map(Into::into).collect(),
             stayed_on_domain: val.stayed_on_domain,
             browser_used: val.browser_used,
+            waf_block: val.waf_block.map(Into::into),
         }
     }
 }
@@ -9214,6 +9386,29 @@ impl From<crawlberg::SsrfPolicy> for SsrfPolicy {
     }
 }
 
+#[allow(clippy::needless_update)]
+impl From<WafBlockDetails> for crawlberg::WafBlockDetails {
+    fn from(val: WafBlockDetails) -> Self {
+        Self {
+            vendor: val.vendor,
+            status: val.status,
+            body_prefix: val.body_prefix,
+            ..Default::default()
+        }
+    }
+}
+
+#[allow(clippy::redundant_closure)]
+impl From<crawlberg::WafBlockDetails> for WafBlockDetails {
+    fn from(val: crawlberg::WafBlockDetails) -> Self {
+        Self {
+            vendor: val.vendor.to_string(),
+            status: val.status,
+            body_prefix: val.body_prefix.map(|v| v.to_string()),
+        }
+    }
+}
+
 impl From<AssetCategory> for crawlberg::AssetCategory {
     fn from(val: AssetCategory) -> Self {
         match val {
@@ -9876,6 +10071,10 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("chrome_args", method!(BrowserConfig::chrome_args, 0))?;
 
+    class.define_method("scroll_containers", method!(BrowserConfig::scroll_containers, 0))?;
+
+    class.define_method("dom_quiet_window", method!(BrowserConfig::dom_quiet_window, 0))?;
+
     let class = module.define_class("BrowserExtras", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(BrowserExtras::new, -1))?;
@@ -10100,6 +10299,11 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
         method!(CrawlConfig::ssrf_deny_private_explicit, 0),
     )?;
 
+    class.define_method(
+        "danger_accept_invalid_certs",
+        method!(CrawlConfig::danger_accept_invalid_certs, 0),
+    )?;
+
     class.define_method("validate", method!(CrawlConfig::validate, 0))?;
 
     let class = module.define_class("CrawlEngineHandle", ruby.class_object())?;
@@ -10185,6 +10389,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("browser_used", method!(CrawlResult::browser_used, 0))?;
 
+    class.define_method("waf_block", method!(CrawlResult::waf_block, 0))?;
+
     class.define_method(
         "unique_normalized_urls",
         method!(CrawlResult::unique_normalized_urls, 0),
@@ -10574,6 +10780,16 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("scheme_allowlist", method!(SsrfPolicy::scheme_allowlist, 0))?;
 
+    let class = module.define_class("WafBlockDetails", ruby.class_object())?;
+
+    class.define_singleton_method("new", function!(WafBlockDetails::new, -1))?;
+
+    class.define_method("vendor", method!(WafBlockDetails::vendor, 0))?;
+
+    class.define_method("status", method!(WafBlockDetails::status, 0))?;
+
+    class.define_method("body_prefix", method!(WafBlockDetails::body_prefix, 0))?;
+
     let class = module.define_class("CrawlStreamIterator", ruby.class_object())?;
     class.define_method("next_chunk", method!(CrawlStreamIterator::next_chunk, 0))?;
     class.define_method("each", method!(CrawlStreamIterator::each, 0))?;

data/lib/crawlberg/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:d323ed0aab9764791b48678b68c8fedcf8e7b3e1dbe50ec190a9023e68acc3bb
+# alef:hash:60c8949f72e7c9412406a7c721b2968d8acafde7ead2534f00d423542e33bade
 # To regenerate: alef generate
 # To verify freshness: alef verify
 # frozen_string_literal: true
@@ -164,6 +164,10 @@ module Crawlberg
     end
   end
   ## A single page has been crawled.
+  ##
+  ## A link whose redirect lands on a URL the crawl already queued or visited gets no event of
+  ## its own: it is requested, then dropped as a duplicate of the destination, which is reported
+  ## once under its own URL. The request count can therefore exceed the event count.
   CrawlEventPage = Data.define(:result) do
     include CrawlEvent
     extend T::Sig

data/lib/crawlberg/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:0fcaeaf159ae13b7e3d56e201029880b4cf648beeb0bb8efc6d0ff175d94e894
+# alef:hash:ed92ed554ddf1dcb4d5a619d4a13d01d9287c96310437df1c7a4c484d27b969c
 # To regenerate: alef generate
 # To verify freshness: alef verify
 # frozen_string_literal: true
 
 module Crawlberg
   ## The version string for this package.
-  VERSION = "1.11.1"
+  VERSION = "1.12.0"
 end

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:ad470f62b746634fbd629f163e7bae4a88fb011bbde28d998e38ade4877adf3f
+# alef:hash:a2ea8938a47348aa37d91448784e4263e6b5f4df1fe3a7ff1f9a8bdd3bf04009
 # To regenerate: alef generate
 # To verify freshness: alef verify
 
@@ -87,8 +87,10 @@ def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?com
         attr_reader session_affinity: bool
         attr_reader chrome_path: String?
     attr_reader chrome_args: Array[String]
+        attr_reader scroll_containers: bool
+        attr_reader dom_quiet_window: Integer
 
-def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool, ?chrome_path: String, ?chrome_args: Array[String]) -> void
+def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool, ?chrome_path: String, ?chrome_args: Array[String], ?scroll_containers: bool, ?dom_quiet_window: Integer) -> void
     end
 
     class BrowserExtras
@@ -204,8 +206,9 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader save_browser_profile: bool
         attr_reader ssrf: SsrfPolicy
         attr_reader ssrf_deny_private_explicit: bool?
+        attr_reader danger_accept_invalid_certs: bool
 
-def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?path_patterns_match_url: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
+def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?path_patterns_match_url: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool, ?danger_accept_invalid_certs: bool) -> void
     def validate: () -> void
     end
 
@@ -255,8 +258,9 @@ def initialize: (?url: String, ?normalized_url: String, ?status_code: Integer, ?
     attr_reader cookies: Array[CookieInfo]
         attr_reader stayed_on_domain: bool
         attr_reader browser_used: bool
+        attr_reader waf_block: WafBlockDetails?
 
-def initialize: (?pages: Array[CrawlPageResult], ?final_url: String, ?redirect_count: Integer, ?was_skipped: bool, ?error: String, ?cookies: Array[CookieInfo], ?stayed_on_domain: bool, ?browser_used: bool) -> void
+def initialize: (?pages: Array[CrawlPageResult], ?final_url: String, ?redirect_count: Integer, ?was_skipped: bool, ?error: String, ?cookies: Array[CookieInfo], ?stayed_on_domain: bool, ?browser_used: bool, ?waf_block: WafBlockDetails) -> void
     def unique_normalized_urls: () -> Integer
     end
 
@@ -510,6 +514,14 @@ def initialize: (?status_code: Integer, ?final_url: String, ?content_type: Strin
 def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist: Array[HostMatcher], ?max_redirects: Integer, ?scheme_allowlist: Array[String]) -> void
     end
 
+    class WafBlockDetails
+        attr_reader vendor: String
+        attr_reader status: Integer?
+        attr_reader body_prefix: String?
+
+    def initialize: (?vendor: String, ?status: Integer, ?body_prefix: String) -> void
+    end
+
     type enum_AssetCategory = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other
 
     class AuthConfig

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlberg
 version: !ruby/object:Gem::Version
-  version: 1.11.1
+  version: 1.12.0
 platform: ruby
 authors:
 - Na'aman Hirschfeld
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-16 00:00:00.000000000 Z
+date: 2026-10-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys