crawlberg 1.11.1-aarch64-linux → 1.12.2-aarch64-linux

data/ext/crawlberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:2ec1fdc93f72b06b0d045fd1f6b6288fad256babb9754b0e89759ebc3fd4b1f8
+// alef:hash:5766b3fb05a187f88eec0ae0ee3c5e32d6bede7f1a3bc38b4305587073983bbd
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -843,6 +843,9 @@ pub struct BrowserConfig {
     session_affinity: bool,
     chrome_path: Option<String>,
     chrome_args: Vec<String>,
+    scroll_containers: bool,
+    dom_quiet_window: u64,
+    dom_quiet_cap: u64,
 }
 
 unsafe impl IntoValueFromNative for BrowserConfig {}
@@ -892,6 +895,9 @@ impl std::fmt::Debug for BrowserConfig {
         debug.field("session_affinity", &self.session_affinity);
         debug.field("chrome_path", &self.chrome_path);
         debug.field("chrome_args", &"<redacted>");
+        debug.field("scroll_containers", &self.scroll_containers);
+        debug.field("dom_quiet_window", &self.dom_quiet_window);
+        debug.field("dom_quiet_cap", &self.dom_quiet_cap);
         debug.finish()
     }
 }
@@ -1070,6 +1076,33 @@ impl BrowserConfig {
                 })?,
                 None => Default::default(),
             },
+            scroll_containers: match kwargs.get(ruby.to_symbol("scroll_containers")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `scroll_containers`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            dom_quiet_window: match kwargs.get(ruby.to_symbol("dom_quiet_window")) {
+                Some(v) => u64::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `dom_quiet_window`: {}", e),
+                    )
+                })?,
+                None => 150,
+            },
+            dom_quiet_cap: match kwargs.get(ruby.to_symbol("dom_quiet_cap")) {
+                Some(v) => u64::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `dom_quiet_cap`: {}", e),
+                    )
+                })?,
+                None => 4000,
+            },
         })
     }
 
@@ -1140,6 +1173,18 @@ impl BrowserConfig {
     fn chrome_args(&self) -> Vec<String> {
         self.chrome_args.clone()
     }
+
+    fn scroll_containers(&self) -> bool {
+        self.scroll_containers
+    }
+
+    fn dom_quiet_window(&self) -> u64 {
+        self.dom_quiet_window.clone()
+    }
+
+    fn dom_quiet_cap(&self) -> u64 {
+        self.dom_quiet_cap.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize)]
@@ -1444,6 +1489,10 @@ pub struct ContentConfig {
     compact_tables: bool,
     keep_inline_images_in: Vec<String>,
     keep_hidden: bool,
+    expand_abbreviations: bool,
+    escape_misc: bool,
+    escape_asterisks: bool,
+    include_icon_labels: bool,
 }
 
 unsafe impl IntoValueFromNative for ContentConfig {}
@@ -1675,6 +1724,42 @@ impl ContentConfig {
                 })?,
                 None => false,
             },
+            expand_abbreviations: match kwargs.get(ruby.to_symbol("expand_abbreviations")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `expand_abbreviations`: {}", e),
+                    )
+                })?,
+                None => true,
+            },
+            escape_misc: match kwargs.get(ruby.to_symbol("escape_misc")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `escape_misc`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            escape_asterisks: match kwargs.get(ruby.to_symbol("escape_asterisks")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `escape_asterisks`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            include_icon_labels: match kwargs.get(ruby.to_symbol("include_icon_labels")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `include_icon_labels`: {}", e),
+                    )
+                })?,
+                None => true,
+            },
         })
     }
 
@@ -1757,6 +1842,22 @@ impl ContentConfig {
     fn keep_hidden(&self) -> bool {
         self.keep_hidden
     }
+
+    fn expand_abbreviations(&self) -> bool {
+        self.expand_abbreviations
+    }
+
+    fn escape_misc(&self) -> bool {
+        self.escape_misc
+    }
+
+    fn escape_asterisks(&self) -> bool {
+        self.escape_asterisks
+    }
+
+    fn include_icon_labels(&self) -> bool {
+        self.include_icon_labels
+    }
 }
 
 #[derive(Clone, serde::Serialize, serde::Deserialize)]
@@ -1936,6 +2037,8 @@ pub struct CrawlConfig {
     save_browser_profile: bool,
     ssrf: SsrfPolicy,
     ssrf_deny_private_explicit: Option<bool>,
+    danger_accept_invalid_certs: bool,
+    subdomain_header_allowlist: Vec<String>,
 }
 
 unsafe impl IntoValueFromNative for CrawlConfig {}
@@ -2023,6 +2126,8 @@ impl std::fmt::Debug for CrawlConfig {
         debug.field("save_browser_profile", &self.save_browser_profile);
         debug.field("ssrf", &self.ssrf);
         debug.field("ssrf_deny_private_explicit", &self.ssrf_deny_private_explicit);
+        debug.field("danger_accept_invalid_certs", &self.danger_accept_invalid_certs);
+        debug.field("subdomain_header_allowlist", &self.subdomain_header_allowlist);
         debug.finish()
     }
 }
@@ -2549,6 +2654,24 @@ impl CrawlConfig {
                 })?),
                 None => None,
             },
+            danger_accept_invalid_certs: match kwargs.get(ruby.to_symbol("danger_accept_invalid_certs")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `danger_accept_invalid_certs`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            subdomain_header_allowlist: match kwargs.get(ruby.to_symbol("subdomain_header_allowlist")) {
+                Some(v) => <Vec<String>>::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `subdomain_header_allowlist`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
         })
     }
 
@@ -2772,6 +2895,14 @@ impl CrawlConfig {
         self.ssrf_deny_private_explicit
     }
 
+    fn danger_accept_invalid_certs(&self) -> bool {
+        self.danger_accept_invalid_certs
+    }
+
+    fn subdomain_header_allowlist(&self) -> Vec<String> {
+        self.subdomain_header_allowlist.clone()
+    }
+
     fn validate(&self) -> Result<(), Error> {
         #[allow(clippy::needless_update)]
         let core_self = crawlberg::CrawlConfig {
@@ -2890,14 +3021,13 @@ impl CrawlConfig {
 
             ssrf_deny_private_explicit: self.ssrf_deny_private_explicit,
 
+            danger_accept_invalid_certs: self.danger_accept_invalid_certs,
+
+            subdomain_header_allowlist: self.subdomain_header_allowlist.clone(),
+
             ..Default::default()
         };
-        let result = core_self.validate().map_err(|e| {
-            magnus::Error::new(
-                unsafe { Ruby::get_unchecked() }.exception_runtime_error(),
-                e.to_string(),
-            )
-        })?;
+        let result = core_self.validate().map_err(alef_magnus_err)?;
         Ok(result)
     }
 }
@@ -2948,12 +3078,7 @@ impl CrawlEngineHandle {
         );
         let stream = runtime
             .block_on(async { inner.crawl_stream(core_req).await })
-            .map_err(|e| {
-                magnus::Error::new(
-                    unsafe { Ruby::get_unchecked() }.exception_runtime_error(),
-                    e.to_string(),
-                )
-            })?;
+            .map_err(alef_magnus_err)?;
         let iterator = CrawlStreamIterator {
             inner: Arc::new(tokio::sync::Mutex::new(Some(stream))),
             runtime,
@@ -2993,12 +3118,7 @@ impl CrawlEngineHandle {
         );
         let stream = runtime
             .block_on(async { inner.batch_crawl_stream(core_req).await })
-            .map_err(|e| {
-                magnus::Error::new(
-                    unsafe { Ruby::get_unchecked() }.exception_runtime_error(),
-                    e.to_string(),
-                )
-            })?;
+            .map_err(alef_magnus_err)?;
         let iterator = BatchCrawlStreamIterator {
             inner: Arc::new(tokio::sync::Mutex::new(Some(stream))),
             runtime,
@@ -3454,7 +3574,7 @@ impl CrawlPageResult {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::CrawlResult")]
 pub struct CrawlResult {
@@ -3466,6 +3586,7 @@ pub struct CrawlResult {
     cookies: Vec<CookieInfo>,
     stayed_on_domain: bool,
     browser_used: bool,
+    waf_block: Option<WafBlockDetails>,
 }
 
 unsafe impl IntoValueFromNative for CrawlResult {}
@@ -3495,6 +3616,14 @@ impl magnus::TryConvert for CrawlResult {
 }
 
 unsafe impl TryConvertOwned for CrawlResult {}
+impl<'de> serde::Deserialize<'de> for CrawlResult {
+    fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        <crawlberg::CrawlResult as serde::Deserialize>::deserialize(deserializer).map(Into::into)
+    }
+}
 
 impl Default for CrawlResult {
     fn default() -> Self {
@@ -3581,6 +3710,15 @@ impl CrawlResult {
                 })?,
                 None => Default::default(),
             },
+            waf_block: match kwargs.get(ruby.to_symbol("waf_block")).filter(|v| !v.is_nil()) {
+                Some(v) => Some(WafBlockDetails::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `waf_block`: {}", e),
+                    )
+                })?),
+                None => None,
+            },
         })
     }
 
@@ -3616,6 +3754,10 @@ impl CrawlResult {
         self.browser_used
     }
 
+    fn waf_block(&self) -> Option<WafBlockDetails> {
+        self.waf_block.clone()
+    }
+
     fn unique_normalized_urls(&self) -> usize {
         #[allow(clippy::needless_update)]
         let core_self = crawlberg::CrawlResult {
@@ -3635,6 +3777,8 @@ impl CrawlResult {
 
             browser_used: self.browser_used,
 
+            waf_block: self.waf_block.clone().map(Into::into),
+
             ..Default::default()
         };
         core_self.unique_normalized_urls()
@@ -6845,6 +6989,99 @@ impl SsrfPolicy {
     }
 }
 
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(default)]
+#[magnus::wrap(class = "Crawlberg::WafBlockDetails")]
+pub struct WafBlockDetails {
+    vendor: String,
+    status: Option<u16>,
+    body_prefix: Option<String>,
+}
+
+unsafe impl IntoValueFromNative for WafBlockDetails {}
+
+impl magnus::TryConvert for WafBlockDetails {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        if let Ok(r) = <&WafBlockDetails as magnus::TryConvert>::try_convert(val) {
+            return Ok(r.clone());
+        }
+        let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
+            s
+        } else {
+            val.funcall::<_, _, String>("to_json", ()).map_err(|e| {
+                magnus::Error::new(
+                    unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                    format!("no implicit conversion into WafBlockDetails: {}", e),
+                )
+            })?
+        };
+        serde_json::from_str::<WafBlockDetails>(&json_str).map_err(|e| {
+            magnus::Error::new(
+                unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                format!("failed to deserialize WafBlockDetails: {}", e),
+            )
+        })
+    }
+}
+
+unsafe impl TryConvertOwned for WafBlockDetails {}
+
+impl Default for WafBlockDetails {
+    fn default() -> Self {
+        crawlberg::WafBlockDetails::default().into()
+    }
+}
+
+impl WafBlockDetails {
+    fn new(args: &[magnus::Value]) -> Result<Self, magnus::Error> {
+        let ruby = unsafe { magnus::Ruby::get_unchecked() };
+        let args = magnus::scan_args::scan_args::<(), (Option<magnus::RHash>,), (), (), (), ()>(args)?;
+        let (kwargs_opt,) = args.optional;
+        let kwargs = kwargs_opt.unwrap_or_else(|| ruby.hash_new());
+        Ok(Self {
+            vendor: match kwargs.get(ruby.to_symbol("vendor")) {
+                Some(v) => String::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `vendor`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
+            status: match kwargs.get(ruby.to_symbol("status")).filter(|v| !v.is_nil()) {
+                Some(v) => Some(u16::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `status`: {}", e),
+                    )
+                })?),
+                None => None,
+            },
+            body_prefix: match kwargs.get(ruby.to_symbol("body_prefix")).filter(|v| !v.is_nil()) {
+                Some(v) => Some(String::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `body_prefix`: {}", e),
+                    )
+                })?),
+                None => None,
+            },
+        })
+    }
+
+    fn vendor(&self) -> String {
+        self.vendor.clone()
+    }
+
+    fn status(&self) -> Option<u16> {
+        self.status
+    }
+
+    fn body_prefix(&self) -> Option<String> {
+        self.body_prefix.clone()
+    }
+}
+
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum AssetCategory {
@@ -7612,6 +7849,8 @@ pub enum PageAction {
         selector: Option<String>,
     },
     Screenshot {
+        #[serde(rename = "fullPage")]
+        #[serde(alias = "full_page")]
         full_page: Option<bool>,
     },
     #[serde(rename = "executeJs")]
@@ -7735,11 +7974,11 @@ fn batch_crawl(engine: CrawlEngineHandle, urls: Vec<String>) -> Result<BatchCraw
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::batch_crawl(&engine.inner, urls).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::batch_crawl(&engine.inner, urls).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7749,11 +7988,11 @@ fn batch_crawl_async(engine: CrawlEngineHandle, urls: Vec<String>) -> Result<Bat
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::batch_crawl(&engine.inner, urls).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::batch_crawl(&engine.inner, urls).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7763,11 +8002,11 @@ fn batch_scrape(engine: CrawlEngineHandle, urls: Vec<String>) -> Result<BatchScr
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::batch_scrape(&engine.inner, urls).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::batch_scrape(&engine.inner, urls).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7777,11 +8016,11 @@ fn batch_scrape_async(engine: CrawlEngineHandle, urls: Vec<String>) -> Result<Ba
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::batch_scrape(&engine.inner, urls).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::batch_scrape(&engine.inner, urls).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7791,11 +8030,11 @@ fn crawl(engine: CrawlEngineHandle, url: String) -> Result<CrawlResult, Error> {
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::crawl(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::crawl(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7805,11 +8044,11 @@ fn crawl_async(engine: CrawlEngineHandle, url: String) -> Result<CrawlResult, Er
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::crawl(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::crawl(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7826,12 +8065,7 @@ fn create_engine(args: &[magnus::Value]) -> Result<CrawlEngineHandle, Error> {
         }
         _ => None,
     };
-    let result = crawlberg::create_engine(config.map(Into::into)).map_err(|e| {
-        magnus::Error::new(
-            unsafe { Ruby::get_unchecked() }.exception_runtime_error(),
-            e.to_string(),
-        )
-    })?;
+    let result = crawlberg::create_engine(config.map(Into::into)).map_err(alef_magnus_err)?;
     Ok(CrawlEngineHandle {
         inner: Arc::new(result),
     })
@@ -7848,11 +8082,11 @@ fn interact(engine: CrawlEngineHandle, url: String, actions: Vec<PageAction>) ->
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::interact(&engine.inner, &url, actions_core).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::interact(&engine.inner, &url, actions_core).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7867,11 +8101,11 @@ fn interact_async(
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::interact(&engine.inner, &url, actions_core).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::interact(&engine.inner, &url, actions_core).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7881,11 +8115,11 @@ fn map_urls(engine: CrawlEngineHandle, url: String) -> Result<MapResult, Error>
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::map_urls(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::map_urls(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7895,11 +8129,11 @@ fn map_urls_async(engine: CrawlEngineHandle, url: String) -> Result<MapResult, E
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::map_urls(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::map_urls(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7909,11 +8143,11 @@ fn scrape(engine: CrawlEngineHandle, url: String) -> Result<ScrapeResult, Error>
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::scrape(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::scrape(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7923,11 +8157,11 @@ fn scrape_async(engine: CrawlEngineHandle, url: String) -> Result<ScrapeResult,
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::scrape(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::scrape(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -8136,6 +8370,9 @@ impl From<BrowserConfig> for crawlberg::BrowserConfig {
             session_affinity: val.session_affinity,
             chrome_path: val.chrome_path.map(Into::into),
             chrome_args: val.chrome_args.into_iter().collect(),
+            scroll_containers: val.scroll_containers,
+            dom_quiet_window: std::time::Duration::from_millis(val.dom_quiet_window),
+            dom_quiet_cap: std::time::Duration::from_millis(val.dom_quiet_cap),
             ..Default::default()
         }
     }
@@ -8162,6 +8399,9 @@ impl From<crawlberg::BrowserConfig> for BrowserConfig {
             session_affinity: val.session_affinity,
             chrome_path: val.chrome_path.map(|p| p.to_string_lossy().to_string()),
             chrome_args: val.chrome_args.into_iter().collect(),
+            scroll_containers: val.scroll_containers,
+            dom_quiet_window: val.dom_quiet_window.as_millis() as u64,
+            dom_quiet_cap: val.dom_quiet_cap.as_millis() as u64,
         }
     }
 }
@@ -8258,6 +8498,10 @@ impl From<ContentConfig> for crawlberg::ContentConfig {
             compact_tables: val.compact_tables,
             keep_inline_images_in: val.keep_inline_images_in.into_iter().collect(),
             keep_hidden: val.keep_hidden,
+            expand_abbreviations: val.expand_abbreviations,
+            escape_misc: val.escape_misc,
+            escape_asterisks: val.escape_asterisks,
+            include_icon_labels: val.include_icon_labels,
             ..Default::default()
         }
     }
@@ -8286,6 +8530,10 @@ impl From<crawlberg::ContentConfig> for ContentConfig {
             compact_tables: val.compact_tables,
             keep_inline_images_in: val.keep_inline_images_in.into_iter().collect(),
             keep_hidden: val.keep_hidden,
+            expand_abbreviations: val.expand_abbreviations,
+            escape_misc: val.escape_misc,
+            escape_asterisks: val.escape_asterisks,
+            include_icon_labels: val.include_icon_labels,
         }
     }
 }
@@ -8379,6 +8627,8 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
             save_browser_profile: val.save_browser_profile,
             ssrf: val.ssrf.into(),
             ssrf_deny_private_explicit: val.ssrf_deny_private_explicit,
+            danger_accept_invalid_certs: val.danger_accept_invalid_certs,
+            subdomain_header_allowlist: val.subdomain_header_allowlist.into_iter().collect(),
             ..Default::default()
         }
     }
@@ -8447,6 +8697,8 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
             save_browser_profile: val.save_browser_profile,
             ssrf: val.ssrf.into(),
             ssrf_deny_private_explicit: val.ssrf_deny_private_explicit,
+            danger_accept_invalid_certs: val.danger_accept_invalid_certs,
+            subdomain_header_allowlist: val.subdomain_header_allowlist.into_iter().collect(),
         }
     }
 }
@@ -8536,6 +8788,7 @@ impl From<CrawlResult> for crawlberg::CrawlResult {
             cookies: val.cookies.into_iter().map(Into::into).collect(),
             stayed_on_domain: val.stayed_on_domain,
             browser_used: val.browser_used,
+            waf_block: val.waf_block.map(Into::into),
             ..Default::default()
         }
     }
@@ -8553,6 +8806,7 @@ impl From<crawlberg::CrawlResult> for CrawlResult {
             cookies: val.cookies.into_iter().map(Into::into).collect(),
             stayed_on_domain: val.stayed_on_domain,
             browser_used: val.browser_used,
+            waf_block: val.waf_block.map(Into::into),
         }
     }
 }
@@ -9214,6 +9468,29 @@ impl From<crawlberg::SsrfPolicy> for SsrfPolicy {
     }
 }
 
+#[allow(clippy::needless_update)]
+impl From<WafBlockDetails> for crawlberg::WafBlockDetails {
+    fn from(val: WafBlockDetails) -> Self {
+        Self {
+            vendor: val.vendor,
+            status: val.status,
+            body_prefix: val.body_prefix,
+            ..Default::default()
+        }
+    }
+}
+
+#[allow(clippy::redundant_closure)]
+impl From<crawlberg::WafBlockDetails> for WafBlockDetails {
+    fn from(val: crawlberg::WafBlockDetails) -> Self {
+        Self {
+            vendor: val.vendor.to_string(),
+            status: val.status,
+            body_prefix: val.body_prefix.map(|v| v.to_string()),
+        }
+    }
+}
+
 impl From<AssetCategory> for crawlberg::AssetCategory {
     fn from(val: AssetCategory) -> Self {
         match val {
@@ -9570,18 +9847,92 @@ impl From<crawlberg::ScrollDirection> for ScrollDirection {
     }
 }
 
-/// Convert a `crawlberg::CrawlError` error to a Magnus runtime error.
+/// Convert the error a core call returned into the typed Ruby exception of its family. An error
+/// of a type with no converter becomes a `RuntimeError` with the message.
+#[allow(dead_code)]
+fn alef_magnus_err<E: std::fmt::Display + 'static>(e: E) -> magnus::Error {
+    let message = e.to_string();
+    #[allow(unused_variables)]
+    let any: Box<dyn std::any::Any> = Box::new(e);
+    let any = match any.downcast::<crawlberg::CrawlError>() {
+        Ok(typed) => return crawl_error_to_magnus_err(*typed),
+        Err(any) => any,
+    };
+    let any = match any.downcast::<crawlberg::SsrfError>() {
+        Ok(typed) => return ssrf_error_to_magnus_err(*typed),
+        Err(any) => any,
+    };
+    let _ = any;
+    magnus::Error::new(
+        unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(),
+        message,
+    )
+}
+
+/// Convert a `crawlberg::CrawlError` error to a typed Ruby exception.
 #[allow(dead_code)]
 fn crawl_error_to_magnus_err(e: crawlberg::CrawlError) -> magnus::Error {
-    let msg = e.to_string();
-    magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
+    use magnus::prelude::*;
+    let message = e.to_string();
+    let ruby = unsafe { magnus::Ruby::get_unchecked() };
+    #[allow(unreachable_patterns)]
+    let class_name = match &e {
+        crawlberg::CrawlError::NotFound { .. } => "NotFoundError",
+        crawlberg::CrawlError::Unauthorized { .. } => "UnauthorizedError",
+        crawlberg::CrawlError::Forbidden { .. } => "ForbiddenError",
+        crawlberg::CrawlError::WafBlocked { .. } => "WafBlockedError",
+        crawlberg::CrawlError::Timeout { .. } => "CrawlTimeoutError",
+        crawlberg::CrawlError::RateLimited { .. } => "RateLimitedError",
+        crawlberg::CrawlError::ServerError { .. } => "ServerError",
+        crawlberg::CrawlError::BadGateway { .. } => "BadGatewayError",
+        crawlberg::CrawlError::Gone { .. } => "GoneError",
+        crawlberg::CrawlError::Connection { .. } => "CrawlConnectionError",
+        crawlberg::CrawlError::Dns { .. } => "DnsError",
+        crawlberg::CrawlError::Ssl { .. } => "SslError",
+        crawlberg::CrawlError::DataLoss { .. } => "DataLossError",
+        crawlberg::CrawlError::ConversionFailed { .. } => "ConversionFailedError",
+        crawlberg::CrawlError::BrowserError { .. } => "BrowserError",
+        crawlberg::CrawlError::BrowserTimeout { .. } => "BrowserTimeoutError",
+        crawlberg::CrawlError::InvalidConfig { .. } => "InvalidConfigError",
+        crawlberg::CrawlError::Unsupported { .. } => "UnsupportedError",
+        crawlberg::CrawlError::TooManyRedirects { .. } => "TooManyRedirectsError",
+        crawlberg::CrawlError::SsrfPolicyViolation { .. } => "SsrfPolicyViolationError",
+        crawlberg::CrawlError::Other { .. } => "OtherError",
+        _ => "CrawlError",
+    };
+    let build = || -> Result<magnus::Error, magnus::Error> {
+        let module = ruby.define_module("Crawlberg")?;
+        let class: magnus::ExceptionClass = module.const_get(class_name)?;
+        let exception = class.new_instance((message.as_str(),))?;
+        Ok(magnus::Error::from(exception))
+    };
+    build().unwrap_or_else(|_| magnus::Error::new(ruby.exception_runtime_error(), message))
 }
 
-/// Convert a `crawlberg::SsrfError` error to a Magnus runtime error.
+/// Convert a `crawlberg::SsrfError` error to a typed Ruby exception.
 #[allow(dead_code)]
 fn ssrf_error_to_magnus_err(e: crawlberg::SsrfError) -> magnus::Error {
-    let msg = e.to_string();
-    magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
+    use magnus::prelude::*;
+    let message = e.to_string();
+    let ruby = unsafe { magnus::Ruby::get_unchecked() };
+    #[allow(unreachable_patterns)]
+    let class_name = match &e {
+        crawlberg::SsrfError::DeniedByPolicy { .. } => "DeniedByPolicyError",
+        crawlberg::SsrfError::NotOnAllowlist => "NotOnAllowlistError",
+        crawlberg::SsrfError::InvalidCidr(..) => "InvalidCidrError",
+        crawlberg::SsrfError::DnsResolutionFailed(..) => "DnsResolutionFailedError",
+        crawlberg::SsrfError::InvalidUrl(..) => "InvalidUrlError",
+        crawlberg::SsrfError::DisallowedScheme(..) => "DisallowedSchemeError",
+        crawlberg::SsrfError::TooManyRedirects => "TooManyRedirectsError",
+        _ => "SsrfError",
+    };
+    let build = || -> Result<magnus::Error, magnus::Error> {
+        let module = ruby.define_module("Crawlberg")?;
+        let class: magnus::ExceptionClass = module.const_get(class_name)?;
+        let exception = class.new_instance((message.as_str(),))?;
+        Ok(magnus::Error::from(exception))
+    };
+    build().unwrap_or_else(|_| magnus::Error::new(ruby.exception_runtime_error(), message))
 }
 
 #[derive(Clone)]
@@ -9629,7 +9980,7 @@ impl CrawlStreamIterator {
                 let binding: CrawlEvent = chunk.into();
                 Ok(binding.into_value_with(&ruby))
             }
-            Some(Err(e)) => Err(Error::new(ruby.exception_runtime_error(), e.to_string())),
+            Some(Err(e)) => Err(alef_magnus_err(e)),
             None => {
                 // Drop the stream to release any resources.
                 let inner = self.inner.clone();
@@ -9711,7 +10062,7 @@ impl BatchCrawlStreamIterator {
                 let binding: CrawlEvent = chunk.into();
                 Ok(binding.into_value_with(&ruby))
             }
-            Some(Err(e)) => Err(Error::new(ruby.exception_runtime_error(), e.to_string())),
+            Some(Err(e)) => Err(alef_magnus_err(e)),
             None => {
                 // Drop the stream to release any resources.
                 let inner = self.inner.clone();
@@ -9876,6 +10227,12 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("chrome_args", method!(BrowserConfig::chrome_args, 0))?;
 
+    class.define_method("scroll_containers", method!(BrowserConfig::scroll_containers, 0))?;
+
+    class.define_method("dom_quiet_window", method!(BrowserConfig::dom_quiet_window, 0))?;
+
+    class.define_method("dom_quiet_cap", method!(BrowserConfig::dom_quiet_cap, 0))?;
+
     let class = module.define_class("BrowserExtras", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(BrowserExtras::new, -1))?;
@@ -9956,6 +10313,14 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("keep_hidden", method!(ContentConfig::keep_hidden, 0))?;
 
+    class.define_method("expand_abbreviations", method!(ContentConfig::expand_abbreviations, 0))?;
+
+    class.define_method("escape_misc", method!(ContentConfig::escape_misc, 0))?;
+
+    class.define_method("escape_asterisks", method!(ContentConfig::escape_asterisks, 0))?;
+
+    class.define_method("include_icon_labels", method!(ContentConfig::include_icon_labels, 0))?;
+
     let class = module.define_class("CookieInfo", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(CookieInfo::new, -1))?;
@@ -10100,6 +10465,16 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
         method!(CrawlConfig::ssrf_deny_private_explicit, 0),
     )?;
 
+    class.define_method(
+        "danger_accept_invalid_certs",
+        method!(CrawlConfig::danger_accept_invalid_certs, 0),
+    )?;
+
+    class.define_method(
+        "subdomain_header_allowlist",
+        method!(CrawlConfig::subdomain_header_allowlist, 0),
+    )?;
+
     class.define_method("validate", method!(CrawlConfig::validate, 0))?;
 
     let class = module.define_class("CrawlEngineHandle", ruby.class_object())?;
@@ -10185,6 +10560,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("browser_used", method!(CrawlResult::browser_used, 0))?;
 
+    class.define_method("waf_block", method!(CrawlResult::waf_block, 0))?;
+
     class.define_method(
         "unique_normalized_urls",
         method!(CrawlResult::unique_normalized_urls, 0),
@@ -10574,6 +10951,16 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("scheme_allowlist", method!(SsrfPolicy::scheme_allowlist, 0))?;
 
+    let class = module.define_class("WafBlockDetails", ruby.class_object())?;
+
+    class.define_singleton_method("new", function!(WafBlockDetails::new, -1))?;
+
+    class.define_method("vendor", method!(WafBlockDetails::vendor, 0))?;
+
+    class.define_method("status", method!(WafBlockDetails::status, 0))?;
+
+    class.define_method("body_prefix", method!(WafBlockDetails::body_prefix, 0))?;
+
     let class = module.define_class("CrawlStreamIterator", ruby.class_object())?;
     class.define_method("next_chunk", method!(CrawlStreamIterator::next_chunk, 0))?;
     class.define_method("each", method!(CrawlStreamIterator::each, 0))?;
@@ -10603,5 +10990,35 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     module.define_module_function("batch_crawl_stream", function!(batch_crawl_stream, 2))?;
 
+    let crawl_error_class = module.define_error("CrawlError", ruby.exception_standard_error())?;
+    let not_found_error_class = module.define_error("NotFoundError", crawl_error_class)?;
+    let unauthorized_error_class = module.define_error("UnauthorizedError", crawl_error_class)?;
+    let forbidden_error_class = module.define_error("ForbiddenError", crawl_error_class)?;
+    let waf_blocked_error_class = module.define_error("WafBlockedError", crawl_error_class)?;
+    let crawl_timeout_error_class = module.define_error("CrawlTimeoutError", crawl_error_class)?;
+    let rate_limited_error_class = module.define_error("RateLimitedError", crawl_error_class)?;
+    let server_error_class = module.define_error("ServerError", crawl_error_class)?;
+    let bad_gateway_error_class = module.define_error("BadGatewayError", crawl_error_class)?;
+    let gone_error_class = module.define_error("GoneError", crawl_error_class)?;
+    let crawl_connection_error_class = module.define_error("CrawlConnectionError", crawl_error_class)?;
+    let dns_error_class = module.define_error("DnsError", crawl_error_class)?;
+    let ssl_error_class = module.define_error("SslError", crawl_error_class)?;
+    let data_loss_error_class = module.define_error("DataLossError", crawl_error_class)?;
+    let conversion_failed_error_class = module.define_error("ConversionFailedError", crawl_error_class)?;
+    let browser_error_class = module.define_error("BrowserError", crawl_error_class)?;
+    let browser_timeout_error_class = module.define_error("BrowserTimeoutError", crawl_error_class)?;
+    let invalid_config_error_class = module.define_error("InvalidConfigError", crawl_error_class)?;
+    let unsupported_error_class = module.define_error("UnsupportedError", crawl_error_class)?;
+    let ssrf_policy_violation_error_class = module.define_error("SsrfPolicyViolationError", crawl_error_class)?;
+    let other_error_class = module.define_error("OtherError", crawl_error_class)?;
+    let ssrf_error_class = module.define_error("SsrfError", crawl_error_class)?;
+    let denied_by_policy_error_class = module.define_error("DeniedByPolicyError", ssrf_error_class)?;
+    let not_on_allowlist_error_class = module.define_error("NotOnAllowlistError", ssrf_error_class)?;
+    let invalid_cidr_error_class = module.define_error("InvalidCidrError", ssrf_error_class)?;
+    let dns_resolution_failed_error_class = module.define_error("DnsResolutionFailedError", ssrf_error_class)?;
+    let invalid_url_error_class = module.define_error("InvalidUrlError", ssrf_error_class)?;
+    let disallowed_scheme_error_class = module.define_error("DisallowedSchemeError", ssrf_error_class)?;
+    let too_many_redirects_error_class = module.define_error("TooManyRedirectsError", ssrf_error_class)?;
+
     Ok(())
 }