crawlberg 1.12.0 → 1.12.3

data/ext/crawlberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:582ed106e4be9129fdfae3b203c9f9ccb338efae1989ad0fdde9d99890d81bd7
+// alef:hash:5766b3fb05a187f88eec0ae0ee3c5e32d6bede7f1a3bc38b4305587073983bbd
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -845,6 +845,7 @@ pub struct BrowserConfig {
     chrome_args: Vec<String>,
     scroll_containers: bool,
     dom_quiet_window: u64,
+    dom_quiet_cap: u64,
 }
 
 unsafe impl IntoValueFromNative for BrowserConfig {}
@@ -896,6 +897,7 @@ impl std::fmt::Debug for BrowserConfig {
         debug.field("chrome_args", &"<redacted>");
         debug.field("scroll_containers", &self.scroll_containers);
         debug.field("dom_quiet_window", &self.dom_quiet_window);
+        debug.field("dom_quiet_cap", &self.dom_quiet_cap);
         debug.finish()
     }
 }
@@ -1092,6 +1094,15 @@ impl BrowserConfig {
                 })?,
                 None => 150,
             },
+            dom_quiet_cap: match kwargs.get(ruby.to_symbol("dom_quiet_cap")) {
+                Some(v) => u64::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `dom_quiet_cap`: {}", e),
+                    )
+                })?,
+                None => 4000,
+            },
         })
     }
 
@@ -1170,6 +1181,10 @@ impl BrowserConfig {
     fn dom_quiet_window(&self) -> u64 {
         self.dom_quiet_window.clone()
     }
+
+    fn dom_quiet_cap(&self) -> u64 {
+        self.dom_quiet_cap.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize)]
@@ -1474,6 +1489,10 @@ pub struct ContentConfig {
     compact_tables: bool,
     keep_inline_images_in: Vec<String>,
     keep_hidden: bool,
+    expand_abbreviations: bool,
+    escape_misc: bool,
+    escape_asterisks: bool,
+    include_icon_labels: bool,
 }
 
 unsafe impl IntoValueFromNative for ContentConfig {}
@@ -1705,6 +1724,42 @@ impl ContentConfig {
                 })?,
                 None => false,
             },
+            expand_abbreviations: match kwargs.get(ruby.to_symbol("expand_abbreviations")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `expand_abbreviations`: {}", e),
+                    )
+                })?,
+                None => true,
+            },
+            escape_misc: match kwargs.get(ruby.to_symbol("escape_misc")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `escape_misc`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            escape_asterisks: match kwargs.get(ruby.to_symbol("escape_asterisks")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `escape_asterisks`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            include_icon_labels: match kwargs.get(ruby.to_symbol("include_icon_labels")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `include_icon_labels`: {}", e),
+                    )
+                })?,
+                None => true,
+            },
         })
     }
 
@@ -1787,6 +1842,22 @@ impl ContentConfig {
     fn keep_hidden(&self) -> bool {
         self.keep_hidden
     }
+
+    fn expand_abbreviations(&self) -> bool {
+        self.expand_abbreviations
+    }
+
+    fn escape_misc(&self) -> bool {
+        self.escape_misc
+    }
+
+    fn escape_asterisks(&self) -> bool {
+        self.escape_asterisks
+    }
+
+    fn include_icon_labels(&self) -> bool {
+        self.include_icon_labels
+    }
 }
 
 #[derive(Clone, serde::Serialize, serde::Deserialize)]
@@ -1967,6 +2038,7 @@ pub struct CrawlConfig {
     ssrf: SsrfPolicy,
     ssrf_deny_private_explicit: Option<bool>,
     danger_accept_invalid_certs: bool,
+    subdomain_header_allowlist: Vec<String>,
 }
 
 unsafe impl IntoValueFromNative for CrawlConfig {}
@@ -2055,6 +2127,7 @@ impl std::fmt::Debug for CrawlConfig {
         debug.field("ssrf", &self.ssrf);
         debug.field("ssrf_deny_private_explicit", &self.ssrf_deny_private_explicit);
         debug.field("danger_accept_invalid_certs", &self.danger_accept_invalid_certs);
+        debug.field("subdomain_header_allowlist", &self.subdomain_header_allowlist);
         debug.finish()
     }
 }
@@ -2590,6 +2663,15 @@ impl CrawlConfig {
                 })?,
                 None => false,
             },
+            subdomain_header_allowlist: match kwargs.get(ruby.to_symbol("subdomain_header_allowlist")) {
+                Some(v) => <Vec<String>>::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `subdomain_header_allowlist`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
         })
     }
 
@@ -2817,6 +2899,10 @@ impl CrawlConfig {
         self.danger_accept_invalid_certs
     }
 
+    fn subdomain_header_allowlist(&self) -> Vec<String> {
+        self.subdomain_header_allowlist.clone()
+    }
+
     fn validate(&self) -> Result<(), Error> {
         #[allow(clippy::needless_update)]
         let core_self = crawlberg::CrawlConfig {
@@ -2937,14 +3023,11 @@ impl CrawlConfig {
 
             danger_accept_invalid_certs: self.danger_accept_invalid_certs,
 
+            subdomain_header_allowlist: self.subdomain_header_allowlist.clone(),
+
             ..Default::default()
         };
-        let result = core_self.validate().map_err(|e| {
-            magnus::Error::new(
-                unsafe { Ruby::get_unchecked() }.exception_runtime_error(),
-                e.to_string(),
-            )
-        })?;
+        let result = core_self.validate().map_err(alef_magnus_err)?;
         Ok(result)
     }
 }
@@ -2995,12 +3078,7 @@ impl CrawlEngineHandle {
         );
         let stream = runtime
             .block_on(async { inner.crawl_stream(core_req).await })
-            .map_err(|e| {
-                magnus::Error::new(
-                    unsafe { Ruby::get_unchecked() }.exception_runtime_error(),
-                    e.to_string(),
-                )
-            })?;
+            .map_err(alef_magnus_err)?;
         let iterator = CrawlStreamIterator {
             inner: Arc::new(tokio::sync::Mutex::new(Some(stream))),
             runtime,
@@ -3040,12 +3118,7 @@ impl CrawlEngineHandle {
         );
         let stream = runtime
             .block_on(async { inner.batch_crawl_stream(core_req).await })
-            .map_err(|e| {
-                magnus::Error::new(
-                    unsafe { Ruby::get_unchecked() }.exception_runtime_error(),
-                    e.to_string(),
-                )
-            })?;
+            .map_err(alef_magnus_err)?;
         let iterator = BatchCrawlStreamIterator {
             inner: Arc::new(tokio::sync::Mutex::new(Some(stream))),
             runtime,
@@ -7776,6 +7849,8 @@ pub enum PageAction {
         selector: Option<String>,
     },
     Screenshot {
+        #[serde(rename = "fullPage")]
+        #[serde(alias = "full_page")]
         full_page: Option<bool>,
     },
     #[serde(rename = "executeJs")]
@@ -7899,11 +7974,11 @@ fn batch_crawl(engine: CrawlEngineHandle, urls: Vec<String>) -> Result<BatchCraw
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::batch_crawl(&engine.inner, urls).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::batch_crawl(&engine.inner, urls).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7913,11 +7988,11 @@ fn batch_crawl_async(engine: CrawlEngineHandle, urls: Vec<String>) -> Result<Bat
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::batch_crawl(&engine.inner, urls).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::batch_crawl(&engine.inner, urls).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7927,11 +8002,11 @@ fn batch_scrape(engine: CrawlEngineHandle, urls: Vec<String>) -> Result<BatchScr
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::batch_scrape(&engine.inner, urls).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::batch_scrape(&engine.inner, urls).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7941,11 +8016,11 @@ fn batch_scrape_async(engine: CrawlEngineHandle, urls: Vec<String>) -> Result<Ba
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::batch_scrape(&engine.inner, urls).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::batch_scrape(&engine.inner, urls).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7955,11 +8030,11 @@ fn crawl(engine: CrawlEngineHandle, url: String) -> Result<CrawlResult, Error> {
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::crawl(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::crawl(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7969,11 +8044,11 @@ fn crawl_async(engine: CrawlEngineHandle, url: String) -> Result<CrawlResult, Er
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::crawl(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::crawl(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -7990,12 +8065,7 @@ fn create_engine(args: &[magnus::Value]) -> Result<CrawlEngineHandle, Error> {
         }
         _ => None,
     };
-    let result = crawlberg::create_engine(config.map(Into::into)).map_err(|e| {
-        magnus::Error::new(
-            unsafe { Ruby::get_unchecked() }.exception_runtime_error(),
-            e.to_string(),
-        )
-    })?;
+    let result = crawlberg::create_engine(config.map(Into::into)).map_err(alef_magnus_err)?;
     Ok(CrawlEngineHandle {
         inner: Arc::new(result),
     })
@@ -8012,11 +8082,11 @@ fn interact(engine: CrawlEngineHandle, url: String, actions: Vec<PageAction>) ->
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::interact(&engine.inner, &url, actions_core).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::interact(&engine.inner, &url, actions_core).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -8031,11 +8101,11 @@ fn interact_async(
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::interact(&engine.inner, &url, actions_core).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::interact(&engine.inner, &url, actions_core).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -8045,11 +8115,11 @@ fn map_urls(engine: CrawlEngineHandle, url: String) -> Result<MapResult, Error>
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::map_urls(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::map_urls(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -8059,11 +8129,11 @@ fn map_urls_async(engine: CrawlEngineHandle, url: String) -> Result<MapResult, E
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::map_urls(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::map_urls(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -8073,11 +8143,11 @@ fn scrape(engine: CrawlEngineHandle, url: String) -> Result<ScrapeResult, Error>
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::scrape(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::scrape(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -8087,11 +8157,11 @@ fn scrape_async(engine: CrawlEngineHandle, url: String) -> Result<ScrapeResult,
             .enable_all()
             .build()
             .map_err(|e| e.to_string())?;
-        rt.block_on(async { crawlberg::scrape(&engine.inner, &url).await })
-            .map_err(|e| e.to_string())
+        Ok(rt.block_on(async { crawlberg::scrape(&engine.inner, &url).await }))
     });
     let result = async_result
         .map_err(|message| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_runtime_error(), message))?;
+    let result = result.map_err(alef_magnus_err)?;
     Ok(result.into())
 }
 
@@ -8302,6 +8372,7 @@ impl From<BrowserConfig> for crawlberg::BrowserConfig {
             chrome_args: val.chrome_args.into_iter().collect(),
             scroll_containers: val.scroll_containers,
             dom_quiet_window: std::time::Duration::from_millis(val.dom_quiet_window),
+            dom_quiet_cap: std::time::Duration::from_millis(val.dom_quiet_cap),
             ..Default::default()
         }
     }
@@ -8330,6 +8401,7 @@ impl From<crawlberg::BrowserConfig> for BrowserConfig {
             chrome_args: val.chrome_args.into_iter().collect(),
             scroll_containers: val.scroll_containers,
             dom_quiet_window: val.dom_quiet_window.as_millis() as u64,
+            dom_quiet_cap: val.dom_quiet_cap.as_millis() as u64,
         }
     }
 }
@@ -8426,6 +8498,10 @@ impl From<ContentConfig> for crawlberg::ContentConfig {
             compact_tables: val.compact_tables,
             keep_inline_images_in: val.keep_inline_images_in.into_iter().collect(),
             keep_hidden: val.keep_hidden,
+            expand_abbreviations: val.expand_abbreviations,
+            escape_misc: val.escape_misc,
+            escape_asterisks: val.escape_asterisks,
+            include_icon_labels: val.include_icon_labels,
             ..Default::default()
         }
     }
@@ -8454,6 +8530,10 @@ impl From<crawlberg::ContentConfig> for ContentConfig {
             compact_tables: val.compact_tables,
             keep_inline_images_in: val.keep_inline_images_in.into_iter().collect(),
             keep_hidden: val.keep_hidden,
+            expand_abbreviations: val.expand_abbreviations,
+            escape_misc: val.escape_misc,
+            escape_asterisks: val.escape_asterisks,
+            include_icon_labels: val.include_icon_labels,
         }
     }
 }
@@ -8548,6 +8628,7 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
             ssrf: val.ssrf.into(),
             ssrf_deny_private_explicit: val.ssrf_deny_private_explicit,
             danger_accept_invalid_certs: val.danger_accept_invalid_certs,
+            subdomain_header_allowlist: val.subdomain_header_allowlist.into_iter().collect(),
             ..Default::default()
         }
     }
@@ -8617,6 +8698,7 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
             ssrf: val.ssrf.into(),
             ssrf_deny_private_explicit: val.ssrf_deny_private_explicit,
             danger_accept_invalid_certs: val.danger_accept_invalid_certs,
+            subdomain_header_allowlist: val.subdomain_header_allowlist.into_iter().collect(),
         }
     }
 }
@@ -9765,18 +9847,92 @@ impl From<crawlberg::ScrollDirection> for ScrollDirection {
     }
 }
 
-/// Convert a `crawlberg::CrawlError` error to a Magnus runtime error.
+/// Convert the error a core call returned into the typed Ruby exception of its family. An error
+/// of a type with no converter becomes a `RuntimeError` with the message.
+#[allow(dead_code)]
+fn alef_magnus_err<E: std::fmt::Display + 'static>(e: E) -> magnus::Error {
+    let message = e.to_string();
+    #[allow(unused_variables)]
+    let any: Box<dyn std::any::Any> = Box::new(e);
+    let any = match any.downcast::<crawlberg::CrawlError>() {
+        Ok(typed) => return crawl_error_to_magnus_err(*typed),
+        Err(any) => any,
+    };
+    let any = match any.downcast::<crawlberg::SsrfError>() {
+        Ok(typed) => return ssrf_error_to_magnus_err(*typed),
+        Err(any) => any,
+    };
+    let _ = any;
+    magnus::Error::new(
+        unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(),
+        message,
+    )
+}
+
+/// Convert a `crawlberg::CrawlError` error to a typed Ruby exception.
 #[allow(dead_code)]
 fn crawl_error_to_magnus_err(e: crawlberg::CrawlError) -> magnus::Error {
-    let msg = e.to_string();
-    magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
+    use magnus::prelude::*;
+    let message = e.to_string();
+    let ruby = unsafe { magnus::Ruby::get_unchecked() };
+    #[allow(unreachable_patterns)]
+    let class_name = match &e {
+        crawlberg::CrawlError::NotFound { .. } => "NotFoundError",
+        crawlberg::CrawlError::Unauthorized { .. } => "UnauthorizedError",
+        crawlberg::CrawlError::Forbidden { .. } => "ForbiddenError",
+        crawlberg::CrawlError::WafBlocked { .. } => "WafBlockedError",
+        crawlberg::CrawlError::Timeout { .. } => "CrawlTimeoutError",
+        crawlberg::CrawlError::RateLimited { .. } => "RateLimitedError",
+        crawlberg::CrawlError::ServerError { .. } => "ServerError",
+        crawlberg::CrawlError::BadGateway { .. } => "BadGatewayError",
+        crawlberg::CrawlError::Gone { .. } => "GoneError",
+        crawlberg::CrawlError::Connection { .. } => "CrawlConnectionError",
+        crawlberg::CrawlError::Dns { .. } => "DnsError",
+        crawlberg::CrawlError::Ssl { .. } => "SslError",
+        crawlberg::CrawlError::DataLoss { .. } => "DataLossError",
+        crawlberg::CrawlError::ConversionFailed { .. } => "ConversionFailedError",
+        crawlberg::CrawlError::BrowserError { .. } => "BrowserError",
+        crawlberg::CrawlError::BrowserTimeout { .. } => "BrowserTimeoutError",
+        crawlberg::CrawlError::InvalidConfig { .. } => "InvalidConfigError",
+        crawlberg::CrawlError::Unsupported { .. } => "UnsupportedError",
+        crawlberg::CrawlError::TooManyRedirects { .. } => "TooManyRedirectsError",
+        crawlberg::CrawlError::SsrfPolicyViolation { .. } => "SsrfPolicyViolationError",
+        crawlberg::CrawlError::Other { .. } => "OtherError",
+        _ => "CrawlError",
+    };
+    let build = || -> Result<magnus::Error, magnus::Error> {
+        let module = ruby.define_module("Crawlberg")?;
+        let class: magnus::ExceptionClass = module.const_get(class_name)?;
+        let exception = class.new_instance((message.as_str(),))?;
+        Ok(magnus::Error::from(exception))
+    };
+    build().unwrap_or_else(|_| magnus::Error::new(ruby.exception_runtime_error(), message))
 }
 
-/// Convert a `crawlberg::SsrfError` error to a Magnus runtime error.
+/// Convert a `crawlberg::SsrfError` error to a typed Ruby exception.
 #[allow(dead_code)]
 fn ssrf_error_to_magnus_err(e: crawlberg::SsrfError) -> magnus::Error {
-    let msg = e.to_string();
-    magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
+    use magnus::prelude::*;
+    let message = e.to_string();
+    let ruby = unsafe { magnus::Ruby::get_unchecked() };
+    #[allow(unreachable_patterns)]
+    let class_name = match &e {
+        crawlberg::SsrfError::DeniedByPolicy { .. } => "DeniedByPolicyError",
+        crawlberg::SsrfError::NotOnAllowlist => "NotOnAllowlistError",
+        crawlberg::SsrfError::InvalidCidr(..) => "InvalidCidrError",
+        crawlberg::SsrfError::DnsResolutionFailed(..) => "DnsResolutionFailedError",
+        crawlberg::SsrfError::InvalidUrl(..) => "InvalidUrlError",
+        crawlberg::SsrfError::DisallowedScheme(..) => "DisallowedSchemeError",
+        crawlberg::SsrfError::TooManyRedirects => "TooManyRedirectsError",
+        _ => "SsrfError",
+    };
+    let build = || -> Result<magnus::Error, magnus::Error> {
+        let module = ruby.define_module("Crawlberg")?;
+        let class: magnus::ExceptionClass = module.const_get(class_name)?;
+        let exception = class.new_instance((message.as_str(),))?;
+        Ok(magnus::Error::from(exception))
+    };
+    build().unwrap_or_else(|_| magnus::Error::new(ruby.exception_runtime_error(), message))
 }
 
 #[derive(Clone)]
@@ -9824,7 +9980,7 @@ impl CrawlStreamIterator {
                 let binding: CrawlEvent = chunk.into();
                 Ok(binding.into_value_with(&ruby))
             }
-            Some(Err(e)) => Err(Error::new(ruby.exception_runtime_error(), e.to_string())),
+            Some(Err(e)) => Err(alef_magnus_err(e)),
             None => {
                 // Drop the stream to release any resources.
                 let inner = self.inner.clone();
@@ -9906,7 +10062,7 @@ impl BatchCrawlStreamIterator {
                 let binding: CrawlEvent = chunk.into();
                 Ok(binding.into_value_with(&ruby))
             }
-            Some(Err(e)) => Err(Error::new(ruby.exception_runtime_error(), e.to_string())),
+            Some(Err(e)) => Err(alef_magnus_err(e)),
             None => {
                 // Drop the stream to release any resources.
                 let inner = self.inner.clone();
@@ -10075,6 +10231,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("dom_quiet_window", method!(BrowserConfig::dom_quiet_window, 0))?;
 
+    class.define_method("dom_quiet_cap", method!(BrowserConfig::dom_quiet_cap, 0))?;
+
     let class = module.define_class("BrowserExtras", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(BrowserExtras::new, -1))?;
@@ -10155,6 +10313,14 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("keep_hidden", method!(ContentConfig::keep_hidden, 0))?;
 
+    class.define_method("expand_abbreviations", method!(ContentConfig::expand_abbreviations, 0))?;
+
+    class.define_method("escape_misc", method!(ContentConfig::escape_misc, 0))?;
+
+    class.define_method("escape_asterisks", method!(ContentConfig::escape_asterisks, 0))?;
+
+    class.define_method("include_icon_labels", method!(ContentConfig::include_icon_labels, 0))?;
+
     let class = module.define_class("CookieInfo", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(CookieInfo::new, -1))?;
@@ -10304,6 +10470,11 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
         method!(CrawlConfig::danger_accept_invalid_certs, 0),
     )?;
 
+    class.define_method(
+        "subdomain_header_allowlist",
+        method!(CrawlConfig::subdomain_header_allowlist, 0),
+    )?;
+
     class.define_method("validate", method!(CrawlConfig::validate, 0))?;
 
     let class = module.define_class("CrawlEngineHandle", ruby.class_object())?;
@@ -10819,5 +10990,35 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     module.define_module_function("batch_crawl_stream", function!(batch_crawl_stream, 2))?;
 
+    let crawl_error_class = module.define_error("CrawlError", ruby.exception_standard_error())?;
+    let not_found_error_class = module.define_error("NotFoundError", crawl_error_class)?;
+    let unauthorized_error_class = module.define_error("UnauthorizedError", crawl_error_class)?;
+    let forbidden_error_class = module.define_error("ForbiddenError", crawl_error_class)?;
+    let waf_blocked_error_class = module.define_error("WafBlockedError", crawl_error_class)?;
+    let crawl_timeout_error_class = module.define_error("CrawlTimeoutError", crawl_error_class)?;
+    let rate_limited_error_class = module.define_error("RateLimitedError", crawl_error_class)?;
+    let server_error_class = module.define_error("ServerError", crawl_error_class)?;
+    let bad_gateway_error_class = module.define_error("BadGatewayError", crawl_error_class)?;
+    let gone_error_class = module.define_error("GoneError", crawl_error_class)?;
+    let crawl_connection_error_class = module.define_error("CrawlConnectionError", crawl_error_class)?;
+    let dns_error_class = module.define_error("DnsError", crawl_error_class)?;
+    let ssl_error_class = module.define_error("SslError", crawl_error_class)?;
+    let data_loss_error_class = module.define_error("DataLossError", crawl_error_class)?;
+    let conversion_failed_error_class = module.define_error("ConversionFailedError", crawl_error_class)?;
+    let browser_error_class = module.define_error("BrowserError", crawl_error_class)?;
+    let browser_timeout_error_class = module.define_error("BrowserTimeoutError", crawl_error_class)?;
+    let invalid_config_error_class = module.define_error("InvalidConfigError", crawl_error_class)?;
+    let unsupported_error_class = module.define_error("UnsupportedError", crawl_error_class)?;
+    let ssrf_policy_violation_error_class = module.define_error("SsrfPolicyViolationError", crawl_error_class)?;
+    let other_error_class = module.define_error("OtherError", crawl_error_class)?;
+    let ssrf_error_class = module.define_error("SsrfError", crawl_error_class)?;
+    let denied_by_policy_error_class = module.define_error("DeniedByPolicyError", ssrf_error_class)?;
+    let not_on_allowlist_error_class = module.define_error("NotOnAllowlistError", ssrf_error_class)?;
+    let invalid_cidr_error_class = module.define_error("InvalidCidrError", ssrf_error_class)?;
+    let dns_resolution_failed_error_class = module.define_error("DnsResolutionFailedError", ssrf_error_class)?;
+    let invalid_url_error_class = module.define_error("InvalidUrlError", ssrf_error_class)?;
+    let disallowed_scheme_error_class = module.define_error("DisallowedSchemeError", ssrf_error_class)?;
+    let too_many_redirects_error_class = module.define_error("TooManyRedirectsError", ssrf_error_class)?;
+
     Ok(())
 }

data/lib/crawlberg/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:ed92ed554ddf1dcb4d5a619d4a13d01d9287c96310437df1c7a4c484d27b969c
+# alef:hash:4b49e60e4c3bdd958dfccc4aee6deec2a78c41bafced6ee25ec8b66aa0b049ca
 # To regenerate: alef generate
 # To verify freshness: alef verify
 # frozen_string_literal: true
 
 module Crawlberg
   ## The version string for this package.
-  VERSION = "1.12.0"
+  VERSION = "1.12.3"
 end