crawlberg 1.7.2 → 1.10.1

data/ext/crawlberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:9563e80a11ae24fda8f480033a40a3dd9382715d77f7a2e60c254f7f3ab71ad5
+// alef:hash:51d55e2247bdc2514829f3eaafb5c8524624f269469856210cf0950e6c206e7f
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -821,7 +821,7 @@ impl BatchScrapeResults {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize)]
+#[derive(Clone, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::BrowserConfig")]
 pub struct BrowserConfig {
@@ -830,6 +830,8 @@ pub struct BrowserConfig {
     endpoint: Option<String>,
     #[serde(default = "default_timeout")]
     timeout: u64,
+    overall_timeout: u64,
+    shutdown_timeout: u64,
     wait: BrowserWait,
     wait_selector: Option<String>,
     extra_wait: Option<u64>,
@@ -839,6 +841,8 @@ pub struct BrowserConfig {
     robots_user_agent: Option<String>,
     capture_network_events: bool,
     session_affinity: bool,
+    chrome_path: Option<String>,
+    chrome_args: Vec<String>,
 }
 
 unsafe impl IntoValueFromNative for BrowserConfig {}
@@ -868,6 +872,29 @@ impl magnus::TryConvert for BrowserConfig {
 }
 
 unsafe impl TryConvertOwned for BrowserConfig {}
+impl std::fmt::Debug for BrowserConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut debug = f.debug_struct("BrowserConfig");
+        debug.field("mode", &self.mode);
+        debug.field("backend", &self.backend);
+        debug.field("endpoint", &"<redacted>");
+        debug.field("timeout", &self.timeout);
+        debug.field("overall_timeout", &self.overall_timeout);
+        debug.field("shutdown_timeout", &self.shutdown_timeout);
+        debug.field("wait", &self.wait);
+        debug.field("wait_selector", &self.wait_selector);
+        debug.field("extra_wait", &self.extra_wait);
+        debug.field("proxy", &self.proxy);
+        debug.field("block_url_patterns", &self.block_url_patterns);
+        debug.field("eval_script", &"<redacted>");
+        debug.field("robots_user_agent", &self.robots_user_agent);
+        debug.field("capture_network_events", &self.capture_network_events);
+        debug.field("session_affinity", &self.session_affinity);
+        debug.field("chrome_path", &self.chrome_path);
+        debug.field("chrome_args", &"<redacted>");
+        debug.finish()
+    }
+}
 impl<'de> serde::Deserialize<'de> for BrowserConfig {
     fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
     where
@@ -926,6 +953,24 @@ impl BrowserConfig {
                 })?,
                 None => 30000,
             },
+            overall_timeout: match kwargs.get(ruby.to_symbol("overall_timeout")) {
+                Some(v) => u64::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `overall_timeout`: {}", e),
+                    )
+                })?,
+                None => 60000,
+            },
+            shutdown_timeout: match kwargs.get(ruby.to_symbol("shutdown_timeout")) {
+                Some(v) => u64::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `shutdown_timeout`: {}", e),
+                    )
+                })?,
+                None => 5000,
+            },
             wait: match kwargs.get(ruby.to_symbol("wait")) {
                 Some(v) => BrowserWait::try_convert(v).map_err(|e| {
                     magnus::Error::new(
@@ -1007,6 +1052,24 @@ impl BrowserConfig {
                 })?,
                 None => true,
             },
+            chrome_path: match kwargs.get(ruby.to_symbol("chrome_path")).filter(|v| !v.is_nil()) {
+                Some(v) => Some(String::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `chrome_path`: {}", e),
+                    )
+                })?),
+                None => None,
+            },
+            chrome_args: match kwargs.get(ruby.to_symbol("chrome_args")) {
+                Some(v) => <Vec<String>>::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `chrome_args`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
         })
     }
 
@@ -1026,6 +1089,14 @@ impl BrowserConfig {
         self.timeout.clone()
     }
 
+    fn overall_timeout(&self) -> u64 {
+        self.overall_timeout.clone()
+    }
+
+    fn shutdown_timeout(&self) -> u64 {
+        self.shutdown_timeout.clone()
+    }
+
     fn wait(&self) -> BrowserWait {
         self.wait.clone()
     }
@@ -1061,6 +1132,14 @@ impl BrowserConfig {
     fn session_affinity(&self) -> bool {
         self.session_affinity
     }
+
+    fn chrome_path(&self) -> Option<String> {
+        self.chrome_path.clone()
+    }
+
+    fn chrome_args(&self) -> Vec<String> {
+        self.chrome_args.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize)]
@@ -1357,6 +1436,7 @@ pub struct ContentConfig {
     wrap: bool,
     wrap_width: usize,
     include_document_structure: bool,
+    extract_metadata: bool,
 }
 
 unsafe impl IntoValueFromNative for ContentConfig {}
@@ -1516,6 +1596,15 @@ impl ContentConfig {
                 })?,
                 None => true,
             },
+            extract_metadata: match kwargs.get(ruby.to_symbol("extract_metadata")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `extract_metadata`: {}", e),
+                    )
+                })?,
+                None => true,
+            },
         })
     }
 
@@ -1566,9 +1655,13 @@ impl ContentConfig {
     fn include_document_structure(&self) -> bool {
         self.include_document_structure
     }
+
+    fn extract_metadata(&self) -> bool {
+        self.extract_metadata
+    }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, serde::Serialize, serde::Deserialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::CookieInfo")]
 pub struct CookieInfo {
@@ -1605,6 +1698,16 @@ impl magnus::TryConvert for CookieInfo {
 }
 
 unsafe impl TryConvertOwned for CookieInfo {}
+impl std::fmt::Debug for CookieInfo {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut debug = f.debug_struct("CookieInfo");
+        debug.field("name", &self.name);
+        debug.field("value", &"<redacted>");
+        debug.field("domain", &self.domain);
+        debug.field("path", &self.path);
+        debug.finish()
+    }
+}
 
 impl Default for CookieInfo {
     fn default() -> Self {
@@ -1675,7 +1778,7 @@ impl CookieInfo {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, serde::Serialize, serde::Deserialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::CrawlConfig")]
 pub struct CrawlConfig {
@@ -1694,6 +1797,11 @@ pub struct CrawlConfig {
     allow_subdomains: bool,
     include_paths: Vec<String>,
     exclude_paths: Vec<String>,
+    path_patterns_match_query: bool,
+    path_patterns_match_url: bool,
+    dedup_include_query: bool,
+    strip_tracking_params: bool,
+    tracking_params: Vec<String>,
     custom_headers: HashMap<String, String>,
     #[serde(default = "default_timeout")]
     request_timeout: u64,
@@ -1701,6 +1809,9 @@ pub struct CrawlConfig {
     max_redirects: usize,
     retry_count: usize,
     retry_codes: Vec<u16>,
+    retry_initial_delay_ms: u64,
+    retry_max_delay_ms: u64,
+    rate_limit_jitter_ratio: f64,
     cookies_enabled: bool,
     auth: Option<AuthConfig>,
     max_body_size: Option<usize>,
@@ -1756,6 +1867,67 @@ impl magnus::TryConvert for CrawlConfig {
 }
 
 unsafe impl TryConvertOwned for CrawlConfig {}
+impl std::fmt::Debug for CrawlConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut debug = f.debug_struct("CrawlConfig");
+        debug.field("max_depth", &self.max_depth);
+        debug.field("max_pages", &self.max_pages);
+        debug.field("max_links_per_page", &self.max_links_per_page);
+        debug.field("max_concurrent", &self.max_concurrent);
+        debug.field("crawl_strategy", &self.crawl_strategy);
+        debug.field("content_filter", &self.content_filter);
+        debug.field("bm25_query", &self.bm25_query);
+        debug.field("bm25_threshold", &self.bm25_threshold);
+        debug.field("respect_robots_txt", &self.respect_robots_txt);
+        debug.field("soft_http_errors", &self.soft_http_errors);
+        debug.field("user_agent", &self.user_agent);
+        debug.field("stay_on_domain", &self.stay_on_domain);
+        debug.field("allow_subdomains", &self.allow_subdomains);
+        debug.field("include_paths", &self.include_paths);
+        debug.field("exclude_paths", &self.exclude_paths);
+        debug.field("path_patterns_match_query", &self.path_patterns_match_query);
+        debug.field("path_patterns_match_url", &self.path_patterns_match_url);
+        debug.field("dedup_include_query", &self.dedup_include_query);
+        debug.field("strip_tracking_params", &self.strip_tracking_params);
+        debug.field("tracking_params", &self.tracking_params);
+        debug.field("custom_headers", &"<redacted>");
+        debug.field("request_timeout", &self.request_timeout);
+        debug.field("rate_limit_ms", &self.rate_limit_ms);
+        debug.field("max_redirects", &self.max_redirects);
+        debug.field("retry_count", &self.retry_count);
+        debug.field("retry_codes", &self.retry_codes);
+        debug.field("retry_initial_delay_ms", &self.retry_initial_delay_ms);
+        debug.field("retry_max_delay_ms", &self.retry_max_delay_ms);
+        debug.field("rate_limit_jitter_ratio", &self.rate_limit_jitter_ratio);
+        debug.field("cookies_enabled", &self.cookies_enabled);
+        debug.field("auth", &self.auth);
+        debug.field("max_body_size", &self.max_body_size);
+        debug.field("remove_tags", &self.remove_tags);
+        debug.field("content", &self.content);
+        debug.field("map_limit", &self.map_limit);
+        debug.field("map_search", &self.map_search);
+        debug.field("download_assets", &self.download_assets);
+        debug.field("asset_types", &self.asset_types);
+        debug.field("max_asset_size", &self.max_asset_size);
+        debug.field("browser", &self.browser);
+        debug.field("proxy", &self.proxy);
+        debug.field("user_agents", &self.user_agents);
+        debug.field("capture_screenshot", &self.capture_screenshot);
+        debug.field("follow_document_urls", &self.follow_document_urls);
+        debug.field("document_url_depth", &self.document_url_depth);
+        debug.field("download_documents", &self.download_documents);
+        debug.field("document_max_size", &self.document_max_size);
+        debug.field("document_mime_types", &self.document_mime_types);
+        debug.field("document_output_dir", &self.document_output_dir);
+        debug.field("document_content_encoding", &self.document_content_encoding);
+        debug.field("warc_output", &self.warc_output);
+        debug.field("browser_profile", &self.browser_profile);
+        debug.field("save_browser_profile", &self.save_browser_profile);
+        debug.field("ssrf", &self.ssrf);
+        debug.field("ssrf_deny_private_explicit", &self.ssrf_deny_private_explicit);
+        debug.finish()
+    }
+}
 
 impl Default for CrawlConfig {
     fn default() -> Self {
@@ -1905,6 +2077,56 @@ impl CrawlConfig {
                 })?,
                 None => Default::default(),
             },
+            path_patterns_match_query: match kwargs.get(ruby.to_symbol("path_patterns_match_query")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `path_patterns_match_query`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            path_patterns_match_url: match kwargs.get(ruby.to_symbol("path_patterns_match_url")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `path_patterns_match_url`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            dedup_include_query: match kwargs.get(ruby.to_symbol("dedup_include_query")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `dedup_include_query`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            strip_tracking_params: match kwargs.get(ruby.to_symbol("strip_tracking_params")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `strip_tracking_params`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
+            tracking_params: match kwargs.get(ruby.to_symbol("tracking_params")) {
+                Some(v) => <Vec<String>>::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `tracking_params`: {}", e),
+                    )
+                })?,
+                None => vec![
+                    "utm_*".to_string(),
+                    "fbclid".to_string(),
+                    "gclid".to_string(),
+                    "ref".to_string(),
+                ],
+            },
             custom_headers: match kwargs.get(ruby.to_symbol("custom_headers")) {
                 Some(v) => <HashMap<String, String>>::try_convert(v).map_err(|e| {
                     magnus::Error::new(
@@ -1959,6 +2181,33 @@ impl CrawlConfig {
                 })?,
                 None => Default::default(),
             },
+            retry_initial_delay_ms: match kwargs.get(ruby.to_symbol("retry_initial_delay_ms")) {
+                Some(v) => u64::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `retry_initial_delay_ms`: {}", e),
+                    )
+                })?,
+                None => 100,
+            },
+            retry_max_delay_ms: match kwargs.get(ruby.to_symbol("retry_max_delay_ms")) {
+                Some(v) => u64::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `retry_max_delay_ms`: {}", e),
+                    )
+                })?,
+                None => 60000,
+            },
+            rate_limit_jitter_ratio: match kwargs.get(ruby.to_symbol("rate_limit_jitter_ratio")) {
+                Some(v) => f64::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `rate_limit_jitter_ratio`: {}", e),
+                    )
+                })?,
+                None => 0.0,
+            },
             cookies_enabled: match kwargs.get(ruby.to_symbol("cookies_enabled")) {
                 Some(v) => bool::try_convert(v).map_err(|e| {
                     magnus::Error::new(
@@ -2265,6 +2514,26 @@ impl CrawlConfig {
         self.exclude_paths.clone()
     }
 
+    fn path_patterns_match_query(&self) -> bool {
+        self.path_patterns_match_query
+    }
+
+    fn path_patterns_match_url(&self) -> bool {
+        self.path_patterns_match_url
+    }
+
+    fn dedup_include_query(&self) -> bool {
+        self.dedup_include_query
+    }
+
+    fn strip_tracking_params(&self) -> bool {
+        self.strip_tracking_params
+    }
+
+    fn tracking_params(&self) -> Vec<String> {
+        self.tracking_params.clone()
+    }
+
     fn custom_headers(&self) -> HashMap<String, String> {
         self.custom_headers.clone()
     }
@@ -2289,6 +2558,18 @@ impl CrawlConfig {
         self.retry_codes.clone()
     }
 
+    fn retry_initial_delay_ms(&self) -> u64 {
+        self.retry_initial_delay_ms
+    }
+
+    fn retry_max_delay_ms(&self) -> u64 {
+        self.retry_max_delay_ms
+    }
+
+    fn rate_limit_jitter_ratio(&self) -> f64 {
+        self.rate_limit_jitter_ratio
+    }
+
     fn cookies_enabled(&self) -> bool {
         self.cookies_enabled
     }
@@ -2426,6 +2707,16 @@ impl CrawlConfig {
 
             exclude_paths: self.exclude_paths.clone(),
 
+            path_patterns_match_query: self.path_patterns_match_query,
+
+            path_patterns_match_url: self.path_patterns_match_url,
+
+            dedup_include_query: self.dedup_include_query,
+
+            strip_tracking_params: self.strip_tracking_params,
+
+            tracking_params: self.tracking_params.clone(),
+
             custom_headers: self
                 .custom_headers
                 .clone()
@@ -2443,6 +2734,12 @@ impl CrawlConfig {
 
             retry_codes: self.retry_codes.clone(),
 
+            retry_initial_delay_ms: self.retry_initial_delay_ms,
+
+            retry_max_delay_ms: self.retry_max_delay_ms,
+
+            rate_limit_jitter_ratio: self.rate_limit_jitter_ratio,
+
             cookies_enabled: self.cookies_enabled,
 
             auth: self.auth.clone().map(Into::into),
@@ -2619,7 +2916,7 @@ impl CrawlEngineHandle {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::CrawlPageResult")]
 pub struct CrawlPageResult {
@@ -2644,6 +2941,11 @@ pub struct CrawlPageResult {
     extraction_meta: Option<ExtractionMeta>,
     downloaded_document: Option<DownloadedDocument>,
     browser_used: bool,
+    final_url: String,
+    redirect_count: usize,
+    noindex_detected: bool,
+    nofollow_detected: bool,
+    ssrf_refused_urls: Vec<String>,
 }
 
 unsafe impl IntoValueFromNative for CrawlPageResult {}
@@ -2673,6 +2975,14 @@ impl magnus::TryConvert for CrawlPageResult {
 }
 
 unsafe impl TryConvertOwned for CrawlPageResult {}
+impl<'de> serde::Deserialize<'de> for CrawlPageResult {
+    fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        <crawlberg::CrawlPageResult as serde::Deserialize>::deserialize(deserializer).map(Into::into)
+    }
+}
 
 impl Default for CrawlPageResult {
     fn default() -> Self {
@@ -2879,6 +3189,51 @@ impl CrawlPageResult {
                 })?,
                 None => Default::default(),
             },
+            final_url: match kwargs.get(ruby.to_symbol("final_url")) {
+                Some(v) => String::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `final_url`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
+            redirect_count: match kwargs.get(ruby.to_symbol("redirect_count")) {
+                Some(v) => usize::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `redirect_count`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
+            noindex_detected: match kwargs.get(ruby.to_symbol("noindex_detected")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `noindex_detected`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
+            nofollow_detected: match kwargs.get(ruby.to_symbol("nofollow_detected")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `nofollow_detected`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
+            ssrf_refused_urls: match kwargs.get(ruby.to_symbol("ssrf_refused_urls")) {
+                Some(v) => <Vec<String>>::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `ssrf_refused_urls`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
         })
     }
 
@@ -2965,6 +3320,26 @@ impl CrawlPageResult {
     fn browser_used(&self) -> bool {
         self.browser_used
     }
+
+    fn final_url(&self) -> String {
+        self.final_url.clone()
+    }
+
+    fn redirect_count(&self) -> usize {
+        self.redirect_count
+    }
+
+    fn noindex_detected(&self) -> bool {
+        self.noindex_detected
+    }
+
+    fn nofollow_detected(&self) -> bool {
+        self.nofollow_detected
+    }
+
+    fn ssrf_refused_urls(&self) -> Vec<String> {
+        self.ssrf_refused_urls.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
@@ -3360,7 +3735,7 @@ impl DownloadedAsset {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize)]
+#[derive(Clone, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::DownloadedDocument")]
 pub struct DownloadedDocument {
@@ -3402,6 +3777,21 @@ impl magnus::TryConvert for DownloadedDocument {
 }
 
 unsafe impl TryConvertOwned for DownloadedDocument {}
+impl std::fmt::Debug for DownloadedDocument {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut debug = f.debug_struct("DownloadedDocument");
+        debug.field("url", &self.url);
+        debug.field("mime_type", &self.mime_type);
+        debug.field("size", &self.size);
+        debug.field("filename", &self.filename);
+        debug.field("content_hash", &self.content_hash);
+        debug.field("headers", &"<redacted>");
+        debug.field("truncated", &self.truncated);
+        debug.field("content_path", &self.content_path);
+        debug.field("content_base64", &self.content_base64);
+        debug.finish()
+    }
+}
 impl<'de> serde::Deserialize<'de> for DownloadedDocument {
     fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
     where
@@ -4145,7 +4535,7 @@ impl ImageInfo {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::InteractionResult")]
 pub struct InteractionResult {
@@ -4153,6 +4543,7 @@ pub struct InteractionResult {
     final_html: String,
     final_url: String,
     screenshot_base64: Option<String>,
+    ssrf_refused_urls: Vec<String>,
 }
 
 unsafe impl IntoValueFromNative for InteractionResult {}
@@ -4182,6 +4573,14 @@ impl magnus::TryConvert for InteractionResult {
 }
 
 unsafe impl TryConvertOwned for InteractionResult {}
+impl<'de> serde::Deserialize<'de> for InteractionResult {
+    fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        <crawlberg::InteractionResult as serde::Deserialize>::deserialize(deserializer).map(Into::into)
+    }
+}
 
 impl Default for InteractionResult {
     fn default() -> Self {
@@ -4232,6 +4631,15 @@ impl InteractionResult {
                 })?),
                 None => None,
             },
+            ssrf_refused_urls: match kwargs.get(ruby.to_symbol("ssrf_refused_urls")) {
+                Some(v) => <Vec<String>>::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `ssrf_refused_urls`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
         })
     }
 
@@ -4250,6 +4658,10 @@ impl InteractionResult {
     fn screenshot_base64(&self) -> Option<String> {
         self.screenshot_base64.clone()
     }
+
+    fn ssrf_refused_urls(&self) -> Vec<String> {
+        self.ssrf_refused_urls.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
@@ -5330,7 +5742,7 @@ impl PageMetadata {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, serde::Serialize, serde::Deserialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::ProxyConfig")]
 pub struct ProxyConfig {
@@ -5366,6 +5778,15 @@ impl magnus::TryConvert for ProxyConfig {
 }
 
 unsafe impl TryConvertOwned for ProxyConfig {}
+impl std::fmt::Debug for ProxyConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut debug = f.debug_struct("ProxyConfig");
+        debug.field("url", &"<redacted>");
+        debug.field("username", &self.username);
+        debug.field("password", &"<redacted>");
+        debug.finish()
+    }
+}
 
 impl Default for ProxyConfig {
     fn default() -> Self {
@@ -5572,7 +5993,7 @@ impl ResponseMeta {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::ScrapeResult")]
 pub struct ScrapeResult {
@@ -5605,6 +6026,7 @@ pub struct ScrapeResult {
     screenshot_base64: Option<String>,
     downloaded_document: Option<DownloadedDocument>,
     browser: Option<BrowserExtras>,
+    ssrf_refused_urls: Vec<String>,
 }
 
 unsafe impl IntoValueFromNative for ScrapeResult {}
@@ -5634,6 +6056,14 @@ impl magnus::TryConvert for ScrapeResult {
 }
 
 unsafe impl TryConvertOwned for ScrapeResult {}
+impl<'de> serde::Deserialize<'de> for ScrapeResult {
+    fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        <crawlberg::ScrapeResult as serde::Deserialize>::deserialize(deserializer).map(Into::into)
+    }
+}
 
 impl Default for ScrapeResult {
     fn default() -> Self {
@@ -5912,6 +6342,15 @@ impl ScrapeResult {
                 })?),
                 None => None,
             },
+            ssrf_refused_urls: match kwargs.get(ruby.to_symbol("ssrf_refused_urls")) {
+                Some(v) => <Vec<String>>::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `ssrf_refused_urls`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
         })
     }
 
@@ -6030,6 +6469,10 @@ impl ScrapeResult {
     fn browser(&self) -> Option<BrowserExtras> {
         self.browser.clone()
     }
+
+    fn ssrf_refused_urls(&self) -> Vec<String> {
+        self.ssrf_refused_urls.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
@@ -6145,6 +6588,7 @@ impl SitemapUrl {
 pub struct SsrfPolicy {
     deny_private: bool,
     allowlist: Vec<HostMatcher>,
+    denylist: Vec<HostMatcher>,
     max_redirects: u8,
     scheme_allowlist: Vec<String>,
 }
@@ -6216,6 +6660,15 @@ impl SsrfPolicy {
                 })?,
                 None => Default::default(),
             },
+            denylist: match kwargs.get(ruby.to_symbol("denylist")) {
+                Some(v) => <Vec<HostMatcher>>::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `denylist`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
             max_redirects: match kwargs.get(ruby.to_symbol("max_redirects")) {
                 Some(v) => u8::try_convert(v).map_err(|e| {
                     magnus::Error::new(
@@ -6245,6 +6698,10 @@ impl SsrfPolicy {
         self.allowlist.clone()
     }
 
+    fn denylist(&self) -> Vec<HostMatcher> {
+        self.denylist.clone()
+    }
+
     fn max_redirects(&self) -> u8 {
         self.max_redirects
     }
@@ -6328,7 +6785,7 @@ impl magnus::TryConvert for AssetCategory {
 unsafe impl IntoValueFromNative for AssetCategory {}
 unsafe impl TryConvertOwned for AssetCategory {}
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, serde::Serialize, serde::Deserialize)]
 #[serde(tag = "type")]
 pub enum AuthConfig {
     #[serde(rename = "basic")]
@@ -6394,6 +6851,11 @@ unsafe impl IntoValueFromNative for AuthConfig {}
 unsafe impl TryConvertOwned for AuthConfig {}
 
 impl AuthConfig {}
+impl std::fmt::Debug for AuthConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("AuthConfig(<redacted>)")
+    }
+}
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "snake_case")]
@@ -6991,7 +7453,7 @@ impl magnus::TryConvert for LinkType {
 unsafe impl IntoValueFromNative for LinkType {}
 unsafe impl TryConvertOwned for LinkType {}
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, serde::Serialize, serde::Deserialize)]
 #[serde(tag = "type")]
 #[serde(rename_all = "camelCase")]
 pub enum PageAction {
@@ -7077,6 +7539,11 @@ unsafe impl IntoValueFromNative for PageAction {}
 unsafe impl TryConvertOwned for PageAction {}
 
 impl PageAction {}
+impl std::fmt::Debug for PageAction {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("PageAction(<redacted>)")
+    }
+}
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
@@ -7522,6 +7989,8 @@ impl From<BrowserConfig> for crawlberg::BrowserConfig {
             backend: val.backend.into(),
             endpoint: val.endpoint,
             timeout: std::time::Duration::from_millis(val.timeout),
+            overall_timeout: std::time::Duration::from_millis(val.overall_timeout),
+            shutdown_timeout: std::time::Duration::from_millis(val.shutdown_timeout),
             wait: val.wait.into(),
             wait_selector: val.wait_selector,
             extra_wait: val.extra_wait.map(std::time::Duration::from_millis),
@@ -7531,6 +8000,8 @@ impl From<BrowserConfig> for crawlberg::BrowserConfig {
             robots_user_agent: val.robots_user_agent,
             capture_network_events: val.capture_network_events,
             session_affinity: val.session_affinity,
+            chrome_path: val.chrome_path.map(Into::into),
+            chrome_args: val.chrome_args.into_iter().collect(),
             ..Default::default()
         }
     }
@@ -7544,6 +8015,8 @@ impl From<crawlberg::BrowserConfig> for BrowserConfig {
             backend: val.backend.into(),
             endpoint: val.endpoint.map(|v| v.to_string()),
             timeout: val.timeout.as_millis() as u64,
+            overall_timeout: val.overall_timeout.as_millis() as u64,
+            shutdown_timeout: val.shutdown_timeout.as_millis() as u64,
             wait: val.wait.into(),
             wait_selector: val.wait_selector.map(|v| v.to_string()),
             extra_wait: val.extra_wait.map(|d| d.as_millis() as u64),
@@ -7553,6 +8026,8 @@ impl From<crawlberg::BrowserConfig> for BrowserConfig {
             robots_user_agent: val.robots_user_agent.map(|v| v.to_string()),
             capture_network_events: val.capture_network_events,
             session_affinity: val.session_affinity,
+            chrome_path: val.chrome_path.map(|p| p.to_string_lossy().to_string()),
+            chrome_args: val.chrome_args.into_iter().collect(),
         }
     }
 }
@@ -7641,6 +8116,7 @@ impl From<ContentConfig> for crawlberg::ContentConfig {
             wrap: val.wrap,
             wrap_width: val.wrap_width,
             include_document_structure: val.include_document_structure,
+            extract_metadata: val.extract_metadata,
             ..Default::default()
         }
     }
@@ -7661,6 +8137,7 @@ impl From<crawlberg::ContentConfig> for ContentConfig {
             wrap: val.wrap,
             wrap_width: val.wrap_width,
             include_document_structure: val.include_document_structure,
+            extract_metadata: val.extract_metadata,
         }
     }
 }
@@ -7710,6 +8187,11 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
             allow_subdomains: val.allow_subdomains,
             include_paths: val.include_paths.into_iter().collect(),
             exclude_paths: val.exclude_paths.into_iter().collect(),
+            path_patterns_match_query: val.path_patterns_match_query,
+            path_patterns_match_url: val.path_patterns_match_url,
+            dedup_include_query: val.dedup_include_query,
+            strip_tracking_params: val.strip_tracking_params,
+            tracking_params: val.tracking_params.into_iter().collect(),
             custom_headers: val
                 .custom_headers
                 .into_iter()
@@ -7720,6 +8202,9 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
             max_redirects: val.max_redirects,
             retry_count: val.retry_count,
             retry_codes: val.retry_codes.into_iter().collect(),
+            retry_initial_delay_ms: val.retry_initial_delay_ms,
+            retry_max_delay_ms: val.retry_max_delay_ms,
+            rate_limit_jitter_ratio: val.rate_limit_jitter_ratio,
             cookies_enabled: val.cookies_enabled,
             auth: val.auth.map(Into::into),
             max_body_size: val.max_body_size,
@@ -7770,6 +8255,11 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
             allow_subdomains: val.allow_subdomains,
             include_paths: val.include_paths.into_iter().collect(),
             exclude_paths: val.exclude_paths.into_iter().collect(),
+            path_patterns_match_query: val.path_patterns_match_query,
+            path_patterns_match_url: val.path_patterns_match_url,
+            dedup_include_query: val.dedup_include_query,
+            strip_tracking_params: val.strip_tracking_params,
+            tracking_params: val.tracking_params.into_iter().collect(),
             custom_headers: val
                 .custom_headers
                 .into_iter()
@@ -7780,6 +8270,9 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
             max_redirects: val.max_redirects,
             retry_count: val.retry_count,
             retry_codes: val.retry_codes.into_iter().collect(),
+            retry_initial_delay_ms: val.retry_initial_delay_ms,
+            retry_max_delay_ms: val.retry_max_delay_ms,
+            rate_limit_jitter_ratio: val.rate_limit_jitter_ratio,
             cookies_enabled: val.cookies_enabled,
             auth: val.auth.map(Into::into),
             max_body_size: val.max_body_size,
@@ -7836,6 +8329,11 @@ impl From<CrawlPageResult> for crawlberg::CrawlPageResult {
             extraction_meta: val.extraction_meta.map(Into::into),
             downloaded_document: val.downloaded_document.map(Into::into),
             browser_used: val.browser_used,
+            final_url: val.final_url,
+            redirect_count: val.redirect_count,
+            noindex_detected: val.noindex_detected,
+            nofollow_detected: val.nofollow_detected,
+            ssrf_refused_urls: val.ssrf_refused_urls.into_iter().collect(),
             ..Default::default()
         }
     }
@@ -7866,6 +8364,11 @@ impl From<crawlberg::CrawlPageResult> for CrawlPageResult {
             extraction_meta: val.extraction_meta.map(Into::into),
             downloaded_document: val.downloaded_document.map(Into::into),
             browser_used: val.browser_used,
+            final_url: val.final_url.to_string(),
+            redirect_count: val.redirect_count,
+            noindex_detected: val.noindex_detected,
+            nofollow_detected: val.nofollow_detected,
+            ssrf_refused_urls: val.ssrf_refused_urls.into_iter().collect(),
         }
     }
 }
@@ -8143,6 +8646,7 @@ impl From<InteractionResult> for crawlberg::InteractionResult {
             final_html: val.final_html,
             final_url: val.final_url,
             screenshot_base64: val.screenshot_base64,
+            ssrf_refused_urls: val.ssrf_refused_urls.into_iter().collect(),
             ..Default::default()
         }
     }
@@ -8156,6 +8660,7 @@ impl From<crawlberg::InteractionResult> for InteractionResult {
             final_html: val.final_html.to_string(),
             final_url: val.final_url.to_string(),
             screenshot_base64: val.screenshot_base64.map(|v| v.to_string()),
+            ssrf_refused_urls: val.ssrf_refused_urls.into_iter().collect(),
         }
     }
 }
@@ -8460,6 +8965,7 @@ impl From<ScrapeResult> for crawlberg::ScrapeResult {
             screenshot_base64: val.screenshot_base64,
             downloaded_document: val.downloaded_document.map(Into::into),
             browser: val.browser.map(Into::into),
+            ssrf_refused_urls: val.ssrf_refused_urls.into_iter().collect(),
             ..Default::default()
         }
     }
@@ -8498,6 +9004,7 @@ impl From<crawlberg::ScrapeResult> for ScrapeResult {
             screenshot_base64: val.screenshot_base64.map(|v| v.to_string()),
             downloaded_document: val.downloaded_document.map(Into::into),
             browser: val.browser.map(Into::into),
+            ssrf_refused_urls: val.ssrf_refused_urls.into_iter().collect(),
         }
     }
 }
@@ -8534,6 +9041,7 @@ impl From<SsrfPolicy> for crawlberg::SsrfPolicy {
         Self {
             deny_private: val.deny_private,
             allowlist: val.allowlist.into_iter().map(Into::into).collect(),
+            denylist: val.denylist.into_iter().map(Into::into).collect(),
             max_redirects: val.max_redirects,
             scheme_allowlist: val.scheme_allowlist.into_iter().collect(),
             ..Default::default()
@@ -8547,6 +9055,7 @@ impl From<crawlberg::SsrfPolicy> for SsrfPolicy {
         Self {
             deny_private: val.deny_private,
             allowlist: val.allowlist.into_iter().map(Into::into).collect(),
+            denylist: val.denylist.into_iter().map(Into::into).collect(),
             max_redirects: val.max_redirects,
             scheme_allowlist: val.scheme_allowlist.into_iter().collect(),
         }
@@ -9186,6 +9695,10 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("timeout", method!(BrowserConfig::timeout, 0))?;
 
+    class.define_method("overall_timeout", method!(BrowserConfig::overall_timeout, 0))?;
+
+    class.define_method("shutdown_timeout", method!(BrowserConfig::shutdown_timeout, 0))?;
+
     class.define_method("wait", method!(BrowserConfig::wait, 0))?;
 
     class.define_method("wait_selector", method!(BrowserConfig::wait_selector, 0))?;
@@ -9207,6 +9720,10 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("session_affinity", method!(BrowserConfig::session_affinity, 0))?;
 
+    class.define_method("chrome_path", method!(BrowserConfig::chrome_path, 0))?;
+
+    class.define_method("chrome_args", method!(BrowserConfig::chrome_args, 0))?;
+
     let class = module.define_class("BrowserExtras", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(BrowserExtras::new, -1))?;
@@ -9268,6 +9785,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
         method!(ContentConfig::include_document_structure, 0),
     )?;
 
+    class.define_method("extract_metadata", method!(ContentConfig::extract_metadata, 0))?;
+
     let class = module.define_class("CookieInfo", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(CookieInfo::new, -1))?;
@@ -9314,6 +9833,22 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("exclude_paths", method!(CrawlConfig::exclude_paths, 0))?;
 
+    class.define_method(
+        "path_patterns_match_query",
+        method!(CrawlConfig::path_patterns_match_query, 0),
+    )?;
+
+    class.define_method(
+        "path_patterns_match_url",
+        method!(CrawlConfig::path_patterns_match_url, 0),
+    )?;
+
+    class.define_method("dedup_include_query", method!(CrawlConfig::dedup_include_query, 0))?;
+
+    class.define_method("strip_tracking_params", method!(CrawlConfig::strip_tracking_params, 0))?;
+
+    class.define_method("tracking_params", method!(CrawlConfig::tracking_params, 0))?;
+
     class.define_method("custom_headers", method!(CrawlConfig::custom_headers, 0))?;
 
     class.define_method("request_timeout", method!(CrawlConfig::request_timeout, 0))?;
@@ -9326,6 +9861,18 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("retry_codes", method!(CrawlConfig::retry_codes, 0))?;
 
+    class.define_method(
+        "retry_initial_delay_ms",
+        method!(CrawlConfig::retry_initial_delay_ms, 0),
+    )?;
+
+    class.define_method("retry_max_delay_ms", method!(CrawlConfig::retry_max_delay_ms, 0))?;
+
+    class.define_method(
+        "rate_limit_jitter_ratio",
+        method!(CrawlConfig::rate_limit_jitter_ratio, 0),
+    )?;
+
     class.define_method("cookies_enabled", method!(CrawlConfig::cookies_enabled, 0))?;
 
     class.define_method("auth", method!(CrawlConfig::auth, 0))?;
@@ -9437,6 +9984,16 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("browser_used", method!(CrawlPageResult::browser_used, 0))?;
 
+    class.define_method("final_url", method!(CrawlPageResult::final_url, 0))?;
+
+    class.define_method("redirect_count", method!(CrawlPageResult::redirect_count, 0))?;
+
+    class.define_method("noindex_detected", method!(CrawlPageResult::noindex_detected, 0))?;
+
+    class.define_method("nofollow_detected", method!(CrawlPageResult::nofollow_detected, 0))?;
+
+    class.define_method("ssrf_refused_urls", method!(CrawlPageResult::ssrf_refused_urls, 0))?;
+
     let class = module.define_class("CrawlResult", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(CrawlResult::new, -1))?;
@@ -9586,6 +10143,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("screenshot_base64", method!(InteractionResult::screenshot_base64, 0))?;
 
+    class.define_method("ssrf_refused_urls", method!(InteractionResult::ssrf_refused_urls, 0))?;
+
     let class = module.define_class("JsonLdEntry", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(JsonLdEntry::new, -1))?;
@@ -9814,6 +10373,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("browser", method!(ScrapeResult::browser, 0))?;
 
+    class.define_method("ssrf_refused_urls", method!(ScrapeResult::ssrf_refused_urls, 0))?;
+
     let class = module.define_class("SitemapUrl", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(SitemapUrl::new, -1))?;
@@ -9834,6 +10395,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("allowlist", method!(SsrfPolicy::allowlist, 0))?;
 
+    class.define_method("denylist", method!(SsrfPolicy::denylist, 0))?;
+
     class.define_method("max_redirects", method!(SsrfPolicy::max_redirects, 0))?;
 
     class.define_method("scheme_allowlist", method!(SsrfPolicy::scheme_allowlist, 0))?;