crawlberg 1.1.4 → 1.3.3

data/ext/crawlberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:899a9b0b3964b8cc10628ac686b74140e934aa910533d6b9333c2acf97f2773b
+// alef:hash:9312cb415995255492557432651036cfe6b0edf4ccd0b27a88d0cb3c5a7d1466
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -316,7 +316,7 @@ impl ContentConfig {
             exclude_selectors: kwargs
                 .get(ruby.to_symbol("exclude_selectors"))
                 .and_then(|v| <Vec<String>>::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(vec!["noscript".to_string()]),
             skip_images: kwargs
                 .get(ruby.to_symbol("skip_images"))
                 .and_then(|v| bool::try_convert(v).ok())
@@ -467,7 +467,7 @@ impl BrowserConfig {
             wait: kwargs
                 .get(ruby.to_symbol("wait"))
                 .and_then(|v| BrowserWait::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(BrowserWait::NetworkIdle),
             wait_selector: kwargs
                 .get(ruby.to_symbol("wait_selector"))
                 .and_then(|v| String::try_convert(v).ok()),
@@ -557,7 +557,12 @@ impl BrowserConfig {
 pub struct CrawlConfig {
     max_depth: Option<usize>,
     max_pages: Option<usize>,
+    max_links_per_page: Option<usize>,
     max_concurrent: Option<usize>,
+    crawl_strategy: CrawlStrategyKind,
+    content_filter: Option<ContentFilterKind>,
+    bm25_query: Option<String>,
+    bm25_threshold: Option<f64>,
     respect_robots_txt: bool,
     soft_http_errors: bool,
     user_agent: Option<String>,
@@ -591,10 +596,13 @@ pub struct CrawlConfig {
     download_documents: bool,
     document_max_size: Option<usize>,
     document_mime_types: Vec<String>,
+    document_output_dir: Option<String>,
+    document_content_encoding: Option<DocumentContentEncoding>,
     warc_output: Option<String>,
     browser_profile: Option<String>,
     save_browser_profile: bool,
     ssrf: SsrfPolicy,
+    ssrf_deny_private_explicit: Option<bool>,
 }
 
 unsafe impl IntoValueFromNative for CrawlConfig {}
@@ -644,9 +652,25 @@ impl CrawlConfig {
             max_pages: kwargs
                 .get(ruby.to_symbol("max_pages"))
                 .and_then(|v| usize::try_convert(v).ok()),
+            max_links_per_page: kwargs
+                .get(ruby.to_symbol("max_links_per_page"))
+                .and_then(|v| usize::try_convert(v).ok()),
             max_concurrent: kwargs
                 .get(ruby.to_symbol("max_concurrent"))
                 .and_then(|v| usize::try_convert(v).ok()),
+            crawl_strategy: kwargs
+                .get(ruby.to_symbol("crawl_strategy"))
+                .and_then(|v| CrawlStrategyKind::try_convert(v).ok())
+                .unwrap_or(CrawlStrategyKind::Bfs),
+            content_filter: kwargs
+                .get(ruby.to_symbol("content_filter"))
+                .and_then(|v| ContentFilterKind::try_convert(v).ok()),
+            bm25_query: kwargs
+                .get(ruby.to_symbol("bm25_query"))
+                .and_then(|v| String::try_convert(v).ok()),
+            bm25_threshold: kwargs
+                .get(ruby.to_symbol("bm25_threshold"))
+                .and_then(|v| f64::try_convert(v).ok()),
             respect_robots_txt: kwargs
                 .get(ruby.to_symbol("respect_robots_txt"))
                 .and_then(|v| bool::try_convert(v).ok())
@@ -765,6 +789,12 @@ impl CrawlConfig {
                 .get(ruby.to_symbol("document_mime_types"))
                 .and_then(|v| <Vec<String>>::try_convert(v).ok())
                 .unwrap_or_default(),
+            document_output_dir: kwargs
+                .get(ruby.to_symbol("document_output_dir"))
+                .and_then(|v| String::try_convert(v).ok()),
+            document_content_encoding: kwargs
+                .get(ruby.to_symbol("document_content_encoding"))
+                .and_then(|v| DocumentContentEncoding::try_convert(v).ok()),
             warc_output: kwargs
                 .get(ruby.to_symbol("warc_output"))
                 .and_then(|v| String::try_convert(v).ok()),
@@ -778,7 +808,10 @@ impl CrawlConfig {
             ssrf: kwargs
                 .get(ruby.to_symbol("ssrf"))
                 .and_then(|v| SsrfPolicy::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(crawlberg::SsrfPolicy::from_env().into()),
+            ssrf_deny_private_explicit: kwargs
+                .get(ruby.to_symbol("ssrf_deny_private_explicit"))
+                .and_then(|v| bool::try_convert(v).ok()),
         })
     }
 
@@ -790,10 +823,30 @@ impl CrawlConfig {
         self.max_pages
     }
 
+    fn max_links_per_page(&self) -> Option<usize> {
+        self.max_links_per_page
+    }
+
     fn max_concurrent(&self) -> Option<usize> {
         self.max_concurrent
     }
 
+    fn crawl_strategy(&self) -> CrawlStrategyKind {
+        self.crawl_strategy.clone()
+    }
+
+    fn content_filter(&self) -> Option<ContentFilterKind> {
+        self.content_filter.clone()
+    }
+
+    fn bm25_query(&self) -> Option<String> {
+        self.bm25_query.clone()
+    }
+
+    fn bm25_threshold(&self) -> Option<f64> {
+        self.bm25_threshold
+    }
+
     fn respect_robots_txt(&self) -> bool {
         self.respect_robots_txt
     }
@@ -922,6 +975,14 @@ impl CrawlConfig {
         self.document_mime_types.clone()
     }
 
+    fn document_output_dir(&self) -> Option<String> {
+        self.document_output_dir.clone()
+    }
+
+    fn document_content_encoding(&self) -> Option<DocumentContentEncoding> {
+        self.document_content_encoding.clone()
+    }
+
     fn warc_output(&self) -> Option<String> {
         self.warc_output.clone()
     }
@@ -938,6 +999,10 @@ impl CrawlConfig {
         self.ssrf.clone()
     }
 
+    fn ssrf_deny_private_explicit(&self) -> Option<bool> {
+        self.ssrf_deny_private_explicit
+    }
+
     fn validate(&self) -> Result<(), Error> {
         #[allow(clippy::needless_update)]
         let core_self = crawlberg::CrawlConfig {
@@ -945,8 +1010,18 @@ impl CrawlConfig {
 
             max_pages: self.max_pages,
 
+            max_links_per_page: self.max_links_per_page,
+
             max_concurrent: self.max_concurrent,
 
+            crawl_strategy: self.crawl_strategy.clone().into(),
+
+            content_filter: self.content_filter.clone().map(Into::into),
+
+            bm25_query: self.bm25_query.clone(),
+
+            bm25_threshold: self.bm25_threshold,
+
             respect_robots_txt: self.respect_robots_txt,
 
             soft_http_errors: self.soft_http_errors,
@@ -1016,6 +1091,10 @@ impl CrawlConfig {
 
             document_mime_types: self.document_mime_types.clone(),
 
+            document_output_dir: self.document_output_dir.clone().map(Into::into),
+
+            document_content_encoding: self.document_content_encoding.clone().map(Into::into),
+
             warc_output: self.warc_output.clone().map(Into::into),
 
             browser_profile: self.browser_profile.clone(),
@@ -1024,6 +1103,8 @@ impl CrawlConfig {
 
             ssrf: self.ssrf.clone().into(),
 
+            ssrf_deny_private_explicit: self.ssrf_deny_private_explicit,
+
             ..Default::default()
         };
         let result = core_self.validate().map_err(|e| {
@@ -1123,6 +1204,9 @@ pub struct DownloadedDocument {
     filename: Option<String>,
     content_hash: String,
     headers: HashMap<String, String>,
+    truncated: bool,
+    content_path: Option<String>,
+    content_base64: Option<String>,
 }
 
 unsafe impl IntoValueFromNative for DownloadedDocument {}
@@ -1189,6 +1273,16 @@ impl DownloadedDocument {
                 .get(ruby.to_symbol("headers"))
                 .and_then(|v| <HashMap<String, String>>::try_convert(v).ok())
                 .unwrap_or_default(),
+            truncated: kwargs
+                .get(ruby.to_symbol("truncated"))
+                .and_then(|v| bool::try_convert(v).ok())
+                .unwrap_or_default(),
+            content_path: kwargs
+                .get(ruby.to_symbol("content_path"))
+                .and_then(|v| String::try_convert(v).ok()),
+            content_base64: kwargs
+                .get(ruby.to_symbol("content_base64"))
+                .and_then(|v| String::try_convert(v).ok()),
         })
     }
 
@@ -1215,6 +1309,18 @@ impl DownloadedDocument {
     fn headers(&self) -> HashMap<String, String> {
         self.headers.clone()
     }
+
+    fn truncated(&self) -> bool {
+        self.truncated
+    }
+
+    fn content_path(&self) -> Option<String> {
+        self.content_path.clone()
+    }
+
+    fn content_base64(&self) -> Option<String> {
+        self.content_base64.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
@@ -1224,6 +1330,7 @@ pub struct InteractionResult {
     action_results: Vec<ActionResult>,
     final_html: String,
     final_url: String,
+    screenshot_base64: Option<String>,
 }
 
 unsafe impl IntoValueFromNative for InteractionResult {}
@@ -1279,6 +1386,9 @@ impl InteractionResult {
                 .get(ruby.to_symbol("final_url"))
                 .and_then(|v| String::try_convert(v).ok())
                 .unwrap_or_default(),
+            screenshot_base64: kwargs
+                .get(ruby.to_symbol("screenshot_base64"))
+                .and_then(|v| String::try_convert(v).ok()),
         })
     }
 
@@ -1293,6 +1403,10 @@ impl InteractionResult {
     fn final_url(&self) -> String {
         self.final_url.clone()
     }
+
+    fn screenshot_base64(&self) -> Option<String> {
+        self.screenshot_base64.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
@@ -1419,6 +1533,7 @@ pub struct ScrapeResult {
     markdown: Option<MarkdownResult>,
     extracted_data: Option<String>,
     extraction_meta: Option<ExtractionMeta>,
+    screenshot_base64: Option<String>,
     downloaded_document: Option<DownloadedDocument>,
     browser: Option<BrowserExtras>,
 }
@@ -1561,6 +1676,9 @@ impl ScrapeResult {
             extraction_meta: kwargs
                 .get(ruby.to_symbol("extraction_meta"))
                 .and_then(|v| ExtractionMeta::try_convert(v).ok()),
+            screenshot_base64: kwargs
+                .get(ruby.to_symbol("screenshot_base64"))
+                .and_then(|v| String::try_convert(v).ok()),
             downloaded_document: kwargs
                 .get(ruby.to_symbol("downloaded_document"))
                 .and_then(|v| DownloadedDocument::try_convert(v).ok()),
@@ -1674,6 +1792,10 @@ impl ScrapeResult {
         self.extraction_meta.clone()
     }
 
+    fn screenshot_base64(&self) -> Option<String> {
+        self.screenshot_base64.clone()
+    }
+
     fn downloaded_document(&self) -> Option<DownloadedDocument> {
         self.downloaded_document.clone()
     }
@@ -2379,7 +2501,7 @@ impl LinkInfo {
             link_type: kwargs
                 .get(ruby.to_symbol("link_type"))
                 .and_then(|v| LinkType::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(LinkType::Internal),
             rel: kwargs
                 .get(ruby.to_symbol("rel"))
                 .and_then(|v| String::try_convert(v).ok()),
@@ -2479,7 +2601,7 @@ impl ImageInfo {
             source: kwargs
                 .get(ruby.to_symbol("source"))
                 .and_then(|v| ImageSource::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(ImageSource::Img),
         })
     }
 
@@ -2564,7 +2686,7 @@ impl FeedInfo {
             feed_type: kwargs
                 .get(ruby.to_symbol("feed_type"))
                 .and_then(|v| FeedType::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(FeedType::Rss),
         })
     }
 
@@ -2814,7 +2936,7 @@ impl DownloadedAsset {
             asset_category: kwargs
                 .get(ruby.to_symbol("asset_category"))
                 .and_then(|v| AssetCategory::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(AssetCategory::Image),
             html_tag: kwargs
                 .get(ruby.to_symbol("html_tag"))
                 .and_then(|v| String::try_convert(v).ok()),
@@ -4358,6 +4480,7 @@ impl BatchCrawlResults {
 #[magnus::wrap(class = "Crawlberg::SsrfPolicy")]
 pub struct SsrfPolicy {
     deny_private: bool,
+    allowlist: Vec<HostMatcher>,
     max_redirects: u8,
 }
 
@@ -4406,6 +4529,10 @@ impl SsrfPolicy {
                 .get(ruby.to_symbol("deny_private"))
                 .and_then(|v| bool::try_convert(v).ok())
                 .unwrap_or(true),
+            allowlist: kwargs
+                .get(ruby.to_symbol("allowlist"))
+                .and_then(|v| <Vec<HostMatcher>>::try_convert(v).ok())
+                .unwrap_or_default(),
             max_redirects: kwargs
                 .get(ruby.to_symbol("max_redirects"))
                 .and_then(|v| u8::try_convert(v).ok())
@@ -4417,6 +4544,10 @@ impl SsrfPolicy {
         self.deny_private
     }
 
+    fn allowlist(&self) -> Vec<HostMatcher> {
+        self.allowlist.clone()
+    }
+
     fn max_redirects(&self) -> u8 {
         self.max_redirects
     }
@@ -4452,8 +4583,9 @@ impl magnus::IntoValue for BrowserMode {
 impl magnus::TryConvert for BrowserMode {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "auto" | "Auto" => Ok(BrowserMode::Auto),
             "always" | "Always" => Ok(BrowserMode::Always),
@@ -4498,8 +4630,9 @@ impl magnus::IntoValue for BrowserWait {
 impl magnus::TryConvert for BrowserWait {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "network_idle" | "NetworkIdle" => Ok(BrowserWait::NetworkIdle),
             "selector" | "Selector" => Ok(BrowserWait::Selector),
@@ -4541,8 +4674,9 @@ impl magnus::IntoValue for BrowserBackend {
 impl magnus::TryConvert for BrowserBackend {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "chromiumoxide" | "Chromiumoxide" => Ok(BrowserBackend::Chromiumoxide),
             "native" | "Native" => Ok(BrowserBackend::Native),
@@ -4557,6 +4691,135 @@ impl magnus::TryConvert for BrowserBackend {
 unsafe impl IntoValueFromNative for BrowserBackend {}
 unsafe impl TryConvertOwned for BrowserBackend {}
 
+#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DocumentContentEncoding {
+    Base64,
+}
+
+impl Default for DocumentContentEncoding {
+    fn default() -> Self {
+        Self::Base64
+    }
+}
+
+impl magnus::IntoValue for DocumentContentEncoding {
+    fn into_value_with(self, handle: &Ruby) -> magnus::Value {
+        let sym = match self {
+            DocumentContentEncoding::Base64 => "base64",
+        };
+        handle.to_symbol(sym).into_value_with(handle)
+    }
+}
+
+impl magnus::TryConvert for DocumentContentEncoding {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        let s: String = magnus::TryConvert::try_convert(val)?;
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
+        match s.as_str() {
+            "base64" | "Base64" => Ok(DocumentContentEncoding::Base64),
+            other => Err(magnus::Error::new(
+                unsafe { Ruby::get_unchecked() }.exception_arg_error(),
+                format!("invalid DocumentContentEncoding value: {other}"),
+            )),
+        }
+    }
+}
+
+unsafe impl IntoValueFromNative for DocumentContentEncoding {}
+unsafe impl TryConvertOwned for DocumentContentEncoding {}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum CrawlStrategyKind {
+    Bfs,
+    Dfs,
+    BestFirst,
+    Adaptive,
+}
+
+impl Default for CrawlStrategyKind {
+    fn default() -> Self {
+        Self::Bfs
+    }
+}
+
+impl magnus::IntoValue for CrawlStrategyKind {
+    fn into_value_with(self, handle: &Ruby) -> magnus::Value {
+        let sym = match self {
+            CrawlStrategyKind::Bfs => "bfs",
+            CrawlStrategyKind::Dfs => "dfs",
+            CrawlStrategyKind::BestFirst => "best_first",
+            CrawlStrategyKind::Adaptive => "adaptive",
+        };
+        handle.to_symbol(sym).into_value_with(handle)
+    }
+}
+
+impl magnus::TryConvert for CrawlStrategyKind {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        let s: String = magnus::TryConvert::try_convert(val)?;
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
+        match s.as_str() {
+            "bfs" | "Bfs" => Ok(CrawlStrategyKind::Bfs),
+            "dfs" | "Dfs" => Ok(CrawlStrategyKind::Dfs),
+            "best_first" | "BestFirst" => Ok(CrawlStrategyKind::BestFirst),
+            "adaptive" | "Adaptive" => Ok(CrawlStrategyKind::Adaptive),
+            other => Err(magnus::Error::new(
+                unsafe { Ruby::get_unchecked() }.exception_arg_error(),
+                format!("invalid CrawlStrategyKind value: {other}"),
+            )),
+        }
+    }
+}
+
+unsafe impl IntoValueFromNative for CrawlStrategyKind {}
+unsafe impl TryConvertOwned for CrawlStrategyKind {}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ContentFilterKind {
+    Bm25,
+}
+
+impl Default for ContentFilterKind {
+    fn default() -> Self {
+        Self::Bm25
+    }
+}
+
+impl magnus::IntoValue for ContentFilterKind {
+    fn into_value_with(self, handle: &Ruby) -> magnus::Value {
+        let sym = match self {
+            ContentFilterKind::Bm25 => "bm25",
+        };
+        handle.to_symbol(sym).into_value_with(handle)
+    }
+}
+
+impl magnus::TryConvert for ContentFilterKind {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        let s: String = magnus::TryConvert::try_convert(val)?;
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
+        match s.as_str() {
+            "bm25" | "Bm25" => Ok(ContentFilterKind::Bm25),
+            other => Err(magnus::Error::new(
+                unsafe { Ruby::get_unchecked() }.exception_arg_error(),
+                format!("invalid ContentFilterKind value: {other}"),
+            )),
+        }
+    }
+}
+
+unsafe impl IntoValueFromNative for ContentFilterKind {}
+unsafe impl TryConvertOwned for ContentFilterKind {}
+
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(tag = "type")]
 pub enum AuthConfig {
@@ -4652,8 +4915,9 @@ impl magnus::IntoValue for LinkType {
 impl magnus::TryConvert for LinkType {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "internal" | "Internal" => Ok(LinkType::Internal),
             "external" | "External" => Ok(LinkType::External),
@@ -4692,8 +4956,8 @@ impl magnus::IntoValue for ImageSource {
         let sym = match self {
             ImageSource::Img => "img",
             ImageSource::PictureSource => "picture_source",
-            ImageSource::OgImage => "og_image",
-            ImageSource::TwitterImage => "twitter_image",
+            ImageSource::OgImage => "og:image",
+            ImageSource::TwitterImage => "twitter:image",
         };
         handle.to_symbol(sym).into_value_with(handle)
     }
@@ -4702,13 +4966,14 @@ impl magnus::IntoValue for ImageSource {
 impl magnus::TryConvert for ImageSource {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "img" | "Img" => Ok(ImageSource::Img),
             "picture_source" | "PictureSource" => Ok(ImageSource::PictureSource),
-            "og_image" | "OgImage" => Ok(ImageSource::OgImage),
-            "twitter_image" | "TwitterImage" => Ok(ImageSource::TwitterImage),
+            "og:image" | "og_image" | "OgImage" => Ok(ImageSource::OgImage),
+            "twitter:image" | "twitter_image" | "TwitterImage" => Ok(ImageSource::TwitterImage),
             other => Err(magnus::Error::new(
                 unsafe { Ruby::get_unchecked() }.exception_arg_error(),
                 format!("invalid ImageSource value: {other}"),
@@ -4748,8 +5013,9 @@ impl magnus::IntoValue for FeedType {
 impl magnus::TryConvert for FeedType {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "rss" | "Rss" => Ok(FeedType::Rss),
             "atom" | "Atom" => Ok(FeedType::Atom),
@@ -4807,8 +5073,9 @@ impl magnus::IntoValue for AssetCategory {
 impl magnus::TryConvert for AssetCategory {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "document" | "Document" => Ok(AssetCategory::Document),
             "image" | "Image" => Ok(AssetCategory::Image),
@@ -5004,8 +5271,9 @@ impl magnus::IntoValue for ScrollDirection {
 impl magnus::TryConvert for ScrollDirection {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "up" | "Up" => Ok(ScrollDirection::Up),
             "down" | "Down" => Ok(ScrollDirection::Down),
@@ -5020,6 +5288,68 @@ impl magnus::TryConvert for ScrollDirection {
 unsafe impl IntoValueFromNative for ScrollDirection {}
 unsafe impl TryConvertOwned for ScrollDirection {}
 
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(tag = "type")]
+#[serde(rename_all = "snake_case")]
+pub enum HostMatcher {
+    Exact { value: String },
+    Suffix { value: String },
+    Cidr { value: String },
+}
+
+impl Default for HostMatcher {
+    fn default() -> Self {
+        Self::Exact {
+            value: Default::default(),
+        }
+    }
+}
+
+impl magnus::IntoValue for HostMatcher {
+    fn into_value_with(self, handle: &Ruby) -> magnus::Value {
+        match serde_json::to_value(&self) {
+            Ok(v) => json_to_ruby(handle, v),
+            Err(_) => handle.qnil().into_value_with(handle),
+        }
+    }
+}
+
+impl magnus::TryConvert for HostMatcher {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        // For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
+        // For unit enums or when passed as a string, fall back to string-based conversion.
+        let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
+            s
+        } else {
+            val.funcall::<_, _, String>("to_json", ()).map_err(|e| {
+                magnus::Error::new(
+                    unsafe { Ruby::get_unchecked() }.exception_type_error(),
+                    format!("no implicit conversion into HostMatcher: {}", e),
+                )
+            })?
+        };
+        // Try deserializing as JSON first (handles JSON strings like "\"markdown\"" or "{\"click\":{\"selector\":\"...\"}}\"")
+        // For internally-tagged enums, a bare variant string is wrapped as {"<tag>": value}.
+        // If that fails, try treating it as a plain string value and wrap in quotes
+        // If both fail, try as Custom variant (for untagged enum support)
+        serde_json::from_str(&json_str)
+            .or_else(|_| serde_json::from_value(serde_json::json!({ "type": json_str })))
+            .or_else(|_| serde_json::from_str(&format!("\"{json_str}\"")))
+            .or_else(|_| {
+                // Try as a JSON string for Custom variant (untagged enums accept any remaining value)
+                match serde_json::to_value(&json_str) {
+                    Ok(val) => serde_json::from_value(val),
+                    Err(e) => Err(e),
+                }
+            })
+            .map_err(|e| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_type_error(), e.to_string()))
+    }
+}
+
+unsafe impl IntoValueFromNative for HostMatcher {}
+unsafe impl TryConvertOwned for HostMatcher {}
+impl HostMatcher {}
+
 fn generate_citations(markdown: String) -> CitationResult {
     crawlberg::generate_citations(&markdown).into()
 }
@@ -5279,7 +5609,6 @@ fn batch_crawl_stream(engine: CrawlEngineHandle, req: BatchCrawlStreamRequest) -
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ExtractionMeta> for crawlberg::ExtractionMeta {
     fn from(val: ExtractionMeta) -> Self {
         Self {
@@ -5293,7 +5622,7 @@ impl From<ExtractionMeta> for crawlberg::ExtractionMeta {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::ExtractionMeta> for ExtractionMeta {
     fn from(val: crawlberg::ExtractionMeta) -> Self {
         Self {
@@ -5307,7 +5636,6 @@ impl From<crawlberg::ExtractionMeta> for ExtractionMeta {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ProxyConfig> for crawlberg::ProxyConfig {
     fn from(val: ProxyConfig) -> Self {
         Self {
@@ -5319,7 +5647,7 @@ impl From<ProxyConfig> for crawlberg::ProxyConfig {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::ProxyConfig> for ProxyConfig {
     fn from(val: crawlberg::ProxyConfig) -> Self {
         Self {
@@ -5331,7 +5659,6 @@ impl From<crawlberg::ProxyConfig> for ProxyConfig {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ContentConfig> for crawlberg::ContentConfig {
     fn from(val: ContentConfig) -> Self {
         Self {
@@ -5352,7 +5679,6 @@ impl From<ContentConfig> for crawlberg::ContentConfig {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::ContentConfig> for ContentConfig {
     fn from(val: crawlberg::ContentConfig) -> Self {
         Self {
@@ -5373,7 +5699,7 @@ impl From<crawlberg::ContentConfig> for ContentConfig {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<BrowserConfig> for crawlberg::BrowserConfig {
     fn from(val: BrowserConfig) -> Self {
         Self {
@@ -5423,7 +5749,12 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
         Self {
             max_depth: val.max_depth,
             max_pages: val.max_pages,
+            max_links_per_page: val.max_links_per_page,
             max_concurrent: val.max_concurrent,
+            crawl_strategy: val.crawl_strategy.into(),
+            content_filter: val.content_filter.map(Into::into),
+            bm25_query: val.bm25_query,
+            bm25_threshold: val.bm25_threshold,
             respect_robots_txt: val.respect_robots_txt,
             soft_http_errors: val.soft_http_errors,
             user_agent: val.user_agent,
@@ -5460,10 +5791,13 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
             download_documents: val.download_documents,
             document_max_size: val.document_max_size,
             document_mime_types: val.document_mime_types.into_iter().collect(),
+            document_output_dir: val.document_output_dir.map(Into::into),
+            document_content_encoding: val.document_content_encoding.map(Into::into),
             warc_output: val.warc_output.map(Into::into),
             browser_profile: val.browser_profile,
             save_browser_profile: val.save_browser_profile,
             ssrf: val.ssrf.into(),
+            ssrf_deny_private_explicit: val.ssrf_deny_private_explicit,
             ..Default::default()
         }
     }
@@ -5475,7 +5809,12 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
         Self {
             max_depth: val.max_depth,
             max_pages: val.max_pages,
+            max_links_per_page: val.max_links_per_page,
             max_concurrent: val.max_concurrent,
+            crawl_strategy: val.crawl_strategy.into(),
+            content_filter: val.content_filter.map(Into::into),
+            bm25_query: val.bm25_query.map(|v| v.to_string()),
+            bm25_threshold: val.bm25_threshold,
             respect_robots_txt: val.respect_robots_txt,
             soft_http_errors: val.soft_http_errors,
             user_agent: val.user_agent.map(|v| v.to_string()),
@@ -5512,10 +5851,13 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
             download_documents: val.download_documents,
             document_max_size: val.document_max_size,
             document_mime_types: val.document_mime_types.into_iter().collect(),
+            document_output_dir: val.document_output_dir.map(|p| p.to_string_lossy().to_string()),
+            document_content_encoding: val.document_content_encoding.map(Into::into),
             warc_output: val.warc_output.map(|p| p.to_string_lossy().to_string()),
             browser_profile: val.browser_profile.map(|v| v.to_string()),
             save_browser_profile: val.save_browser_profile,
             ssrf: val.ssrf.into(),
+            ssrf_deny_private_explicit: val.ssrf_deny_private_explicit,
         }
     }
 }
@@ -5533,7 +5875,7 @@ impl From<BrowserExtras> for crawlberg::BrowserExtras {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::BrowserExtras> for BrowserExtras {
     fn from(val: crawlberg::BrowserExtras) -> Self {
         Self {
@@ -5555,6 +5897,9 @@ impl From<DownloadedDocument> for crawlberg::DownloadedDocument {
             filename: val.filename.map(Into::into),
             content_hash: val.content_hash.into(),
             headers: val.headers.into_iter().map(|(k, v)| (k.into(), v.into())).collect(),
+            truncated: val.truncated,
+            content_path: val.content_path,
+            content_base64: val.content_base64,
             ..Default::default()
         }
     }
@@ -5570,18 +5915,22 @@ impl From<crawlberg::DownloadedDocument> for DownloadedDocument {
             filename: val.filename.as_ref().map(|v| v.to_string()),
             content_hash: val.content_hash.to_string(),
             headers: val.headers.into_iter().map(|(k, v)| (k.into(), v.into())).collect(),
+            truncated: val.truncated,
+            content_path: val.content_path.map(|v| v.to_string()),
+            content_base64: val.content_base64.map(|v| v.to_string()),
         }
     }
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<InteractionResult> for crawlberg::InteractionResult {
     fn from(val: InteractionResult) -> Self {
         Self {
             action_results: val.action_results.into_iter().map(Into::into).collect(),
             final_html: val.final_html,
             final_url: val.final_url,
+            screenshot_base64: val.screenshot_base64,
             ..Default::default()
         }
     }
@@ -5594,6 +5943,7 @@ impl From<crawlberg::InteractionResult> for InteractionResult {
             action_results: val.action_results.into_iter().map(Into::into).collect(),
             final_html: val.final_html.to_string(),
             final_url: val.final_url.to_string(),
+            screenshot_base64: val.screenshot_base64.map(|v| v.to_string()),
         }
     }
 }
@@ -5613,7 +5963,7 @@ impl From<ActionResult> for crawlberg::ActionResult {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::ActionResult> for ActionResult {
     fn from(val: crawlberg::ActionResult) -> Self {
         Self {
@@ -5657,6 +6007,7 @@ impl From<ScrapeResult> for crawlberg::ScrapeResult {
             markdown: val.markdown.map(Into::into),
             extracted_data: val.extracted_data.as_ref().and_then(|s| serde_json::from_str(s).ok()),
             extraction_meta: val.extraction_meta.map(Into::into),
+            screenshot_base64: val.screenshot_base64,
             downloaded_document: val.downloaded_document.map(Into::into),
             browser: val.browser.map(Into::into),
             ..Default::default()
@@ -5694,6 +6045,7 @@ impl From<crawlberg::ScrapeResult> for ScrapeResult {
             markdown: val.markdown.map(Into::into),
             extracted_data: val.extracted_data.as_ref().map(ToString::to_string),
             extraction_meta: val.extraction_meta.map(Into::into),
+            screenshot_base64: val.screenshot_base64.map(|v| v.to_string()),
             downloaded_document: val.downloaded_document.map(Into::into),
             browser: val.browser.map(Into::into),
         }
@@ -5761,7 +6113,7 @@ impl From<crawlberg::CrawlPageResult> for CrawlPageResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<CrawlResult> for crawlberg::CrawlResult {
     fn from(val: CrawlResult) -> Self {
         Self {
@@ -5795,7 +6147,6 @@ impl From<crawlberg::CrawlResult> for CrawlResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<SitemapUrl> for crawlberg::SitemapUrl {
     fn from(val: SitemapUrl) -> Self {
         Self {
@@ -5808,7 +6159,7 @@ impl From<SitemapUrl> for crawlberg::SitemapUrl {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::SitemapUrl> for SitemapUrl {
     fn from(val: crawlberg::SitemapUrl) -> Self {
         Self {
@@ -5821,7 +6172,7 @@ impl From<crawlberg::SitemapUrl> for SitemapUrl {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<MapResult> for crawlberg::MapResult {
     fn from(val: MapResult) -> Self {
         Self {
@@ -5831,7 +6182,7 @@ impl From<MapResult> for crawlberg::MapResult {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::MapResult> for MapResult {
     fn from(val: crawlberg::MapResult) -> Self {
         Self {
@@ -5841,7 +6192,7 @@ impl From<crawlberg::MapResult> for MapResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<MarkdownResult> for crawlberg::MarkdownResult {
     fn from(val: MarkdownResult) -> Self {
         Self {
@@ -5863,7 +6214,7 @@ impl From<MarkdownResult> for crawlberg::MarkdownResult {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::MarkdownResult> for MarkdownResult {
     fn from(val: crawlberg::MarkdownResult) -> Self {
         Self {
@@ -5878,7 +6229,7 @@ impl From<crawlberg::MarkdownResult> for MarkdownResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<LinkInfo> for crawlberg::LinkInfo {
     fn from(val: LinkInfo) -> Self {
         Self {
@@ -5906,7 +6257,7 @@ impl From<crawlberg::LinkInfo> for LinkInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<ImageInfo> for crawlberg::ImageInfo {
     fn from(val: ImageInfo) -> Self {
         Self {
@@ -5934,7 +6285,7 @@ impl From<crawlberg::ImageInfo> for ImageInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<FeedInfo> for crawlberg::FeedInfo {
     fn from(val: FeedInfo) -> Self {
         Self {
@@ -5958,7 +6309,6 @@ impl From<crawlberg::FeedInfo> for FeedInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<JsonLdEntry> for crawlberg::JsonLdEntry {
     fn from(val: JsonLdEntry) -> Self {
         Self {
@@ -5970,7 +6320,7 @@ impl From<JsonLdEntry> for crawlberg::JsonLdEntry {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::JsonLdEntry> for JsonLdEntry {
     fn from(val: crawlberg::JsonLdEntry) -> Self {
         Self {
@@ -5982,7 +6332,6 @@ impl From<crawlberg::JsonLdEntry> for JsonLdEntry {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CookieInfo> for crawlberg::CookieInfo {
     fn from(val: CookieInfo) -> Self {
         Self {
@@ -5995,7 +6344,7 @@ impl From<CookieInfo> for crawlberg::CookieInfo {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::CookieInfo> for CookieInfo {
     fn from(val: crawlberg::CookieInfo) -> Self {
         Self {
@@ -6008,7 +6357,7 @@ impl From<crawlberg::CookieInfo> for CookieInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<DownloadedAsset> for crawlberg::DownloadedAsset {
     fn from(val: DownloadedAsset) -> Self {
         Self {
@@ -6038,7 +6387,6 @@ impl From<crawlberg::DownloadedAsset> for DownloadedAsset {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ArticleMetadata> for crawlberg::ArticleMetadata {
     fn from(val: ArticleMetadata) -> Self {
         Self {
@@ -6052,7 +6400,7 @@ impl From<ArticleMetadata> for crawlberg::ArticleMetadata {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::ArticleMetadata> for ArticleMetadata {
     fn from(val: crawlberg::ArticleMetadata) -> Self {
         Self {
@@ -6066,7 +6414,6 @@ impl From<crawlberg::ArticleMetadata> for ArticleMetadata {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<HreflangEntry> for crawlberg::HreflangEntry {
     fn from(val: HreflangEntry) -> Self {
         Self {
@@ -6077,7 +6424,6 @@ impl From<HreflangEntry> for crawlberg::HreflangEntry {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::HreflangEntry> for HreflangEntry {
     fn from(val: crawlberg::HreflangEntry) -> Self {
         Self {
@@ -6088,7 +6434,6 @@ impl From<crawlberg::HreflangEntry> for HreflangEntry {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<FaviconInfo> for crawlberg::FaviconInfo {
     fn from(val: FaviconInfo) -> Self {
         Self {
@@ -6101,7 +6446,7 @@ impl From<FaviconInfo> for crawlberg::FaviconInfo {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::FaviconInfo> for FaviconInfo {
     fn from(val: crawlberg::FaviconInfo) -> Self {
         Self {
@@ -6114,7 +6459,6 @@ impl From<crawlberg::FaviconInfo> for FaviconInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<HeadingInfo> for crawlberg::HeadingInfo {
     fn from(val: HeadingInfo) -> Self {
         Self {
@@ -6125,7 +6469,6 @@ impl From<HeadingInfo> for crawlberg::HeadingInfo {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::HeadingInfo> for HeadingInfo {
     fn from(val: crawlberg::HeadingInfo) -> Self {
         Self {
@@ -6136,7 +6479,6 @@ impl From<crawlberg::HeadingInfo> for HeadingInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ResponseMeta> for crawlberg::ResponseMeta {
     fn from(val: ResponseMeta) -> Self {
         Self {
@@ -6152,7 +6494,7 @@ impl From<ResponseMeta> for crawlberg::ResponseMeta {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::ResponseMeta> for ResponseMeta {
     fn from(val: crawlberg::ResponseMeta) -> Self {
         Self {
@@ -6272,7 +6614,6 @@ impl From<crawlberg::PageMetadata> for PageMetadata {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
     fn from(val: CrawlStreamRequest) -> Self {
         Self {
@@ -6282,7 +6623,6 @@ impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::CrawlStreamRequest> for CrawlStreamRequest {
     fn from(val: crawlberg::CrawlStreamRequest) -> Self {
         Self {
@@ -6292,7 +6632,6 @@ impl From<crawlberg::CrawlStreamRequest> for CrawlStreamRequest {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
     fn from(val: BatchCrawlStreamRequest) -> Self {
         Self {
@@ -6302,7 +6641,6 @@ impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::BatchCrawlStreamRequest> for BatchCrawlStreamRequest {
     fn from(val: crawlberg::BatchCrawlStreamRequest) -> Self {
         Self {
@@ -6312,7 +6650,7 @@ impl From<crawlberg::BatchCrawlStreamRequest> for BatchCrawlStreamRequest {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<CitationResult> for crawlberg::CitationResult {
     fn from(val: CitationResult) -> Self {
         Self {
@@ -6323,7 +6661,7 @@ impl From<CitationResult> for crawlberg::CitationResult {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::CitationResult> for CitationResult {
     fn from(val: crawlberg::CitationResult) -> Self {
         Self {
@@ -6334,7 +6672,6 @@ impl From<crawlberg::CitationResult> for CitationResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CitationReference> for crawlberg::CitationReference {
     fn from(val: CitationReference) -> Self {
         Self {
@@ -6346,7 +6683,6 @@ impl From<CitationReference> for crawlberg::CitationReference {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::CitationReference> for CitationReference {
     fn from(val: crawlberg::CitationReference) -> Self {
         Self {
@@ -6358,7 +6694,7 @@ impl From<crawlberg::CitationReference> for CitationReference {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<BatchScrapeResult> for crawlberg::BatchScrapeResult {
     fn from(val: BatchScrapeResult) -> Self {
         Self {
@@ -6382,7 +6718,7 @@ impl From<crawlberg::BatchScrapeResult> for BatchScrapeResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<BatchCrawlResult> for crawlberg::BatchCrawlResult {
     fn from(val: BatchCrawlResult) -> Self {
         Self {
@@ -6406,7 +6742,7 @@ impl From<crawlberg::BatchCrawlResult> for BatchCrawlResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<BatchScrapeResults> for crawlberg::BatchScrapeResults {
     fn from(val: BatchScrapeResults) -> Self {
         Self {
@@ -6419,7 +6755,7 @@ impl From<BatchScrapeResults> for crawlberg::BatchScrapeResults {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::BatchScrapeResults> for BatchScrapeResults {
     fn from(val: crawlberg::BatchScrapeResults) -> Self {
         Self {
@@ -6432,7 +6768,7 @@ impl From<crawlberg::BatchScrapeResults> for BatchScrapeResults {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<BatchCrawlResults> for crawlberg::BatchCrawlResults {
     fn from(val: BatchCrawlResults) -> Self {
         Self {
@@ -6445,7 +6781,7 @@ impl From<BatchCrawlResults> for crawlberg::BatchCrawlResults {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::BatchCrawlResults> for BatchCrawlResults {
     fn from(val: crawlberg::BatchCrawlResults) -> Self {
         Self {
@@ -6458,22 +6794,24 @@ impl From<crawlberg::BatchCrawlResults> for BatchCrawlResults {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<SsrfPolicy> for crawlberg::SsrfPolicy {
     fn from(val: SsrfPolicy) -> Self {
         Self {
             deny_private: val.deny_private,
+            allowlist: val.allowlist.into_iter().map(Into::into).collect(),
             max_redirects: val.max_redirects,
             ..Default::default()
         }
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::SsrfPolicy> for SsrfPolicy {
     fn from(val: crawlberg::SsrfPolicy) -> Self {
         Self {
             deny_private: val.deny_private,
+            allowlist: val.allowlist.into_iter().map(Into::into).collect(),
             max_redirects: val.max_redirects,
         }
     }
@@ -6539,6 +6877,60 @@ impl From<crawlberg::BrowserBackend> for BrowserBackend {
     }
 }
 
+impl From<DocumentContentEncoding> for crawlberg::DocumentContentEncoding {
+    fn from(val: DocumentContentEncoding) -> Self {
+        match val {
+            DocumentContentEncoding::Base64 => Self::Base64,
+        }
+    }
+}
+
+impl From<crawlberg::DocumentContentEncoding> for DocumentContentEncoding {
+    fn from(val: crawlberg::DocumentContentEncoding) -> Self {
+        match val {
+            crawlberg::DocumentContentEncoding::Base64 => Self::Base64,
+        }
+    }
+}
+
+impl From<CrawlStrategyKind> for crawlberg::CrawlStrategyKind {
+    fn from(val: CrawlStrategyKind) -> Self {
+        match val {
+            CrawlStrategyKind::Bfs => Self::Bfs,
+            CrawlStrategyKind::Dfs => Self::Dfs,
+            CrawlStrategyKind::BestFirst => Self::BestFirst,
+            CrawlStrategyKind::Adaptive => Self::Adaptive,
+        }
+    }
+}
+
+impl From<crawlberg::CrawlStrategyKind> for CrawlStrategyKind {
+    fn from(val: crawlberg::CrawlStrategyKind) -> Self {
+        match val {
+            crawlberg::CrawlStrategyKind::Bfs => Self::Bfs,
+            crawlberg::CrawlStrategyKind::Dfs => Self::Dfs,
+            crawlberg::CrawlStrategyKind::BestFirst => Self::BestFirst,
+            crawlberg::CrawlStrategyKind::Adaptive => Self::Adaptive,
+        }
+    }
+}
+
+impl From<ContentFilterKind> for crawlberg::ContentFilterKind {
+    fn from(val: ContentFilterKind) -> Self {
+        match val {
+            ContentFilterKind::Bm25 => Self::Bm25,
+        }
+    }
+}
+
+impl From<crawlberg::ContentFilterKind> for ContentFilterKind {
+    fn from(val: crawlberg::ContentFilterKind) -> Self {
+        match val {
+            crawlberg::ContentFilterKind::Bm25 => Self::Bm25,
+        }
+    }
+}
+
 impl From<AuthConfig> for crawlberg::AuthConfig {
     fn from(val: AuthConfig) -> Self {
         match val {
@@ -6768,6 +7160,32 @@ impl From<crawlberg::ScrollDirection> for ScrollDirection {
     }
 }
 
+impl From<HostMatcher> for crawlberg::HostMatcher {
+    fn from(val: HostMatcher) -> Self {
+        match val {
+            HostMatcher::Exact { value } => Self::Exact { value: value },
+            HostMatcher::Suffix { value } => Self::Suffix { value: value },
+            HostMatcher::Cidr { value } => Self::Cidr { value: value },
+        }
+    }
+}
+
+impl From<crawlberg::HostMatcher> for HostMatcher {
+    fn from(val: crawlberg::HostMatcher) -> Self {
+        match val {
+            crawlberg::HostMatcher::Exact { value } => Self::Exact {
+                value: value.to_string(),
+            },
+            crawlberg::HostMatcher::Suffix { value } => Self::Suffix {
+                value: value.to_string(),
+            },
+            crawlberg::HostMatcher::Cidr { value } => Self::Cidr {
+                value: value.to_string(),
+            },
+        }
+    }
+}
+
 /// Convert a `crawlberg::CrawlError` error to a Magnus runtime error.
 #[allow(dead_code)]
 fn crawl_error_to_magnus_err(e: crawlberg::CrawlError) -> magnus::Error {
@@ -7049,8 +7467,18 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("max_pages", method!(CrawlConfig::max_pages, 0))?;
 
+    class.define_method("max_links_per_page", method!(CrawlConfig::max_links_per_page, 0))?;
+
     class.define_method("max_concurrent", method!(CrawlConfig::max_concurrent, 0))?;
 
+    class.define_method("crawl_strategy", method!(CrawlConfig::crawl_strategy, 0))?;
+
+    class.define_method("content_filter", method!(CrawlConfig::content_filter, 0))?;
+
+    class.define_method("bm25_query", method!(CrawlConfig::bm25_query, 0))?;
+
+    class.define_method("bm25_threshold", method!(CrawlConfig::bm25_threshold, 0))?;
+
     class.define_method("respect_robots_txt", method!(CrawlConfig::respect_robots_txt, 0))?;
 
     class.define_method("soft_http_errors", method!(CrawlConfig::soft_http_errors, 0))?;
@@ -7115,6 +7543,13 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("document_mime_types", method!(CrawlConfig::document_mime_types, 0))?;
 
+    class.define_method("document_output_dir", method!(CrawlConfig::document_output_dir, 0))?;
+
+    class.define_method(
+        "document_content_encoding",
+        method!(CrawlConfig::document_content_encoding, 0),
+    )?;
+
     class.define_method("warc_output", method!(CrawlConfig::warc_output, 0))?;
 
     class.define_method("browser_profile", method!(CrawlConfig::browser_profile, 0))?;
@@ -7123,6 +7558,11 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("ssrf", method!(CrawlConfig::ssrf, 0))?;
 
+    class.define_method(
+        "ssrf_deny_private_explicit",
+        method!(CrawlConfig::ssrf_deny_private_explicit, 0),
+    )?;
+
     class.define_method("validate", method!(CrawlConfig::validate, 0))?;
 
     let class = module.define_class("BrowserExtras", ruby.class_object())?;
@@ -7151,6 +7591,12 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("headers", method!(DownloadedDocument::headers, 0))?;
 
+    class.define_method("truncated", method!(DownloadedDocument::truncated, 0))?;
+
+    class.define_method("content_path", method!(DownloadedDocument::content_path, 0))?;
+
+    class.define_method("content_base64", method!(DownloadedDocument::content_base64, 0))?;
+
     let class = module.define_class("InteractionResult", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(InteractionResult::new, -1))?;
@@ -7161,6 +7607,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("final_url", method!(InteractionResult::final_url, 0))?;
 
+    class.define_method("screenshot_base64", method!(InteractionResult::screenshot_base64, 0))?;
+
     let class = module.define_class("ActionResult", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(ActionResult::new, -1))?;
@@ -7231,6 +7679,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("extraction_meta", method!(ScrapeResult::extraction_meta, 0))?;
 
+    class.define_method("screenshot_base64", method!(ScrapeResult::screenshot_base64, 0))?;
+
     class.define_method("downloaded_document", method!(ScrapeResult::downloaded_document, 0))?;
 
     class.define_method("browser", method!(ScrapeResult::browser, 0))?;
@@ -7655,6 +8105,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("deny_private", method!(SsrfPolicy::deny_private, 0))?;
 
+    class.define_method("allowlist", method!(SsrfPolicy::allowlist, 0))?;
+
     class.define_method("max_redirects", method!(SsrfPolicy::max_redirects, 0))?;
 
     let class = module.define_class("CrawlStreamIterator", ruby.class_object())?;