crawlberg 1.1.3 → 1.2.1

data/ext/crawlberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:9239f0c4a976848daf64796cbf08d6835862f17985c5f163c603390b305b904e
+// alef:hash:0ce4d753fdb4854e44358639dcbaebee3449a4afa142dbc4f0a72aa72c214648
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -557,6 +557,7 @@ impl BrowserConfig {
 pub struct CrawlConfig {
     max_depth: Option<usize>,
     max_pages: Option<usize>,
+    max_links_per_page: Option<usize>,
     max_concurrent: Option<usize>,
     respect_robots_txt: bool,
     soft_http_errors: bool,
@@ -591,10 +592,13 @@ pub struct CrawlConfig {
     download_documents: bool,
     document_max_size: Option<usize>,
     document_mime_types: Vec<String>,
+    document_output_dir: Option<String>,
+    document_content_encoding: Option<DocumentContentEncoding>,
     warc_output: Option<String>,
     browser_profile: Option<String>,
     save_browser_profile: bool,
     ssrf: SsrfPolicy,
+    ssrf_deny_private_explicit: Option<bool>,
 }
 
 unsafe impl IntoValueFromNative for CrawlConfig {}
@@ -644,6 +648,9 @@ impl CrawlConfig {
             max_pages: kwargs
                 .get(ruby.to_symbol("max_pages"))
                 .and_then(|v| usize::try_convert(v).ok()),
+            max_links_per_page: kwargs
+                .get(ruby.to_symbol("max_links_per_page"))
+                .and_then(|v| usize::try_convert(v).ok()),
             max_concurrent: kwargs
                 .get(ruby.to_symbol("max_concurrent"))
                 .and_then(|v| usize::try_convert(v).ok()),
@@ -765,6 +772,12 @@ impl CrawlConfig {
                 .get(ruby.to_symbol("document_mime_types"))
                 .and_then(|v| <Vec<String>>::try_convert(v).ok())
                 .unwrap_or_default(),
+            document_output_dir: kwargs
+                .get(ruby.to_symbol("document_output_dir"))
+                .and_then(|v| String::try_convert(v).ok()),
+            document_content_encoding: kwargs
+                .get(ruby.to_symbol("document_content_encoding"))
+                .and_then(|v| DocumentContentEncoding::try_convert(v).ok()),
             warc_output: kwargs
                 .get(ruby.to_symbol("warc_output"))
                 .and_then(|v| String::try_convert(v).ok()),
@@ -779,6 +792,9 @@ impl CrawlConfig {
                 .get(ruby.to_symbol("ssrf"))
                 .and_then(|v| SsrfPolicy::try_convert(v).ok())
                 .unwrap_or_default(),
+            ssrf_deny_private_explicit: kwargs
+                .get(ruby.to_symbol("ssrf_deny_private_explicit"))
+                .and_then(|v| bool::try_convert(v).ok()),
         })
     }
 
@@ -790,6 +806,10 @@ impl CrawlConfig {
         self.max_pages
     }
 
+    fn max_links_per_page(&self) -> Option<usize> {
+        self.max_links_per_page
+    }
+
     fn max_concurrent(&self) -> Option<usize> {
         self.max_concurrent
     }
@@ -922,6 +942,14 @@ impl CrawlConfig {
         self.document_mime_types.clone()
     }
 
+    fn document_output_dir(&self) -> Option<String> {
+        self.document_output_dir.clone()
+    }
+
+    fn document_content_encoding(&self) -> Option<DocumentContentEncoding> {
+        self.document_content_encoding.clone()
+    }
+
     fn warc_output(&self) -> Option<String> {
         self.warc_output.clone()
     }
@@ -938,6 +966,10 @@ impl CrawlConfig {
         self.ssrf.clone()
     }
 
+    fn ssrf_deny_private_explicit(&self) -> Option<bool> {
+        self.ssrf_deny_private_explicit
+    }
+
     fn validate(&self) -> Result<(), Error> {
         #[allow(clippy::needless_update)]
         let core_self = crawlberg::CrawlConfig {
@@ -945,6 +977,8 @@ impl CrawlConfig {
 
             max_pages: self.max_pages,
 
+            max_links_per_page: self.max_links_per_page,
+
             max_concurrent: self.max_concurrent,
 
             respect_robots_txt: self.respect_robots_txt,
@@ -1016,6 +1050,10 @@ impl CrawlConfig {
 
             document_mime_types: self.document_mime_types.clone(),
 
+            document_output_dir: self.document_output_dir.clone().map(Into::into),
+
+            document_content_encoding: self.document_content_encoding.clone().map(Into::into),
+
             warc_output: self.warc_output.clone().map(Into::into),
 
             browser_profile: self.browser_profile.clone(),
@@ -1024,6 +1062,8 @@ impl CrawlConfig {
 
             ssrf: self.ssrf.clone().into(),
 
+            ssrf_deny_private_explicit: self.ssrf_deny_private_explicit,
+
             ..Default::default()
         };
         let result = core_self.validate().map_err(|e| {
@@ -1123,6 +1163,9 @@ pub struct DownloadedDocument {
     filename: Option<String>,
     content_hash: String,
     headers: HashMap<String, String>,
+    truncated: bool,
+    content_path: Option<String>,
+    content_base64: Option<String>,
 }
 
 unsafe impl IntoValueFromNative for DownloadedDocument {}
@@ -1189,6 +1232,16 @@ impl DownloadedDocument {
                 .get(ruby.to_symbol("headers"))
                 .and_then(|v| <HashMap<String, String>>::try_convert(v).ok())
                 .unwrap_or_default(),
+            truncated: kwargs
+                .get(ruby.to_symbol("truncated"))
+                .and_then(|v| bool::try_convert(v).ok())
+                .unwrap_or_default(),
+            content_path: kwargs
+                .get(ruby.to_symbol("content_path"))
+                .and_then(|v| String::try_convert(v).ok()),
+            content_base64: kwargs
+                .get(ruby.to_symbol("content_base64"))
+                .and_then(|v| String::try_convert(v).ok()),
         })
     }
 
@@ -1215,6 +1268,18 @@ impl DownloadedDocument {
     fn headers(&self) -> HashMap<String, String> {
         self.headers.clone()
     }
+
+    fn truncated(&self) -> bool {
+        self.truncated
+    }
+
+    fn content_path(&self) -> Option<String> {
+        self.content_path.clone()
+    }
+
+    fn content_base64(&self) -> Option<String> {
+        self.content_base64.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
@@ -1224,6 +1289,7 @@ pub struct InteractionResult {
     action_results: Vec<ActionResult>,
     final_html: String,
     final_url: String,
+    screenshot_base64: Option<String>,
 }
 
 unsafe impl IntoValueFromNative for InteractionResult {}
@@ -1279,6 +1345,9 @@ impl InteractionResult {
                 .get(ruby.to_symbol("final_url"))
                 .and_then(|v| String::try_convert(v).ok())
                 .unwrap_or_default(),
+            screenshot_base64: kwargs
+                .get(ruby.to_symbol("screenshot_base64"))
+                .and_then(|v| String::try_convert(v).ok()),
         })
     }
 
@@ -1293,6 +1362,10 @@ impl InteractionResult {
     fn final_url(&self) -> String {
         self.final_url.clone()
     }
+
+    fn screenshot_base64(&self) -> Option<String> {
+        self.screenshot_base64.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
@@ -1419,6 +1492,7 @@ pub struct ScrapeResult {
     markdown: Option<MarkdownResult>,
     extracted_data: Option<String>,
     extraction_meta: Option<ExtractionMeta>,
+    screenshot_base64: Option<String>,
     downloaded_document: Option<DownloadedDocument>,
     browser: Option<BrowserExtras>,
 }
@@ -1561,6 +1635,9 @@ impl ScrapeResult {
             extraction_meta: kwargs
                 .get(ruby.to_symbol("extraction_meta"))
                 .and_then(|v| ExtractionMeta::try_convert(v).ok()),
+            screenshot_base64: kwargs
+                .get(ruby.to_symbol("screenshot_base64"))
+                .and_then(|v| String::try_convert(v).ok()),
             downloaded_document: kwargs
                 .get(ruby.to_symbol("downloaded_document"))
                 .and_then(|v| DownloadedDocument::try_convert(v).ok()),
@@ -1674,6 +1751,10 @@ impl ScrapeResult {
         self.extraction_meta.clone()
     }
 
+    fn screenshot_base64(&self) -> Option<String> {
+        self.screenshot_base64.clone()
+    }
+
     fn downloaded_document(&self) -> Option<DownloadedDocument> {
         self.downloaded_document.clone()
     }
@@ -4358,6 +4439,7 @@ impl BatchCrawlResults {
 #[magnus::wrap(class = "Crawlberg::SsrfPolicy")]
 pub struct SsrfPolicy {
     deny_private: bool,
+    allowlist: Vec<HostMatcher>,
     max_redirects: u8,
 }
 
@@ -4406,6 +4488,10 @@ impl SsrfPolicy {
                 .get(ruby.to_symbol("deny_private"))
                 .and_then(|v| bool::try_convert(v).ok())
                 .unwrap_or(true),
+            allowlist: kwargs
+                .get(ruby.to_symbol("allowlist"))
+                .and_then(|v| <Vec<HostMatcher>>::try_convert(v).ok())
+                .unwrap_or_default(),
             max_redirects: kwargs
                 .get(ruby.to_symbol("max_redirects"))
                 .and_then(|v| u8::try_convert(v).ok())
@@ -4417,6 +4503,10 @@ impl SsrfPolicy {
         self.deny_private
     }
 
+    fn allowlist(&self) -> Vec<HostMatcher> {
+        self.allowlist.clone()
+    }
+
     fn max_redirects(&self) -> u8 {
         self.max_redirects
     }
@@ -4557,6 +4647,45 @@ impl magnus::TryConvert for BrowserBackend {
 unsafe impl IntoValueFromNative for BrowserBackend {}
 unsafe impl TryConvertOwned for BrowserBackend {}
 
+#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum DocumentContentEncoding {
+    Base64,
+}
+
+impl Default for DocumentContentEncoding {
+    fn default() -> Self {
+        Self::Base64
+    }
+}
+
+impl magnus::IntoValue for DocumentContentEncoding {
+    fn into_value_with(self, handle: &Ruby) -> magnus::Value {
+        let sym = match self {
+            DocumentContentEncoding::Base64 => "base64",
+        };
+        handle.to_symbol(sym).into_value_with(handle)
+    }
+}
+
+impl magnus::TryConvert for DocumentContentEncoding {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        let s: String = magnus::TryConvert::try_convert(val)?;
+        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
+        // and a lowercase fallback so fixtures written in any of those styles work.
+        match s.as_str() {
+            "base64" | "Base64" => Ok(DocumentContentEncoding::Base64),
+            other => Err(magnus::Error::new(
+                unsafe { Ruby::get_unchecked() }.exception_arg_error(),
+                format!("invalid DocumentContentEncoding value: {other}"),
+            )),
+        }
+    }
+}
+
+unsafe impl IntoValueFromNative for DocumentContentEncoding {}
+unsafe impl TryConvertOwned for DocumentContentEncoding {}
+
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(tag = "type")]
 pub enum AuthConfig {
@@ -5020,6 +5149,68 @@ impl magnus::TryConvert for ScrollDirection {
 unsafe impl IntoValueFromNative for ScrollDirection {}
 unsafe impl TryConvertOwned for ScrollDirection {}
 
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(tag = "type")]
+#[serde(rename_all = "snake_case")]
+pub enum HostMatcher {
+    Exact { value: String },
+    Suffix { value: String },
+    Cidr { value: String },
+}
+
+impl Default for HostMatcher {
+    fn default() -> Self {
+        Self::Exact {
+            value: Default::default(),
+        }
+    }
+}
+
+impl magnus::IntoValue for HostMatcher {
+    fn into_value_with(self, handle: &Ruby) -> magnus::Value {
+        match serde_json::to_value(&self) {
+            Ok(v) => json_to_ruby(handle, v),
+            Err(_) => handle.qnil().into_value_with(handle),
+        }
+    }
+}
+
+impl magnus::TryConvert for HostMatcher {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        // For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
+        // For unit enums or when passed as a string, fall back to string-based conversion.
+        let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
+            s
+        } else {
+            val.funcall::<_, _, String>("to_json", ()).map_err(|e| {
+                magnus::Error::new(
+                    unsafe { Ruby::get_unchecked() }.exception_type_error(),
+                    format!("no implicit conversion into HostMatcher: {}", e),
+                )
+            })?
+        };
+        // Try deserializing as JSON first (handles JSON strings like "\"markdown\"" or "{\"click\":{\"selector\":\"...\"}}\"")
+        // For internally-tagged enums, a bare variant string is wrapped as {"<tag>": value}.
+        // If that fails, try treating it as a plain string value and wrap in quotes
+        // If both fail, try as Custom variant (for untagged enum support)
+        serde_json::from_str(&json_str)
+            .or_else(|_| serde_json::from_value(serde_json::json!({ "type": json_str })))
+            .or_else(|_| serde_json::from_str(&format!("\"{json_str}\"")))
+            .or_else(|_| {
+                // Try as a JSON string for Custom variant (untagged enums accept any remaining value)
+                match serde_json::to_value(&json_str) {
+                    Ok(val) => serde_json::from_value(val),
+                    Err(e) => Err(e),
+                }
+            })
+            .map_err(|e| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_type_error(), e.to_string()))
+    }
+}
+
+unsafe impl IntoValueFromNative for HostMatcher {}
+unsafe impl TryConvertOwned for HostMatcher {}
+impl HostMatcher {}
+
 fn generate_citations(markdown: String) -> CitationResult {
     crawlberg::generate_citations(&markdown).into()
 }
@@ -5279,7 +5470,6 @@ fn batch_crawl_stream(engine: CrawlEngineHandle, req: BatchCrawlStreamRequest) -
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ExtractionMeta> for crawlberg::ExtractionMeta {
     fn from(val: ExtractionMeta) -> Self {
         Self {
@@ -5293,7 +5483,7 @@ impl From<ExtractionMeta> for crawlberg::ExtractionMeta {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::ExtractionMeta> for ExtractionMeta {
     fn from(val: crawlberg::ExtractionMeta) -> Self {
         Self {
@@ -5307,7 +5497,6 @@ impl From<crawlberg::ExtractionMeta> for ExtractionMeta {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ProxyConfig> for crawlberg::ProxyConfig {
     fn from(val: ProxyConfig) -> Self {
         Self {
@@ -5319,7 +5508,7 @@ impl From<ProxyConfig> for crawlberg::ProxyConfig {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::ProxyConfig> for ProxyConfig {
     fn from(val: crawlberg::ProxyConfig) -> Self {
         Self {
@@ -5331,7 +5520,6 @@ impl From<crawlberg::ProxyConfig> for ProxyConfig {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ContentConfig> for crawlberg::ContentConfig {
     fn from(val: ContentConfig) -> Self {
         Self {
@@ -5352,7 +5540,6 @@ impl From<ContentConfig> for crawlberg::ContentConfig {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::ContentConfig> for ContentConfig {
     fn from(val: crawlberg::ContentConfig) -> Self {
         Self {
@@ -5373,7 +5560,7 @@ impl From<crawlberg::ContentConfig> for ContentConfig {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<BrowserConfig> for crawlberg::BrowserConfig {
     fn from(val: BrowserConfig) -> Self {
         Self {
@@ -5423,6 +5610,7 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
         Self {
             max_depth: val.max_depth,
             max_pages: val.max_pages,
+            max_links_per_page: val.max_links_per_page,
             max_concurrent: val.max_concurrent,
             respect_robots_txt: val.respect_robots_txt,
             soft_http_errors: val.soft_http_errors,
@@ -5460,10 +5648,13 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
             download_documents: val.download_documents,
             document_max_size: val.document_max_size,
             document_mime_types: val.document_mime_types.into_iter().collect(),
+            document_output_dir: val.document_output_dir.map(Into::into),
+            document_content_encoding: val.document_content_encoding.map(Into::into),
             warc_output: val.warc_output.map(Into::into),
             browser_profile: val.browser_profile,
             save_browser_profile: val.save_browser_profile,
             ssrf: val.ssrf.into(),
+            ssrf_deny_private_explicit: val.ssrf_deny_private_explicit,
             ..Default::default()
         }
     }
@@ -5475,6 +5666,7 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
         Self {
             max_depth: val.max_depth,
             max_pages: val.max_pages,
+            max_links_per_page: val.max_links_per_page,
             max_concurrent: val.max_concurrent,
             respect_robots_txt: val.respect_robots_txt,
             soft_http_errors: val.soft_http_errors,
@@ -5512,10 +5704,13 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
             download_documents: val.download_documents,
             document_max_size: val.document_max_size,
             document_mime_types: val.document_mime_types.into_iter().collect(),
+            document_output_dir: val.document_output_dir.map(|p| p.to_string_lossy().to_string()),
+            document_content_encoding: val.document_content_encoding.map(Into::into),
             warc_output: val.warc_output.map(|p| p.to_string_lossy().to_string()),
             browser_profile: val.browser_profile.map(|v| v.to_string()),
             save_browser_profile: val.save_browser_profile,
             ssrf: val.ssrf.into(),
+            ssrf_deny_private_explicit: val.ssrf_deny_private_explicit,
         }
     }
 }
@@ -5533,7 +5728,7 @@ impl From<BrowserExtras> for crawlberg::BrowserExtras {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::BrowserExtras> for BrowserExtras {
     fn from(val: crawlberg::BrowserExtras) -> Self {
         Self {
@@ -5555,6 +5750,9 @@ impl From<DownloadedDocument> for crawlberg::DownloadedDocument {
             filename: val.filename.map(Into::into),
             content_hash: val.content_hash.into(),
             headers: val.headers.into_iter().map(|(k, v)| (k.into(), v.into())).collect(),
+            truncated: val.truncated,
+            content_path: val.content_path,
+            content_base64: val.content_base64,
             ..Default::default()
         }
     }
@@ -5570,18 +5768,22 @@ impl From<crawlberg::DownloadedDocument> for DownloadedDocument {
             filename: val.filename.as_ref().map(|v| v.to_string()),
             content_hash: val.content_hash.to_string(),
             headers: val.headers.into_iter().map(|(k, v)| (k.into(), v.into())).collect(),
+            truncated: val.truncated,
+            content_path: val.content_path.map(|v| v.to_string()),
+            content_base64: val.content_base64.map(|v| v.to_string()),
         }
     }
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<InteractionResult> for crawlberg::InteractionResult {
     fn from(val: InteractionResult) -> Self {
         Self {
             action_results: val.action_results.into_iter().map(Into::into).collect(),
             final_html: val.final_html,
             final_url: val.final_url,
+            screenshot_base64: val.screenshot_base64,
             ..Default::default()
         }
     }
@@ -5594,6 +5796,7 @@ impl From<crawlberg::InteractionResult> for InteractionResult {
             action_results: val.action_results.into_iter().map(Into::into).collect(),
             final_html: val.final_html.to_string(),
             final_url: val.final_url.to_string(),
+            screenshot_base64: val.screenshot_base64.map(|v| v.to_string()),
         }
     }
 }
@@ -5613,7 +5816,7 @@ impl From<ActionResult> for crawlberg::ActionResult {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::ActionResult> for ActionResult {
     fn from(val: crawlberg::ActionResult) -> Self {
         Self {
@@ -5657,6 +5860,7 @@ impl From<ScrapeResult> for crawlberg::ScrapeResult {
             markdown: val.markdown.map(Into::into),
             extracted_data: val.extracted_data.as_ref().and_then(|s| serde_json::from_str(s).ok()),
             extraction_meta: val.extraction_meta.map(Into::into),
+            screenshot_base64: val.screenshot_base64,
             downloaded_document: val.downloaded_document.map(Into::into),
             browser: val.browser.map(Into::into),
             ..Default::default()
@@ -5694,6 +5898,7 @@ impl From<crawlberg::ScrapeResult> for ScrapeResult {
             markdown: val.markdown.map(Into::into),
             extracted_data: val.extracted_data.as_ref().map(ToString::to_string),
             extraction_meta: val.extraction_meta.map(Into::into),
+            screenshot_base64: val.screenshot_base64.map(|v| v.to_string()),
             downloaded_document: val.downloaded_document.map(Into::into),
             browser: val.browser.map(Into::into),
         }
@@ -5761,7 +5966,7 @@ impl From<crawlberg::CrawlPageResult> for CrawlPageResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<CrawlResult> for crawlberg::CrawlResult {
     fn from(val: CrawlResult) -> Self {
         Self {
@@ -5795,7 +6000,6 @@ impl From<crawlberg::CrawlResult> for CrawlResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<SitemapUrl> for crawlberg::SitemapUrl {
     fn from(val: SitemapUrl) -> Self {
         Self {
@@ -5808,7 +6012,7 @@ impl From<SitemapUrl> for crawlberg::SitemapUrl {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::SitemapUrl> for SitemapUrl {
     fn from(val: crawlberg::SitemapUrl) -> Self {
         Self {
@@ -5821,7 +6025,7 @@ impl From<crawlberg::SitemapUrl> for SitemapUrl {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<MapResult> for crawlberg::MapResult {
     fn from(val: MapResult) -> Self {
         Self {
@@ -5831,7 +6035,7 @@ impl From<MapResult> for crawlberg::MapResult {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::MapResult> for MapResult {
     fn from(val: crawlberg::MapResult) -> Self {
         Self {
@@ -5841,7 +6045,7 @@ impl From<crawlberg::MapResult> for MapResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<MarkdownResult> for crawlberg::MarkdownResult {
     fn from(val: MarkdownResult) -> Self {
         Self {
@@ -5863,7 +6067,7 @@ impl From<MarkdownResult> for crawlberg::MarkdownResult {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::MarkdownResult> for MarkdownResult {
     fn from(val: crawlberg::MarkdownResult) -> Self {
         Self {
@@ -5878,7 +6082,7 @@ impl From<crawlberg::MarkdownResult> for MarkdownResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<LinkInfo> for crawlberg::LinkInfo {
     fn from(val: LinkInfo) -> Self {
         Self {
@@ -5906,7 +6110,7 @@ impl From<crawlberg::LinkInfo> for LinkInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<ImageInfo> for crawlberg::ImageInfo {
     fn from(val: ImageInfo) -> Self {
         Self {
@@ -5934,7 +6138,7 @@ impl From<crawlberg::ImageInfo> for ImageInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<FeedInfo> for crawlberg::FeedInfo {
     fn from(val: FeedInfo) -> Self {
         Self {
@@ -5958,7 +6162,6 @@ impl From<crawlberg::FeedInfo> for FeedInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<JsonLdEntry> for crawlberg::JsonLdEntry {
     fn from(val: JsonLdEntry) -> Self {
         Self {
@@ -5970,7 +6173,7 @@ impl From<JsonLdEntry> for crawlberg::JsonLdEntry {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::JsonLdEntry> for JsonLdEntry {
     fn from(val: crawlberg::JsonLdEntry) -> Self {
         Self {
@@ -5982,7 +6185,6 @@ impl From<crawlberg::JsonLdEntry> for JsonLdEntry {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CookieInfo> for crawlberg::CookieInfo {
     fn from(val: CookieInfo) -> Self {
         Self {
@@ -5995,7 +6197,7 @@ impl From<CookieInfo> for crawlberg::CookieInfo {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::CookieInfo> for CookieInfo {
     fn from(val: crawlberg::CookieInfo) -> Self {
         Self {
@@ -6008,7 +6210,7 @@ impl From<crawlberg::CookieInfo> for CookieInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<DownloadedAsset> for crawlberg::DownloadedAsset {
     fn from(val: DownloadedAsset) -> Self {
         Self {
@@ -6038,7 +6240,6 @@ impl From<crawlberg::DownloadedAsset> for DownloadedAsset {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ArticleMetadata> for crawlberg::ArticleMetadata {
     fn from(val: ArticleMetadata) -> Self {
         Self {
@@ -6052,7 +6253,7 @@ impl From<ArticleMetadata> for crawlberg::ArticleMetadata {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::ArticleMetadata> for ArticleMetadata {
     fn from(val: crawlberg::ArticleMetadata) -> Self {
         Self {
@@ -6066,7 +6267,6 @@ impl From<crawlberg::ArticleMetadata> for ArticleMetadata {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<HreflangEntry> for crawlberg::HreflangEntry {
     fn from(val: HreflangEntry) -> Self {
         Self {
@@ -6077,7 +6277,6 @@ impl From<HreflangEntry> for crawlberg::HreflangEntry {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::HreflangEntry> for HreflangEntry {
     fn from(val: crawlberg::HreflangEntry) -> Self {
         Self {
@@ -6088,7 +6287,6 @@ impl From<crawlberg::HreflangEntry> for HreflangEntry {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<FaviconInfo> for crawlberg::FaviconInfo {
     fn from(val: FaviconInfo) -> Self {
         Self {
@@ -6101,7 +6299,7 @@ impl From<FaviconInfo> for crawlberg::FaviconInfo {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::FaviconInfo> for FaviconInfo {
     fn from(val: crawlberg::FaviconInfo) -> Self {
         Self {
@@ -6114,7 +6312,6 @@ impl From<crawlberg::FaviconInfo> for FaviconInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<HeadingInfo> for crawlberg::HeadingInfo {
     fn from(val: HeadingInfo) -> Self {
         Self {
@@ -6125,7 +6322,6 @@ impl From<HeadingInfo> for crawlberg::HeadingInfo {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::HeadingInfo> for HeadingInfo {
     fn from(val: crawlberg::HeadingInfo) -> Self {
         Self {
@@ -6136,7 +6332,6 @@ impl From<crawlberg::HeadingInfo> for HeadingInfo {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ResponseMeta> for crawlberg::ResponseMeta {
     fn from(val: ResponseMeta) -> Self {
         Self {
@@ -6152,7 +6347,7 @@ impl From<ResponseMeta> for crawlberg::ResponseMeta {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::redundant_closure)]
 impl From<crawlberg::ResponseMeta> for ResponseMeta {
     fn from(val: crawlberg::ResponseMeta) -> Self {
         Self {
@@ -6272,7 +6467,6 @@ impl From<crawlberg::PageMetadata> for PageMetadata {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
     fn from(val: CrawlStreamRequest) -> Self {
         Self {
@@ -6282,7 +6476,6 @@ impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::CrawlStreamRequest> for CrawlStreamRequest {
     fn from(val: crawlberg::CrawlStreamRequest) -> Self {
         Self {
@@ -6292,7 +6485,6 @@ impl From<crawlberg::CrawlStreamRequest> for CrawlStreamRequest {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
     fn from(val: BatchCrawlStreamRequest) -> Self {
         Self {
@@ -6302,7 +6494,6 @@ impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::BatchCrawlStreamRequest> for BatchCrawlStreamRequest {
     fn from(val: crawlberg::BatchCrawlStreamRequest) -> Self {
         Self {
@@ -6312,7 +6503,7 @@ impl From<crawlberg::BatchCrawlStreamRequest> for BatchCrawlStreamRequest {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<CitationResult> for crawlberg::CitationResult {
     fn from(val: CitationResult) -> Self {
         Self {
@@ -6323,7 +6514,7 @@ impl From<CitationResult> for crawlberg::CitationResult {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::CitationResult> for CitationResult {
     fn from(val: crawlberg::CitationResult) -> Self {
         Self {
@@ -6334,7 +6525,6 @@ impl From<crawlberg::CitationResult> for CitationResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CitationReference> for crawlberg::CitationReference {
     fn from(val: CitationReference) -> Self {
         Self {
@@ -6346,7 +6536,6 @@ impl From<CitationReference> for crawlberg::CitationReference {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<crawlberg::CitationReference> for CitationReference {
     fn from(val: crawlberg::CitationReference) -> Self {
         Self {
@@ -6358,7 +6547,7 @@ impl From<crawlberg::CitationReference> for CitationReference {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<BatchScrapeResult> for crawlberg::BatchScrapeResult {
     fn from(val: BatchScrapeResult) -> Self {
         Self {
@@ -6382,7 +6571,7 @@ impl From<crawlberg::BatchScrapeResult> for BatchScrapeResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<BatchCrawlResult> for crawlberg::BatchCrawlResult {
     fn from(val: BatchCrawlResult) -> Self {
         Self {
@@ -6406,7 +6595,7 @@ impl From<crawlberg::BatchCrawlResult> for BatchCrawlResult {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<BatchScrapeResults> for crawlberg::BatchScrapeResults {
     fn from(val: BatchScrapeResults) -> Self {
         Self {
@@ -6419,7 +6608,7 @@ impl From<BatchScrapeResults> for crawlberg::BatchScrapeResults {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::BatchScrapeResults> for BatchScrapeResults {
     fn from(val: crawlberg::BatchScrapeResults) -> Self {
         Self {
@@ -6432,7 +6621,7 @@ impl From<crawlberg::BatchScrapeResults> for BatchScrapeResults {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<BatchCrawlResults> for crawlberg::BatchCrawlResults {
     fn from(val: BatchCrawlResults) -> Self {
         Self {
@@ -6445,7 +6634,7 @@ impl From<BatchCrawlResults> for crawlberg::BatchCrawlResults {
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::BatchCrawlResults> for BatchCrawlResults {
     fn from(val: crawlberg::BatchCrawlResults) -> Self {
         Self {
@@ -6458,22 +6647,24 @@ impl From<crawlberg::BatchCrawlResults> for BatchCrawlResults {
 }
 
 #[allow(clippy::needless_update)]
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<SsrfPolicy> for crawlberg::SsrfPolicy {
     fn from(val: SsrfPolicy) -> Self {
         Self {
             deny_private: val.deny_private,
+            allowlist: val.allowlist.into_iter().map(Into::into).collect(),
             max_redirects: val.max_redirects,
             ..Default::default()
         }
     }
 }
 
-#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+#[allow(clippy::useless_conversion)]
 impl From<crawlberg::SsrfPolicy> for SsrfPolicy {
     fn from(val: crawlberg::SsrfPolicy) -> Self {
         Self {
             deny_private: val.deny_private,
+            allowlist: val.allowlist.into_iter().map(Into::into).collect(),
             max_redirects: val.max_redirects,
         }
     }
@@ -6539,6 +6730,22 @@ impl From<crawlberg::BrowserBackend> for BrowserBackend {
     }
 }
 
+impl From<DocumentContentEncoding> for crawlberg::DocumentContentEncoding {
+    fn from(val: DocumentContentEncoding) -> Self {
+        match val {
+            DocumentContentEncoding::Base64 => Self::Base64,
+        }
+    }
+}
+
+impl From<crawlberg::DocumentContentEncoding> for DocumentContentEncoding {
+    fn from(val: crawlberg::DocumentContentEncoding) -> Self {
+        match val {
+            crawlberg::DocumentContentEncoding::Base64 => Self::Base64,
+        }
+    }
+}
+
 impl From<AuthConfig> for crawlberg::AuthConfig {
     fn from(val: AuthConfig) -> Self {
         match val {
@@ -6768,6 +6975,32 @@ impl From<crawlberg::ScrollDirection> for ScrollDirection {
     }
 }
 
+impl From<HostMatcher> for crawlberg::HostMatcher {
+    fn from(val: HostMatcher) -> Self {
+        match val {
+            HostMatcher::Exact { value } => Self::Exact { value: value },
+            HostMatcher::Suffix { value } => Self::Suffix { value: value },
+            HostMatcher::Cidr { value } => Self::Cidr { value: value },
+        }
+    }
+}
+
+impl From<crawlberg::HostMatcher> for HostMatcher {
+    fn from(val: crawlberg::HostMatcher) -> Self {
+        match val {
+            crawlberg::HostMatcher::Exact { value } => Self::Exact {
+                value: value.to_string(),
+            },
+            crawlberg::HostMatcher::Suffix { value } => Self::Suffix {
+                value: value.to_string(),
+            },
+            crawlberg::HostMatcher::Cidr { value } => Self::Cidr {
+                value: value.to_string(),
+            },
+        }
+    }
+}
+
 /// Convert a `crawlberg::CrawlError` error to a Magnus runtime error.
 #[allow(dead_code)]
 fn crawl_error_to_magnus_err(e: crawlberg::CrawlError) -> magnus::Error {
@@ -7049,6 +7282,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("max_pages", method!(CrawlConfig::max_pages, 0))?;
 
+    class.define_method("max_links_per_page", method!(CrawlConfig::max_links_per_page, 0))?;
+
     class.define_method("max_concurrent", method!(CrawlConfig::max_concurrent, 0))?;
 
     class.define_method("respect_robots_txt", method!(CrawlConfig::respect_robots_txt, 0))?;
@@ -7115,6 +7350,13 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("document_mime_types", method!(CrawlConfig::document_mime_types, 0))?;
 
+    class.define_method("document_output_dir", method!(CrawlConfig::document_output_dir, 0))?;
+
+    class.define_method(
+        "document_content_encoding",
+        method!(CrawlConfig::document_content_encoding, 0),
+    )?;
+
     class.define_method("warc_output", method!(CrawlConfig::warc_output, 0))?;
 
     class.define_method("browser_profile", method!(CrawlConfig::browser_profile, 0))?;
@@ -7123,6 +7365,11 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("ssrf", method!(CrawlConfig::ssrf, 0))?;
 
+    class.define_method(
+        "ssrf_deny_private_explicit",
+        method!(CrawlConfig::ssrf_deny_private_explicit, 0),
+    )?;
+
     class.define_method("validate", method!(CrawlConfig::validate, 0))?;
 
     let class = module.define_class("BrowserExtras", ruby.class_object())?;
@@ -7151,6 +7398,12 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("headers", method!(DownloadedDocument::headers, 0))?;
 
+    class.define_method("truncated", method!(DownloadedDocument::truncated, 0))?;
+
+    class.define_method("content_path", method!(DownloadedDocument::content_path, 0))?;
+
+    class.define_method("content_base64", method!(DownloadedDocument::content_base64, 0))?;
+
     let class = module.define_class("InteractionResult", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(InteractionResult::new, -1))?;
@@ -7161,6 +7414,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("final_url", method!(InteractionResult::final_url, 0))?;
 
+    class.define_method("screenshot_base64", method!(InteractionResult::screenshot_base64, 0))?;
+
     let class = module.define_class("ActionResult", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(ActionResult::new, -1))?;
@@ -7231,6 +7486,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("extraction_meta", method!(ScrapeResult::extraction_meta, 0))?;
 
+    class.define_method("screenshot_base64", method!(ScrapeResult::screenshot_base64, 0))?;
+
     class.define_method("downloaded_document", method!(ScrapeResult::downloaded_document, 0))?;
 
     class.define_method("browser", method!(ScrapeResult::browser, 0))?;
@@ -7655,6 +7912,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("deny_private", method!(SsrfPolicy::deny_private, 0))?;
 
+    class.define_method("allowlist", method!(SsrfPolicy::allowlist, 0))?;
+
     class.define_method("max_redirects", method!(SsrfPolicy::max_redirects, 0))?;
 
     let class = module.define_class("CrawlStreamIterator", ruby.class_object())?;