xberg 1.3.4 → 1.3.6

data/ext/xberg_rb/native/Cargo.toml

@@ -1,11 +1,11 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:715ad8642ffff546e63c46773a283e4a03bcb5b87f1ccadddc688baa7e1c4d1d
+# alef:hash:c27e6904867bcdf10220f00395fbf4b5c652b935305f84d03c851f2a5ed1ef8f
 # To regenerate: alef generate
 # To verify freshness: alef verify
 
 [package]
 name = "xberg-rb"
-version = "1.3.4"
+version = "1.3.6"
 edition = "2024"
 license = "MIT"
 description = "High-performance document intelligence library"
@@ -72,8 +72,8 @@ xml = ["xberg/xml"]
 
 [dependencies]
 async-trait = "0.1"
-crawlberg = { default-features = false, version = "1.8.0" }
-html-to-markdown-rs = { default-features = false, version = "3.16" }
+crawlberg = { default-features = false, version = "1.10.2" }
+html-to-markdown-rs = { default-features = false, version = "3.17.2" }
 magnus = "0.9"
 rb-sys = ">=0.9.130, <0.10"
 serde = { version = "1", features = ["derive"] }
@@ -82,13 +82,13 @@ tokio = { version = "1", features = ["rt-multi-thread"] }
 tracing = "0.1"
 
 [target.'cfg(all(target_os = "macos", target_arch = "x86_64"))'.dependencies]
-xberg = { version = "1.3.4", default-features = false, features = ["macos-intel-target", "simd-utf8"] }
+xberg = { version = "1.3.6", default-features = false, features = ["macos-intel-target", "simd-utf8"] }
 
 [target.'cfg(not(any(target_os = "windows", all(target_os = "macos", target_arch = "x86_64"))))'.dependencies]
-xberg = { version = "1.3.4", features = ["full", "pdf", "pdf-pdfium", "ocr", "paddle-ocr", "paddle-ocr-types", "layout-detection", "layout-types", "embeddings", "embedding-presets", "reranker", "reranker-presets", "sparse-embeddings", "sparse-embedding-presets", "late-interaction", "late-interaction-presets", "chunking", "keywords-yake", "keywords-rake", "language-detection", "html", "tree-sitter", "office", "email", "archives", "stopwords", "auto-rotate", "auto-rotate-types", "tokio-runtime", "api", "api-types", "mcp", "liter-llm", "quality", "svg", "transcription", "transcription-types", "classification", "captioning", "formula-recognition", "ner-onnx", "ner-llm", "diff", "markdown-footnotes", "redaction", "enrichment", "heuristics", "presets", "structured", "url-ingestion"] }
+xberg = { version = "1.3.6", features = ["full", "pdf", "pdf-pdfium", "ocr", "paddle-ocr", "paddle-ocr-types", "layout-detection", "layout-types", "embeddings", "embedding-presets", "reranker", "reranker-presets", "sparse-embeddings", "sparse-embedding-presets", "late-interaction", "late-interaction-presets", "chunking", "keywords-yake", "keywords-rake", "language-detection", "html", "tree-sitter", "office", "email", "archives", "stopwords", "auto-rotate", "auto-rotate-types", "tokio-runtime", "api", "api-types", "mcp", "liter-llm", "quality", "svg", "transcription", "transcription-types", "classification", "captioning", "formula-recognition", "ner-onnx", "ner-llm", "diff", "markdown-footnotes", "redaction", "enrichment", "heuristics", "presets", "structured", "url-ingestion"] }
 
 [target.'cfg(target_os = "windows")'.dependencies]
-xberg = { version = "1.3.4", default-features = false, features = ["windows-gnu-target", "simd-utf8"] }
+xberg = { version = "1.3.6", default-features = false, features = ["windows-gnu-target", "simd-utf8"] }
 
 # This crate deliberately does not use `[lints]` / `workspace = true`: its C-ABI /
 # PyO3 / napi / ext-php-rs / NIF boundary requires `unsafe` code, and the workspace's

data/ext/xberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:3e862eca7b1250c4aa8d40b5563353572b94bd205d31f9f57c9efc950fcd26cb
+// alef:hash:3751950d58564a09b72eb0e65f2f992e39f3dcd2cd851cd4493626ba6caef9a0
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -1126,7 +1126,7 @@ impl BoundingBox {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize)]
+#[derive(Clone, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Xberg::BrowserConfig")]
 pub struct BrowserConfig {
@@ -1146,6 +1146,8 @@ pub struct BrowserConfig {
     robots_user_agent: Option<String>,
     capture_network_events: bool,
     session_affinity: bool,
+    chrome_path: Option<String>,
+    chrome_args: Vec<String>,
 }
 
 unsafe impl IntoValueFromNative for BrowserConfig {}
@@ -1175,6 +1177,29 @@ impl magnus::TryConvert for BrowserConfig {
 }
 
 unsafe impl TryConvertOwned for BrowserConfig {}
+impl std::fmt::Debug for BrowserConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut debug = f.debug_struct("BrowserConfig");
+        debug.field("mode", &self.mode);
+        debug.field("backend", &self.backend);
+        debug.field("endpoint", &"<redacted>");
+        debug.field("timeout", &self.timeout);
+        debug.field("overall_timeout", &self.overall_timeout);
+        debug.field("shutdown_timeout", &self.shutdown_timeout);
+        debug.field("wait", &self.wait);
+        debug.field("wait_selector", &self.wait_selector);
+        debug.field("extra_wait", &self.extra_wait);
+        debug.field("proxy", &self.proxy);
+        debug.field("block_url_patterns", &self.block_url_patterns);
+        debug.field("eval_script", &"<redacted>");
+        debug.field("robots_user_agent", &self.robots_user_agent);
+        debug.field("capture_network_events", &self.capture_network_events);
+        debug.field("session_affinity", &self.session_affinity);
+        debug.field("chrome_path", &self.chrome_path);
+        debug.field("chrome_args", &"<redacted>");
+        debug.finish()
+    }
+}
 impl<'de> serde::Deserialize<'de> for BrowserConfig {
     fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
     where
@@ -1332,6 +1357,24 @@ impl BrowserConfig {
                 })?,
                 None => true,
             },
+            chrome_path: match kwargs.get(ruby.to_symbol("chrome_path")).filter(|v| !v.is_nil()) {
+                Some(v) => Some(String::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `chrome_path`: {}", e),
+                    )
+                })?),
+                None => None,
+            },
+            chrome_args: match kwargs.get(ruby.to_symbol("chrome_args")) {
+                Some(v) => <Vec<String>>::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `chrome_args`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
         })
     }
 
@@ -1394,6 +1437,14 @@ impl BrowserConfig {
     fn session_affinity(&self) -> bool {
         self.session_affinity
     }
+
+    fn chrome_path(&self) -> Option<String> {
+        self.chrome_path.clone()
+    }
+
+    fn chrome_args(&self) -> Vec<String> {
+        self.chrome_args.clone()
+    }
 }
 
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
@@ -4090,6 +4141,7 @@ pub struct ConversionOptions {
     include_document_structure: bool,
     extract_images: bool,
     max_image_size: u64,
+    max_input_size: Option<u64>,
     capture_svg: bool,
     infer_dimensions: bool,
     max_depth: Option<usize>,
@@ -4490,6 +4542,15 @@ impl ConversionOptions {
                 })?,
                 None => 5242880,
             },
+            max_input_size: match kwargs.get(ruby.to_symbol("max_input_size")).filter(|v| !v.is_nil()) {
+                Some(v) => Some(u64::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `max_input_size`: {}", e),
+                    )
+                })?),
+                None => None,
+            },
             capture_svg: match kwargs.get(ruby.to_symbol("capture_svg")) {
                 Some(v) => bool::try_convert(v).map_err(|e| {
                     magnus::Error::new(
@@ -4703,6 +4764,10 @@ impl ConversionOptions {
         self.max_image_size
     }
 
+    fn max_input_size(&self) -> Option<u64> {
+        self.max_input_size
+    }
+
     fn capture_svg(&self) -> bool {
         self.capture_svg
     }
@@ -4995,7 +5060,7 @@ impl CoreProperties {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, serde::Serialize, serde::Deserialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Xberg::CrawlConfig")]
 pub struct CrawlConfig {
@@ -5015,6 +5080,7 @@ pub struct CrawlConfig {
     include_paths: Vec<String>,
     exclude_paths: Vec<String>,
     path_patterns_match_query: bool,
+    path_patterns_match_url: bool,
     dedup_include_query: bool,
     strip_tracking_params: bool,
     tracking_params: Vec<String>,
@@ -5083,6 +5149,67 @@ impl magnus::TryConvert for CrawlConfig {
 }
 
 unsafe impl TryConvertOwned for CrawlConfig {}
+impl std::fmt::Debug for CrawlConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut debug = f.debug_struct("CrawlConfig");
+        debug.field("max_depth", &self.max_depth);
+        debug.field("max_pages", &self.max_pages);
+        debug.field("max_links_per_page", &self.max_links_per_page);
+        debug.field("max_concurrent", &self.max_concurrent);
+        debug.field("crawl_strategy", &self.crawl_strategy);
+        debug.field("content_filter", &self.content_filter);
+        debug.field("bm25_query", &self.bm25_query);
+        debug.field("bm25_threshold", &self.bm25_threshold);
+        debug.field("respect_robots_txt", &self.respect_robots_txt);
+        debug.field("soft_http_errors", &self.soft_http_errors);
+        debug.field("user_agent", &self.user_agent);
+        debug.field("stay_on_domain", &self.stay_on_domain);
+        debug.field("allow_subdomains", &self.allow_subdomains);
+        debug.field("include_paths", &self.include_paths);
+        debug.field("exclude_paths", &self.exclude_paths);
+        debug.field("path_patterns_match_query", &self.path_patterns_match_query);
+        debug.field("path_patterns_match_url", &self.path_patterns_match_url);
+        debug.field("dedup_include_query", &self.dedup_include_query);
+        debug.field("strip_tracking_params", &self.strip_tracking_params);
+        debug.field("tracking_params", &self.tracking_params);
+        debug.field("custom_headers", &"<redacted>");
+        debug.field("request_timeout", &self.request_timeout);
+        debug.field("rate_limit_ms", &self.rate_limit_ms);
+        debug.field("max_redirects", &self.max_redirects);
+        debug.field("retry_count", &self.retry_count);
+        debug.field("retry_codes", &self.retry_codes);
+        debug.field("retry_initial_delay_ms", &self.retry_initial_delay_ms);
+        debug.field("retry_max_delay_ms", &self.retry_max_delay_ms);
+        debug.field("rate_limit_jitter_ratio", &self.rate_limit_jitter_ratio);
+        debug.field("cookies_enabled", &self.cookies_enabled);
+        debug.field("auth", &self.auth);
+        debug.field("max_body_size", &self.max_body_size);
+        debug.field("remove_tags", &self.remove_tags);
+        debug.field("content", &self.content);
+        debug.field("map_limit", &self.map_limit);
+        debug.field("map_search", &self.map_search);
+        debug.field("download_assets", &self.download_assets);
+        debug.field("asset_types", &self.asset_types);
+        debug.field("max_asset_size", &self.max_asset_size);
+        debug.field("browser", &self.browser);
+        debug.field("proxy", &self.proxy);
+        debug.field("user_agents", &self.user_agents);
+        debug.field("capture_screenshot", &self.capture_screenshot);
+        debug.field("follow_document_urls", &self.follow_document_urls);
+        debug.field("document_url_depth", &self.document_url_depth);
+        debug.field("download_documents", &self.download_documents);
+        debug.field("document_max_size", &self.document_max_size);
+        debug.field("document_mime_types", &self.document_mime_types);
+        debug.field("document_output_dir", &self.document_output_dir);
+        debug.field("document_content_encoding", &self.document_content_encoding);
+        debug.field("warc_output", &self.warc_output);
+        debug.field("browser_profile", &self.browser_profile);
+        debug.field("save_browser_profile", &self.save_browser_profile);
+        debug.field("ssrf", &self.ssrf);
+        debug.field("ssrf_deny_private_explicit", &self.ssrf_deny_private_explicit);
+        debug.finish()
+    }
+}
 
 impl Default for CrawlConfig {
     fn default() -> Self {
@@ -5241,6 +5368,15 @@ impl CrawlConfig {
                 })?,
                 None => false,
             },
+            path_patterns_match_url: match kwargs.get(ruby.to_symbol("path_patterns_match_url")) {
+                Some(v) => bool::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `path_patterns_match_url`: {}", e),
+                    )
+                })?,
+                None => false,
+            },
             dedup_include_query: match kwargs.get(ruby.to_symbol("dedup_include_query")) {
                 Some(v) => bool::try_convert(v).map_err(|e| {
                     magnus::Error::new(
@@ -5664,6 +5800,10 @@ impl CrawlConfig {
         self.path_patterns_match_query
     }
 
+    fn path_patterns_match_url(&self) -> bool {
+        self.path_patterns_match_url
+    }
+
     fn dedup_include_query(&self) -> bool {
         self.dedup_include_query
     }
@@ -29315,7 +29455,7 @@ impl PropertyChange {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, serde::Serialize, serde::Deserialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Xberg::ProxyConfig")]
 pub struct ProxyConfig {
@@ -29351,6 +29491,15 @@ impl magnus::TryConvert for ProxyConfig {
 }
 
 unsafe impl TryConvertOwned for ProxyConfig {}
+impl std::fmt::Debug for ProxyConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let mut debug = f.debug_struct("ProxyConfig");
+        debug.field("url", &"<redacted>");
+        debug.field("username", &self.username);
+        debug.field("password", &"<redacted>");
+        debug.finish()
+    }
+}
 
 impl Default for ProxyConfig {
     fn default() -> Self {
@@ -32038,6 +32187,7 @@ impl SparseEmbeddingPreset {
 pub struct SsrfPolicy {
     deny_private: bool,
     allowlist: Vec<HostMatcher>,
+    denylist: Vec<HostMatcher>,
     max_redirects: u8,
     scheme_allowlist: Vec<String>,
 }
@@ -32109,6 +32259,15 @@ impl SsrfPolicy {
                 })?,
                 None => Default::default(),
             },
+            denylist: match kwargs.get(ruby.to_symbol("denylist")) {
+                Some(v) => <Vec<HostMatcher>>::try_convert(v).map_err(|e| {
+                    magnus::Error::new(
+                        unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                        format!("invalid value for `denylist`: {}", e),
+                    )
+                })?,
+                None => Default::default(),
+            },
             max_redirects: match kwargs.get(ruby.to_symbol("max_redirects")) {
                 Some(v) => u8::try_convert(v).map_err(|e| {
                     magnus::Error::new(
@@ -32138,6 +32297,10 @@ impl SsrfPolicy {
         self.allowlist.clone()
     }
 
+    fn denylist(&self) -> Vec<HostMatcher> {
+        self.denylist.clone()
+    }
+
     fn max_redirects(&self) -> u8 {
         self.max_redirects
     }
@@ -36266,7 +36429,7 @@ impl magnus::TryConvert for AssetCategory {
 unsafe impl IntoValueFromNative for AssetCategory {}
 unsafe impl TryConvertOwned for AssetCategory {}
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, serde::Serialize, serde::Deserialize)]
 #[serde(tag = "type")]
 pub enum AuthConfig {
     #[serde(rename = "basic")]
@@ -36332,6 +36495,11 @@ unsafe impl IntoValueFromNative for AuthConfig {}
 unsafe impl TryConvertOwned for AuthConfig {}
 
 impl AuthConfig {}
+impl std::fmt::Debug for AuthConfig {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.write_str("AuthConfig(<redacted>)")
+    }
+}
 
 #[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "snake_case")]
@@ -48325,6 +48493,8 @@ impl From<BrowserConfig> for crawlberg::BrowserConfig {
             robots_user_agent: val.robots_user_agent,
             capture_network_events: val.capture_network_events,
             session_affinity: val.session_affinity,
+            chrome_path: val.chrome_path.map(Into::into),
+            chrome_args: val.chrome_args.into_iter().collect(),
             ..Default::default()
         }
     }
@@ -48349,6 +48519,8 @@ impl From<crawlberg::BrowserConfig> for BrowserConfig {
             robots_user_agent: val.robots_user_agent.map(|v| v.to_string()),
             capture_network_events: val.capture_network_events,
             session_affinity: val.session_affinity,
+            chrome_path: val.chrome_path.map(|p| p.to_string_lossy().to_string()),
+            chrome_args: val.chrome_args.into_iter().collect(),
         }
     }
 }
@@ -48918,6 +49090,7 @@ impl From<ConversionOptions> for html_to_markdown_rs::ConversionOptions {
             include_document_structure: val.include_document_structure,
             extract_images: val.extract_images,
             max_image_size: val.max_image_size,
+            max_input_size: val.max_input_size,
             capture_svg: val.capture_svg,
             infer_dimensions: val.infer_dimensions,
             max_depth: val.max_depth,
@@ -48972,6 +49145,7 @@ impl From<html_to_markdown_rs::ConversionOptions> for ConversionOptions {
             include_document_structure: val.include_document_structure,
             extract_images: val.extract_images,
             max_image_size: val.max_image_size,
+            max_input_size: val.max_input_size,
             capture_svg: val.capture_svg,
             infer_dimensions: val.infer_dimensions,
             max_depth: val.max_depth,
@@ -49052,6 +49226,7 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
             include_paths: val.include_paths.into_iter().collect(),
             exclude_paths: val.exclude_paths.into_iter().collect(),
             path_patterns_match_query: val.path_patterns_match_query,
+            path_patterns_match_url: val.path_patterns_match_url,
             dedup_include_query: val.dedup_include_query,
             strip_tracking_params: val.strip_tracking_params,
             tracking_params: val.tracking_params.into_iter().collect(),
@@ -49119,6 +49294,7 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
             include_paths: val.include_paths.into_iter().collect(),
             exclude_paths: val.exclude_paths.into_iter().collect(),
             path_patterns_match_query: val.path_patterns_match_query,
+            path_patterns_match_url: val.path_patterns_match_url,
             dedup_include_query: val.dedup_include_query,
             strip_tracking_params: val.strip_tracking_params,
             tracking_params: val.tracking_params.into_iter().collect(),
@@ -53939,6 +54115,7 @@ impl From<SsrfPolicy> for crawlberg::SsrfPolicy {
         Self {
             deny_private: val.deny_private,
             allowlist: val.allowlist.into_iter().map(Into::into).collect(),
+            denylist: val.denylist.into_iter().map(Into::into).collect(),
             max_redirects: val.max_redirects,
             scheme_allowlist: val.scheme_allowlist.into_iter().collect(),
             ..Default::default()
@@ -53952,6 +54129,7 @@ impl From<crawlberg::SsrfPolicy> for SsrfPolicy {
         Self {
             deny_private: val.deny_private,
             allowlist: val.allowlist.into_iter().map(Into::into).collect(),
+            denylist: val.denylist.into_iter().map(Into::into).collect(),
             max_redirects: val.max_redirects,
             scheme_allowlist: val.scheme_allowlist.into_iter().collect(),
         }
@@ -57397,30 +57575,30 @@ impl From<html_to_markdown_rs::WhitespaceMode> for WhitespaceMode {
     }
 }
 
-/// Convert a `xberg::heuristics::error::HeuristicsError` error to a Magnus runtime error.
+/// Convert a `xberg::HeuristicsError` error to a Magnus runtime error.
 #[allow(dead_code)]
-fn heuristics_error_to_magnus_err(e: xberg::heuristics::error::HeuristicsError) -> magnus::Error {
+fn heuristics_error_to_magnus_err(e: xberg::HeuristicsError) -> magnus::Error {
     let msg = e.to_string();
     magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
 }
 
-/// Convert a `xberg::presets::loader::LoadError` error to a Magnus runtime error.
+/// Convert a `xberg::LoadError` error to a Magnus runtime error.
 #[allow(dead_code)]
-fn load_error_to_magnus_err(e: xberg::presets::loader::LoadError) -> magnus::Error {
+fn load_error_to_magnus_err(e: xberg::LoadError) -> magnus::Error {
     let msg = e.to_string();
     magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
 }
 
-/// Convert a `xberg::presets::resolve::ResolveError` error to a Magnus runtime error.
+/// Convert a `xberg::ResolveError` error to a Magnus runtime error.
 #[allow(dead_code)]
-fn resolve_error_to_magnus_err(e: xberg::presets::resolve::ResolveError) -> magnus::Error {
+fn resolve_error_to_magnus_err(e: xberg::ResolveError) -> magnus::Error {
     let msg = e.to_string();
     magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
 }
 
-/// Convert a `xberg::error::XbergError` error to a Magnus runtime error.
+/// Convert a `xberg::XbergError` error to a Magnus runtime error.
 #[allow(dead_code)]
-fn xberg_error_to_magnus_err(e: xberg::error::XbergError) -> magnus::Error {
+fn xberg_error_to_magnus_err(e: xberg::XbergError) -> magnus::Error {
     let msg = e.to_string();
     magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
 }
@@ -57591,6 +57769,10 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("session_affinity", method!(BrowserConfig::session_affinity, 0))?;
 
+    class.define_method("chrome_path", method!(BrowserConfig::chrome_path, 0))?;
+
+    class.define_method("chrome_args", method!(BrowserConfig::chrome_args, 0))?;
+
     let class = module.define_class("CacheStats", ruby.class_object())?;
 
     class.define_singleton_method("new", function!(CacheStats::new, -1))?;
@@ -58027,6 +58209,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("max_image_size", method!(ConversionOptions::max_image_size, 0))?;
 
+    class.define_method("max_input_size", method!(ConversionOptions::max_input_size, 0))?;
+
     class.define_method("capture_svg", method!(ConversionOptions::capture_svg, 0))?;
 
     class.define_method("infer_dimensions", method!(ConversionOptions::infer_dimensions, 0))?;
@@ -58128,6 +58312,11 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
         method!(CrawlConfig::path_patterns_match_query, 0),
     )?;
 
+    class.define_method(
+        "path_patterns_match_url",
+        method!(CrawlConfig::path_patterns_match_url, 0),
+    )?;
+
     class.define_method("dedup_include_query", method!(CrawlConfig::dedup_include_query, 0))?;
 
     class.define_method("strip_tracking_params", method!(CrawlConfig::strip_tracking_params, 0))?;
@@ -61805,6 +61994,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("allowlist", method!(SsrfPolicy::allowlist, 0))?;
 
+    class.define_method("denylist", method!(SsrfPolicy::denylist, 0))?;
+
     class.define_method("max_redirects", method!(SsrfPolicy::max_redirects, 0))?;
 
     class.define_method("scheme_allowlist", method!(SsrfPolicy::scheme_allowlist, 0))?;

data/lib/xberg/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:a73b2b7b23be41b809d7a63c6e84f6b22dc213b4d2d018e2004ef2a2f0e9410e
+# alef:hash:988034bb9172ee9a4ecaa49221da1ceed7f562d9d0ef1192f4a91b54dfed0ed1
 # To regenerate: alef generate
 # To verify freshness: alef verify
 # frozen_string_literal: true
 
 module Xberg
   ## The version string for this package.
-  VERSION = "1.3.4"
+  VERSION = "1.3.6"
 end

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:4d6e82c5b0834d91c2575ca21fe65ed7dc5bb424ab80623cb550f60e8cd268ad
+# alef:hash:d01bd27d5fa9368a18dda14ba966abab79e6f319668303eaa72b202e75e41946
 # To regenerate: alef generate
 # To verify freshness: alef verify
 
@@ -107,8 +107,10 @@ def initialize: (?entry_count: Integer, ?citation_keys: Array[String], ?authors:
         attr_reader robots_user_agent: String?
         attr_reader capture_network_events: bool
         attr_reader session_affinity: bool
+        attr_reader chrome_path: String?
+    attr_reader chrome_args: Array[String]
 
-def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool) -> void
+def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?overall_timeout: Integer, ?shutdown_timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool, ?chrome_path: String, ?chrome_args: Array[String]) -> void
     end
 
     class CacheStats
@@ -362,6 +364,7 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader include_document_structure: bool
         attr_reader extract_images: bool
         attr_reader max_image_size: Integer
+        attr_reader max_input_size: Integer?
         attr_reader capture_svg: bool
         attr_reader infer_dimensions: bool
         attr_reader max_depth: Integer?
@@ -369,7 +372,7 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
         attr_reader tier_strategy: enum_TierStrategy
         attr_reader base_url: String?
 
-def initialize: (?heading_style: enum_HeadingStyle, ?list_indent_type: enum_ListIndentType, ?list_indent_width: Integer, ?bullets: String, ?strong_em_symbol: String, ?escape_asterisks: bool, ?escape_underscores: bool, ?escape_misc: bool, ?escape_ascii: bool, ?code_language: String, ?autolinks: bool, ?default_title: bool, ?br_in_tables: bool, ?compact_tables: bool, ?highlight_style: enum_HighlightStyle, ?extract_metadata: bool, ?whitespace_mode: enum_WhitespaceMode, ?strip_newlines: bool, ?wrap: bool, ?wrap_width: Integer, ?convert_as_inline: bool, ?sub_symbol: String, ?sup_symbol: String, ?newline_style: enum_NewlineStyle, ?code_block_style: enum_CodeBlockStyle, ?keep_inline_images_in: Array[String], ?preprocessing: PreprocessingOptions, ?encoding: String, ?debug: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?skip_images: bool, ?inline_data_media: enum_InlineDataMedia, ?url_escape_style: enum_UrlEscapeStyle, ?link_style: enum_LinkStyle, ?output_format: enum_HtmlToMarkdownRsOutputFormat, ?include_document_structure: bool, ?extract_images: bool, ?max_image_size: Integer, ?capture_svg: bool, ?infer_dimensions: bool, ?max_depth: Integer, ?exclude_selectors: Array[String], ?tier_strategy: enum_TierStrategy, ?base_url: String) -> void
+def initialize: (?heading_style: enum_HeadingStyle, ?list_indent_type: enum_ListIndentType, ?list_indent_width: Integer, ?bullets: String, ?strong_em_symbol: String, ?escape_asterisks: bool, ?escape_underscores: bool, ?escape_misc: bool, ?escape_ascii: bool, ?code_language: String, ?autolinks: bool, ?default_title: bool, ?br_in_tables: bool, ?compact_tables: bool, ?highlight_style: enum_HighlightStyle, ?extract_metadata: bool, ?whitespace_mode: enum_WhitespaceMode, ?strip_newlines: bool, ?wrap: bool, ?wrap_width: Integer, ?convert_as_inline: bool, ?sub_symbol: String, ?sup_symbol: String, ?newline_style: enum_NewlineStyle, ?code_block_style: enum_CodeBlockStyle, ?keep_inline_images_in: Array[String], ?preprocessing: PreprocessingOptions, ?encoding: String, ?debug: bool, ?strip_tags: Array[String], ?preserve_tags: Array[String], ?skip_images: bool, ?inline_data_media: enum_InlineDataMedia, ?url_escape_style: enum_UrlEscapeStyle, ?link_style: enum_LinkStyle, ?output_format: enum_HtmlToMarkdownRsOutputFormat, ?include_document_structure: bool, ?extract_images: bool, ?max_image_size: Integer, ?max_input_size: Integer, ?capture_svg: bool, ?infer_dimensions: bool, ?max_depth: Integer, ?exclude_selectors: Array[String], ?tier_strategy: enum_TierStrategy, ?base_url: String) -> void
     end
 
     class CoreProperties
@@ -409,6 +412,7 @@ def initialize: (?heading_style: enum_HeadingStyle, ?list_indent_type: enum_List
     attr_reader include_paths: Array[String]
     attr_reader exclude_paths: Array[String]
         attr_reader path_patterns_match_query: bool
+        attr_reader path_patterns_match_url: bool
         attr_reader dedup_include_query: bool
         attr_reader strip_tracking_params: bool
     attr_reader tracking_params: Array[String]
@@ -448,7 +452,7 @@ def initialize: (?heading_style: enum_HeadingStyle, ?list_indent_type: enum_List
         attr_reader ssrf: SsrfPolicy
         attr_reader ssrf_deny_private_explicit: bool?
 
-def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
+def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?path_patterns_match_query: bool, ?path_patterns_match_url: bool, ?dedup_include_query: bool, ?strip_tracking_params: bool, ?tracking_params: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?retry_initial_delay_ms: Integer, ?retry_max_delay_ms: Integer, ?rate_limit_jitter_ratio: Float, ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
     end
 
     class CsvConfig
@@ -2444,10 +2448,11 @@ def initialize: (name: String, model_repo: String, model_file: String, additiona
     class SsrfPolicy
         attr_reader deny_private: bool
     attr_reader allowlist: Array[HostMatcher]
+    attr_reader denylist: Array[HostMatcher]
         attr_reader max_redirects: Integer
     attr_reader scheme_allowlist: Array[String]
 
-def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?max_redirects: Integer, ?scheme_allowlist: Array[String]) -> void
+def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist: Array[HostMatcher], ?max_redirects: Integer, ?scheme_allowlist: Array[String]) -> void
     end
 
     class StructuredData

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: xberg
 version: !ruby/object:Gem::Version
-  version: 1.3.4
+  version: 1.3.6
 platform: ruby
 authors:
 - Na'aman Hirschfeld <naaman@xberg.io>
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-10-05 00:00:00.000000000 Z
+date: 2026-10-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys