kreuzcrawl 0.3.0.pre.rc.72 → 0.3.0.pre.rc.76

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8edea9a39a5cbe88bec892ae7fc8de37f0a4b4d5dc3430f5a6e4c0ccb475e443
-  data.tar.gz: 3dd64e9b4d361cc4d3ec82178260aaa018b0ca1d1d2f76034cb20ed05c23cefd
+  metadata.gz: 2e042443c6e9e74a3efa5090e5a9007bf95b3a29326d9d9c49a8b5310df1892d
+  data.tar.gz: 33d32333a455f6330dcec4266db79dafb758c0014c99d7d62af0f608befa811f
 SHA512:
-  metadata.gz: 56ac6e332c9e4cba97c1adb271a6e36dd1fb0c8eb1b9549950d7048c208e9ecf2f9bdb8529dd326d2b5bece9fa2d56f961e8199b2c5667167ee5965ffa3f7596
-  data.tar.gz: b84490766d8ade6a489ba72ee7711c25b6e16e75861073f9b3c4346c8d0da4ecdca2321c9f590860070260ab9ae144429e4837ae9fced893b0983feee3718dd0
+  metadata.gz: 946f2b86e1bfcf44cee9623c154dd8a0cc3fbec4860e5131fddc96d6aa8a7a4ff38933e2d8172d21ad7fe17dcc60382424100ea3404e314f800a59d740b2eab9
+  data.tar.gz: c860e9ab287b354203960a631d81b0bf5e1786b9bdb1263465598922d2ad9c76c25986747a28df0c37451f0f931666df444256b2d467c57c6f30d58c97d78df1

data/ext/kreuzcrawl_rb/native/Cargo.lock

@@ -1460,9 +1460,9 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl"
-version = "0.3.0-rc.72"
+version = "0.3.0-rc.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8a83f0115e5fce77f5f6a4c53d572b8c151038c0fcd16c641db2e8b499603582"
+checksum = "b620f0f12386898f22a5e8e2ab8ba1066d3a304c3b6e8847d5ee8e51cd068de1"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -1502,7 +1502,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl-rb"
-version = "0.3.0-rc.72"
+version = "0.3.0-rc.76"
 dependencies = [
  "futures",
  "kreuzcrawl",

data/ext/kreuzcrawl_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzcrawl-rb"
-version = "0.3.0-rc.72"
+version = "0.3.0-rc.76"
 edition = "2024"
 license = "Elastic-2.0"
 description = "High-performance web crawling engine"
@@ -18,7 +18,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 futures = "0.3"
-kreuzcrawl = { version = "0.3.0-rc.72", features = ["interact", "browser-chromiumoxide"] }
+kreuzcrawl = { version = "0.3.0-rc.76", features = ["interact", "browser-chromiumoxide"] }
 magnus = "0.8"
 rb-sys = ">=0.9, <0.9.128"
 serde = { version = "1", features = ["derive"] }

data/ext/kreuzcrawl_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:bcc3d5bd544a6e7593876ab5de51eb8c1f53d0d8d2c8c3d31c436d54ead6b3ec
+// alef:hash:8896b7e192e408f916c81acf04c97fc5ee10b269fe22c287e9eefab02fada615
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -608,6 +608,7 @@ pub struct CrawlConfig {
     warc_output: Option<String>,
     browser_profile: Option<String>,
     save_browser_profile: bool,
+    ssrf: SsrfPolicy,
 }
 
 unsafe impl IntoValueFromNative for CrawlConfig {}
@@ -679,6 +680,7 @@ impl Default for CrawlConfig {
             warc_output: None,
             browser_profile: None,
             save_browser_profile: false,
+            ssrf: Default::default(),
         }
     }
 }
@@ -827,6 +829,10 @@ impl CrawlConfig {
                 .get(ruby.to_symbol("save_browser_profile"))
                 .and_then(|v| bool::try_convert(v).ok())
                 .unwrap_or(false),
+            ssrf: kwargs
+                .get(ruby.to_symbol("ssrf"))
+                .and_then(|v| SsrfPolicy::try_convert(v).ok())
+                .unwrap_or_default(),
         })
     }
 
@@ -982,6 +988,10 @@ impl CrawlConfig {
         self.save_browser_profile
     }
 
+    fn ssrf(&self) -> SsrfPolicy {
+        self.ssrf.clone()
+    }
+
     fn validate(&self) -> Result<(), Error> {
         #[allow(clippy::needless_update)]
         let core_self = kreuzcrawl::CrawlConfig {
@@ -1066,8 +1076,8 @@ impl CrawlConfig {
 
             save_browser_profile: self.save_browser_profile,
 
-            ssrf: Default::default(),
-            dispatch: Default::default(),
+            ssrf: self.ssrf.clone().into(),
+
             ..Default::default()
         };
         let result = core_self.validate().map_err(|e| {
@@ -2101,7 +2111,7 @@ impl CrawlResult {
 
             browser_used: self.browser_used,
 
-            normalized_urls: Default::default(),
+            ..Default::default()
         };
         core_self.unique_normalized_urls()
     }
@@ -4419,6 +4429,75 @@ impl BatchCrawlResults {
     }
 }
 
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(default)]
+#[magnus::wrap(class = "Kreuzcrawl::SsrfPolicy")]
+pub struct SsrfPolicy {
+    deny_private: bool,
+    max_redirects: u8,
+}
+
+unsafe impl IntoValueFromNative for SsrfPolicy {}
+
+impl magnus::TryConvert for SsrfPolicy {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        if let Ok(r) = <&SsrfPolicy as magnus::TryConvert>::try_convert(val) {
+            return Ok(r.clone());
+        }
+        let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
+            s
+        } else {
+            val.funcall::<_, _, String>("to_json", ()).map_err(|e| {
+                magnus::Error::new(
+                    unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                    format!("no implicit conversion into SsrfPolicy: {}", e),
+                )
+            })?
+        };
+        serde_json::from_str::<SsrfPolicy>(&json_str).map_err(|e| {
+            magnus::Error::new(
+                unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                format!("failed to deserialize SsrfPolicy: {}", e),
+            )
+        })
+    }
+}
+
+unsafe impl TryConvertOwned for SsrfPolicy {}
+
+impl Default for SsrfPolicy {
+    fn default() -> Self {
+        kreuzcrawl::SsrfPolicy::default().into()
+    }
+}
+
+impl SsrfPolicy {
+    fn new(args: &[magnus::Value]) -> Result<Self, magnus::Error> {
+        let ruby = unsafe { magnus::Ruby::get_unchecked() };
+        let args = magnus::scan_args::scan_args::<(), (Option<magnus::RHash>,), (), (), (), ()>(args)?;
+        let (kwargs_opt,) = args.optional;
+        let kwargs = kwargs_opt.unwrap_or_else(|| ruby.hash_new());
+        Ok(Self {
+            deny_private: kwargs
+                .get(ruby.to_symbol("deny_private"))
+                .and_then(|v| bool::try_convert(v).ok())
+                .unwrap_or(true),
+            max_redirects: kwargs
+                .get(ruby.to_symbol("max_redirects"))
+                .and_then(|v| u8::try_convert(v).ok())
+                .unwrap_or(5),
+        })
+    }
+
+    fn deny_private(&self) -> bool {
+        self.deny_private
+    }
+
+    fn max_redirects(&self) -> u8 {
+        self.max_redirects
+    }
+}
+
 #[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum BrowserMode {
@@ -5446,6 +5525,7 @@ impl From<CrawlConfig> for kreuzcrawl::CrawlConfig {
             warc_output: val.warc_output.map(Into::into),
             browser_profile: val.browser_profile,
             save_browser_profile: val.save_browser_profile,
+            ssrf: val.ssrf.into(),
             ..Default::default()
         }
     }
@@ -5497,6 +5577,7 @@ impl From<kreuzcrawl::CrawlConfig> for CrawlConfig {
             warc_output: val.warc_output.map(|p| p.to_string_lossy().to_string()),
             browser_profile: val.browser_profile.map(|v| v.to_string()),
             save_browser_profile: val.save_browser_profile,
+            ssrf: val.ssrf.into(),
         }
     }
 }
@@ -6384,6 +6465,28 @@ impl From<kreuzcrawl::BatchCrawlResults> for BatchCrawlResults {
     }
 }
 
+#[allow(clippy::needless_update)]
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+impl From<SsrfPolicy> for kreuzcrawl::SsrfPolicy {
+    fn from(val: SsrfPolicy) -> Self {
+        Self {
+            deny_private: val.deny_private,
+            max_redirects: val.max_redirects,
+            ..Default::default()
+        }
+    }
+}
+
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+impl From<kreuzcrawl::SsrfPolicy> for SsrfPolicy {
+    fn from(val: kreuzcrawl::SsrfPolicy) -> Self {
+        Self {
+            deny_private: val.deny_private,
+            max_redirects: val.max_redirects,
+        }
+    }
+}
+
 impl From<BrowserMode> for kreuzcrawl::BrowserMode {
     fn from(val: BrowserMode) -> Self {
         match val {
@@ -6666,6 +6769,13 @@ fn crawl_error_to_magnus_err(e: kreuzcrawl::CrawlError) -> magnus::Error {
     magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
 }
 
+/// Convert a `kreuzcrawl::SsrfError` error to a Magnus runtime error.
+#[allow(dead_code)]
+fn ssrf_error_to_magnus_err(e: kreuzcrawl::SsrfError) -> magnus::Error {
+    let msg = e.to_string();
+    magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
+}
+
 #[derive(Clone)]
 #[magnus::wrap(class = "Kreuzcrawl::CrawlStreamIterator")]
 pub struct CrawlStreamIterator {
@@ -7011,6 +7121,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("save_browser_profile", method!(CrawlConfig::save_browser_profile, 0))?;
 
+    class.define_method("ssrf", method!(CrawlConfig::ssrf, 0))?;
+
     class.define_method("validate", method!(CrawlConfig::validate, 0))?;
 
     let class = module.define_class("BrowserExtras", ruby.class_object())?;
@@ -7537,6 +7649,14 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("failed_count", method!(BatchCrawlResults::failed_count, 0))?;
 
+    let class = module.define_class("SsrfPolicy", ruby.class_object())?;
+
+    class.define_singleton_method("new", function!(SsrfPolicy::new, -1))?;
+
+    class.define_method("deny_private", method!(SsrfPolicy::deny_private, 0))?;
+
+    class.define_method("max_redirects", method!(SsrfPolicy::max_redirects, 0))?;
+
     let class = module.define_class("CrawlStreamIterator", ruby.class_object())?;
     class.define_method("next_chunk", method!(CrawlStreamIterator::next_chunk, 0))?;
     class.define_method("each", method!(CrawlStreamIterator::each, 0))?;

data/lib/kreuzcrawl/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:bcc3d5bd544a6e7593876ab5de51eb8c1f53d0d8d2c8c3d31c436d54ead6b3ec
+# alef:hash:8896b7e192e408f916c81acf04c97fc5ee10b269fe22c287e9eefab02fada615
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true

data/lib/kreuzcrawl/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:bcc3d5bd544a6e7593876ab5de51eb8c1f53d0d8d2c8c3d31c436d54ead6b3ec
+# alef:hash:8896b7e192e408f916c81acf04c97fc5ee10b269fe22c287e9eefab02fada615
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
 
 module Kreuzcrawl
   ## The version string for this package.
-  VERSION = "0.3.0.pre.rc.72"
+  VERSION = "0.3.0.pre.rc.76"
 end

data/lib/kreuzcrawl.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:bcc3d5bd544a6e7593876ab5de51eb8c1f53d0d8d2c8c3d31c436d54ead6b3ec
+# alef:hash:8896b7e192e408f916c81acf04c97fc5ee10b269fe22c287e9eefab02fada615
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:bcc3d5bd544a6e7593876ab5de51eb8c1f53d0d8d2c8c3d31c436d54ead6b3ec
+# alef:hash:8896b7e192e408f916c81acf04c97fc5ee10b269fe22c287e9eefab02fada615
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 
@@ -103,8 +103,9 @@ module Kreuzcrawl
     attr_accessor warc_output: String?
     attr_accessor browser_profile: String?
     attr_accessor save_browser_profile: bool?
+    attr_accessor ssrf: SsrfPolicy?
 
-    def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_concurrent: Integer, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool) -> void
+    def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_concurrent: Integer, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy) -> void
     def validate: () -> void
     def self.default: () -> CrawlConfig
   end
@@ -460,6 +461,15 @@ module Kreuzcrawl
     def initialize: (?results: Array[BatchCrawlResult], ?total_count: Integer, ?completed_count: Integer, ?failed_count: Integer) -> void
   end
 
+  class SsrfPolicy
+    attr_accessor deny_private: bool?
+    attr_accessor max_redirects: Integer?
+
+    def initialize: (?deny_private: bool, ?max_redirects: Integer) -> void
+    def self.default: () -> SsrfPolicy
+    def self.from_env: () -> SsrfPolicy
+  end
+
   class BrowserMode
     type value = :auto | :always | :never | :stealth
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kreuzcrawl
 version: !ruby/object:Gem::Version
-  version: 0.3.0.pre.rc.72
+  version: 0.3.0.pre.rc.76
 platform: ruby
 authors:
 - Kreuzberg Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-06-16 00:00:00.000000000 Z
+date: 2026-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys