kreuzcrawl 0.3.0.pre.rc.71 → 0.3.0.pre.rc.76

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f9dc9a2b925bd356635ef537594ccf72049f8bbfa47754f348fe0643443a4c9f
-  data.tar.gz: 06b86af2e1ef230a61a1c9c81d84115e2346e558de94df0d17c08097e63c84e1
+  metadata.gz: 2e042443c6e9e74a3efa5090e5a9007bf95b3a29326d9d9c49a8b5310df1892d
+  data.tar.gz: 33d32333a455f6330dcec4266db79dafb758c0014c99d7d62af0f608befa811f
 SHA512:
-  metadata.gz: 490013779c30677fd6f3618275bb0bd9f11f0da30c43b320e2fda3b814762d18fac68391c9b6e91c9b9f80a5f3cc4ee6a008d48e4897b6dd79c314af085e5d5b
-  data.tar.gz: 6c8b027fc5db3dd04aa20b034a09e80a4041d527e688131a0bdfbc2f87a9390bef47adc14db5e096a195d69ddcc45ad17d68f43dbb91846ce09fba8919d6d5c0
+  metadata.gz: 946f2b86e1bfcf44cee9623c154dd8a0cc3fbec4860e5131fddc96d6aa8a7a4ff38933e2d8172d21ad7fe17dcc60382424100ea3404e314f800a59d740b2eab9
+  data.tar.gz: c860e9ab287b354203960a631d81b0bf5e1786b9bdb1263465598922d2ad9c76c25986747a28df0c37451f0f931666df444256b2d467c57c6f30d58c97d78df1

data/README.md

@@ -141,12 +141,13 @@ Contributions are welcome! Please see our [Contributing Guide](https://github.co
 
 ## Part of Kreuzberg.dev
 
-- [Kreuzberg](https://github.com/kreuzberg-dev/kreuzberg) — document intelligence: text, tables, metadata from 90+ formats with optional OCR.
+- [Kreuzberg](https://github.com/kreuzberg-dev/kreuzberg) — document intelligence: text, tables, metadata from 91+ formats with optional OCR.
 - [Kreuzberg Cloud](https://github.com/kreuzberg-dev/kreuzberg-cloud) — managed extraction API with SDKs, dashboards, and observability.
+- [kreuzcrawl](https://github.com/kreuzberg-dev/kreuzcrawl) — web crawling and scraping with HTML→Markdown and headless-Chrome fallback.
 - [html-to-markdown](https://github.com/kreuzberg-dev/html-to-markdown) — fast, lossless HTML→Markdown engine.
 - [liter-llm](https://github.com/kreuzberg-dev/liter-llm) — universal LLM API client with native bindings for 14 languages and 143 providers.
 - [tree-sitter-language-pack](https://github.com/kreuzberg-dev/tree-sitter-language-pack) — tree-sitter grammars and code-intelligence primitives.
-- [alef](https://github.com/kreuzberg-dev/alef) — the polyglot binding generator that produces this README and all per-language bindings.
+- [alef](https://github.com/kreuzberg-dev/alef) — the polyglot binding generator that produces every per-language binding across the 5 polyglot repos.
 - [Discord](https://discord.gg/xt9WY3GnKR) — community, roadmap, announcements.
 
 ## License
@@ -158,4 +159,3 @@ This project is licensed under [Elastic License 2.0](https://github.com/kreuzber
 - [Documentation](https://docs.kreuzcrawl.kreuzberg.dev)
 - [GitHub Repository](https://github.com/kreuzberg-dev/kreuzcrawl)
 - [Issue Tracker](https://github.com/kreuzberg-dev/kreuzcrawl/issues)
-- [Issues](https://github.com/kreuzberg-dev/kreuzcrawl/issues)

data/ext/kreuzcrawl_rb/native/Cargo.lock

@@ -1460,9 +1460,9 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl"
-version = "0.3.0-rc.71"
+version = "0.3.0-rc.76"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8c85e79454c048ec41289dfb0168e87ff1c5a0a28997de8cc0dbd3ab331c8e2"
+checksum = "b620f0f12386898f22a5e8e2ab8ba1066d3a304c3b6e8847d5ee8e51cd068de1"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -1502,7 +1502,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl-rb"
-version = "0.3.0-rc.71"
+version = "0.3.0-rc.76"
 dependencies = [
  "futures",
  "kreuzcrawl",

data/ext/kreuzcrawl_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzcrawl-rb"
-version = "0.3.0-rc.71"
+version = "0.3.0-rc.76"
 edition = "2024"
 license = "Elastic-2.0"
 description = "High-performance web crawling engine"
@@ -18,7 +18,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 futures = "0.3"
-kreuzcrawl = { version = "0.3.0-rc.71", features = ["interact", "browser-chromiumoxide"] }
+kreuzcrawl = { version = "0.3.0-rc.76", features = ["interact", "browser-chromiumoxide"] }
 magnus = "0.8"
 rb-sys = ">=0.9, <0.9.128"
 serde = { version = "1", features = ["derive"] }

data/ext/kreuzcrawl_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:f2acf0fe68bedde7da1909fa6feedaffb995d5aec49220ec23bf64125a392d19
+// alef:hash:8896b7e192e408f916c81acf04c97fc5ee10b269fe22c287e9eefab02fada615
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -608,6 +608,7 @@ pub struct CrawlConfig {
     warc_output: Option<String>,
     browser_profile: Option<String>,
     save_browser_profile: bool,
+    ssrf: SsrfPolicy,
 }
 
 unsafe impl IntoValueFromNative for CrawlConfig {}
@@ -679,6 +680,7 @@ impl Default for CrawlConfig {
             warc_output: None,
             browser_profile: None,
             save_browser_profile: false,
+            ssrf: Default::default(),
         }
     }
 }
@@ -827,6 +829,10 @@ impl CrawlConfig {
                 .get(ruby.to_symbol("save_browser_profile"))
                 .and_then(|v| bool::try_convert(v).ok())
                 .unwrap_or(false),
+            ssrf: kwargs
+                .get(ruby.to_symbol("ssrf"))
+                .and_then(|v| SsrfPolicy::try_convert(v).ok())
+                .unwrap_or_default(),
         })
     }
 
@@ -982,6 +988,10 @@ impl CrawlConfig {
         self.save_browser_profile
     }
 
+    fn ssrf(&self) -> SsrfPolicy {
+        self.ssrf.clone()
+    }
+
     fn validate(&self) -> Result<(), Error> {
         #[allow(clippy::needless_update)]
         let core_self = kreuzcrawl::CrawlConfig {
@@ -1066,8 +1076,8 @@ impl CrawlConfig {
 
             save_browser_profile: self.save_browser_profile,
 
-            ssrf: Default::default(),
-            dispatch: Default::default(),
+            ssrf: self.ssrf.clone().into(),
+
             ..Default::default()
         };
         let result = core_self.validate().map_err(|e| {
@@ -2101,7 +2111,7 @@ impl CrawlResult {
 
             browser_used: self.browser_used,
 
-            normalized_urls: Default::default(),
+            ..Default::default()
         };
         core_self.unique_normalized_urls()
     }
@@ -4419,6 +4429,75 @@ impl BatchCrawlResults {
     }
 }
 
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(default)]
+#[magnus::wrap(class = "Kreuzcrawl::SsrfPolicy")]
+pub struct SsrfPolicy {
+    deny_private: bool,
+    max_redirects: u8,
+}
+
+unsafe impl IntoValueFromNative for SsrfPolicy {}
+
+impl magnus::TryConvert for SsrfPolicy {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        if let Ok(r) = <&SsrfPolicy as magnus::TryConvert>::try_convert(val) {
+            return Ok(r.clone());
+        }
+        let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
+            s
+        } else {
+            val.funcall::<_, _, String>("to_json", ()).map_err(|e| {
+                magnus::Error::new(
+                    unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                    format!("no implicit conversion into SsrfPolicy: {}", e),
+                )
+            })?
+        };
+        serde_json::from_str::<SsrfPolicy>(&json_str).map_err(|e| {
+            magnus::Error::new(
+                unsafe { magnus::Ruby::get_unchecked() }.exception_type_error(),
+                format!("failed to deserialize SsrfPolicy: {}", e),
+            )
+        })
+    }
+}
+
+unsafe impl TryConvertOwned for SsrfPolicy {}
+
+impl Default for SsrfPolicy {
+    fn default() -> Self {
+        kreuzcrawl::SsrfPolicy::default().into()
+    }
+}
+
+impl SsrfPolicy {
+    fn new(args: &[magnus::Value]) -> Result<Self, magnus::Error> {
+        let ruby = unsafe { magnus::Ruby::get_unchecked() };
+        let args = magnus::scan_args::scan_args::<(), (Option<magnus::RHash>,), (), (), (), ()>(args)?;
+        let (kwargs_opt,) = args.optional;
+        let kwargs = kwargs_opt.unwrap_or_else(|| ruby.hash_new());
+        Ok(Self {
+            deny_private: kwargs
+                .get(ruby.to_symbol("deny_private"))
+                .and_then(|v| bool::try_convert(v).ok())
+                .unwrap_or(true),
+            max_redirects: kwargs
+                .get(ruby.to_symbol("max_redirects"))
+                .and_then(|v| u8::try_convert(v).ok())
+                .unwrap_or(5),
+        })
+    }
+
+    fn deny_private(&self) -> bool {
+        self.deny_private
+    }
+
+    fn max_redirects(&self) -> u8 {
+        self.max_redirects
+    }
+}
+
 #[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(rename_all = "snake_case")]
 pub enum BrowserMode {
@@ -5446,8 +5525,7 @@ impl From<CrawlConfig> for kreuzcrawl::CrawlConfig {
             warc_output: val.warc_output.map(Into::into),
             browser_profile: val.browser_profile,
             save_browser_profile: val.save_browser_profile,
-            ssrf: Default::default(),
-            dispatch: Default::default(),
+            ssrf: val.ssrf.into(),
             ..Default::default()
         }
     }
@@ -5499,6 +5577,7 @@ impl From<kreuzcrawl::CrawlConfig> for CrawlConfig {
             warc_output: val.warc_output.map(|p| p.to_string_lossy().to_string()),
             browser_profile: val.browser_profile.map(|v| v.to_string()),
             save_browser_profile: val.save_browser_profile,
+            ssrf: val.ssrf.into(),
         }
     }
 }
@@ -5525,17 +5604,18 @@ impl From<kreuzcrawl::BrowserExtras> for BrowserExtras {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<DownloadedDocument> for kreuzcrawl::DownloadedDocument {
     fn from(val: DownloadedDocument) -> Self {
         Self {
             url: val.url,
             mime_type: val.mime_type.into(),
-            content: Default::default(),
             size: val.size,
             filename: val.filename.map(Into::into),
             content_hash: val.content_hash.into(),
             headers: val.headers.into_iter().map(|(k, v)| (k.into(), v.into())).collect(),
+            ..Default::default()
         }
     }
 }
@@ -5554,6 +5634,7 @@ impl From<kreuzcrawl::DownloadedDocument> for DownloadedDocument {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<InteractionResult> for kreuzcrawl::InteractionResult {
     fn from(val: InteractionResult) -> Self {
@@ -5561,7 +5642,7 @@ impl From<InteractionResult> for kreuzcrawl::InteractionResult {
             action_results: val.action_results.into_iter().map(Into::into).collect(),
             final_html: val.final_html,
             final_url: val.final_url,
-            screenshot: Default::default(),
+            ..Default::default()
         }
     }
 }
@@ -5603,6 +5684,7 @@ impl From<kreuzcrawl::ActionResult> for ActionResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ScrapeResult> for kreuzcrawl::ScrapeResult {
     fn from(val: ScrapeResult) -> Self {
@@ -5633,9 +5715,9 @@ impl From<ScrapeResult> for kreuzcrawl::ScrapeResult {
             markdown: val.markdown.map(Into::into),
             extracted_data: val.extracted_data.as_ref().and_then(|s| serde_json::from_str(s).ok()),
             extraction_meta: val.extraction_meta.map(Into::into),
-            screenshot: Default::default(),
             downloaded_document: val.downloaded_document.map(Into::into),
             browser: val.browser.map(Into::into),
+            ..Default::default()
         }
     }
 }
@@ -5734,6 +5816,7 @@ impl From<kreuzcrawl::CrawlPageResult> for CrawlPageResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CrawlResult> for kreuzcrawl::CrawlResult {
     fn from(val: CrawlResult) -> Self {
@@ -5746,7 +5829,7 @@ impl From<CrawlResult> for kreuzcrawl::CrawlResult {
             cookies: val.cookies.into_iter().map(Into::into).collect(),
             stayed_on_domain: val.stayed_on_domain,
             browser_used: val.browser_used,
-            normalized_urls: Default::default(),
+            ..Default::default()
         }
     }
 }
@@ -6382,6 +6465,28 @@ impl From<kreuzcrawl::BatchCrawlResults> for BatchCrawlResults {
     }
 }
 
+#[allow(clippy::needless_update)]
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+impl From<SsrfPolicy> for kreuzcrawl::SsrfPolicy {
+    fn from(val: SsrfPolicy) -> Self {
+        Self {
+            deny_private: val.deny_private,
+            max_redirects: val.max_redirects,
+            ..Default::default()
+        }
+    }
+}
+
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+impl From<kreuzcrawl::SsrfPolicy> for SsrfPolicy {
+    fn from(val: kreuzcrawl::SsrfPolicy) -> Self {
+        Self {
+            deny_private: val.deny_private,
+            max_redirects: val.max_redirects,
+        }
+    }
+}
+
 impl From<BrowserMode> for kreuzcrawl::BrowserMode {
     fn from(val: BrowserMode) -> Self {
         match val {
@@ -6664,6 +6769,13 @@ fn crawl_error_to_magnus_err(e: kreuzcrawl::CrawlError) -> magnus::Error {
     magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
 }
 
+/// Convert a `kreuzcrawl::SsrfError` error to a Magnus runtime error.
+#[allow(dead_code)]
+fn ssrf_error_to_magnus_err(e: kreuzcrawl::SsrfError) -> magnus::Error {
+    let msg = e.to_string();
+    magnus::Error::new(unsafe { magnus::Ruby::get_unchecked() }.exception_runtime_error(), msg)
+}
+
 #[derive(Clone)]
 #[magnus::wrap(class = "Kreuzcrawl::CrawlStreamIterator")]
 pub struct CrawlStreamIterator {
@@ -7009,6 +7121,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("save_browser_profile", method!(CrawlConfig::save_browser_profile, 0))?;
 
+    class.define_method("ssrf", method!(CrawlConfig::ssrf, 0))?;
+
     class.define_method("validate", method!(CrawlConfig::validate, 0))?;
 
     let class = module.define_class("BrowserExtras", ruby.class_object())?;
@@ -7535,6 +7649,14 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("failed_count", method!(BatchCrawlResults::failed_count, 0))?;
 
+    let class = module.define_class("SsrfPolicy", ruby.class_object())?;
+
+    class.define_singleton_method("new", function!(SsrfPolicy::new, -1))?;
+
+    class.define_method("deny_private", method!(SsrfPolicy::deny_private, 0))?;
+
+    class.define_method("max_redirects", method!(SsrfPolicy::max_redirects, 0))?;
+
     let class = module.define_class("CrawlStreamIterator", ruby.class_object())?;
     class.define_method("next_chunk", method!(CrawlStreamIterator::next_chunk, 0))?;
     class.define_method("each", method!(CrawlStreamIterator::each, 0))?;

data/lib/kreuzcrawl/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:f2acf0fe68bedde7da1909fa6feedaffb995d5aec49220ec23bf64125a392d19
+# alef:hash:8896b7e192e408f916c81acf04c97fc5ee10b269fe22c287e9eefab02fada615
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
@@ -106,10 +106,7 @@ module Kreuzcrawl
   # Not available on `wasm32` targets — streaming requires native concurrency
   # primitives (tokio channels, `JoinSet`) that are not supported on wasm32.
   #
-  # Delivered to bindings via alef's streaming-adapter pattern. The
-  # `crawl_stream` / `batch_crawl_stream` binding wrappers in `bindings.rs`
-  # expose this as the per-language streaming idiom (Python `AsyncIterator`,
-  # Ruby `Enumerator`, PHP `Generator`, Elixir `Stream.unfold`, etc.).
+  # Delivered to bindings through each target's native streaming idiom.
   module CrawlEvent
     extend T::Helpers
     extend T::Sig

data/lib/kreuzcrawl/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:f2acf0fe68bedde7da1909fa6feedaffb995d5aec49220ec23bf64125a392d19
+# alef:hash:8896b7e192e408f916c81acf04c97fc5ee10b269fe22c287e9eefab02fada615
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
 
 module Kreuzcrawl
   ## The version string for this package.
-  VERSION = "0.3.0.pre.rc.71"
+  VERSION = "0.3.0.pre.rc.76"
 end

data/lib/kreuzcrawl.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:f2acf0fe68bedde7da1909fa6feedaffb995d5aec49220ec23bf64125a392d19
+# alef:hash:8896b7e192e408f916c81acf04c97fc5ee10b269fe22c287e9eefab02fada615
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:f2acf0fe68bedde7da1909fa6feedaffb995d5aec49220ec23bf64125a392d19
+# alef:hash:8896b7e192e408f916c81acf04c97fc5ee10b269fe22c287e9eefab02fada615
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 
@@ -103,8 +103,9 @@ module Kreuzcrawl
     attr_accessor warc_output: String?
     attr_accessor browser_profile: String?
     attr_accessor save_browser_profile: bool?
+    attr_accessor ssrf: SsrfPolicy?
 
-    def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_concurrent: Integer, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool) -> void
+    def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_concurrent: Integer, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy) -> void
     def validate: () -> void
     def self.default: () -> CrawlConfig
   end
@@ -460,6 +461,15 @@ module Kreuzcrawl
     def initialize: (?results: Array[BatchCrawlResult], ?total_count: Integer, ?completed_count: Integer, ?failed_count: Integer) -> void
   end
 
+  class SsrfPolicy
+    attr_accessor deny_private: bool?
+    attr_accessor max_redirects: Integer?
+
+    def initialize: (?deny_private: bool, ?max_redirects: Integer) -> void
+    def self.default: () -> SsrfPolicy
+    def self.from_env: () -> SsrfPolicy
+  end
+
   class BrowserMode
     type value = :auto | :always | :never | :stealth
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kreuzcrawl
 version: !ruby/object:Gem::Version
-  version: 0.3.0.pre.rc.71
+  version: 0.3.0.pre.rc.76
 platform: ruby
 authors:
 - Kreuzberg Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-06-16 00:00:00.000000000 Z
+date: 2026-06-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys