kreuzcrawl 0.3.0.pre.rc.55 → 0.3.0.pre.rc.59

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fa116f68f2098dce4abbe8ff92f44b9cff4b805c002d3cc8b8878c0124bb63c7
-  data.tar.gz: 05e13d15d8bdb411f82e08645e56b25fe50f2aeb03e7ca47556daf8a838f6079
+  metadata.gz: 119450f2df01a74e99d218b681740eb7367fccbfaea5754437df4f266b765a2e
+  data.tar.gz: ec3bd96a79e88e273c3c75a19e1e1630a46b7dde9f205455e7b64cf642a5e3ad
 SHA512:
-  metadata.gz: f759525a4253d622f2f52851aae6349e56d8e9c0edd6573aafa3b89551b582f57983a7fbe153ad61689584372aeeb396fd7a982ff7d84435021002ee6bac1e3f
-  data.tar.gz: '014803027bbf92c00ec72c96df1a587244752ebc6579a5508781baca9441ce521918547fe8a956fed933f072fbf4b4e245d20d3d4e6f144bc9513c7612748dcc'
+  metadata.gz: 473048d12a1cde342c72722cdd730efa94d95ab48583261f869cb68ab913805d44f821424d8ac3ede99f132400f6d79ba79d72ed9002faa454b1d17b3692d712
+  data.tar.gz: '0109b1c7a3f3430d1396127924b17937de44c7d9a699e3c6bd61acc2a27d4169510fe7fd027f6726eff3016a976c5d64dcd61b065d70915efae18cc55dd9f5e1'

data/ext/kreuzcrawl_rb/native/Cargo.lock

@@ -280,9 +280,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
 
 [[package]]
 name = "cc"
-version = "1.2.63"
+version = "1.2.64"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f"
+checksum = "dad887fd958be91b5098c0248def011f4523ab786cd411be668777e55063501f"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1013,19 +1013,21 @@ dependencies = [
 
 [[package]]
 name = "html-to-markdown-rs"
-version = "3.5.7"
+version = "3.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6cbbfb183e8634cb956309c6bbd781d9ddae068d376fb9eb1451ac49cf4cbba7"
+checksum = "567f72be80982d7d1c9d46730506847e8782e62ff95bf8d3ede3489b3a8e963a"
 dependencies = [
  "ahash",
  "astral-tl",
  "base64",
+ "bitflags",
  "html-escape",
  "html5ever",
  "image",
  "lru",
  "memchr",
  "once_cell",
+ "phf",
  "regex",
  "serde",
  "serde_json",
@@ -1426,9 +1428,9 @@ dependencies = [
 
 [[package]]
 name = "js-sys"
-version = "0.3.100"
+version = "0.3.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f2025f20d7a4fa7785846e7b63d10a76d3f1cee98ee5cb79ea59703f95e42162"
+checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31"
 dependencies = [
  "cfg-if",
  "futures-util",
@@ -1457,9 +1459,9 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl"
-version = "0.3.0-rc.55"
+version = "0.3.0-rc.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d82133a5044ee918cd9f2c3523bf2cae2f9efedb4503039bfbb4f583f509969"
+checksum = "b7e01bcf1ea739b309c26f952a9ee4981d9080e9915863dc92ba3803df265c0e"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -1480,6 +1482,7 @@ dependencies = [
  "memchr",
  "notify",
  "opentelemetry",
+ "opentelemetry-semantic-conventions",
  "quick-xml",
  "regex",
  "reqwest",
@@ -1491,12 +1494,13 @@ dependencies = [
  "tokio-stream",
  "toml",
  "tower",
+ "tracing",
  "url",
 ]
 
 [[package]]
 name = "kreuzcrawl-rb"
-version = "0.3.0-rc.55"
+version = "0.3.0-rc.59"
 dependencies = [
  "futures",
  "kreuzcrawl",
@@ -1754,6 +1758,12 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "opentelemetry-semantic-conventions"
+version = "0.32.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6ca2f98a0437b427b4b08f19f1caa3c44db885a202bc12cfea13d6c702243d68"
+
 [[package]]
 name = "option-ext"
 version = "0.2.0"
@@ -1795,6 +1805,7 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
 dependencies = [
+ "phf_macros",
  "phf_shared",
  "serde",
 ]
@@ -1819,6 +1830,19 @@ dependencies = [
  "phf_shared",
 ]
 
+[[package]]
+name = "phf_macros"
+version = "0.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
+dependencies = [
+ "phf_generator",
+ "phf_shared",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "phf_shared"
 version = "0.13.1"
@@ -2955,9 +2979,9 @@ checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
 
 [[package]]
 name = "wasip2"
-version = "1.0.3+wasi-0.2.9"
+version = "1.0.4+wasi-0.2.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "20064672db26d7cdc89c7798c48a0fdfac8213434a1186e5ef29fd560ae223d6"
+checksum = "b67efb37e106e55ce722a510d6b5f9c17f083e5fc79afc2badeb12cc313d9487"
 dependencies = [
  "wit-bindgen 0.57.1",
 ]
@@ -2973,9 +2997,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen"
-version = "0.2.123"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a254a4b10c19a76f09a27640e7ffbf9bc30bf67e16a3bf28aaefa4920fe81563"
+checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a"
 dependencies = [
  "cfg-if",
  "once_cell",
@@ -2986,9 +3010,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.73"
+version = "0.4.75"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "54568702fabf5d4849ce2b90fadfa64168a097eaf4b351ce9df8b687a0086aaf"
+checksum = "503b14d284f2c8dac03b819967e155ea753f573586193b2b2c95990cb5d69280"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -2996,9 +3020,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.123"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "24a40fc75b0ec6f3746ceb10d36f53a93dcd68a93b11b6445983945d79eba0dc"
+checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d"
 dependencies = [
  "quote",
  "wasm-bindgen-macro-support",
@@ -3006,9 +3030,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.123"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "908f34bd9b9ce3d4caf07b72dfab63d61504d156856c6bd3cd87fa350cf3985b"
+checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd"
 dependencies = [
  "bumpalo",
  "proc-macro2",
@@ -3019,9 +3043,9 @@ dependencies = [
 
 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.123"
+version = "0.2.125"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7acbf7616c27b194bbb550bf77ed0c2c3e5b7fd1260a93082b95fb7f47959b92"
+checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f"
 dependencies = [
  "unicode-ident",
 ]
@@ -3062,9 +3086,9 @@ dependencies = [
 
 [[package]]
 name = "web-sys"
-version = "0.3.100"
+version = "0.3.102"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6e0871acf327f283dc6da28a1696cdc64fb355ba9f935d052021fa77f35cce69"
+checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d"
 dependencies = [
  "js-sys",
  "wasm-bindgen",
@@ -3523,9 +3547,9 @@ dependencies = [
 
 [[package]]
 name = "zeroize"
-version = "1.8.2"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
+checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e"
 
 [[package]]
 name = "zerotrie"

data/ext/kreuzcrawl_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzcrawl-rb"
-version = "0.3.0-rc.55"
+version = "0.3.0-rc.59"
 edition = "2024"
 license = "Elastic-2.0"
 description = "High-performance web crawling engine"
@@ -18,7 +18,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 futures = "0.3"
-kreuzcrawl = { version = "0.3.0-rc.55", features = ["interact", "browser-chromiumoxide"] }
+kreuzcrawl = { version = "0.3.0-rc.59", features = ["interact", "browser-chromiumoxide"] }
 magnus = "0.8"
 rb-sys = ">=0.9, <0.9.128"
 serde = { version = "1", features = ["derive"] }

data/ext/kreuzcrawl_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:882a6f513499cb08d9cbeb641424fe238f26f3653e161241824c4ea057a44c4f
+// alef:hash:4a1130eaa7f8e3312e90b66f3eb0a52e657e6378ee013ed7eeaada8817434913
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -400,7 +400,6 @@ pub struct BrowserConfig {
     wait: BrowserWait,
     wait_selector: Option<String>,
     extra_wait: Option<u64>,
-    stealth: bool,
     proxy: Option<ProxyConfig>,
     block_url_patterns: Vec<String>,
     eval_script: Option<String>,
@@ -447,7 +446,6 @@ impl Default for BrowserConfig {
             wait: Default::default(),
             wait_selector: None,
             extra_wait: None,
-            stealth: false,
             proxy: None,
             block_url_patterns: vec![],
             eval_script: None,
@@ -490,10 +488,6 @@ impl BrowserConfig {
             extra_wait: kwargs
                 .get(ruby.to_symbol("extra_wait"))
                 .and_then(|v| u64::try_convert(v).ok()),
-            stealth: kwargs
-                .get(ruby.to_symbol("stealth"))
-                .and_then(|v| bool::try_convert(v).ok())
-                .unwrap_or(false),
             proxy: kwargs
                 .get(ruby.to_symbol("proxy"))
                 .and_then(|v| ProxyConfig::try_convert(v).ok()),
@@ -546,10 +540,6 @@ impl BrowserConfig {
         self.extra_wait.clone()
     }
 
-    fn stealth(&self) -> bool {
-        self.stealth
-    }
-
     fn proxy(&self) -> Option<ProxyConfig> {
         self.proxy.clone()
     }
@@ -610,6 +600,8 @@ pub struct CrawlConfig {
     proxy: Option<ProxyConfig>,
     user_agents: Vec<String>,
     capture_screenshot: bool,
+    follow_document_urls: bool,
+    document_url_depth: Option<u32>,
     download_documents: bool,
     document_max_size: Option<usize>,
     document_mime_types: Vec<String>,
@@ -679,6 +671,8 @@ impl Default for CrawlConfig {
             proxy: None,
             user_agents: vec![],
             capture_screenshot: false,
+            follow_document_urls: false,
+            document_url_depth: None,
             download_documents: true,
             document_max_size: None,
             document_mime_types: vec![],
@@ -805,6 +799,13 @@ impl CrawlConfig {
                 .get(ruby.to_symbol("capture_screenshot"))
                 .and_then(|v| bool::try_convert(v).ok())
                 .unwrap_or(false),
+            follow_document_urls: kwargs
+                .get(ruby.to_symbol("follow_document_urls"))
+                .and_then(|v| bool::try_convert(v).ok())
+                .unwrap_or(false),
+            document_url_depth: kwargs
+                .get(ruby.to_symbol("document_url_depth"))
+                .and_then(|v| u32::try_convert(v).ok()),
             download_documents: kwargs
                 .get(ruby.to_symbol("download_documents"))
                 .and_then(|v| bool::try_convert(v).ok())
@@ -949,6 +950,14 @@ impl CrawlConfig {
         self.capture_screenshot
     }
 
+    fn follow_document_urls(&self) -> bool {
+        self.follow_document_urls
+    }
+
+    fn document_url_depth(&self) -> Option<u32> {
+        self.document_url_depth
+    }
+
     fn download_documents(&self) -> bool {
         self.download_documents
     }
@@ -1041,6 +1050,10 @@ impl CrawlConfig {
 
             capture_screenshot: self.capture_screenshot,
 
+            follow_document_urls: self.follow_document_urls,
+
+            document_url_depth: self.document_url_depth,
+
             download_documents: self.download_documents,
 
             document_max_size: self.document_max_size,
@@ -4411,6 +4424,7 @@ pub enum BrowserMode {
     Auto,
     Always,
     Never,
+    Stealth,
 }
 
 impl Default for BrowserMode {
@@ -4425,6 +4439,7 @@ impl magnus::IntoValue for BrowserMode {
             BrowserMode::Auto => "auto",
             BrowserMode::Always => "always",
             BrowserMode::Never => "never",
+            BrowserMode::Stealth => "stealth",
         };
         handle.to_symbol(sym).into_value_with(handle)
     }
@@ -4439,6 +4454,7 @@ impl magnus::TryConvert for BrowserMode {
             "auto" | "Auto" => Ok(BrowserMode::Auto),
             "always" | "Always" => Ok(BrowserMode::Always),
             "never" | "Never" => Ok(BrowserMode::Never),
+            "stealth" | "Stealth" => Ok(BrowserMode::Stealth),
             other => Err(magnus::Error::new(
                 unsafe { Ruby::get_unchecked() }.exception_arg_error(),
                 format!("invalid BrowserMode value: {other}"),
@@ -5351,7 +5367,6 @@ impl From<BrowserConfig> for kreuzcrawl::BrowserConfig {
             wait: val.wait.into(),
             wait_selector: val.wait_selector,
             extra_wait: val.extra_wait.map(std::time::Duration::from_millis),
-            stealth: val.stealth,
             proxy: val.proxy.map(Into::into),
             block_url_patterns: val.block_url_patterns.into_iter().collect(),
             eval_script: val.eval_script,
@@ -5373,7 +5388,6 @@ impl From<kreuzcrawl::BrowserConfig> for BrowserConfig {
             wait: val.wait.into(),
             wait_selector: val.wait_selector.map(|v| v.to_string()),
             extra_wait: val.extra_wait.map(|d| d.as_millis() as u64),
-            stealth: val.stealth,
             proxy: val.proxy.map(Into::into),
             block_url_patterns: val.block_url_patterns.into_iter().collect(),
             eval_script: val.eval_script.map(|v| v.to_string()),
@@ -5423,6 +5437,8 @@ impl From<CrawlConfig> for kreuzcrawl::CrawlConfig {
             proxy: val.proxy.map(Into::into),
             user_agents: val.user_agents.into_iter().collect(),
             capture_screenshot: val.capture_screenshot,
+            follow_document_urls: val.follow_document_urls,
+            document_url_depth: val.document_url_depth,
             download_documents: val.download_documents,
             document_max_size: val.document_max_size,
             document_mime_types: val.document_mime_types.into_iter().collect(),
@@ -5473,6 +5489,8 @@ impl From<kreuzcrawl::CrawlConfig> for CrawlConfig {
             proxy: val.proxy.map(Into::into),
             user_agents: val.user_agents.into_iter().collect(),
             capture_screenshot: val.capture_screenshot,
+            follow_document_urls: val.follow_document_urls,
+            document_url_depth: val.document_url_depth,
             download_documents: val.download_documents,
             document_max_size: val.document_max_size,
             document_mime_types: val.document_mime_types.into_iter().collect(),
@@ -6368,6 +6386,7 @@ impl From<BrowserMode> for kreuzcrawl::BrowserMode {
             BrowserMode::Auto => Self::Auto,
             BrowserMode::Always => Self::Always,
             BrowserMode::Never => Self::Never,
+            BrowserMode::Stealth => Self::Stealth,
         }
     }
 }
@@ -6378,6 +6397,7 @@ impl From<kreuzcrawl::BrowserMode> for BrowserMode {
             kreuzcrawl::BrowserMode::Auto => Self::Auto,
             kreuzcrawl::BrowserMode::Always => Self::Always,
             kreuzcrawl::BrowserMode::Never => Self::Never,
+            kreuzcrawl::BrowserMode::Stealth => Self::Stealth,
         }
     }
 }
@@ -6892,8 +6912,6 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("extra_wait", method!(BrowserConfig::extra_wait, 0))?;
 
-    class.define_method("stealth", method!(BrowserConfig::stealth, 0))?;
-
     class.define_method("proxy", method!(BrowserConfig::proxy, 0))?;
 
     class.define_method("block_url_patterns", method!(BrowserConfig::block_url_patterns, 0))?;
@@ -6973,6 +6991,10 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("capture_screenshot", method!(CrawlConfig::capture_screenshot, 0))?;
 
+    class.define_method("follow_document_urls", method!(CrawlConfig::follow_document_urls, 0))?;
+
+    class.define_method("document_url_depth", method!(CrawlConfig::document_url_depth, 0))?;
+
     class.define_method("download_documents", method!(CrawlConfig::download_documents, 0))?;
 
     class.define_method("document_max_size", method!(CrawlConfig::document_max_size, 0))?;

data/lib/kreuzcrawl/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:882a6f513499cb08d9cbeb641424fe238f26f3653e161241824c4ea057a44c4f
+# alef:hash:4a1130eaa7f8e3312e90b66f3eb0a52e657e6378ee013ed7eeaada8817434913
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true

data/lib/kreuzcrawl/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:882a6f513499cb08d9cbeb641424fe238f26f3653e161241824c4ea057a44c4f
+# alef:hash:4a1130eaa7f8e3312e90b66f3eb0a52e657e6378ee013ed7eeaada8817434913
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
 
 module Kreuzcrawl
   ## The version string for this package.
-  VERSION = "0.3.0.pre.rc.55"
+  VERSION = "0.3.0.pre.rc.59"
 end

data/lib/kreuzcrawl.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:882a6f513499cb08d9cbeb641424fe238f26f3653e161241824c4ea057a44c4f
+# alef:hash:4a1130eaa7f8e3312e90b66f3eb0a52e657e6378ee013ed7eeaada8817434913
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:882a6f513499cb08d9cbeb641424fe238f26f3653e161241824c4ea057a44c4f
+# alef:hash:4a1130eaa7f8e3312e90b66f3eb0a52e657e6378ee013ed7eeaada8817434913
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 
@@ -53,7 +53,6 @@ module Kreuzcrawl
     attr_accessor wait: BrowserWait?
     attr_accessor wait_selector: String?
     attr_accessor extra_wait: Integer?
-    attr_accessor stealth: bool?
     attr_accessor proxy: ProxyConfig?
     attr_accessor block_url_patterns: Array[String]?
     attr_accessor eval_script: String?
@@ -61,7 +60,7 @@ module Kreuzcrawl
     attr_accessor capture_network_events: bool?
     attr_accessor session_affinity: bool?
 
-    def initialize: (?mode: BrowserMode, ?backend: BrowserBackend, ?endpoint: String, ?timeout: Integer, ?wait: BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?stealth: bool, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool) -> void
+    def initialize: (?mode: BrowserMode, ?backend: BrowserBackend, ?endpoint: String, ?timeout: Integer, ?wait: BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool) -> void
     def self.default: () -> BrowserConfig
   end
 
@@ -96,6 +95,8 @@ module Kreuzcrawl
     attr_accessor proxy: ProxyConfig?
     attr_accessor user_agents: Array[String]?
     attr_accessor capture_screenshot: bool?
+    attr_accessor follow_document_urls: bool?
+    attr_accessor document_url_depth: Integer?
     attr_accessor download_documents: bool?
     attr_accessor document_max_size: Integer?
     attr_accessor document_mime_types: Array[String]?
@@ -103,7 +104,7 @@ module Kreuzcrawl
     attr_accessor browser_profile: String?
     attr_accessor save_browser_profile: bool?
 
-    def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_concurrent: Integer, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool) -> void
+    def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_concurrent: Integer, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool) -> void
     def validate: () -> void
     def self.default: () -> CrawlConfig
   end
@@ -460,7 +461,7 @@ module Kreuzcrawl
   end
 
   class BrowserMode
-    type value = :auto | :always | :never
+    type value = :auto | :always | :never | :stealth
   end
 
   class BrowserWait

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kreuzcrawl
 version: !ruby/object:Gem::Version
-  version: 0.3.0.pre.rc.55
+  version: 0.3.0.pre.rc.59
 platform: ruby
 authors:
 - Kreuzberg Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-06-12 00:00:00.000000000 Z
+date: 2026-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -78,6 +78,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ">="
     - !ruby/object:Gem::Version
       version: 3.2.0
+  - - "<"
+    - !ruby/object:Gem::Version
+      version: '4.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="