crawlberg 1.4.2 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f597e0e18bfca18b073ac6647f09704f8f811d3083175ef73a01e1d0a467355a
-  data.tar.gz: 034ab10f2cd798d6bf32c457b7b5abbae47737332c43c492b80345fa85eef7ac
+  metadata.gz: 1086b94d47584857a4dd092fcae70e6f2abba176dc9c45ad32cc2e18b5c5860a
+  data.tar.gz: b66c5106dc19d75ac9db8760192586e69d77639535adde26cb6c05fea6c856bc
 SHA512:
-  metadata.gz: bf52c93eee44c22122a61090b0a7df4824674f24e7718308f24f9c0d535e7f589ddbc903f1baa915842923e460f4dcd5b9995cbf185d629d6151571a9c431947
-  data.tar.gz: 8e7e088921ac8276371fa54c61429b746d2bce838bb37e643f12e2ef579e515e08fa526eaabf6a0e0aa9b27e01c6cfa9edb97d0eff70de1771fe632c1bb36780
+  metadata.gz: b3e758e72fca31dbc7ca62a8b753ca46266040d5ac10cd119c58ba4650ccab9a1ec805a4acc11ccf3403904e20f1377a2ca8bb47ad33e7c9d12ffeb4822b42e1
+  data.tar.gz: 3d86a33d4d74a623747bc588a58c131ad90307c9cfcccd39190e3dd4bada811155777f46811086f41e88dbe78aa9d9c394fff5c674a898ead2dfd776e553b046

data/ext/crawlberg_rb/native/Cargo.lock

@@ -72,9 +72,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
 
 [[package]]
 name = "astral-tl"
-version = "0.7.11"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d90933ffb0f97e2fc2e0de21da9d3f20597b804012d199843a6fe7c2810d28f3"
+checksum = "04d4bb5df351abb6758a04240b0193794ddaf34a7c0b9452fab3c25dcce39e88"
 dependencies = [
  "memchr",
 ]
@@ -200,7 +200,7 @@ dependencies = [
  "cc",
  "cfg-if",
  "constant_time_eq",
- "cpufeatures 0.3.0",
+ "cpufeatures 0.3.1",
 ]
 
 [[package]]
@@ -301,12 +301,12 @@ checksum = "f079e83a288787bcd14a6aea84cee5c87a67c5a3e660c30f557a3d24761b3527"
 
 [[package]]
 name = "chacha20"
-version = "0.10.1"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d524456ba66e72eb8b115ff89e01e497f8e6d11d78b70b1aa13c0fbd97540a81"
+checksum = "65c35e4b699c7e15ccbe7ee35c005e4fc0a278d22238a2857e6ce2dadeda1b06"
 dependencies = [
  "cfg-if",
- "cpufeatures 0.3.0",
+ "cpufeatures 0.3.1",
  "rand_core 0.10.1",
 ]
 
@@ -416,9 +416,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
 
 [[package]]
 name = "combine"
-version = "4.6.7"
+version = "4.6.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd"
+checksum = "cfc320937d09e6de266b31b9afb480f197d7a861be86be7cb2ea7e5d1bfffc5e"
 dependencies = [
  "bytes",
  "memchr",
@@ -540,18 +540,18 @@ dependencies = [
 
 [[package]]
 name = "cpufeatures"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b2a41393f66f16b0823bb79094d54ac5fbd34ab292ddafb9a0456ac9f87d201"
+checksum = "5ca28b0ae3115b884660db4118d803791fd6756b6e88f39c0f3f7859060d7566"
 dependencies = [
  "libc",
 ]
 
 [[package]]
 name = "crawlberg"
-version = "1.4.2"
+version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "723dbd626a68331c8b235e0c1878d38630fc827b9df6e86bf176250e87a54520"
+checksum = "1a63c394ff80d371a01d8365821c5cdb5d9a854085742df07212c4746d2c0213"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -593,7 +593,7 @@ dependencies = [
 
 [[package]]
 name = "crawlberg-rb"
-version = "1.4.2"
+version = "1.5.0"
 dependencies = [
  "crawlberg",
  "futures",
@@ -792,12 +792,13 @@ checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890"
 
 [[package]]
 name = "flate2"
-version = "1.1.9"
+version = "1.1.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c"
+checksum = "6e634e2e0ebac1ee034020da1ca582e17ffe4e0f5e985823721e168928136dcb"
 dependencies = [
  "crc32fast",
- "miniz_oxide",
+ "miniz_oxide 0.9.1",
+ "zlib-rs",
 ]
 
 [[package]]
@@ -993,9 +994,9 @@ checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
 
 [[package]]
 name = "h2"
-version = "0.4.18"
+version = "0.4.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "839c0e8a181239723652be9062bb56ca5bf5f64011f73b623f6f4fc59086a228"
+checksum = "ef8e5e5a340588f4452631496976cf8636d4a7ecf600239fdc27615d2530bc16"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -1036,9 +1037,9 @@ checksum = "c9356095b4b41197bba32173600e1582792cda618f65d12f68e2e77d273413c5"
 
 [[package]]
 name = "html-to-markdown-rs"
-version = "3.11.6"
+version = "3.12.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c5fe2583aad3e87439e40d841fed327a60892bb9946da1e3a5788ee7f0f4fe3c"
+checksum = "9fd381d291246b2df83535a2f2896dc32633a4cb69a14861f29f9dc0c7fbab77"
 dependencies = [
  "ahash",
  "astral-tl",
@@ -1117,9 +1118,9 @@ dependencies = [
 
 [[package]]
 name = "hyper"
-version = "1.11.0"
+version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d22053281f852e11534f5198498373cbb59295120a20771d90f7ed1897490a72"
+checksum = "27b501faa50e7a26c3d3560ca625132f4078a17771f4810baf70475ae48cbe43"
 dependencies = [
  "atomic-waker",
  "bytes",
@@ -1334,9 +1335,9 @@ dependencies = [
 
 [[package]]
 name = "indexmap"
-version = "2.14.0"
+version = "2.14.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9"
+checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb"
 dependencies = [
  "equivalent",
  "hashbrown 0.17.1",
@@ -1497,9 +1498,9 @@ dependencies = [
 
 [[package]]
 name = "libredox"
-version = "0.1.20"
+version = "0.1.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a"
+checksum = "d7955dfc218a8afb29dfeffd540e3a6e96baeb94fe7138228dd7cc6937fbbf96"
 dependencies = [
  "libc",
 ]
@@ -1599,6 +1600,16 @@ dependencies = [
  "simd-adler32",
 ]
 
+[[package]]
+name = "miniz_oxide"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c"
+dependencies = [
+ "adler2",
+ "simd-adler32",
+]
+
 [[package]]
 name = "mio"
 version = "1.2.2"
@@ -1850,7 +1861,7 @@ dependencies = [
  "crc32fast",
  "fdeflate",
  "flate2",
- "miniz_oxide",
+ "miniz_oxide 0.8.9",
 ]
 
 [[package]]
@@ -2064,9 +2075,9 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.127"
+version = "0.9.130"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7d7c9560fe42dcffa576941394075f18a17dce89fcf718a2fa90b7dc2134d12"
+checksum = "02faf625bb10ba893e3ae620f19c9fb1b5f8fcae0fe4eb86bb3f2230fad75edb"
 dependencies = [
  "rb-sys-build",
 ]
@@ -2275,9 +2286,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f"
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.14"
+version = "0.103.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0527518605e68109d875e248ea259b6758801cf165e4b2c2733ae3b51f12535a"
+checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2"
 dependencies = [
  "aws-lc-rs",
  "ring",
@@ -2420,7 +2431,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4"
 dependencies = [
  "cfg-if",
- "cpufeatures 0.3.0",
+ "cpufeatures 0.3.1",
  "digest 0.11.3",
 ]
 
@@ -3087,9 +3098,9 @@ checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88"
 
 [[package]]
 name = "which"
-version = "8.0.5"
+version = "8.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f3ef584124b911bcc3875c2f1472e80f24361ceb789bd1c62b3e9a3df9ff43c"
+checksum = "bae2f2b2b816647a1cab1acc91f5bd20812d53cb344382635ec2181940c8034f"
 dependencies = [
  "libc",
 ]
@@ -3450,6 +3461,12 @@ dependencies = [
  "syn 3.0.4",
 ]
 
+[[package]]
+name = "zlib-rs"
+version = "0.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34b31d188d9d685a4f9c7b46d6e36631b07058d2cfe190267adce54dc230bf12"
+
 [[package]]
 name = "zmij"
 version = "1.0.23"

data/ext/crawlberg_rb/native/Cargo.toml

@@ -1,11 +1,11 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:7cb6aa801d5f904cfc9a0079f817dfe7816c1d5b05bfd24c2eec864119ad7ee6
+# alef:hash:3c4fd09d678021d0e0e1fbe3ef980db4e5fdc7f30760cded4652482133197e05
 # To regenerate: alef generate
 # To verify freshness: alef verify
 
 [package]
 name = "crawlberg-rb"
-version = "1.4.2"
+version = "1.5.0"
 edition = "2024"
 license = "MIT"
 description = "High-performance web crawling engine"
@@ -22,10 +22,10 @@ path = "../src/lib.rs"
 crate-type = ["cdylib"]
 
 [dependencies]
-crawlberg = { version = "1.4.2", features = ["interact", "browser-chromiumoxide"] }
+crawlberg = { version = "1.5.0", features = ["interact", "browser-chromiumoxide"] }
 futures = "0.3"
 magnus = "0.8"
-rb-sys = ">=0.9, <0.9.128"
+rb-sys = ">=0.9.130, <0.10"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 tokio = { version = "1", features = ["rt-multi-thread"] }

data/ext/crawlberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:c8bd1f23995c2c54aa56bd9e9dd95391dff81fd7eefe7a65306513caf8564400
+// alef:hash:45a77cf71e37c29533a0d31a68c73f8d55de1a5aebed5f50b90fd1d2d0790b2f
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -411,6 +411,7 @@ impl BatchCrawlResults {
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::BatchCrawlStreamRequest")]
@@ -446,12 +447,14 @@ impl magnus::TryConvert for BatchCrawlStreamRequest {
 
 unsafe impl TryConvertOwned for BatchCrawlStreamRequest {}
 
+#[cfg(not(target_arch = "wasm32"))]
 impl Default for BatchCrawlStreamRequest {
     fn default() -> Self {
         crawlberg::BatchCrawlStreamRequest::default().into()
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
 impl BatchCrawlStreamRequest {
     fn new(args: &[magnus::Value]) -> Result<Self, magnus::Error> {
         let ruby = unsafe { magnus::Ruby::get_unchecked() };
@@ -634,7 +637,7 @@ impl BatchScrapeResults {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::BrowserConfig")]
 pub struct BrowserConfig {
@@ -681,6 +684,14 @@ impl magnus::TryConvert for BrowserConfig {
 }
 
 unsafe impl TryConvertOwned for BrowserConfig {}
+impl<'de> serde::Deserialize<'de> for BrowserConfig {
+    fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        <crawlberg::BrowserConfig as serde::Deserialize>::deserialize(deserializer).map(Into::into)
+    }
+}
 
 impl Default for BrowserConfig {
     fn default() -> Self {
@@ -797,7 +808,7 @@ impl BrowserConfig {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::BrowserExtras")]
 pub struct BrowserExtras {
@@ -833,6 +844,14 @@ impl magnus::TryConvert for BrowserExtras {
 }
 
 unsafe impl TryConvertOwned for BrowserExtras {}
+impl<'de> serde::Deserialize<'de> for BrowserExtras {
+    fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        <crawlberg::BrowserExtras as serde::Deserialize>::deserialize(deserializer).map(Into::into)
+    }
+}
 
 impl Default for BrowserExtras {
     fn default() -> Self {
@@ -1026,7 +1045,7 @@ impl CitationResult {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::ContentConfig")]
 pub struct ContentConfig {
@@ -1071,6 +1090,14 @@ impl magnus::TryConvert for ContentConfig {
 }
 
 unsafe impl TryConvertOwned for ContentConfig {}
+impl<'de> serde::Deserialize<'de> for ContentConfig {
+    fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        <crawlberg::ContentConfig as serde::Deserialize>::deserialize(deserializer).map(Into::into)
+    }
+}
 
 impl Default for ContentConfig {
     fn default() -> Self {
@@ -2308,6 +2335,7 @@ impl CrawlResult {
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::CrawlStreamRequest")]
@@ -2343,12 +2371,14 @@ impl magnus::TryConvert for CrawlStreamRequest {
 
 unsafe impl TryConvertOwned for CrawlStreamRequest {}
 
+#[cfg(not(target_arch = "wasm32"))]
 impl Default for CrawlStreamRequest {
     fn default() -> Self {
         crawlberg::CrawlStreamRequest::default().into()
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
 impl CrawlStreamRequest {
     fn new(args: &[magnus::Value]) -> Result<Self, magnus::Error> {
         let ruby = unsafe { magnus::Ruby::get_unchecked() };
@@ -2471,7 +2501,7 @@ impl DownloadedAsset {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::DownloadedDocument")]
 pub struct DownloadedDocument {
@@ -2513,6 +2543,14 @@ impl magnus::TryConvert for DownloadedDocument {
 }
 
 unsafe impl TryConvertOwned for DownloadedDocument {}
+impl<'de> serde::Deserialize<'de> for DownloadedDocument {
+    fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        <crawlberg::DownloadedDocument as serde::Deserialize>::deserialize(deserializer).map(Into::into)
+    }
+}
 
 impl Default for DownloadedDocument {
     fn default() -> Self {
@@ -4475,7 +4513,7 @@ impl SitemapUrl {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[derive(Clone, Debug, serde::Serialize)]
 #[serde(default)]
 #[magnus::wrap(class = "Crawlberg::SsrfPolicy")]
 pub struct SsrfPolicy {
@@ -4512,6 +4550,14 @@ impl magnus::TryConvert for SsrfPolicy {
 }
 
 unsafe impl TryConvertOwned for SsrfPolicy {}
+impl<'de> serde::Deserialize<'de> for SsrfPolicy {
+    fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
+    where
+        D: serde::Deserializer<'de>,
+    {
+        <crawlberg::SsrfPolicy as serde::Deserialize>::deserialize(deserializer).map(Into::into)
+    }
+}
 
 impl Default for SsrfPolicy {
     fn default() -> Self {
@@ -5726,6 +5772,7 @@ impl From<crawlberg::BatchCrawlResults> for BatchCrawlResults {
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
 #[allow(clippy::needless_update)]
 impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
     fn from(val: BatchCrawlStreamRequest) -> Self {
@@ -5736,6 +5783,7 @@ impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
 impl From<crawlberg::BatchCrawlStreamRequest> for BatchCrawlStreamRequest {
     fn from(val: crawlberg::BatchCrawlStreamRequest) -> Self {
         Self {
@@ -6185,6 +6233,7 @@ impl From<crawlberg::CrawlResult> for CrawlResult {
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
 #[allow(clippy::needless_update)]
 impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
     fn from(val: CrawlStreamRequest) -> Self {
@@ -6195,6 +6244,7 @@ impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
     }
 }
 
+#[cfg(not(target_arch = "wasm32"))]
 impl From<crawlberg::CrawlStreamRequest> for CrawlStreamRequest {
     fn from(val: crawlberg::CrawlStreamRequest) -> Self {
         Self {

data/lib/crawlberg/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:9ea9a14d259e4988466100a061689d975727c16c521c140aef2205c9b97bf831
+# alef:hash:7234c62471e6f1812ca4c07d492c86782a033ac7e8d8fc64dcbb4dc0ca728eeb
 # To regenerate: alef generate
 # To verify freshness: alef verify
 # frozen_string_literal: true
 
 module Crawlberg
   ## The version string for this package.
-  VERSION = "1.4.2"
+  VERSION = "1.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crawlberg
 version: !ruby/object:Gem::Version
-  version: 1.4.2
+  version: 1.5.0
 platform: ruby
 authors:
 - Xberg Team
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-08-29 00:00:00.000000000 Z
+date: 2026-09-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -16,20 +16,20 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.130
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.9.128
+        version: '0.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0.9'
+        version: 0.9.130
     - - "<"
       - !ruby/object:Gem::Version
-        version: 0.9.128
+        version: '0.10'
 - !ruby/object:Gem::Dependency
   name: sorbet-runtime
   requirement: !ruby/object:Gem::Requirement