crawlberg 1.12.4 → 1.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/ext/crawlberg_rb/native/Cargo.lock +18 -8
  3. data/ext/crawlberg_rb/native/Cargo.toml +3 -3
  4. data/ext/crawlberg_rb/src/lib.rs +30 -28
  5. data/lib/crawlberg/version.rb +2 -2
  6. metadata +3 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 747021fcab944856af87da80556698102fb7ee7f3220c354e571fa59280b1160
4
- data.tar.gz: 54b65fe2b38f69ecfc1d562e0709e866d9a8a553e33efee72ad790496152cca7
3
+ metadata.gz: 3ca93fca8c6d8f1755a66c9a48c0316120da289fd3536e989e1d46a4f689b05a
4
+ data.tar.gz: d0bd7f659086bf5648e55ae68f9bce7206fef597da8407ed129fcaf4f7baa905
5
5
  SHA512:
6
- metadata.gz: 4505534058307fe2ccc412b11218a57529760ee573e5e1115908d08c94ec52968de2fb48abc939cd4c101770446d61813376b4c064505c5137c4294754464f5b
7
- data.tar.gz: 67f418fd655c3e05874f4fd41faa22bfb71363a800131ae886d5317225f606d600fcbc67bc52e2142c5e5867553d410bb0c196c4742f36ee35e52c13fea7f7bd
6
+ metadata.gz: 6e6fa6272d0bbd099ad9d106173c18e55dfdfbb3766f7b20dc2a40419ca57e308cb267109a5509f7bd6edc54a5aa644e8ee74aa4c0504b5f2113b86deae97975
7
+ data.tar.gz: 732ad605a1cddf8b2dfc0a73f08e7b1e834f0d53a6a142e8dfc41a758cbfdc4c03a43a28a9461d82a6993b11fbd00e80d953f4d0147e884c0d7fd00d1dc53f7d
data/ext/crawlberg_rb/native/Cargo.lock CHANGED
@@ -76,6 +76,15 @@ version = "0.7.8"
76
76
  source = "registry+https://github.com/rust-lang/crates.io-index"
77
77
  checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
78
78
 
79
+ [[package]]
80
+ name = "astral-html"
81
+ version = "0.0.4"
82
+ source = "registry+https://github.com/rust-lang/crates.io-index"
83
+ checksum = "d050c4dc17bfd8bf50da08bda9ef724b44a1b4ec09a5811f193f978cb6fbadbb"
84
+ dependencies = [
85
+ "memchr",
86
+ ]
87
+
79
88
  [[package]]
80
89
  name = "astral-tl"
81
90
  version = "0.8.0"
@@ -537,9 +546,9 @@ dependencies = [
537
546
 
538
547
  [[package]]
539
548
  name = "crawlberg"
540
- version = "1.12.4"
549
+ version = "1.13.0"
541
550
  source = "registry+https://github.com/rust-lang/crates.io-index"
542
- checksum = "e345a707346726368541ecc67fb348671c883b049aa266153858010514ee20ee"
551
+ checksum = "6056c5ff22fb7e7f4a3b1e30376217a728b3a9f7532459f6b8d162af6798cbef"
543
552
  dependencies = [
544
553
  "ahash",
545
554
  "aho-corasick",
@@ -635,7 +644,7 @@ dependencies = [
635
644
 
636
645
  [[package]]
637
646
  name = "crawlberg-rb"
638
- version = "1.12.4"
647
+ version = "1.13.0"
639
648
  dependencies = [
640
649
  "crawlberg",
641
650
  "futures",
@@ -648,9 +657,9 @@ dependencies = [
648
657
 
649
658
  [[package]]
650
659
  name = "crawlberg-robots"
651
- version = "1.12.4"
660
+ version = "1.13.0"
652
661
  source = "registry+https://github.com/rust-lang/crates.io-index"
653
- checksum = "7115fd4903fd4d7ffa0193824de27f3a407f6322d8ca2c7bde1025c4c7df841c"
662
+ checksum = "4c503c57dc1adab25a8d384b7b00d0e4bb9febe6b2d60fb34af67f1bdd8a4907"
654
663
 
655
664
  [[package]]
656
665
  name = "crc32fast"
@@ -1077,12 +1086,12 @@ checksum = "491194d601b4fa9fac3c9571e014070a0ad1bc3392d1bc10419695196a8b3cc3"
1077
1086
 
1078
1087
  [[package]]
1079
1088
  name = "html-to-markdown-rs"
1080
- version = "3.19.1"
1089
+ version = "3.21.0"
1081
1090
  source = "registry+https://github.com/rust-lang/crates.io-index"
1082
- checksum = "e0c9d1d13553220b2dd9bbbd135b6d8d9dfcb8c1893a842e93f93b9f59bcc5a4"
1091
+ checksum = "1508f37b10916e398625bcce1e4d19da5a04786d25de2996e2e4579c0e800a22"
1083
1092
  dependencies = [
1084
1093
  "ahash",
1085
- "astral-tl",
1094
+ "astral-html",
1086
1095
  "base64",
1087
1096
  "bitflags",
1088
1097
  "html-escape",
@@ -1094,6 +1103,7 @@ dependencies = [
1094
1103
  "regex",
1095
1104
  "serde",
1096
1105
  "serde_json",
1106
+ "smallvec",
1097
1107
  "thiserror 2.0.21",
1098
1108
  "tracing",
1099
1109
  "url",
data/ext/crawlberg_rb/native/Cargo.toml CHANGED
@@ -1,11 +1,11 @@
1
1
  # This file is auto-generated by alef — DO NOT EDIT.
2
- # alef:hash:7503feae8b3539c3d4ba3358faff2fcf06f713448e257078862d2e3b8f2e5ebb
2
+ # alef:hash:6910a56855d9d9eb2b2bb3ed737d02d2d3a19845e19a66335e9dd048307e5b85
3
3
  # To regenerate: alef generate
4
4
  # To verify freshness: alef verify
5
5
 
6
6
  [package]
7
7
  name = "crawlberg-rb"
8
- version = "1.12.4"
8
+ version = "1.13.0"
9
9
  edition = "2024"
10
10
  authors = ["Na'aman Hirschfeld <naaman@xberg.io>", "Tobias Perlstein <tobias@xberg.io>"]
11
11
  license = "MIT"
@@ -23,7 +23,7 @@ path = "../src/lib.rs"
23
23
  crate-type = ["cdylib"]
24
24
 
25
25
  [dependencies]
26
- crawlberg = { version = "1.12.4", features = ["interact", "browser-chromiumoxide"] }
26
+ crawlberg = { version = "1.13.0", features = ["interact", "browser-chromiumoxide"] }
27
27
  futures = "0.3"
28
28
  magnus = "0.9"
29
29
  rb-sys = ">=0.9.130, <0.10"
data/ext/crawlberg_rb/src/lib.rs CHANGED
@@ -1,5 +1,5 @@
1
1
  // This file is auto-generated by alef. DO NOT EDIT.
2
- // alef:hash:5766b3fb05a187f88eec0ae0ee3c5e32d6bede7f1a3bc38b4305587073983bbd
2
+ // alef:hash:2095bb341eb34e18eedd6431c932be6d804a83d47584faab078f93f8f4012a26
3
3
  // Re-generate with: alef generate
4
4
  #![allow(dead_code, unused_imports, unused_variables)]
5
5
  #![allow(
@@ -10991,34 +10991,36 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
10991
10991
  module.define_module_function("batch_crawl_stream", function!(batch_crawl_stream, 2))?;
10992
10992
 
10993
10993
  let crawl_error_class = module.define_error("CrawlError", ruby.exception_standard_error())?;
10994
- let not_found_error_class = module.define_error("NotFoundError", crawl_error_class)?;
10995
- let unauthorized_error_class = module.define_error("UnauthorizedError", crawl_error_class)?;
10996
- let forbidden_error_class = module.define_error("ForbiddenError", crawl_error_class)?;
10997
- let waf_blocked_error_class = module.define_error("WafBlockedError", crawl_error_class)?;
10998
- let crawl_timeout_error_class = module.define_error("CrawlTimeoutError", crawl_error_class)?;
10999
- let rate_limited_error_class = module.define_error("RateLimitedError", crawl_error_class)?;
11000
- let server_error_class = module.define_error("ServerError", crawl_error_class)?;
11001
- let bad_gateway_error_class = module.define_error("BadGatewayError", crawl_error_class)?;
11002
- let gone_error_class = module.define_error("GoneError", crawl_error_class)?;
11003
- let crawl_connection_error_class = module.define_error("CrawlConnectionError", crawl_error_class)?;
11004
- let dns_error_class = module.define_error("DnsError", crawl_error_class)?;
11005
- let ssl_error_class = module.define_error("SslError", crawl_error_class)?;
11006
- let data_loss_error_class = module.define_error("DataLossError", crawl_error_class)?;
11007
- let conversion_failed_error_class = module.define_error("ConversionFailedError", crawl_error_class)?;
11008
- let browser_error_class = module.define_error("BrowserError", crawl_error_class)?;
11009
- let browser_timeout_error_class = module.define_error("BrowserTimeoutError", crawl_error_class)?;
11010
- let invalid_config_error_class = module.define_error("InvalidConfigError", crawl_error_class)?;
11011
- let unsupported_error_class = module.define_error("UnsupportedError", crawl_error_class)?;
11012
- let ssrf_policy_violation_error_class = module.define_error("SsrfPolicyViolationError", crawl_error_class)?;
11013
- let other_error_class = module.define_error("OtherError", crawl_error_class)?;
10994
+ module.define_error("NotFoundError", crawl_error_class)?;
10995
+ module.define_error("UnauthorizedError", crawl_error_class)?;
10996
+ module.define_error("ForbiddenError", crawl_error_class)?;
10997
+ module.define_error("WafBlockedError", crawl_error_class)?;
10998
+ module.define_error("CrawlTimeoutError", crawl_error_class)?;
10999
+ module.define_error("RateLimitedError", crawl_error_class)?;
11000
+ module.define_error("ServerError", crawl_error_class)?;
11001
+ module.define_error("BadGatewayError", crawl_error_class)?;
11002
+ module.define_error("GoneError", crawl_error_class)?;
11003
+ module.define_error("CrawlConnectionError", crawl_error_class)?;
11004
+ module.define_error("DnsError", crawl_error_class)?;
11005
+ module.define_error("SslError", crawl_error_class)?;
11006
+ module.define_error("DataLossError", crawl_error_class)?;
11007
+ module.define_error("ConversionFailedError", crawl_error_class)?;
11008
+ module.define_error("BrowserError", crawl_error_class)?;
11009
+ module.define_error("BrowserTimeoutError", crawl_error_class)?;
11010
+ module.define_error("InvalidConfigError", crawl_error_class)?;
11011
+ module.define_error("UnsupportedError", crawl_error_class)?;
11012
+ module.define_error("SsrfPolicyViolationError", crawl_error_class)?;
11013
+ module.define_error("OtherError", crawl_error_class)?;
11014
11014
  let ssrf_error_class = module.define_error("SsrfError", crawl_error_class)?;
11015
- let denied_by_policy_error_class = module.define_error("DeniedByPolicyError", ssrf_error_class)?;
11016
- let not_on_allowlist_error_class = module.define_error("NotOnAllowlistError", ssrf_error_class)?;
11017
- let invalid_cidr_error_class = module.define_error("InvalidCidrError", ssrf_error_class)?;
11018
- let dns_resolution_failed_error_class = module.define_error("DnsResolutionFailedError", ssrf_error_class)?;
11019
- let invalid_url_error_class = module.define_error("InvalidUrlError", ssrf_error_class)?;
11020
- let disallowed_scheme_error_class = module.define_error("DisallowedSchemeError", ssrf_error_class)?;
11021
- let too_many_redirects_error_class = module.define_error("TooManyRedirectsError", ssrf_error_class)?;
11015
+ module.define_error("DeniedByPolicyError", ssrf_error_class)?;
11016
+ module.define_error("NotOnAllowlistError", ssrf_error_class)?;
11017
+ module.define_error("InvalidCidrError", ssrf_error_class)?;
11018
+ module.define_error("DnsResolutionFailedError", ssrf_error_class)?;
11019
+ module.define_error("InvalidUrlError", ssrf_error_class)?;
11020
+ module.define_error("DisallowedSchemeError", ssrf_error_class)?;
11021
+ module.define_error("TooManyRedirectsError", ssrf_error_class)?;
11022
+ module.funcall::<_, _, magnus::Value>("module_eval", ("class CrawlError\n module Family; end\n include Family\n def self.===(other)\n super || (equal?(CrawlError) && Family === other)\n end\nend",))?;
11023
+ module.funcall::<_, _, magnus::Value>("module_eval", ("class SsrfError\n module Family; include CrawlError::Family; end\n include Family\n def self.===(other)\n super || (equal?(SsrfError) && Family === other)\n end\nend",))?;
11022
11024
 
11023
11025
  Ok(())
11024
11026
  }
data/lib/crawlberg/version.rb CHANGED
@@ -1,10 +1,10 @@
1
1
  # This file is auto-generated by alef — DO NOT EDIT.
2
- # alef:hash:27817e744b9371747239d3dbd9b752d1075b40b343e776252ec09010abe2a16b
2
+ # alef:hash:3387d9d8b581cdfb271ae374ad39a3901c8d3fd155a33f971e8a2c5b10a30cad
3
3
  # To regenerate: alef generate
4
4
  # To verify freshness: alef verify
5
5
  # frozen_string_literal: true
6
6
 
7
7
  module Crawlberg
8
8
  ## The version string for this package.
9
- VERSION = "1.12.4"
9
+ VERSION = "1.13.0"
10
10
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: crawlberg
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.12.4
4
+ version: 1.13.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Na'aman Hirschfeld
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2026-10-31 00:00:00.000000000 Z
12
+ date: 2026-11-06 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: rb_sys
@@ -73,7 +73,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
73
73
  requirements:
74
74
  - - ">="
75
75
  - !ruby/object:Gem::Version
76
- version: 3.2.0
76
+ version: 3.3.0
77
77
  required_rubygems_version: !ruby/object:Gem::Requirement
78
78
  requirements:
79
79
  - - ">="