crawlberg 1.12.3 → 1.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/crawlberg_rb/native/Cargo.lock +55 -37
- data/ext/crawlberg_rb/native/Cargo.toml +3 -3
- data/ext/crawlberg_rb/src/lib.rs +30 -28
- data/lib/crawlberg/native.rb +28 -19
- data/lib/crawlberg/version.rb +2 -2
- data/sig/types.rbs +4 -4
- metadata +3 -3
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3ca93fca8c6d8f1755a66c9a48c0316120da289fd3536e989e1d46a4f689b05a
|
|
4
|
+
data.tar.gz: d0bd7f659086bf5648e55ae68f9bce7206fef597da8407ed129fcaf4f7baa905
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 6e6fa6272d0bbd099ad9d106173c18e55dfdfbb3766f7b20dc2a40419ca57e308cb267109a5509f7bd6edc54a5aa644e8ee74aa4c0504b5f2113b86deae97975
|
|
7
|
+
data.tar.gz: 732ad605a1cddf8b2dfc0a73f08e7b1e834f0d53a6a142e8dfc41a758cbfdc4c03a43a28a9461d82a6993b11fbd00e80d953f4d0147e884c0d7fd00d1dc53f7d
|
|
@@ -76,6 +76,15 @@ version = "0.7.8"
|
|
|
76
76
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
77
77
|
checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
|
|
78
78
|
|
|
79
|
+
[[package]]
|
|
80
|
+
name = "astral-html"
|
|
81
|
+
version = "0.0.4"
|
|
82
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
83
|
+
checksum = "d050c4dc17bfd8bf50da08bda9ef724b44a1b4ec09a5811f193f978cb6fbadbb"
|
|
84
|
+
dependencies = [
|
|
85
|
+
"memchr",
|
|
86
|
+
]
|
|
87
|
+
|
|
79
88
|
[[package]]
|
|
80
89
|
name = "astral-tl"
|
|
81
90
|
version = "0.8.0"
|
|
@@ -139,9 +148,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
|
|
|
139
148
|
|
|
140
149
|
[[package]]
|
|
141
150
|
name = "aws-lc-rs"
|
|
142
|
-
version = "1.
|
|
151
|
+
version = "1.19.0"
|
|
143
152
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
144
|
-
checksum = "
|
|
153
|
+
checksum = "048006790404103c488e6c5616b3e7e702a71292aadda08e8f7698a9e0d2eb53"
|
|
145
154
|
dependencies = [
|
|
146
155
|
"aws-lc-sys",
|
|
147
156
|
"zeroize",
|
|
@@ -149,9 +158,9 @@ dependencies = [
|
|
|
149
158
|
|
|
150
159
|
[[package]]
|
|
151
160
|
name = "aws-lc-sys"
|
|
152
|
-
version = "0.
|
|
161
|
+
version = "0.47.0"
|
|
153
162
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
154
|
-
checksum = "
|
|
163
|
+
checksum = "8bfbe32ac36b3c428696ce315464181bc4029d689fd58e3f2a4e58c252242f52"
|
|
155
164
|
dependencies = [
|
|
156
165
|
"cc",
|
|
157
166
|
"cmake",
|
|
@@ -537,9 +546,9 @@ dependencies = [
|
|
|
537
546
|
|
|
538
547
|
[[package]]
|
|
539
548
|
name = "crawlberg"
|
|
540
|
-
version = "1.
|
|
549
|
+
version = "1.13.0"
|
|
541
550
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
542
|
-
checksum = "
|
|
551
|
+
checksum = "6056c5ff22fb7e7f4a3b1e30376217a728b3a9f7532459f6b8d162af6798cbef"
|
|
543
552
|
dependencies = [
|
|
544
553
|
"ahash",
|
|
545
554
|
"aho-corasick",
|
|
@@ -635,7 +644,7 @@ dependencies = [
|
|
|
635
644
|
|
|
636
645
|
[[package]]
|
|
637
646
|
name = "crawlberg-rb"
|
|
638
|
-
version = "1.
|
|
647
|
+
version = "1.13.0"
|
|
639
648
|
dependencies = [
|
|
640
649
|
"crawlberg",
|
|
641
650
|
"futures",
|
|
@@ -648,9 +657,9 @@ dependencies = [
|
|
|
648
657
|
|
|
649
658
|
[[package]]
|
|
650
659
|
name = "crawlberg-robots"
|
|
651
|
-
version = "1.
|
|
660
|
+
version = "1.13.0"
|
|
652
661
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
653
|
-
checksum = "
|
|
662
|
+
checksum = "4c503c57dc1adab25a8d384b7b00d0e4bb9febe6b2d60fb34af67f1bdd8a4907"
|
|
654
663
|
|
|
655
664
|
[[package]]
|
|
656
665
|
name = "crc32fast"
|
|
@@ -774,9 +783,9 @@ checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be"
|
|
|
774
783
|
|
|
775
784
|
[[package]]
|
|
776
785
|
name = "encoding_rs"
|
|
777
|
-
version = "0.8.
|
|
786
|
+
version = "0.8.44"
|
|
778
787
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
779
|
-
checksum = "
|
|
788
|
+
checksum = "89253175e704ab869b04f9db224d38630e0211a53841a79181deae0889e169f5"
|
|
780
789
|
dependencies = [
|
|
781
790
|
"cfg-if",
|
|
782
791
|
"core_detect",
|
|
@@ -1077,12 +1086,12 @@ checksum = "491194d601b4fa9fac3c9571e014070a0ad1bc3392d1bc10419695196a8b3cc3"
|
|
|
1077
1086
|
|
|
1078
1087
|
[[package]]
|
|
1079
1088
|
name = "html-to-markdown-rs"
|
|
1080
|
-
version = "3.
|
|
1089
|
+
version = "3.21.0"
|
|
1081
1090
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1082
|
-
checksum = "
|
|
1091
|
+
checksum = "1508f37b10916e398625bcce1e4d19da5a04786d25de2996e2e4579c0e800a22"
|
|
1083
1092
|
dependencies = [
|
|
1084
1093
|
"ahash",
|
|
1085
|
-
"astral-
|
|
1094
|
+
"astral-html",
|
|
1086
1095
|
"base64",
|
|
1087
1096
|
"bitflags",
|
|
1088
1097
|
"html-escape",
|
|
@@ -1094,6 +1103,7 @@ dependencies = [
|
|
|
1094
1103
|
"regex",
|
|
1095
1104
|
"serde",
|
|
1096
1105
|
"serde_json",
|
|
1106
|
+
"smallvec",
|
|
1097
1107
|
"thiserror 2.0.21",
|
|
1098
1108
|
"tracing",
|
|
1099
1109
|
"url",
|
|
@@ -1521,9 +1531,9 @@ dependencies = [
|
|
|
1521
1531
|
|
|
1522
1532
|
[[package]]
|
|
1523
1533
|
name = "js-sys"
|
|
1524
|
-
version = "0.3.
|
|
1534
|
+
version = "0.3.107"
|
|
1525
1535
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1526
|
-
checksum = "
|
|
1536
|
+
checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a"
|
|
1527
1537
|
dependencies = [
|
|
1528
1538
|
"cfg-if",
|
|
1529
1539
|
"futures-util",
|
|
@@ -1610,9 +1620,9 @@ dependencies = [
|
|
|
1610
1620
|
|
|
1611
1621
|
[[package]]
|
|
1612
1622
|
name = "log"
|
|
1613
|
-
version = "0.4.
|
|
1623
|
+
version = "0.4.35"
|
|
1614
1624
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1615
|
-
checksum = "
|
|
1625
|
+
checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c"
|
|
1616
1626
|
|
|
1617
1627
|
[[package]]
|
|
1618
1628
|
name = "lru-slab"
|
|
@@ -1757,9 +1767,9 @@ dependencies = [
|
|
|
1757
1767
|
|
|
1758
1768
|
[[package]]
|
|
1759
1769
|
name = "notify-types"
|
|
1760
|
-
version = "2.
|
|
1770
|
+
version = "2.2.0"
|
|
1761
1771
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1762
|
-
checksum = "
|
|
1772
|
+
checksum = "0a437a9308232ce794b1e7fdf540d38ea37efc3c73492640c13b4c8568b8f108"
|
|
1763
1773
|
dependencies = [
|
|
1764
1774
|
"bitflags",
|
|
1765
1775
|
]
|
|
@@ -2686,9 +2696,9 @@ dependencies = [
|
|
|
2686
2696
|
|
|
2687
2697
|
[[package]]
|
|
2688
2698
|
name = "sysinfo"
|
|
2689
|
-
version = "0.
|
|
2699
|
+
version = "0.40.1"
|
|
2690
2700
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2691
|
-
checksum = "
|
|
2701
|
+
checksum = "0e8a3edbe0394ce115f3917a00349dd242be09eb5a091ed509eaf67b55cfb8a8"
|
|
2692
2702
|
dependencies = [
|
|
2693
2703
|
"libc",
|
|
2694
2704
|
"memchr",
|
|
@@ -3087,9 +3097,18 @@ dependencies = [
|
|
|
3087
3097
|
|
|
3088
3098
|
[[package]]
|
|
3089
3099
|
name = "utf8_iter"
|
|
3090
|
-
version = "1.0
|
|
3100
|
+
version = "1.1.0"
|
|
3101
|
+
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3102
|
+
checksum = "af3bd9d3e571c26fcce9d033b87ea5ab061b8d2ffe9b7a5675d810d68f03edf0"
|
|
3103
|
+
dependencies = [
|
|
3104
|
+
"utf_types",
|
|
3105
|
+
]
|
|
3106
|
+
|
|
3107
|
+
[[package]]
|
|
3108
|
+
name = "utf_types"
|
|
3109
|
+
version = "1.0.0"
|
|
3091
3110
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3092
|
-
checksum = "
|
|
3111
|
+
checksum = "3a3b425286980a367278453f3b6d28cdcb134b574e5a35824a6ff5eebad50bb6"
|
|
3093
3112
|
|
|
3094
3113
|
[[package]]
|
|
3095
3114
|
name = "version_check"
|
|
@@ -3133,12 +3152,11 @@ dependencies = [
|
|
|
3133
3152
|
|
|
3134
3153
|
[[package]]
|
|
3135
3154
|
name = "wasm-bindgen"
|
|
3136
|
-
version = "0.2.
|
|
3155
|
+
version = "0.2.130"
|
|
3137
3156
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3138
|
-
checksum = "
|
|
3157
|
+
checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c"
|
|
3139
3158
|
dependencies = [
|
|
3140
3159
|
"cfg-if",
|
|
3141
|
-
"once_cell",
|
|
3142
3160
|
"rustversion",
|
|
3143
3161
|
"wasm-bindgen-macro",
|
|
3144
3162
|
"wasm-bindgen-shared",
|
|
@@ -3146,9 +3164,9 @@ dependencies = [
|
|
|
3146
3164
|
|
|
3147
3165
|
[[package]]
|
|
3148
3166
|
name = "wasm-bindgen-futures"
|
|
3149
|
-
version = "0.4.
|
|
3167
|
+
version = "0.4.80"
|
|
3150
3168
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3151
|
-
checksum = "
|
|
3169
|
+
checksum = "5e14e0327cb5368ad60ccb22d04a9827a693c95a926d8d125e9fa39e0baec501"
|
|
3152
3170
|
dependencies = [
|
|
3153
3171
|
"js-sys",
|
|
3154
3172
|
"tokio",
|
|
@@ -3157,9 +3175,9 @@ dependencies = [
|
|
|
3157
3175
|
|
|
3158
3176
|
[[package]]
|
|
3159
3177
|
name = "wasm-bindgen-macro"
|
|
3160
|
-
version = "0.2.
|
|
3178
|
+
version = "0.2.130"
|
|
3161
3179
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3162
|
-
checksum = "
|
|
3180
|
+
checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9"
|
|
3163
3181
|
dependencies = [
|
|
3164
3182
|
"quote",
|
|
3165
3183
|
"wasm-bindgen-macro-support",
|
|
@@ -3167,9 +3185,9 @@ dependencies = [
|
|
|
3167
3185
|
|
|
3168
3186
|
[[package]]
|
|
3169
3187
|
name = "wasm-bindgen-macro-support"
|
|
3170
|
-
version = "0.2.
|
|
3188
|
+
version = "0.2.130"
|
|
3171
3189
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3172
|
-
checksum = "
|
|
3190
|
+
checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a"
|
|
3173
3191
|
dependencies = [
|
|
3174
3192
|
"bumpalo",
|
|
3175
3193
|
"proc-macro2",
|
|
@@ -3180,18 +3198,18 @@ dependencies = [
|
|
|
3180
3198
|
|
|
3181
3199
|
[[package]]
|
|
3182
3200
|
name = "wasm-bindgen-shared"
|
|
3183
|
-
version = "0.2.
|
|
3201
|
+
version = "0.2.130"
|
|
3184
3202
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3185
|
-
checksum = "
|
|
3203
|
+
checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e"
|
|
3186
3204
|
dependencies = [
|
|
3187
3205
|
"unicode-ident",
|
|
3188
3206
|
]
|
|
3189
3207
|
|
|
3190
3208
|
[[package]]
|
|
3191
3209
|
name = "web-sys"
|
|
3192
|
-
version = "0.3.
|
|
3210
|
+
version = "0.3.107"
|
|
3193
3211
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
3194
|
-
checksum = "
|
|
3212
|
+
checksum = "d7d179596ff5ec0f6de4db64d54011bde6289cc49603d3a6e4ce3f329207ab88"
|
|
3195
3213
|
dependencies = [
|
|
3196
3214
|
"js-sys",
|
|
3197
3215
|
"wasm-bindgen",
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:6910a56855d9d9eb2b2bb3ed737d02d2d3a19845e19a66335e9dd048307e5b85
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
|
|
6
6
|
[package]
|
|
7
7
|
name = "crawlberg-rb"
|
|
8
|
-
version = "1.
|
|
8
|
+
version = "1.13.0"
|
|
9
9
|
edition = "2024"
|
|
10
10
|
authors = ["Na'aman Hirschfeld <naaman@xberg.io>", "Tobias Perlstein <tobias@xberg.io>"]
|
|
11
11
|
license = "MIT"
|
|
@@ -23,7 +23,7 @@ path = "../src/lib.rs"
|
|
|
23
23
|
crate-type = ["cdylib"]
|
|
24
24
|
|
|
25
25
|
[dependencies]
|
|
26
|
-
crawlberg = { version = "1.
|
|
26
|
+
crawlberg = { version = "1.13.0", features = ["interact", "browser-chromiumoxide"] }
|
|
27
27
|
futures = "0.3"
|
|
28
28
|
magnus = "0.9"
|
|
29
29
|
rb-sys = ">=0.9.130, <0.10"
|
data/ext/crawlberg_rb/src/lib.rs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef. DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:2095bb341eb34e18eedd6431c932be6d804a83d47584faab078f93f8f4012a26
|
|
3
3
|
// Re-generate with: alef generate
|
|
4
4
|
#![allow(dead_code, unused_imports, unused_variables)]
|
|
5
5
|
#![allow(
|
|
@@ -10991,34 +10991,36 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
10991
10991
|
module.define_module_function("batch_crawl_stream", function!(batch_crawl_stream, 2))?;
|
|
10992
10992
|
|
|
10993
10993
|
let crawl_error_class = module.define_error("CrawlError", ruby.exception_standard_error())?;
|
|
10994
|
-
|
|
10995
|
-
|
|
10996
|
-
|
|
10997
|
-
|
|
10998
|
-
|
|
10999
|
-
|
|
11000
|
-
|
|
11001
|
-
|
|
11002
|
-
|
|
11003
|
-
|
|
11004
|
-
|
|
11005
|
-
|
|
11006
|
-
|
|
11007
|
-
|
|
11008
|
-
|
|
11009
|
-
|
|
11010
|
-
|
|
11011
|
-
|
|
11012
|
-
|
|
11013
|
-
|
|
10994
|
+
module.define_error("NotFoundError", crawl_error_class)?;
|
|
10995
|
+
module.define_error("UnauthorizedError", crawl_error_class)?;
|
|
10996
|
+
module.define_error("ForbiddenError", crawl_error_class)?;
|
|
10997
|
+
module.define_error("WafBlockedError", crawl_error_class)?;
|
|
10998
|
+
module.define_error("CrawlTimeoutError", crawl_error_class)?;
|
|
10999
|
+
module.define_error("RateLimitedError", crawl_error_class)?;
|
|
11000
|
+
module.define_error("ServerError", crawl_error_class)?;
|
|
11001
|
+
module.define_error("BadGatewayError", crawl_error_class)?;
|
|
11002
|
+
module.define_error("GoneError", crawl_error_class)?;
|
|
11003
|
+
module.define_error("CrawlConnectionError", crawl_error_class)?;
|
|
11004
|
+
module.define_error("DnsError", crawl_error_class)?;
|
|
11005
|
+
module.define_error("SslError", crawl_error_class)?;
|
|
11006
|
+
module.define_error("DataLossError", crawl_error_class)?;
|
|
11007
|
+
module.define_error("ConversionFailedError", crawl_error_class)?;
|
|
11008
|
+
module.define_error("BrowserError", crawl_error_class)?;
|
|
11009
|
+
module.define_error("BrowserTimeoutError", crawl_error_class)?;
|
|
11010
|
+
module.define_error("InvalidConfigError", crawl_error_class)?;
|
|
11011
|
+
module.define_error("UnsupportedError", crawl_error_class)?;
|
|
11012
|
+
module.define_error("SsrfPolicyViolationError", crawl_error_class)?;
|
|
11013
|
+
module.define_error("OtherError", crawl_error_class)?;
|
|
11014
11014
|
let ssrf_error_class = module.define_error("SsrfError", crawl_error_class)?;
|
|
11015
|
-
|
|
11016
|
-
|
|
11017
|
-
|
|
11018
|
-
|
|
11019
|
-
|
|
11020
|
-
|
|
11021
|
-
|
|
11015
|
+
module.define_error("DeniedByPolicyError", ssrf_error_class)?;
|
|
11016
|
+
module.define_error("NotOnAllowlistError", ssrf_error_class)?;
|
|
11017
|
+
module.define_error("InvalidCidrError", ssrf_error_class)?;
|
|
11018
|
+
module.define_error("DnsResolutionFailedError", ssrf_error_class)?;
|
|
11019
|
+
module.define_error("InvalidUrlError", ssrf_error_class)?;
|
|
11020
|
+
module.define_error("DisallowedSchemeError", ssrf_error_class)?;
|
|
11021
|
+
module.define_error("TooManyRedirectsError", ssrf_error_class)?;
|
|
11022
|
+
module.funcall::<_, _, magnus::Value>("module_eval", ("class CrawlError\n module Family; end\n include Family\n def self.===(other)\n super || (equal?(CrawlError) && Family === other)\n end\nend",))?;
|
|
11023
|
+
module.funcall::<_, _, magnus::Value>("module_eval", ("class SsrfError\n module Family; include CrawlError::Family; end\n include Family\n def self.===(other)\n super || (equal?(SsrfError) && Family === other)\n end\nend",))?;
|
|
11022
11024
|
|
|
11023
11025
|
Ok(())
|
|
11024
11026
|
}
|
data/lib/crawlberg/native.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:69b28e0471fcb4accb00c4c7601e854143fc70114da129bcb4f7853d1a92c044
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
@@ -82,7 +82,10 @@ module Crawlberg
|
|
|
82
82
|
# @return [self]
|
|
83
83
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
84
84
|
def self.from_hash(hash)
|
|
85
|
-
new(
|
|
85
|
+
new(
|
|
86
|
+
username: hash.key?(:username) ? hash[:username] : hash["username"],
|
|
87
|
+
password: hash.key?(:password) ? hash[:password] : hash["password"]
|
|
88
|
+
)
|
|
86
89
|
end
|
|
87
90
|
end
|
|
88
91
|
## Bearer token authentication.
|
|
@@ -103,7 +106,7 @@ module Crawlberg
|
|
|
103
106
|
# @return [self]
|
|
104
107
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
105
108
|
def self.from_hash(hash)
|
|
106
|
-
new(token: hash[:token]
|
|
109
|
+
new(token: hash.key?(:token) ? hash[:token] : hash["token"])
|
|
107
110
|
end
|
|
108
111
|
end
|
|
109
112
|
## Custom authentication header.
|
|
@@ -127,7 +130,7 @@ module Crawlberg
|
|
|
127
130
|
# @return [self]
|
|
128
131
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
129
132
|
def self.from_hash(hash)
|
|
130
|
-
new(name: hash[:name]
|
|
133
|
+
new(name: hash.key?(:name) ? hash[:name] : hash["name"], value: hash.key?(:value) ? hash[:value] : hash["value"])
|
|
131
134
|
end
|
|
132
135
|
end
|
|
133
136
|
end
|
|
@@ -185,7 +188,7 @@ module Crawlberg
|
|
|
185
188
|
# @return [self]
|
|
186
189
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
187
190
|
def self.from_hash(hash)
|
|
188
|
-
new(result: hash[:result]
|
|
191
|
+
new(result: hash.key?(:result) ? hash[:result] : hash["result"])
|
|
189
192
|
end
|
|
190
193
|
end
|
|
191
194
|
## An error occurred while crawling a URL.
|
|
@@ -209,7 +212,7 @@ module Crawlberg
|
|
|
209
212
|
# @return [self]
|
|
210
213
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
211
214
|
def self.from_hash(hash)
|
|
212
|
-
new(url: hash[:url]
|
|
215
|
+
new(url: hash.key?(:url) ? hash[:url] : hash["url"], error: hash.key?(:error) ? hash[:error] : hash["error"])
|
|
213
216
|
end
|
|
214
217
|
end
|
|
215
218
|
## The crawl has completed.
|
|
@@ -230,7 +233,7 @@ module Crawlberg
|
|
|
230
233
|
# @return [self]
|
|
231
234
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
232
235
|
def self.from_hash(hash)
|
|
233
|
-
new(pages_crawled: hash[:pages_crawled]
|
|
236
|
+
new(pages_crawled: hash.key?(:pages_crawled) ? hash[:pages_crawled] : hash["pages_crawled"])
|
|
234
237
|
end
|
|
235
238
|
end
|
|
236
239
|
end
|
|
@@ -293,7 +296,7 @@ module Crawlberg
|
|
|
293
296
|
# @return [self]
|
|
294
297
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
295
298
|
def self.from_hash(hash)
|
|
296
|
-
new(value: hash[:value]
|
|
299
|
+
new(value: hash.key?(:value) ? hash[:value] : hash["value"])
|
|
297
300
|
end
|
|
298
301
|
end
|
|
299
302
|
## Suffix match: ".xberg.io" matches "api.xberg.io" and "xberg.io".
|
|
@@ -314,7 +317,7 @@ module Crawlberg
|
|
|
314
317
|
# @return [self]
|
|
315
318
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
316
319
|
def self.from_hash(hash)
|
|
317
|
-
new(value: hash[:value]
|
|
320
|
+
new(value: hash.key?(:value) ? hash[:value] : hash["value"])
|
|
318
321
|
end
|
|
319
322
|
end
|
|
320
323
|
## CIDR match: "10.0.0.0/8" matches IP addresses in that range.
|
|
@@ -336,7 +339,7 @@ module Crawlberg
|
|
|
336
339
|
# @return [self]
|
|
337
340
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
338
341
|
def self.from_hash(hash)
|
|
339
|
-
new(value: hash[:value]
|
|
342
|
+
new(value: hash.key?(:value) ? hash[:value] : hash["value"])
|
|
340
343
|
end
|
|
341
344
|
end
|
|
342
345
|
end
|
|
@@ -408,7 +411,7 @@ module Crawlberg
|
|
|
408
411
|
# @return [self]
|
|
409
412
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
410
413
|
def self.from_hash(hash)
|
|
411
|
-
new(selector: hash[:selector]
|
|
414
|
+
new(selector: hash.key?(:selector) ? hash[:selector] : hash["selector"])
|
|
412
415
|
end
|
|
413
416
|
end
|
|
414
417
|
## Type text into an element matching the given CSS selector.
|
|
@@ -442,7 +445,10 @@ module Crawlberg
|
|
|
442
445
|
# @return [self]
|
|
443
446
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
444
447
|
def self.from_hash(hash)
|
|
445
|
-
new(
|
|
448
|
+
new(
|
|
449
|
+
selector: hash.key?(:selector) ? hash[:selector] : hash["selector"],
|
|
450
|
+
text: hash.key?(:text) ? hash[:text] : hash["text"]
|
|
451
|
+
)
|
|
446
452
|
end
|
|
447
453
|
end
|
|
448
454
|
## Press a keyboard key (e.g. "Enter", "Tab", "Escape").
|
|
@@ -473,7 +479,7 @@ module Crawlberg
|
|
|
473
479
|
# @return [self]
|
|
474
480
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
475
481
|
def self.from_hash(hash)
|
|
476
|
-
new(key: hash[:key]
|
|
482
|
+
new(key: hash.key?(:key) ? hash[:key] : hash["key"])
|
|
477
483
|
end
|
|
478
484
|
end
|
|
479
485
|
## Scroll the page or a specific element.
|
|
@@ -511,9 +517,9 @@ module Crawlberg
|
|
|
511
517
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
512
518
|
def self.from_hash(hash)
|
|
513
519
|
new(
|
|
514
|
-
direction: hash[:direction]
|
|
515
|
-
selector: hash[:selector]
|
|
516
|
-
amount: hash[:amount]
|
|
520
|
+
direction: hash.key?(:direction) ? hash[:direction] : hash["direction"],
|
|
521
|
+
selector: hash.key?(:selector) ? hash[:selector] : hash["selector"],
|
|
522
|
+
amount: hash.key?(:amount) ? hash[:amount] : hash["amount"]
|
|
517
523
|
)
|
|
518
524
|
end
|
|
519
525
|
end
|
|
@@ -548,7 +554,10 @@ module Crawlberg
|
|
|
548
554
|
# @return [self]
|
|
549
555
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
550
556
|
def self.from_hash(hash)
|
|
551
|
-
new(
|
|
557
|
+
new(
|
|
558
|
+
milliseconds: hash.key?(:milliseconds) ? hash[:milliseconds] : hash["milliseconds"],
|
|
559
|
+
selector: hash.key?(:selector) ? hash[:selector] : hash["selector"]
|
|
560
|
+
)
|
|
552
561
|
end
|
|
553
562
|
end
|
|
554
563
|
## Take a screenshot of the current page.
|
|
@@ -583,7 +592,7 @@ module Crawlberg
|
|
|
583
592
|
# @return [self]
|
|
584
593
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
585
594
|
def self.from_hash(hash)
|
|
586
|
-
new(full_page: hash[:full_page]
|
|
595
|
+
new(full_page: hash.key?(:full_page) ? hash[:full_page] : hash["full_page"])
|
|
587
596
|
end
|
|
588
597
|
end
|
|
589
598
|
## Execute arbitrary JavaScript in the page context.
|
|
@@ -619,7 +628,7 @@ module Crawlberg
|
|
|
619
628
|
# @return [self]
|
|
620
629
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
621
630
|
def self.from_hash(hash)
|
|
622
|
-
new(script: hash[:script]
|
|
631
|
+
new(script: hash.key?(:script) ? hash[:script] : hash["script"])
|
|
623
632
|
end
|
|
624
633
|
end
|
|
625
634
|
## Scrape the current page HTML.
|
data/lib/crawlberg/version.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:3387d9d8b581cdfb271ae374ad39a3901c8d3fd155a33f971e8a2c5b10a30cad
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
module Crawlberg
|
|
8
8
|
## The version string for this package.
|
|
9
|
-
VERSION = "1.
|
|
9
|
+
VERSION = "1.13.0"
|
|
10
10
|
end
|
data/sig/types.rbs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:2a433f35126235c3682b8cfe4eaad04f121e9210241b4afaa525a7175302dca4
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
|
|
@@ -743,7 +743,7 @@ def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist:
|
|
|
743
743
|
def scrape?: () -> bool
|
|
744
744
|
|
|
745
745
|
def self.from_hash: (untyped hash) -> untyped
|
|
746
|
-
def self.new: (direction: enum_ScrollDirection, selector: String
|
|
746
|
+
def self.new: (direction: enum_ScrollDirection, selector: String?, amount: Integer?) -> PageActionScroll
|
|
747
747
|
end
|
|
748
748
|
|
|
749
749
|
class PageActionWait
|
|
@@ -761,7 +761,7 @@ def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist:
|
|
|
761
761
|
def scrape?: () -> bool
|
|
762
762
|
|
|
763
763
|
def self.from_hash: (untyped hash) -> untyped
|
|
764
|
-
def self.new: (milliseconds: Integer
|
|
764
|
+
def self.new: (milliseconds: Integer?, selector: String?) -> PageActionWait
|
|
765
765
|
end
|
|
766
766
|
|
|
767
767
|
class PageActionScreenshot
|
|
@@ -778,7 +778,7 @@ def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?denylist:
|
|
|
778
778
|
def scrape?: () -> bool
|
|
779
779
|
|
|
780
780
|
def self.from_hash: (untyped hash) -> untyped
|
|
781
|
-
def self.new: (full_page: bool) -> PageActionScreenshot
|
|
781
|
+
def self.new: (full_page: bool?) -> PageActionScreenshot
|
|
782
782
|
end
|
|
783
783
|
|
|
784
784
|
class PageActionExecuteJs
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: crawlberg
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.13.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Na'aman Hirschfeld
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2026-
|
|
12
|
+
date: 2026-11-06 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: rb_sys
|
|
@@ -73,7 +73,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
73
73
|
requirements:
|
|
74
74
|
- - ">="
|
|
75
75
|
- !ruby/object:Gem::Version
|
|
76
|
-
version: 3.
|
|
76
|
+
version: 3.3.0
|
|
77
77
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
78
78
|
requirements:
|
|
79
79
|
- - ">="
|