crawlberg 1.4.0 → 1.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/ext/crawlberg_rb/native/Cargo.lock +18 -18
- data/ext/crawlberg_rb/native/Cargo.toml +3 -3
- data/ext/crawlberg_rb/native/extconf.rb +1 -1
- data/ext/crawlberg_rb/src/lib.rs +37 -34
- data/lib/crawlberg/native.rb +96 -47
- data/lib/crawlberg/version.rb +2 -2
- data/lib/crawlberg.rb +1 -1
- data/sig/types.rbs +404 -403
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: f597e0e18bfca18b073ac6647f09704f8f811d3083175ef73a01e1d0a467355a
|
|
4
|
+
data.tar.gz: 034ab10f2cd798d6bf32c457b7b5abbae47737332c43c492b80345fa85eef7ac
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: bf52c93eee44c22122a61090b0a7df4824674f24e7718308f24f9c0d535e7f589ddbc903f1baa915842923e460f4dcd5b9995cbf185d629d6151571a9c431947
|
|
7
|
+
data.tar.gz: 8e7e088921ac8276371fa54c61429b746d2bce838bb37e643f12e2ef579e515e08fa526eaabf6a0e0aa9b27e01c6cfa9edb97d0eff70de1771fe632c1bb36780
|
data/README.md
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
<!-- This file is auto-generated by alef — DO NOT EDIT. -->
|
|
2
|
-
<!-- alef:hash:
|
|
2
|
+
<!-- alef:hash:442342be79ebc6986eda548e4fe12a6ce7adfbf0ae84fde27ddcc064f25ff0be -->
|
|
3
3
|
<!-- To regenerate: alef readme -->
|
|
4
4
|
<!-- To verify freshness: alef verify -->
|
|
5
5
|
|
|
@@ -99,7 +99,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667"
|
|
|
99
99
|
dependencies = [
|
|
100
100
|
"proc-macro2",
|
|
101
101
|
"quote",
|
|
102
|
-
"syn 3.0.
|
|
102
|
+
"syn 3.0.4",
|
|
103
103
|
]
|
|
104
104
|
|
|
105
105
|
[[package]]
|
|
@@ -549,9 +549,9 @@ dependencies = [
|
|
|
549
549
|
|
|
550
550
|
[[package]]
|
|
551
551
|
name = "crawlberg"
|
|
552
|
-
version = "1.4.
|
|
552
|
+
version = "1.4.2"
|
|
553
553
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
554
|
-
checksum = "
|
|
554
|
+
checksum = "723dbd626a68331c8b235e0c1878d38630fc827b9df6e86bf176250e87a54520"
|
|
555
555
|
dependencies = [
|
|
556
556
|
"ahash",
|
|
557
557
|
"aho-corasick",
|
|
@@ -593,7 +593,7 @@ dependencies = [
|
|
|
593
593
|
|
|
594
594
|
[[package]]
|
|
595
595
|
name = "crawlberg-rb"
|
|
596
|
-
version = "1.4.
|
|
596
|
+
version = "1.4.2"
|
|
597
597
|
dependencies = [
|
|
598
598
|
"crawlberg",
|
|
599
599
|
"futures",
|
|
@@ -606,9 +606,9 @@ dependencies = [
|
|
|
606
606
|
|
|
607
607
|
[[package]]
|
|
608
608
|
name = "crc32fast"
|
|
609
|
-
version = "1.5.
|
|
609
|
+
version = "1.5.1"
|
|
610
610
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
611
|
-
checksum = "
|
|
611
|
+
checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550"
|
|
612
612
|
dependencies = [
|
|
613
613
|
"cfg-if",
|
|
614
614
|
]
|
|
@@ -720,7 +720,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8"
|
|
|
720
720
|
dependencies = [
|
|
721
721
|
"proc-macro2",
|
|
722
722
|
"quote",
|
|
723
|
-
"syn 3.0.
|
|
723
|
+
"syn 3.0.4",
|
|
724
724
|
]
|
|
725
725
|
|
|
726
726
|
[[package]]
|
|
@@ -886,7 +886,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44"
|
|
|
886
886
|
dependencies = [
|
|
887
887
|
"proc-macro2",
|
|
888
888
|
"quote",
|
|
889
|
-
"syn 3.0.
|
|
889
|
+
"syn 3.0.4",
|
|
890
890
|
]
|
|
891
891
|
|
|
892
892
|
[[package]]
|
|
@@ -1036,9 +1036,9 @@ checksum = "c9356095b4b41197bba32173600e1582792cda618f65d12f68e2e77d273413c5"
|
|
|
1036
1036
|
|
|
1037
1037
|
[[package]]
|
|
1038
1038
|
name = "html-to-markdown-rs"
|
|
1039
|
-
version = "3.11.
|
|
1039
|
+
version = "3.11.6"
|
|
1040
1040
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1041
|
-
checksum = "
|
|
1041
|
+
checksum = "c5fe2583aad3e87439e40d841fed327a60892bb9946da1e3a5788ee7f0f4fe3c"
|
|
1042
1042
|
dependencies = [
|
|
1043
1043
|
"ahash",
|
|
1044
1044
|
"astral-tl",
|
|
@@ -1527,9 +1527,9 @@ dependencies = [
|
|
|
1527
1527
|
|
|
1528
1528
|
[[package]]
|
|
1529
1529
|
name = "log"
|
|
1530
|
-
version = "0.4.
|
|
1530
|
+
version = "0.4.34"
|
|
1531
1531
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1532
|
-
checksum = "
|
|
1532
|
+
checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6"
|
|
1533
1533
|
|
|
1534
1534
|
[[package]]
|
|
1535
1535
|
name = "lru-slab"
|
|
@@ -2377,7 +2377,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348"
|
|
|
2377
2377
|
dependencies = [
|
|
2378
2378
|
"proc-macro2",
|
|
2379
2379
|
"quote",
|
|
2380
|
-
"syn 3.0.
|
|
2380
|
+
"syn 3.0.4",
|
|
2381
2381
|
]
|
|
2382
2382
|
|
|
2383
2383
|
[[package]]
|
|
@@ -2551,9 +2551,9 @@ dependencies = [
|
|
|
2551
2551
|
|
|
2552
2552
|
[[package]]
|
|
2553
2553
|
name = "syn"
|
|
2554
|
-
version = "3.0.
|
|
2554
|
+
version = "3.0.4"
|
|
2555
2555
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
2556
|
-
checksum = "
|
|
2556
|
+
checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f"
|
|
2557
2557
|
dependencies = [
|
|
2558
2558
|
"proc-macro2",
|
|
2559
2559
|
"quote",
|
|
@@ -2627,7 +2627,7 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af"
|
|
|
2627
2627
|
dependencies = [
|
|
2628
2628
|
"proc-macro2",
|
|
2629
2629
|
"quote",
|
|
2630
|
-
"syn 3.0.
|
|
2630
|
+
"syn 3.0.4",
|
|
2631
2631
|
]
|
|
2632
2632
|
|
|
2633
2633
|
[[package]]
|
|
@@ -2718,7 +2718,7 @@ checksum = "78773a2a397f451582ce068015985c33193cf6dea8b74d2a639fe457b2f07b0e"
|
|
|
2718
2718
|
dependencies = [
|
|
2719
2719
|
"proc-macro2",
|
|
2720
2720
|
"quote",
|
|
2721
|
-
"syn 3.0.
|
|
2721
|
+
"syn 3.0.4",
|
|
2722
2722
|
]
|
|
2723
2723
|
|
|
2724
2724
|
[[package]]
|
|
@@ -3447,7 +3447,7 @@ checksum = "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da"
|
|
|
3447
3447
|
dependencies = [
|
|
3448
3448
|
"proc-macro2",
|
|
3449
3449
|
"quote",
|
|
3450
|
-
"syn 3.0.
|
|
3450
|
+
"syn 3.0.4",
|
|
3451
3451
|
]
|
|
3452
3452
|
|
|
3453
3453
|
[[package]]
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:7cb6aa801d5f904cfc9a0079f817dfe7816c1d5b05bfd24c2eec864119ad7ee6
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
|
|
6
6
|
[package]
|
|
7
7
|
name = "crawlberg-rb"
|
|
8
|
-
version = "1.4.
|
|
8
|
+
version = "1.4.2"
|
|
9
9
|
edition = "2024"
|
|
10
10
|
license = "MIT"
|
|
11
11
|
description = "High-performance web crawling engine"
|
|
@@ -22,7 +22,7 @@ path = "../src/lib.rs"
|
|
|
22
22
|
crate-type = ["cdylib"]
|
|
23
23
|
|
|
24
24
|
[dependencies]
|
|
25
|
-
crawlberg = { version = "1.4.
|
|
25
|
+
crawlberg = { version = "1.4.2", features = ["interact", "browser-chromiumoxide"] }
|
|
26
26
|
futures = "0.3"
|
|
27
27
|
magnus = "0.8"
|
|
28
28
|
rb-sys = ">=0.9, <0.9.128"
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:5326e03e26baf331d1f85eb82de9b52903317cf694aa9d920b1b22dc6c1f009f
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
|
data/ext/crawlberg_rb/src/lib.rs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef. DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:c8bd1f23995c2c54aa56bd9e9dd95391dff81fd7eefe7a65306513caf8564400
|
|
3
3
|
// Re-generate with: alef generate
|
|
4
4
|
#![allow(dead_code, unused_imports, unused_variables)]
|
|
5
5
|
#![allow(
|
|
@@ -4482,6 +4482,7 @@ pub struct SsrfPolicy {
|
|
|
4482
4482
|
deny_private: bool,
|
|
4483
4483
|
allowlist: Vec<HostMatcher>,
|
|
4484
4484
|
max_redirects: u8,
|
|
4485
|
+
scheme_allowlist: Vec<String>,
|
|
4485
4486
|
}
|
|
4486
4487
|
|
|
4487
4488
|
unsafe impl IntoValueFromNative for SsrfPolicy {}
|
|
@@ -4537,6 +4538,14 @@ impl SsrfPolicy {
|
|
|
4537
4538
|
.get(ruby.to_symbol("max_redirects"))
|
|
4538
4539
|
.and_then(|v| u8::try_convert(v).ok())
|
|
4539
4540
|
.unwrap_or(5),
|
|
4541
|
+
scheme_allowlist: kwargs
|
|
4542
|
+
.get(ruby.to_symbol("scheme_allowlist"))
|
|
4543
|
+
.and_then(|v| <Vec<String>>::try_convert(v).ok())
|
|
4544
|
+
.unwrap_or(
|
|
4545
|
+
serde_json::from_str::<crawlberg::SsrfPolicy>(r#"{}"#)
|
|
4546
|
+
.expect("alef-generated default JSON for `SsrfPolicy` failed to deserialize")
|
|
4547
|
+
.scheme_allowlist,
|
|
4548
|
+
),
|
|
4540
4549
|
})
|
|
4541
4550
|
}
|
|
4542
4551
|
|
|
@@ -4551,6 +4560,10 @@ impl SsrfPolicy {
|
|
|
4551
4560
|
fn max_redirects(&self) -> u8 {
|
|
4552
4561
|
self.max_redirects
|
|
4553
4562
|
}
|
|
4563
|
+
|
|
4564
|
+
fn scheme_allowlist(&self) -> Vec<String> {
|
|
4565
|
+
self.scheme_allowlist.clone()
|
|
4566
|
+
}
|
|
4554
4567
|
}
|
|
4555
4568
|
|
|
4556
4569
|
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
@@ -6510,7 +6523,7 @@ impl From<MarkdownResult> for crawlberg::MarkdownResult {
|
|
|
6510
6523
|
tables: val
|
|
6511
6524
|
.tables
|
|
6512
6525
|
.into_iter()
|
|
6513
|
-
.
|
|
6526
|
+
.map(|s| serde_json::from_str(&s).unwrap_or_default())
|
|
6514
6527
|
.collect(),
|
|
6515
6528
|
warnings: val.warnings.into_iter().collect(),
|
|
6516
6529
|
citations: val.citations,
|
|
@@ -6801,6 +6814,7 @@ impl From<SsrfPolicy> for crawlberg::SsrfPolicy {
|
|
|
6801
6814
|
deny_private: val.deny_private,
|
|
6802
6815
|
allowlist: val.allowlist.into_iter().map(Into::into).collect(),
|
|
6803
6816
|
max_redirects: val.max_redirects,
|
|
6817
|
+
scheme_allowlist: val.scheme_allowlist.into_iter().collect(),
|
|
6804
6818
|
..Default::default()
|
|
6805
6819
|
}
|
|
6806
6820
|
}
|
|
@@ -6813,6 +6827,7 @@ impl From<crawlberg::SsrfPolicy> for SsrfPolicy {
|
|
|
6813
6827
|
deny_private: val.deny_private,
|
|
6814
6828
|
allowlist: val.allowlist.into_iter().map(Into::into).collect(),
|
|
6815
6829
|
max_redirects: val.max_redirects,
|
|
6830
|
+
scheme_allowlist: val.scheme_allowlist.into_iter().collect(),
|
|
6816
6831
|
}
|
|
6817
6832
|
}
|
|
6818
6833
|
}
|
|
@@ -6854,15 +6869,9 @@ impl From<crawlberg::AssetCategory> for AssetCategory {
|
|
|
6854
6869
|
impl From<AuthConfig> for crawlberg::AuthConfig {
|
|
6855
6870
|
fn from(val: AuthConfig) -> Self {
|
|
6856
6871
|
match val {
|
|
6857
|
-
AuthConfig::Basic { username, password } => Self::Basic {
|
|
6858
|
-
|
|
6859
|
-
|
|
6860
|
-
},
|
|
6861
|
-
AuthConfig::Bearer { token } => Self::Bearer { token: token },
|
|
6862
|
-
AuthConfig::Header { name, value } => Self::Header {
|
|
6863
|
-
name: name,
|
|
6864
|
-
value: value,
|
|
6865
|
-
},
|
|
6872
|
+
AuthConfig::Basic { username, password } => Self::Basic { username, password },
|
|
6873
|
+
AuthConfig::Bearer { token } => Self::Bearer { token },
|
|
6874
|
+
AuthConfig::Header { name, value } => Self::Header { name, value },
|
|
6866
6875
|
}
|
|
6867
6876
|
}
|
|
6868
6877
|
}
|
|
@@ -6971,9 +6980,7 @@ impl From<crawlberg::CrawlEvent> for CrawlEvent {
|
|
|
6971
6980
|
url: url.to_string(),
|
|
6972
6981
|
error: error.to_string(),
|
|
6973
6982
|
},
|
|
6974
|
-
crawlberg::CrawlEvent::Complete { pages_crawled } => Self::Complete {
|
|
6975
|
-
pages_crawled: pages_crawled,
|
|
6976
|
-
},
|
|
6983
|
+
crawlberg::CrawlEvent::Complete { pages_crawled } => Self::Complete { pages_crawled },
|
|
6977
6984
|
}
|
|
6978
6985
|
}
|
|
6979
6986
|
}
|
|
@@ -7039,9 +7046,9 @@ impl From<crawlberg::FeedType> for FeedType {
|
|
|
7039
7046
|
impl From<HostMatcher> for crawlberg::HostMatcher {
|
|
7040
7047
|
fn from(val: HostMatcher) -> Self {
|
|
7041
7048
|
match val {
|
|
7042
|
-
HostMatcher::Exact { value } => Self::Exact { value
|
|
7043
|
-
HostMatcher::Suffix { value } => Self::Suffix { value
|
|
7044
|
-
HostMatcher::Cidr { value } => Self::Cidr { value
|
|
7049
|
+
HostMatcher::Exact { value } => Self::Exact { value },
|
|
7050
|
+
HostMatcher::Suffix { value } => Self::Suffix { value },
|
|
7051
|
+
HostMatcher::Cidr { value } => Self::Cidr { value },
|
|
7045
7052
|
}
|
|
7046
7053
|
}
|
|
7047
7054
|
}
|
|
@@ -7109,27 +7116,21 @@ impl From<crawlberg::LinkType> for LinkType {
|
|
|
7109
7116
|
impl From<PageAction> for crawlberg::PageAction {
|
|
7110
7117
|
fn from(val: PageAction) -> Self {
|
|
7111
7118
|
match val {
|
|
7112
|
-
PageAction::Click { selector } => Self::Click { selector
|
|
7113
|
-
PageAction::TypeText { selector, text } => Self::TypeText {
|
|
7114
|
-
|
|
7115
|
-
text: text,
|
|
7116
|
-
},
|
|
7117
|
-
PageAction::Press { key } => Self::Press { key: key },
|
|
7119
|
+
PageAction::Click { selector } => Self::Click { selector },
|
|
7120
|
+
PageAction::TypeText { selector, text } => Self::TypeText { selector, text },
|
|
7121
|
+
PageAction::Press { key } => Self::Press { key },
|
|
7118
7122
|
PageAction::Scroll {
|
|
7119
7123
|
direction,
|
|
7120
7124
|
selector,
|
|
7121
7125
|
amount,
|
|
7122
7126
|
} => Self::Scroll {
|
|
7123
7127
|
direction: direction.into(),
|
|
7124
|
-
selector
|
|
7125
|
-
amount
|
|
7126
|
-
},
|
|
7127
|
-
PageAction::Wait { milliseconds, selector } => Self::Wait {
|
|
7128
|
-
milliseconds: milliseconds,
|
|
7129
|
-
selector: selector,
|
|
7128
|
+
selector,
|
|
7129
|
+
amount,
|
|
7130
7130
|
},
|
|
7131
|
-
PageAction::
|
|
7132
|
-
PageAction::
|
|
7131
|
+
PageAction::Wait { milliseconds, selector } => Self::Wait { milliseconds, selector },
|
|
7132
|
+
PageAction::Screenshot { full_page } => Self::Screenshot { full_page },
|
|
7133
|
+
PageAction::ExecuteJs { script } => Self::ExecuteJs { script },
|
|
7133
7134
|
PageAction::Scrape => Self::Scrape,
|
|
7134
7135
|
}
|
|
7135
7136
|
}
|
|
@@ -7153,13 +7154,13 @@ impl From<crawlberg::PageAction> for PageAction {
|
|
|
7153
7154
|
} => Self::Scroll {
|
|
7154
7155
|
direction: direction.into(),
|
|
7155
7156
|
selector: selector.map(|v| v.to_string()),
|
|
7156
|
-
amount
|
|
7157
|
+
amount,
|
|
7157
7158
|
},
|
|
7158
7159
|
crawlberg::PageAction::Wait { milliseconds, selector } => Self::Wait {
|
|
7159
|
-
milliseconds
|
|
7160
|
+
milliseconds,
|
|
7160
7161
|
selector: selector.map(|v| v.to_string()),
|
|
7161
7162
|
},
|
|
7162
|
-
crawlberg::PageAction::Screenshot { full_page } => Self::Screenshot { full_page
|
|
7163
|
+
crawlberg::PageAction::Screenshot { full_page } => Self::Screenshot { full_page },
|
|
7163
7164
|
crawlberg::PageAction::ExecuteJs { script } => Self::ExecuteJs {
|
|
7164
7165
|
script: script.to_string(),
|
|
7165
7166
|
},
|
|
@@ -8109,6 +8110,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
8109
8110
|
|
|
8110
8111
|
class.define_method("max_redirects", method!(SsrfPolicy::max_redirects, 0))?;
|
|
8111
8112
|
|
|
8113
|
+
class.define_method("scheme_allowlist", method!(SsrfPolicy::scheme_allowlist, 0))?;
|
|
8114
|
+
|
|
8112
8115
|
let class = module.define_class("CrawlStreamIterator", ruby.class_object())?;
|
|
8113
8116
|
class.define_method("next_chunk", method!(CrawlStreamIterator::next_chunk, 0))?;
|
|
8114
8117
|
class.define_method("each", method!(CrawlStreamIterator::each, 0))?;
|
data/lib/crawlberg/native.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:925532ba2f606ce36a0d3444d5c96407ca56a1c55bcd256372a1ea2a383d11e5
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
@@ -18,7 +18,7 @@ library_candidates = dlexts.flat_map do |dlext|
|
|
|
18
18
|
File.join(__dir__, "..", extension_name, ruby_abi, "#{extension_name}.#{dlext}"),
|
|
19
19
|
File.join(__dir__, "..", extension_name, ruby_abi, "lib#{extension_name}.#{dlext}"),
|
|
20
20
|
File.join(__dir__, "..", "#{extension_name}.#{dlext}"),
|
|
21
|
-
File.join(__dir__, "..", "lib#{extension_name}.#{dlext}")
|
|
21
|
+
File.join(__dir__, "..", "lib#{extension_name}.#{dlext}")
|
|
22
22
|
]
|
|
23
23
|
end
|
|
24
24
|
|
|
@@ -34,6 +34,7 @@ unless native_extension
|
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
require native_extension
|
|
37
|
+
|
|
37
38
|
module Crawlberg
|
|
38
39
|
# Authentication configuration.
|
|
39
40
|
module AuthConfig
|
|
@@ -49,10 +50,14 @@ module Crawlberg
|
|
|
49
50
|
def self.from_hash(hash)
|
|
50
51
|
discriminator = hash[:type] || hash["type"]
|
|
51
52
|
case discriminator
|
|
52
|
-
when "basic"
|
|
53
|
-
|
|
54
|
-
when "
|
|
55
|
-
|
|
53
|
+
when "basic"
|
|
54
|
+
AuthConfigBasic.from_hash(hash)
|
|
55
|
+
when "bearer"
|
|
56
|
+
AuthConfigBearer.from_hash(hash)
|
|
57
|
+
when "header"
|
|
58
|
+
AuthConfigHeader.from_hash(hash)
|
|
59
|
+
else
|
|
60
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
56
61
|
end
|
|
57
62
|
end
|
|
58
63
|
end
|
|
@@ -63,10 +68,12 @@ module Crawlberg
|
|
|
63
68
|
|
|
64
69
|
# Username sent in the `Authorization: Basic` header.
|
|
65
70
|
sig { returns(String) }
|
|
66
|
-
|
|
71
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
72
|
+
def username = super
|
|
67
73
|
# Password sent in the `Authorization: Basic` header.
|
|
68
74
|
sig { returns(String) }
|
|
69
|
-
|
|
75
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
76
|
+
def password = super
|
|
70
77
|
sig { returns(T::Boolean) }
|
|
71
78
|
def basic? = true
|
|
72
79
|
sig { returns(T::Boolean) }
|
|
@@ -87,7 +94,8 @@ module Crawlberg
|
|
|
87
94
|
|
|
88
95
|
# Token sent in the `Authorization: Bearer` header.
|
|
89
96
|
sig { returns(String) }
|
|
90
|
-
|
|
97
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
98
|
+
def token = super
|
|
91
99
|
sig { returns(T::Boolean) }
|
|
92
100
|
def basic? = false
|
|
93
101
|
sig { returns(T::Boolean) }
|
|
@@ -108,10 +116,12 @@ module Crawlberg
|
|
|
108
116
|
|
|
109
117
|
# HTTP header name to set on each request.
|
|
110
118
|
sig { returns(String) }
|
|
111
|
-
|
|
119
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
120
|
+
def name = super
|
|
112
121
|
# HTTP header value to send.
|
|
113
122
|
sig { returns(String) }
|
|
114
|
-
|
|
123
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
124
|
+
def value = super
|
|
115
125
|
sig { returns(T::Boolean) }
|
|
116
126
|
def basic? = false
|
|
117
127
|
sig { returns(T::Boolean) }
|
|
@@ -147,10 +157,14 @@ module Crawlberg
|
|
|
147
157
|
def self.from_hash(hash)
|
|
148
158
|
discriminator = hash[:type] || hash["type"]
|
|
149
159
|
case discriminator
|
|
150
|
-
when "page"
|
|
151
|
-
|
|
152
|
-
when "
|
|
153
|
-
|
|
160
|
+
when "page"
|
|
161
|
+
CrawlEventPage.from_hash(hash)
|
|
162
|
+
when "error"
|
|
163
|
+
CrawlEventError.from_hash(hash)
|
|
164
|
+
when "complete"
|
|
165
|
+
CrawlEventComplete.from_hash(hash)
|
|
166
|
+
else
|
|
167
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
154
168
|
end
|
|
155
169
|
end
|
|
156
170
|
end
|
|
@@ -161,7 +175,8 @@ module Crawlberg
|
|
|
161
175
|
|
|
162
176
|
# The crawled page result.
|
|
163
177
|
sig { returns(CrawlPageResult) }
|
|
164
|
-
|
|
178
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
179
|
+
def result = super
|
|
165
180
|
sig { returns(T::Boolean) }
|
|
166
181
|
def page? = true
|
|
167
182
|
sig { returns(T::Boolean) }
|
|
@@ -182,10 +197,12 @@ module Crawlberg
|
|
|
182
197
|
|
|
183
198
|
# The URL that failed.
|
|
184
199
|
sig { returns(String) }
|
|
185
|
-
|
|
200
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
201
|
+
def url = super
|
|
186
202
|
# The error message.
|
|
187
203
|
sig { returns(String) }
|
|
188
|
-
|
|
204
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
205
|
+
def error = super
|
|
189
206
|
sig { returns(T::Boolean) }
|
|
190
207
|
def page? = false
|
|
191
208
|
sig { returns(T::Boolean) }
|
|
@@ -206,7 +223,8 @@ module Crawlberg
|
|
|
206
223
|
|
|
207
224
|
# Total number of pages crawled.
|
|
208
225
|
sig { returns(Integer) }
|
|
209
|
-
|
|
226
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
227
|
+
def pages_crawled = super
|
|
210
228
|
sig { returns(T::Boolean) }
|
|
211
229
|
def page? = false
|
|
212
230
|
sig { returns(T::Boolean) }
|
|
@@ -251,10 +269,14 @@ module Crawlberg
|
|
|
251
269
|
def self.from_hash(hash)
|
|
252
270
|
discriminator = hash[:type] || hash["type"]
|
|
253
271
|
case discriminator
|
|
254
|
-
when "exact"
|
|
255
|
-
|
|
256
|
-
when "
|
|
257
|
-
|
|
272
|
+
when "exact"
|
|
273
|
+
HostMatcherExact.from_hash(hash)
|
|
274
|
+
when "suffix"
|
|
275
|
+
HostMatcherSuffix.from_hash(hash)
|
|
276
|
+
when "cidr"
|
|
277
|
+
HostMatcherCidr.from_hash(hash)
|
|
278
|
+
else
|
|
279
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
258
280
|
end
|
|
259
281
|
end
|
|
260
282
|
end
|
|
@@ -265,7 +287,8 @@ module Crawlberg
|
|
|
265
287
|
|
|
266
288
|
# The hostname to match.
|
|
267
289
|
sig { returns(String) }
|
|
268
|
-
|
|
290
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
291
|
+
def value = super
|
|
269
292
|
sig { returns(T::Boolean) }
|
|
270
293
|
def exact? = true
|
|
271
294
|
sig { returns(T::Boolean) }
|
|
@@ -286,7 +309,8 @@ module Crawlberg
|
|
|
286
309
|
|
|
287
310
|
# The dot-prefixed suffix to match. A leading dot is optional.
|
|
288
311
|
sig { returns(String) }
|
|
289
|
-
|
|
312
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
313
|
+
def value = super
|
|
290
314
|
sig { returns(T::Boolean) }
|
|
291
315
|
def exact? = false
|
|
292
316
|
sig { returns(T::Boolean) }
|
|
@@ -308,7 +332,8 @@ module Crawlberg
|
|
|
308
332
|
# The CIDR block. Validated when built through [`HostMatcher::cidr`] or
|
|
309
333
|
# deserialization.
|
|
310
334
|
sig { returns(String) }
|
|
311
|
-
|
|
335
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
336
|
+
def value = super
|
|
312
337
|
sig { returns(T::Boolean) }
|
|
313
338
|
def exact? = false
|
|
314
339
|
sig { returns(T::Boolean) }
|
|
@@ -342,15 +367,24 @@ module Crawlberg
|
|
|
342
367
|
def self.from_hash(hash)
|
|
343
368
|
discriminator = hash[:type] || hash["type"]
|
|
344
369
|
case discriminator
|
|
345
|
-
when "click"
|
|
346
|
-
|
|
347
|
-
when "
|
|
348
|
-
|
|
349
|
-
when "
|
|
350
|
-
|
|
351
|
-
when "
|
|
352
|
-
|
|
353
|
-
|
|
370
|
+
when "click"
|
|
371
|
+
PageActionClick.from_hash(hash)
|
|
372
|
+
when "type"
|
|
373
|
+
PageActionTypeText.from_hash(hash)
|
|
374
|
+
when "press"
|
|
375
|
+
PageActionPress.from_hash(hash)
|
|
376
|
+
when "scroll"
|
|
377
|
+
PageActionScroll.from_hash(hash)
|
|
378
|
+
when "wait"
|
|
379
|
+
PageActionWait.from_hash(hash)
|
|
380
|
+
when "screenshot"
|
|
381
|
+
PageActionScreenshot.from_hash(hash)
|
|
382
|
+
when "executeJs"
|
|
383
|
+
PageActionExecuteJs.from_hash(hash)
|
|
384
|
+
when "scrape"
|
|
385
|
+
PageActionScrape.from_hash(hash)
|
|
386
|
+
else
|
|
387
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
354
388
|
end
|
|
355
389
|
end
|
|
356
390
|
end
|
|
@@ -361,7 +395,8 @@ module Crawlberg
|
|
|
361
395
|
|
|
362
396
|
# CSS selector for the element to click.
|
|
363
397
|
sig { returns(String) }
|
|
364
|
-
|
|
398
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
399
|
+
def selector = super
|
|
365
400
|
sig { returns(T::Boolean) }
|
|
366
401
|
def click? = true
|
|
367
402
|
sig { returns(T::Boolean) }
|
|
@@ -392,10 +427,12 @@ module Crawlberg
|
|
|
392
427
|
|
|
393
428
|
# CSS selector for the input element.
|
|
394
429
|
sig { returns(String) }
|
|
395
|
-
|
|
430
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
431
|
+
def selector = super
|
|
396
432
|
# Text to type into the element.
|
|
397
433
|
sig { returns(String) }
|
|
398
|
-
|
|
434
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
435
|
+
def text = super
|
|
399
436
|
sig { returns(T::Boolean) }
|
|
400
437
|
def click? = false
|
|
401
438
|
sig { returns(T::Boolean) }
|
|
@@ -426,7 +463,8 @@ module Crawlberg
|
|
|
426
463
|
|
|
427
464
|
# Key name to press.
|
|
428
465
|
sig { returns(String) }
|
|
429
|
-
|
|
466
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
467
|
+
def key = super
|
|
430
468
|
sig { returns(T::Boolean) }
|
|
431
469
|
def click? = false
|
|
432
470
|
sig { returns(T::Boolean) }
|
|
@@ -457,13 +495,16 @@ module Crawlberg
|
|
|
457
495
|
|
|
458
496
|
# Direction to scroll.
|
|
459
497
|
sig { returns(ScrollDirection) }
|
|
460
|
-
|
|
498
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
499
|
+
def direction = super
|
|
461
500
|
# Optional CSS selector for a scrollable element. Scrolls the page if absent.
|
|
462
501
|
sig { returns(T.nilable(String)) }
|
|
463
|
-
|
|
502
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
503
|
+
def selector = super
|
|
464
504
|
# Optional pixel amount to scroll. Uses a default if absent.
|
|
465
505
|
sig { returns(T.nilable(Integer)) }
|
|
466
|
-
|
|
506
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
507
|
+
def amount = super
|
|
467
508
|
sig { returns(T::Boolean) }
|
|
468
509
|
def click? = false
|
|
469
510
|
sig { returns(T::Boolean) }
|
|
@@ -484,7 +525,11 @@ module Crawlberg
|
|
|
484
525
|
# @return [self]
|
|
485
526
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
486
527
|
def self.from_hash(hash)
|
|
487
|
-
new(
|
|
528
|
+
new(
|
|
529
|
+
direction: hash[:direction] || hash["direction"],
|
|
530
|
+
selector: hash[:selector] || hash["selector"],
|
|
531
|
+
amount: hash[:amount] || hash["amount"]
|
|
532
|
+
)
|
|
488
533
|
end
|
|
489
534
|
end
|
|
490
535
|
## Wait for a duration or for an element to appear.
|
|
@@ -494,10 +539,12 @@ module Crawlberg
|
|
|
494
539
|
|
|
495
540
|
# Milliseconds to wait. Ignored if `selector` is provided.
|
|
496
541
|
sig { returns(T.nilable(Integer)) }
|
|
497
|
-
|
|
542
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
543
|
+
def milliseconds = super
|
|
498
544
|
# CSS selector to wait for.
|
|
499
545
|
sig { returns(T.nilable(String)) }
|
|
500
|
-
|
|
546
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
547
|
+
def selector = super
|
|
501
548
|
sig { returns(T::Boolean) }
|
|
502
549
|
def click? = false
|
|
503
550
|
sig { returns(T::Boolean) }
|
|
@@ -532,7 +579,8 @@ module Crawlberg
|
|
|
532
579
|
# `full_page` (snake_case) alias so language bindings and fixtures can
|
|
533
580
|
# use either convention without error.
|
|
534
581
|
sig { returns(T.nilable(T::Boolean)) }
|
|
535
|
-
|
|
582
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
583
|
+
def full_page = super
|
|
536
584
|
sig { returns(T::Boolean) }
|
|
537
585
|
def click? = false
|
|
538
586
|
sig { returns(T::Boolean) }
|
|
@@ -568,7 +616,8 @@ module Crawlberg
|
|
|
568
616
|
|
|
569
617
|
# JavaScript source code to execute. Max 1 MB.
|
|
570
618
|
sig { returns(String) }
|
|
571
|
-
|
|
619
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
620
|
+
def script = super
|
|
572
621
|
sig { returns(T::Boolean) }
|
|
573
622
|
def click? = false
|
|
574
623
|
sig { returns(T::Boolean) }
|
data/lib/crawlberg/version.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:9ea9a14d259e4988466100a061689d975727c16c521c140aef2205c9b97bf831
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
module Crawlberg
|
|
8
8
|
## The version string for this package.
|
|
9
|
-
VERSION = "1.4.
|
|
9
|
+
VERSION = "1.4.2"
|
|
10
10
|
end
|