crawlberg 1.4.0 → 1.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/ext/crawlberg_rb/native/Cargo.lock +64 -47
- data/ext/crawlberg_rb/native/Cargo.toml +4 -4
- data/ext/crawlberg_rb/native/extconf.rb +1 -1
- data/ext/crawlberg_rb/src/lib.rs +92 -39
- data/lib/crawlberg/native.rb +96 -47
- data/lib/crawlberg/version.rb +2 -2
- data/lib/crawlberg.rb +1 -1
- data/sig/types.rbs +404 -403
- metadata +6 -6
data/ext/crawlberg_rb/src/lib.rs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef. DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:45a77cf71e37c29533a0d31a68c73f8d55de1a5aebed5f50b90fd1d2d0790b2f
|
|
3
3
|
// Re-generate with: alef generate
|
|
4
4
|
#![allow(dead_code, unused_imports, unused_variables)]
|
|
5
5
|
#![allow(
|
|
@@ -411,6 +411,7 @@ impl BatchCrawlResults {
|
|
|
411
411
|
}
|
|
412
412
|
}
|
|
413
413
|
|
|
414
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
414
415
|
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
|
415
416
|
#[serde(default)]
|
|
416
417
|
#[magnus::wrap(class = "Crawlberg::BatchCrawlStreamRequest")]
|
|
@@ -446,12 +447,14 @@ impl magnus::TryConvert for BatchCrawlStreamRequest {
|
|
|
446
447
|
|
|
447
448
|
unsafe impl TryConvertOwned for BatchCrawlStreamRequest {}
|
|
448
449
|
|
|
450
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
449
451
|
impl Default for BatchCrawlStreamRequest {
|
|
450
452
|
fn default() -> Self {
|
|
451
453
|
crawlberg::BatchCrawlStreamRequest::default().into()
|
|
452
454
|
}
|
|
453
455
|
}
|
|
454
456
|
|
|
457
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
455
458
|
impl BatchCrawlStreamRequest {
|
|
456
459
|
fn new(args: &[magnus::Value]) -> Result<Self, magnus::Error> {
|
|
457
460
|
let ruby = unsafe { magnus::Ruby::get_unchecked() };
|
|
@@ -634,7 +637,7 @@ impl BatchScrapeResults {
|
|
|
634
637
|
}
|
|
635
638
|
}
|
|
636
639
|
|
|
637
|
-
#[derive(Clone, Debug, serde::Serialize
|
|
640
|
+
#[derive(Clone, Debug, serde::Serialize)]
|
|
638
641
|
#[serde(default)]
|
|
639
642
|
#[magnus::wrap(class = "Crawlberg::BrowserConfig")]
|
|
640
643
|
pub struct BrowserConfig {
|
|
@@ -681,6 +684,14 @@ impl magnus::TryConvert for BrowserConfig {
|
|
|
681
684
|
}
|
|
682
685
|
|
|
683
686
|
unsafe impl TryConvertOwned for BrowserConfig {}
|
|
687
|
+
impl<'de> serde::Deserialize<'de> for BrowserConfig {
|
|
688
|
+
fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
|
|
689
|
+
where
|
|
690
|
+
D: serde::Deserializer<'de>,
|
|
691
|
+
{
|
|
692
|
+
<crawlberg::BrowserConfig as serde::Deserialize>::deserialize(deserializer).map(Into::into)
|
|
693
|
+
}
|
|
694
|
+
}
|
|
684
695
|
|
|
685
696
|
impl Default for BrowserConfig {
|
|
686
697
|
fn default() -> Self {
|
|
@@ -797,7 +808,7 @@ impl BrowserConfig {
|
|
|
797
808
|
}
|
|
798
809
|
}
|
|
799
810
|
|
|
800
|
-
#[derive(Clone, Debug, serde::Serialize
|
|
811
|
+
#[derive(Clone, Debug, serde::Serialize)]
|
|
801
812
|
#[serde(default)]
|
|
802
813
|
#[magnus::wrap(class = "Crawlberg::BrowserExtras")]
|
|
803
814
|
pub struct BrowserExtras {
|
|
@@ -833,6 +844,14 @@ impl magnus::TryConvert for BrowserExtras {
|
|
|
833
844
|
}
|
|
834
845
|
|
|
835
846
|
unsafe impl TryConvertOwned for BrowserExtras {}
|
|
847
|
+
impl<'de> serde::Deserialize<'de> for BrowserExtras {
|
|
848
|
+
fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
|
|
849
|
+
where
|
|
850
|
+
D: serde::Deserializer<'de>,
|
|
851
|
+
{
|
|
852
|
+
<crawlberg::BrowserExtras as serde::Deserialize>::deserialize(deserializer).map(Into::into)
|
|
853
|
+
}
|
|
854
|
+
}
|
|
836
855
|
|
|
837
856
|
impl Default for BrowserExtras {
|
|
838
857
|
fn default() -> Self {
|
|
@@ -1026,7 +1045,7 @@ impl CitationResult {
|
|
|
1026
1045
|
}
|
|
1027
1046
|
}
|
|
1028
1047
|
|
|
1029
|
-
#[derive(Clone, Debug, serde::Serialize
|
|
1048
|
+
#[derive(Clone, Debug, serde::Serialize)]
|
|
1030
1049
|
#[serde(default)]
|
|
1031
1050
|
#[magnus::wrap(class = "Crawlberg::ContentConfig")]
|
|
1032
1051
|
pub struct ContentConfig {
|
|
@@ -1071,6 +1090,14 @@ impl magnus::TryConvert for ContentConfig {
|
|
|
1071
1090
|
}
|
|
1072
1091
|
|
|
1073
1092
|
unsafe impl TryConvertOwned for ContentConfig {}
|
|
1093
|
+
impl<'de> serde::Deserialize<'de> for ContentConfig {
|
|
1094
|
+
fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
|
|
1095
|
+
where
|
|
1096
|
+
D: serde::Deserializer<'de>,
|
|
1097
|
+
{
|
|
1098
|
+
<crawlberg::ContentConfig as serde::Deserialize>::deserialize(deserializer).map(Into::into)
|
|
1099
|
+
}
|
|
1100
|
+
}
|
|
1074
1101
|
|
|
1075
1102
|
impl Default for ContentConfig {
|
|
1076
1103
|
fn default() -> Self {
|
|
@@ -2308,6 +2335,7 @@ impl CrawlResult {
|
|
|
2308
2335
|
}
|
|
2309
2336
|
}
|
|
2310
2337
|
|
|
2338
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
2311
2339
|
#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
|
|
2312
2340
|
#[serde(default)]
|
|
2313
2341
|
#[magnus::wrap(class = "Crawlberg::CrawlStreamRequest")]
|
|
@@ -2343,12 +2371,14 @@ impl magnus::TryConvert for CrawlStreamRequest {
|
|
|
2343
2371
|
|
|
2344
2372
|
unsafe impl TryConvertOwned for CrawlStreamRequest {}
|
|
2345
2373
|
|
|
2374
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
2346
2375
|
impl Default for CrawlStreamRequest {
|
|
2347
2376
|
fn default() -> Self {
|
|
2348
2377
|
crawlberg::CrawlStreamRequest::default().into()
|
|
2349
2378
|
}
|
|
2350
2379
|
}
|
|
2351
2380
|
|
|
2381
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
2352
2382
|
impl CrawlStreamRequest {
|
|
2353
2383
|
fn new(args: &[magnus::Value]) -> Result<Self, magnus::Error> {
|
|
2354
2384
|
let ruby = unsafe { magnus::Ruby::get_unchecked() };
|
|
@@ -2471,7 +2501,7 @@ impl DownloadedAsset {
|
|
|
2471
2501
|
}
|
|
2472
2502
|
}
|
|
2473
2503
|
|
|
2474
|
-
#[derive(Clone, Debug, serde::Serialize
|
|
2504
|
+
#[derive(Clone, Debug, serde::Serialize)]
|
|
2475
2505
|
#[serde(default)]
|
|
2476
2506
|
#[magnus::wrap(class = "Crawlberg::DownloadedDocument")]
|
|
2477
2507
|
pub struct DownloadedDocument {
|
|
@@ -2513,6 +2543,14 @@ impl magnus::TryConvert for DownloadedDocument {
|
|
|
2513
2543
|
}
|
|
2514
2544
|
|
|
2515
2545
|
unsafe impl TryConvertOwned for DownloadedDocument {}
|
|
2546
|
+
impl<'de> serde::Deserialize<'de> for DownloadedDocument {
|
|
2547
|
+
fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
|
|
2548
|
+
where
|
|
2549
|
+
D: serde::Deserializer<'de>,
|
|
2550
|
+
{
|
|
2551
|
+
<crawlberg::DownloadedDocument as serde::Deserialize>::deserialize(deserializer).map(Into::into)
|
|
2552
|
+
}
|
|
2553
|
+
}
|
|
2516
2554
|
|
|
2517
2555
|
impl Default for DownloadedDocument {
|
|
2518
2556
|
fn default() -> Self {
|
|
@@ -4475,13 +4513,14 @@ impl SitemapUrl {
|
|
|
4475
4513
|
}
|
|
4476
4514
|
}
|
|
4477
4515
|
|
|
4478
|
-
#[derive(Clone, Debug, serde::Serialize
|
|
4516
|
+
#[derive(Clone, Debug, serde::Serialize)]
|
|
4479
4517
|
#[serde(default)]
|
|
4480
4518
|
#[magnus::wrap(class = "Crawlberg::SsrfPolicy")]
|
|
4481
4519
|
pub struct SsrfPolicy {
|
|
4482
4520
|
deny_private: bool,
|
|
4483
4521
|
allowlist: Vec<HostMatcher>,
|
|
4484
4522
|
max_redirects: u8,
|
|
4523
|
+
scheme_allowlist: Vec<String>,
|
|
4485
4524
|
}
|
|
4486
4525
|
|
|
4487
4526
|
unsafe impl IntoValueFromNative for SsrfPolicy {}
|
|
@@ -4511,6 +4550,14 @@ impl magnus::TryConvert for SsrfPolicy {
|
|
|
4511
4550
|
}
|
|
4512
4551
|
|
|
4513
4552
|
unsafe impl TryConvertOwned for SsrfPolicy {}
|
|
4553
|
+
impl<'de> serde::Deserialize<'de> for SsrfPolicy {
|
|
4554
|
+
fn deserialize<D>(deserializer: D) -> ::core::result::Result<Self, D::Error>
|
|
4555
|
+
where
|
|
4556
|
+
D: serde::Deserializer<'de>,
|
|
4557
|
+
{
|
|
4558
|
+
<crawlberg::SsrfPolicy as serde::Deserialize>::deserialize(deserializer).map(Into::into)
|
|
4559
|
+
}
|
|
4560
|
+
}
|
|
4514
4561
|
|
|
4515
4562
|
impl Default for SsrfPolicy {
|
|
4516
4563
|
fn default() -> Self {
|
|
@@ -4537,6 +4584,14 @@ impl SsrfPolicy {
|
|
|
4537
4584
|
.get(ruby.to_symbol("max_redirects"))
|
|
4538
4585
|
.and_then(|v| u8::try_convert(v).ok())
|
|
4539
4586
|
.unwrap_or(5),
|
|
4587
|
+
scheme_allowlist: kwargs
|
|
4588
|
+
.get(ruby.to_symbol("scheme_allowlist"))
|
|
4589
|
+
.and_then(|v| <Vec<String>>::try_convert(v).ok())
|
|
4590
|
+
.unwrap_or(
|
|
4591
|
+
serde_json::from_str::<crawlberg::SsrfPolicy>(r#"{}"#)
|
|
4592
|
+
.expect("alef-generated default JSON for `SsrfPolicy` failed to deserialize")
|
|
4593
|
+
.scheme_allowlist,
|
|
4594
|
+
),
|
|
4540
4595
|
})
|
|
4541
4596
|
}
|
|
4542
4597
|
|
|
@@ -4551,6 +4606,10 @@ impl SsrfPolicy {
|
|
|
4551
4606
|
fn max_redirects(&self) -> u8 {
|
|
4552
4607
|
self.max_redirects
|
|
4553
4608
|
}
|
|
4609
|
+
|
|
4610
|
+
fn scheme_allowlist(&self) -> Vec<String> {
|
|
4611
|
+
self.scheme_allowlist.clone()
|
|
4612
|
+
}
|
|
4554
4613
|
}
|
|
4555
4614
|
|
|
4556
4615
|
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
@@ -5713,6 +5772,7 @@ impl From<crawlberg::BatchCrawlResults> for BatchCrawlResults {
|
|
|
5713
5772
|
}
|
|
5714
5773
|
}
|
|
5715
5774
|
|
|
5775
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
5716
5776
|
#[allow(clippy::needless_update)]
|
|
5717
5777
|
impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
|
|
5718
5778
|
fn from(val: BatchCrawlStreamRequest) -> Self {
|
|
@@ -5723,6 +5783,7 @@ impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
|
|
|
5723
5783
|
}
|
|
5724
5784
|
}
|
|
5725
5785
|
|
|
5786
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
5726
5787
|
impl From<crawlberg::BatchCrawlStreamRequest> for BatchCrawlStreamRequest {
|
|
5727
5788
|
fn from(val: crawlberg::BatchCrawlStreamRequest) -> Self {
|
|
5728
5789
|
Self {
|
|
@@ -6172,6 +6233,7 @@ impl From<crawlberg::CrawlResult> for CrawlResult {
|
|
|
6172
6233
|
}
|
|
6173
6234
|
}
|
|
6174
6235
|
|
|
6236
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
6175
6237
|
#[allow(clippy::needless_update)]
|
|
6176
6238
|
impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
|
|
6177
6239
|
fn from(val: CrawlStreamRequest) -> Self {
|
|
@@ -6182,6 +6244,7 @@ impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
|
|
|
6182
6244
|
}
|
|
6183
6245
|
}
|
|
6184
6246
|
|
|
6247
|
+
#[cfg(not(target_arch = "wasm32"))]
|
|
6185
6248
|
impl From<crawlberg::CrawlStreamRequest> for CrawlStreamRequest {
|
|
6186
6249
|
fn from(val: crawlberg::CrawlStreamRequest) -> Self {
|
|
6187
6250
|
Self {
|
|
@@ -6510,7 +6573,7 @@ impl From<MarkdownResult> for crawlberg::MarkdownResult {
|
|
|
6510
6573
|
tables: val
|
|
6511
6574
|
.tables
|
|
6512
6575
|
.into_iter()
|
|
6513
|
-
.
|
|
6576
|
+
.map(|s| serde_json::from_str(&s).unwrap_or_default())
|
|
6514
6577
|
.collect(),
|
|
6515
6578
|
warnings: val.warnings.into_iter().collect(),
|
|
6516
6579
|
citations: val.citations,
|
|
@@ -6801,6 +6864,7 @@ impl From<SsrfPolicy> for crawlberg::SsrfPolicy {
|
|
|
6801
6864
|
deny_private: val.deny_private,
|
|
6802
6865
|
allowlist: val.allowlist.into_iter().map(Into::into).collect(),
|
|
6803
6866
|
max_redirects: val.max_redirects,
|
|
6867
|
+
scheme_allowlist: val.scheme_allowlist.into_iter().collect(),
|
|
6804
6868
|
..Default::default()
|
|
6805
6869
|
}
|
|
6806
6870
|
}
|
|
@@ -6813,6 +6877,7 @@ impl From<crawlberg::SsrfPolicy> for SsrfPolicy {
|
|
|
6813
6877
|
deny_private: val.deny_private,
|
|
6814
6878
|
allowlist: val.allowlist.into_iter().map(Into::into).collect(),
|
|
6815
6879
|
max_redirects: val.max_redirects,
|
|
6880
|
+
scheme_allowlist: val.scheme_allowlist.into_iter().collect(),
|
|
6816
6881
|
}
|
|
6817
6882
|
}
|
|
6818
6883
|
}
|
|
@@ -6854,15 +6919,9 @@ impl From<crawlberg::AssetCategory> for AssetCategory {
|
|
|
6854
6919
|
impl From<AuthConfig> for crawlberg::AuthConfig {
|
|
6855
6920
|
fn from(val: AuthConfig) -> Self {
|
|
6856
6921
|
match val {
|
|
6857
|
-
AuthConfig::Basic { username, password } => Self::Basic {
|
|
6858
|
-
|
|
6859
|
-
|
|
6860
|
-
},
|
|
6861
|
-
AuthConfig::Bearer { token } => Self::Bearer { token: token },
|
|
6862
|
-
AuthConfig::Header { name, value } => Self::Header {
|
|
6863
|
-
name: name,
|
|
6864
|
-
value: value,
|
|
6865
|
-
},
|
|
6922
|
+
AuthConfig::Basic { username, password } => Self::Basic { username, password },
|
|
6923
|
+
AuthConfig::Bearer { token } => Self::Bearer { token },
|
|
6924
|
+
AuthConfig::Header { name, value } => Self::Header { name, value },
|
|
6866
6925
|
}
|
|
6867
6926
|
}
|
|
6868
6927
|
}
|
|
@@ -6971,9 +7030,7 @@ impl From<crawlberg::CrawlEvent> for CrawlEvent {
|
|
|
6971
7030
|
url: url.to_string(),
|
|
6972
7031
|
error: error.to_string(),
|
|
6973
7032
|
},
|
|
6974
|
-
crawlberg::CrawlEvent::Complete { pages_crawled } => Self::Complete {
|
|
6975
|
-
pages_crawled: pages_crawled,
|
|
6976
|
-
},
|
|
7033
|
+
crawlberg::CrawlEvent::Complete { pages_crawled } => Self::Complete { pages_crawled },
|
|
6977
7034
|
}
|
|
6978
7035
|
}
|
|
6979
7036
|
}
|
|
@@ -7039,9 +7096,9 @@ impl From<crawlberg::FeedType> for FeedType {
|
|
|
7039
7096
|
impl From<HostMatcher> for crawlberg::HostMatcher {
|
|
7040
7097
|
fn from(val: HostMatcher) -> Self {
|
|
7041
7098
|
match val {
|
|
7042
|
-
HostMatcher::Exact { value } => Self::Exact { value
|
|
7043
|
-
HostMatcher::Suffix { value } => Self::Suffix { value
|
|
7044
|
-
HostMatcher::Cidr { value } => Self::Cidr { value
|
|
7099
|
+
HostMatcher::Exact { value } => Self::Exact { value },
|
|
7100
|
+
HostMatcher::Suffix { value } => Self::Suffix { value },
|
|
7101
|
+
HostMatcher::Cidr { value } => Self::Cidr { value },
|
|
7045
7102
|
}
|
|
7046
7103
|
}
|
|
7047
7104
|
}
|
|
@@ -7109,27 +7166,21 @@ impl From<crawlberg::LinkType> for LinkType {
|
|
|
7109
7166
|
impl From<PageAction> for crawlberg::PageAction {
|
|
7110
7167
|
fn from(val: PageAction) -> Self {
|
|
7111
7168
|
match val {
|
|
7112
|
-
PageAction::Click { selector } => Self::Click { selector
|
|
7113
|
-
PageAction::TypeText { selector, text } => Self::TypeText {
|
|
7114
|
-
|
|
7115
|
-
text: text,
|
|
7116
|
-
},
|
|
7117
|
-
PageAction::Press { key } => Self::Press { key: key },
|
|
7169
|
+
PageAction::Click { selector } => Self::Click { selector },
|
|
7170
|
+
PageAction::TypeText { selector, text } => Self::TypeText { selector, text },
|
|
7171
|
+
PageAction::Press { key } => Self::Press { key },
|
|
7118
7172
|
PageAction::Scroll {
|
|
7119
7173
|
direction,
|
|
7120
7174
|
selector,
|
|
7121
7175
|
amount,
|
|
7122
7176
|
} => Self::Scroll {
|
|
7123
7177
|
direction: direction.into(),
|
|
7124
|
-
selector
|
|
7125
|
-
amount
|
|
7126
|
-
},
|
|
7127
|
-
PageAction::Wait { milliseconds, selector } => Self::Wait {
|
|
7128
|
-
milliseconds: milliseconds,
|
|
7129
|
-
selector: selector,
|
|
7178
|
+
selector,
|
|
7179
|
+
amount,
|
|
7130
7180
|
},
|
|
7131
|
-
PageAction::
|
|
7132
|
-
PageAction::
|
|
7181
|
+
PageAction::Wait { milliseconds, selector } => Self::Wait { milliseconds, selector },
|
|
7182
|
+
PageAction::Screenshot { full_page } => Self::Screenshot { full_page },
|
|
7183
|
+
PageAction::ExecuteJs { script } => Self::ExecuteJs { script },
|
|
7133
7184
|
PageAction::Scrape => Self::Scrape,
|
|
7134
7185
|
}
|
|
7135
7186
|
}
|
|
@@ -7153,13 +7204,13 @@ impl From<crawlberg::PageAction> for PageAction {
|
|
|
7153
7204
|
} => Self::Scroll {
|
|
7154
7205
|
direction: direction.into(),
|
|
7155
7206
|
selector: selector.map(|v| v.to_string()),
|
|
7156
|
-
amount
|
|
7207
|
+
amount,
|
|
7157
7208
|
},
|
|
7158
7209
|
crawlberg::PageAction::Wait { milliseconds, selector } => Self::Wait {
|
|
7159
|
-
milliseconds
|
|
7210
|
+
milliseconds,
|
|
7160
7211
|
selector: selector.map(|v| v.to_string()),
|
|
7161
7212
|
},
|
|
7162
|
-
crawlberg::PageAction::Screenshot { full_page } => Self::Screenshot { full_page
|
|
7213
|
+
crawlberg::PageAction::Screenshot { full_page } => Self::Screenshot { full_page },
|
|
7163
7214
|
crawlberg::PageAction::ExecuteJs { script } => Self::ExecuteJs {
|
|
7164
7215
|
script: script.to_string(),
|
|
7165
7216
|
},
|
|
@@ -8109,6 +8160,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
|
|
|
8109
8160
|
|
|
8110
8161
|
class.define_method("max_redirects", method!(SsrfPolicy::max_redirects, 0))?;
|
|
8111
8162
|
|
|
8163
|
+
class.define_method("scheme_allowlist", method!(SsrfPolicy::scheme_allowlist, 0))?;
|
|
8164
|
+
|
|
8112
8165
|
let class = module.define_class("CrawlStreamIterator", ruby.class_object())?;
|
|
8113
8166
|
class.define_method("next_chunk", method!(CrawlStreamIterator::next_chunk, 0))?;
|
|
8114
8167
|
class.define_method("each", method!(CrawlStreamIterator::each, 0))?;
|
data/lib/crawlberg/native.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:925532ba2f606ce36a0d3444d5c96407ca56a1c55bcd256372a1ea2a383d11e5
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
@@ -18,7 +18,7 @@ library_candidates = dlexts.flat_map do |dlext|
|
|
|
18
18
|
File.join(__dir__, "..", extension_name, ruby_abi, "#{extension_name}.#{dlext}"),
|
|
19
19
|
File.join(__dir__, "..", extension_name, ruby_abi, "lib#{extension_name}.#{dlext}"),
|
|
20
20
|
File.join(__dir__, "..", "#{extension_name}.#{dlext}"),
|
|
21
|
-
File.join(__dir__, "..", "lib#{extension_name}.#{dlext}")
|
|
21
|
+
File.join(__dir__, "..", "lib#{extension_name}.#{dlext}")
|
|
22
22
|
]
|
|
23
23
|
end
|
|
24
24
|
|
|
@@ -34,6 +34,7 @@ unless native_extension
|
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
require native_extension
|
|
37
|
+
|
|
37
38
|
module Crawlberg
|
|
38
39
|
# Authentication configuration.
|
|
39
40
|
module AuthConfig
|
|
@@ -49,10 +50,14 @@ module Crawlberg
|
|
|
49
50
|
def self.from_hash(hash)
|
|
50
51
|
discriminator = hash[:type] || hash["type"]
|
|
51
52
|
case discriminator
|
|
52
|
-
when "basic"
|
|
53
|
-
|
|
54
|
-
when "
|
|
55
|
-
|
|
53
|
+
when "basic"
|
|
54
|
+
AuthConfigBasic.from_hash(hash)
|
|
55
|
+
when "bearer"
|
|
56
|
+
AuthConfigBearer.from_hash(hash)
|
|
57
|
+
when "header"
|
|
58
|
+
AuthConfigHeader.from_hash(hash)
|
|
59
|
+
else
|
|
60
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
56
61
|
end
|
|
57
62
|
end
|
|
58
63
|
end
|
|
@@ -63,10 +68,12 @@ module Crawlberg
|
|
|
63
68
|
|
|
64
69
|
# Username sent in the `Authorization: Basic` header.
|
|
65
70
|
sig { returns(String) }
|
|
66
|
-
|
|
71
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
72
|
+
def username = super
|
|
67
73
|
# Password sent in the `Authorization: Basic` header.
|
|
68
74
|
sig { returns(String) }
|
|
69
|
-
|
|
75
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
76
|
+
def password = super
|
|
70
77
|
sig { returns(T::Boolean) }
|
|
71
78
|
def basic? = true
|
|
72
79
|
sig { returns(T::Boolean) }
|
|
@@ -87,7 +94,8 @@ module Crawlberg
|
|
|
87
94
|
|
|
88
95
|
# Token sent in the `Authorization: Bearer` header.
|
|
89
96
|
sig { returns(String) }
|
|
90
|
-
|
|
97
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
98
|
+
def token = super
|
|
91
99
|
sig { returns(T::Boolean) }
|
|
92
100
|
def basic? = false
|
|
93
101
|
sig { returns(T::Boolean) }
|
|
@@ -108,10 +116,12 @@ module Crawlberg
|
|
|
108
116
|
|
|
109
117
|
# HTTP header name to set on each request.
|
|
110
118
|
sig { returns(String) }
|
|
111
|
-
|
|
119
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
120
|
+
def name = super
|
|
112
121
|
# HTTP header value to send.
|
|
113
122
|
sig { returns(String) }
|
|
114
|
-
|
|
123
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
124
|
+
def value = super
|
|
115
125
|
sig { returns(T::Boolean) }
|
|
116
126
|
def basic? = false
|
|
117
127
|
sig { returns(T::Boolean) }
|
|
@@ -147,10 +157,14 @@ module Crawlberg
|
|
|
147
157
|
def self.from_hash(hash)
|
|
148
158
|
discriminator = hash[:type] || hash["type"]
|
|
149
159
|
case discriminator
|
|
150
|
-
when "page"
|
|
151
|
-
|
|
152
|
-
when "
|
|
153
|
-
|
|
160
|
+
when "page"
|
|
161
|
+
CrawlEventPage.from_hash(hash)
|
|
162
|
+
when "error"
|
|
163
|
+
CrawlEventError.from_hash(hash)
|
|
164
|
+
when "complete"
|
|
165
|
+
CrawlEventComplete.from_hash(hash)
|
|
166
|
+
else
|
|
167
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
154
168
|
end
|
|
155
169
|
end
|
|
156
170
|
end
|
|
@@ -161,7 +175,8 @@ module Crawlberg
|
|
|
161
175
|
|
|
162
176
|
# The crawled page result.
|
|
163
177
|
sig { returns(CrawlPageResult) }
|
|
164
|
-
|
|
178
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
179
|
+
def result = super
|
|
165
180
|
sig { returns(T::Boolean) }
|
|
166
181
|
def page? = true
|
|
167
182
|
sig { returns(T::Boolean) }
|
|
@@ -182,10 +197,12 @@ module Crawlberg
|
|
|
182
197
|
|
|
183
198
|
# The URL that failed.
|
|
184
199
|
sig { returns(String) }
|
|
185
|
-
|
|
200
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
201
|
+
def url = super
|
|
186
202
|
# The error message.
|
|
187
203
|
sig { returns(String) }
|
|
188
|
-
|
|
204
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
205
|
+
def error = super
|
|
189
206
|
sig { returns(T::Boolean) }
|
|
190
207
|
def page? = false
|
|
191
208
|
sig { returns(T::Boolean) }
|
|
@@ -206,7 +223,8 @@ module Crawlberg
|
|
|
206
223
|
|
|
207
224
|
# Total number of pages crawled.
|
|
208
225
|
sig { returns(Integer) }
|
|
209
|
-
|
|
226
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
227
|
+
def pages_crawled = super
|
|
210
228
|
sig { returns(T::Boolean) }
|
|
211
229
|
def page? = false
|
|
212
230
|
sig { returns(T::Boolean) }
|
|
@@ -251,10 +269,14 @@ module Crawlberg
|
|
|
251
269
|
def self.from_hash(hash)
|
|
252
270
|
discriminator = hash[:type] || hash["type"]
|
|
253
271
|
case discriminator
|
|
254
|
-
when "exact"
|
|
255
|
-
|
|
256
|
-
when "
|
|
257
|
-
|
|
272
|
+
when "exact"
|
|
273
|
+
HostMatcherExact.from_hash(hash)
|
|
274
|
+
when "suffix"
|
|
275
|
+
HostMatcherSuffix.from_hash(hash)
|
|
276
|
+
when "cidr"
|
|
277
|
+
HostMatcherCidr.from_hash(hash)
|
|
278
|
+
else
|
|
279
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
258
280
|
end
|
|
259
281
|
end
|
|
260
282
|
end
|
|
@@ -265,7 +287,8 @@ module Crawlberg
|
|
|
265
287
|
|
|
266
288
|
# The hostname to match.
|
|
267
289
|
sig { returns(String) }
|
|
268
|
-
|
|
290
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
291
|
+
def value = super
|
|
269
292
|
sig { returns(T::Boolean) }
|
|
270
293
|
def exact? = true
|
|
271
294
|
sig { returns(T::Boolean) }
|
|
@@ -286,7 +309,8 @@ module Crawlberg
|
|
|
286
309
|
|
|
287
310
|
# The dot-prefixed suffix to match. A leading dot is optional.
|
|
288
311
|
sig { returns(String) }
|
|
289
|
-
|
|
312
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
313
|
+
def value = super
|
|
290
314
|
sig { returns(T::Boolean) }
|
|
291
315
|
def exact? = false
|
|
292
316
|
sig { returns(T::Boolean) }
|
|
@@ -308,7 +332,8 @@ module Crawlberg
|
|
|
308
332
|
# The CIDR block. Validated when built through [`HostMatcher::cidr`] or
|
|
309
333
|
# deserialization.
|
|
310
334
|
sig { returns(String) }
|
|
311
|
-
|
|
335
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
336
|
+
def value = super
|
|
312
337
|
sig { returns(T::Boolean) }
|
|
313
338
|
def exact? = false
|
|
314
339
|
sig { returns(T::Boolean) }
|
|
@@ -342,15 +367,24 @@ module Crawlberg
|
|
|
342
367
|
def self.from_hash(hash)
|
|
343
368
|
discriminator = hash[:type] || hash["type"]
|
|
344
369
|
case discriminator
|
|
345
|
-
when "click"
|
|
346
|
-
|
|
347
|
-
when "
|
|
348
|
-
|
|
349
|
-
when "
|
|
350
|
-
|
|
351
|
-
when "
|
|
352
|
-
|
|
353
|
-
|
|
370
|
+
when "click"
|
|
371
|
+
PageActionClick.from_hash(hash)
|
|
372
|
+
when "type"
|
|
373
|
+
PageActionTypeText.from_hash(hash)
|
|
374
|
+
when "press"
|
|
375
|
+
PageActionPress.from_hash(hash)
|
|
376
|
+
when "scroll"
|
|
377
|
+
PageActionScroll.from_hash(hash)
|
|
378
|
+
when "wait"
|
|
379
|
+
PageActionWait.from_hash(hash)
|
|
380
|
+
when "screenshot"
|
|
381
|
+
PageActionScreenshot.from_hash(hash)
|
|
382
|
+
when "executeJs"
|
|
383
|
+
PageActionExecuteJs.from_hash(hash)
|
|
384
|
+
when "scrape"
|
|
385
|
+
PageActionScrape.from_hash(hash)
|
|
386
|
+
else
|
|
387
|
+
raise "Unknown discriminator: #{discriminator}"
|
|
354
388
|
end
|
|
355
389
|
end
|
|
356
390
|
end
|
|
@@ -361,7 +395,8 @@ module Crawlberg
|
|
|
361
395
|
|
|
362
396
|
# CSS selector for the element to click.
|
|
363
397
|
sig { returns(String) }
|
|
364
|
-
|
|
398
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
399
|
+
def selector = super
|
|
365
400
|
sig { returns(T::Boolean) }
|
|
366
401
|
def click? = true
|
|
367
402
|
sig { returns(T::Boolean) }
|
|
@@ -392,10 +427,12 @@ module Crawlberg
|
|
|
392
427
|
|
|
393
428
|
# CSS selector for the input element.
|
|
394
429
|
sig { returns(String) }
|
|
395
|
-
|
|
430
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
431
|
+
def selector = super
|
|
396
432
|
# Text to type into the element.
|
|
397
433
|
sig { returns(String) }
|
|
398
|
-
|
|
434
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
435
|
+
def text = super
|
|
399
436
|
sig { returns(T::Boolean) }
|
|
400
437
|
def click? = false
|
|
401
438
|
sig { returns(T::Boolean) }
|
|
@@ -426,7 +463,8 @@ module Crawlberg
|
|
|
426
463
|
|
|
427
464
|
# Key name to press.
|
|
428
465
|
sig { returns(String) }
|
|
429
|
-
|
|
466
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
467
|
+
def key = super
|
|
430
468
|
sig { returns(T::Boolean) }
|
|
431
469
|
def click? = false
|
|
432
470
|
sig { returns(T::Boolean) }
|
|
@@ -457,13 +495,16 @@ module Crawlberg
|
|
|
457
495
|
|
|
458
496
|
# Direction to scroll.
|
|
459
497
|
sig { returns(ScrollDirection) }
|
|
460
|
-
|
|
498
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
499
|
+
def direction = super
|
|
461
500
|
# Optional CSS selector for a scrollable element. Scrolls the page if absent.
|
|
462
501
|
sig { returns(T.nilable(String)) }
|
|
463
|
-
|
|
502
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
503
|
+
def selector = super
|
|
464
504
|
# Optional pixel amount to scroll. Uses a default if absent.
|
|
465
505
|
sig { returns(T.nilable(Integer)) }
|
|
466
|
-
|
|
506
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
507
|
+
def amount = super
|
|
467
508
|
sig { returns(T::Boolean) }
|
|
468
509
|
def click? = false
|
|
469
510
|
sig { returns(T::Boolean) }
|
|
@@ -484,7 +525,11 @@ module Crawlberg
|
|
|
484
525
|
# @return [self]
|
|
485
526
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.attached_class) }
|
|
486
527
|
def self.from_hash(hash)
|
|
487
|
-
new(
|
|
528
|
+
new(
|
|
529
|
+
direction: hash[:direction] || hash["direction"],
|
|
530
|
+
selector: hash[:selector] || hash["selector"],
|
|
531
|
+
amount: hash[:amount] || hash["amount"]
|
|
532
|
+
)
|
|
488
533
|
end
|
|
489
534
|
end
|
|
490
535
|
## Wait for a duration or for an element to appear.
|
|
@@ -494,10 +539,12 @@ module Crawlberg
|
|
|
494
539
|
|
|
495
540
|
# Milliseconds to wait. Ignored if `selector` is provided.
|
|
496
541
|
sig { returns(T.nilable(Integer)) }
|
|
497
|
-
|
|
542
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
543
|
+
def milliseconds = super
|
|
498
544
|
# CSS selector to wait for.
|
|
499
545
|
sig { returns(T.nilable(String)) }
|
|
500
|
-
|
|
546
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
547
|
+
def selector = super
|
|
501
548
|
sig { returns(T::Boolean) }
|
|
502
549
|
def click? = false
|
|
503
550
|
sig { returns(T::Boolean) }
|
|
@@ -532,7 +579,8 @@ module Crawlberg
|
|
|
532
579
|
# `full_page` (snake_case) alias so language bindings and fixtures can
|
|
533
580
|
# use either convention without error.
|
|
534
581
|
sig { returns(T.nilable(T::Boolean)) }
|
|
535
|
-
|
|
582
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
583
|
+
def full_page = super
|
|
536
584
|
sig { returns(T::Boolean) }
|
|
537
585
|
def click? = false
|
|
538
586
|
sig { returns(T::Boolean) }
|
|
@@ -568,7 +616,8 @@ module Crawlberg
|
|
|
568
616
|
|
|
569
617
|
# JavaScript source code to execute. Max 1 MB.
|
|
570
618
|
sig { returns(String) }
|
|
571
|
-
|
|
619
|
+
# rubocop:disable Lint/UselessMethodDefinition
|
|
620
|
+
def script = super
|
|
572
621
|
sig { returns(T::Boolean) }
|
|
573
622
|
def click? = false
|
|
574
623
|
sig { returns(T::Boolean) }
|
data/lib/crawlberg/version.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:7234c62471e6f1812ca4c07d492c86782a033ac7e8d8fc64dcbb4dc0ca728eeb
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
module Crawlberg
|
|
8
8
|
## The version string for this package.
|
|
9
|
-
VERSION = "1.
|
|
9
|
+
VERSION = "1.5.0"
|
|
10
10
|
end
|