crawlberg 1.7.0 → 1.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/crawlberg_rb/native/Cargo.lock +7 -7
- data/ext/crawlberg_rb/native/Cargo.toml +4 -4
- data/ext/crawlberg_rb/src/lib.rs +114 -37
- data/lib/crawlberg/native.rb +24 -47
- data/lib/crawlberg/version.rb +2 -2
- data/sig/types.rbs +29 -51
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 154f92c622f76efb16d88fb2075772251365a09de713990bbb1b63668ee22065
|
|
4
|
+
data.tar.gz: e60ce1d8f947cf1ea3afe0b3cf203467dc2fb1dc5d2a942bdf166a5ff2e198ea
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: dea0dc85482926a5bd305ba5176ef280e6eca5d192ceb4ce4c24bcd49cb93383ec1358b3e3940ae712aadd22fbe32373efccc6032bbdeee89b1472281b598f33
|
|
7
|
+
data.tar.gz: a6ed384d2269b3dab4ca43ec5e5f5417b26e2e0391eb0329a64686d2d06e4f0b8bf89de0f5dccf3ca0a123114c0cc47aa4684c6dd6433ed0a7bc0b6a6f462d2c
|
|
@@ -555,9 +555,9 @@ dependencies = [
|
|
|
555
555
|
|
|
556
556
|
[[package]]
|
|
557
557
|
name = "crawlberg"
|
|
558
|
-
version = "1.7.
|
|
558
|
+
version = "1.7.2"
|
|
559
559
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
560
|
-
checksum = "
|
|
560
|
+
checksum = "f65b7f99dea838e39d35c03e0acef6b16103a9626881e07b846d027fcd02d62f"
|
|
561
561
|
dependencies = [
|
|
562
562
|
"ahash",
|
|
563
563
|
"aho-corasick",
|
|
@@ -599,7 +599,7 @@ dependencies = [
|
|
|
599
599
|
|
|
600
600
|
[[package]]
|
|
601
601
|
name = "crawlberg-rb"
|
|
602
|
-
version = "1.7.
|
|
602
|
+
version = "1.7.2"
|
|
603
603
|
dependencies = [
|
|
604
604
|
"crawlberg",
|
|
605
605
|
"futures",
|
|
@@ -1553,9 +1553,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
|
|
|
1553
1553
|
|
|
1554
1554
|
[[package]]
|
|
1555
1555
|
name = "magnus"
|
|
1556
|
-
version = "0.
|
|
1556
|
+
version = "0.9.0"
|
|
1557
1557
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1558
|
-
checksum = "
|
|
1558
|
+
checksum = "379d50b6eea4ca84074ccd3e551eb9b07f7865194d5b3198325317e9ef78e5e9"
|
|
1559
1559
|
dependencies = [
|
|
1560
1560
|
"magnus-macros",
|
|
1561
1561
|
"rb-sys",
|
|
@@ -1565,9 +1565,9 @@ dependencies = [
|
|
|
1565
1565
|
|
|
1566
1566
|
[[package]]
|
|
1567
1567
|
name = "magnus-macros"
|
|
1568
|
-
version = "0.
|
|
1568
|
+
version = "0.9.0"
|
|
1569
1569
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
1570
|
-
checksum = "
|
|
1570
|
+
checksum = "4ca70566af98184bb92d4d9853daaf533c1a3418b9570d0547ea44970035f3fe"
|
|
1571
1571
|
dependencies = [
|
|
1572
1572
|
"proc-macro2",
|
|
1573
1573
|
"quote",
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:5335ae796018deac59899b2632bbcbf919394a15070c42cd96d6ad1719190d45
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
|
|
6
6
|
[package]
|
|
7
7
|
name = "crawlberg-rb"
|
|
8
|
-
version = "1.7.
|
|
8
|
+
version = "1.7.2"
|
|
9
9
|
edition = "2024"
|
|
10
10
|
license = "MIT"
|
|
11
11
|
description = "High-performance web crawling engine"
|
|
@@ -22,9 +22,9 @@ path = "../src/lib.rs"
|
|
|
22
22
|
crate-type = ["cdylib"]
|
|
23
23
|
|
|
24
24
|
[dependencies]
|
|
25
|
-
crawlberg = { version = "1.7.
|
|
25
|
+
crawlberg = { version = "1.7.2", features = ["interact", "browser-chromiumoxide"] }
|
|
26
26
|
futures = "0.3"
|
|
27
|
-
magnus = "0.
|
|
27
|
+
magnus = "0.9"
|
|
28
28
|
rb-sys = ">=0.9.130, <0.10"
|
|
29
29
|
serde = { version = "1", features = ["derive"] }
|
|
30
30
|
serde_json = "1"
|
data/ext/crawlberg_rb/src/lib.rs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef. DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:9563e80a11ae24fda8f480033a40a3dd9382715d77f7a2e60c254f7f3ab71ad5
|
|
3
3
|
// Re-generate with: alef generate
|
|
4
4
|
#![allow(dead_code, unused_imports, unused_variables)]
|
|
5
5
|
#![allow(
|
|
@@ -6254,7 +6254,7 @@ impl SsrfPolicy {
|
|
|
6254
6254
|
}
|
|
6255
6255
|
}
|
|
6256
6256
|
|
|
6257
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
6257
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
6258
6258
|
#[serde(rename_all = "snake_case")]
|
|
6259
6259
|
pub enum AssetCategory {
|
|
6260
6260
|
Document,
|
|
@@ -6295,7 +6295,14 @@ impl magnus::IntoValue for AssetCategory {
|
|
|
6295
6295
|
|
|
6296
6296
|
impl magnus::TryConvert for AssetCategory {
|
|
6297
6297
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6298
|
-
|
|
6298
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
6299
|
+
// Accept that exact representation as well as String inputs; do not
|
|
6300
|
+
// coerce arbitrary objects through to_s.
|
|
6301
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
6302
|
+
symbol.name()?.into_owned()
|
|
6303
|
+
} else {
|
|
6304
|
+
magnus::TryConvert::try_convert(val)?
|
|
6305
|
+
};
|
|
6299
6306
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
6300
6307
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
6301
6308
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -6352,7 +6359,8 @@ impl magnus::IntoValue for AuthConfig {
|
|
|
6352
6359
|
|
|
6353
6360
|
impl magnus::TryConvert for AuthConfig {
|
|
6354
6361
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6355
|
-
//
|
|
6362
|
+
// Generated Data variants carry native DTOs, not JSON-compatible display strings.
|
|
6363
|
+
let class_name: Option<String> = val.class().funcall("name", ())?; // For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
|
|
6356
6364
|
// For unit enums or when passed as a string, fall back to string-based conversion.
|
|
6357
6365
|
let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
|
|
6358
6366
|
s
|
|
@@ -6384,9 +6392,10 @@ impl magnus::TryConvert for AuthConfig {
|
|
|
6384
6392
|
|
|
6385
6393
|
unsafe impl IntoValueFromNative for AuthConfig {}
|
|
6386
6394
|
unsafe impl TryConvertOwned for AuthConfig {}
|
|
6395
|
+
|
|
6387
6396
|
impl AuthConfig {}
|
|
6388
6397
|
|
|
6389
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
6398
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
6390
6399
|
#[serde(rename_all = "snake_case")]
|
|
6391
6400
|
pub enum BrowserBackend {
|
|
6392
6401
|
Chromiumoxide,
|
|
@@ -6411,7 +6420,14 @@ impl magnus::IntoValue for BrowserBackend {
|
|
|
6411
6420
|
|
|
6412
6421
|
impl magnus::TryConvert for BrowserBackend {
|
|
6413
6422
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6414
|
-
|
|
6423
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
6424
|
+
// Accept that exact representation as well as String inputs; do not
|
|
6425
|
+
// coerce arbitrary objects through to_s.
|
|
6426
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
6427
|
+
symbol.name()?.into_owned()
|
|
6428
|
+
} else {
|
|
6429
|
+
magnus::TryConvert::try_convert(val)?
|
|
6430
|
+
};
|
|
6415
6431
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
6416
6432
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
6417
6433
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -6429,7 +6445,7 @@ impl magnus::TryConvert for BrowserBackend {
|
|
|
6429
6445
|
unsafe impl IntoValueFromNative for BrowserBackend {}
|
|
6430
6446
|
unsafe impl TryConvertOwned for BrowserBackend {}
|
|
6431
6447
|
|
|
6432
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
6448
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
6433
6449
|
#[serde(rename_all = "snake_case")]
|
|
6434
6450
|
pub enum BrowserMode {
|
|
6435
6451
|
Auto,
|
|
@@ -6458,7 +6474,14 @@ impl magnus::IntoValue for BrowserMode {
|
|
|
6458
6474
|
|
|
6459
6475
|
impl magnus::TryConvert for BrowserMode {
|
|
6460
6476
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6461
|
-
|
|
6477
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
6478
|
+
// Accept that exact representation as well as String inputs; do not
|
|
6479
|
+
// coerce arbitrary objects through to_s.
|
|
6480
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
6481
|
+
symbol.name()?.into_owned()
|
|
6482
|
+
} else {
|
|
6483
|
+
magnus::TryConvert::try_convert(val)?
|
|
6484
|
+
};
|
|
6462
6485
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
6463
6486
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
6464
6487
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -6478,7 +6501,7 @@ impl magnus::TryConvert for BrowserMode {
|
|
|
6478
6501
|
unsafe impl IntoValueFromNative for BrowserMode {}
|
|
6479
6502
|
unsafe impl TryConvertOwned for BrowserMode {}
|
|
6480
6503
|
|
|
6481
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
6504
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
6482
6505
|
#[serde(rename_all = "snake_case")]
|
|
6483
6506
|
pub enum BrowserWait {
|
|
6484
6507
|
NetworkIdle,
|
|
@@ -6505,7 +6528,14 @@ impl magnus::IntoValue for BrowserWait {
|
|
|
6505
6528
|
|
|
6506
6529
|
impl magnus::TryConvert for BrowserWait {
|
|
6507
6530
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6508
|
-
|
|
6531
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
6532
|
+
// Accept that exact representation as well as String inputs; do not
|
|
6533
|
+
// coerce arbitrary objects through to_s.
|
|
6534
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
6535
|
+
symbol.name()?.into_owned()
|
|
6536
|
+
} else {
|
|
6537
|
+
magnus::TryConvert::try_convert(val)?
|
|
6538
|
+
};
|
|
6509
6539
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
6510
6540
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
6511
6541
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -6524,7 +6554,7 @@ impl magnus::TryConvert for BrowserWait {
|
|
|
6524
6554
|
unsafe impl IntoValueFromNative for BrowserWait {}
|
|
6525
6555
|
unsafe impl TryConvertOwned for BrowserWait {}
|
|
6526
6556
|
|
|
6527
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
6557
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
6528
6558
|
#[serde(rename_all = "snake_case")]
|
|
6529
6559
|
pub enum ContentFilterKind {
|
|
6530
6560
|
Bm25,
|
|
@@ -6547,7 +6577,14 @@ impl magnus::IntoValue for ContentFilterKind {
|
|
|
6547
6577
|
|
|
6548
6578
|
impl magnus::TryConvert for ContentFilterKind {
|
|
6549
6579
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6550
|
-
|
|
6580
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
6581
|
+
// Accept that exact representation as well as String inputs; do not
|
|
6582
|
+
// coerce arbitrary objects through to_s.
|
|
6583
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
6584
|
+
symbol.name()?.into_owned()
|
|
6585
|
+
} else {
|
|
6586
|
+
magnus::TryConvert::try_convert(val)?
|
|
6587
|
+
};
|
|
6551
6588
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
6552
6589
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
6553
6590
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -6568,7 +6605,7 @@ unsafe impl TryConvertOwned for ContentFilterKind {}
|
|
|
6568
6605
|
#[serde(tag = "type")]
|
|
6569
6606
|
#[serde(rename_all = "snake_case")]
|
|
6570
6607
|
pub enum CrawlEvent {
|
|
6571
|
-
Page { result: CrawlPageResult },
|
|
6608
|
+
Page { result: Box<CrawlPageResult> },
|
|
6572
6609
|
Error { url: String, error: String },
|
|
6573
6610
|
Complete { pages_crawled: usize },
|
|
6574
6611
|
}
|
|
@@ -6592,7 +6629,8 @@ impl magnus::IntoValue for CrawlEvent {
|
|
|
6592
6629
|
|
|
6593
6630
|
impl magnus::TryConvert for CrawlEvent {
|
|
6594
6631
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6595
|
-
//
|
|
6632
|
+
// Generated Data variants carry native DTOs, not JSON-compatible display strings.
|
|
6633
|
+
let class_name: Option<String> = val.class().funcall("name", ())?; // For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
|
|
6596
6634
|
// For unit enums or when passed as a string, fall back to string-based conversion.
|
|
6597
6635
|
let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
|
|
6598
6636
|
s
|
|
@@ -6624,9 +6662,10 @@ impl magnus::TryConvert for CrawlEvent {
|
|
|
6624
6662
|
|
|
6625
6663
|
unsafe impl IntoValueFromNative for CrawlEvent {}
|
|
6626
6664
|
unsafe impl TryConvertOwned for CrawlEvent {}
|
|
6665
|
+
|
|
6627
6666
|
impl CrawlEvent {}
|
|
6628
6667
|
|
|
6629
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
6668
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
6630
6669
|
#[serde(rename_all = "snake_case")]
|
|
6631
6670
|
pub enum CrawlStrategyKind {
|
|
6632
6671
|
Bfs,
|
|
@@ -6655,7 +6694,14 @@ impl magnus::IntoValue for CrawlStrategyKind {
|
|
|
6655
6694
|
|
|
6656
6695
|
impl magnus::TryConvert for CrawlStrategyKind {
|
|
6657
6696
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6658
|
-
|
|
6697
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
6698
|
+
// Accept that exact representation as well as String inputs; do not
|
|
6699
|
+
// coerce arbitrary objects through to_s.
|
|
6700
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
6701
|
+
symbol.name()?.into_owned()
|
|
6702
|
+
} else {
|
|
6703
|
+
magnus::TryConvert::try_convert(val)?
|
|
6704
|
+
};
|
|
6659
6705
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
6660
6706
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
6661
6707
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -6675,7 +6721,7 @@ impl magnus::TryConvert for CrawlStrategyKind {
|
|
|
6675
6721
|
unsafe impl IntoValueFromNative for CrawlStrategyKind {}
|
|
6676
6722
|
unsafe impl TryConvertOwned for CrawlStrategyKind {}
|
|
6677
6723
|
|
|
6678
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
6724
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
6679
6725
|
#[serde(rename_all = "snake_case")]
|
|
6680
6726
|
pub enum DocumentContentEncoding {
|
|
6681
6727
|
Base64,
|
|
@@ -6698,7 +6744,14 @@ impl magnus::IntoValue for DocumentContentEncoding {
|
|
|
6698
6744
|
|
|
6699
6745
|
impl magnus::TryConvert for DocumentContentEncoding {
|
|
6700
6746
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6701
|
-
|
|
6747
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
6748
|
+
// Accept that exact representation as well as String inputs; do not
|
|
6749
|
+
// coerce arbitrary objects through to_s.
|
|
6750
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
6751
|
+
symbol.name()?.into_owned()
|
|
6752
|
+
} else {
|
|
6753
|
+
magnus::TryConvert::try_convert(val)?
|
|
6754
|
+
};
|
|
6702
6755
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
6703
6756
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
6704
6757
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -6715,7 +6768,7 @@ impl magnus::TryConvert for DocumentContentEncoding {
|
|
|
6715
6768
|
unsafe impl IntoValueFromNative for DocumentContentEncoding {}
|
|
6716
6769
|
unsafe impl TryConvertOwned for DocumentContentEncoding {}
|
|
6717
6770
|
|
|
6718
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
6771
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
6719
6772
|
#[serde(rename_all = "snake_case")]
|
|
6720
6773
|
pub enum FeedType {
|
|
6721
6774
|
Rss,
|
|
@@ -6742,7 +6795,14 @@ impl magnus::IntoValue for FeedType {
|
|
|
6742
6795
|
|
|
6743
6796
|
impl magnus::TryConvert for FeedType {
|
|
6744
6797
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6745
|
-
|
|
6798
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
6799
|
+
// Accept that exact representation as well as String inputs; do not
|
|
6800
|
+
// coerce arbitrary objects through to_s.
|
|
6801
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
6802
|
+
symbol.name()?.into_owned()
|
|
6803
|
+
} else {
|
|
6804
|
+
magnus::TryConvert::try_convert(val)?
|
|
6805
|
+
};
|
|
6746
6806
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
6747
6807
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
6748
6808
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -6770,14 +6830,6 @@ pub enum HostMatcher {
|
|
|
6770
6830
|
Cidr { value: String },
|
|
6771
6831
|
}
|
|
6772
6832
|
|
|
6773
|
-
impl Default for HostMatcher {
|
|
6774
|
-
fn default() -> Self {
|
|
6775
|
-
Self::Exact {
|
|
6776
|
-
value: Default::default(),
|
|
6777
|
-
}
|
|
6778
|
-
}
|
|
6779
|
-
}
|
|
6780
|
-
|
|
6781
6833
|
impl magnus::IntoValue for HostMatcher {
|
|
6782
6834
|
fn into_value_with(self, handle: &Ruby) -> magnus::Value {
|
|
6783
6835
|
match serde_json::to_value(&self) {
|
|
@@ -6789,7 +6841,8 @@ impl magnus::IntoValue for HostMatcher {
|
|
|
6789
6841
|
|
|
6790
6842
|
impl magnus::TryConvert for HostMatcher {
|
|
6791
6843
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6792
|
-
//
|
|
6844
|
+
// Generated Data variants carry native DTOs, not JSON-compatible display strings.
|
|
6845
|
+
let class_name: Option<String> = val.class().funcall("name", ())?; // For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
|
|
6793
6846
|
// For unit enums or when passed as a string, fall back to string-based conversion.
|
|
6794
6847
|
let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
|
|
6795
6848
|
s
|
|
@@ -6821,9 +6874,10 @@ impl magnus::TryConvert for HostMatcher {
|
|
|
6821
6874
|
|
|
6822
6875
|
unsafe impl IntoValueFromNative for HostMatcher {}
|
|
6823
6876
|
unsafe impl TryConvertOwned for HostMatcher {}
|
|
6877
|
+
|
|
6824
6878
|
impl HostMatcher {}
|
|
6825
6879
|
|
|
6826
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
6880
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
6827
6881
|
#[serde(rename_all = "snake_case")]
|
|
6828
6882
|
pub enum ImageSource {
|
|
6829
6883
|
Img,
|
|
@@ -6854,7 +6908,14 @@ impl magnus::IntoValue for ImageSource {
|
|
|
6854
6908
|
|
|
6855
6909
|
impl magnus::TryConvert for ImageSource {
|
|
6856
6910
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6857
|
-
|
|
6911
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
6912
|
+
// Accept that exact representation as well as String inputs; do not
|
|
6913
|
+
// coerce arbitrary objects through to_s.
|
|
6914
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
6915
|
+
symbol.name()?.into_owned()
|
|
6916
|
+
} else {
|
|
6917
|
+
magnus::TryConvert::try_convert(val)?
|
|
6918
|
+
};
|
|
6858
6919
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
6859
6920
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
6860
6921
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -6874,7 +6935,7 @@ impl magnus::TryConvert for ImageSource {
|
|
|
6874
6935
|
unsafe impl IntoValueFromNative for ImageSource {}
|
|
6875
6936
|
unsafe impl TryConvertOwned for ImageSource {}
|
|
6876
6937
|
|
|
6877
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
6938
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
6878
6939
|
#[serde(rename_all = "snake_case")]
|
|
6879
6940
|
pub enum LinkType {
|
|
6880
6941
|
Internal,
|
|
@@ -6903,7 +6964,14 @@ impl magnus::IntoValue for LinkType {
|
|
|
6903
6964
|
|
|
6904
6965
|
impl magnus::TryConvert for LinkType {
|
|
6905
6966
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6906
|
-
|
|
6967
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
6968
|
+
// Accept that exact representation as well as String inputs; do not
|
|
6969
|
+
// coerce arbitrary objects through to_s.
|
|
6970
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
6971
|
+
symbol.name()?.into_owned()
|
|
6972
|
+
} else {
|
|
6973
|
+
magnus::TryConvert::try_convert(val)?
|
|
6974
|
+
};
|
|
6907
6975
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
6908
6976
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
6909
6977
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -6974,7 +7042,8 @@ impl magnus::IntoValue for PageAction {
|
|
|
6974
7042
|
|
|
6975
7043
|
impl magnus::TryConvert for PageAction {
|
|
6976
7044
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
6977
|
-
//
|
|
7045
|
+
// Generated Data variants carry native DTOs, not JSON-compatible display strings.
|
|
7046
|
+
let class_name: Option<String> = val.class().funcall("name", ())?; // For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
|
|
6978
7047
|
// For unit enums or when passed as a string, fall back to string-based conversion.
|
|
6979
7048
|
let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
|
|
6980
7049
|
s
|
|
@@ -7006,9 +7075,10 @@ impl magnus::TryConvert for PageAction {
|
|
|
7006
7075
|
|
|
7007
7076
|
unsafe impl IntoValueFromNative for PageAction {}
|
|
7008
7077
|
unsafe impl TryConvertOwned for PageAction {}
|
|
7078
|
+
|
|
7009
7079
|
impl PageAction {}
|
|
7010
7080
|
|
|
7011
|
-
#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
|
|
7081
|
+
#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug, serde::Serialize, serde::Deserialize)]
|
|
7012
7082
|
#[serde(rename_all = "camelCase")]
|
|
7013
7083
|
pub enum ScrollDirection {
|
|
7014
7084
|
Up,
|
|
@@ -7033,7 +7103,14 @@ impl magnus::IntoValue for ScrollDirection {
|
|
|
7033
7103
|
|
|
7034
7104
|
impl magnus::TryConvert for ScrollDirection {
|
|
7035
7105
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
7036
|
-
|
|
7106
|
+
// IntoValue emits Symbols, including when this enum is a Hash key.
|
|
7107
|
+
// Accept that exact representation as well as String inputs; do not
|
|
7108
|
+
// coerce arbitrary objects through to_s.
|
|
7109
|
+
let s: String = if let Some(symbol) = magnus::Symbol::from_value(val) {
|
|
7110
|
+
symbol.name()?.into_owned()
|
|
7111
|
+
} else {
|
|
7112
|
+
magnus::TryConvert::try_convert(val)?
|
|
7113
|
+
};
|
|
7037
7114
|
// Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
|
|
7038
7115
|
// emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
|
|
7039
7116
|
// existing consumer code written in any of those styles keep working.
|
|
@@ -8619,7 +8696,7 @@ impl From<crawlberg::CrawlEvent> for CrawlEvent {
|
|
|
8619
8696
|
fn from(val: crawlberg::CrawlEvent) -> Self {
|
|
8620
8697
|
match val {
|
|
8621
8698
|
crawlberg::CrawlEvent::Page { result } => Self::Page {
|
|
8622
|
-
result: (*result).into(),
|
|
8699
|
+
result: Box::new((*result).into()),
|
|
8623
8700
|
},
|
|
8624
8701
|
crawlberg::CrawlEvent::Error { url, error } => Self::Error {
|
|
8625
8702
|
url: url.to_string(),
|
data/lib/crawlberg/native.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:d323ed0aab9764791b48678b68c8fedcf8e7b3e1dbe50ec190a9023e68acc3bb
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
@@ -68,12 +68,10 @@ module Crawlberg
|
|
|
68
68
|
|
|
69
69
|
# Username sent in the `Authorization: Basic` header.
|
|
70
70
|
sig { returns(String) }
|
|
71
|
-
|
|
72
|
-
def username = super
|
|
71
|
+
def username = Data.instance_method(:to_h).bind_call(self).fetch(:username)
|
|
73
72
|
# Password sent in the `Authorization: Basic` header.
|
|
74
73
|
sig { returns(String) }
|
|
75
|
-
|
|
76
|
-
def password = super
|
|
74
|
+
def password = Data.instance_method(:to_h).bind_call(self).fetch(:password)
|
|
77
75
|
sig { returns(T::Boolean) }
|
|
78
76
|
def basic? = true
|
|
79
77
|
sig { returns(T::Boolean) }
|
|
@@ -94,8 +92,7 @@ module Crawlberg
|
|
|
94
92
|
|
|
95
93
|
# Token sent in the `Authorization: Bearer` header.
|
|
96
94
|
sig { returns(String) }
|
|
97
|
-
|
|
98
|
-
def token = super
|
|
95
|
+
def token = Data.instance_method(:to_h).bind_call(self).fetch(:token)
|
|
99
96
|
sig { returns(T::Boolean) }
|
|
100
97
|
def basic? = false
|
|
101
98
|
sig { returns(T::Boolean) }
|
|
@@ -116,12 +113,10 @@ module Crawlberg
|
|
|
116
113
|
|
|
117
114
|
# HTTP header name to set on each request.
|
|
118
115
|
sig { returns(String) }
|
|
119
|
-
|
|
120
|
-
def name = super
|
|
116
|
+
def name = Data.instance_method(:to_h).bind_call(self).fetch(:name)
|
|
121
117
|
# HTTP header value to send.
|
|
122
118
|
sig { returns(String) }
|
|
123
|
-
|
|
124
|
-
def value = super
|
|
119
|
+
def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
|
|
125
120
|
sig { returns(T::Boolean) }
|
|
126
121
|
def basic? = false
|
|
127
122
|
sig { returns(T::Boolean) }
|
|
@@ -175,8 +170,7 @@ module Crawlberg
|
|
|
175
170
|
|
|
176
171
|
# The crawled page result.
|
|
177
172
|
sig { returns(CrawlPageResult) }
|
|
178
|
-
|
|
179
|
-
def result = super
|
|
173
|
+
def result = Data.instance_method(:to_h).bind_call(self).fetch(:result)
|
|
180
174
|
sig { returns(T::Boolean) }
|
|
181
175
|
def page? = true
|
|
182
176
|
sig { returns(T::Boolean) }
|
|
@@ -197,12 +191,10 @@ module Crawlberg
|
|
|
197
191
|
|
|
198
192
|
# The URL that failed.
|
|
199
193
|
sig { returns(String) }
|
|
200
|
-
|
|
201
|
-
def url = super
|
|
194
|
+
def url = Data.instance_method(:to_h).bind_call(self).fetch(:url)
|
|
202
195
|
# The error message.
|
|
203
196
|
sig { returns(String) }
|
|
204
|
-
|
|
205
|
-
def error = super
|
|
197
|
+
def error = Data.instance_method(:to_h).bind_call(self).fetch(:error)
|
|
206
198
|
sig { returns(T::Boolean) }
|
|
207
199
|
def page? = false
|
|
208
200
|
sig { returns(T::Boolean) }
|
|
@@ -223,8 +215,7 @@ module Crawlberg
|
|
|
223
215
|
|
|
224
216
|
# Total number of pages crawled.
|
|
225
217
|
sig { returns(Integer) }
|
|
226
|
-
|
|
227
|
-
def pages_crawled = super
|
|
218
|
+
def pages_crawled = Data.instance_method(:to_h).bind_call(self).fetch(:pages_crawled)
|
|
228
219
|
sig { returns(T::Boolean) }
|
|
229
220
|
def page? = false
|
|
230
221
|
sig { returns(T::Boolean) }
|
|
@@ -287,8 +278,7 @@ module Crawlberg
|
|
|
287
278
|
|
|
288
279
|
# The hostname to match.
|
|
289
280
|
sig { returns(String) }
|
|
290
|
-
|
|
291
|
-
def value = super
|
|
281
|
+
def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
|
|
292
282
|
sig { returns(T::Boolean) }
|
|
293
283
|
def exact? = true
|
|
294
284
|
sig { returns(T::Boolean) }
|
|
@@ -309,8 +299,7 @@ module Crawlberg
|
|
|
309
299
|
|
|
310
300
|
# The dot-prefixed suffix to match. A leading dot is optional.
|
|
311
301
|
sig { returns(String) }
|
|
312
|
-
|
|
313
|
-
def value = super
|
|
302
|
+
def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
|
|
314
303
|
sig { returns(T::Boolean) }
|
|
315
304
|
def exact? = false
|
|
316
305
|
sig { returns(T::Boolean) }
|
|
@@ -332,8 +321,7 @@ module Crawlberg
|
|
|
332
321
|
# The CIDR block. Validated when built through [`HostMatcher::cidr`] or
|
|
333
322
|
# deserialization.
|
|
334
323
|
sig { returns(String) }
|
|
335
|
-
|
|
336
|
-
def value = super
|
|
324
|
+
def value = Data.instance_method(:to_h).bind_call(self).fetch(:value)
|
|
337
325
|
sig { returns(T::Boolean) }
|
|
338
326
|
def exact? = false
|
|
339
327
|
sig { returns(T::Boolean) }
|
|
@@ -395,8 +383,7 @@ module Crawlberg
|
|
|
395
383
|
|
|
396
384
|
# CSS selector for the element to click.
|
|
397
385
|
sig { returns(String) }
|
|
398
|
-
|
|
399
|
-
def selector = super
|
|
386
|
+
def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
|
|
400
387
|
sig { returns(T::Boolean) }
|
|
401
388
|
def click? = true
|
|
402
389
|
sig { returns(T::Boolean) }
|
|
@@ -427,12 +414,10 @@ module Crawlberg
|
|
|
427
414
|
|
|
428
415
|
# CSS selector for the input element.
|
|
429
416
|
sig { returns(String) }
|
|
430
|
-
|
|
431
|
-
def selector = super
|
|
417
|
+
def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
|
|
432
418
|
# Text to type into the element.
|
|
433
419
|
sig { returns(String) }
|
|
434
|
-
|
|
435
|
-
def text = super
|
|
420
|
+
def text = Data.instance_method(:to_h).bind_call(self).fetch(:text)
|
|
436
421
|
sig { returns(T::Boolean) }
|
|
437
422
|
def click? = false
|
|
438
423
|
sig { returns(T::Boolean) }
|
|
@@ -463,8 +448,7 @@ module Crawlberg
|
|
|
463
448
|
|
|
464
449
|
# Key name to press.
|
|
465
450
|
sig { returns(String) }
|
|
466
|
-
|
|
467
|
-
def key = super
|
|
451
|
+
def key = Data.instance_method(:to_h).bind_call(self).fetch(:key)
|
|
468
452
|
sig { returns(T::Boolean) }
|
|
469
453
|
def click? = false
|
|
470
454
|
sig { returns(T::Boolean) }
|
|
@@ -495,16 +479,13 @@ module Crawlberg
|
|
|
495
479
|
|
|
496
480
|
# Direction to scroll.
|
|
497
481
|
sig { returns(ScrollDirection) }
|
|
498
|
-
|
|
499
|
-
def direction = super
|
|
482
|
+
def direction = Data.instance_method(:to_h).bind_call(self).fetch(:direction)
|
|
500
483
|
# Optional CSS selector for a scrollable element. Scrolls the page if absent.
|
|
501
484
|
sig { returns(T.nilable(String)) }
|
|
502
|
-
|
|
503
|
-
def selector = super
|
|
485
|
+
def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
|
|
504
486
|
# Optional pixel amount to scroll. Uses a default if absent.
|
|
505
487
|
sig { returns(T.nilable(Integer)) }
|
|
506
|
-
|
|
507
|
-
def amount = super
|
|
488
|
+
def amount = Data.instance_method(:to_h).bind_call(self).fetch(:amount)
|
|
508
489
|
sig { returns(T::Boolean) }
|
|
509
490
|
def click? = false
|
|
510
491
|
sig { returns(T::Boolean) }
|
|
@@ -539,12 +520,10 @@ module Crawlberg
|
|
|
539
520
|
|
|
540
521
|
# Milliseconds to wait. Ignored if `selector` is provided.
|
|
541
522
|
sig { returns(T.nilable(Integer)) }
|
|
542
|
-
|
|
543
|
-
def milliseconds = super
|
|
523
|
+
def milliseconds = Data.instance_method(:to_h).bind_call(self).fetch(:milliseconds)
|
|
544
524
|
# CSS selector to wait for.
|
|
545
525
|
sig { returns(T.nilable(String)) }
|
|
546
|
-
|
|
547
|
-
def selector = super
|
|
526
|
+
def selector = Data.instance_method(:to_h).bind_call(self).fetch(:selector)
|
|
548
527
|
sig { returns(T::Boolean) }
|
|
549
528
|
def click? = false
|
|
550
529
|
sig { returns(T::Boolean) }
|
|
@@ -579,8 +558,7 @@ module Crawlberg
|
|
|
579
558
|
# `full_page` (snake_case) alias so language bindings and fixtures can
|
|
580
559
|
# use either convention without error.
|
|
581
560
|
sig { returns(T.nilable(T::Boolean)) }
|
|
582
|
-
|
|
583
|
-
def full_page = super
|
|
561
|
+
def full_page = Data.instance_method(:to_h).bind_call(self).fetch(:full_page)
|
|
584
562
|
sig { returns(T::Boolean) }
|
|
585
563
|
def click? = false
|
|
586
564
|
sig { returns(T::Boolean) }
|
|
@@ -616,8 +594,7 @@ module Crawlberg
|
|
|
616
594
|
|
|
617
595
|
# JavaScript source code to execute. Max 1 MB.
|
|
618
596
|
sig { returns(String) }
|
|
619
|
-
|
|
620
|
-
def script = super
|
|
597
|
+
def script = Data.instance_method(:to_h).bind_call(self).fetch(:script)
|
|
621
598
|
sig { returns(T::Boolean) }
|
|
622
599
|
def click? = false
|
|
623
600
|
sig { returns(T::Boolean) }
|
data/lib/crawlberg/version.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:fdaf5ebf5fb30639d0b25b5f62a7044ed59029769d49609135c3bfd37b6f18e8
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
module Crawlberg
|
|
8
8
|
## The version string for this package.
|
|
9
|
-
VERSION = "1.7.
|
|
9
|
+
VERSION = "1.7.2"
|
|
10
10
|
end
|
data/sig/types.rbs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:ddb9c01b1ccd8b583329fc083873cbeaec5fed7790bc44b0db41df5ccc4b7d91
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify
|
|
5
5
|
|
|
@@ -70,11 +70,11 @@ def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?com
|
|
|
70
70
|
end
|
|
71
71
|
|
|
72
72
|
class BrowserConfig
|
|
73
|
-
attr_reader mode:
|
|
74
|
-
attr_reader backend:
|
|
73
|
+
attr_reader mode: enum_BrowserMode
|
|
74
|
+
attr_reader backend: enum_BrowserBackend
|
|
75
75
|
attr_reader endpoint: String?
|
|
76
76
|
attr_reader timeout: Integer
|
|
77
|
-
attr_reader wait:
|
|
77
|
+
attr_reader wait: enum_BrowserWait
|
|
78
78
|
attr_reader wait_selector: String?
|
|
79
79
|
attr_reader extra_wait: Integer?
|
|
80
80
|
attr_reader proxy: ProxyConfig?
|
|
@@ -84,7 +84,7 @@ def initialize: (?results: Array[BatchScrapeResult], ?total_count: Integer, ?com
|
|
|
84
84
|
attr_reader capture_network_events: bool
|
|
85
85
|
attr_reader session_affinity: bool
|
|
86
86
|
|
|
87
|
-
def initialize: (?mode:
|
|
87
|
+
def initialize: (?mode: enum_BrowserMode, ?backend: enum_BrowserBackend, ?endpoint: String, ?timeout: Integer, ?wait: enum_BrowserWait, ?wait_selector: String, ?extra_wait: Integer, ?proxy: ProxyConfig, ?block_url_patterns: Array[String], ?eval_script: String, ?robots_user_agent: String, ?capture_network_events: bool, ?session_affinity: bool) -> void
|
|
88
88
|
end
|
|
89
89
|
|
|
90
90
|
class BrowserExtras
|
|
@@ -141,8 +141,8 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
|
|
|
141
141
|
attr_reader max_pages: Integer?
|
|
142
142
|
attr_reader max_links_per_page: Integer?
|
|
143
143
|
attr_reader max_concurrent: Integer?
|
|
144
|
-
attr_reader crawl_strategy:
|
|
145
|
-
attr_reader content_filter:
|
|
144
|
+
attr_reader crawl_strategy: enum_CrawlStrategyKind
|
|
145
|
+
attr_reader content_filter: enum_ContentFilterKind?
|
|
146
146
|
attr_reader bm25_query: String?
|
|
147
147
|
attr_reader bm25_threshold: Float?
|
|
148
148
|
attr_reader respect_robots_txt: bool
|
|
@@ -166,7 +166,7 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
|
|
|
166
166
|
attr_reader map_limit: Integer?
|
|
167
167
|
attr_reader map_search: String?
|
|
168
168
|
attr_reader download_assets: bool
|
|
169
|
-
attr_reader asset_types: Array[
|
|
169
|
+
attr_reader asset_types: Array[enum_AssetCategory]
|
|
170
170
|
attr_reader max_asset_size: Integer?
|
|
171
171
|
attr_reader browser: BrowserConfig
|
|
172
172
|
attr_reader proxy: ProxyConfig?
|
|
@@ -178,14 +178,14 @@ def initialize: (?output_format: String, ?preprocessing_preset: String, ?remove_
|
|
|
178
178
|
attr_reader document_max_size: Integer?
|
|
179
179
|
attr_reader document_mime_types: Array[String]
|
|
180
180
|
attr_reader document_output_dir: String?
|
|
181
|
-
attr_reader document_content_encoding:
|
|
181
|
+
attr_reader document_content_encoding: enum_DocumentContentEncoding?
|
|
182
182
|
attr_reader warc_output: String?
|
|
183
183
|
attr_reader browser_profile: String?
|
|
184
184
|
attr_reader save_browser_profile: bool
|
|
185
185
|
attr_reader ssrf: SsrfPolicy
|
|
186
186
|
attr_reader ssrf_deny_private_explicit: bool?
|
|
187
187
|
|
|
188
|
-
def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy:
|
|
188
|
+
def initialize: (?max_depth: Integer, ?max_pages: Integer, ?max_links_per_page: Integer, ?max_concurrent: Integer, ?crawl_strategy: enum_CrawlStrategyKind, ?content_filter: enum_ContentFilterKind, ?bm25_query: String, ?bm25_threshold: Float, ?respect_robots_txt: bool, ?soft_http_errors: bool, ?user_agent: String, ?stay_on_domain: bool, ?allow_subdomains: bool, ?include_paths: Array[String], ?exclude_paths: Array[String], ?custom_headers: Hash[String, String], ?request_timeout: Integer, ?rate_limit_ms: Integer, ?max_redirects: Integer, ?retry_count: Integer, ?retry_codes: Array[Integer], ?cookies_enabled: bool, ?auth: AuthConfig, ?max_body_size: Integer, ?remove_tags: Array[String], ?content: ContentConfig, ?map_limit: Integer, ?map_search: String, ?download_assets: bool, ?asset_types: Array[enum_AssetCategory], ?max_asset_size: Integer, ?browser: BrowserConfig, ?proxy: ProxyConfig, ?user_agents: Array[String], ?capture_screenshot: bool, ?follow_document_urls: bool, ?document_url_depth: Integer, ?download_documents: bool, ?document_max_size: Integer, ?document_mime_types: Array[String], ?document_output_dir: String, ?document_content_encoding: enum_DocumentContentEncoding, ?warc_output: String, ?browser_profile: String, ?save_browser_profile: bool, ?ssrf: SsrfPolicy, ?ssrf_deny_private_explicit: bool) -> void
|
|
189
189
|
def validate: () -> void
|
|
190
190
|
end
|
|
191
191
|
|
|
@@ -245,10 +245,10 @@ def initialize: (?pages: Array[CrawlPageResult], ?final_url: String, ?redirect_c
|
|
|
245
245
|
attr_reader content_hash: String
|
|
246
246
|
attr_reader mime_type: String?
|
|
247
247
|
attr_reader size: Integer
|
|
248
|
-
attr_reader asset_category:
|
|
248
|
+
attr_reader asset_category: enum_AssetCategory
|
|
249
249
|
attr_reader html_tag: String?
|
|
250
250
|
|
|
251
|
-
def initialize: (?url: String, ?content_hash: String, ?mime_type: String, ?size: Integer, ?asset_category:
|
|
251
|
+
def initialize: (?url: String, ?content_hash: String, ?mime_type: String, ?size: Integer, ?asset_category: enum_AssetCategory, ?html_tag: String) -> void
|
|
252
252
|
end
|
|
253
253
|
|
|
254
254
|
class DownloadedDocument
|
|
@@ -287,9 +287,9 @@ def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: St
|
|
|
287
287
|
class FeedInfo
|
|
288
288
|
attr_reader url: String
|
|
289
289
|
attr_reader title: String?
|
|
290
|
-
attr_reader feed_type:
|
|
290
|
+
attr_reader feed_type: enum_FeedType
|
|
291
291
|
|
|
292
|
-
def initialize: (?url: String, ?title: String, ?feed_type:
|
|
292
|
+
def initialize: (?url: String, ?title: String, ?feed_type: enum_FeedType) -> void
|
|
293
293
|
end
|
|
294
294
|
|
|
295
295
|
class HeadingInfo
|
|
@@ -311,9 +311,9 @@ def initialize: (?url: String, ?mime_type: String, ?size: Integer, ?filename: St
|
|
|
311
311
|
attr_reader alt: String?
|
|
312
312
|
attr_reader width: Integer?
|
|
313
313
|
attr_reader height: Integer?
|
|
314
|
-
attr_reader source:
|
|
314
|
+
attr_reader source: enum_ImageSource
|
|
315
315
|
|
|
316
|
-
def initialize: (?url: String, ?alt: String, ?width: Integer, ?height: Integer, ?source:
|
|
316
|
+
def initialize: (?url: String, ?alt: String, ?width: Integer, ?height: Integer, ?source: enum_ImageSource) -> void
|
|
317
317
|
end
|
|
318
318
|
|
|
319
319
|
class InteractionResult
|
|
@@ -336,11 +336,11 @@ def initialize: (?action_results: Array[ActionResult], ?final_html: String, ?fin
|
|
|
336
336
|
class LinkInfo
|
|
337
337
|
attr_reader url: String
|
|
338
338
|
attr_reader text: String
|
|
339
|
-
attr_reader link_type:
|
|
339
|
+
attr_reader link_type: enum_LinkType
|
|
340
340
|
attr_reader rel: String?
|
|
341
341
|
attr_reader nofollow: bool
|
|
342
342
|
|
|
343
|
-
def initialize: (?url: String, ?text: String, ?link_type:
|
|
343
|
+
def initialize: (?url: String, ?text: String, ?link_type: enum_LinkType, ?rel: String, ?nofollow: bool) -> void
|
|
344
344
|
end
|
|
345
345
|
|
|
346
346
|
class MapResult
|
|
@@ -480,61 +480,39 @@ def initialize: (?status_code: Integer, ?final_url: String, ?content_type: Strin
|
|
|
480
480
|
def initialize: (?deny_private: bool, ?allowlist: Array[HostMatcher], ?max_redirects: Integer, ?scheme_allowlist: Array[String]) -> void
|
|
481
481
|
end
|
|
482
482
|
|
|
483
|
-
|
|
484
|
-
type value = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other
|
|
485
|
-
end
|
|
483
|
+
type enum_AssetCategory = :document | :image | :audio | :video | :font | :stylesheet | :script | :archive | :data | :other
|
|
486
484
|
|
|
487
485
|
class AuthConfig
|
|
488
486
|
end
|
|
489
487
|
|
|
490
|
-
|
|
491
|
-
type value = :chromiumoxide | :native
|
|
492
|
-
end
|
|
488
|
+
type enum_BrowserBackend = :chromiumoxide | :native
|
|
493
489
|
|
|
494
|
-
|
|
495
|
-
type value = :auto | :always | :never | :stealth
|
|
496
|
-
end
|
|
490
|
+
type enum_BrowserMode = :auto | :always | :never | :stealth
|
|
497
491
|
|
|
498
|
-
|
|
499
|
-
type value = :network_idle | :selector | :fixed
|
|
500
|
-
end
|
|
492
|
+
type enum_BrowserWait = :network_idle | :selector | :fixed
|
|
501
493
|
|
|
502
|
-
|
|
503
|
-
type value = :bm25
|
|
504
|
-
end
|
|
494
|
+
type enum_ContentFilterKind = :bm25
|
|
505
495
|
|
|
506
496
|
class CrawlEvent
|
|
507
497
|
end
|
|
508
498
|
|
|
509
|
-
|
|
510
|
-
type value = :bfs | :dfs | :best_first | :adaptive
|
|
511
|
-
end
|
|
499
|
+
type enum_CrawlStrategyKind = :bfs | :dfs | :best_first | :adaptive
|
|
512
500
|
|
|
513
|
-
|
|
514
|
-
type value = :base64
|
|
515
|
-
end
|
|
501
|
+
type enum_DocumentContentEncoding = :base64
|
|
516
502
|
|
|
517
|
-
|
|
518
|
-
type value = :rss | :atom | :json_feed
|
|
519
|
-
end
|
|
503
|
+
type enum_FeedType = :rss | :atom | :json_feed
|
|
520
504
|
|
|
521
505
|
class HostMatcher
|
|
522
506
|
end
|
|
523
507
|
|
|
524
|
-
|
|
525
|
-
type value = :img | :picture_source | :"og:image" | :"twitter:image"
|
|
526
|
-
end
|
|
508
|
+
type enum_ImageSource = :img | :picture_source | :"og:image" | :"twitter:image"
|
|
527
509
|
|
|
528
|
-
|
|
529
|
-
type value = :internal | :external | :anchor | :document
|
|
530
|
-
end
|
|
510
|
+
type enum_LinkType = :internal | :external | :anchor | :document
|
|
531
511
|
|
|
532
512
|
class PageAction
|
|
533
513
|
end
|
|
534
514
|
|
|
535
|
-
|
|
536
|
-
type value = :up | :down
|
|
537
|
-
end
|
|
515
|
+
type enum_ScrollDirection = :up | :down
|
|
538
516
|
|
|
539
517
|
def self.batch_crawl: (CrawlEngineHandle engine, Array[String] urls) -> BatchCrawlResults
|
|
540
518
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: crawlberg
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.7.
|
|
4
|
+
version: 1.7.2
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Xberg Team
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-09-
|
|
11
|
+
date: 2026-09-24 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rb_sys
|