crawlberg 1.0.6 → 1.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/crawlberg_rb/native/Cargo.lock +3 -3
- data/ext/crawlberg_rb/native/Cargo.toml +2 -2
- data/ext/crawlberg_rb/native/extconf.rb +3 -0
- data/ext/crawlberg_rb/src/lib.rs +63 -9
- data/lib/crawlberg/native.rb +41 -0
- data/lib/crawlberg/version.rb +2 -2
- data/lib/crawlberg.rb +8 -0
- data/lib/crawlberg_rb.so +0 -0
- data/sig/types.rbs +1 -1
- metadata +2 -5
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: abc390524f35e43793e7b5ad676200fe7a930f7845bb3c3cc22c881559575542
|
|
4
|
+
data.tar.gz: 1b9b907f80532b305c1102fc3d62da76220c8947ee791b6bcba7a71341d18a05
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0465e2a4003698f03bb26279e5428de63c81fad640a8f335271f0c3557be47d41f0d9176dd694b3a8fdcde4348e90d7a9e445d7b954bfbfbb15752b184fe6027
|
|
7
|
+
data.tar.gz: 0d97333565483576663ec454e5c00caf1d24a2ec000d988f682d0672c42262393df7e93c18a96117e6e277a10b14d2f3a90227649d9c7c498620c29c4266c600
|
|
@@ -545,9 +545,9 @@ dependencies = [
|
|
|
545
545
|
|
|
546
546
|
[[package]]
|
|
547
547
|
name = "crawlberg"
|
|
548
|
-
version = "1.0.
|
|
548
|
+
version = "1.0.8"
|
|
549
549
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
550
|
-
checksum = "
|
|
550
|
+
checksum = "fc6e5f06202d56f7b1b16399841241b810db3e305401bb401599565b07c92e10"
|
|
551
551
|
dependencies = [
|
|
552
552
|
"ahash",
|
|
553
553
|
"aho-corasick",
|
|
@@ -587,7 +587,7 @@ dependencies = [
|
|
|
587
587
|
|
|
588
588
|
[[package]]
|
|
589
589
|
name = "crawlberg-rb"
|
|
590
|
-
version = "1.0.
|
|
590
|
+
version = "1.0.7"
|
|
591
591
|
dependencies = [
|
|
592
592
|
"crawlberg",
|
|
593
593
|
"futures",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "crawlberg-rb"
|
|
3
|
-
version = "1.0.
|
|
3
|
+
version = "1.0.7"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
license = "MIT"
|
|
6
6
|
description = "High-performance web crawling engine"
|
|
@@ -17,7 +17,7 @@ path = "../src/lib.rs"
|
|
|
17
17
|
crate-type = ["cdylib"]
|
|
18
18
|
|
|
19
19
|
[dependencies]
|
|
20
|
-
crawlberg = { version = "1.0.
|
|
20
|
+
crawlberg = { version = "1.0.8", features = ["interact", "browser-chromiumoxide"] }
|
|
21
21
|
futures = "0.3"
|
|
22
22
|
magnus = "0.8"
|
|
23
23
|
rb-sys = ">=0.9, <0.9.128"
|
|
@@ -7,5 +7,8 @@ default_profile = ENV.fetch("CARGO_PROFILE", "release")
|
|
|
7
7
|
|
|
8
8
|
create_rust_makefile("crawlberg_rb") do |config|
|
|
9
9
|
config.profile = default_profile.to_sym
|
|
10
|
+
# extconf.rb and Cargo.toml are siblings under ext/crawlberg_rb/native/; rb_sys interprets
|
|
11
|
+
# ext_dir relative to extconf.rb, so "." finds the sibling Cargo.toml. "native" would
|
|
12
|
+
# resolve to native/native/Cargo.toml and break `gem install` on end-user machines.
|
|
10
13
|
config.ext_dir = "."
|
|
11
14
|
end
|
data/ext/crawlberg_rb/src/lib.rs
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
// This file is auto-generated by alef. DO NOT EDIT.
|
|
2
|
+
// alef:hash:75130b11765cbed0a1ea2b5bcffe59620c4045cee1c7cece391c5e3fa47b10e0
|
|
3
|
+
// Re-generate with: alef generate
|
|
2
4
|
#![allow(dead_code, unused_imports, unused_variables)]
|
|
3
5
|
#![allow(
|
|
4
6
|
clippy::too_many_arguments,
|
|
@@ -3981,6 +3983,7 @@ impl CrawlEngineHandle {
|
|
|
3981
3983
|
};
|
|
3982
3984
|
let ruby = unsafe { Ruby::get_unchecked() };
|
|
3983
3985
|
if ruby.block_given() {
|
|
3986
|
+
// Drive the stream synchronously, yielding each chunk to the block.
|
|
3984
3987
|
iterator.each()?;
|
|
3985
3988
|
Ok(ruby.qnil().as_value())
|
|
3986
3989
|
} else {
|
|
@@ -4015,6 +4018,7 @@ impl CrawlEngineHandle {
|
|
|
4015
4018
|
};
|
|
4016
4019
|
let ruby = unsafe { Ruby::get_unchecked() };
|
|
4017
4020
|
if ruby.block_given() {
|
|
4021
|
+
// Drive the stream synchronously, yielding each chunk to the block.
|
|
4018
4022
|
iterator.each()?;
|
|
4019
4023
|
Ok(ruby.qnil().as_value())
|
|
4020
4024
|
} else {
|
|
@@ -4448,6 +4452,8 @@ impl magnus::IntoValue for BrowserMode {
|
|
|
4448
4452
|
impl magnus::TryConvert for BrowserMode {
|
|
4449
4453
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4450
4454
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4455
|
+
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4456
|
+
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4451
4457
|
match s.as_str() {
|
|
4452
4458
|
"auto" | "Auto" => Ok(BrowserMode::Auto),
|
|
4453
4459
|
"always" | "Always" => Ok(BrowserMode::Always),
|
|
@@ -4492,6 +4498,8 @@ impl magnus::IntoValue for BrowserWait {
|
|
|
4492
4498
|
impl magnus::TryConvert for BrowserWait {
|
|
4493
4499
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4494
4500
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4501
|
+
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4502
|
+
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4495
4503
|
match s.as_str() {
|
|
4496
4504
|
"network_idle" | "NetworkIdle" => Ok(BrowserWait::NetworkIdle),
|
|
4497
4505
|
"selector" | "Selector" => Ok(BrowserWait::Selector),
|
|
@@ -4533,6 +4541,8 @@ impl magnus::IntoValue for BrowserBackend {
|
|
|
4533
4541
|
impl magnus::TryConvert for BrowserBackend {
|
|
4534
4542
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4535
4543
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4544
|
+
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4545
|
+
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4536
4546
|
match s.as_str() {
|
|
4537
4547
|
"chromiumoxide" | "Chromiumoxide" => Ok(BrowserBackend::Chromiumoxide),
|
|
4538
4548
|
"native" | "Native" => Ok(BrowserBackend::Native),
|
|
@@ -4578,6 +4588,8 @@ impl magnus::IntoValue for AuthConfig {
|
|
|
4578
4588
|
|
|
4579
4589
|
impl magnus::TryConvert for AuthConfig {
|
|
4580
4590
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4591
|
+
// For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
|
|
4592
|
+
// For unit enums or when passed as a string, fall back to string-based conversion.
|
|
4581
4593
|
let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
|
|
4582
4594
|
s
|
|
4583
4595
|
} else {
|
|
@@ -4588,12 +4600,19 @@ impl magnus::TryConvert for AuthConfig {
|
|
|
4588
4600
|
)
|
|
4589
4601
|
})?
|
|
4590
4602
|
};
|
|
4603
|
+
// Try deserializing as JSON first (handles JSON strings like "\"markdown\"" or "{\"click\":{\"selector\":\"...\"}}\"")
|
|
4604
|
+
// For internally-tagged enums, a bare variant string is wrapped as {"<tag>": value}.
|
|
4605
|
+
// If that fails, try treating it as a plain string value and wrap in quotes
|
|
4606
|
+
// If both fail, try as Custom variant (for untagged enum support)
|
|
4591
4607
|
serde_json::from_str(&json_str)
|
|
4592
4608
|
.or_else(|_| serde_json::from_value(serde_json::json!({ "type": json_str })))
|
|
4593
4609
|
.or_else(|_| serde_json::from_str(&format!("\"{json_str}\"")))
|
|
4594
|
-
.or_else(|_|
|
|
4595
|
-
|
|
4596
|
-
|
|
4610
|
+
.or_else(|_| {
|
|
4611
|
+
// Try as a JSON string for Custom variant (untagged enums accept any remaining value)
|
|
4612
|
+
match serde_json::to_value(&json_str) {
|
|
4613
|
+
Ok(val) => serde_json::from_value(val),
|
|
4614
|
+
Err(e) => Err(e),
|
|
4615
|
+
}
|
|
4597
4616
|
})
|
|
4598
4617
|
.map_err(|e| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_type_error(), e.to_string()))
|
|
4599
4618
|
}
|
|
@@ -4633,6 +4652,8 @@ impl magnus::IntoValue for LinkType {
|
|
|
4633
4652
|
impl magnus::TryConvert for LinkType {
|
|
4634
4653
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4635
4654
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4655
|
+
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4656
|
+
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4636
4657
|
match s.as_str() {
|
|
4637
4658
|
"internal" | "Internal" => Ok(LinkType::Internal),
|
|
4638
4659
|
"external" | "External" => Ok(LinkType::External),
|
|
@@ -4681,6 +4702,8 @@ impl magnus::IntoValue for ImageSource {
|
|
|
4681
4702
|
impl magnus::TryConvert for ImageSource {
|
|
4682
4703
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4683
4704
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4705
|
+
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4706
|
+
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4684
4707
|
match s.as_str() {
|
|
4685
4708
|
"img" | "Img" => Ok(ImageSource::Img),
|
|
4686
4709
|
"picture_source" | "PictureSource" => Ok(ImageSource::PictureSource),
|
|
@@ -4725,6 +4748,8 @@ impl magnus::IntoValue for FeedType {
|
|
|
4725
4748
|
impl magnus::TryConvert for FeedType {
|
|
4726
4749
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4727
4750
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4751
|
+
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4752
|
+
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4728
4753
|
match s.as_str() {
|
|
4729
4754
|
"rss" | "Rss" => Ok(FeedType::Rss),
|
|
4730
4755
|
"atom" | "Atom" => Ok(FeedType::Atom),
|
|
@@ -4782,6 +4807,8 @@ impl magnus::IntoValue for AssetCategory {
|
|
|
4782
4807
|
impl magnus::TryConvert for AssetCategory {
|
|
4783
4808
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4784
4809
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4810
|
+
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4811
|
+
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4785
4812
|
match s.as_str() {
|
|
4786
4813
|
"document" | "Document" => Ok(AssetCategory::Document),
|
|
4787
4814
|
"image" | "Image" => Ok(AssetCategory::Image),
|
|
@@ -4832,6 +4859,8 @@ impl magnus::IntoValue for CrawlEvent {
|
|
|
4832
4859
|
|
|
4833
4860
|
impl magnus::TryConvert for CrawlEvent {
|
|
4834
4861
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4862
|
+
// For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
|
|
4863
|
+
// For unit enums or when passed as a string, fall back to string-based conversion.
|
|
4835
4864
|
let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
|
|
4836
4865
|
s
|
|
4837
4866
|
} else {
|
|
@@ -4842,12 +4871,19 @@ impl magnus::TryConvert for CrawlEvent {
|
|
|
4842
4871
|
)
|
|
4843
4872
|
})?
|
|
4844
4873
|
};
|
|
4874
|
+
// Try deserializing as JSON first (handles JSON strings like "\"markdown\"" or "{\"click\":{\"selector\":\"...\"}}\"")
|
|
4875
|
+
// For internally-tagged enums, a bare variant string is wrapped as {"<tag>": value}.
|
|
4876
|
+
// If that fails, try treating it as a plain string value and wrap in quotes
|
|
4877
|
+
// If both fail, try as Custom variant (for untagged enum support)
|
|
4845
4878
|
serde_json::from_str(&json_str)
|
|
4846
4879
|
.or_else(|_| serde_json::from_value(serde_json::json!({ "type": json_str })))
|
|
4847
4880
|
.or_else(|_| serde_json::from_str(&format!("\"{json_str}\"")))
|
|
4848
|
-
.or_else(|_|
|
|
4849
|
-
|
|
4850
|
-
|
|
4881
|
+
.or_else(|_| {
|
|
4882
|
+
// Try as a JSON string for Custom variant (untagged enums accept any remaining value)
|
|
4883
|
+
match serde_json::to_value(&json_str) {
|
|
4884
|
+
Ok(val) => serde_json::from_value(val),
|
|
4885
|
+
Err(e) => Err(e),
|
|
4886
|
+
}
|
|
4851
4887
|
})
|
|
4852
4888
|
.map_err(|e| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_type_error(), e.to_string()))
|
|
4853
4889
|
}
|
|
@@ -4908,6 +4944,8 @@ impl magnus::IntoValue for PageAction {
|
|
|
4908
4944
|
|
|
4909
4945
|
impl magnus::TryConvert for PageAction {
|
|
4910
4946
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4947
|
+
// For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
|
|
4948
|
+
// For unit enums or when passed as a string, fall back to string-based conversion.
|
|
4911
4949
|
let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
|
|
4912
4950
|
s
|
|
4913
4951
|
} else {
|
|
@@ -4918,12 +4956,19 @@ impl magnus::TryConvert for PageAction {
|
|
|
4918
4956
|
)
|
|
4919
4957
|
})?
|
|
4920
4958
|
};
|
|
4959
|
+
// Try deserializing as JSON first (handles JSON strings like "\"markdown\"" or "{\"click\":{\"selector\":\"...\"}}\"")
|
|
4960
|
+
// For internally-tagged enums, a bare variant string is wrapped as {"<tag>": value}.
|
|
4961
|
+
// If that fails, try treating it as a plain string value and wrap in quotes
|
|
4962
|
+
// If both fail, try as Custom variant (for untagged enum support)
|
|
4921
4963
|
serde_json::from_str(&json_str)
|
|
4922
4964
|
.or_else(|_| serde_json::from_value(serde_json::json!({ "type": json_str })))
|
|
4923
4965
|
.or_else(|_| serde_json::from_str(&format!("\"{json_str}\"")))
|
|
4924
|
-
.or_else(|_|
|
|
4925
|
-
|
|
4926
|
-
|
|
4966
|
+
.or_else(|_| {
|
|
4967
|
+
// Try as a JSON string for Custom variant (untagged enums accept any remaining value)
|
|
4968
|
+
match serde_json::to_value(&json_str) {
|
|
4969
|
+
Ok(val) => serde_json::from_value(val),
|
|
4970
|
+
Err(e) => Err(e),
|
|
4971
|
+
}
|
|
4927
4972
|
})
|
|
4928
4973
|
.map_err(|e| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_type_error(), e.to_string()))
|
|
4929
4974
|
}
|
|
@@ -4959,6 +5004,8 @@ impl magnus::IntoValue for ScrollDirection {
|
|
|
4959
5004
|
impl magnus::TryConvert for ScrollDirection {
|
|
4960
5005
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4961
5006
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
5007
|
+
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
5008
|
+
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4962
5009
|
match s.as_str() {
|
|
4963
5010
|
"up" | "Up" => Ok(ScrollDirection::Up),
|
|
4964
5011
|
"down" | "Down" => Ok(ScrollDirection::Down),
|
|
@@ -6782,6 +6829,7 @@ impl CrawlStreamIterator {
|
|
|
6782
6829
|
}
|
|
6783
6830
|
Some(Err(e)) => Err(Error::new(ruby.exception_runtime_error(), e.to_string())),
|
|
6784
6831
|
None => {
|
|
6832
|
+
// Drop the stream to release any resources.
|
|
6785
6833
|
let inner = self.inner.clone();
|
|
6786
6834
|
let runtime = self.runtime.clone();
|
|
6787
6835
|
runtime.block_on(async move {
|
|
@@ -6799,6 +6847,8 @@ impl CrawlStreamIterator {
|
|
|
6799
6847
|
use magnus::value::ReprValue;
|
|
6800
6848
|
let ruby = unsafe { Ruby::get_unchecked() };
|
|
6801
6849
|
if !ruby.block_given() {
|
|
6850
|
+
// Without a block, return an Enumerator over `each` so the caller can
|
|
6851
|
+
// call `.to_a`, `.lazy`, etc.
|
|
6802
6852
|
let self_val: magnus::Value = self.clone().into_value_with(&ruby);
|
|
6803
6853
|
let enumerator = self_val.enumeratorize(ruby.to_symbol("each"), ());
|
|
6804
6854
|
return Ok(enumerator.as_value());
|
|
@@ -6861,6 +6911,7 @@ impl BatchCrawlStreamIterator {
|
|
|
6861
6911
|
}
|
|
6862
6912
|
Some(Err(e)) => Err(Error::new(ruby.exception_runtime_error(), e.to_string())),
|
|
6863
6913
|
None => {
|
|
6914
|
+
// Drop the stream to release any resources.
|
|
6864
6915
|
let inner = self.inner.clone();
|
|
6865
6916
|
let runtime = self.runtime.clone();
|
|
6866
6917
|
runtime.block_on(async move {
|
|
@@ -6878,6 +6929,8 @@ impl BatchCrawlStreamIterator {
|
|
|
6878
6929
|
use magnus::value::ReprValue;
|
|
6879
6930
|
let ruby = unsafe { Ruby::get_unchecked() };
|
|
6880
6931
|
if !ruby.block_given() {
|
|
6932
|
+
// Without a block, return an Enumerator over `each` so the caller can
|
|
6933
|
+
// call `.to_a`, `.lazy`, etc.
|
|
6881
6934
|
let self_val: magnus::Value = self.clone().into_value_with(&ruby);
|
|
6882
6935
|
let enumerator = self_val.enumeratorize(ruby.to_symbol("each"), ());
|
|
6883
6936
|
return Ok(enumerator.as_value());
|
|
@@ -6897,6 +6950,7 @@ impl BatchCrawlStreamIterator {
|
|
|
6897
6950
|
fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
|
|
6898
6951
|
let module = ruby.define_module("Crawlberg")?;
|
|
6899
6952
|
|
|
6953
|
+
// Ensure JSON library is loaded for Hash#to_json
|
|
6900
6954
|
let _ = ruby.eval::<magnus::Value>("require \"json\"");
|
|
6901
6955
|
|
|
6902
6956
|
let class = module.define_class("ExtractionMeta", ruby.class_object())?;
|
data/lib/crawlberg/native.rb
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
+
# alef:hash:75130b11765cbed0a1ea2b5bcffe59620c4045cee1c7cece391c5e3fa47b10e0
|
|
3
|
+
# To regenerate: alef generate
|
|
4
|
+
# To verify freshness: alef verify --exit-code
|
|
2
5
|
# frozen_string_literal: true
|
|
3
6
|
|
|
4
7
|
require "json"
|
|
@@ -6,12 +9,14 @@ require "sorbet-runtime"
|
|
|
6
9
|
require "crawlberg_rb"
|
|
7
10
|
|
|
8
11
|
module Crawlberg
|
|
12
|
+
# Authentication configuration.
|
|
9
13
|
module AuthConfig
|
|
10
14
|
extend T::Helpers
|
|
11
15
|
extend T::Sig
|
|
12
16
|
|
|
13
17
|
interface!
|
|
14
18
|
|
|
19
|
+
# Dispatch from a Hash to the appropriate variant constructor.
|
|
15
20
|
# @param hash [Hash] with discriminator field and variant-specific fields
|
|
16
21
|
# @return [variant_class] an instance of the appropriate variant
|
|
17
22
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.untyped) }
|
|
@@ -34,9 +39,11 @@ module Crawlberg
|
|
|
34
39
|
include AuthConfig
|
|
35
40
|
extend T::Sig
|
|
36
41
|
|
|
42
|
+
# Username sent in the `Authorization: Basic` header.
|
|
37
43
|
sig { returns(String) }
|
|
38
44
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
39
45
|
def username = super
|
|
46
|
+
# Password sent in the `Authorization: Basic` header.
|
|
40
47
|
sig { returns(String) }
|
|
41
48
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
42
49
|
def password = super
|
|
@@ -58,6 +65,7 @@ module Crawlberg
|
|
|
58
65
|
include AuthConfig
|
|
59
66
|
extend T::Sig
|
|
60
67
|
|
|
68
|
+
# Token sent in the `Authorization: Bearer` header.
|
|
61
69
|
sig { returns(String) }
|
|
62
70
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
63
71
|
def token = super
|
|
@@ -79,9 +87,11 @@ module Crawlberg
|
|
|
79
87
|
include AuthConfig
|
|
80
88
|
extend T::Sig
|
|
81
89
|
|
|
90
|
+
# HTTP header name to set on each request.
|
|
82
91
|
sig { returns(String) }
|
|
83
92
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
84
93
|
def name = super
|
|
94
|
+
# HTTP header value to send.
|
|
85
95
|
sig { returns(String) }
|
|
86
96
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
87
97
|
def value = super
|
|
@@ -101,12 +111,19 @@ module Crawlberg
|
|
|
101
111
|
end
|
|
102
112
|
|
|
103
113
|
module Crawlberg
|
|
114
|
+
# An event emitted during a streaming crawl operation.
|
|
115
|
+
#
|
|
116
|
+
# Not available on `wasm32` targets — streaming requires native concurrency
|
|
117
|
+
# primitives (tokio channels, `JoinSet`) that are not supported on wasm32.
|
|
118
|
+
#
|
|
119
|
+
# Delivered to bindings through each target's native streaming idiom.
|
|
104
120
|
module CrawlEvent
|
|
105
121
|
extend T::Helpers
|
|
106
122
|
extend T::Sig
|
|
107
123
|
|
|
108
124
|
interface!
|
|
109
125
|
|
|
126
|
+
# Dispatch from a Hash to the appropriate variant constructor.
|
|
110
127
|
# @param hash [Hash] with discriminator field and variant-specific fields
|
|
111
128
|
# @return [variant_class] an instance of the appropriate variant
|
|
112
129
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.untyped) }
|
|
@@ -129,6 +146,7 @@ module Crawlberg
|
|
|
129
146
|
include CrawlEvent
|
|
130
147
|
extend T::Sig
|
|
131
148
|
|
|
149
|
+
# The crawled page result.
|
|
132
150
|
sig { returns(CrawlPageResult) }
|
|
133
151
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
134
152
|
def result = super
|
|
@@ -150,9 +168,11 @@ module Crawlberg
|
|
|
150
168
|
include CrawlEvent
|
|
151
169
|
extend T::Sig
|
|
152
170
|
|
|
171
|
+
# The URL that failed.
|
|
153
172
|
sig { returns(String) }
|
|
154
173
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
155
174
|
def url = super
|
|
175
|
+
# The error message.
|
|
156
176
|
sig { returns(String) }
|
|
157
177
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
158
178
|
def error = super
|
|
@@ -174,6 +194,7 @@ module Crawlberg
|
|
|
174
194
|
include CrawlEvent
|
|
175
195
|
extend T::Sig
|
|
176
196
|
|
|
197
|
+
# Total number of pages crawled.
|
|
177
198
|
sig { returns(Integer) }
|
|
178
199
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
179
200
|
def pages_crawled = super
|
|
@@ -193,12 +214,17 @@ module Crawlberg
|
|
|
193
214
|
end
|
|
194
215
|
|
|
195
216
|
module Crawlberg
|
|
217
|
+
# A single page interaction action.
|
|
218
|
+
#
|
|
219
|
+
# Actions are serialized with a `type` tag using camelCase naming,
|
|
220
|
+
# except `ExecuteJs` which is explicitly renamed to `"executeJs"`.
|
|
196
221
|
module PageAction
|
|
197
222
|
extend T::Helpers
|
|
198
223
|
extend T::Sig
|
|
199
224
|
|
|
200
225
|
interface!
|
|
201
226
|
|
|
227
|
+
# Dispatch from a Hash to the appropriate variant constructor.
|
|
202
228
|
# @param hash [Hash] with discriminator field and variant-specific fields
|
|
203
229
|
# @return [variant_class] an instance of the appropriate variant
|
|
204
230
|
sig { params(hash: T::Hash[T.untyped, T.untyped]).returns(T.untyped) }
|
|
@@ -231,6 +257,7 @@ module Crawlberg
|
|
|
231
257
|
include PageAction
|
|
232
258
|
extend T::Sig
|
|
233
259
|
|
|
260
|
+
# CSS selector for the element to click.
|
|
234
261
|
sig { returns(String) }
|
|
235
262
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
236
263
|
def selector = super
|
|
@@ -262,9 +289,11 @@ module Crawlberg
|
|
|
262
289
|
include PageAction
|
|
263
290
|
extend T::Sig
|
|
264
291
|
|
|
292
|
+
# CSS selector for the input element.
|
|
265
293
|
sig { returns(String) }
|
|
266
294
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
267
295
|
def selector = super
|
|
296
|
+
# Text to type into the element.
|
|
268
297
|
sig { returns(String) }
|
|
269
298
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
270
299
|
def text = super
|
|
@@ -296,6 +325,7 @@ module Crawlberg
|
|
|
296
325
|
include PageAction
|
|
297
326
|
extend T::Sig
|
|
298
327
|
|
|
328
|
+
# Key name to press.
|
|
299
329
|
sig { returns(String) }
|
|
300
330
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
301
331
|
def key = super
|
|
@@ -327,12 +357,15 @@ module Crawlberg
|
|
|
327
357
|
include PageAction
|
|
328
358
|
extend T::Sig
|
|
329
359
|
|
|
360
|
+
# Direction to scroll.
|
|
330
361
|
sig { returns(ScrollDirection) }
|
|
331
362
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
332
363
|
def direction = super
|
|
364
|
+
# Optional CSS selector for a scrollable element. Scrolls the page if absent.
|
|
333
365
|
sig { returns(T.nilable(String)) }
|
|
334
366
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
335
367
|
def selector = super
|
|
368
|
+
# Optional pixel amount to scroll. Uses a default if absent.
|
|
336
369
|
sig { returns(T.nilable(Integer)) }
|
|
337
370
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
338
371
|
def amount = super
|
|
@@ -368,9 +401,11 @@ module Crawlberg
|
|
|
368
401
|
include PageAction
|
|
369
402
|
extend T::Sig
|
|
370
403
|
|
|
404
|
+
# Milliseconds to wait. Ignored if `selector` is provided.
|
|
371
405
|
sig { returns(T.nilable(Integer)) }
|
|
372
406
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
373
407
|
def milliseconds = super
|
|
408
|
+
# CSS selector to wait for.
|
|
374
409
|
sig { returns(T.nilable(String)) }
|
|
375
410
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
376
411
|
def selector = super
|
|
@@ -402,6 +437,11 @@ module Crawlberg
|
|
|
402
437
|
include PageAction
|
|
403
438
|
extend T::Sig
|
|
404
439
|
|
|
440
|
+
# Whether to capture the full scrollable page. Defaults to viewport only.
|
|
441
|
+
#
|
|
442
|
+
# Accepts both the canonical `fullPage` (camelCase) form and the
|
|
443
|
+
# `full_page` (snake_case) alias so language bindings and fixtures can
|
|
444
|
+
# use either convention without error.
|
|
405
445
|
sig { returns(T.nilable(T::Boolean)) }
|
|
406
446
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
407
447
|
def full_page = super
|
|
@@ -438,6 +478,7 @@ module Crawlberg
|
|
|
438
478
|
include PageAction
|
|
439
479
|
extend T::Sig
|
|
440
480
|
|
|
481
|
+
# JavaScript source code to execute. Max 1 MB.
|
|
441
482
|
sig { returns(String) }
|
|
442
483
|
# rubocop:disable Lint/UselessMethodDefinition
|
|
443
484
|
def script = super
|
data/lib/crawlberg/version.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:75130b11765cbed0a1ea2b5bcffe59620c4045cee1c7cece391c5e3fa47b10e0
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify --exit-code
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
module Crawlberg
|
|
8
8
|
## The version string for this package.
|
|
9
|
-
VERSION = "1.0.
|
|
9
|
+
VERSION = "1.0.8"
|
|
10
10
|
end
|
data/lib/crawlberg.rb
CHANGED
|
@@ -1,4 +1,7 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
+
# alef:hash:75130b11765cbed0a1ea2b5bcffe59620c4045cee1c7cece391c5e3fa47b10e0
|
|
3
|
+
# To regenerate: alef generate
|
|
4
|
+
# To verify freshness: alef verify --exit-code
|
|
2
5
|
# frozen_string_literal: true
|
|
3
6
|
|
|
4
7
|
require_relative "crawlberg/version"
|
|
@@ -11,8 +14,13 @@ require_relative "crawlberg/native"
|
|
|
11
14
|
# {file:README.md} for usage examples and the upstream documentation
|
|
12
15
|
# for the full API reference.
|
|
13
16
|
module Crawlberg
|
|
17
|
+
# Re-export all types and functions from native extension
|
|
14
18
|
end
|
|
15
19
|
|
|
20
|
+
# Bring top-level Crawlberg classes into the global namespace so callers
|
|
21
|
+
# (and the generated e2e suite) can reference them unqualified. The native
|
|
22
|
+
# extension has already been required above, so every type constant is defined
|
|
23
|
+
# under Crawlberg by this point.
|
|
16
24
|
Crawlberg.constants.each do |const_name|
|
|
17
25
|
value = Crawlberg.const_get(const_name)
|
|
18
26
|
::Object.const_set(const_name, value) if value.is_a?(Module) && !::Object.const_defined?(const_name)
|
data/lib/crawlberg_rb.so
CHANGED
|
Binary file
|
data/sig/types.rbs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:75130b11765cbed0a1ea2b5bcffe59620c4045cee1c7cece391c5e3fa47b10e0
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify --exit-code
|
|
5
5
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: crawlberg
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Xberg Team
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-07-
|
|
11
|
+
date: 2026-07-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rb_sys
|
|
@@ -78,9 +78,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
78
78
|
- - ">="
|
|
79
79
|
- !ruby/object:Gem::Version
|
|
80
80
|
version: 3.2.0
|
|
81
|
-
- - "<"
|
|
82
|
-
- !ruby/object:Gem::Version
|
|
83
|
-
version: '4.0'
|
|
84
81
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
85
82
|
requirements:
|
|
86
83
|
- - ">="
|