crawlberg 1.0.4 → 1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/ext/crawlberg_rb/native/Cargo.lock +110 -99
- data/ext/crawlberg_rb/native/Cargo.toml +2 -2
- data/ext/crawlberg_rb/native/extconf.rb +0 -3
- data/ext/crawlberg_rb/src/lib.rs +73 -64
- data/lib/crawlberg/native.rb +0 -41
- data/lib/crawlberg/version.rb +2 -2
- data/lib/crawlberg.rb +0 -8
- data/lib/crawlberg_rb.so +0 -0
- data/sig/types.rbs +1 -1
- metadata +2 -2
data/ext/crawlberg_rb/src/lib.rs
CHANGED
|
@@ -1,6 +1,4 @@
|
|
|
1
1
|
// This file is auto-generated by alef. DO NOT EDIT.
|
|
2
|
-
// alef:hash:e0600612767fdd3d51e9128afdb0ffcd1fb490e9942f3839a5120ab7a4e3b658
|
|
3
|
-
// Re-generate with: alef generate
|
|
4
2
|
#![allow(dead_code, unused_imports, unused_variables)]
|
|
5
3
|
#![allow(
|
|
6
4
|
clippy::too_many_arguments,
|
|
@@ -2040,6 +2038,7 @@ impl CrawlResult {
|
|
|
2040
2038
|
}
|
|
2041
2039
|
|
|
2042
2040
|
fn unique_normalized_urls(&self) -> usize {
|
|
2041
|
+
#[allow(clippy::needless_update)]
|
|
2043
2042
|
let core_self = crawlberg::CrawlResult {
|
|
2044
2043
|
pages: self.pages.clone().into_iter().map(Into::into).collect(),
|
|
2045
2044
|
|
|
@@ -3982,7 +3981,6 @@ impl CrawlEngineHandle {
|
|
|
3982
3981
|
};
|
|
3983
3982
|
let ruby = unsafe { Ruby::get_unchecked() };
|
|
3984
3983
|
if ruby.block_given() {
|
|
3985
|
-
// Drive the stream synchronously, yielding each chunk to the block.
|
|
3986
3984
|
iterator.each()?;
|
|
3987
3985
|
Ok(ruby.qnil().as_value())
|
|
3988
3986
|
} else {
|
|
@@ -4017,7 +4015,6 @@ impl CrawlEngineHandle {
|
|
|
4017
4015
|
};
|
|
4018
4016
|
let ruby = unsafe { Ruby::get_unchecked() };
|
|
4019
4017
|
if ruby.block_given() {
|
|
4020
|
-
// Drive the stream synchronously, yielding each chunk to the block.
|
|
4021
4018
|
iterator.each()?;
|
|
4022
4019
|
Ok(ruby.qnil().as_value())
|
|
4023
4020
|
} else {
|
|
@@ -4451,8 +4448,6 @@ impl magnus::IntoValue for BrowserMode {
|
|
|
4451
4448
|
impl magnus::TryConvert for BrowserMode {
|
|
4452
4449
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4453
4450
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4454
|
-
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4455
|
-
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4456
4451
|
match s.as_str() {
|
|
4457
4452
|
"auto" | "Auto" => Ok(BrowserMode::Auto),
|
|
4458
4453
|
"always" | "Always" => Ok(BrowserMode::Always),
|
|
@@ -4497,8 +4492,6 @@ impl magnus::IntoValue for BrowserWait {
|
|
|
4497
4492
|
impl magnus::TryConvert for BrowserWait {
|
|
4498
4493
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4499
4494
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4500
|
-
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4501
|
-
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4502
4495
|
match s.as_str() {
|
|
4503
4496
|
"network_idle" | "NetworkIdle" => Ok(BrowserWait::NetworkIdle),
|
|
4504
4497
|
"selector" | "Selector" => Ok(BrowserWait::Selector),
|
|
@@ -4540,8 +4533,6 @@ impl magnus::IntoValue for BrowserBackend {
|
|
|
4540
4533
|
impl magnus::TryConvert for BrowserBackend {
|
|
4541
4534
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4542
4535
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4543
|
-
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4544
|
-
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4545
4536
|
match s.as_str() {
|
|
4546
4537
|
"chromiumoxide" | "Chromiumoxide" => Ok(BrowserBackend::Chromiumoxide),
|
|
4547
4538
|
"native" | "Native" => Ok(BrowserBackend::Native),
|
|
@@ -4587,8 +4578,6 @@ impl magnus::IntoValue for AuthConfig {
|
|
|
4587
4578
|
|
|
4588
4579
|
impl magnus::TryConvert for AuthConfig {
|
|
4589
4580
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4590
|
-
// For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
|
|
4591
|
-
// For unit enums or when passed as a string, fall back to string-based conversion.
|
|
4592
4581
|
let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
|
|
4593
4582
|
s
|
|
4594
4583
|
} else {
|
|
@@ -4599,19 +4588,12 @@ impl magnus::TryConvert for AuthConfig {
|
|
|
4599
4588
|
)
|
|
4600
4589
|
})?
|
|
4601
4590
|
};
|
|
4602
|
-
// Try deserializing as JSON first (handles JSON strings like "\"markdown\"" or "{\"click\":{\"selector\":\"...\"}}\"")
|
|
4603
|
-
// For internally-tagged enums, a bare variant string is wrapped as {"<tag>": value}.
|
|
4604
|
-
// If that fails, try treating it as a plain string value and wrap in quotes
|
|
4605
|
-
// If both fail, try as Custom variant (for untagged enum support)
|
|
4606
4591
|
serde_json::from_str(&json_str)
|
|
4607
4592
|
.or_else(|_| serde_json::from_value(serde_json::json!({ "type": json_str })))
|
|
4608
4593
|
.or_else(|_| serde_json::from_str(&format!("\"{json_str}\"")))
|
|
4609
|
-
.or_else(|_| {
|
|
4610
|
-
|
|
4611
|
-
|
|
4612
|
-
Ok(val) => serde_json::from_value(val),
|
|
4613
|
-
Err(e) => Err(e),
|
|
4614
|
-
}
|
|
4594
|
+
.or_else(|_| match serde_json::to_value(&json_str) {
|
|
4595
|
+
Ok(val) => serde_json::from_value(val),
|
|
4596
|
+
Err(e) => Err(e),
|
|
4615
4597
|
})
|
|
4616
4598
|
.map_err(|e| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_type_error(), e.to_string()))
|
|
4617
4599
|
}
|
|
@@ -4651,8 +4633,6 @@ impl magnus::IntoValue for LinkType {
|
|
|
4651
4633
|
impl magnus::TryConvert for LinkType {
|
|
4652
4634
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4653
4635
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4654
|
-
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4655
|
-
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4656
4636
|
match s.as_str() {
|
|
4657
4637
|
"internal" | "Internal" => Ok(LinkType::Internal),
|
|
4658
4638
|
"external" | "External" => Ok(LinkType::External),
|
|
@@ -4701,8 +4681,6 @@ impl magnus::IntoValue for ImageSource {
|
|
|
4701
4681
|
impl magnus::TryConvert for ImageSource {
|
|
4702
4682
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4703
4683
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4704
|
-
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4705
|
-
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4706
4684
|
match s.as_str() {
|
|
4707
4685
|
"img" | "Img" => Ok(ImageSource::Img),
|
|
4708
4686
|
"picture_source" | "PictureSource" => Ok(ImageSource::PictureSource),
|
|
@@ -4747,8 +4725,6 @@ impl magnus::IntoValue for FeedType {
|
|
|
4747
4725
|
impl magnus::TryConvert for FeedType {
|
|
4748
4726
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4749
4727
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4750
|
-
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4751
|
-
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4752
4728
|
match s.as_str() {
|
|
4753
4729
|
"rss" | "Rss" => Ok(FeedType::Rss),
|
|
4754
4730
|
"atom" | "Atom" => Ok(FeedType::Atom),
|
|
@@ -4806,8 +4782,6 @@ impl magnus::IntoValue for AssetCategory {
|
|
|
4806
4782
|
impl magnus::TryConvert for AssetCategory {
|
|
4807
4783
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4808
4784
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
4809
|
-
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
4810
|
-
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
4811
4785
|
match s.as_str() {
|
|
4812
4786
|
"document" | "Document" => Ok(AssetCategory::Document),
|
|
4813
4787
|
"image" | "Image" => Ok(AssetCategory::Image),
|
|
@@ -4858,8 +4832,6 @@ impl magnus::IntoValue for CrawlEvent {
|
|
|
4858
4832
|
|
|
4859
4833
|
impl magnus::TryConvert for CrawlEvent {
|
|
4860
4834
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4861
|
-
// For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
|
|
4862
|
-
// For unit enums or when passed as a string, fall back to string-based conversion.
|
|
4863
4835
|
let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
|
|
4864
4836
|
s
|
|
4865
4837
|
} else {
|
|
@@ -4870,19 +4842,12 @@ impl magnus::TryConvert for CrawlEvent {
|
|
|
4870
4842
|
)
|
|
4871
4843
|
})?
|
|
4872
4844
|
};
|
|
4873
|
-
// Try deserializing as JSON first (handles JSON strings like "\"markdown\"" or "{\"click\":{\"selector\":\"...\"}}\"")
|
|
4874
|
-
// For internally-tagged enums, a bare variant string is wrapped as {"<tag>": value}.
|
|
4875
|
-
// If that fails, try treating it as a plain string value and wrap in quotes
|
|
4876
|
-
// If both fail, try as Custom variant (for untagged enum support)
|
|
4877
4845
|
serde_json::from_str(&json_str)
|
|
4878
4846
|
.or_else(|_| serde_json::from_value(serde_json::json!({ "type": json_str })))
|
|
4879
4847
|
.or_else(|_| serde_json::from_str(&format!("\"{json_str}\"")))
|
|
4880
|
-
.or_else(|_| {
|
|
4881
|
-
|
|
4882
|
-
|
|
4883
|
-
Ok(val) => serde_json::from_value(val),
|
|
4884
|
-
Err(e) => Err(e),
|
|
4885
|
-
}
|
|
4848
|
+
.or_else(|_| match serde_json::to_value(&json_str) {
|
|
4849
|
+
Ok(val) => serde_json::from_value(val),
|
|
4850
|
+
Err(e) => Err(e),
|
|
4886
4851
|
})
|
|
4887
4852
|
.map_err(|e| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_type_error(), e.to_string()))
|
|
4888
4853
|
}
|
|
@@ -4943,8 +4908,6 @@ impl magnus::IntoValue for PageAction {
|
|
|
4943
4908
|
|
|
4944
4909
|
impl magnus::TryConvert for PageAction {
|
|
4945
4910
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
4946
|
-
// For data enums with fields (e.g., PageAction), try to deserialize from JSON first.
|
|
4947
|
-
// For unit enums or when passed as a string, fall back to string-based conversion.
|
|
4948
4911
|
let json_str: String = if let Ok(s) = <String as magnus::TryConvert>::try_convert(val) {
|
|
4949
4912
|
s
|
|
4950
4913
|
} else {
|
|
@@ -4955,19 +4918,12 @@ impl magnus::TryConvert for PageAction {
|
|
|
4955
4918
|
)
|
|
4956
4919
|
})?
|
|
4957
4920
|
};
|
|
4958
|
-
// Try deserializing as JSON first (handles JSON strings like "\"markdown\"" or "{\"click\":{\"selector\":\"...\"}}\"")
|
|
4959
|
-
// For internally-tagged enums, a bare variant string is wrapped as {"<tag>": value}.
|
|
4960
|
-
// If that fails, try treating it as a plain string value and wrap in quotes
|
|
4961
|
-
// If both fail, try as Custom variant (for untagged enum support)
|
|
4962
4921
|
serde_json::from_str(&json_str)
|
|
4963
4922
|
.or_else(|_| serde_json::from_value(serde_json::json!({ "type": json_str })))
|
|
4964
4923
|
.or_else(|_| serde_json::from_str(&format!("\"{json_str}\"")))
|
|
4965
|
-
.or_else(|_| {
|
|
4966
|
-
|
|
4967
|
-
|
|
4968
|
-
Ok(val) => serde_json::from_value(val),
|
|
4969
|
-
Err(e) => Err(e),
|
|
4970
|
-
}
|
|
4924
|
+
.or_else(|_| match serde_json::to_value(&json_str) {
|
|
4925
|
+
Ok(val) => serde_json::from_value(val),
|
|
4926
|
+
Err(e) => Err(e),
|
|
4971
4927
|
})
|
|
4972
4928
|
.map_err(|e| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_type_error(), e.to_string()))
|
|
4973
4929
|
}
|
|
@@ -5003,8 +4959,6 @@ impl magnus::IntoValue for ScrollDirection {
|
|
|
5003
4959
|
impl magnus::TryConvert for ScrollDirection {
|
|
5004
4960
|
fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
|
|
5005
4961
|
let s: String = magnus::TryConvert::try_convert(val)?;
|
|
5006
|
-
// Accept the serde wire name (snake_case), the PascalCase Rust variant name,
|
|
5007
|
-
// and a lowercase fallback so fixtures written in any of those styles work.
|
|
5008
4962
|
match s.as_str() {
|
|
5009
4963
|
"up" | "Up" => Ok(ScrollDirection::Up),
|
|
5010
4964
|
"down" | "Down" => Ok(ScrollDirection::Down),
|
|
@@ -5277,6 +5231,7 @@ fn batch_crawl_stream(engine: CrawlEngineHandle, req: BatchCrawlStreamRequest) -
|
|
|
5277
5231
|
engine.batch_crawl_stream(req)
|
|
5278
5232
|
}
|
|
5279
5233
|
|
|
5234
|
+
#[allow(clippy::needless_update)]
|
|
5280
5235
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5281
5236
|
impl From<ExtractionMeta> for crawlberg::ExtractionMeta {
|
|
5282
5237
|
fn from(val: ExtractionMeta) -> Self {
|
|
@@ -5286,6 +5241,7 @@ impl From<ExtractionMeta> for crawlberg::ExtractionMeta {
|
|
|
5286
5241
|
completion_tokens: val.completion_tokens,
|
|
5287
5242
|
model: val.model,
|
|
5288
5243
|
chunks_processed: val.chunks_processed,
|
|
5244
|
+
..Default::default()
|
|
5289
5245
|
}
|
|
5290
5246
|
}
|
|
5291
5247
|
}
|
|
@@ -5303,6 +5259,7 @@ impl From<crawlberg::ExtractionMeta> for ExtractionMeta {
|
|
|
5303
5259
|
}
|
|
5304
5260
|
}
|
|
5305
5261
|
|
|
5262
|
+
#[allow(clippy::needless_update)]
|
|
5306
5263
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5307
5264
|
impl From<ProxyConfig> for crawlberg::ProxyConfig {
|
|
5308
5265
|
fn from(val: ProxyConfig) -> Self {
|
|
@@ -5310,6 +5267,7 @@ impl From<ProxyConfig> for crawlberg::ProxyConfig {
|
|
|
5310
5267
|
url: val.url,
|
|
5311
5268
|
username: val.username,
|
|
5312
5269
|
password: val.password,
|
|
5270
|
+
..Default::default()
|
|
5313
5271
|
}
|
|
5314
5272
|
}
|
|
5315
5273
|
}
|
|
@@ -5325,6 +5283,7 @@ impl From<crawlberg::ProxyConfig> for ProxyConfig {
|
|
|
5325
5283
|
}
|
|
5326
5284
|
}
|
|
5327
5285
|
|
|
5286
|
+
#[allow(clippy::needless_update)]
|
|
5328
5287
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5329
5288
|
impl From<ContentConfig> for crawlberg::ContentConfig {
|
|
5330
5289
|
fn from(val: ContentConfig) -> Self {
|
|
@@ -5341,6 +5300,7 @@ impl From<ContentConfig> for crawlberg::ContentConfig {
|
|
|
5341
5300
|
wrap: val.wrap,
|
|
5342
5301
|
wrap_width: val.wrap_width,
|
|
5343
5302
|
include_document_structure: val.include_document_structure,
|
|
5303
|
+
..Default::default()
|
|
5344
5304
|
}
|
|
5345
5305
|
}
|
|
5346
5306
|
}
|
|
@@ -5365,6 +5325,7 @@ impl From<crawlberg::ContentConfig> for ContentConfig {
|
|
|
5365
5325
|
}
|
|
5366
5326
|
}
|
|
5367
5327
|
|
|
5328
|
+
#[allow(clippy::needless_update)]
|
|
5368
5329
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5369
5330
|
impl From<BrowserConfig> for crawlberg::BrowserConfig {
|
|
5370
5331
|
fn from(val: BrowserConfig) -> Self {
|
|
@@ -5382,6 +5343,7 @@ impl From<BrowserConfig> for crawlberg::BrowserConfig {
|
|
|
5382
5343
|
robots_user_agent: val.robots_user_agent,
|
|
5383
5344
|
capture_network_events: val.capture_network_events,
|
|
5384
5345
|
session_affinity: val.session_affinity,
|
|
5346
|
+
..Default::default()
|
|
5385
5347
|
}
|
|
5386
5348
|
}
|
|
5387
5349
|
}
|
|
@@ -5511,6 +5473,7 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
|
|
|
5511
5473
|
}
|
|
5512
5474
|
}
|
|
5513
5475
|
|
|
5476
|
+
#[allow(clippy::needless_update)]
|
|
5514
5477
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5515
5478
|
impl From<BrowserExtras> for crawlberg::BrowserExtras {
|
|
5516
5479
|
fn from(val: BrowserExtras) -> Self {
|
|
@@ -5518,6 +5481,7 @@ impl From<BrowserExtras> for crawlberg::BrowserExtras {
|
|
|
5518
5481
|
eval_result: val.eval_result.as_ref().and_then(|s| serde_json::from_str(s).ok()),
|
|
5519
5482
|
network_events: val.network_events.into_iter().map(Into::into).collect(),
|
|
5520
5483
|
cookies: val.cookies.into_iter().map(Into::into).collect(),
|
|
5484
|
+
..Default::default()
|
|
5521
5485
|
}
|
|
5522
5486
|
}
|
|
5523
5487
|
}
|
|
@@ -5587,6 +5551,7 @@ impl From<crawlberg::InteractionResult> for InteractionResult {
|
|
|
5587
5551
|
}
|
|
5588
5552
|
}
|
|
5589
5553
|
|
|
5554
|
+
#[allow(clippy::needless_update)]
|
|
5590
5555
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5591
5556
|
impl From<ActionResult> for crawlberg::ActionResult {
|
|
5592
5557
|
fn from(val: ActionResult) -> Self {
|
|
@@ -5596,6 +5561,7 @@ impl From<ActionResult> for crawlberg::ActionResult {
|
|
|
5596
5561
|
success: val.success,
|
|
5597
5562
|
data: val.data.as_ref().and_then(|s| serde_json::from_str(s).ok()),
|
|
5598
5563
|
error: val.error,
|
|
5564
|
+
..Default::default()
|
|
5599
5565
|
}
|
|
5600
5566
|
}
|
|
5601
5567
|
}
|
|
@@ -5687,6 +5653,7 @@ impl From<crawlberg::ScrapeResult> for ScrapeResult {
|
|
|
5687
5653
|
}
|
|
5688
5654
|
}
|
|
5689
5655
|
|
|
5656
|
+
#[allow(clippy::needless_update)]
|
|
5690
5657
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5691
5658
|
impl From<CrawlPageResult> for crawlberg::CrawlPageResult {
|
|
5692
5659
|
fn from(val: CrawlPageResult) -> Self {
|
|
@@ -5712,6 +5679,7 @@ impl From<CrawlPageResult> for crawlberg::CrawlPageResult {
|
|
|
5712
5679
|
extraction_meta: val.extraction_meta.map(Into::into),
|
|
5713
5680
|
downloaded_document: val.downloaded_document.map(Into::into),
|
|
5714
5681
|
browser_used: val.browser_used,
|
|
5682
|
+
..Default::default()
|
|
5715
5683
|
}
|
|
5716
5684
|
}
|
|
5717
5685
|
}
|
|
@@ -5779,6 +5747,7 @@ impl From<crawlberg::CrawlResult> for CrawlResult {
|
|
|
5779
5747
|
}
|
|
5780
5748
|
}
|
|
5781
5749
|
|
|
5750
|
+
#[allow(clippy::needless_update)]
|
|
5782
5751
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5783
5752
|
impl From<SitemapUrl> for crawlberg::SitemapUrl {
|
|
5784
5753
|
fn from(val: SitemapUrl) -> Self {
|
|
@@ -5787,6 +5756,7 @@ impl From<SitemapUrl> for crawlberg::SitemapUrl {
|
|
|
5787
5756
|
lastmod: val.lastmod,
|
|
5788
5757
|
changefreq: val.changefreq,
|
|
5789
5758
|
priority: val.priority,
|
|
5759
|
+
..Default::default()
|
|
5790
5760
|
}
|
|
5791
5761
|
}
|
|
5792
5762
|
}
|
|
@@ -5803,11 +5773,13 @@ impl From<crawlberg::SitemapUrl> for SitemapUrl {
|
|
|
5803
5773
|
}
|
|
5804
5774
|
}
|
|
5805
5775
|
|
|
5776
|
+
#[allow(clippy::needless_update)]
|
|
5806
5777
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5807
5778
|
impl From<MapResult> for crawlberg::MapResult {
|
|
5808
5779
|
fn from(val: MapResult) -> Self {
|
|
5809
5780
|
Self {
|
|
5810
5781
|
urls: val.urls.into_iter().map(Into::into).collect(),
|
|
5782
|
+
..Default::default()
|
|
5811
5783
|
}
|
|
5812
5784
|
}
|
|
5813
5785
|
}
|
|
@@ -5821,6 +5793,7 @@ impl From<crawlberg::MapResult> for MapResult {
|
|
|
5821
5793
|
}
|
|
5822
5794
|
}
|
|
5823
5795
|
|
|
5796
|
+
#[allow(clippy::needless_update)]
|
|
5824
5797
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5825
5798
|
impl From<MarkdownResult> for crawlberg::MarkdownResult {
|
|
5826
5799
|
fn from(val: MarkdownResult) -> Self {
|
|
@@ -5838,6 +5811,7 @@ impl From<MarkdownResult> for crawlberg::MarkdownResult {
|
|
|
5838
5811
|
warnings: val.warnings.into_iter().collect(),
|
|
5839
5812
|
citations: val.citations,
|
|
5840
5813
|
fit_content: val.fit_content,
|
|
5814
|
+
..Default::default()
|
|
5841
5815
|
}
|
|
5842
5816
|
}
|
|
5843
5817
|
}
|
|
@@ -5856,6 +5830,7 @@ impl From<crawlberg::MarkdownResult> for MarkdownResult {
|
|
|
5856
5830
|
}
|
|
5857
5831
|
}
|
|
5858
5832
|
|
|
5833
|
+
#[allow(clippy::needless_update)]
|
|
5859
5834
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5860
5835
|
impl From<LinkInfo> for crawlberg::LinkInfo {
|
|
5861
5836
|
fn from(val: LinkInfo) -> Self {
|
|
@@ -5865,6 +5840,7 @@ impl From<LinkInfo> for crawlberg::LinkInfo {
|
|
|
5865
5840
|
link_type: val.link_type.into(),
|
|
5866
5841
|
rel: val.rel,
|
|
5867
5842
|
nofollow: val.nofollow,
|
|
5843
|
+
..Default::default()
|
|
5868
5844
|
}
|
|
5869
5845
|
}
|
|
5870
5846
|
}
|
|
@@ -5882,6 +5858,7 @@ impl From<crawlberg::LinkInfo> for LinkInfo {
|
|
|
5882
5858
|
}
|
|
5883
5859
|
}
|
|
5884
5860
|
|
|
5861
|
+
#[allow(clippy::needless_update)]
|
|
5885
5862
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5886
5863
|
impl From<ImageInfo> for crawlberg::ImageInfo {
|
|
5887
5864
|
fn from(val: ImageInfo) -> Self {
|
|
@@ -5891,6 +5868,7 @@ impl From<ImageInfo> for crawlberg::ImageInfo {
|
|
|
5891
5868
|
width: val.width,
|
|
5892
5869
|
height: val.height,
|
|
5893
5870
|
source: val.source.into(),
|
|
5871
|
+
..Default::default()
|
|
5894
5872
|
}
|
|
5895
5873
|
}
|
|
5896
5874
|
}
|
|
@@ -5908,6 +5886,7 @@ impl From<crawlberg::ImageInfo> for ImageInfo {
|
|
|
5908
5886
|
}
|
|
5909
5887
|
}
|
|
5910
5888
|
|
|
5889
|
+
#[allow(clippy::needless_update)]
|
|
5911
5890
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5912
5891
|
impl From<FeedInfo> for crawlberg::FeedInfo {
|
|
5913
5892
|
fn from(val: FeedInfo) -> Self {
|
|
@@ -5915,6 +5894,7 @@ impl From<FeedInfo> for crawlberg::FeedInfo {
|
|
|
5915
5894
|
url: val.url,
|
|
5916
5895
|
title: val.title,
|
|
5917
5896
|
feed_type: val.feed_type.into(),
|
|
5897
|
+
..Default::default()
|
|
5918
5898
|
}
|
|
5919
5899
|
}
|
|
5920
5900
|
}
|
|
@@ -5930,6 +5910,7 @@ impl From<crawlberg::FeedInfo> for FeedInfo {
|
|
|
5930
5910
|
}
|
|
5931
5911
|
}
|
|
5932
5912
|
|
|
5913
|
+
#[allow(clippy::needless_update)]
|
|
5933
5914
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5934
5915
|
impl From<JsonLdEntry> for crawlberg::JsonLdEntry {
|
|
5935
5916
|
fn from(val: JsonLdEntry) -> Self {
|
|
@@ -5937,6 +5918,7 @@ impl From<JsonLdEntry> for crawlberg::JsonLdEntry {
|
|
|
5937
5918
|
schema_type: val.schema_type,
|
|
5938
5919
|
name: val.name,
|
|
5939
5920
|
raw: val.raw,
|
|
5921
|
+
..Default::default()
|
|
5940
5922
|
}
|
|
5941
5923
|
}
|
|
5942
5924
|
}
|
|
@@ -5952,6 +5934,7 @@ impl From<crawlberg::JsonLdEntry> for JsonLdEntry {
|
|
|
5952
5934
|
}
|
|
5953
5935
|
}
|
|
5954
5936
|
|
|
5937
|
+
#[allow(clippy::needless_update)]
|
|
5955
5938
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5956
5939
|
impl From<CookieInfo> for crawlberg::CookieInfo {
|
|
5957
5940
|
fn from(val: CookieInfo) -> Self {
|
|
@@ -5960,6 +5943,7 @@ impl From<CookieInfo> for crawlberg::CookieInfo {
|
|
|
5960
5943
|
value: val.value,
|
|
5961
5944
|
domain: val.domain,
|
|
5962
5945
|
path: val.path,
|
|
5946
|
+
..Default::default()
|
|
5963
5947
|
}
|
|
5964
5948
|
}
|
|
5965
5949
|
}
|
|
@@ -5976,6 +5960,7 @@ impl From<crawlberg::CookieInfo> for CookieInfo {
|
|
|
5976
5960
|
}
|
|
5977
5961
|
}
|
|
5978
5962
|
|
|
5963
|
+
#[allow(clippy::needless_update)]
|
|
5979
5964
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5980
5965
|
impl From<DownloadedAsset> for crawlberg::DownloadedAsset {
|
|
5981
5966
|
fn from(val: DownloadedAsset) -> Self {
|
|
@@ -5986,6 +5971,7 @@ impl From<DownloadedAsset> for crawlberg::DownloadedAsset {
|
|
|
5986
5971
|
size: val.size,
|
|
5987
5972
|
asset_category: val.asset_category.into(),
|
|
5988
5973
|
html_tag: val.html_tag,
|
|
5974
|
+
..Default::default()
|
|
5989
5975
|
}
|
|
5990
5976
|
}
|
|
5991
5977
|
}
|
|
@@ -6004,6 +5990,7 @@ impl From<crawlberg::DownloadedAsset> for DownloadedAsset {
|
|
|
6004
5990
|
}
|
|
6005
5991
|
}
|
|
6006
5992
|
|
|
5993
|
+
#[allow(clippy::needless_update)]
|
|
6007
5994
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6008
5995
|
impl From<ArticleMetadata> for crawlberg::ArticleMetadata {
|
|
6009
5996
|
fn from(val: ArticleMetadata) -> Self {
|
|
@@ -6013,6 +6000,7 @@ impl From<ArticleMetadata> for crawlberg::ArticleMetadata {
|
|
|
6013
6000
|
author: val.author,
|
|
6014
6001
|
section: val.section,
|
|
6015
6002
|
tags: val.tags.into_iter().collect(),
|
|
6003
|
+
..Default::default()
|
|
6016
6004
|
}
|
|
6017
6005
|
}
|
|
6018
6006
|
}
|
|
@@ -6030,12 +6018,14 @@ impl From<crawlberg::ArticleMetadata> for ArticleMetadata {
|
|
|
6030
6018
|
}
|
|
6031
6019
|
}
|
|
6032
6020
|
|
|
6021
|
+
#[allow(clippy::needless_update)]
|
|
6033
6022
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6034
6023
|
impl From<HreflangEntry> for crawlberg::HreflangEntry {
|
|
6035
6024
|
fn from(val: HreflangEntry) -> Self {
|
|
6036
6025
|
Self {
|
|
6037
6026
|
lang: val.lang,
|
|
6038
6027
|
url: val.url,
|
|
6028
|
+
..Default::default()
|
|
6039
6029
|
}
|
|
6040
6030
|
}
|
|
6041
6031
|
}
|
|
@@ -6050,6 +6040,7 @@ impl From<crawlberg::HreflangEntry> for HreflangEntry {
|
|
|
6050
6040
|
}
|
|
6051
6041
|
}
|
|
6052
6042
|
|
|
6043
|
+
#[allow(clippy::needless_update)]
|
|
6053
6044
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6054
6045
|
impl From<FaviconInfo> for crawlberg::FaviconInfo {
|
|
6055
6046
|
fn from(val: FaviconInfo) -> Self {
|
|
@@ -6058,6 +6049,7 @@ impl From<FaviconInfo> for crawlberg::FaviconInfo {
|
|
|
6058
6049
|
rel: val.rel,
|
|
6059
6050
|
sizes: val.sizes,
|
|
6060
6051
|
mime_type: val.mime_type,
|
|
6052
|
+
..Default::default()
|
|
6061
6053
|
}
|
|
6062
6054
|
}
|
|
6063
6055
|
}
|
|
@@ -6074,12 +6066,14 @@ impl From<crawlberg::FaviconInfo> for FaviconInfo {
|
|
|
6074
6066
|
}
|
|
6075
6067
|
}
|
|
6076
6068
|
|
|
6069
|
+
#[allow(clippy::needless_update)]
|
|
6077
6070
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6078
6071
|
impl From<HeadingInfo> for crawlberg::HeadingInfo {
|
|
6079
6072
|
fn from(val: HeadingInfo) -> Self {
|
|
6080
6073
|
Self {
|
|
6081
6074
|
level: val.level,
|
|
6082
6075
|
text: val.text,
|
|
6076
|
+
..Default::default()
|
|
6083
6077
|
}
|
|
6084
6078
|
}
|
|
6085
6079
|
}
|
|
@@ -6094,6 +6088,7 @@ impl From<crawlberg::HeadingInfo> for HeadingInfo {
|
|
|
6094
6088
|
}
|
|
6095
6089
|
}
|
|
6096
6090
|
|
|
6091
|
+
#[allow(clippy::needless_update)]
|
|
6097
6092
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6098
6093
|
impl From<ResponseMeta> for crawlberg::ResponseMeta {
|
|
6099
6094
|
fn from(val: ResponseMeta) -> Self {
|
|
@@ -6105,6 +6100,7 @@ impl From<ResponseMeta> for crawlberg::ResponseMeta {
|
|
|
6105
6100
|
x_powered_by: val.x_powered_by,
|
|
6106
6101
|
content_language: val.content_language,
|
|
6107
6102
|
content_encoding: val.content_encoding,
|
|
6103
|
+
..Default::default()
|
|
6108
6104
|
}
|
|
6109
6105
|
}
|
|
6110
6106
|
}
|
|
@@ -6124,6 +6120,7 @@ impl From<crawlberg::ResponseMeta> for ResponseMeta {
|
|
|
6124
6120
|
}
|
|
6125
6121
|
}
|
|
6126
6122
|
|
|
6123
|
+
#[allow(clippy::needless_update)]
|
|
6127
6124
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6128
6125
|
impl From<PageMetadata> for crawlberg::PageMetadata {
|
|
6129
6126
|
fn from(val: PageMetadata) -> Self {
|
|
@@ -6171,6 +6168,7 @@ impl From<PageMetadata> for crawlberg::PageMetadata {
|
|
|
6171
6168
|
favicons: val.favicons.map(|v| v.into_iter().map(Into::into).collect()),
|
|
6172
6169
|
headings: val.headings.map(|v| v.into_iter().map(Into::into).collect()),
|
|
6173
6170
|
word_count: val.word_count,
|
|
6171
|
+
..Default::default()
|
|
6174
6172
|
}
|
|
6175
6173
|
}
|
|
6176
6174
|
}
|
|
@@ -6226,10 +6224,14 @@ impl From<crawlberg::PageMetadata> for PageMetadata {
|
|
|
6226
6224
|
}
|
|
6227
6225
|
}
|
|
6228
6226
|
|
|
6227
|
+
#[allow(clippy::needless_update)]
|
|
6229
6228
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6230
6229
|
impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
|
|
6231
6230
|
fn from(val: CrawlStreamRequest) -> Self {
|
|
6232
|
-
Self {
|
|
6231
|
+
Self {
|
|
6232
|
+
url: val.url,
|
|
6233
|
+
..Default::default()
|
|
6234
|
+
}
|
|
6233
6235
|
}
|
|
6234
6236
|
}
|
|
6235
6237
|
|
|
@@ -6242,11 +6244,13 @@ impl From<crawlberg::CrawlStreamRequest> for CrawlStreamRequest {
|
|
|
6242
6244
|
}
|
|
6243
6245
|
}
|
|
6244
6246
|
|
|
6247
|
+
#[allow(clippy::needless_update)]
|
|
6245
6248
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6246
6249
|
impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
|
|
6247
6250
|
fn from(val: BatchCrawlStreamRequest) -> Self {
|
|
6248
6251
|
Self {
|
|
6249
6252
|
urls: val.urls.into_iter().collect(),
|
|
6253
|
+
..Default::default()
|
|
6250
6254
|
}
|
|
6251
6255
|
}
|
|
6252
6256
|
}
|
|
@@ -6260,12 +6264,14 @@ impl From<crawlberg::BatchCrawlStreamRequest> for BatchCrawlStreamRequest {
|
|
|
6260
6264
|
}
|
|
6261
6265
|
}
|
|
6262
6266
|
|
|
6267
|
+
#[allow(clippy::needless_update)]
|
|
6263
6268
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6264
6269
|
impl From<CitationResult> for crawlberg::CitationResult {
|
|
6265
6270
|
fn from(val: CitationResult) -> Self {
|
|
6266
6271
|
Self {
|
|
6267
6272
|
content: val.content,
|
|
6268
6273
|
references: val.references.into_iter().map(Into::into).collect(),
|
|
6274
|
+
..Default::default()
|
|
6269
6275
|
}
|
|
6270
6276
|
}
|
|
6271
6277
|
}
|
|
@@ -6280,6 +6286,7 @@ impl From<crawlberg::CitationResult> for CitationResult {
|
|
|
6280
6286
|
}
|
|
6281
6287
|
}
|
|
6282
6288
|
|
|
6289
|
+
#[allow(clippy::needless_update)]
|
|
6283
6290
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6284
6291
|
impl From<CitationReference> for crawlberg::CitationReference {
|
|
6285
6292
|
fn from(val: CitationReference) -> Self {
|
|
@@ -6287,6 +6294,7 @@ impl From<CitationReference> for crawlberg::CitationReference {
|
|
|
6287
6294
|
index: val.index,
|
|
6288
6295
|
url: val.url,
|
|
6289
6296
|
text: val.text,
|
|
6297
|
+
..Default::default()
|
|
6290
6298
|
}
|
|
6291
6299
|
}
|
|
6292
6300
|
}
|
|
@@ -6302,6 +6310,7 @@ impl From<crawlberg::CitationReference> for CitationReference {
|
|
|
6302
6310
|
}
|
|
6303
6311
|
}
|
|
6304
6312
|
|
|
6313
|
+
#[allow(clippy::needless_update)]
|
|
6305
6314
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6306
6315
|
impl From<BatchScrapeResult> for crawlberg::BatchScrapeResult {
|
|
6307
6316
|
fn from(val: BatchScrapeResult) -> Self {
|
|
@@ -6309,6 +6318,7 @@ impl From<BatchScrapeResult> for crawlberg::BatchScrapeResult {
|
|
|
6309
6318
|
url: val.url,
|
|
6310
6319
|
result: val.result.map(Into::into),
|
|
6311
6320
|
error: val.error,
|
|
6321
|
+
..Default::default()
|
|
6312
6322
|
}
|
|
6313
6323
|
}
|
|
6314
6324
|
}
|
|
@@ -6324,6 +6334,7 @@ impl From<crawlberg::BatchScrapeResult> for BatchScrapeResult {
|
|
|
6324
6334
|
}
|
|
6325
6335
|
}
|
|
6326
6336
|
|
|
6337
|
+
#[allow(clippy::needless_update)]
|
|
6327
6338
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6328
6339
|
impl From<BatchCrawlResult> for crawlberg::BatchCrawlResult {
|
|
6329
6340
|
fn from(val: BatchCrawlResult) -> Self {
|
|
@@ -6331,6 +6342,7 @@ impl From<BatchCrawlResult> for crawlberg::BatchCrawlResult {
|
|
|
6331
6342
|
url: val.url,
|
|
6332
6343
|
result: val.result.map(Into::into),
|
|
6333
6344
|
error: val.error,
|
|
6345
|
+
..Default::default()
|
|
6334
6346
|
}
|
|
6335
6347
|
}
|
|
6336
6348
|
}
|
|
@@ -6346,6 +6358,7 @@ impl From<crawlberg::BatchCrawlResult> for BatchCrawlResult {
|
|
|
6346
6358
|
}
|
|
6347
6359
|
}
|
|
6348
6360
|
|
|
6361
|
+
#[allow(clippy::needless_update)]
|
|
6349
6362
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6350
6363
|
impl From<BatchScrapeResults> for crawlberg::BatchScrapeResults {
|
|
6351
6364
|
fn from(val: BatchScrapeResults) -> Self {
|
|
@@ -6354,6 +6367,7 @@ impl From<BatchScrapeResults> for crawlberg::BatchScrapeResults {
|
|
|
6354
6367
|
total_count: val.total_count,
|
|
6355
6368
|
completed_count: val.completed_count,
|
|
6356
6369
|
failed_count: val.failed_count,
|
|
6370
|
+
..Default::default()
|
|
6357
6371
|
}
|
|
6358
6372
|
}
|
|
6359
6373
|
}
|
|
@@ -6370,6 +6384,7 @@ impl From<crawlberg::BatchScrapeResults> for BatchScrapeResults {
|
|
|
6370
6384
|
}
|
|
6371
6385
|
}
|
|
6372
6386
|
|
|
6387
|
+
#[allow(clippy::needless_update)]
|
|
6373
6388
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6374
6389
|
impl From<BatchCrawlResults> for crawlberg::BatchCrawlResults {
|
|
6375
6390
|
fn from(val: BatchCrawlResults) -> Self {
|
|
@@ -6378,6 +6393,7 @@ impl From<BatchCrawlResults> for crawlberg::BatchCrawlResults {
|
|
|
6378
6393
|
total_count: val.total_count,
|
|
6379
6394
|
completed_count: val.completed_count,
|
|
6380
6395
|
failed_count: val.failed_count,
|
|
6396
|
+
..Default::default()
|
|
6381
6397
|
}
|
|
6382
6398
|
}
|
|
6383
6399
|
}
|
|
@@ -6766,7 +6782,6 @@ impl CrawlStreamIterator {
|
|
|
6766
6782
|
}
|
|
6767
6783
|
Some(Err(e)) => Err(Error::new(ruby.exception_runtime_error(), e.to_string())),
|
|
6768
6784
|
None => {
|
|
6769
|
-
// Drop the stream to release any resources.
|
|
6770
6785
|
let inner = self.inner.clone();
|
|
6771
6786
|
let runtime = self.runtime.clone();
|
|
6772
6787
|
runtime.block_on(async move {
|
|
@@ -6784,8 +6799,6 @@ impl CrawlStreamIterator {
|
|
|
6784
6799
|
use magnus::value::ReprValue;
|
|
6785
6800
|
let ruby = unsafe { Ruby::get_unchecked() };
|
|
6786
6801
|
if !ruby.block_given() {
|
|
6787
|
-
// Without a block, return an Enumerator over `each` so the caller can
|
|
6788
|
-
// call `.to_a`, `.lazy`, etc.
|
|
6789
6802
|
let self_val: magnus::Value = self.clone().into_value_with(&ruby);
|
|
6790
6803
|
let enumerator = self_val.enumeratorize(ruby.to_symbol("each"), ());
|
|
6791
6804
|
return Ok(enumerator.as_value());
|
|
@@ -6848,7 +6861,6 @@ impl BatchCrawlStreamIterator {
|
|
|
6848
6861
|
}
|
|
6849
6862
|
Some(Err(e)) => Err(Error::new(ruby.exception_runtime_error(), e.to_string())),
|
|
6850
6863
|
None => {
|
|
6851
|
-
// Drop the stream to release any resources.
|
|
6852
6864
|
let inner = self.inner.clone();
|
|
6853
6865
|
let runtime = self.runtime.clone();
|
|
6854
6866
|
runtime.block_on(async move {
|
|
@@ -6866,8 +6878,6 @@ impl BatchCrawlStreamIterator {
|
|
|
6866
6878
|
use magnus::value::ReprValue;
|
|
6867
6879
|
let ruby = unsafe { Ruby::get_unchecked() };
|
|
6868
6880
|
if !ruby.block_given() {
|
|
6869
|
-
// Without a block, return an Enumerator over `each` so the caller can
|
|
6870
|
-
// call `.to_a`, `.lazy`, etc.
|
|
6871
6881
|
let self_val: magnus::Value = self.clone().into_value_with(&ruby);
|
|
6872
6882
|
let enumerator = self_val.enumeratorize(ruby.to_symbol("each"), ());
|
|
6873
6883
|
return Ok(enumerator.as_value());
|
|
@@ -6887,7 +6897,6 @@ impl BatchCrawlStreamIterator {
|
|
|
6887
6897
|
fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
|
|
6888
6898
|
let module = ruby.define_module("Crawlberg")?;
|
|
6889
6899
|
|
|
6890
|
-
// Ensure JSON library is loaded for Hash#to_json
|
|
6891
6900
|
let _ = ruby.eval::<magnus::Value>("require \"json\"");
|
|
6892
6901
|
|
|
6893
6902
|
let class = module.define_class("ExtractionMeta", ruby.class_object())?;
|