crawlberg 1.0.3 → 1.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/ext/crawlberg_rb/native/Cargo.lock +3 -3
- data/ext/crawlberg_rb/native/Cargo.toml +2 -2
- data/ext/crawlberg_rb/src/lib.rs +65 -2
- data/lib/crawlberg/native.rb +1 -1
- data/lib/crawlberg/version.rb +2 -2
- data/lib/crawlberg.rb +1 -1
- data/lib/crawlberg_rb.so +0 -0
- data/sig/types.rbs +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 55dbf400c68cc45ca0e71f057db854c1723c355bc6be94e672cacc23cdd87494
|
|
4
|
+
data.tar.gz: 27a4fe5a9c53c859a54577a2c0f05ae5c3648d0252c20880db3d919c9d2f26d9
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a76604a73de927275a7a477d557e7cb206164b7db4219241b0a4ad4397ac4db924a9b084009392d98736e983767056088bf19f3a600f86339cd3066dbed66c74
|
|
7
|
+
data.tar.gz: 00c05f3ec86ecffdd90006f740b6847875ca7f23872a9709d3d8612aa25503dcf394647e63a32730ba9f7b1e142f81691de67f6fc0e21231692781715d0694a2
|
data/README.md
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
|
|
10
10
|
<div align="center" style="display: flex; flex-wrap: wrap; gap: 8px; justify-content: center; margin: 20px 0;">
|
|
11
11
|
<a href="https://github.com/xberg-io/alef">
|
|
12
|
-
<img src="https://img.shields.io/badge/
|
|
12
|
+
<img src="https://img.shields.io/badge/built%20with-alef%20%D7%90-007ec6" alt="Built with alef">
|
|
13
13
|
</a>
|
|
14
14
|
<!-- Language Bindings -->
|
|
15
15
|
<a href="https://crates.io/crates/crawlberg">
|
|
@@ -545,9 +545,9 @@ dependencies = [
|
|
|
545
545
|
|
|
546
546
|
[[package]]
|
|
547
547
|
name = "crawlberg"
|
|
548
|
-
version = "1.0.
|
|
548
|
+
version = "1.0.5"
|
|
549
549
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
|
550
|
-
checksum = "
|
|
550
|
+
checksum = "2200ee1bcca212516ab5d0900a0c108c50e999295316c8351c52b7ea8e2a504c"
|
|
551
551
|
dependencies = [
|
|
552
552
|
"ahash",
|
|
553
553
|
"aho-corasick",
|
|
@@ -587,7 +587,7 @@ dependencies = [
|
|
|
587
587
|
|
|
588
588
|
[[package]]
|
|
589
589
|
name = "crawlberg-rb"
|
|
590
|
-
version = "1.0.
|
|
590
|
+
version = "1.0.4"
|
|
591
591
|
dependencies = [
|
|
592
592
|
"crawlberg",
|
|
593
593
|
"futures",
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[package]
|
|
2
2
|
name = "crawlberg-rb"
|
|
3
|
-
version = "1.0.
|
|
3
|
+
version = "1.0.4"
|
|
4
4
|
edition = "2024"
|
|
5
5
|
license = "MIT"
|
|
6
6
|
description = "High-performance web crawling engine"
|
|
@@ -17,7 +17,7 @@ path = "../src/lib.rs"
|
|
|
17
17
|
crate-type = ["cdylib"]
|
|
18
18
|
|
|
19
19
|
[dependencies]
|
|
20
|
-
crawlberg = { version = "1.0.
|
|
20
|
+
crawlberg = { version = "1.0.5", features = ["interact", "browser-chromiumoxide"] }
|
|
21
21
|
futures = "0.3"
|
|
22
22
|
magnus = "0.8"
|
|
23
23
|
rb-sys = ">=0.9, <0.9.128"
|
data/ext/crawlberg_rb/src/lib.rs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
// This file is auto-generated by alef. DO NOT EDIT.
|
|
2
|
-
// alef:hash:
|
|
2
|
+
// alef:hash:ed6753aa35e27ad4764a749b35d849473835fb530fcba666ac607635b1333c06
|
|
3
3
|
// Re-generate with: alef generate
|
|
4
4
|
#![allow(dead_code, unused_imports, unused_variables)]
|
|
5
5
|
#![allow(
|
|
@@ -2040,6 +2040,7 @@ impl CrawlResult {
|
|
|
2040
2040
|
}
|
|
2041
2041
|
|
|
2042
2042
|
fn unique_normalized_urls(&self) -> usize {
|
|
2043
|
+
#[allow(clippy::needless_update)]
|
|
2043
2044
|
let core_self = crawlberg::CrawlResult {
|
|
2044
2045
|
pages: self.pages.clone().into_iter().map(Into::into).collect(),
|
|
2045
2046
|
|
|
@@ -5277,6 +5278,7 @@ fn batch_crawl_stream(engine: CrawlEngineHandle, req: BatchCrawlStreamRequest) -
|
|
|
5277
5278
|
engine.batch_crawl_stream(req)
|
|
5278
5279
|
}
|
|
5279
5280
|
|
|
5281
|
+
#[allow(clippy::needless_update)]
|
|
5280
5282
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5281
5283
|
impl From<ExtractionMeta> for crawlberg::ExtractionMeta {
|
|
5282
5284
|
fn from(val: ExtractionMeta) -> Self {
|
|
@@ -5286,6 +5288,7 @@ impl From<ExtractionMeta> for crawlberg::ExtractionMeta {
|
|
|
5286
5288
|
completion_tokens: val.completion_tokens,
|
|
5287
5289
|
model: val.model,
|
|
5288
5290
|
chunks_processed: val.chunks_processed,
|
|
5291
|
+
..Default::default()
|
|
5289
5292
|
}
|
|
5290
5293
|
}
|
|
5291
5294
|
}
|
|
@@ -5303,6 +5306,7 @@ impl From<crawlberg::ExtractionMeta> for ExtractionMeta {
|
|
|
5303
5306
|
}
|
|
5304
5307
|
}
|
|
5305
5308
|
|
|
5309
|
+
#[allow(clippy::needless_update)]
|
|
5306
5310
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5307
5311
|
impl From<ProxyConfig> for crawlberg::ProxyConfig {
|
|
5308
5312
|
fn from(val: ProxyConfig) -> Self {
|
|
@@ -5310,6 +5314,7 @@ impl From<ProxyConfig> for crawlberg::ProxyConfig {
|
|
|
5310
5314
|
url: val.url,
|
|
5311
5315
|
username: val.username,
|
|
5312
5316
|
password: val.password,
|
|
5317
|
+
..Default::default()
|
|
5313
5318
|
}
|
|
5314
5319
|
}
|
|
5315
5320
|
}
|
|
@@ -5325,6 +5330,7 @@ impl From<crawlberg::ProxyConfig> for ProxyConfig {
|
|
|
5325
5330
|
}
|
|
5326
5331
|
}
|
|
5327
5332
|
|
|
5333
|
+
#[allow(clippy::needless_update)]
|
|
5328
5334
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5329
5335
|
impl From<ContentConfig> for crawlberg::ContentConfig {
|
|
5330
5336
|
fn from(val: ContentConfig) -> Self {
|
|
@@ -5341,6 +5347,7 @@ impl From<ContentConfig> for crawlberg::ContentConfig {
|
|
|
5341
5347
|
wrap: val.wrap,
|
|
5342
5348
|
wrap_width: val.wrap_width,
|
|
5343
5349
|
include_document_structure: val.include_document_structure,
|
|
5350
|
+
..Default::default()
|
|
5344
5351
|
}
|
|
5345
5352
|
}
|
|
5346
5353
|
}
|
|
@@ -5365,6 +5372,7 @@ impl From<crawlberg::ContentConfig> for ContentConfig {
|
|
|
5365
5372
|
}
|
|
5366
5373
|
}
|
|
5367
5374
|
|
|
5375
|
+
#[allow(clippy::needless_update)]
|
|
5368
5376
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5369
5377
|
impl From<BrowserConfig> for crawlberg::BrowserConfig {
|
|
5370
5378
|
fn from(val: BrowserConfig) -> Self {
|
|
@@ -5382,6 +5390,7 @@ impl From<BrowserConfig> for crawlberg::BrowserConfig {
|
|
|
5382
5390
|
robots_user_agent: val.robots_user_agent,
|
|
5383
5391
|
capture_network_events: val.capture_network_events,
|
|
5384
5392
|
session_affinity: val.session_affinity,
|
|
5393
|
+
..Default::default()
|
|
5385
5394
|
}
|
|
5386
5395
|
}
|
|
5387
5396
|
}
|
|
@@ -5511,6 +5520,7 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
|
|
|
5511
5520
|
}
|
|
5512
5521
|
}
|
|
5513
5522
|
|
|
5523
|
+
#[allow(clippy::needless_update)]
|
|
5514
5524
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5515
5525
|
impl From<BrowserExtras> for crawlberg::BrowserExtras {
|
|
5516
5526
|
fn from(val: BrowserExtras) -> Self {
|
|
@@ -5518,6 +5528,7 @@ impl From<BrowserExtras> for crawlberg::BrowserExtras {
|
|
|
5518
5528
|
eval_result: val.eval_result.as_ref().and_then(|s| serde_json::from_str(s).ok()),
|
|
5519
5529
|
network_events: val.network_events.into_iter().map(Into::into).collect(),
|
|
5520
5530
|
cookies: val.cookies.into_iter().map(Into::into).collect(),
|
|
5531
|
+
..Default::default()
|
|
5521
5532
|
}
|
|
5522
5533
|
}
|
|
5523
5534
|
}
|
|
@@ -5587,6 +5598,7 @@ impl From<crawlberg::InteractionResult> for InteractionResult {
|
|
|
5587
5598
|
}
|
|
5588
5599
|
}
|
|
5589
5600
|
|
|
5601
|
+
#[allow(clippy::needless_update)]
|
|
5590
5602
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5591
5603
|
impl From<ActionResult> for crawlberg::ActionResult {
|
|
5592
5604
|
fn from(val: ActionResult) -> Self {
|
|
@@ -5596,6 +5608,7 @@ impl From<ActionResult> for crawlberg::ActionResult {
|
|
|
5596
5608
|
success: val.success,
|
|
5597
5609
|
data: val.data.as_ref().and_then(|s| serde_json::from_str(s).ok()),
|
|
5598
5610
|
error: val.error,
|
|
5611
|
+
..Default::default()
|
|
5599
5612
|
}
|
|
5600
5613
|
}
|
|
5601
5614
|
}
|
|
@@ -5687,6 +5700,7 @@ impl From<crawlberg::ScrapeResult> for ScrapeResult {
|
|
|
5687
5700
|
}
|
|
5688
5701
|
}
|
|
5689
5702
|
|
|
5703
|
+
#[allow(clippy::needless_update)]
|
|
5690
5704
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5691
5705
|
impl From<CrawlPageResult> for crawlberg::CrawlPageResult {
|
|
5692
5706
|
fn from(val: CrawlPageResult) -> Self {
|
|
@@ -5712,6 +5726,7 @@ impl From<CrawlPageResult> for crawlberg::CrawlPageResult {
|
|
|
5712
5726
|
extraction_meta: val.extraction_meta.map(Into::into),
|
|
5713
5727
|
downloaded_document: val.downloaded_document.map(Into::into),
|
|
5714
5728
|
browser_used: val.browser_used,
|
|
5729
|
+
..Default::default()
|
|
5715
5730
|
}
|
|
5716
5731
|
}
|
|
5717
5732
|
}
|
|
@@ -5779,6 +5794,7 @@ impl From<crawlberg::CrawlResult> for CrawlResult {
|
|
|
5779
5794
|
}
|
|
5780
5795
|
}
|
|
5781
5796
|
|
|
5797
|
+
#[allow(clippy::needless_update)]
|
|
5782
5798
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5783
5799
|
impl From<SitemapUrl> for crawlberg::SitemapUrl {
|
|
5784
5800
|
fn from(val: SitemapUrl) -> Self {
|
|
@@ -5787,6 +5803,7 @@ impl From<SitemapUrl> for crawlberg::SitemapUrl {
|
|
|
5787
5803
|
lastmod: val.lastmod,
|
|
5788
5804
|
changefreq: val.changefreq,
|
|
5789
5805
|
priority: val.priority,
|
|
5806
|
+
..Default::default()
|
|
5790
5807
|
}
|
|
5791
5808
|
}
|
|
5792
5809
|
}
|
|
@@ -5803,11 +5820,13 @@ impl From<crawlberg::SitemapUrl> for SitemapUrl {
|
|
|
5803
5820
|
}
|
|
5804
5821
|
}
|
|
5805
5822
|
|
|
5823
|
+
#[allow(clippy::needless_update)]
|
|
5806
5824
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5807
5825
|
impl From<MapResult> for crawlberg::MapResult {
|
|
5808
5826
|
fn from(val: MapResult) -> Self {
|
|
5809
5827
|
Self {
|
|
5810
5828
|
urls: val.urls.into_iter().map(Into::into).collect(),
|
|
5829
|
+
..Default::default()
|
|
5811
5830
|
}
|
|
5812
5831
|
}
|
|
5813
5832
|
}
|
|
@@ -5821,6 +5840,7 @@ impl From<crawlberg::MapResult> for MapResult {
|
|
|
5821
5840
|
}
|
|
5822
5841
|
}
|
|
5823
5842
|
|
|
5843
|
+
#[allow(clippy::needless_update)]
|
|
5824
5844
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5825
5845
|
impl From<MarkdownResult> for crawlberg::MarkdownResult {
|
|
5826
5846
|
fn from(val: MarkdownResult) -> Self {
|
|
@@ -5838,6 +5858,7 @@ impl From<MarkdownResult> for crawlberg::MarkdownResult {
|
|
|
5838
5858
|
warnings: val.warnings.into_iter().collect(),
|
|
5839
5859
|
citations: val.citations,
|
|
5840
5860
|
fit_content: val.fit_content,
|
|
5861
|
+
..Default::default()
|
|
5841
5862
|
}
|
|
5842
5863
|
}
|
|
5843
5864
|
}
|
|
@@ -5856,6 +5877,7 @@ impl From<crawlberg::MarkdownResult> for MarkdownResult {
|
|
|
5856
5877
|
}
|
|
5857
5878
|
}
|
|
5858
5879
|
|
|
5880
|
+
#[allow(clippy::needless_update)]
|
|
5859
5881
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5860
5882
|
impl From<LinkInfo> for crawlberg::LinkInfo {
|
|
5861
5883
|
fn from(val: LinkInfo) -> Self {
|
|
@@ -5865,6 +5887,7 @@ impl From<LinkInfo> for crawlberg::LinkInfo {
|
|
|
5865
5887
|
link_type: val.link_type.into(),
|
|
5866
5888
|
rel: val.rel,
|
|
5867
5889
|
nofollow: val.nofollow,
|
|
5890
|
+
..Default::default()
|
|
5868
5891
|
}
|
|
5869
5892
|
}
|
|
5870
5893
|
}
|
|
@@ -5882,6 +5905,7 @@ impl From<crawlberg::LinkInfo> for LinkInfo {
|
|
|
5882
5905
|
}
|
|
5883
5906
|
}
|
|
5884
5907
|
|
|
5908
|
+
#[allow(clippy::needless_update)]
|
|
5885
5909
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5886
5910
|
impl From<ImageInfo> for crawlberg::ImageInfo {
|
|
5887
5911
|
fn from(val: ImageInfo) -> Self {
|
|
@@ -5891,6 +5915,7 @@ impl From<ImageInfo> for crawlberg::ImageInfo {
|
|
|
5891
5915
|
width: val.width,
|
|
5892
5916
|
height: val.height,
|
|
5893
5917
|
source: val.source.into(),
|
|
5918
|
+
..Default::default()
|
|
5894
5919
|
}
|
|
5895
5920
|
}
|
|
5896
5921
|
}
|
|
@@ -5908,6 +5933,7 @@ impl From<crawlberg::ImageInfo> for ImageInfo {
|
|
|
5908
5933
|
}
|
|
5909
5934
|
}
|
|
5910
5935
|
|
|
5936
|
+
#[allow(clippy::needless_update)]
|
|
5911
5937
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5912
5938
|
impl From<FeedInfo> for crawlberg::FeedInfo {
|
|
5913
5939
|
fn from(val: FeedInfo) -> Self {
|
|
@@ -5915,6 +5941,7 @@ impl From<FeedInfo> for crawlberg::FeedInfo {
|
|
|
5915
5941
|
url: val.url,
|
|
5916
5942
|
title: val.title,
|
|
5917
5943
|
feed_type: val.feed_type.into(),
|
|
5944
|
+
..Default::default()
|
|
5918
5945
|
}
|
|
5919
5946
|
}
|
|
5920
5947
|
}
|
|
@@ -5930,6 +5957,7 @@ impl From<crawlberg::FeedInfo> for FeedInfo {
|
|
|
5930
5957
|
}
|
|
5931
5958
|
}
|
|
5932
5959
|
|
|
5960
|
+
#[allow(clippy::needless_update)]
|
|
5933
5961
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5934
5962
|
impl From<JsonLdEntry> for crawlberg::JsonLdEntry {
|
|
5935
5963
|
fn from(val: JsonLdEntry) -> Self {
|
|
@@ -5937,6 +5965,7 @@ impl From<JsonLdEntry> for crawlberg::JsonLdEntry {
|
|
|
5937
5965
|
schema_type: val.schema_type,
|
|
5938
5966
|
name: val.name,
|
|
5939
5967
|
raw: val.raw,
|
|
5968
|
+
..Default::default()
|
|
5940
5969
|
}
|
|
5941
5970
|
}
|
|
5942
5971
|
}
|
|
@@ -5952,6 +5981,7 @@ impl From<crawlberg::JsonLdEntry> for JsonLdEntry {
|
|
|
5952
5981
|
}
|
|
5953
5982
|
}
|
|
5954
5983
|
|
|
5984
|
+
#[allow(clippy::needless_update)]
|
|
5955
5985
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5956
5986
|
impl From<CookieInfo> for crawlberg::CookieInfo {
|
|
5957
5987
|
fn from(val: CookieInfo) -> Self {
|
|
@@ -5960,6 +5990,7 @@ impl From<CookieInfo> for crawlberg::CookieInfo {
|
|
|
5960
5990
|
value: val.value,
|
|
5961
5991
|
domain: val.domain,
|
|
5962
5992
|
path: val.path,
|
|
5993
|
+
..Default::default()
|
|
5963
5994
|
}
|
|
5964
5995
|
}
|
|
5965
5996
|
}
|
|
@@ -5976,6 +6007,7 @@ impl From<crawlberg::CookieInfo> for CookieInfo {
|
|
|
5976
6007
|
}
|
|
5977
6008
|
}
|
|
5978
6009
|
|
|
6010
|
+
#[allow(clippy::needless_update)]
|
|
5979
6011
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
5980
6012
|
impl From<DownloadedAsset> for crawlberg::DownloadedAsset {
|
|
5981
6013
|
fn from(val: DownloadedAsset) -> Self {
|
|
@@ -5986,6 +6018,7 @@ impl From<DownloadedAsset> for crawlberg::DownloadedAsset {
|
|
|
5986
6018
|
size: val.size,
|
|
5987
6019
|
asset_category: val.asset_category.into(),
|
|
5988
6020
|
html_tag: val.html_tag,
|
|
6021
|
+
..Default::default()
|
|
5989
6022
|
}
|
|
5990
6023
|
}
|
|
5991
6024
|
}
|
|
@@ -6004,6 +6037,7 @@ impl From<crawlberg::DownloadedAsset> for DownloadedAsset {
|
|
|
6004
6037
|
}
|
|
6005
6038
|
}
|
|
6006
6039
|
|
|
6040
|
+
#[allow(clippy::needless_update)]
|
|
6007
6041
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6008
6042
|
impl From<ArticleMetadata> for crawlberg::ArticleMetadata {
|
|
6009
6043
|
fn from(val: ArticleMetadata) -> Self {
|
|
@@ -6013,6 +6047,7 @@ impl From<ArticleMetadata> for crawlberg::ArticleMetadata {
|
|
|
6013
6047
|
author: val.author,
|
|
6014
6048
|
section: val.section,
|
|
6015
6049
|
tags: val.tags.into_iter().collect(),
|
|
6050
|
+
..Default::default()
|
|
6016
6051
|
}
|
|
6017
6052
|
}
|
|
6018
6053
|
}
|
|
@@ -6030,12 +6065,14 @@ impl From<crawlberg::ArticleMetadata> for ArticleMetadata {
|
|
|
6030
6065
|
}
|
|
6031
6066
|
}
|
|
6032
6067
|
|
|
6068
|
+
#[allow(clippy::needless_update)]
|
|
6033
6069
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6034
6070
|
impl From<HreflangEntry> for crawlberg::HreflangEntry {
|
|
6035
6071
|
fn from(val: HreflangEntry) -> Self {
|
|
6036
6072
|
Self {
|
|
6037
6073
|
lang: val.lang,
|
|
6038
6074
|
url: val.url,
|
|
6075
|
+
..Default::default()
|
|
6039
6076
|
}
|
|
6040
6077
|
}
|
|
6041
6078
|
}
|
|
@@ -6050,6 +6087,7 @@ impl From<crawlberg::HreflangEntry> for HreflangEntry {
|
|
|
6050
6087
|
}
|
|
6051
6088
|
}
|
|
6052
6089
|
|
|
6090
|
+
#[allow(clippy::needless_update)]
|
|
6053
6091
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6054
6092
|
impl From<FaviconInfo> for crawlberg::FaviconInfo {
|
|
6055
6093
|
fn from(val: FaviconInfo) -> Self {
|
|
@@ -6058,6 +6096,7 @@ impl From<FaviconInfo> for crawlberg::FaviconInfo {
|
|
|
6058
6096
|
rel: val.rel,
|
|
6059
6097
|
sizes: val.sizes,
|
|
6060
6098
|
mime_type: val.mime_type,
|
|
6099
|
+
..Default::default()
|
|
6061
6100
|
}
|
|
6062
6101
|
}
|
|
6063
6102
|
}
|
|
@@ -6074,12 +6113,14 @@ impl From<crawlberg::FaviconInfo> for FaviconInfo {
|
|
|
6074
6113
|
}
|
|
6075
6114
|
}
|
|
6076
6115
|
|
|
6116
|
+
#[allow(clippy::needless_update)]
|
|
6077
6117
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6078
6118
|
impl From<HeadingInfo> for crawlberg::HeadingInfo {
|
|
6079
6119
|
fn from(val: HeadingInfo) -> Self {
|
|
6080
6120
|
Self {
|
|
6081
6121
|
level: val.level,
|
|
6082
6122
|
text: val.text,
|
|
6123
|
+
..Default::default()
|
|
6083
6124
|
}
|
|
6084
6125
|
}
|
|
6085
6126
|
}
|
|
@@ -6094,6 +6135,7 @@ impl From<crawlberg::HeadingInfo> for HeadingInfo {
|
|
|
6094
6135
|
}
|
|
6095
6136
|
}
|
|
6096
6137
|
|
|
6138
|
+
#[allow(clippy::needless_update)]
|
|
6097
6139
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6098
6140
|
impl From<ResponseMeta> for crawlberg::ResponseMeta {
|
|
6099
6141
|
fn from(val: ResponseMeta) -> Self {
|
|
@@ -6105,6 +6147,7 @@ impl From<ResponseMeta> for crawlberg::ResponseMeta {
|
|
|
6105
6147
|
x_powered_by: val.x_powered_by,
|
|
6106
6148
|
content_language: val.content_language,
|
|
6107
6149
|
content_encoding: val.content_encoding,
|
|
6150
|
+
..Default::default()
|
|
6108
6151
|
}
|
|
6109
6152
|
}
|
|
6110
6153
|
}
|
|
@@ -6124,6 +6167,7 @@ impl From<crawlberg::ResponseMeta> for ResponseMeta {
|
|
|
6124
6167
|
}
|
|
6125
6168
|
}
|
|
6126
6169
|
|
|
6170
|
+
#[allow(clippy::needless_update)]
|
|
6127
6171
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6128
6172
|
impl From<PageMetadata> for crawlberg::PageMetadata {
|
|
6129
6173
|
fn from(val: PageMetadata) -> Self {
|
|
@@ -6171,6 +6215,7 @@ impl From<PageMetadata> for crawlberg::PageMetadata {
|
|
|
6171
6215
|
favicons: val.favicons.map(|v| v.into_iter().map(Into::into).collect()),
|
|
6172
6216
|
headings: val.headings.map(|v| v.into_iter().map(Into::into).collect()),
|
|
6173
6217
|
word_count: val.word_count,
|
|
6218
|
+
..Default::default()
|
|
6174
6219
|
}
|
|
6175
6220
|
}
|
|
6176
6221
|
}
|
|
@@ -6226,10 +6271,14 @@ impl From<crawlberg::PageMetadata> for PageMetadata {
|
|
|
6226
6271
|
}
|
|
6227
6272
|
}
|
|
6228
6273
|
|
|
6274
|
+
#[allow(clippy::needless_update)]
|
|
6229
6275
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6230
6276
|
impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
|
|
6231
6277
|
fn from(val: CrawlStreamRequest) -> Self {
|
|
6232
|
-
Self {
|
|
6278
|
+
Self {
|
|
6279
|
+
url: val.url,
|
|
6280
|
+
..Default::default()
|
|
6281
|
+
}
|
|
6233
6282
|
}
|
|
6234
6283
|
}
|
|
6235
6284
|
|
|
@@ -6242,11 +6291,13 @@ impl From<crawlberg::CrawlStreamRequest> for CrawlStreamRequest {
|
|
|
6242
6291
|
}
|
|
6243
6292
|
}
|
|
6244
6293
|
|
|
6294
|
+
#[allow(clippy::needless_update)]
|
|
6245
6295
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6246
6296
|
impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
|
|
6247
6297
|
fn from(val: BatchCrawlStreamRequest) -> Self {
|
|
6248
6298
|
Self {
|
|
6249
6299
|
urls: val.urls.into_iter().collect(),
|
|
6300
|
+
..Default::default()
|
|
6250
6301
|
}
|
|
6251
6302
|
}
|
|
6252
6303
|
}
|
|
@@ -6260,12 +6311,14 @@ impl From<crawlberg::BatchCrawlStreamRequest> for BatchCrawlStreamRequest {
|
|
|
6260
6311
|
}
|
|
6261
6312
|
}
|
|
6262
6313
|
|
|
6314
|
+
#[allow(clippy::needless_update)]
|
|
6263
6315
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6264
6316
|
impl From<CitationResult> for crawlberg::CitationResult {
|
|
6265
6317
|
fn from(val: CitationResult) -> Self {
|
|
6266
6318
|
Self {
|
|
6267
6319
|
content: val.content,
|
|
6268
6320
|
references: val.references.into_iter().map(Into::into).collect(),
|
|
6321
|
+
..Default::default()
|
|
6269
6322
|
}
|
|
6270
6323
|
}
|
|
6271
6324
|
}
|
|
@@ -6280,6 +6333,7 @@ impl From<crawlberg::CitationResult> for CitationResult {
|
|
|
6280
6333
|
}
|
|
6281
6334
|
}
|
|
6282
6335
|
|
|
6336
|
+
#[allow(clippy::needless_update)]
|
|
6283
6337
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6284
6338
|
impl From<CitationReference> for crawlberg::CitationReference {
|
|
6285
6339
|
fn from(val: CitationReference) -> Self {
|
|
@@ -6287,6 +6341,7 @@ impl From<CitationReference> for crawlberg::CitationReference {
|
|
|
6287
6341
|
index: val.index,
|
|
6288
6342
|
url: val.url,
|
|
6289
6343
|
text: val.text,
|
|
6344
|
+
..Default::default()
|
|
6290
6345
|
}
|
|
6291
6346
|
}
|
|
6292
6347
|
}
|
|
@@ -6302,6 +6357,7 @@ impl From<crawlberg::CitationReference> for CitationReference {
|
|
|
6302
6357
|
}
|
|
6303
6358
|
}
|
|
6304
6359
|
|
|
6360
|
+
#[allow(clippy::needless_update)]
|
|
6305
6361
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6306
6362
|
impl From<BatchScrapeResult> for crawlberg::BatchScrapeResult {
|
|
6307
6363
|
fn from(val: BatchScrapeResult) -> Self {
|
|
@@ -6309,6 +6365,7 @@ impl From<BatchScrapeResult> for crawlberg::BatchScrapeResult {
|
|
|
6309
6365
|
url: val.url,
|
|
6310
6366
|
result: val.result.map(Into::into),
|
|
6311
6367
|
error: val.error,
|
|
6368
|
+
..Default::default()
|
|
6312
6369
|
}
|
|
6313
6370
|
}
|
|
6314
6371
|
}
|
|
@@ -6324,6 +6381,7 @@ impl From<crawlberg::BatchScrapeResult> for BatchScrapeResult {
|
|
|
6324
6381
|
}
|
|
6325
6382
|
}
|
|
6326
6383
|
|
|
6384
|
+
#[allow(clippy::needless_update)]
|
|
6327
6385
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6328
6386
|
impl From<BatchCrawlResult> for crawlberg::BatchCrawlResult {
|
|
6329
6387
|
fn from(val: BatchCrawlResult) -> Self {
|
|
@@ -6331,6 +6389,7 @@ impl From<BatchCrawlResult> for crawlberg::BatchCrawlResult {
|
|
|
6331
6389
|
url: val.url,
|
|
6332
6390
|
result: val.result.map(Into::into),
|
|
6333
6391
|
error: val.error,
|
|
6392
|
+
..Default::default()
|
|
6334
6393
|
}
|
|
6335
6394
|
}
|
|
6336
6395
|
}
|
|
@@ -6346,6 +6405,7 @@ impl From<crawlberg::BatchCrawlResult> for BatchCrawlResult {
|
|
|
6346
6405
|
}
|
|
6347
6406
|
}
|
|
6348
6407
|
|
|
6408
|
+
#[allow(clippy::needless_update)]
|
|
6349
6409
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6350
6410
|
impl From<BatchScrapeResults> for crawlberg::BatchScrapeResults {
|
|
6351
6411
|
fn from(val: BatchScrapeResults) -> Self {
|
|
@@ -6354,6 +6414,7 @@ impl From<BatchScrapeResults> for crawlberg::BatchScrapeResults {
|
|
|
6354
6414
|
total_count: val.total_count,
|
|
6355
6415
|
completed_count: val.completed_count,
|
|
6356
6416
|
failed_count: val.failed_count,
|
|
6417
|
+
..Default::default()
|
|
6357
6418
|
}
|
|
6358
6419
|
}
|
|
6359
6420
|
}
|
|
@@ -6370,6 +6431,7 @@ impl From<crawlberg::BatchScrapeResults> for BatchScrapeResults {
|
|
|
6370
6431
|
}
|
|
6371
6432
|
}
|
|
6372
6433
|
|
|
6434
|
+
#[allow(clippy::needless_update)]
|
|
6373
6435
|
#[allow(clippy::redundant_closure, clippy::useless_conversion)]
|
|
6374
6436
|
impl From<BatchCrawlResults> for crawlberg::BatchCrawlResults {
|
|
6375
6437
|
fn from(val: BatchCrawlResults) -> Self {
|
|
@@ -6378,6 +6440,7 @@ impl From<BatchCrawlResults> for crawlberg::BatchCrawlResults {
|
|
|
6378
6440
|
total_count: val.total_count,
|
|
6379
6441
|
completed_count: val.completed_count,
|
|
6380
6442
|
failed_count: val.failed_count,
|
|
6443
|
+
..Default::default()
|
|
6381
6444
|
}
|
|
6382
6445
|
}
|
|
6383
6446
|
}
|
data/lib/crawlberg/native.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:ed6753aa35e27ad4764a749b35d849473835fb530fcba666ac607635b1333c06
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify --exit-code
|
|
5
5
|
# frozen_string_literal: true
|
data/lib/crawlberg/version.rb
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:ed6753aa35e27ad4764a749b35d849473835fb530fcba666ac607635b1333c06
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify --exit-code
|
|
5
5
|
# frozen_string_literal: true
|
|
6
6
|
|
|
7
7
|
module Crawlberg
|
|
8
8
|
## The version string for this package.
|
|
9
|
-
VERSION = "1.0.
|
|
9
|
+
VERSION = "1.0.5"
|
|
10
10
|
end
|
data/lib/crawlberg.rb
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:ed6753aa35e27ad4764a749b35d849473835fb530fcba666ac607635b1333c06
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify --exit-code
|
|
5
5
|
# frozen_string_literal: true
|
data/lib/crawlberg_rb.so
CHANGED
|
Binary file
|
data/sig/types.rbs
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# This file is auto-generated by alef — DO NOT EDIT.
|
|
2
|
-
# alef:hash:
|
|
2
|
+
# alef:hash:ed6753aa35e27ad4764a749b35d849473835fb530fcba666ac607635b1333c06
|
|
3
3
|
# To regenerate: alef generate
|
|
4
4
|
# To verify freshness: alef verify --exit-code
|
|
5
5
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: crawlberg
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.0.
|
|
4
|
+
version: 1.0.5
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Xberg Team
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2026-07-
|
|
11
|
+
date: 2026-07-09 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: rb_sys
|