crawlberg 1.0.4 → 1.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f778a8beaecb355878084fef537cad125194cdc38f55a384bb328be27723a530
-  data.tar.gz: f494cc2344805c7f58a8f2cd43b85a20f0a86b92fcdf586f947ff06732b7e610
+  metadata.gz: 55dbf400c68cc45ca0e71f057db854c1723c355bc6be94e672cacc23cdd87494
+  data.tar.gz: 27a4fe5a9c53c859a54577a2c0f05ae5c3648d0252c20880db3d919c9d2f26d9
 SHA512:
-  metadata.gz: d0b7a63ca773ccdc7ce5753a45e76711005510216214ccf60dee17f864ed483b6cfdcafb33920730ca2ae5c794dcb99719d55794cf26bbb50b6d9d752f9c0bcc
-  data.tar.gz: df8b06f3f515933440cc3882df3933628a9625e837e46ac3ea0b4ac2f2923c7559b01c5bedde289f00051251ad1ee6595ba2595b697d388d60fd3363207564bb
+  metadata.gz: a76604a73de927275a7a477d557e7cb206164b7db4219241b0a4ad4397ac4db924a9b084009392d98736e983767056088bf19f3a600f86339cd3066dbed66c74
+  data.tar.gz: 00c05f3ec86ecffdd90006f740b6847875ca7f23872a9709d3d8612aa25503dcf394647e63a32730ba9f7b1e142f81691de67f6fc0e21231692781715d0694a2

data/ext/crawlberg_rb/native/Cargo.lock

@@ -545,9 +545,9 @@ dependencies = [
 
 [[package]]
 name = "crawlberg"
-version = "1.0.4"
+version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7e73be1f747c594bfabc2596ef3c162e35fa50e4616d3e3af85821823f5424d0"
+checksum = "2200ee1bcca212516ab5d0900a0c108c50e999295316c8351c52b7ea8e2a504c"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -587,7 +587,7 @@ dependencies = [
 
 [[package]]
 name = "crawlberg-rb"
-version = "1.0.3"
+version = "1.0.4"
 dependencies = [
  "crawlberg",
  "futures",

data/ext/crawlberg_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "crawlberg-rb"
-version = "1.0.3"
+version = "1.0.4"
 edition = "2024"
 license = "MIT"
 description = "High-performance web crawling engine"
@@ -17,7 +17,7 @@ path = "../src/lib.rs"
 crate-type = ["cdylib"]
 
 [dependencies]
-crawlberg = { version = "1.0.4", features = ["interact", "browser-chromiumoxide"] }
+crawlberg = { version = "1.0.5", features = ["interact", "browser-chromiumoxide"] }
 futures = "0.3"
 magnus = "0.8"
 rb-sys = ">=0.9, <0.9.128"

data/ext/crawlberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:e0600612767fdd3d51e9128afdb0ffcd1fb490e9942f3839a5120ab7a4e3b658
+// alef:hash:ed6753aa35e27ad4764a749b35d849473835fb530fcba666ac607635b1333c06
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -2040,6 +2040,7 @@ impl CrawlResult {
     }
 
     fn unique_normalized_urls(&self) -> usize {
+        #[allow(clippy::needless_update)]
         let core_self = crawlberg::CrawlResult {
             pages: self.pages.clone().into_iter().map(Into::into).collect(),
 
@@ -5277,6 +5278,7 @@ fn batch_crawl_stream(engine: CrawlEngineHandle, req: BatchCrawlStreamRequest) -
     engine.batch_crawl_stream(req)
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ExtractionMeta> for crawlberg::ExtractionMeta {
     fn from(val: ExtractionMeta) -> Self {
@@ -5286,6 +5288,7 @@ impl From<ExtractionMeta> for crawlberg::ExtractionMeta {
             completion_tokens: val.completion_tokens,
             model: val.model,
             chunks_processed: val.chunks_processed,
+            ..Default::default()
         }
     }
 }
@@ -5303,6 +5306,7 @@ impl From<crawlberg::ExtractionMeta> for ExtractionMeta {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ProxyConfig> for crawlberg::ProxyConfig {
     fn from(val: ProxyConfig) -> Self {
@@ -5310,6 +5314,7 @@ impl From<ProxyConfig> for crawlberg::ProxyConfig {
             url: val.url,
             username: val.username,
             password: val.password,
+            ..Default::default()
         }
     }
 }
@@ -5325,6 +5330,7 @@ impl From<crawlberg::ProxyConfig> for ProxyConfig {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ContentConfig> for crawlberg::ContentConfig {
     fn from(val: ContentConfig) -> Self {
@@ -5341,6 +5347,7 @@ impl From<ContentConfig> for crawlberg::ContentConfig {
             wrap: val.wrap,
             wrap_width: val.wrap_width,
             include_document_structure: val.include_document_structure,
+            ..Default::default()
         }
     }
 }
@@ -5365,6 +5372,7 @@ impl From<crawlberg::ContentConfig> for ContentConfig {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BrowserConfig> for crawlberg::BrowserConfig {
     fn from(val: BrowserConfig) -> Self {
@@ -5382,6 +5390,7 @@ impl From<BrowserConfig> for crawlberg::BrowserConfig {
             robots_user_agent: val.robots_user_agent,
             capture_network_events: val.capture_network_events,
             session_affinity: val.session_affinity,
+            ..Default::default()
         }
     }
 }
@@ -5511,6 +5520,7 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BrowserExtras> for crawlberg::BrowserExtras {
     fn from(val: BrowserExtras) -> Self {
@@ -5518,6 +5528,7 @@ impl From<BrowserExtras> for crawlberg::BrowserExtras {
             eval_result: val.eval_result.as_ref().and_then(|s| serde_json::from_str(s).ok()),
             network_events: val.network_events.into_iter().map(Into::into).collect(),
             cookies: val.cookies.into_iter().map(Into::into).collect(),
+            ..Default::default()
         }
     }
 }
@@ -5587,6 +5598,7 @@ impl From<crawlberg::InteractionResult> for InteractionResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ActionResult> for crawlberg::ActionResult {
     fn from(val: ActionResult) -> Self {
@@ -5596,6 +5608,7 @@ impl From<ActionResult> for crawlberg::ActionResult {
             success: val.success,
             data: val.data.as_ref().and_then(|s| serde_json::from_str(s).ok()),
             error: val.error,
+            ..Default::default()
         }
     }
 }
@@ -5687,6 +5700,7 @@ impl From<crawlberg::ScrapeResult> for ScrapeResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CrawlPageResult> for crawlberg::CrawlPageResult {
     fn from(val: CrawlPageResult) -> Self {
@@ -5712,6 +5726,7 @@ impl From<CrawlPageResult> for crawlberg::CrawlPageResult {
             extraction_meta: val.extraction_meta.map(Into::into),
             downloaded_document: val.downloaded_document.map(Into::into),
             browser_used: val.browser_used,
+            ..Default::default()
         }
     }
 }
@@ -5779,6 +5794,7 @@ impl From<crawlberg::CrawlResult> for CrawlResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<SitemapUrl> for crawlberg::SitemapUrl {
     fn from(val: SitemapUrl) -> Self {
@@ -5787,6 +5803,7 @@ impl From<SitemapUrl> for crawlberg::SitemapUrl {
             lastmod: val.lastmod,
             changefreq: val.changefreq,
             priority: val.priority,
+            ..Default::default()
         }
     }
 }
@@ -5803,11 +5820,13 @@ impl From<crawlberg::SitemapUrl> for SitemapUrl {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<MapResult> for crawlberg::MapResult {
     fn from(val: MapResult) -> Self {
         Self {
             urls: val.urls.into_iter().map(Into::into).collect(),
+            ..Default::default()
         }
     }
 }
@@ -5821,6 +5840,7 @@ impl From<crawlberg::MapResult> for MapResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<MarkdownResult> for crawlberg::MarkdownResult {
     fn from(val: MarkdownResult) -> Self {
@@ -5838,6 +5858,7 @@ impl From<MarkdownResult> for crawlberg::MarkdownResult {
             warnings: val.warnings.into_iter().collect(),
             citations: val.citations,
             fit_content: val.fit_content,
+            ..Default::default()
         }
     }
 }
@@ -5856,6 +5877,7 @@ impl From<crawlberg::MarkdownResult> for MarkdownResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<LinkInfo> for crawlberg::LinkInfo {
     fn from(val: LinkInfo) -> Self {
@@ -5865,6 +5887,7 @@ impl From<LinkInfo> for crawlberg::LinkInfo {
             link_type: val.link_type.into(),
             rel: val.rel,
             nofollow: val.nofollow,
+            ..Default::default()
         }
     }
 }
@@ -5882,6 +5905,7 @@ impl From<crawlberg::LinkInfo> for LinkInfo {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ImageInfo> for crawlberg::ImageInfo {
     fn from(val: ImageInfo) -> Self {
@@ -5891,6 +5915,7 @@ impl From<ImageInfo> for crawlberg::ImageInfo {
             width: val.width,
             height: val.height,
             source: val.source.into(),
+            ..Default::default()
         }
     }
 }
@@ -5908,6 +5933,7 @@ impl From<crawlberg::ImageInfo> for ImageInfo {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<FeedInfo> for crawlberg::FeedInfo {
     fn from(val: FeedInfo) -> Self {
@@ -5915,6 +5941,7 @@ impl From<FeedInfo> for crawlberg::FeedInfo {
             url: val.url,
             title: val.title,
             feed_type: val.feed_type.into(),
+            ..Default::default()
         }
     }
 }
@@ -5930,6 +5957,7 @@ impl From<crawlberg::FeedInfo> for FeedInfo {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<JsonLdEntry> for crawlberg::JsonLdEntry {
     fn from(val: JsonLdEntry) -> Self {
@@ -5937,6 +5965,7 @@ impl From<JsonLdEntry> for crawlberg::JsonLdEntry {
             schema_type: val.schema_type,
             name: val.name,
             raw: val.raw,
+            ..Default::default()
         }
     }
 }
@@ -5952,6 +5981,7 @@ impl From<crawlberg::JsonLdEntry> for JsonLdEntry {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CookieInfo> for crawlberg::CookieInfo {
     fn from(val: CookieInfo) -> Self {
@@ -5960,6 +5990,7 @@ impl From<CookieInfo> for crawlberg::CookieInfo {
             value: val.value,
             domain: val.domain,
             path: val.path,
+            ..Default::default()
         }
     }
 }
@@ -5976,6 +6007,7 @@ impl From<crawlberg::CookieInfo> for CookieInfo {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<DownloadedAsset> for crawlberg::DownloadedAsset {
     fn from(val: DownloadedAsset) -> Self {
@@ -5986,6 +6018,7 @@ impl From<DownloadedAsset> for crawlberg::DownloadedAsset {
             size: val.size,
             asset_category: val.asset_category.into(),
             html_tag: val.html_tag,
+            ..Default::default()
         }
     }
 }
@@ -6004,6 +6037,7 @@ impl From<crawlberg::DownloadedAsset> for DownloadedAsset {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ArticleMetadata> for crawlberg::ArticleMetadata {
     fn from(val: ArticleMetadata) -> Self {
@@ -6013,6 +6047,7 @@ impl From<ArticleMetadata> for crawlberg::ArticleMetadata {
             author: val.author,
             section: val.section,
             tags: val.tags.into_iter().collect(),
+            ..Default::default()
         }
     }
 }
@@ -6030,12 +6065,14 @@ impl From<crawlberg::ArticleMetadata> for ArticleMetadata {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<HreflangEntry> for crawlberg::HreflangEntry {
     fn from(val: HreflangEntry) -> Self {
         Self {
             lang: val.lang,
             url: val.url,
+            ..Default::default()
         }
     }
 }
@@ -6050,6 +6087,7 @@ impl From<crawlberg::HreflangEntry> for HreflangEntry {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<FaviconInfo> for crawlberg::FaviconInfo {
     fn from(val: FaviconInfo) -> Self {
@@ -6058,6 +6096,7 @@ impl From<FaviconInfo> for crawlberg::FaviconInfo {
             rel: val.rel,
             sizes: val.sizes,
             mime_type: val.mime_type,
+            ..Default::default()
         }
     }
 }
@@ -6074,12 +6113,14 @@ impl From<crawlberg::FaviconInfo> for FaviconInfo {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<HeadingInfo> for crawlberg::HeadingInfo {
     fn from(val: HeadingInfo) -> Self {
         Self {
             level: val.level,
             text: val.text,
+            ..Default::default()
         }
     }
 }
@@ -6094,6 +6135,7 @@ impl From<crawlberg::HeadingInfo> for HeadingInfo {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ResponseMeta> for crawlberg::ResponseMeta {
     fn from(val: ResponseMeta) -> Self {
@@ -6105,6 +6147,7 @@ impl From<ResponseMeta> for crawlberg::ResponseMeta {
             x_powered_by: val.x_powered_by,
             content_language: val.content_language,
             content_encoding: val.content_encoding,
+            ..Default::default()
         }
     }
 }
@@ -6124,6 +6167,7 @@ impl From<crawlberg::ResponseMeta> for ResponseMeta {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<PageMetadata> for crawlberg::PageMetadata {
     fn from(val: PageMetadata) -> Self {
@@ -6171,6 +6215,7 @@ impl From<PageMetadata> for crawlberg::PageMetadata {
             favicons: val.favicons.map(|v| v.into_iter().map(Into::into).collect()),
             headings: val.headings.map(|v| v.into_iter().map(Into::into).collect()),
             word_count: val.word_count,
+            ..Default::default()
         }
     }
 }
@@ -6226,10 +6271,14 @@ impl From<crawlberg::PageMetadata> for PageMetadata {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CrawlStreamRequest> for crawlberg::CrawlStreamRequest {
     fn from(val: CrawlStreamRequest) -> Self {
-        Self { url: val.url }
+        Self {
+            url: val.url,
+            ..Default::default()
+        }
     }
 }
 
@@ -6242,11 +6291,13 @@ impl From<crawlberg::CrawlStreamRequest> for CrawlStreamRequest {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BatchCrawlStreamRequest> for crawlberg::BatchCrawlStreamRequest {
     fn from(val: BatchCrawlStreamRequest) -> Self {
         Self {
             urls: val.urls.into_iter().collect(),
+            ..Default::default()
         }
     }
 }
@@ -6260,12 +6311,14 @@ impl From<crawlberg::BatchCrawlStreamRequest> for BatchCrawlStreamRequest {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CitationResult> for crawlberg::CitationResult {
     fn from(val: CitationResult) -> Self {
         Self {
             content: val.content,
             references: val.references.into_iter().map(Into::into).collect(),
+            ..Default::default()
         }
     }
 }
@@ -6280,6 +6333,7 @@ impl From<crawlberg::CitationResult> for CitationResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CitationReference> for crawlberg::CitationReference {
     fn from(val: CitationReference) -> Self {
@@ -6287,6 +6341,7 @@ impl From<CitationReference> for crawlberg::CitationReference {
             index: val.index,
             url: val.url,
             text: val.text,
+            ..Default::default()
         }
     }
 }
@@ -6302,6 +6357,7 @@ impl From<crawlberg::CitationReference> for CitationReference {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BatchScrapeResult> for crawlberg::BatchScrapeResult {
     fn from(val: BatchScrapeResult) -> Self {
@@ -6309,6 +6365,7 @@ impl From<BatchScrapeResult> for crawlberg::BatchScrapeResult {
             url: val.url,
             result: val.result.map(Into::into),
             error: val.error,
+            ..Default::default()
         }
     }
 }
@@ -6324,6 +6381,7 @@ impl From<crawlberg::BatchScrapeResult> for BatchScrapeResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BatchCrawlResult> for crawlberg::BatchCrawlResult {
     fn from(val: BatchCrawlResult) -> Self {
@@ -6331,6 +6389,7 @@ impl From<BatchCrawlResult> for crawlberg::BatchCrawlResult {
             url: val.url,
             result: val.result.map(Into::into),
             error: val.error,
+            ..Default::default()
         }
     }
 }
@@ -6346,6 +6405,7 @@ impl From<crawlberg::BatchCrawlResult> for BatchCrawlResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BatchScrapeResults> for crawlberg::BatchScrapeResults {
     fn from(val: BatchScrapeResults) -> Self {
@@ -6354,6 +6414,7 @@ impl From<BatchScrapeResults> for crawlberg::BatchScrapeResults {
             total_count: val.total_count,
             completed_count: val.completed_count,
             failed_count: val.failed_count,
+            ..Default::default()
         }
     }
 }
@@ -6370,6 +6431,7 @@ impl From<crawlberg::BatchScrapeResults> for BatchScrapeResults {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BatchCrawlResults> for crawlberg::BatchCrawlResults {
     fn from(val: BatchCrawlResults) -> Self {
@@ -6378,6 +6440,7 @@ impl From<BatchCrawlResults> for crawlberg::BatchCrawlResults {
             total_count: val.total_count,
             completed_count: val.completed_count,
             failed_count: val.failed_count,
+            ..Default::default()
         }
     }
 }

data/lib/crawlberg/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:e0600612767fdd3d51e9128afdb0ffcd1fb490e9942f3839a5120ab7a4e3b658
+# alef:hash:ed6753aa35e27ad4764a749b35d849473835fb530fcba666ac607635b1333c06
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true

data/lib/crawlberg/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:297f8acf68782dcba88253e72a4650a57f0615157fd2c848c6867ea6e6e40585
+# alef:hash:ed6753aa35e27ad4764a749b35d849473835fb530fcba666ac607635b1333c06
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
 
 module Crawlberg
   ## The version string for this package.
-  VERSION = "1.0.4"
+  VERSION = "1.0.5"
 end

data/lib/crawlberg.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:e0600612767fdd3d51e9128afdb0ffcd1fb490e9942f3839a5120ab7a4e3b658
+# alef:hash:ed6753aa35e27ad4764a749b35d849473835fb530fcba666ac607635b1333c06
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:e0600612767fdd3d51e9128afdb0ffcd1fb490e9942f3839a5120ab7a4e3b658
+# alef:hash:ed6753aa35e27ad4764a749b35d849473835fb530fcba666ac607635b1333c06
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crawlberg
 version: !ruby/object:Gem::Version
-  version: 1.0.4
+  version: 1.0.5
 platform: ruby
 authors:
 - Xberg Team