crawlberg 1.0.1 → 1.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3fef501414af0f060bbd74518d19f132b39b1b4ab48e517c26f8c4d588f1cbe7
-  data.tar.gz: 2937571ed0798968eb0567df042837ff65a8603de68882017099b969045395a2
+  metadata.gz: 14f3cbbce5b53366e44498f012fe7b79b45b8a11a4c5cd712dfd4f5dcbd18934
+  data.tar.gz: c18b8a4575a244bd54fff1db81e1b21266a6a52d54cec99e7c760f1abad6f076
 SHA512:
-  metadata.gz: d74f342935558ffadd242c187138704ae15b787f66262001ff9341c66e0ed05ad28ec83fad781f4d82af72ee3579f56dac643ed2e2a4f36960b7ac609ce432cd
-  data.tar.gz: f3125487a1167bed9fc2b3b119bb50bc2419dc7cb8cc2678c0d676ab44a7ae5ca3ba0843041bbf6e8981d9305611b8fc512f250819e3c994463551a87655a795
+  metadata.gz: d0a381e2451d54f1e1a016d3d832df167f5aaee8fac4b0eb81631b32cebac8b8000bc6d3f5119d51b9f29a8a92c9b0707ed33aebdf76d7f97a3809dc1c5fb047
+  data.tar.gz: 40ff0204c26d0dc39c5b9161036cda1b91157f9e3a9d7e1ac06be03806c6322f68ebd43f5fd8357d1ee75f69d0682e89c702cdac32f25a59f4eb5a59f7dc37a7

data/ext/crawlberg_rb/native/Cargo.lock

@@ -145,9 +145,9 @@ checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53"
 
 [[package]]
 name = "aws-lc-rs"
-version = "1.17.0"
+version = "1.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ec2f1fc3ec205783a5da9a7e6c1509cc69dedf09a1949e412c1e18469326d00"
+checksum = "4342d8937fc7e5dd9b1c60292261c0670c882a2cd1719cfc11b1af41731e32ad"
 dependencies = [
  "aws-lc-sys",
  "zeroize",
@@ -155,14 +155,15 @@ dependencies = [
 
 [[package]]
 name = "aws-lc-sys"
-version = "0.41.0"
+version = "0.42.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1a2f9779ce85b93ab6170dd940ad0169b5766ff848247aff13bb788b832fe3f4"
+checksum = "6d9ceb1da931507a12f4fccea479dccd00da1943e1b4ae72d8e502d707361444"
 dependencies = [
  "cc",
  "cmake",
  "dunce",
  "fs_extra",
+ "pkg-config",
 ]
 
 [[package]]
@@ -544,9 +545,9 @@ dependencies = [
 
 [[package]]
 name = "crawlberg"
-version = "1.0.1"
+version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3b29d0b66d3886745ab12e56e17a9127dc91f30b7261a2080c9eed67074c34f6"
+checksum = "46b3f6dfec53e26f1f989b4685943153a54e1d2101f8bd3ae5afd810b7957423"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -586,7 +587,7 @@ dependencies = [
 
 [[package]]
 name = "crawlberg-rb"
-version = "1.0.0"
+version = "1.0.3"
 dependencies = [
  "crawlberg",
  "futures",
@@ -1042,9 +1043,9 @@ dependencies = [
 
 [[package]]
 name = "html-to-markdown-rs"
-version = "3.7.2"
+version = "3.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e80e5d74dbe02fc52264d6a1d21c12bbc0cb062bca64caa295e4703590f1aabe"
+checksum = "da9935e81a6fee38ebbc5a2ca4e9590f99f650ffb8a08734c4115a3e06c3c5d3"
 dependencies = [
  "ahash",
  "astral-tl",
@@ -1056,7 +1057,7 @@ dependencies = [
  "lru",
  "memchr",
  "once_cell",
- "phf",
+ "phf 0.14.0",
  "regex",
  "serde",
  "serde_json",
@@ -1765,9 +1766,19 @@ name = "phf"
 version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf"
+dependencies = [
+ "phf_shared 0.13.1",
+ "serde",
+]
+
+[[package]]
+name = "phf"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6"
 dependencies = [
  "phf_macros",
- "phf_shared",
+ "phf_shared 0.14.0",
  "serde",
 ]
 
@@ -1777,8 +1788,8 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "49aa7f9d80421bca176ca8dbfebe668cc7a2684708594ec9f3c0db0805d5d6e1"
 dependencies = [
- "phf_generator",
- "phf_shared",
+ "phf_generator 0.13.1",
+ "phf_shared 0.13.1",
 ]
 
 [[package]]
@@ -1788,17 +1799,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737"
 dependencies = [
  "fastrand",
- "phf_shared",
+ "phf_shared 0.13.1",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100"
+dependencies = [
+ "fastrand",
+ "phf_shared 0.14.0",
 ]
 
 [[package]]
 name = "phf_macros"
-version = "0.13.1"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef"
+checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085"
 dependencies = [
- "phf_generator",
- "phf_shared",
+ "phf_generator 0.14.0",
+ "phf_shared 0.14.0",
  "proc-macro2",
  "quote",
  "syn",
@@ -1813,12 +1834,27 @@ dependencies = [
  "siphasher",
 ]
 
+[[package]]
+name = "phf_shared"
+version = "0.14.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89"
+dependencies = [
+ "siphasher",
+]
+
 [[package]]
 name = "pin-project-lite"
 version = "0.2.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.33"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e"
+
 [[package]]
 name = "png"
 version = "0.18.1"
@@ -1901,9 +1937,9 @@ checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3"
 
 [[package]]
 name = "quick-xml"
-version = "0.40.1"
+version = "0.41.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f"
+checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1"
 dependencies = [
  "memchr",
 ]
@@ -2149,9 +2185,9 @@ dependencies = [
 
 [[package]]
 name = "rustc-hash"
-version = "2.1.2"
+version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "94300abf3f1ae2e2b8ffb7b58043de3d399c73fa6f4b73826402a5c457614dbe"
+checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
 [[package]]
 name = "rustc_version"
@@ -2190,9 +2226,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-pki-types"
-version = "1.14.1"
+version = "1.15.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9"
+checksum = "764899a24af3980067ee14bc143654f297b22eaebfe3c7b6b211920a5a59b046"
 dependencies = [
  "web-time",
  "zeroize",
@@ -2468,7 +2504,7 @@ checksum = "a18596f8c785a729f2819c0f6a7eae6ebeebdfffbfe4214ae6b087f690e31901"
 dependencies = [
  "new_debug_unreachable",
  "parking_lot",
- "phf_shared",
+ "phf_shared 0.13.1",
  "precomputed-hash",
 ]
 
@@ -2478,8 +2514,8 @@ version = "0.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "585635e46db231059f76c5849798146164652513eb9e8ab2685939dd90f29b69"
 dependencies = [
- "phf_generator",
- "phf_shared",
+ "phf_generator 0.13.1",
+ "phf_shared 0.13.1",
  "proc-macro2",
  "quote",
 ]
@@ -2574,9 +2610,9 @@ dependencies = [
 
 [[package]]
 name = "time"
-version = "0.3.51"
+version = "0.3.53"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85c17d80feb7334b40c484e45ed1a5273dfd8bfda537c3be2e74a06a6686f327"
+checksum = "18dfaaeddcb932337b5e7866ee7d0ce9b76d2fd092997146f187ec09b4558a50"
 dependencies = [
  "deranged",
  "num-conv",
@@ -2594,9 +2630,9 @@ checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109"
 
 [[package]]
 name = "time-macros"
-version = "0.2.30"
+version = "0.2.31"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcef1a61bdb119096e153208ec5cbec23944ce8bca13be5c7f60c634f7403935"
+checksum = "c431b87111666e491a90baa837f914fb45cd5dc3c268591b0220ff5057f2085f"
 dependencies = [
  "num-conv",
  "time-core",
@@ -3011,7 +3047,7 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "075474b12bcb3d2e3d4546580e9de478eeeead668a1761e2a8860c836b7ef297"
 dependencies = [
- "phf",
+ "phf 0.13.1",
  "phf_codegen",
  "string_cache",
  "string_cache_codegen",

data/ext/crawlberg_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "crawlberg-rb"
-version = "1.0.0"
+version = "1.0.3"
 edition = "2024"
 license = "MIT"
 description = "High-performance web crawling engine"
@@ -17,7 +17,7 @@ path = "../src/lib.rs"
 crate-type = ["cdylib"]
 
 [dependencies]
-crawlberg = { version = "1.0.1", features = ["interact", "browser-chromiumoxide"] }
+crawlberg = { version = "1.0.3", features = ["interact", "browser-chromiumoxide"] }
 futures = "0.3"
 magnus = "0.8"
 rb-sys = ">=0.9, <0.9.128"

data/ext/crawlberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:06831f8166c6d860691af36ee02b72ae3246568eb2e5c67ed5d11da71d02afeb
+// alef:hash:e0600612767fdd3d51e9128afdb0ffcd1fb490e9942f3839a5120ab7a4e3b658
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -438,21 +438,7 @@ unsafe impl TryConvertOwned for BrowserConfig {}
 
 impl Default for BrowserConfig {
     fn default() -> Self {
-        Self {
-            mode: BrowserMode::Auto,
-            backend: BrowserBackend::Chromiumoxide,
-            endpoint: None,
-            timeout: 30000,
-            wait: Default::default(),
-            wait_selector: None,
-            extra_wait: None,
-            proxy: None,
-            block_url_patterns: vec![],
-            eval_script: None,
-            robots_user_agent: None,
-            capture_network_events: false,
-            session_affinity: true,
-        }
+        crawlberg::BrowserConfig::default().into()
     }
 }
 
@@ -641,47 +627,7 @@ unsafe impl TryConvertOwned for CrawlConfig {}
 
 impl Default for CrawlConfig {
     fn default() -> Self {
-        Self {
-            max_depth: None,
-            max_pages: None,
-            max_concurrent: None,
-            respect_robots_txt: false,
-            soft_http_errors: false,
-            user_agent: None,
-            stay_on_domain: false,
-            allow_subdomains: false,
-            include_paths: vec![],
-            exclude_paths: vec![],
-            custom_headers: Default::default(),
-            request_timeout: 30000,
-            rate_limit_ms: None,
-            max_redirects: 10,
-            retry_count: 0,
-            retry_codes: vec![],
-            cookies_enabled: false,
-            auth: None,
-            max_body_size: None,
-            remove_tags: vec![],
-            content: Default::default(),
-            map_limit: None,
-            map_search: None,
-            download_assets: false,
-            asset_types: vec![],
-            max_asset_size: None,
-            browser: Default::default(),
-            proxy: None,
-            user_agents: vec![],
-            capture_screenshot: false,
-            follow_document_urls: false,
-            document_url_depth: None,
-            download_documents: true,
-            document_max_size: None,
-            document_mime_types: vec![],
-            warc_output: None,
-            browser_profile: None,
-            save_browser_profile: false,
-            ssrf: Default::default(),
-        }
+        crawlberg::CrawlConfig::default().into()
     }
 }
 
@@ -2410,13 +2356,7 @@ unsafe impl TryConvertOwned for LinkInfo {}
 
 impl Default for LinkInfo {
     fn default() -> Self {
-        Self {
-            url: String::new(),
-            text: String::new(),
-            link_type: Default::default(),
-            rel: None,
-            nofollow: false,
-        }
+        crawlberg::LinkInfo::default().into()
     }
 }
 
@@ -2511,13 +2451,7 @@ unsafe impl TryConvertOwned for ImageInfo {}
 
 impl Default for ImageInfo {
     fn default() -> Self {
-        Self {
-            url: String::new(),
-            alt: None,
-            width: None,
-            height: None,
-            source: Default::default(),
-        }
+        crawlberg::ImageInfo::default().into()
     }
 }
 
@@ -2608,11 +2542,7 @@ unsafe impl TryConvertOwned for FeedInfo {}
 
 impl Default for FeedInfo {
     fn default() -> Self {
-        Self {
-            url: String::new(),
-            title: None,
-            feed_type: Default::default(),
-        }
+        crawlberg::FeedInfo::default().into()
     }
 }
 
@@ -2854,14 +2784,7 @@ unsafe impl TryConvertOwned for DownloadedAsset {}
 
 impl Default for DownloadedAsset {
     fn default() -> Self {
-        Self {
-            url: String::new(),
-            content_hash: String::new(),
-            mime_type: None,
-            size: 0,
-            asset_category: Default::default(),
-            html_tag: None,
-        }
+        crawlberg::DownloadedAsset::default().into()
     }
 }
 
@@ -6556,9 +6479,15 @@ impl From<crawlberg::BrowserBackend> for BrowserBackend {
 impl From<AuthConfig> for crawlberg::AuthConfig {
     fn from(val: AuthConfig) -> Self {
         match val {
-            AuthConfig::Basic { username, password } => Self::Basic { username, password },
-            AuthConfig::Bearer { token } => Self::Bearer { token },
-            AuthConfig::Header { name, value } => Self::Header { name, value },
+            AuthConfig::Basic { username, password } => Self::Basic {
+                username: username,
+                password: password,
+            },
+            AuthConfig::Bearer { token } => Self::Bearer { token: token },
+            AuthConfig::Header { name, value } => Self::Header {
+                name: name,
+                value: value,
+            },
         }
     }
 }
@@ -6689,7 +6618,9 @@ impl From<crawlberg::CrawlEvent> for CrawlEvent {
                 url: url.to_string(),
                 error: error.to_string(),
             },
-            crawlberg::CrawlEvent::Complete { pages_crawled } => Self::Complete { pages_crawled },
+            crawlberg::CrawlEvent::Complete { pages_crawled } => Self::Complete {
+                pages_crawled: pages_crawled,
+            },
         }
     }
 }
@@ -6697,21 +6628,27 @@ impl From<crawlberg::CrawlEvent> for CrawlEvent {
 impl From<PageAction> for crawlberg::PageAction {
     fn from(val: PageAction) -> Self {
         match val {
-            PageAction::Click { selector } => Self::Click { selector },
-            PageAction::TypeText { selector, text } => Self::TypeText { selector, text },
-            PageAction::Press { key } => Self::Press { key },
+            PageAction::Click { selector } => Self::Click { selector: selector },
+            PageAction::TypeText { selector, text } => Self::TypeText {
+                selector: selector,
+                text: text,
+            },
+            PageAction::Press { key } => Self::Press { key: key },
             PageAction::Scroll {
                 direction,
                 selector,
                 amount,
             } => Self::Scroll {
                 direction: direction.into(),
-                selector,
-                amount,
+                selector: selector,
+                amount: amount,
             },
-            PageAction::Wait { milliseconds, selector } => Self::Wait { milliseconds, selector },
-            PageAction::Screenshot { full_page } => Self::Screenshot { full_page },
-            PageAction::ExecuteJs { script } => Self::ExecuteJs { script },
+            PageAction::Wait { milliseconds, selector } => Self::Wait {
+                milliseconds: milliseconds,
+                selector: selector,
+            },
+            PageAction::Screenshot { full_page } => Self::Screenshot { full_page: full_page },
+            PageAction::ExecuteJs { script } => Self::ExecuteJs { script: script },
             PageAction::Scrape => Self::Scrape,
         }
     }
@@ -6735,13 +6672,13 @@ impl From<crawlberg::PageAction> for PageAction {
             } => Self::Scroll {
                 direction: direction.into(),
                 selector: selector.map(|v| v.to_string()),
-                amount,
+                amount: amount,
             },
             crawlberg::PageAction::Wait { milliseconds, selector } => Self::Wait {
-                milliseconds,
+                milliseconds: milliseconds,
                 selector: selector.map(|v| v.to_string()),
             },
-            crawlberg::PageAction::Screenshot { full_page } => Self::Screenshot { full_page },
+            crawlberg::PageAction::Screenshot { full_page } => Self::Screenshot { full_page: full_page },
             crawlberg::PageAction::ExecuteJs { script } => Self::ExecuteJs {
                 script: script.to_string(),
             },