crawlberg 1.2.1 → 1.3.3

data/ext/crawlberg_rb/native/Cargo.toml

@@ -1,6 +1,11 @@
+# This file is auto-generated by alef — DO NOT EDIT.
+# alef:hash:6d3e12955a76fb5c98a03dc656a5b76ae41ba095ad388a22cb35ce647d7159fc
+# To regenerate: alef generate
+# To verify freshness: alef verify
+
 [package]
 name = "crawlberg-rb"
-version = "1.2.1"
+version = "1.3.3"
 edition = "2024"
 license = "MIT"
 description = "High-performance web crawling engine"
@@ -17,10 +22,22 @@ path = "../src/lib.rs"
 crate-type = ["cdylib"]
 
 [dependencies]
-crawlberg = { version = "1.2.1", features = ["interact", "browser-chromiumoxide"] }
+crawlberg = { version = "1.3.3", features = ["interact", "browser-chromiumoxide"] }
 futures = "0.3"
 magnus = "0.8"
 rb-sys = ">=0.9, <0.9.128"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 tokio = { version = "1", features = ["rt-multi-thread"] }
+
+# This crate deliberately does not use `[lints]` / `workspace = true`: its C-ABI /
+# PyO3 / napi / ext-php-rs / NIF boundary requires `unsafe` code, and the workspace's
+# `[workspace.lints.rust]` sets `unsafe_code = "deny"` -- an all-or-nothing table that
+# would turn every such boundary into a compile error. The `[lints.clippy]` block below
+# instead carries the subset of the workspace's deny-by-default lint policy this crate
+# can actually satisfy. ~keep
+[lints.clippy]
+all = { level = "warn", priority = -1 }
+dbg_macro = "deny"
+print_stderr = "deny"
+print_stdout = "deny"

data/ext/crawlberg_rb/native/extconf.rb

@@ -1,3 +1,8 @@
+# This file is auto-generated by alef — DO NOT EDIT.
+# alef:hash:7e8b3cef132610f31102ee2069cf33d39e5e1ece68f870166e669e04f476544e
+# To regenerate: alef generate
+# To verify freshness: alef verify
+
 # frozen_string_literal: true
 
 require "mkmf"

data/ext/crawlberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:0ce4d753fdb4854e44358639dcbaebee3449a4afa142dbc4f0a72aa72c214648
+// alef:hash:9312cb415995255492557432651036cfe6b0edf4ccd0b27a88d0cb3c5a7d1466
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -316,7 +316,7 @@ impl ContentConfig {
             exclude_selectors: kwargs
                 .get(ruby.to_symbol("exclude_selectors"))
                 .and_then(|v| <Vec<String>>::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(vec!["noscript".to_string()]),
             skip_images: kwargs
                 .get(ruby.to_symbol("skip_images"))
                 .and_then(|v| bool::try_convert(v).ok())
@@ -467,7 +467,7 @@ impl BrowserConfig {
             wait: kwargs
                 .get(ruby.to_symbol("wait"))
                 .and_then(|v| BrowserWait::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(BrowserWait::NetworkIdle),
             wait_selector: kwargs
                 .get(ruby.to_symbol("wait_selector"))
                 .and_then(|v| String::try_convert(v).ok()),
@@ -559,6 +559,10 @@ pub struct CrawlConfig {
     max_pages: Option<usize>,
     max_links_per_page: Option<usize>,
     max_concurrent: Option<usize>,
+    crawl_strategy: CrawlStrategyKind,
+    content_filter: Option<ContentFilterKind>,
+    bm25_query: Option<String>,
+    bm25_threshold: Option<f64>,
     respect_robots_txt: bool,
     soft_http_errors: bool,
     user_agent: Option<String>,
@@ -654,6 +658,19 @@ impl CrawlConfig {
             max_concurrent: kwargs
                 .get(ruby.to_symbol("max_concurrent"))
                 .and_then(|v| usize::try_convert(v).ok()),
+            crawl_strategy: kwargs
+                .get(ruby.to_symbol("crawl_strategy"))
+                .and_then(|v| CrawlStrategyKind::try_convert(v).ok())
+                .unwrap_or(CrawlStrategyKind::Bfs),
+            content_filter: kwargs
+                .get(ruby.to_symbol("content_filter"))
+                .and_then(|v| ContentFilterKind::try_convert(v).ok()),
+            bm25_query: kwargs
+                .get(ruby.to_symbol("bm25_query"))
+                .and_then(|v| String::try_convert(v).ok()),
+            bm25_threshold: kwargs
+                .get(ruby.to_symbol("bm25_threshold"))
+                .and_then(|v| f64::try_convert(v).ok()),
             respect_robots_txt: kwargs
                 .get(ruby.to_symbol("respect_robots_txt"))
                 .and_then(|v| bool::try_convert(v).ok())
@@ -791,7 +808,7 @@ impl CrawlConfig {
             ssrf: kwargs
                 .get(ruby.to_symbol("ssrf"))
                 .and_then(|v| SsrfPolicy::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(crawlberg::SsrfPolicy::from_env().into()),
             ssrf_deny_private_explicit: kwargs
                 .get(ruby.to_symbol("ssrf_deny_private_explicit"))
                 .and_then(|v| bool::try_convert(v).ok()),
@@ -814,6 +831,22 @@ impl CrawlConfig {
         self.max_concurrent
     }
 
+    fn crawl_strategy(&self) -> CrawlStrategyKind {
+        self.crawl_strategy.clone()
+    }
+
+    fn content_filter(&self) -> Option<ContentFilterKind> {
+        self.content_filter.clone()
+    }
+
+    fn bm25_query(&self) -> Option<String> {
+        self.bm25_query.clone()
+    }
+
+    fn bm25_threshold(&self) -> Option<f64> {
+        self.bm25_threshold
+    }
+
     fn respect_robots_txt(&self) -> bool {
         self.respect_robots_txt
     }
@@ -981,6 +1014,14 @@ impl CrawlConfig {
 
             max_concurrent: self.max_concurrent,
 
+            crawl_strategy: self.crawl_strategy.clone().into(),
+
+            content_filter: self.content_filter.clone().map(Into::into),
+
+            bm25_query: self.bm25_query.clone(),
+
+            bm25_threshold: self.bm25_threshold,
+
             respect_robots_txt: self.respect_robots_txt,
 
             soft_http_errors: self.soft_http_errors,
@@ -2460,7 +2501,7 @@ impl LinkInfo {
             link_type: kwargs
                 .get(ruby.to_symbol("link_type"))
                 .and_then(|v| LinkType::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(LinkType::Internal),
             rel: kwargs
                 .get(ruby.to_symbol("rel"))
                 .and_then(|v| String::try_convert(v).ok()),
@@ -2560,7 +2601,7 @@ impl ImageInfo {
             source: kwargs
                 .get(ruby.to_symbol("source"))
                 .and_then(|v| ImageSource::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(ImageSource::Img),
         })
     }
 
@@ -2645,7 +2686,7 @@ impl FeedInfo {
             feed_type: kwargs
                 .get(ruby.to_symbol("feed_type"))
                 .and_then(|v| FeedType::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(FeedType::Rss),
         })
     }
 
@@ -2895,7 +2936,7 @@ impl DownloadedAsset {
             asset_category: kwargs
                 .get(ruby.to_symbol("asset_category"))
                 .and_then(|v| AssetCategory::try_convert(v).ok())
-                .unwrap_or_default(),
+                .unwrap_or(AssetCategory::Image),
             html_tag: kwargs
                 .get(ruby.to_symbol("html_tag"))
                 .and_then(|v| String::try_convert(v).ok()),
@@ -4542,8 +4583,9 @@ impl magnus::IntoValue for BrowserMode {
 impl magnus::TryConvert for BrowserMode {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "auto" | "Auto" => Ok(BrowserMode::Auto),
             "always" | "Always" => Ok(BrowserMode::Always),
@@ -4588,8 +4630,9 @@ impl magnus::IntoValue for BrowserWait {
 impl magnus::TryConvert for BrowserWait {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "network_idle" | "NetworkIdle" => Ok(BrowserWait::NetworkIdle),
             "selector" | "Selector" => Ok(BrowserWait::Selector),
@@ -4631,8 +4674,9 @@ impl magnus::IntoValue for BrowserBackend {
 impl magnus::TryConvert for BrowserBackend {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "chromiumoxide" | "Chromiumoxide" => Ok(BrowserBackend::Chromiumoxide),
             "native" | "Native" => Ok(BrowserBackend::Native),
@@ -4671,8 +4715,9 @@ impl magnus::IntoValue for DocumentContentEncoding {
 impl magnus::TryConvert for DocumentContentEncoding {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "base64" | "Base64" => Ok(DocumentContentEncoding::Base64),
             other => Err(magnus::Error::new(
@@ -4686,6 +4731,95 @@ impl magnus::TryConvert for DocumentContentEncoding {
 unsafe impl IntoValueFromNative for DocumentContentEncoding {}
 unsafe impl TryConvertOwned for DocumentContentEncoding {}
 
+#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum CrawlStrategyKind {
+    Bfs,
+    Dfs,
+    BestFirst,
+    Adaptive,
+}
+
+impl Default for CrawlStrategyKind {
+    fn default() -> Self {
+        Self::Bfs
+    }
+}
+
+impl magnus::IntoValue for CrawlStrategyKind {
+    fn into_value_with(self, handle: &Ruby) -> magnus::Value {
+        let sym = match self {
+            CrawlStrategyKind::Bfs => "bfs",
+            CrawlStrategyKind::Dfs => "dfs",
+            CrawlStrategyKind::BestFirst => "best_first",
+            CrawlStrategyKind::Adaptive => "adaptive",
+        };
+        handle.to_symbol(sym).into_value_with(handle)
+    }
+}
+
+impl magnus::TryConvert for CrawlStrategyKind {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        let s: String = magnus::TryConvert::try_convert(val)?;
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
+        match s.as_str() {
+            "bfs" | "Bfs" => Ok(CrawlStrategyKind::Bfs),
+            "dfs" | "Dfs" => Ok(CrawlStrategyKind::Dfs),
+            "best_first" | "BestFirst" => Ok(CrawlStrategyKind::BestFirst),
+            "adaptive" | "Adaptive" => Ok(CrawlStrategyKind::Adaptive),
+            other => Err(magnus::Error::new(
+                unsafe { Ruby::get_unchecked() }.exception_arg_error(),
+                format!("invalid CrawlStrategyKind value: {other}"),
+            )),
+        }
+    }
+}
+
+unsafe impl IntoValueFromNative for CrawlStrategyKind {}
+unsafe impl TryConvertOwned for CrawlStrategyKind {}
+
+#[derive(Clone, Copy, PartialEq, Eq, Debug, serde::Serialize, serde::Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ContentFilterKind {
+    Bm25,
+}
+
+impl Default for ContentFilterKind {
+    fn default() -> Self {
+        Self::Bm25
+    }
+}
+
+impl magnus::IntoValue for ContentFilterKind {
+    fn into_value_with(self, handle: &Ruby) -> magnus::Value {
+        let sym = match self {
+            ContentFilterKind::Bm25 => "bm25",
+        };
+        handle.to_symbol(sym).into_value_with(handle)
+    }
+}
+
+impl magnus::TryConvert for ContentFilterKind {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        let s: String = magnus::TryConvert::try_convert(val)?;
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
+        match s.as_str() {
+            "bm25" | "Bm25" => Ok(ContentFilterKind::Bm25),
+            other => Err(magnus::Error::new(
+                unsafe { Ruby::get_unchecked() }.exception_arg_error(),
+                format!("invalid ContentFilterKind value: {other}"),
+            )),
+        }
+    }
+}
+
+unsafe impl IntoValueFromNative for ContentFilterKind {}
+unsafe impl TryConvertOwned for ContentFilterKind {}
+
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
 #[serde(tag = "type")]
 pub enum AuthConfig {
@@ -4781,8 +4915,9 @@ impl magnus::IntoValue for LinkType {
 impl magnus::TryConvert for LinkType {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "internal" | "Internal" => Ok(LinkType::Internal),
             "external" | "External" => Ok(LinkType::External),
@@ -4821,8 +4956,8 @@ impl magnus::IntoValue for ImageSource {
         let sym = match self {
             ImageSource::Img => "img",
             ImageSource::PictureSource => "picture_source",
-            ImageSource::OgImage => "og_image",
-            ImageSource::TwitterImage => "twitter_image",
+            ImageSource::OgImage => "og:image",
+            ImageSource::TwitterImage => "twitter:image",
         };
         handle.to_symbol(sym).into_value_with(handle)
     }
@@ -4831,13 +4966,14 @@ impl magnus::IntoValue for ImageSource {
 impl magnus::TryConvert for ImageSource {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "img" | "Img" => Ok(ImageSource::Img),
             "picture_source" | "PictureSource" => Ok(ImageSource::PictureSource),
-            "og_image" | "OgImage" => Ok(ImageSource::OgImage),
-            "twitter_image" | "TwitterImage" => Ok(ImageSource::TwitterImage),
+            "og:image" | "og_image" | "OgImage" => Ok(ImageSource::OgImage),
+            "twitter:image" | "twitter_image" | "TwitterImage" => Ok(ImageSource::TwitterImage),
             other => Err(magnus::Error::new(
                 unsafe { Ruby::get_unchecked() }.exception_arg_error(),
                 format!("invalid ImageSource value: {other}"),
@@ -4877,8 +5013,9 @@ impl magnus::IntoValue for FeedType {
 impl magnus::TryConvert for FeedType {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "rss" | "Rss" => Ok(FeedType::Rss),
             "atom" | "Atom" => Ok(FeedType::Atom),
@@ -4936,8 +5073,9 @@ impl magnus::IntoValue for AssetCategory {
 impl magnus::TryConvert for AssetCategory {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "document" | "Document" => Ok(AssetCategory::Document),
             "image" | "Image" => Ok(AssetCategory::Image),
@@ -5133,8 +5271,9 @@ impl magnus::IntoValue for ScrollDirection {
 impl magnus::TryConvert for ScrollDirection {
     fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
         let s: String = magnus::TryConvert::try_convert(val)?;
-        // Accept the serde wire name (snake_case), the PascalCase Rust variant name,
-        // and a lowercase fallback so fixtures written in any of those styles work.
+        // Accept the real serde wire value, the legacy always-snake_case symbol Magnus used to
+        // emit unconditionally, and the verbatim PascalCase Rust variant name, so fixtures and
+        // existing consumer code written in any of those styles keep working.
         match s.as_str() {
             "up" | "Up" => Ok(ScrollDirection::Up),
             "down" | "Down" => Ok(ScrollDirection::Down),
@@ -5612,6 +5751,10 @@ impl From<CrawlConfig> for crawlberg::CrawlConfig {
             max_pages: val.max_pages,
             max_links_per_page: val.max_links_per_page,
             max_concurrent: val.max_concurrent,
+            crawl_strategy: val.crawl_strategy.into(),
+            content_filter: val.content_filter.map(Into::into),
+            bm25_query: val.bm25_query,
+            bm25_threshold: val.bm25_threshold,
             respect_robots_txt: val.respect_robots_txt,
             soft_http_errors: val.soft_http_errors,
             user_agent: val.user_agent,
@@ -5668,6 +5811,10 @@ impl From<crawlberg::CrawlConfig> for CrawlConfig {
             max_pages: val.max_pages,
             max_links_per_page: val.max_links_per_page,
             max_concurrent: val.max_concurrent,
+            crawl_strategy: val.crawl_strategy.into(),
+            content_filter: val.content_filter.map(Into::into),
+            bm25_query: val.bm25_query.map(|v| v.to_string()),
+            bm25_threshold: val.bm25_threshold,
             respect_robots_txt: val.respect_robots_txt,
             soft_http_errors: val.soft_http_errors,
             user_agent: val.user_agent.map(|v| v.to_string()),
@@ -6746,6 +6893,44 @@ impl From<crawlberg::DocumentContentEncoding> for DocumentContentEncoding {
     }
 }
 
+impl From<CrawlStrategyKind> for crawlberg::CrawlStrategyKind {
+    fn from(val: CrawlStrategyKind) -> Self {
+        match val {
+            CrawlStrategyKind::Bfs => Self::Bfs,
+            CrawlStrategyKind::Dfs => Self::Dfs,
+            CrawlStrategyKind::BestFirst => Self::BestFirst,
+            CrawlStrategyKind::Adaptive => Self::Adaptive,
+        }
+    }
+}
+
+impl From<crawlberg::CrawlStrategyKind> for CrawlStrategyKind {
+    fn from(val: crawlberg::CrawlStrategyKind) -> Self {
+        match val {
+            crawlberg::CrawlStrategyKind::Bfs => Self::Bfs,
+            crawlberg::CrawlStrategyKind::Dfs => Self::Dfs,
+            crawlberg::CrawlStrategyKind::BestFirst => Self::BestFirst,
+            crawlberg::CrawlStrategyKind::Adaptive => Self::Adaptive,
+        }
+    }
+}
+
+impl From<ContentFilterKind> for crawlberg::ContentFilterKind {
+    fn from(val: ContentFilterKind) -> Self {
+        match val {
+            ContentFilterKind::Bm25 => Self::Bm25,
+        }
+    }
+}
+
+impl From<crawlberg::ContentFilterKind> for ContentFilterKind {
+    fn from(val: crawlberg::ContentFilterKind) -> Self {
+        match val {
+            crawlberg::ContentFilterKind::Bm25 => Self::Bm25,
+        }
+    }
+}
+
 impl From<AuthConfig> for crawlberg::AuthConfig {
     fn from(val: AuthConfig) -> Self {
         match val {
@@ -7286,6 +7471,14 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     class.define_method("max_concurrent", method!(CrawlConfig::max_concurrent, 0))?;
 
+    class.define_method("crawl_strategy", method!(CrawlConfig::crawl_strategy, 0))?;
+
+    class.define_method("content_filter", method!(CrawlConfig::content_filter, 0))?;
+
+    class.define_method("bm25_query", method!(CrawlConfig::bm25_query, 0))?;
+
+    class.define_method("bm25_threshold", method!(CrawlConfig::bm25_threshold, 0))?;
+
     class.define_method("respect_robots_txt", method!(CrawlConfig::respect_robots_txt, 0))?;
 
     class.define_method("soft_http_errors", method!(CrawlConfig::soft_http_errors, 0))?;