kreuzcrawl 0.1.2 → 0.3.0.pre.rc.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2f4027aef59737e93add85d5da2398c4a95767fb41766b86822f50199331e32
-  data.tar.gz: 82f38cf86e988321ef9bc11f5a42a023df464e79d0d0ad646edeae564cabfec6
+  metadata.gz: 92258fdaf73e3d2a7c3d2973905388ea408deb41a035d5d5a35613924c9417da
+  data.tar.gz: a907f79b86c3511a7c572dfcf07fa342d19f1e2455c25f47a7ea75b2b543c171
 SHA512:
-  metadata.gz: 1ebcebff06f45e809441c0cae1d9f14be48a79b6d1439b3421ebedd4808d9c71e6ffa86d51a6c1d64ba57a0bc2eeffa24a672a4af3db5659185ba5c8cca8443a
-  data.tar.gz: af29b39fdf985044e579d71064af9eb10dc348c81a82e7f27dc0650c4720ee1164fd8fe5529358cc9ba494b0fc296e9a425b17182794d001b3bcee526f737d35
+  metadata.gz: 86aa4b0c8f043de802d7d844167b351b0e140a46b2f2d04664e0710104e3c305b5e156a709aeb0303a534cb7a53818180015c6b82bebc3ce41f84f48ae0feded
+  data.tar.gz: 2d3747ecd2e4a410e93f894e1ca16a48ad881735eaed33d0989d8abfd4278cb55970cafa5c925e8ce3b0ef5b2b9d15bdb79827fe8451b1281f6ba9501f9f326b

data/ext/kreuzcrawl_rb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzcrawl-rb"
-version = "0.1.2"
+version = "0.3.0-rc.2"
 edition = "2024"
 license = "Elastic-2.0"
 

data/ext/kreuzcrawl_rb/src/kreuzcrawl/version.rb

@@ -1,6 +1,10 @@
-# This file is auto-generated by alef. DO NOT EDIT.
+# This file is auto-generated by alef — DO NOT EDIT.
+# alef:hash:b59e800fdddf59213911a2309f9e365fa6993399f1c12fbe68bd27b269cff2d9
+# To regenerate: alef generate
+# To verify freshness: alef verify --exit-code
+# Issues & docs: https://github.com/kreuzberg-dev/alef
 # frozen_string_literal: true
 
 module Kreuzcrawl
-  VERSION = "0.1.2"
+  VERSION = "0.3.0-rc.2"
 end

data/ext/kreuzcrawl_rb/src/kreuzcrawl.rb

@@ -1,4 +1,8 @@
-# This file is auto-generated by alef. DO NOT EDIT.
+# This file is auto-generated by alef — DO NOT EDIT.
+# alef:hash:cb606b2a7daa26b35ced54ce0131bc649210d7c9396f1309bf155a45e52ef34e
+# To regenerate: alef generate
+# To verify freshness: alef verify --exit-code
+# Issues & docs: https://github.com/kreuzberg-dev/alef
 # frozen_string_literal: true
 
 require_relative 'kreuzcrawl/version'

data/ext/kreuzcrawl_rb/src/lib.rs

@@ -1,19 +1,7 @@
 // This file is auto-generated by alef. DO NOT EDIT.
+// alef:hash:38edd0f53a45bf9e74fc6f8570ccc68607d810942abb51abaedc20c4afa2ccda
 // Re-generate with: alef generate
 #![allow(dead_code)]
-#![allow(
-    clippy::too_many_arguments,
-    clippy::let_unit_value,
-    clippy::needless_borrow,
-    clippy::map_identity,
-    clippy::just_underscores_and_digits,
-    clippy::unused_unit,
-    clippy::unnecessary_cast,
-    clippy::unwrap_or_default,
-    clippy::derivable_impls,
-    clippy::needless_borrows_for_generic_args,
-    clippy::unnecessary_fallible_conversions
-)]
 
 use magnus::{Error, IntoValueFromNative, Ruby, function, method, prelude::*, try_convert::TryConvertOwned};
 use std::collections::HashMap;
@@ -175,6 +163,133 @@ impl ProxyConfig {
     }
 }
 
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[magnus::wrap(class = "Kreuzcrawl::ContentConfig")]
+#[serde(default)]
+pub struct ContentConfig {
+    pub output_format: String,
+    pub preprocessing_preset: String,
+    pub remove_navigation: bool,
+    pub remove_forms: bool,
+    pub strip_tags: Vec<String>,
+    pub preserve_tags: Vec<String>,
+    pub exclude_selectors: Vec<String>,
+    pub skip_images: bool,
+    pub max_depth: Option<usize>,
+    pub wrap: bool,
+    pub wrap_width: usize,
+    pub include_document_structure: bool,
+}
+
+unsafe impl IntoValueFromNative for ContentConfig {}
+
+impl magnus::TryConvert for ContentConfig {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        let r: &ContentConfig = magnus::TryConvert::try_convert(val)?;
+        Ok(r.clone())
+    }
+}
+unsafe impl TryConvertOwned for ContentConfig {}
+
+impl Default for ContentConfig {
+    fn default() -> Self {
+        Self {
+            output_format: Default::default(),
+            preprocessing_preset: Default::default(),
+            remove_navigation: Default::default(),
+            remove_forms: Default::default(),
+            strip_tags: Default::default(),
+            preserve_tags: Default::default(),
+            exclude_selectors: Default::default(),
+            skip_images: Default::default(),
+            max_depth: Default::default(),
+            wrap: Default::default(),
+            wrap_width: Default::default(),
+            include_document_structure: Default::default(),
+        }
+    }
+}
+
+impl ContentConfig {
+    fn new(
+        output_format: Option<String>,
+        preprocessing_preset: Option<String>,
+        remove_navigation: Option<bool>,
+        remove_forms: Option<bool>,
+        strip_tags: Option<Vec<String>>,
+        preserve_tags: Option<Vec<String>>,
+        exclude_selectors: Option<Vec<String>>,
+        skip_images: Option<bool>,
+        max_depth: Option<usize>,
+        wrap: Option<bool>,
+        wrap_width: Option<usize>,
+        include_document_structure: Option<bool>,
+    ) -> Self {
+        Self {
+            output_format: output_format.unwrap_or("markdown".to_string()),
+            preprocessing_preset: preprocessing_preset.unwrap_or("standard".to_string()),
+            remove_navigation: remove_navigation.unwrap_or(true),
+            remove_forms: remove_forms.unwrap_or(true),
+            strip_tags: strip_tags.unwrap_or_default(),
+            preserve_tags: preserve_tags.unwrap_or_default(),
+            exclude_selectors: exclude_selectors.unwrap_or_default(),
+            skip_images: skip_images.unwrap_or(false),
+            max_depth,
+            wrap: wrap.unwrap_or(false),
+            wrap_width: wrap_width.unwrap_or(80),
+            include_document_structure: include_document_structure.unwrap_or(true),
+        }
+    }
+
+    fn output_format(&self) -> String {
+        self.output_format.clone()
+    }
+
+    fn preprocessing_preset(&self) -> String {
+        self.preprocessing_preset.clone()
+    }
+
+    fn remove_navigation(&self) -> bool {
+        self.remove_navigation
+    }
+
+    fn remove_forms(&self) -> bool {
+        self.remove_forms
+    }
+
+    fn strip_tags(&self) -> Vec<String> {
+        self.strip_tags.clone()
+    }
+
+    fn preserve_tags(&self) -> Vec<String> {
+        self.preserve_tags.clone()
+    }
+
+    fn exclude_selectors(&self) -> Vec<String> {
+        self.exclude_selectors.clone()
+    }
+
+    fn skip_images(&self) -> bool {
+        self.skip_images
+    }
+
+    fn max_depth(&self) -> Option<usize> {
+        self.max_depth
+    }
+
+    fn wrap(&self) -> bool {
+        self.wrap
+    }
+
+    fn wrap_width(&self) -> usize {
+        self.wrap_width
+    }
+
+    fn include_document_structure(&self) -> bool {
+        self.include_document_structure
+    }
+}
+
 #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
 #[magnus::wrap(class = "Kreuzcrawl::BrowserConfig")]
 #[serde(default)]
@@ -263,8 +378,8 @@ pub struct CrawlConfig {
     pub cookies_enabled: bool,
     pub auth: Option<AuthConfig>,
     pub max_body_size: Option<usize>,
-    pub main_content_only: bool,
     pub remove_tags: Vec<String>,
+    pub content: ContentConfig,
     pub map_limit: Option<usize>,
     pub map_search: Option<String>,
     pub download_assets: bool,
@@ -361,14 +476,14 @@ impl CrawlConfig {
             max_body_size: kwargs
                 .get(ruby.to_symbol("max_body_size"))
                 .and_then(|v| usize::try_convert(v).ok()),
-            main_content_only: kwargs
-                .get(ruby.to_symbol("main_content_only"))
-                .and_then(|v| bool::try_convert(v).ok())
-                .unwrap_or(false),
             remove_tags: kwargs
                 .get(ruby.to_symbol("remove_tags"))
                 .and_then(|v| <Vec<String>>::try_convert(v).ok())
                 .unwrap_or_default(),
+            content: kwargs
+                .get(ruby.to_symbol("content"))
+                .and_then(|v| ContentConfig::try_convert(v).ok())
+                .unwrap_or_default(),
             map_limit: kwargs
                 .get(ruby.to_symbol("map_limit"))
                 .and_then(|v| usize::try_convert(v).ok()),
@@ -497,14 +612,14 @@ impl CrawlConfig {
         self.max_body_size
     }
 
-    fn main_content_only(&self) -> bool {
-        self.main_content_only
-    }
-
     fn remove_tags(&self) -> Vec<String> {
         self.remove_tags.clone()
     }
 
+    fn content(&self) -> ContentConfig {
+        self.content.clone()
+    }
+
     fn map_limit(&self) -> Option<usize> {
         self.map_limit
     }
@@ -586,8 +701,8 @@ impl CrawlConfig {
             cookies_enabled: self.cookies_enabled,
             auth: self.auth.clone().map(Into::into),
             max_body_size: self.max_body_size,
-            main_content_only: self.main_content_only,
             remove_tags: self.remove_tags.clone(),
+            content: self.content.clone().into(),
             map_limit: self.map_limit,
             map_search: self.map_search.clone(),
             download_assets: self.download_assets,
@@ -723,7 +838,6 @@ pub struct ScrapeResult {
     pub is_pdf: bool,
     pub was_skipped: bool,
     pub detected_charset: Option<String>,
-    pub main_content_only: bool,
     pub auth_header_sent: bool,
     pub response_meta: Option<ResponseMeta>,
     pub assets: Vec<DownloadedAsset>,
@@ -766,7 +880,6 @@ impl Default for ScrapeResult {
             is_pdf: Default::default(),
             was_skipped: Default::default(),
             detected_charset: Default::default(),
-            main_content_only: Default::default(),
             auth_header_sent: Default::default(),
             response_meta: Default::default(),
             assets: Default::default(),
@@ -850,10 +963,6 @@ impl ScrapeResult {
             detected_charset: kwargs
                 .get(ruby.to_symbol("detected_charset"))
                 .and_then(|v| String::try_convert(v).ok()),
-            main_content_only: kwargs
-                .get(ruby.to_symbol("main_content_only"))
-                .and_then(|v| bool::try_convert(v).ok())
-                .unwrap_or_default(),
             auth_header_sent: kwargs
                 .get(ruby.to_symbol("auth_header_sent"))
                 .and_then(|v| bool::try_convert(v).ok())
@@ -959,10 +1068,6 @@ impl ScrapeResult {
         self.detected_charset.clone()
     }
 
-    fn main_content_only(&self) -> bool {
-        self.main_content_only
-    }
-
     fn auth_header_sent(&self) -> bool {
         self.auth_header_sent
     }
@@ -3270,6 +3375,7 @@ fn batch_crawl_async(engine: CrawlEngineHandle, urls: Vec<String>) -> Result<Vec
     Ok(result.into_iter().map(Into::into).collect())
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ExtractionMeta> for kreuzcrawl::ExtractionMeta {
     fn from(val: ExtractionMeta) -> Self {
         Self {
@@ -3282,6 +3388,7 @@ impl From<ExtractionMeta> for kreuzcrawl::ExtractionMeta {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::ExtractionMeta> for ExtractionMeta {
     fn from(val: kreuzcrawl::ExtractionMeta) -> Self {
         Self {
@@ -3294,6 +3401,7 @@ impl From<kreuzcrawl::ExtractionMeta> for ExtractionMeta {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ProxyConfig> for kreuzcrawl::ProxyConfig {
     fn from(val: ProxyConfig) -> Self {
         Self {
@@ -3304,6 +3412,7 @@ impl From<ProxyConfig> for kreuzcrawl::ProxyConfig {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::ProxyConfig> for ProxyConfig {
     fn from(val: kreuzcrawl::ProxyConfig) -> Self {
         Self {
@@ -3314,6 +3423,47 @@ impl From<kreuzcrawl::ProxyConfig> for ProxyConfig {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+impl From<ContentConfig> for kreuzcrawl::ContentConfig {
+    fn from(val: ContentConfig) -> Self {
+        Self {
+            output_format: val.output_format,
+            preprocessing_preset: val.preprocessing_preset,
+            remove_navigation: val.remove_navigation,
+            remove_forms: val.remove_forms,
+            strip_tags: val.strip_tags,
+            preserve_tags: val.preserve_tags,
+            exclude_selectors: val.exclude_selectors,
+            skip_images: val.skip_images,
+            max_depth: val.max_depth,
+            wrap: val.wrap,
+            wrap_width: val.wrap_width,
+            include_document_structure: val.include_document_structure,
+        }
+    }
+}
+
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
+impl From<kreuzcrawl::ContentConfig> for ContentConfig {
+    fn from(val: kreuzcrawl::ContentConfig) -> Self {
+        Self {
+            output_format: val.output_format,
+            preprocessing_preset: val.preprocessing_preset,
+            remove_navigation: val.remove_navigation,
+            remove_forms: val.remove_forms,
+            strip_tags: val.strip_tags,
+            preserve_tags: val.preserve_tags,
+            exclude_selectors: val.exclude_selectors,
+            skip_images: val.skip_images,
+            max_depth: val.max_depth,
+            wrap: val.wrap,
+            wrap_width: val.wrap_width,
+            include_document_structure: val.include_document_structure,
+        }
+    }
+}
+
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BrowserConfig> for kreuzcrawl::BrowserConfig {
     fn from(val: BrowserConfig) -> Self {
         Self {
@@ -3327,6 +3477,7 @@ impl From<BrowserConfig> for kreuzcrawl::BrowserConfig {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::BrowserConfig> for BrowserConfig {
     fn from(val: kreuzcrawl::BrowserConfig) -> Self {
         Self {
@@ -3341,6 +3492,7 @@ impl From<kreuzcrawl::BrowserConfig> for BrowserConfig {
 }
 
 #[allow(clippy::needless_update)]
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CrawlConfig> for kreuzcrawl::CrawlConfig {
     fn from(val: CrawlConfig) -> Self {
         Self {
@@ -3362,8 +3514,8 @@ impl From<CrawlConfig> for kreuzcrawl::CrawlConfig {
             cookies_enabled: val.cookies_enabled,
             auth: val.auth.map(Into::into),
             max_body_size: val.max_body_size,
-            main_content_only: val.main_content_only,
             remove_tags: val.remove_tags,
+            content: val.content.into(),
             map_limit: val.map_limit,
             map_search: val.map_search,
             download_assets: val.download_assets,
@@ -3384,6 +3536,7 @@ impl From<CrawlConfig> for kreuzcrawl::CrawlConfig {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::CrawlConfig> for CrawlConfig {
     fn from(val: kreuzcrawl::CrawlConfig) -> Self {
         Self {
@@ -3405,8 +3558,8 @@ impl From<kreuzcrawl::CrawlConfig> for CrawlConfig {
             cookies_enabled: val.cookies_enabled,
             auth: val.auth.map(Into::into),
             max_body_size: val.max_body_size,
-            main_content_only: val.main_content_only,
             remove_tags: val.remove_tags,
+            content: val.content.into(),
             map_limit: val.map_limit,
             map_search: val.map_search,
             download_assets: val.download_assets,
@@ -3426,12 +3579,13 @@ impl From<kreuzcrawl::CrawlConfig> for CrawlConfig {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<DownloadedDocument> for kreuzcrawl::DownloadedDocument {
     fn from(val: DownloadedDocument) -> Self {
         Self {
             url: val.url,
             mime_type: Default::default(),
-            content: val.content,
+            content: val.content.into(),
             size: val.size,
             filename: Default::default(),
             content_hash: Default::default(),
@@ -3440,6 +3594,7 @@ impl From<DownloadedDocument> for kreuzcrawl::DownloadedDocument {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::DownloadedDocument> for DownloadedDocument {
     fn from(val: kreuzcrawl::DownloadedDocument) -> Self {
         Self {
@@ -3458,6 +3613,7 @@ impl From<kreuzcrawl::DownloadedDocument> for DownloadedDocument {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ScrapeResult> for kreuzcrawl::ScrapeResult {
     fn from(val: ScrapeResult) -> Self {
         Self {
@@ -3478,7 +3634,6 @@ impl From<ScrapeResult> for kreuzcrawl::ScrapeResult {
             is_pdf: val.is_pdf,
             was_skipped: val.was_skipped,
             detected_charset: val.detected_charset,
-            main_content_only: val.main_content_only,
             auth_header_sent: val.auth_header_sent,
             response_meta: val.response_meta.map(Into::into),
             assets: val.assets.into_iter().map(Into::into).collect(),
@@ -3487,12 +3642,13 @@ impl From<ScrapeResult> for kreuzcrawl::ScrapeResult {
             markdown: val.markdown.map(Into::into),
             extracted_data: val.extracted_data.as_ref().and_then(|s| serde_json::from_str(s).ok()),
             extraction_meta: val.extraction_meta.map(Into::into),
-            screenshot: val.screenshot,
+            screenshot: val.screenshot.map(Into::into),
             downloaded_document: val.downloaded_document.map(Into::into),
         }
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::ScrapeResult> for ScrapeResult {
     fn from(val: kreuzcrawl::ScrapeResult) -> Self {
         Self {
@@ -3513,7 +3669,6 @@ impl From<kreuzcrawl::ScrapeResult> for ScrapeResult {
             is_pdf: val.is_pdf,
             was_skipped: val.was_skipped,
             detected_charset: val.detected_charset,
-            main_content_only: val.main_content_only,
             auth_header_sent: val.auth_header_sent,
             response_meta: val.response_meta.map(Into::into),
             assets: val.assets.into_iter().map(Into::into).collect(),
@@ -3528,6 +3683,7 @@ impl From<kreuzcrawl::ScrapeResult> for ScrapeResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CrawlPageResult> for kreuzcrawl::CrawlPageResult {
     fn from(val: CrawlPageResult) -> Self {
         Self {
@@ -3555,6 +3711,7 @@ impl From<CrawlPageResult> for kreuzcrawl::CrawlPageResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::CrawlPageResult> for CrawlPageResult {
     fn from(val: kreuzcrawl::CrawlPageResult) -> Self {
         Self {
@@ -3582,6 +3739,7 @@ impl From<kreuzcrawl::CrawlPageResult> for CrawlPageResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CrawlResult> for kreuzcrawl::CrawlResult {
     fn from(val: CrawlResult) -> Self {
         Self {
@@ -3596,6 +3754,7 @@ impl From<CrawlResult> for kreuzcrawl::CrawlResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::CrawlResult> for CrawlResult {
     fn from(val: kreuzcrawl::CrawlResult) -> Self {
         Self {
@@ -3610,6 +3769,7 @@ impl From<kreuzcrawl::CrawlResult> for CrawlResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<SitemapUrl> for kreuzcrawl::SitemapUrl {
     fn from(val: SitemapUrl) -> Self {
         Self {
@@ -3621,6 +3781,7 @@ impl From<SitemapUrl> for kreuzcrawl::SitemapUrl {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::SitemapUrl> for SitemapUrl {
     fn from(val: kreuzcrawl::SitemapUrl) -> Self {
         Self {
@@ -3632,6 +3793,7 @@ impl From<kreuzcrawl::SitemapUrl> for SitemapUrl {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<MapResult> for kreuzcrawl::MapResult {
     fn from(val: MapResult) -> Self {
         Self {
@@ -3640,6 +3802,7 @@ impl From<MapResult> for kreuzcrawl::MapResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::MapResult> for MapResult {
     fn from(val: kreuzcrawl::MapResult) -> Self {
         Self {
@@ -3648,6 +3811,7 @@ impl From<kreuzcrawl::MapResult> for MapResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<MarkdownResult> for kreuzcrawl::MarkdownResult {
     fn from(val: MarkdownResult) -> Self {
         Self {
@@ -3668,6 +3832,7 @@ impl From<MarkdownResult> for kreuzcrawl::MarkdownResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::MarkdownResult> for MarkdownResult {
     fn from(val: kreuzcrawl::MarkdownResult) -> Self {
         Self {
@@ -3681,6 +3846,7 @@ impl From<kreuzcrawl::MarkdownResult> for MarkdownResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<LinkInfo> for kreuzcrawl::LinkInfo {
     fn from(val: LinkInfo) -> Self {
         Self {
@@ -3693,6 +3859,7 @@ impl From<LinkInfo> for kreuzcrawl::LinkInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::LinkInfo> for LinkInfo {
     fn from(val: kreuzcrawl::LinkInfo) -> Self {
         Self {
@@ -3705,6 +3872,7 @@ impl From<kreuzcrawl::LinkInfo> for LinkInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ImageInfo> for kreuzcrawl::ImageInfo {
     fn from(val: ImageInfo) -> Self {
         Self {
@@ -3717,6 +3885,7 @@ impl From<ImageInfo> for kreuzcrawl::ImageInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::ImageInfo> for ImageInfo {
     fn from(val: kreuzcrawl::ImageInfo) -> Self {
         Self {
@@ -3729,6 +3898,7 @@ impl From<kreuzcrawl::ImageInfo> for ImageInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<FeedInfo> for kreuzcrawl::FeedInfo {
     fn from(val: FeedInfo) -> Self {
         Self {
@@ -3739,6 +3909,7 @@ impl From<FeedInfo> for kreuzcrawl::FeedInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::FeedInfo> for FeedInfo {
     fn from(val: kreuzcrawl::FeedInfo) -> Self {
         Self {
@@ -3749,6 +3920,7 @@ impl From<kreuzcrawl::FeedInfo> for FeedInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<JsonLdEntry> for kreuzcrawl::JsonLdEntry {
     fn from(val: JsonLdEntry) -> Self {
         Self {
@@ -3759,6 +3931,7 @@ impl From<JsonLdEntry> for kreuzcrawl::JsonLdEntry {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::JsonLdEntry> for JsonLdEntry {
     fn from(val: kreuzcrawl::JsonLdEntry) -> Self {
         Self {
@@ -3769,6 +3942,7 @@ impl From<kreuzcrawl::JsonLdEntry> for JsonLdEntry {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CookieInfo> for kreuzcrawl::CookieInfo {
     fn from(val: CookieInfo) -> Self {
         Self {
@@ -3780,6 +3954,7 @@ impl From<CookieInfo> for kreuzcrawl::CookieInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::CookieInfo> for CookieInfo {
     fn from(val: kreuzcrawl::CookieInfo) -> Self {
         Self {
@@ -3791,6 +3966,7 @@ impl From<kreuzcrawl::CookieInfo> for CookieInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<DownloadedAsset> for kreuzcrawl::DownloadedAsset {
     fn from(val: DownloadedAsset) -> Self {
         Self {
@@ -3804,6 +3980,7 @@ impl From<DownloadedAsset> for kreuzcrawl::DownloadedAsset {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::DownloadedAsset> for DownloadedAsset {
     fn from(val: kreuzcrawl::DownloadedAsset) -> Self {
         Self {
@@ -3817,6 +3994,7 @@ impl From<kreuzcrawl::DownloadedAsset> for DownloadedAsset {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ArticleMetadata> for kreuzcrawl::ArticleMetadata {
     fn from(val: ArticleMetadata) -> Self {
         Self {
@@ -3829,6 +4007,7 @@ impl From<ArticleMetadata> for kreuzcrawl::ArticleMetadata {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::ArticleMetadata> for ArticleMetadata {
     fn from(val: kreuzcrawl::ArticleMetadata) -> Self {
         Self {
@@ -3841,6 +4020,7 @@ impl From<kreuzcrawl::ArticleMetadata> for ArticleMetadata {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<HreflangEntry> for kreuzcrawl::HreflangEntry {
     fn from(val: HreflangEntry) -> Self {
         Self {
@@ -3850,6 +4030,7 @@ impl From<HreflangEntry> for kreuzcrawl::HreflangEntry {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::HreflangEntry> for HreflangEntry {
     fn from(val: kreuzcrawl::HreflangEntry) -> Self {
         Self {
@@ -3859,6 +4040,7 @@ impl From<kreuzcrawl::HreflangEntry> for HreflangEntry {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<FaviconInfo> for kreuzcrawl::FaviconInfo {
     fn from(val: FaviconInfo) -> Self {
         Self {
@@ -3870,6 +4052,7 @@ impl From<FaviconInfo> for kreuzcrawl::FaviconInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::FaviconInfo> for FaviconInfo {
     fn from(val: kreuzcrawl::FaviconInfo) -> Self {
         Self {
@@ -3881,6 +4064,7 @@ impl From<kreuzcrawl::FaviconInfo> for FaviconInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<HeadingInfo> for kreuzcrawl::HeadingInfo {
     fn from(val: HeadingInfo) -> Self {
         Self {
@@ -3890,6 +4074,7 @@ impl From<HeadingInfo> for kreuzcrawl::HeadingInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::HeadingInfo> for HeadingInfo {
     fn from(val: kreuzcrawl::HeadingInfo) -> Self {
         Self {
@@ -3899,6 +4084,7 @@ impl From<kreuzcrawl::HeadingInfo> for HeadingInfo {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ResponseMeta> for kreuzcrawl::ResponseMeta {
     fn from(val: ResponseMeta) -> Self {
         Self {
@@ -3913,6 +4099,7 @@ impl From<ResponseMeta> for kreuzcrawl::ResponseMeta {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::ResponseMeta> for ResponseMeta {
     fn from(val: kreuzcrawl::ResponseMeta) -> Self {
         Self {
@@ -3927,6 +4114,7 @@ impl From<kreuzcrawl::ResponseMeta> for ResponseMeta {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<PageMetadata> for kreuzcrawl::PageMetadata {
     fn from(val: PageMetadata) -> Self {
         Self {
@@ -3977,6 +4165,7 @@ impl From<PageMetadata> for kreuzcrawl::PageMetadata {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::PageMetadata> for PageMetadata {
     fn from(val: kreuzcrawl::PageMetadata) -> Self {
         Self {
@@ -4027,6 +4216,7 @@ impl From<kreuzcrawl::PageMetadata> for PageMetadata {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CitationResult> for kreuzcrawl::CitationResult {
     fn from(val: CitationResult) -> Self {
         Self {
@@ -4036,6 +4226,7 @@ impl From<CitationResult> for kreuzcrawl::CitationResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::CitationResult> for CitationResult {
     fn from(val: kreuzcrawl::CitationResult) -> Self {
         Self {
@@ -4045,6 +4236,7 @@ impl From<kreuzcrawl::CitationResult> for CitationResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CitationReference> for kreuzcrawl::CitationReference {
     fn from(val: CitationReference) -> Self {
         Self {
@@ -4055,6 +4247,7 @@ impl From<CitationReference> for kreuzcrawl::CitationReference {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::CitationReference> for CitationReference {
     fn from(val: kreuzcrawl::CitationReference) -> Self {
         Self {
@@ -4065,6 +4258,7 @@ impl From<kreuzcrawl::CitationReference> for CitationReference {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BatchScrapeResult> for kreuzcrawl::BatchScrapeResult {
     fn from(val: BatchScrapeResult) -> Self {
         Self {
@@ -4075,6 +4269,7 @@ impl From<BatchScrapeResult> for kreuzcrawl::BatchScrapeResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::BatchScrapeResult> for BatchScrapeResult {
     fn from(val: kreuzcrawl::BatchScrapeResult) -> Self {
         Self {
@@ -4085,6 +4280,7 @@ impl From<kreuzcrawl::BatchScrapeResult> for BatchScrapeResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<BatchCrawlResult> for kreuzcrawl::BatchCrawlResult {
     fn from(val: BatchCrawlResult) -> Self {
         Self {
@@ -4095,6 +4291,7 @@ impl From<BatchCrawlResult> for kreuzcrawl::BatchCrawlResult {
     }
 }
 
+#[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<kreuzcrawl::BatchCrawlResult> for BatchCrawlResult {
     fn from(val: kreuzcrawl::BatchCrawlResult) -> Self {
         Self {
@@ -4288,6 +4485,24 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("username", method!(ProxyConfig::username, 0))?;
     class.define_method("password", method!(ProxyConfig::password, 0))?;
 
+    let class = module.define_class("ContentConfig", ruby.class_object())?;
+    class.define_singleton_method("new", function!(ContentConfig::new, 12))?;
+    class.define_method("output_format", method!(ContentConfig::output_format, 0))?;
+    class.define_method("preprocessing_preset", method!(ContentConfig::preprocessing_preset, 0))?;
+    class.define_method("remove_navigation", method!(ContentConfig::remove_navigation, 0))?;
+    class.define_method("remove_forms", method!(ContentConfig::remove_forms, 0))?;
+    class.define_method("strip_tags", method!(ContentConfig::strip_tags, 0))?;
+    class.define_method("preserve_tags", method!(ContentConfig::preserve_tags, 0))?;
+    class.define_method("exclude_selectors", method!(ContentConfig::exclude_selectors, 0))?;
+    class.define_method("skip_images", method!(ContentConfig::skip_images, 0))?;
+    class.define_method("max_depth", method!(ContentConfig::max_depth, 0))?;
+    class.define_method("wrap", method!(ContentConfig::wrap, 0))?;
+    class.define_method("wrap_width", method!(ContentConfig::wrap_width, 0))?;
+    class.define_method(
+        "include_document_structure",
+        method!(ContentConfig::include_document_structure, 0),
+    )?;
+
     let class = module.define_class("BrowserConfig", ruby.class_object())?;
     class.define_singleton_method("new", function!(BrowserConfig::new, 6))?;
     class.define_method("mode", method!(BrowserConfig::mode, 0))?;
@@ -4317,8 +4532,8 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("cookies_enabled", method!(CrawlConfig::cookies_enabled, 0))?;
     class.define_method("auth", method!(CrawlConfig::auth, 0))?;
     class.define_method("max_body_size", method!(CrawlConfig::max_body_size, 0))?;
-    class.define_method("main_content_only", method!(CrawlConfig::main_content_only, 0))?;
     class.define_method("remove_tags", method!(CrawlConfig::remove_tags, 0))?;
+    class.define_method("content", method!(CrawlConfig::content, 0))?;
     class.define_method("map_limit", method!(CrawlConfig::map_limit, 0))?;
     class.define_method("map_search", method!(CrawlConfig::map_search, 0))?;
     class.define_method("download_assets", method!(CrawlConfig::download_assets, 0))?;
@@ -4365,7 +4580,6 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("is_pdf", method!(ScrapeResult::is_pdf, 0))?;
     class.define_method("was_skipped", method!(ScrapeResult::was_skipped, 0))?;
     class.define_method("detected_charset", method!(ScrapeResult::detected_charset, 0))?;
-    class.define_method("main_content_only", method!(ScrapeResult::main_content_only, 0))?;
     class.define_method("auth_header_sent", method!(ScrapeResult::auth_header_sent, 0))?;
     class.define_method("response_meta", method!(ScrapeResult::response_meta, 0))?;
     class.define_method("assets", method!(ScrapeResult::assets, 0))?;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kreuzcrawl
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.3.0.pre.rc.2
 platform: ruby
 authors:
 - Kreuzberg Team