kreuzcrawl 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 922ae77455c2e0e37df7ddb5a4084ad765187c947427092346d577e577d25be1
-  data.tar.gz: e1e537da083dea650de05a86713fe0b93134751345b7bb9d47ee3760379774c4
+  metadata.gz: 92e6cb8be4ef5d12847d6d632465cf41b18ce6564652049052b8793162204255
+  data.tar.gz: 9bd7b541b57c35b0ddb1cec5d6b119463ca1622db1f57455b340f38b4cf3abd3
 SHA512:
-  metadata.gz: 5499c3684018e4bb69febc2e3c939b9f388c2976636f6dce443866108d898759895e5a0567b8bf9fdfbadf1c35dda8f54cbf73ec2ddfb1f20c39dac7a5a0efd6
-  data.tar.gz: c82c4450e90a1a47216f9240f181791a7fa37089c7d776709bcd0c031949a0043c9045c0919f04ac1f8672f46c9bcb811daed5f01b5e0bde07e8e2f87edcf020
+  metadata.gz: 78ae3990a646e6f0a9ecfdb8355d4bfb11f476d3c0dc498651e5ebeb08f28dd43dd848178c43ee4f5788517e0014b3b20f8827990abe9e8e3377cb43c27be773
+  data.tar.gz: f43d72f320847d7bf872dc7d59d8cf816bf1c2faf05528998773dbd55ab414dd35380e190f7a0f64eb0e60eae4866c22263eceb0ba05da02a47ad5e98aecbeeb

data/ext/kreuzcrawl_rb/Cargo.lock

@@ -1174,7 +1174,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl"
-version = "0.1.0-rc.5"
+version = "0.1.1"
 dependencies = [
  "ahash",
  "astral-tl",
@@ -1200,7 +1200,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl-rb"
-version = "0.1.0-rc.5"
+version = "0.1.1"
 dependencies = [
  "kreuzcrawl",
  "magnus",

data/ext/kreuzcrawl_rb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzcrawl-rb"
-version = "0.1.1"
+version = "0.2.0"
 edition = "2024"
 license = "Elastic-2.0"
 

data/ext/kreuzcrawl_rb/src/kreuzcrawl/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Kreuzcrawl
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/ext/kreuzcrawl_rb/src/lib.rs

@@ -1,6 +1,19 @@
 // This file is auto-generated by alef. DO NOT EDIT.
 // Re-generate with: alef generate
 #![allow(dead_code)]
+#![allow(
+    clippy::too_many_arguments,
+    clippy::let_unit_value,
+    clippy::needless_borrow,
+    clippy::map_identity,
+    clippy::just_underscores_and_digits,
+    clippy::unused_unit,
+    clippy::unnecessary_cast,
+    clippy::unwrap_or_default,
+    clippy::derivable_impls,
+    clippy::needless_borrows_for_generic_args,
+    clippy::unnecessary_fallible_conversions
+)]
 
 use magnus::{Error, IntoValueFromNative, Ruby, function, method, prelude::*, try_convert::TryConvertOwned};
 use std::collections::HashMap;
@@ -162,6 +175,133 @@ impl ProxyConfig {
     }
 }
 
+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
+#[magnus::wrap(class = "Kreuzcrawl::ContentConfig")]
+#[serde(default)]
+pub struct ContentConfig {
+    pub output_format: String,
+    pub preprocessing_preset: String,
+    pub remove_navigation: bool,
+    pub remove_forms: bool,
+    pub strip_tags: Vec<String>,
+    pub preserve_tags: Vec<String>,
+    pub exclude_selectors: Vec<String>,
+    pub skip_images: bool,
+    pub max_depth: Option<usize>,
+    pub wrap: bool,
+    pub wrap_width: usize,
+    pub include_document_structure: bool,
+}
+
+unsafe impl IntoValueFromNative for ContentConfig {}
+
+impl magnus::TryConvert for ContentConfig {
+    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
+        let r: &ContentConfig = magnus::TryConvert::try_convert(val)?;
+        Ok(r.clone())
+    }
+}
+unsafe impl TryConvertOwned for ContentConfig {}
+
+impl Default for ContentConfig {
+    fn default() -> Self {
+        Self {
+            output_format: Default::default(),
+            preprocessing_preset: Default::default(),
+            remove_navigation: Default::default(),
+            remove_forms: Default::default(),
+            strip_tags: Default::default(),
+            preserve_tags: Default::default(),
+            exclude_selectors: Default::default(),
+            skip_images: Default::default(),
+            max_depth: Default::default(),
+            wrap: Default::default(),
+            wrap_width: Default::default(),
+            include_document_structure: Default::default(),
+        }
+    }
+}
+
+impl ContentConfig {
+    fn new(
+        output_format: Option<String>,
+        preprocessing_preset: Option<String>,
+        remove_navigation: Option<bool>,
+        remove_forms: Option<bool>,
+        strip_tags: Option<Vec<String>>,
+        preserve_tags: Option<Vec<String>>,
+        exclude_selectors: Option<Vec<String>>,
+        skip_images: Option<bool>,
+        max_depth: Option<usize>,
+        wrap: Option<bool>,
+        wrap_width: Option<usize>,
+        include_document_structure: Option<bool>,
+    ) -> Self {
+        Self {
+            output_format: output_format.unwrap_or("markdown".to_string()),
+            preprocessing_preset: preprocessing_preset.unwrap_or("standard".to_string()),
+            remove_navigation: remove_navigation.unwrap_or(true),
+            remove_forms: remove_forms.unwrap_or(true),
+            strip_tags: strip_tags.unwrap_or_default(),
+            preserve_tags: preserve_tags.unwrap_or_default(),
+            exclude_selectors: exclude_selectors.unwrap_or_default(),
+            skip_images: skip_images.unwrap_or(false),
+            max_depth,
+            wrap: wrap.unwrap_or(false),
+            wrap_width: wrap_width.unwrap_or(80),
+            include_document_structure: include_document_structure.unwrap_or(true),
+        }
+    }
+
+    fn output_format(&self) -> String {
+        self.output_format.clone()
+    }
+
+    fn preprocessing_preset(&self) -> String {
+        self.preprocessing_preset.clone()
+    }
+
+    fn remove_navigation(&self) -> bool {
+        self.remove_navigation
+    }
+
+    fn remove_forms(&self) -> bool {
+        self.remove_forms
+    }
+
+    fn strip_tags(&self) -> Vec<String> {
+        self.strip_tags.clone()
+    }
+
+    fn preserve_tags(&self) -> Vec<String> {
+        self.preserve_tags.clone()
+    }
+
+    fn exclude_selectors(&self) -> Vec<String> {
+        self.exclude_selectors.clone()
+    }
+
+    fn skip_images(&self) -> bool {
+        self.skip_images
+    }
+
+    fn max_depth(&self) -> Option<usize> {
+        self.max_depth
+    }
+
+    fn wrap(&self) -> bool {
+        self.wrap
+    }
+
+    fn wrap_width(&self) -> usize {
+        self.wrap_width
+    }
+
+    fn include_document_structure(&self) -> bool {
+        self.include_document_structure
+    }
+}
+
 #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
 #[magnus::wrap(class = "Kreuzcrawl::BrowserConfig")]
 #[serde(default)]
@@ -243,14 +383,15 @@ pub struct CrawlConfig {
     pub exclude_paths: Vec<String>,
     pub custom_headers: HashMap<String, String>,
     pub request_timeout: u64,
+    pub rate_limit_ms: Option<u64>,
     pub max_redirects: usize,
     pub retry_count: usize,
     pub retry_codes: Vec<u16>,
     pub cookies_enabled: bool,
     pub auth: Option<AuthConfig>,
     pub max_body_size: Option<usize>,
-    pub main_content_only: bool,
     pub remove_tags: Vec<String>,
+    pub content: ContentConfig,
     pub map_limit: Option<usize>,
     pub map_search: Option<String>,
     pub download_assets: bool,
@@ -322,6 +463,9 @@ impl CrawlConfig {
                 .get(ruby.to_symbol("request_timeout"))
                 .and_then(|v| u64::try_convert(v).ok())
                 .unwrap_or(30000),
+            rate_limit_ms: kwargs
+                .get(ruby.to_symbol("rate_limit_ms"))
+                .and_then(|v| u64::try_convert(v).ok()),
             max_redirects: kwargs
                 .get(ruby.to_symbol("max_redirects"))
                 .and_then(|v| usize::try_convert(v).ok())
@@ -344,14 +488,14 @@ impl CrawlConfig {
             max_body_size: kwargs
                 .get(ruby.to_symbol("max_body_size"))
                 .and_then(|v| usize::try_convert(v).ok()),
-            main_content_only: kwargs
-                .get(ruby.to_symbol("main_content_only"))
-                .and_then(|v| bool::try_convert(v).ok())
-                .unwrap_or(false),
             remove_tags: kwargs
                 .get(ruby.to_symbol("remove_tags"))
                 .and_then(|v| <Vec<String>>::try_convert(v).ok())
                 .unwrap_or_default(),
+            content: kwargs
+                .get(ruby.to_symbol("content"))
+                .and_then(|v| ContentConfig::try_convert(v).ok())
+                .unwrap_or_default(),
             map_limit: kwargs
                 .get(ruby.to_symbol("map_limit"))
                 .and_then(|v| usize::try_convert(v).ok()),
@@ -452,6 +596,10 @@ impl CrawlConfig {
         self.request_timeout.clone()
     }
 
+    fn rate_limit_ms(&self) -> Option<u64> {
+        self.rate_limit_ms
+    }
+
     fn max_redirects(&self) -> usize {
         self.max_redirects
     }
@@ -476,14 +624,14 @@ impl CrawlConfig {
         self.max_body_size
     }
 
-    fn main_content_only(&self) -> bool {
-        self.main_content_only
-    }
-
     fn remove_tags(&self) -> Vec<String> {
         self.remove_tags.clone()
     }
 
+    fn content(&self) -> ContentConfig {
+        self.content.clone()
+    }
+
     fn map_limit(&self) -> Option<usize> {
         self.map_limit
     }
@@ -558,14 +706,15 @@ impl CrawlConfig {
             exclude_paths: self.exclude_paths.clone(),
             custom_headers: self.custom_headers.clone().into_iter().collect(),
             request_timeout: std::time::Duration::from_millis(self.request_timeout),
+            rate_limit_ms: self.rate_limit_ms,
             max_redirects: self.max_redirects,
             retry_count: self.retry_count,
             retry_codes: self.retry_codes.clone(),
             cookies_enabled: self.cookies_enabled,
             auth: self.auth.clone().map(Into::into),
             max_body_size: self.max_body_size,
-            main_content_only: self.main_content_only,
             remove_tags: self.remove_tags.clone(),
+            content: self.content.clone().into(),
             map_limit: self.map_limit,
             map_search: self.map_search.clone(),
             download_assets: self.download_assets,
@@ -680,140 +829,6 @@ impl DownloadedDocument {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-#[magnus::wrap(class = "Kreuzcrawl::InteractionResult")]
-#[serde(default)]
-pub struct InteractionResult {
-    pub action_results: Vec<ActionResult>,
-    pub final_html: String,
-    pub final_url: String,
-    pub screenshot: Option<Vec<u8>>,
-}
-
-unsafe impl IntoValueFromNative for InteractionResult {}
-
-impl magnus::TryConvert for InteractionResult {
-    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
-        let r: &InteractionResult = magnus::TryConvert::try_convert(val)?;
-        Ok(r.clone())
-    }
-}
-unsafe impl TryConvertOwned for InteractionResult {}
-
-impl Default for InteractionResult {
-    fn default() -> Self {
-        Self {
-            action_results: Default::default(),
-            final_html: Default::default(),
-            final_url: Default::default(),
-            screenshot: Default::default(),
-        }
-    }
-}
-
-impl InteractionResult {
-    fn new(
-        action_results: Option<Vec<ActionResult>>,
-        final_html: Option<String>,
-        final_url: Option<String>,
-        screenshot: Option<Vec<u8>>,
-    ) -> Self {
-        Self {
-            action_results: action_results.unwrap_or_default(),
-            final_html: final_html.unwrap_or_default(),
-            final_url: final_url.unwrap_or_default(),
-            screenshot,
-        }
-    }
-
-    fn action_results(&self) -> Vec<ActionResult> {
-        self.action_results.clone()
-    }
-
-    fn final_html(&self) -> String {
-        self.final_html.clone()
-    }
-
-    fn final_url(&self) -> String {
-        self.final_url.clone()
-    }
-
-    fn screenshot(&self) -> Option<Vec<u8>> {
-        self.screenshot.clone()
-    }
-}
-
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-#[magnus::wrap(class = "Kreuzcrawl::ActionResult")]
-#[serde(default)]
-pub struct ActionResult {
-    pub action_index: usize,
-    pub action_type: String,
-    pub success: bool,
-    pub data: Option<String>,
-    pub error: Option<String>,
-}
-
-unsafe impl IntoValueFromNative for ActionResult {}
-
-impl magnus::TryConvert for ActionResult {
-    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
-        let r: &ActionResult = magnus::TryConvert::try_convert(val)?;
-        Ok(r.clone())
-    }
-}
-unsafe impl TryConvertOwned for ActionResult {}
-
-impl Default for ActionResult {
-    fn default() -> Self {
-        Self {
-            action_index: Default::default(),
-            action_type: Default::default(),
-            success: Default::default(),
-            data: Default::default(),
-            error: Default::default(),
-        }
-    }
-}
-
-impl ActionResult {
-    fn new(
-        action_index: Option<usize>,
-        action_type: Option<String>,
-        success: Option<bool>,
-        data: Option<String>,
-        error: Option<String>,
-    ) -> Self {
-        Self {
-            action_index: action_index.unwrap_or_default(),
-            action_type: action_type.unwrap_or_default(),
-            success: success.unwrap_or_default(),
-            data,
-            error,
-        }
-    }
-
-    fn action_index(&self) -> usize {
-        self.action_index
-    }
-
-    fn action_type(&self) -> String {
-        self.action_type.clone()
-    }
-
-    fn success(&self) -> bool {
-        self.success
-    }
-
-    fn data(&self) -> Option<String> {
-        self.data.clone()
-    }
-
-    fn error(&self) -> Option<String> {
-        self.error.clone()
-    }
-}
-
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
 #[magnus::wrap(class = "Kreuzcrawl::ScrapeResult")]
 #[serde(default)]
@@ -835,7 +850,6 @@ pub struct ScrapeResult {
     pub is_pdf: bool,
     pub was_skipped: bool,
     pub detected_charset: Option<String>,
-    pub main_content_only: bool,
     pub auth_header_sent: bool,
     pub response_meta: Option<ResponseMeta>,
     pub assets: Vec<DownloadedAsset>,
@@ -878,7 +892,6 @@ impl Default for ScrapeResult {
             is_pdf: Default::default(),
             was_skipped: Default::default(),
             detected_charset: Default::default(),
-            main_content_only: Default::default(),
             auth_header_sent: Default::default(),
             response_meta: Default::default(),
             assets: Default::default(),
@@ -962,10 +975,6 @@ impl ScrapeResult {
             detected_charset: kwargs
                 .get(ruby.to_symbol("detected_charset"))
                 .and_then(|v| String::try_convert(v).ok()),
-            main_content_only: kwargs
-                .get(ruby.to_symbol("main_content_only"))
-                .and_then(|v| bool::try_convert(v).ok())
-                .unwrap_or_default(),
             auth_header_sent: kwargs
                 .get(ruby.to_symbol("auth_header_sent"))
                 .and_then(|v| bool::try_convert(v).ok())
@@ -1071,10 +1080,6 @@ impl ScrapeResult {
         self.detected_charset.clone()
     }
 
-    fn main_content_only(&self) -> bool {
-        self.main_content_only
-    }
-
     fn auth_header_sent(&self) -> bool {
         self.auth_header_sent
     }
@@ -1616,93 +1621,6 @@ impl MarkdownResult {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-#[magnus::wrap(class = "Kreuzcrawl::CachedPage")]
-#[serde(default)]
-pub struct CachedPage {
-    pub url: String,
-    pub status_code: u16,
-    pub content_type: String,
-    pub body: String,
-    pub etag: Option<String>,
-    pub last_modified: Option<String>,
-    pub cached_at: u64,
-}
-
-unsafe impl IntoValueFromNative for CachedPage {}
-
-impl magnus::TryConvert for CachedPage {
-    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
-        let r: &CachedPage = magnus::TryConvert::try_convert(val)?;
-        Ok(r.clone())
-    }
-}
-unsafe impl TryConvertOwned for CachedPage {}
-
-impl Default for CachedPage {
-    fn default() -> Self {
-        Self {
-            url: Default::default(),
-            status_code: Default::default(),
-            content_type: Default::default(),
-            body: Default::default(),
-            etag: Default::default(),
-            last_modified: Default::default(),
-            cached_at: Default::default(),
-        }
-    }
-}
-
-impl CachedPage {
-    fn new(
-        url: Option<String>,
-        status_code: Option<u16>,
-        content_type: Option<String>,
-        body: Option<String>,
-        etag: Option<String>,
-        last_modified: Option<String>,
-        cached_at: Option<u64>,
-    ) -> Self {
-        Self {
-            url: url.unwrap_or_default(),
-            status_code: status_code.unwrap_or_default(),
-            content_type: content_type.unwrap_or_default(),
-            body: body.unwrap_or_default(),
-            etag,
-            last_modified,
-            cached_at: cached_at.unwrap_or_default(),
-        }
-    }
-
-    fn url(&self) -> String {
-        self.url.clone()
-    }
-
-    fn status_code(&self) -> u16 {
-        self.status_code
-    }
-
-    fn content_type(&self) -> String {
-        self.content_type.clone()
-    }
-
-    fn body(&self) -> String {
-        self.body.clone()
-    }
-
-    fn etag(&self) -> Option<String> {
-        self.etag.clone()
-    }
-
-    fn last_modified(&self) -> Option<String> {
-        self.last_modified.clone()
-    }
-
-    fn cached_at(&self) -> u64 {
-        self.cached_at
-    }
-}
-
 #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
 #[magnus::wrap(class = "Kreuzcrawl::LinkInfo")]
 #[serde(default)]
@@ -3295,39 +3213,6 @@ impl magnus::TryConvert for AssetCategory {
 unsafe impl IntoValueFromNative for AssetCategory {}
 unsafe impl TryConvertOwned for AssetCategory {}
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-pub enum CrawlEvent {
-    Page { _0: CrawlPageResult },
-    Error { url: String, error: String },
-    Complete { pages_crawled: usize },
-}
-
-impl Default for CrawlEvent {
-    fn default() -> Self {
-        Self::Page { _0: Default::default() }
-    }
-}
-
-impl magnus::IntoValue for CrawlEvent {
-    fn into_value_with(self, handle: &Ruby) -> magnus::Value {
-        match serde_json::to_value(&self) {
-            Ok(v) => json_to_ruby(handle, v),
-            Err(_) => handle.qnil().into_value_with(handle),
-        }
-    }
-}
-
-impl magnus::TryConvert for CrawlEvent {
-    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
-        let s: String = magnus::TryConvert::try_convert(val)?;
-        serde_json::from_str(&s)
-            .map_err(|e| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_type_error(), e.to_string()))
-    }
-}
-
-unsafe impl IntoValueFromNative for CrawlEvent {}
-unsafe impl TryConvertOwned for CrawlEvent {}
-
 fn create_engine(config: Option<String>) -> Result<CrawlEngineHandle, Error> {
     let config: Option<CrawlConfig> = config
         .as_deref()
@@ -3546,6 +3431,44 @@ impl From<kreuzcrawl::ProxyConfig> for ProxyConfig {
     }
 }
 
+impl From<ContentConfig> for kreuzcrawl::ContentConfig {
+    fn from(val: ContentConfig) -> Self {
+        Self {
+            output_format: val.output_format,
+            preprocessing_preset: val.preprocessing_preset,
+            remove_navigation: val.remove_navigation,
+            remove_forms: val.remove_forms,
+            strip_tags: val.strip_tags,
+            preserve_tags: val.preserve_tags,
+            exclude_selectors: val.exclude_selectors,
+            skip_images: val.skip_images,
+            max_depth: val.max_depth,
+            wrap: val.wrap,
+            wrap_width: val.wrap_width,
+            include_document_structure: val.include_document_structure,
+        }
+    }
+}
+
+impl From<kreuzcrawl::ContentConfig> for ContentConfig {
+    fn from(val: kreuzcrawl::ContentConfig) -> Self {
+        Self {
+            output_format: val.output_format,
+            preprocessing_preset: val.preprocessing_preset,
+            remove_navigation: val.remove_navigation,
+            remove_forms: val.remove_forms,
+            strip_tags: val.strip_tags,
+            preserve_tags: val.preserve_tags,
+            exclude_selectors: val.exclude_selectors,
+            skip_images: val.skip_images,
+            max_depth: val.max_depth,
+            wrap: val.wrap,
+            wrap_width: val.wrap_width,
+            include_document_structure: val.include_document_structure,
+        }
+    }
+}
+
 impl From<BrowserConfig> for kreuzcrawl::BrowserConfig {
     fn from(val: BrowserConfig) -> Self {
         Self {
@@ -3587,14 +3510,15 @@ impl From<CrawlConfig> for kreuzcrawl::CrawlConfig {
             exclude_paths: val.exclude_paths,
             custom_headers: val.custom_headers.into_iter().collect(),
             request_timeout: std::time::Duration::from_millis(val.request_timeout),
+            rate_limit_ms: val.rate_limit_ms,
             max_redirects: val.max_redirects,
             retry_count: val.retry_count,
             retry_codes: val.retry_codes,
             cookies_enabled: val.cookies_enabled,
             auth: val.auth.map(Into::into),
             max_body_size: val.max_body_size,
-            main_content_only: val.main_content_only,
             remove_tags: val.remove_tags,
+            content: val.content.into(),
             map_limit: val.map_limit,
             map_search: val.map_search,
             download_assets: val.download_assets,
@@ -3629,14 +3553,15 @@ impl From<kreuzcrawl::CrawlConfig> for CrawlConfig {
             exclude_paths: val.exclude_paths,
             custom_headers: val.custom_headers.into_iter().collect(),
             request_timeout: val.request_timeout.as_millis() as u64,
+            rate_limit_ms: val.rate_limit_ms,
             max_redirects: val.max_redirects,
             retry_count: val.retry_count,
             retry_codes: val.retry_codes,
             cookies_enabled: val.cookies_enabled,
             auth: val.auth.map(Into::into),
             max_body_size: val.max_body_size,
-            main_content_only: val.main_content_only,
             remove_tags: val.remove_tags,
+            content: val.content.into(),
             map_limit: val.map_limit,
             map_search: val.map_search,
             download_assets: val.download_assets,
@@ -3677,40 +3602,17 @@ impl From<kreuzcrawl::DownloadedDocument> for DownloadedDocument {
             mime_type: format!("{:?}", val.mime_type),
             content: val.content.to_vec(),
             size: val.size,
-            filename: val.filename.as_ref().map(|v| format!("{:?}", v)),
+            filename: val.filename.as_ref().map(|v| format!("{v:?}")),
             content_hash: format!("{:?}", val.content_hash),
             headers: val
                 .headers
                 .into_iter()
-                .map(|(k, v)| (format!("{:?}", k), format!("{:?}", v)))
+                .map(|(k, v)| (k.to_string(), v.to_string()))
                 .collect(),
         }
     }
 }
 
-impl From<kreuzcrawl::InteractionResult> for InteractionResult {
-    fn from(val: kreuzcrawl::InteractionResult) -> Self {
-        Self {
-            action_results: val.action_results.into_iter().map(Into::into).collect(),
-            final_html: val.final_html,
-            final_url: val.final_url,
-            screenshot: val.screenshot.map(|v| v.to_vec()),
-        }
-    }
-}
-
-impl From<kreuzcrawl::ActionResult> for ActionResult {
-    fn from(val: kreuzcrawl::ActionResult) -> Self {
-        Self {
-            action_index: val.action_index,
-            action_type: format!("{:?}", val.action_type),
-            success: val.success,
-            data: val.data.as_ref().map(ToString::to_string),
-            error: val.error,
-        }
-    }
-}
-
 impl From<ScrapeResult> for kreuzcrawl::ScrapeResult {
     fn from(val: ScrapeResult) -> Self {
         Self {
@@ -3731,7 +3633,6 @@ impl From<ScrapeResult> for kreuzcrawl::ScrapeResult {
             is_pdf: val.is_pdf,
             was_skipped: val.was_skipped,
             detected_charset: val.detected_charset,
-            main_content_only: val.main_content_only,
             auth_header_sent: val.auth_header_sent,
             response_meta: val.response_meta.map(Into::into),
             assets: val.assets.into_iter().map(Into::into).collect(),
@@ -3766,7 +3667,6 @@ impl From<kreuzcrawl::ScrapeResult> for ScrapeResult {
             is_pdf: val.is_pdf,
             was_skipped: val.was_skipped,
             detected_charset: val.detected_charset,
-            main_content_only: val.main_content_only,
             auth_header_sent: val.auth_header_sent,
             response_meta: val.response_meta.map(Into::into),
             assets: val.assets.into_iter().map(Into::into).collect(),
@@ -3934,20 +3834,6 @@ impl From<kreuzcrawl::MarkdownResult> for MarkdownResult {
     }
 }
 
-impl From<kreuzcrawl::CachedPage> for CachedPage {
-    fn from(val: kreuzcrawl::CachedPage) -> Self {
-        Self {
-            url: val.url,
-            status_code: val.status_code,
-            content_type: val.content_type,
-            body: val.body,
-            etag: val.etag,
-            last_modified: val.last_modified,
-            cached_at: val.cached_at,
-        }
-    }
-}
-
 impl From<LinkInfo> for kreuzcrawl::LinkInfo {
     fn from(val: LinkInfo) -> Self {
         Self {
@@ -4530,16 +4416,6 @@ impl From<kreuzcrawl::AssetCategory> for AssetCategory {
     }
 }
 
-impl From<kreuzcrawl::CrawlEvent> for CrawlEvent {
-    fn from(val: kreuzcrawl::CrawlEvent) -> Self {
-        match val {
-            kreuzcrawl::CrawlEvent::Page(_0) => Self::Page { _0: (*_0).into() },
-            kreuzcrawl::CrawlEvent::Error { url, error } => Self::Error { url, error },
-            kreuzcrawl::CrawlEvent::Complete { pages_crawled } => Self::Complete { pages_crawled },
-        }
-    }
-}
-
 /// Convert a `kreuzcrawl::CrawlError` error to a Magnus runtime error.
 #[allow(dead_code)]
 fn crawl_error_to_magnus_err(e: kreuzcrawl::CrawlError) -> magnus::Error {
@@ -4565,6 +4441,24 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("username", method!(ProxyConfig::username, 0))?;
     class.define_method("password", method!(ProxyConfig::password, 0))?;
 
+    let class = module.define_class("ContentConfig", ruby.class_object())?;
+    class.define_singleton_method("new", function!(ContentConfig::new, 12))?;
+    class.define_method("output_format", method!(ContentConfig::output_format, 0))?;
+    class.define_method("preprocessing_preset", method!(ContentConfig::preprocessing_preset, 0))?;
+    class.define_method("remove_navigation", method!(ContentConfig::remove_navigation, 0))?;
+    class.define_method("remove_forms", method!(ContentConfig::remove_forms, 0))?;
+    class.define_method("strip_tags", method!(ContentConfig::strip_tags, 0))?;
+    class.define_method("preserve_tags", method!(ContentConfig::preserve_tags, 0))?;
+    class.define_method("exclude_selectors", method!(ContentConfig::exclude_selectors, 0))?;
+    class.define_method("skip_images", method!(ContentConfig::skip_images, 0))?;
+    class.define_method("max_depth", method!(ContentConfig::max_depth, 0))?;
+    class.define_method("wrap", method!(ContentConfig::wrap, 0))?;
+    class.define_method("wrap_width", method!(ContentConfig::wrap_width, 0))?;
+    class.define_method(
+        "include_document_structure",
+        method!(ContentConfig::include_document_structure, 0),
+    )?;
+
     let class = module.define_class("BrowserConfig", ruby.class_object())?;
     class.define_singleton_method("new", function!(BrowserConfig::new, 6))?;
     class.define_method("mode", method!(BrowserConfig::mode, 0))?;
@@ -4587,14 +4481,15 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("exclude_paths", method!(CrawlConfig::exclude_paths, 0))?;
     class.define_method("custom_headers", method!(CrawlConfig::custom_headers, 0))?;
     class.define_method("request_timeout", method!(CrawlConfig::request_timeout, 0))?;
+    class.define_method("rate_limit_ms", method!(CrawlConfig::rate_limit_ms, 0))?;
     class.define_method("max_redirects", method!(CrawlConfig::max_redirects, 0))?;
     class.define_method("retry_count", method!(CrawlConfig::retry_count, 0))?;
     class.define_method("retry_codes", method!(CrawlConfig::retry_codes, 0))?;
     class.define_method("cookies_enabled", method!(CrawlConfig::cookies_enabled, 0))?;
     class.define_method("auth", method!(CrawlConfig::auth, 0))?;
     class.define_method("max_body_size", method!(CrawlConfig::max_body_size, 0))?;
-    class.define_method("main_content_only", method!(CrawlConfig::main_content_only, 0))?;
     class.define_method("remove_tags", method!(CrawlConfig::remove_tags, 0))?;
+    class.define_method("content", method!(CrawlConfig::content, 0))?;
     class.define_method("map_limit", method!(CrawlConfig::map_limit, 0))?;
     class.define_method("map_search", method!(CrawlConfig::map_search, 0))?;
     class.define_method("download_assets", method!(CrawlConfig::download_assets, 0))?;
@@ -4622,21 +4517,6 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("content_hash", method!(DownloadedDocument::content_hash, 0))?;
     class.define_method("headers", method!(DownloadedDocument::headers, 0))?;
 
-    let class = module.define_class("InteractionResult", ruby.class_object())?;
-    class.define_singleton_method("new", function!(InteractionResult::new, 4))?;
-    class.define_method("action_results", method!(InteractionResult::action_results, 0))?;
-    class.define_method("final_html", method!(InteractionResult::final_html, 0))?;
-    class.define_method("final_url", method!(InteractionResult::final_url, 0))?;
-    class.define_method("screenshot", method!(InteractionResult::screenshot, 0))?;
-
-    let class = module.define_class("ActionResult", ruby.class_object())?;
-    class.define_singleton_method("new", function!(ActionResult::new, 5))?;
-    class.define_method("action_index", method!(ActionResult::action_index, 0))?;
-    class.define_method("action_type", method!(ActionResult::action_type, 0))?;
-    class.define_method("success", method!(ActionResult::success, 0))?;
-    class.define_method("data", method!(ActionResult::data, 0))?;
-    class.define_method("error", method!(ActionResult::error, 0))?;
-
     let class = module.define_class("ScrapeResult", ruby.class_object())?;
     class.define_singleton_method("new", function!(ScrapeResult::new, 1))?;
     class.define_method("status_code", method!(ScrapeResult::status_code, 0))?;
@@ -4656,7 +4536,6 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("is_pdf", method!(ScrapeResult::is_pdf, 0))?;
     class.define_method("was_skipped", method!(ScrapeResult::was_skipped, 0))?;
     class.define_method("detected_charset", method!(ScrapeResult::detected_charset, 0))?;
-    class.define_method("main_content_only", method!(ScrapeResult::main_content_only, 0))?;
     class.define_method("auth_header_sent", method!(ScrapeResult::auth_header_sent, 0))?;
     class.define_method("response_meta", method!(ScrapeResult::response_meta, 0))?;
     class.define_method("assets", method!(ScrapeResult::assets, 0))?;
@@ -4725,16 +4604,6 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("citations", method!(MarkdownResult::citations, 0))?;
     class.define_method("fit_content", method!(MarkdownResult::fit_content, 0))?;
 
-    let class = module.define_class("CachedPage", ruby.class_object())?;
-    class.define_singleton_method("new", function!(CachedPage::new, 7))?;
-    class.define_method("url", method!(CachedPage::url, 0))?;
-    class.define_method("status_code", method!(CachedPage::status_code, 0))?;
-    class.define_method("content_type", method!(CachedPage::content_type, 0))?;
-    class.define_method("body", method!(CachedPage::body, 0))?;
-    class.define_method("etag", method!(CachedPage::etag, 0))?;
-    class.define_method("last_modified", method!(CachedPage::last_modified, 0))?;
-    class.define_method("cached_at", method!(CachedPage::cached_at, 0))?;
-
     let class = module.define_class("LinkInfo", ruby.class_object())?;
     class.define_singleton_method("new", function!(LinkInfo::new, 5))?;
     class.define_method("url", method!(LinkInfo::url, 0))?;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kreuzcrawl
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Kreuzberg Team