kreuzcrawl 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 922ae77455c2e0e37df7ddb5a4084ad765187c947427092346d577e577d25be1
-  data.tar.gz: e1e537da083dea650de05a86713fe0b93134751345b7bb9d47ee3760379774c4
+  metadata.gz: a2f4027aef59737e93add85d5da2398c4a95767fb41766b86822f50199331e32
+  data.tar.gz: 82f38cf86e988321ef9bc11f5a42a023df464e79d0d0ad646edeae564cabfec6
 SHA512:
-  metadata.gz: 5499c3684018e4bb69febc2e3c939b9f388c2976636f6dce443866108d898759895e5a0567b8bf9fdfbadf1c35dda8f54cbf73ec2ddfb1f20c39dac7a5a0efd6
-  data.tar.gz: c82c4450e90a1a47216f9240f181791a7fa37089c7d776709bcd0c031949a0043c9045c0919f04ac1f8672f46c9bcb811daed5f01b5e0bde07e8e2f87edcf020
+  metadata.gz: 1ebcebff06f45e809441c0cae1d9f14be48a79b6d1439b3421ebedd4808d9c71e6ffa86d51a6c1d64ba57a0bc2eeffa24a672a4af3db5659185ba5c8cca8443a
+  data.tar.gz: af29b39fdf985044e579d71064af9eb10dc348c81a82e7f27dc0650c4720ee1164fd8fe5529358cc9ba494b0fc296e9a425b17182794d001b3bcee526f737d35

data/ext/kreuzcrawl_rb/Cargo.lock

@@ -1174,7 +1174,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl"
-version = "0.1.0-rc.5"
+version = "0.1.1"
 dependencies = [
  "ahash",
  "astral-tl",
@@ -1200,7 +1200,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl-rb"
-version = "0.1.0-rc.5"
+version = "0.1.1"
 dependencies = [
  "kreuzcrawl",
  "magnus",

data/ext/kreuzcrawl_rb/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzcrawl-rb"
-version = "0.1.1"
+version = "0.1.2"
 edition = "2024"
 license = "Elastic-2.0"
 

data/ext/kreuzcrawl_rb/src/kreuzcrawl/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Kreuzcrawl
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/ext/kreuzcrawl_rb/src/lib.rs

@@ -1,6 +1,19 @@
 // This file is auto-generated by alef. DO NOT EDIT.
 // Re-generate with: alef generate
 #![allow(dead_code)]
+#![allow(
+    clippy::too_many_arguments,
+    clippy::let_unit_value,
+    clippy::needless_borrow,
+    clippy::map_identity,
+    clippy::just_underscores_and_digits,
+    clippy::unused_unit,
+    clippy::unnecessary_cast,
+    clippy::unwrap_or_default,
+    clippy::derivable_impls,
+    clippy::needless_borrows_for_generic_args,
+    clippy::unnecessary_fallible_conversions
+)]
 
 use magnus::{Error, IntoValueFromNative, Ruby, function, method, prelude::*, try_convert::TryConvertOwned};
 use std::collections::HashMap;
@@ -243,6 +256,7 @@ pub struct CrawlConfig {
     pub exclude_paths: Vec<String>,
     pub custom_headers: HashMap<String, String>,
     pub request_timeout: u64,
+    pub rate_limit_ms: Option<u64>,
     pub max_redirects: usize,
     pub retry_count: usize,
     pub retry_codes: Vec<u16>,
@@ -322,6 +336,9 @@ impl CrawlConfig {
                 .get(ruby.to_symbol("request_timeout"))
                 .and_then(|v| u64::try_convert(v).ok())
                 .unwrap_or(30000),
+            rate_limit_ms: kwargs
+                .get(ruby.to_symbol("rate_limit_ms"))
+                .and_then(|v| u64::try_convert(v).ok()),
             max_redirects: kwargs
                 .get(ruby.to_symbol("max_redirects"))
                 .and_then(|v| usize::try_convert(v).ok())
@@ -452,6 +469,10 @@ impl CrawlConfig {
         self.request_timeout.clone()
     }
 
+    fn rate_limit_ms(&self) -> Option<u64> {
+        self.rate_limit_ms
+    }
+
     fn max_redirects(&self) -> usize {
         self.max_redirects
     }
@@ -558,6 +579,7 @@ impl CrawlConfig {
             exclude_paths: self.exclude_paths.clone(),
             custom_headers: self.custom_headers.clone().into_iter().collect(),
             request_timeout: std::time::Duration::from_millis(self.request_timeout),
+            rate_limit_ms: self.rate_limit_ms,
             max_redirects: self.max_redirects,
             retry_count: self.retry_count,
             retry_codes: self.retry_codes.clone(),
@@ -680,140 +702,6 @@ impl DownloadedDocument {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-#[magnus::wrap(class = "Kreuzcrawl::InteractionResult")]
-#[serde(default)]
-pub struct InteractionResult {
-    pub action_results: Vec<ActionResult>,
-    pub final_html: String,
-    pub final_url: String,
-    pub screenshot: Option<Vec<u8>>,
-}
-
-unsafe impl IntoValueFromNative for InteractionResult {}
-
-impl magnus::TryConvert for InteractionResult {
-    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
-        let r: &InteractionResult = magnus::TryConvert::try_convert(val)?;
-        Ok(r.clone())
-    }
-}
-unsafe impl TryConvertOwned for InteractionResult {}
-
-impl Default for InteractionResult {
-    fn default() -> Self {
-        Self {
-            action_results: Default::default(),
-            final_html: Default::default(),
-            final_url: Default::default(),
-            screenshot: Default::default(),
-        }
-    }
-}
-
-impl InteractionResult {
-    fn new(
-        action_results: Option<Vec<ActionResult>>,
-        final_html: Option<String>,
-        final_url: Option<String>,
-        screenshot: Option<Vec<u8>>,
-    ) -> Self {
-        Self {
-            action_results: action_results.unwrap_or_default(),
-            final_html: final_html.unwrap_or_default(),
-            final_url: final_url.unwrap_or_default(),
-            screenshot,
-        }
-    }
-
-    fn action_results(&self) -> Vec<ActionResult> {
-        self.action_results.clone()
-    }
-
-    fn final_html(&self) -> String {
-        self.final_html.clone()
-    }
-
-    fn final_url(&self) -> String {
-        self.final_url.clone()
-    }
-
-    fn screenshot(&self) -> Option<Vec<u8>> {
-        self.screenshot.clone()
-    }
-}
-
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-#[magnus::wrap(class = "Kreuzcrawl::ActionResult")]
-#[serde(default)]
-pub struct ActionResult {
-    pub action_index: usize,
-    pub action_type: String,
-    pub success: bool,
-    pub data: Option<String>,
-    pub error: Option<String>,
-}
-
-unsafe impl IntoValueFromNative for ActionResult {}
-
-impl magnus::TryConvert for ActionResult {
-    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
-        let r: &ActionResult = magnus::TryConvert::try_convert(val)?;
-        Ok(r.clone())
-    }
-}
-unsafe impl TryConvertOwned for ActionResult {}
-
-impl Default for ActionResult {
-    fn default() -> Self {
-        Self {
-            action_index: Default::default(),
-            action_type: Default::default(),
-            success: Default::default(),
-            data: Default::default(),
-            error: Default::default(),
-        }
-    }
-}
-
-impl ActionResult {
-    fn new(
-        action_index: Option<usize>,
-        action_type: Option<String>,
-        success: Option<bool>,
-        data: Option<String>,
-        error: Option<String>,
-    ) -> Self {
-        Self {
-            action_index: action_index.unwrap_or_default(),
-            action_type: action_type.unwrap_or_default(),
-            success: success.unwrap_or_default(),
-            data,
-            error,
-        }
-    }
-
-    fn action_index(&self) -> usize {
-        self.action_index
-    }
-
-    fn action_type(&self) -> String {
-        self.action_type.clone()
-    }
-
-    fn success(&self) -> bool {
-        self.success
-    }
-
-    fn data(&self) -> Option<String> {
-        self.data.clone()
-    }
-
-    fn error(&self) -> Option<String> {
-        self.error.clone()
-    }
-}
-
 #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
 #[magnus::wrap(class = "Kreuzcrawl::ScrapeResult")]
 #[serde(default)]
@@ -1616,93 +1504,6 @@ impl MarkdownResult {
     }
 }
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-#[magnus::wrap(class = "Kreuzcrawl::CachedPage")]
-#[serde(default)]
-pub struct CachedPage {
-    pub url: String,
-    pub status_code: u16,
-    pub content_type: String,
-    pub body: String,
-    pub etag: Option<String>,
-    pub last_modified: Option<String>,
-    pub cached_at: u64,
-}
-
-unsafe impl IntoValueFromNative for CachedPage {}
-
-impl magnus::TryConvert for CachedPage {
-    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
-        let r: &CachedPage = magnus::TryConvert::try_convert(val)?;
-        Ok(r.clone())
-    }
-}
-unsafe impl TryConvertOwned for CachedPage {}
-
-impl Default for CachedPage {
-    fn default() -> Self {
-        Self {
-            url: Default::default(),
-            status_code: Default::default(),
-            content_type: Default::default(),
-            body: Default::default(),
-            etag: Default::default(),
-            last_modified: Default::default(),
-            cached_at: Default::default(),
-        }
-    }
-}
-
-impl CachedPage {
-    fn new(
-        url: Option<String>,
-        status_code: Option<u16>,
-        content_type: Option<String>,
-        body: Option<String>,
-        etag: Option<String>,
-        last_modified: Option<String>,
-        cached_at: Option<u64>,
-    ) -> Self {
-        Self {
-            url: url.unwrap_or_default(),
-            status_code: status_code.unwrap_or_default(),
-            content_type: content_type.unwrap_or_default(),
-            body: body.unwrap_or_default(),
-            etag,
-            last_modified,
-            cached_at: cached_at.unwrap_or_default(),
-        }
-    }
-
-    fn url(&self) -> String {
-        self.url.clone()
-    }
-
-    fn status_code(&self) -> u16 {
-        self.status_code
-    }
-
-    fn content_type(&self) -> String {
-        self.content_type.clone()
-    }
-
-    fn body(&self) -> String {
-        self.body.clone()
-    }
-
-    fn etag(&self) -> Option<String> {
-        self.etag.clone()
-    }
-
-    fn last_modified(&self) -> Option<String> {
-        self.last_modified.clone()
-    }
-
-    fn cached_at(&self) -> u64 {
-        self.cached_at
-    }
-}
-
 #[derive(Clone, Debug, Default, serde::Serialize, serde::Deserialize)]
 #[magnus::wrap(class = "Kreuzcrawl::LinkInfo")]
 #[serde(default)]
@@ -3295,39 +3096,6 @@ impl magnus::TryConvert for AssetCategory {
 unsafe impl IntoValueFromNative for AssetCategory {}
 unsafe impl TryConvertOwned for AssetCategory {}
 
-#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)]
-pub enum CrawlEvent {
-    Page { _0: CrawlPageResult },
-    Error { url: String, error: String },
-    Complete { pages_crawled: usize },
-}
-
-impl Default for CrawlEvent {
-    fn default() -> Self {
-        Self::Page { _0: Default::default() }
-    }
-}
-
-impl magnus::IntoValue for CrawlEvent {
-    fn into_value_with(self, handle: &Ruby) -> magnus::Value {
-        match serde_json::to_value(&self) {
-            Ok(v) => json_to_ruby(handle, v),
-            Err(_) => handle.qnil().into_value_with(handle),
-        }
-    }
-}
-
-impl magnus::TryConvert for CrawlEvent {
-    fn try_convert(val: magnus::Value) -> Result<Self, magnus::Error> {
-        let s: String = magnus::TryConvert::try_convert(val)?;
-        serde_json::from_str(&s)
-            .map_err(|e| magnus::Error::new(unsafe { Ruby::get_unchecked() }.exception_type_error(), e.to_string()))
-    }
-}
-
-unsafe impl IntoValueFromNative for CrawlEvent {}
-unsafe impl TryConvertOwned for CrawlEvent {}
-
 fn create_engine(config: Option<String>) -> Result<CrawlEngineHandle, Error> {
     let config: Option<CrawlConfig> = config
         .as_deref()
@@ -3587,6 +3355,7 @@ impl From<CrawlConfig> for kreuzcrawl::CrawlConfig {
             exclude_paths: val.exclude_paths,
             custom_headers: val.custom_headers.into_iter().collect(),
             request_timeout: std::time::Duration::from_millis(val.request_timeout),
+            rate_limit_ms: val.rate_limit_ms,
             max_redirects: val.max_redirects,
             retry_count: val.retry_count,
             retry_codes: val.retry_codes,
@@ -3629,6 +3398,7 @@ impl From<kreuzcrawl::CrawlConfig> for CrawlConfig {
             exclude_paths: val.exclude_paths,
             custom_headers: val.custom_headers.into_iter().collect(),
             request_timeout: val.request_timeout.as_millis() as u64,
+            rate_limit_ms: val.rate_limit_ms,
             max_redirects: val.max_redirects,
             retry_count: val.retry_count,
             retry_codes: val.retry_codes,
@@ -3677,40 +3447,17 @@ impl From<kreuzcrawl::DownloadedDocument> for DownloadedDocument {
             mime_type: format!("{:?}", val.mime_type),
             content: val.content.to_vec(),
             size: val.size,
-            filename: val.filename.as_ref().map(|v| format!("{:?}", v)),
+            filename: val.filename.as_ref().map(|v| format!("{v:?}")),
             content_hash: format!("{:?}", val.content_hash),
             headers: val
                 .headers
                 .into_iter()
-                .map(|(k, v)| (format!("{:?}", k), format!("{:?}", v)))
+                .map(|(k, v)| (k.to_string(), v.to_string()))
                 .collect(),
         }
     }
 }
 
-impl From<kreuzcrawl::InteractionResult> for InteractionResult {
-    fn from(val: kreuzcrawl::InteractionResult) -> Self {
-        Self {
-            action_results: val.action_results.into_iter().map(Into::into).collect(),
-            final_html: val.final_html,
-            final_url: val.final_url,
-            screenshot: val.screenshot.map(|v| v.to_vec()),
-        }
-    }
-}
-
-impl From<kreuzcrawl::ActionResult> for ActionResult {
-    fn from(val: kreuzcrawl::ActionResult) -> Self {
-        Self {
-            action_index: val.action_index,
-            action_type: format!("{:?}", val.action_type),
-            success: val.success,
-            data: val.data.as_ref().map(ToString::to_string),
-            error: val.error,
-        }
-    }
-}
-
 impl From<ScrapeResult> for kreuzcrawl::ScrapeResult {
     fn from(val: ScrapeResult) -> Self {
         Self {
@@ -3934,20 +3681,6 @@ impl From<kreuzcrawl::MarkdownResult> for MarkdownResult {
     }
 }
 
-impl From<kreuzcrawl::CachedPage> for CachedPage {
-    fn from(val: kreuzcrawl::CachedPage) -> Self {
-        Self {
-            url: val.url,
-            status_code: val.status_code,
-            content_type: val.content_type,
-            body: val.body,
-            etag: val.etag,
-            last_modified: val.last_modified,
-            cached_at: val.cached_at,
-        }
-    }
-}
-
 impl From<LinkInfo> for kreuzcrawl::LinkInfo {
     fn from(val: LinkInfo) -> Self {
         Self {
@@ -4530,16 +4263,6 @@ impl From<kreuzcrawl::AssetCategory> for AssetCategory {
     }
 }
 
-impl From<kreuzcrawl::CrawlEvent> for CrawlEvent {
-    fn from(val: kreuzcrawl::CrawlEvent) -> Self {
-        match val {
-            kreuzcrawl::CrawlEvent::Page(_0) => Self::Page { _0: (*_0).into() },
-            kreuzcrawl::CrawlEvent::Error { url, error } => Self::Error { url, error },
-            kreuzcrawl::CrawlEvent::Complete { pages_crawled } => Self::Complete { pages_crawled },
-        }
-    }
-}
-
 /// Convert a `kreuzcrawl::CrawlError` error to a Magnus runtime error.
 #[allow(dead_code)]
 fn crawl_error_to_magnus_err(e: kreuzcrawl::CrawlError) -> magnus::Error {
@@ -4587,6 +4310,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("exclude_paths", method!(CrawlConfig::exclude_paths, 0))?;
     class.define_method("custom_headers", method!(CrawlConfig::custom_headers, 0))?;
     class.define_method("request_timeout", method!(CrawlConfig::request_timeout, 0))?;
+    class.define_method("rate_limit_ms", method!(CrawlConfig::rate_limit_ms, 0))?;
     class.define_method("max_redirects", method!(CrawlConfig::max_redirects, 0))?;
     class.define_method("retry_count", method!(CrawlConfig::retry_count, 0))?;
     class.define_method("retry_codes", method!(CrawlConfig::retry_codes, 0))?;
@@ -4622,21 +4346,6 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("content_hash", method!(DownloadedDocument::content_hash, 0))?;
     class.define_method("headers", method!(DownloadedDocument::headers, 0))?;
 
-    let class = module.define_class("InteractionResult", ruby.class_object())?;
-    class.define_singleton_method("new", function!(InteractionResult::new, 4))?;
-    class.define_method("action_results", method!(InteractionResult::action_results, 0))?;
-    class.define_method("final_html", method!(InteractionResult::final_html, 0))?;
-    class.define_method("final_url", method!(InteractionResult::final_url, 0))?;
-    class.define_method("screenshot", method!(InteractionResult::screenshot, 0))?;
-
-    let class = module.define_class("ActionResult", ruby.class_object())?;
-    class.define_singleton_method("new", function!(ActionResult::new, 5))?;
-    class.define_method("action_index", method!(ActionResult::action_index, 0))?;
-    class.define_method("action_type", method!(ActionResult::action_type, 0))?;
-    class.define_method("success", method!(ActionResult::success, 0))?;
-    class.define_method("data", method!(ActionResult::data, 0))?;
-    class.define_method("error", method!(ActionResult::error, 0))?;
-
     let class = module.define_class("ScrapeResult", ruby.class_object())?;
     class.define_singleton_method("new", function!(ScrapeResult::new, 1))?;
     class.define_method("status_code", method!(ScrapeResult::status_code, 0))?;
@@ -4725,16 +4434,6 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     class.define_method("citations", method!(MarkdownResult::citations, 0))?;
     class.define_method("fit_content", method!(MarkdownResult::fit_content, 0))?;
 
-    let class = module.define_class("CachedPage", ruby.class_object())?;
-    class.define_singleton_method("new", function!(CachedPage::new, 7))?;
-    class.define_method("url", method!(CachedPage::url, 0))?;
-    class.define_method("status_code", method!(CachedPage::status_code, 0))?;
-    class.define_method("content_type", method!(CachedPage::content_type, 0))?;
-    class.define_method("body", method!(CachedPage::body, 0))?;
-    class.define_method("etag", method!(CachedPage::etag, 0))?;
-    class.define_method("last_modified", method!(CachedPage::last_modified, 0))?;
-    class.define_method("cached_at", method!(CachedPage::cached_at, 0))?;
-
     let class = module.define_class("LinkInfo", ruby.class_object())?;
     class.define_singleton_method("new", function!(LinkInfo::new, 5))?;
     class.define_method("url", method!(LinkInfo::url, 0))?;

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kreuzcrawl
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Kreuzberg Team