kreuzcrawl 0.3.0.pre.rc.71 → 0.3.0.pre.rc.72

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f9dc9a2b925bd356635ef537594ccf72049f8bbfa47754f348fe0643443a4c9f
-  data.tar.gz: 06b86af2e1ef230a61a1c9c81d84115e2346e558de94df0d17c08097e63c84e1
+  metadata.gz: 8edea9a39a5cbe88bec892ae7fc8de37f0a4b4d5dc3430f5a6e4c0ccb475e443
+  data.tar.gz: 3dd64e9b4d361cc4d3ec82178260aaa018b0ca1d1d2f76034cb20ed05c23cefd
 SHA512:
-  metadata.gz: 490013779c30677fd6f3618275bb0bd9f11f0da30c43b320e2fda3b814762d18fac68391c9b6e91c9b9f80a5f3cc4ee6a008d48e4897b6dd79c314af085e5d5b
-  data.tar.gz: 6c8b027fc5db3dd04aa20b034a09e80a4041d527e688131a0bdfbc2f87a9390bef47adc14db5e096a195d69ddcc45ad17d68f43dbb91846ce09fba8919d6d5c0
+  metadata.gz: 56ac6e332c9e4cba97c1adb271a6e36dd1fb0c8eb1b9549950d7048c208e9ecf2f9bdb8529dd326d2b5bece9fa2d56f961e8199b2c5667167ee5965ffa3f7596
+  data.tar.gz: b84490766d8ade6a489ba72ee7711c25b6e16e75861073f9b3c4346c8d0da4ecdca2321c9f590860070260ab9ae144429e4837ae9fced893b0983feee3718dd0

data/README.md

@@ -141,12 +141,13 @@ Contributions are welcome! Please see our [Contributing Guide](https://github.co
 
 ## Part of Kreuzberg.dev
 
-- [Kreuzberg](https://github.com/kreuzberg-dev/kreuzberg) — document intelligence: text, tables, metadata from 90+ formats with optional OCR.
+- [Kreuzberg](https://github.com/kreuzberg-dev/kreuzberg) — document intelligence: text, tables, metadata from 91+ formats with optional OCR.
 - [Kreuzberg Cloud](https://github.com/kreuzberg-dev/kreuzberg-cloud) — managed extraction API with SDKs, dashboards, and observability.
+- [kreuzcrawl](https://github.com/kreuzberg-dev/kreuzcrawl) — web crawling and scraping with HTML→Markdown and headless-Chrome fallback.
 - [html-to-markdown](https://github.com/kreuzberg-dev/html-to-markdown) — fast, lossless HTML→Markdown engine.
 - [liter-llm](https://github.com/kreuzberg-dev/liter-llm) — universal LLM API client with native bindings for 14 languages and 143 providers.
 - [tree-sitter-language-pack](https://github.com/kreuzberg-dev/tree-sitter-language-pack) — tree-sitter grammars and code-intelligence primitives.
-- [alef](https://github.com/kreuzberg-dev/alef) — the polyglot binding generator that produces this README and all per-language bindings.
+- [alef](https://github.com/kreuzberg-dev/alef) — the polyglot binding generator that produces every per-language binding across the 5 polyglot repos.
 - [Discord](https://discord.gg/xt9WY3GnKR) — community, roadmap, announcements.
 
 ## License
@@ -158,4 +159,3 @@ This project is licensed under [Elastic License 2.0](https://github.com/kreuzber
 - [Documentation](https://docs.kreuzcrawl.kreuzberg.dev)
 - [GitHub Repository](https://github.com/kreuzberg-dev/kreuzcrawl)
 - [Issue Tracker](https://github.com/kreuzberg-dev/kreuzcrawl/issues)
-- [Issues](https://github.com/kreuzberg-dev/kreuzcrawl/issues)

data/ext/kreuzcrawl_rb/native/Cargo.lock

@@ -1460,9 +1460,9 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl"
-version = "0.3.0-rc.71"
+version = "0.3.0-rc.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e8c85e79454c048ec41289dfb0168e87ff1c5a0a28997de8cc0dbd3ab331c8e2"
+checksum = "8a83f0115e5fce77f5f6a4c53d572b8c151038c0fcd16c641db2e8b499603582"
 dependencies = [
  "ahash",
  "aho-corasick",
@@ -1502,7 +1502,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzcrawl-rb"
-version = "0.3.0-rc.71"
+version = "0.3.0-rc.72"
 dependencies = [
  "futures",
  "kreuzcrawl",

data/ext/kreuzcrawl_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzcrawl-rb"
-version = "0.3.0-rc.71"
+version = "0.3.0-rc.72"
 edition = "2024"
 license = "Elastic-2.0"
 description = "High-performance web crawling engine"
@@ -18,7 +18,7 @@ crate-type = ["cdylib"]
 
 [dependencies]
 futures = "0.3"
-kreuzcrawl = { version = "0.3.0-rc.71", features = ["interact", "browser-chromiumoxide"] }
+kreuzcrawl = { version = "0.3.0-rc.72", features = ["interact", "browser-chromiumoxide"] }
 magnus = "0.8"
 rb-sys = ">=0.9, <0.9.128"
 serde = { version = "1", features = ["derive"] }

data/ext/kreuzcrawl_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:f2acf0fe68bedde7da1909fa6feedaffb995d5aec49220ec23bf64125a392d19
+// alef:hash:bcc3d5bd544a6e7593876ab5de51eb8c1f53d0d8d2c8c3d31c436d54ead6b3ec
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -5446,8 +5446,6 @@ impl From<CrawlConfig> for kreuzcrawl::CrawlConfig {
             warc_output: val.warc_output.map(Into::into),
             browser_profile: val.browser_profile,
             save_browser_profile: val.save_browser_profile,
-            ssrf: Default::default(),
-            dispatch: Default::default(),
             ..Default::default()
         }
     }
@@ -5525,17 +5523,18 @@ impl From<kreuzcrawl::BrowserExtras> for BrowserExtras {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<DownloadedDocument> for kreuzcrawl::DownloadedDocument {
     fn from(val: DownloadedDocument) -> Self {
         Self {
             url: val.url,
             mime_type: val.mime_type.into(),
-            content: Default::default(),
             size: val.size,
             filename: val.filename.map(Into::into),
             content_hash: val.content_hash.into(),
             headers: val.headers.into_iter().map(|(k, v)| (k.into(), v.into())).collect(),
+            ..Default::default()
         }
     }
 }
@@ -5554,6 +5553,7 @@ impl From<kreuzcrawl::DownloadedDocument> for DownloadedDocument {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<InteractionResult> for kreuzcrawl::InteractionResult {
     fn from(val: InteractionResult) -> Self {
@@ -5561,7 +5561,7 @@ impl From<InteractionResult> for kreuzcrawl::InteractionResult {
             action_results: val.action_results.into_iter().map(Into::into).collect(),
             final_html: val.final_html,
             final_url: val.final_url,
-            screenshot: Default::default(),
+            ..Default::default()
         }
     }
 }
@@ -5603,6 +5603,7 @@ impl From<kreuzcrawl::ActionResult> for ActionResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<ScrapeResult> for kreuzcrawl::ScrapeResult {
     fn from(val: ScrapeResult) -> Self {
@@ -5633,9 +5634,9 @@ impl From<ScrapeResult> for kreuzcrawl::ScrapeResult {
             markdown: val.markdown.map(Into::into),
             extracted_data: val.extracted_data.as_ref().and_then(|s| serde_json::from_str(s).ok()),
             extraction_meta: val.extraction_meta.map(Into::into),
-            screenshot: Default::default(),
             downloaded_document: val.downloaded_document.map(Into::into),
             browser: val.browser.map(Into::into),
+            ..Default::default()
         }
     }
 }
@@ -5734,6 +5735,7 @@ impl From<kreuzcrawl::CrawlPageResult> for CrawlPageResult {
     }
 }
 
+#[allow(clippy::needless_update)]
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<CrawlResult> for kreuzcrawl::CrawlResult {
     fn from(val: CrawlResult) -> Self {
@@ -5746,7 +5748,7 @@ impl From<CrawlResult> for kreuzcrawl::CrawlResult {
             cookies: val.cookies.into_iter().map(Into::into).collect(),
             stayed_on_domain: val.stayed_on_domain,
             browser_used: val.browser_used,
-            normalized_urls: Default::default(),
+            ..Default::default()
         }
     }
 }

data/lib/kreuzcrawl/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:f2acf0fe68bedde7da1909fa6feedaffb995d5aec49220ec23bf64125a392d19
+# alef:hash:bcc3d5bd544a6e7593876ab5de51eb8c1f53d0d8d2c8c3d31c436d54ead6b3ec
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
@@ -106,10 +106,7 @@ module Kreuzcrawl
   # Not available on `wasm32` targets — streaming requires native concurrency
   # primitives (tokio channels, `JoinSet`) that are not supported on wasm32.
   #
-  # Delivered to bindings via alef's streaming-adapter pattern. The
-  # `crawl_stream` / `batch_crawl_stream` binding wrappers in `bindings.rs`
-  # expose this as the per-language streaming idiom (Python `AsyncIterator`,
-  # Ruby `Enumerator`, PHP `Generator`, Elixir `Stream.unfold`, etc.).
+  # Delivered to bindings through each target's native streaming idiom.
   module CrawlEvent
     extend T::Helpers
     extend T::Sig

data/lib/kreuzcrawl/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:f2acf0fe68bedde7da1909fa6feedaffb995d5aec49220ec23bf64125a392d19
+# alef:hash:bcc3d5bd544a6e7593876ab5de51eb8c1f53d0d8d2c8c3d31c436d54ead6b3ec
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
 
 module Kreuzcrawl
   ## The version string for this package.
-  VERSION = "0.3.0.pre.rc.71"
+  VERSION = "0.3.0.pre.rc.72"
 end

data/lib/kreuzcrawl.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:f2acf0fe68bedde7da1909fa6feedaffb995d5aec49220ec23bf64125a392d19
+# alef:hash:bcc3d5bd544a6e7593876ab5de51eb8c1f53d0d8d2c8c3d31c436d54ead6b3ec
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:f2acf0fe68bedde7da1909fa6feedaffb995d5aec49220ec23bf64125a392d19
+# alef:hash:bcc3d5bd544a6e7593876ab5de51eb8c1f53d0d8d2c8c3d31c436d54ead6b3ec
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: kreuzcrawl
 version: !ruby/object:Gem::Version
-  version: 0.3.0.pre.rc.71
+  version: 0.3.0.pre.rc.72
 platform: ruby
 authors:
 - Kreuzberg Team