kreuzberg 4.2.6 → 4.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7d0b60c586fdfacb40e14d02a0f21f0d8844e347095a7d566a310ca5d3600e82
-  data.tar.gz: 5319a67e46fa8422b2f37b82b06329dfb4322c4afa0182a5ca434f3068cde531
+  metadata.gz: 2f7a9518bec5df1adf4b81c08136aff44f7ab6b43cd85d7188625a3224e3a0e9
+  data.tar.gz: bd5b5a05b0612d6b66d59d458fd7b0366b0256f290e0158fa86002a64c016429
 SHA512:
-  metadata.gz: 4670ecee61b6ba3f0e13979587a746f3a57e7910408984f73e504258bd1a8ef053421a3e1c6ea99d86cd79f8dae899dc64c13b0ff5e997a2eda2fff2a08ba179
-  data.tar.gz: dd750e605c25e6db09093ea50c9ab44aa1b7c7fd8141ad9bb6c691edabbba553e455a358809bfafbab1a450f6c22127733eb63c5dff4737385dd3f6d200848fb
+  metadata.gz: dec54496350fe7c4ba61171ebb2c145bc7c2612fde428a395c5aa279bd25fcfedf21ddaeb9012f80db78717315200f2d7f4529119f217fc00e772e066a5de6e0
+  data.tar.gz: 68ae3d48391637936e8edd6b16947790bf4bf1da00a1c1efcf23b88da0dec4876d749a358327ea5e5f93e8a8f898a9a34d891bb6ab4311f46fec202de3b2e57a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kreuzberg (4.2.6)
+    kreuzberg (4.2.7)
 
 GEM
   remote: https://rubygems.org/
@@ -80,8 +80,9 @@ GEM
       ffi (~> 1.0)
     rb_sys (0.9.119)
       rake-compiler-dock (= 1.10.0)
-    rbs (3.10.2)
+    rbs (3.10.3)
       logger
+      tsort
     regexp_parser (2.11.3)
     reline (0.6.3)
       io-console (~> 0.5)
@@ -142,6 +143,7 @@ GEM
     strscan (3.1.7)
     terminal-table (4.0.0)
       unicode-display_width (>= 1.1.1, < 4)
+    tsort (0.2.0)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
     unicode-display_width (3.2.0)
@@ -207,7 +209,7 @@ CHECKSUMS
   i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
   json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
-  kreuzberg (4.2.6)
+  kreuzberg (4.2.7)
   language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
   lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
   listen (3.10.0) sha256=c6e182db62143aeccc2e1960033bebe7445309c7272061979bb098d03760c9d2
@@ -228,7 +230,7 @@ CHECKSUMS
   rb-fsevent (0.11.2) sha256=43900b972e7301d6570f64b850a5aa67833ee7d87b458ee92805d56b7318aefe
   rb-inotify (0.11.1) sha256=a0a700441239b0ff18eb65e3866236cd78613d6b9f78fea1f9ac47a85e47be6e
   rb_sys (0.9.119) sha256=64393fa148e402e1b79b64496d2aabfc7df79da6b822b8bb48dc1141eaf40b4b
-  rbs (3.10.2) sha256=bd8a5dc4c62f229f020146b61844a31f9c79e649449d212904a474eb79c846fc
+  rbs (3.10.3) sha256=70627f3919016134d554e6c99195552ae3ef6020fe034c8e983facc9c192daa6
   regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4
   reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
   rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587
@@ -246,6 +248,7 @@ CHECKSUMS
   steep (1.10.0) sha256=1b295b55f9aaff1b8d3ee42453ee55bc2a1078fda0268f288edb2dc014f4d7d1
   strscan (3.1.7) sha256=5f76462b94a3ea50b44973225b7d75b2cb96d4e1bee9ef1319b99ca117b72c8c
   terminal-table (4.0.0) sha256=f504793203f8251b2ea7c7068333053f0beeea26093ec9962e62ea79f94301d2
+  tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f
   tzinfo (2.0.6) sha256=8daf828cc77bcf7d63b0e3bdb6caa47e2272dcfaf4fbfe46f8c3a9df087a829b
   unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
   unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f

data/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.2.6" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.2.7" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/ext/kreuzberg_rb/native/Cargo.lock

@@ -716,9 +716,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.54"
+version = "1.2.55"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6354c81bbfd62d9cfa9cb3c773c2b7b2a3a482d569de977fd0e961f6e7c00583"
+checksum = "47b26a0954ae34af09b50f0de26458fa95369a0d478d8236d3f93082b219bd29"
 dependencies = [
  "find-msvc-tools",
  "jobserver",
@@ -1631,9 +1631,9 @@ dependencies = [
 
 [[package]]
 name = "find-msvc-tools"
-version = "0.1.8"
+version = "0.1.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8591b0bcc8a98a64310a2fae1bb3e9b8564dd10e381e6e28010fde8e8e8568db"
+checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582"
 
 [[package]]
 name = "flate2"
@@ -2027,9 +2027,9 @@ dependencies = [
 
 [[package]]
 name = "html-to-markdown-rs"
-version = "2.23.4"
+version = "2.24.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffbf49450676163bdf69fac2acf72674fcf2d2aaf690aa247368c567cc9afb2a"
+checksum = "51e190e3bcf14728f08547b10ba3afdae24c84299045e0831bb9ee1e199d54ad"
 dependencies = [
  "astral-tl",
  "base64 0.22.1",
@@ -2627,7 +2627,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg"
-version = "4.2.0"
+version = "4.2.6"
 dependencies = [
  "ahash",
  "async-trait",
@@ -2636,6 +2636,7 @@ dependencies = [
  "base64-simd",
  "biblatex",
  "bitvec",
+ "bytes",
  "calamine",
  "chardetng",
  "dashmap",
@@ -2703,6 +2704,7 @@ dependencies = [
  "typst-syntax",
  "unicode-normalization",
  "ureq 3.1.4",
+ "utoipa",
  "uuid",
  "whatlang",
  "yake-rust",
@@ -2711,8 +2713,9 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg-ffi"
-version = "4.2.0"
+version = "4.2.6"
 dependencies = [
+ "ahash",
  "async-trait",
  "cbindgen",
  "html-to-markdown-rs",
@@ -2768,7 +2771,7 @@ dependencies = [
 
 [[package]]
 name = "kreuzberg-tesseract"
-version = "4.2.0"
+version = "4.2.6"
 dependencies = [
  "cc",
  "cmake",
@@ -6704,6 +6707,30 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
 
+[[package]]
+name = "utoipa"
+version = "5.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fcc29c80c21c31608227e0912b2d7fddba57ad76b606890627ba8ee7964e993"
+dependencies = [
+ "indexmap",
+ "serde",
+ "serde_json",
+ "utoipa-gen",
+]
+
+[[package]]
+name = "utoipa-gen"
+version = "5.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d79d08d92ab8af4c5e8a6da20c47ae3f61a0f1dabc1997cdf2d082b757ca08b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "regex",
+ "syn",
+]
+
 [[package]]
 name = "uuid"
 version = "1.20.0"

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -1,5 +1,37 @@
 [workspace]
 
+[workspace.dependencies]
+bytes = { version = "1", features = ["serde"] }
+serde = { version = "1.0.228", features = ["derive"] }
+serde_json = { version = "1.0.149" }
+tokio = { version = "1.49.0", features = [
+    "rt",
+    "rt-multi-thread",
+    "macros",
+    "sync",
+    "process",
+    "fs",
+    "time",
+    "io-util",
+] }
+thiserror = "2.0.18"
+anyhow = "1.0"
+libc = "0.2.180"
+async-trait = "0.1.89"
+tracing = "0.1"
+ahash = "0.8.12"
+base64 = "0.22.1"
+hex = "0.4.3"
+num_cpus = "1.17.0"
+once_cell = "1.21.3"
+parking_lot = "0.12.5"
+html-to-markdown-rs = { version = "2.24.3", default-features = false }
+reqwest = { version = "0.13.1", default-features = false }
+image = { version = "0.25.9", default-features = false }
+toml = "0.9.11"
+tempfile = "3.24.0"
+lzma-rust2 = { version = "0.15.7" }
+
 [workspace.lints.clippy]
 collapsible_if = "allow"
 

data/ext/kreuzberg_rb/native/src/config/types.rs

@@ -93,8 +93,10 @@ pub fn parse_chunking_config(ruby: &Ruby, hash: RHash) -> Result<ChunkingConfig,
     };
 
     let config = ChunkingConfig {
-        max_chars,
-        max_overlap,
+        max_characters: max_chars,
+        overlap: max_overlap,
+        trim: true,
+        chunker_type: kreuzberg::ChunkerType::Text,
         embedding,
         preset,
     };

data/ext/kreuzberg_rb/native/src/plugins/ocr_backend.rs

@@ -90,7 +90,7 @@ impl OcrBackend for RubyOcrBackend {
 
             Ok(ExtractionResult {
                 content,
-                mime_type: "text/plain".to_string(),
+                mime_type: std::borrow::Cow::Borrowed("text/plain"),
                 metadata: Metadata::default(),
                 tables: vec![],
                 detected_languages: None,

data/ext/kreuzberg_rb/native/src/plugins/post_processor.rs

@@ -95,7 +95,7 @@ pub fn register_post_processor(args: &[Value]) -> Result<(), Error> {
                         message: format!("Failed to convert mime_type: {}", e),
                         plugin_name: processor_name.clone(),
                     })?;
-                    updated_result.mime_type = new_mime;
+                    updated_result.mime_type = std::borrow::Cow::Owned(new_mime);
                 }
 
                 Ok::<kreuzberg::ExtractionResult, kreuzberg::KreuzbergError>(updated_result)

data/ext/kreuzberg_rb/native/src/result.rs

@@ -27,7 +27,7 @@ pub fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> R
     let content_value = ruby.str_new(result.content.as_str()).into_value_with(ruby);
     set_hash_entry(ruby, &hash, "content", content_value)?;
 
-    let mime_value = ruby.str_new(result.mime_type.as_str()).into_value_with(ruby);
+    let mime_value = ruby.str_new(result.mime_type.as_ref()).into_value_with(ruby);
     set_hash_entry(ruby, &hash, "mime_type", mime_value)?;
 
     // Set metadata both as JSON string and parsed hash
@@ -117,7 +117,8 @@ pub fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> R
             let image_hash = ruby.hash_new();
             let data_value = ruby.str_from_slice(&image.data).into_value_with(ruby);
             image_hash.aset("data", data_value)?;
-            image_hash.aset("format", image.format)?;
+            let format_value = ruby.str_new(image.format.as_ref()).into_value_with(ruby);
+            image_hash.aset("format", format_value)?;
             image_hash.aset("image_index", image.image_index as i64)?;
             if let Some(page) = image.page_number {
                 image_hash.aset("page_number", page as i64)?;
@@ -200,7 +201,8 @@ pub fn extraction_result_to_ruby(ruby: &Ruby, result: RustExtractionResult) -> R
                 let image_hash = ruby.hash_new();
                 let data_value = ruby.str_from_slice(&image.data).into_value_with(ruby);
                 image_hash.aset("data", data_value)?;
-                image_hash.aset("format", image.format.clone())?;
+                let format_value = ruby.str_new(image.format.as_ref()).into_value_with(ruby);
+                image_hash.aset("format", format_value)?;
                 image_hash.aset("image_index", image.image_index as i64)?;
                 if let Some(page) = image.page_number {
                     image_hash.aset("page_number", page as i64)?;

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.2.6'
+  VERSION = '4.2.7'
 end