kreuzberg 4.3.2 → 4.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a66ef8cb3e024db9b96d346dbc21f2b255cafc56baaede065078cbc225cf571d
-  data.tar.gz: 5ad2fec77bf30f7c91ef8c1618ad1ecdad6714cbfc78046c77a8d5631647e29d
+  metadata.gz: 801d80170e744ae0641ce1fd08c3f09d2bed1a88e41a1ffc533a7d8d8eafe391
+  data.tar.gz: 58437edacbe5d45d45b95695fd75992559ab2f3120d4b69ddfd7153e0adfdd18
 SHA512:
-  metadata.gz: f86b3fa11ca2d638fe3102f5f64757914664f169ffc8d59008d79d7b32d19b06fe65ec609ae22f43bcf4d000260c8a427d15e08cc5135e2d47db14b4ad060ccf
-  data.tar.gz: fc1b33c27e0159999a1e19baf91156325a6959dfde9c9a99e07dc616749ceec7f04ee54c6cbb29b4fd9acd83717012465b17393518f3cda8fa6461c1f66fe49b
+  metadata.gz: 6e7a486189ce2ce1878ef898a9d7784a95666a673177e66e1921ed67f33d82cfb735356890e9dff449c404a45480f94c34794c6c97988a57f59e28d799d43a8b
+  data.tar.gz: 7951e6aaa0c882dd65c702bff65cce5980401a2a31423707b38a9f63d4c8983d919fcfe50b8843ac1e908e9e754d6c2f7e2e52d7b64a66a762fbeeceedb41def

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    kreuzberg (4.3.2)
+    kreuzberg (4.3.3)
       rb_sys (~> 0.9.119)
 
 GEM
@@ -123,7 +123,7 @@ GEM
       rubocop (~> 1.81)
     ruby-progressbar (1.13.0)
     securerandom (0.4.1)
-    sorbet-runtime (0.6.12938)
+    sorbet-runtime (0.6.12940)
     steep (1.10.0)
       activesupport (>= 5.1)
       concurrent-ruby (>= 1.1.10)
@@ -209,7 +209,7 @@ CHECKSUMS
   i18n (1.14.8) sha256=285778639134865c5e0f6269e0b818256017e8cde89993fdfcbfb64d088824a5
   io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
   json (2.18.1) sha256=fe112755501b8d0466b5ada6cf50c8c3f41e897fa128ac5d263ec09eedc9f986
-  kreuzberg (4.3.2)
+  kreuzberg (4.3.3)
   language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
   lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
   listen (3.10.0) sha256=c6e182db62143aeccc2e1960033bebe7445309c7272061979bb098d03760c9d2
@@ -244,7 +244,7 @@ CHECKSUMS
   rubocop-rspec (3.9.0) sha256=8fa70a3619408237d789aeecfb9beef40576acc855173e60939d63332fdb55e2
   ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
   securerandom (0.4.1) sha256=cc5193d414a4341b6e225f0cb4446aceca8e50d5e1888743fac16987638ea0b1
-  sorbet-runtime (0.6.12938) sha256=698a95c71c94beb1d844b78d14d38765bdd423f2a58d05853809fe86a3c56e0d
+  sorbet-runtime (0.6.12940) sha256=e935193a35cc5fc92d0bbe60db50fbdfbb787052a3cf01b2daa7af4cecb318cd
   steep (1.10.0) sha256=1b295b55f9aaff1b8d3ee42453ee55bc2a1078fda0268f288edb2dc014f4d7d1
   strscan (3.1.7) sha256=5f76462b94a3ea50b44973225b7d75b2cb96d4e1bee9ef1319b99ca117b72c8c
   terminal-table (4.0.0) sha256=f504793203f8251b2ea7c7068333053f0beeea26093ec9962e62ea79f94301d2

data/README.md

@@ -22,7 +22,7 @@
     <img src="https://img.shields.io/maven-central/v/dev.kreuzberg/kreuzberg?label=Java&color=007ec6" alt="Java">
   </a>
   <a href="https://github.com/kreuzberg-dev/kreuzberg/releases">
-    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.3.2" alt="Go">
+    <img src="https://img.shields.io/github/v/tag/kreuzberg-dev/kreuzberg?label=Go&color=007ec6&filter=v4.3.3" alt="Go">
   </a>
   <a href="https://www.nuget.org/packages/Kreuzberg/">
     <img src="https://img.shields.io/nuget/v/Kreuzberg?label=C%23&color=007ec6" alt="C#">

data/ext/kreuzberg_rb/native/Cargo.toml

@@ -37,7 +37,7 @@ collapsible_if = "allow"
 
 [package]
 name = "kreuzberg-rb"
-version = "4.3.2"
+version = "4.3.3"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]

data/lib/kreuzberg/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Kreuzberg
-  VERSION = '4.3.2'
+  VERSION = '4.3.3'
 end

data/vendor/Cargo.toml

@@ -2,7 +2,7 @@
 members = ["kreuzberg", "kreuzberg-ffi", "kreuzberg-tesseract", "kreuzberg-paddle-ocr", "kreuzberg-pdfium-render"]
 
 [workspace.package]
-version = "4.3.2"
+version = "4.3.3"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -22,7 +22,7 @@ getrandom = { version = "0.4.1", features = ["wasm_js"] }
 hex = "0.4.3"
 html-to-markdown-rs = { version = "2.24.5", default-features = false }
 image = { version = "0.25.9", default-features = false }
-libc = "0.2.181"
+libc = "0.2.182"
 lzma-rust2 = { version = "0.15.7" }
 num_cpus = "1.17.0"
 once_cell = "1.21.3"
@@ -34,5 +34,5 @@ serde_json = { version = "1.0.149" }
 tempfile = "3.25.0"
 thiserror = "2.0.18"
 tokio = { version = "1.49.0", features = ["rt", "rt-multi-thread", "macros", "sync", "process", "fs", "time", "io-util"] }
-toml = "1.0.0"
+toml = "1.0.1"
 tracing = "0.1"

data/vendor/kreuzberg/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "kreuzberg"
-version = "4.3.2"
+version = "4.3.3"
 edition = "2024"
 rust-version = "1.91"
 authors = ["Na'aman Hirschfeld <nhirschfeld@gmail.com>"]
@@ -142,7 +142,7 @@ dirs = "6.0"
 simdutf8 = { version = "0.1", optional = true }
 hex = "0.4.3"
 lazy_static = "1.5.0"
-libc = "0.2.181"
+libc = "0.2.182"
 memchr = "2.8.0"
 num_cpus = "1.17.0"
 once_cell = "1.21.3"
@@ -154,7 +154,7 @@ serde = { version = "1.0.228", features = ["derive"] }
 serde_json = { version = "1.0.149" }
 serde_yaml_ng = "0.10.0"
 jotdown = "0.9"
-toml = "1.0.0"
+toml = "1.0.1"
 mime_guess = "2.0"
 rmp-serde = "1.3"
 thiserror = "2.0.18"

data/vendor/kreuzberg/README.md

@@ -17,7 +17,7 @@ High-performance document intelligence library for Rust. Extract text, metadata,
 
 This is the core Rust library that powers the Python, TypeScript, and Ruby bindings.
 
-> **🚀 Version 4.3.2 Release**
+> **🚀 Version 4.3.3 Release**
 > This is a pre-release version. We invite you to test the library and [report any issues](https://github.com/kreuzberg-dev/kreuzberg/issues) you encounter.
 >
 > **Note**: The Rust crate is not currently published to crates.io for this RC. Use git dependencies or language bindings (Python, TypeScript, Ruby) instead.

data/vendor/kreuzberg/examples/download_paddle_models.rs

@@ -4,17 +4,6 @@
 //! and cache ONNX models locally. This is useful for offline applications or
 //! pre-warming the model cache before starting document extraction.
 //!
-//! # Security Notice
-//!
-//! **IMPORTANT**: The PaddleOCR models are currently downloaded without SHA256
-//! checksum verification. The model definitions in `paddle_ocr/model_manager.rs`
-//! contain empty checksum strings (lines 59, 66, 73) with a note stating:
-//! "Skip checksum for now - will be updated with actual checksums".
-//!
-//! This is a security concern for production use. Models should be verified
-//! against their known cryptographic signatures before use. See the model manager
-//! module for implementation details and to track when checksums are added.
-//!
 //! # Usage
 //!
 //! ```sh
@@ -36,14 +25,23 @@
 //!
 //! # Language Support
 //!
-//! The current implementation downloads fixed model sets optimized for:
-//! - Detection (PP-OCRv4 English)
-//! - Classification (MobileNet v2.0 Chinese/Universal)
-//! - Recognition (PP-OCRv4 English)
+//! This implementation supports 12 script families covering 106+ languages:
+//! - **English**: English-optimized recognition models
+//! - **Chinese**: Simplified and Traditional Chinese
+//! - **Latin**: European languages using Latin script
+//! - **Korean**: Hangul script
+//! - **Eslav**: Cyrillic-based languages (Russian, Ukrainian, etc.)
+//! - **Thai**: Thai script
+//! - **Greek**: Greek script
+//! - **Arabic**: Arabic and Persian scripts
+//! - **Devanagari**: Hindi and related scripts
+//! - **Tamil**: Tamil script
+//! - **Telugu**: Telugu script
+//! - **Kannada**: Kannada script
 //!
-//! Language-specific model selection is not yet implemented in the ModelManager.
-//! To use models for other languages, you would need to manually download from
-//! the PaddleOCR model repository and configure custom model paths.
+//! Models are downloaded on-demand per script family. The English recognition model
+//! and dictionary are downloaded by default. Other language families are automatically
+//! downloaded when needed during document processing.
 //!
 //! # Examples
 //!
@@ -63,6 +61,7 @@
 //!     println!("Detection model:       {:?}", models.det_model);
 //!     println!("Classification model: {:?}", models.cls_model);
 //!     println!("Recognition model:    {:?}", models.rec_model);
+//!     println!("Dictionary file:      {:?}", models.dict_file);
 //!
 //!     // Show cache statistics
 //!     let stats = manager.cache_stats()?;
@@ -173,13 +172,14 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
             println!("Models being downloaded:");
             println!("  - Detection model (PP-OCRv4 det)");
             println!("  - Classification model (Mobile v2.0 cls)");
-            println!("  - Recognition model (PP-OCRv4 rec)");
-            println!("\nWARNING: Models are downloaded without checksum verification.");
-            println!("For production use, verify model integrity independently.\n");
-
-            // SECURITY: Download and ensure models exist
-            // NOTE: SHA256 checksums are currently empty in model_manager.rs
-            // This should be updated with actual checksums before production deployment
+            println!("  - Recognition model (PP-OCRv4 rec, English)");
+            println!("  - Dictionary file (for text recognition)\n");
+            println!("Additional language family models are downloaded on-demand.");
+            println!("Supported families: English, Chinese, Latin, Korean, Eslav, Thai,");
+            println!("Greek, Arabic, Devanagari, Tamil, Telugu, Kannada.\n");
+
+            // Download and ensure models exist
+            // SHA256 checksums are now embedded and verified automatically
             match manager.ensure_models_exist() {
                 Ok(paths) => {
                     println!("\nModels downloaded successfully!\n");
@@ -187,6 +187,7 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
                     println!("  Detection:       {}", paths.det_model.display());
                     println!("  Classification:  {}", paths.cls_model.display());
                     println!("  Recognition:     {}", paths.rec_model.display());
+                    println!("  Dictionary:      {}", paths.dict_file.display());
                 }
                 Err(e) => {
                     eprintln!("Error downloading models: {}", e);
@@ -285,9 +286,10 @@ fn print_usage(program_name: &str) {
     println!("    --help, -h             Print this help message");
     println!();
     println!("NOTES:");
-    println!("    Language-specific model selection is not yet supported.");
-    println!("    Models downloaded are optimized for English/Chinese OCR.");
-    println!("    See example documentation for security considerations.");
+    println!("    Language-specific models are supported and downloaded on-demand.");
+    println!("    Supported script families: English, Chinese, Latin, Korean, Eslav,");
+    println!("    Thai, Greek, Arabic, Devanagari, Tamil, Telugu, Kannada.");
+    println!("    See example documentation for language support details.");
     println!();
     println!("EXAMPLES:");
     println!("    {} --cache-dir /tmp/models", program_name);
@@ -310,7 +312,7 @@ fn list_cache_contents(cache_dir: &PathBuf) -> Result<(), Box<dyn std::error::Er
         if path.is_dir() {
             println!("  [DIR] {}/", file_name.to_string_lossy());
 
-            // List files in subdirectory
+            // List files in subdirectory (2 levels deep for most, 3 for rec/)
             for sub_entry in fs::read_dir(&path)? {
                 let sub_entry = sub_entry?;
                 let sub_path = sub_entry.path();
@@ -321,7 +323,23 @@ fn list_cache_contents(cache_dir: &PathBuf) -> Result<(), Box<dyn std::error::Er
                     let size_kb = metadata.len() as f64 / 1000.0;
                     println!("      - {} ({:.1} KB)", sub_name.to_string_lossy(), size_kb);
                 } else if sub_path.is_dir() {
+                    // For rec/ directory, we have 3 levels: rec/{family}/{files}
                     println!("      [DIR] {}/", sub_name.to_string_lossy());
+
+                    // List files in the third level
+                    for third_entry in fs::read_dir(&sub_path)? {
+                        let third_entry = third_entry?;
+                        let third_path = third_entry.path();
+                        let third_name = third_entry.file_name();
+
+                        if third_path.is_file() {
+                            let metadata = fs::metadata(&third_path)?;
+                            let size_kb = metadata.len() as f64 / 1000.0;
+                            println!("          - {} ({:.1} KB)", third_name.to_string_lossy(), size_kb);
+                        } else if third_path.is_dir() {
+                            println!("          [DIR] {}/", third_name.to_string_lossy());
+                        }
+                    }
                 }
             }
         } else if path.is_file() {