kreuzberg 4.2.6 → 4.2.7

data/vendor/kreuzberg/src/keywords/processor.rs

@@ -6,6 +6,7 @@
 use crate::plugins::{Plugin, PostProcessor, ProcessingStage};
 use crate::{ExtractionConfig, ExtractionResult, KreuzbergError, Result};
 use async_trait::async_trait;
+use std::borrow::Cow;
 
 /// Post-processor that extracts keywords from document content.
 ///
@@ -65,7 +66,7 @@ impl PostProcessor for KeywordExtractor {
         result
             .metadata
             .additional
-            .insert("keywords".to_string(), serde_json::to_value(&keywords)?);
+            .insert(Cow::Borrowed("keywords"), serde_json::to_value(&keywords)?);
 
         Ok(())
     }
@@ -107,7 +108,7 @@ machine learning that uses neural networks with multiple layers.
 
         let mut result = ExtractionResult {
             content: TEST_TEXT.to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -138,7 +139,7 @@ machine learning that uses neural networks with multiple layers.
 
         let mut result = ExtractionResult {
             content: TEST_TEXT.to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -165,7 +166,7 @@ machine learning that uses neural networks with multiple layers.
 
         let mut result = ExtractionResult {
             content: TEST_TEXT.to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -192,7 +193,7 @@ machine learning that uses neural networks with multiple layers.
 
         let mut result = ExtractionResult {
             content: "Short text".to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -230,7 +231,7 @@ machine learning that uses neural networks with multiple layers.
 
         let result = ExtractionResult {
             content: TEST_TEXT.to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -257,7 +258,7 @@ machine learning that uses neural networks with multiple layers.
 
         let short_result = ExtractionResult {
             content: "Short text with just a few words".to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -270,7 +271,7 @@ machine learning that uses neural networks with multiple layers.
 
         let long_result = ExtractionResult {
             content: "word ".repeat(1000),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,

data/vendor/kreuzberg/src/language_detection/processor.rs

@@ -87,6 +87,7 @@ mod tests {
     use super::*;
     use crate::core::config::LanguageDetectionConfig;
     use crate::types::Metadata;
+    use std::borrow::Cow;
 
     #[tokio::test]
     async fn test_language_detector_processor() {
@@ -102,7 +103,7 @@ mod tests {
 
         let mut result = ExtractionResult {
             content: "Hello world! This is a test of the language detection system.".to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -128,7 +129,7 @@ mod tests {
 
         let mut result = ExtractionResult {
             content: "Hello world!".to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -165,7 +166,7 @@ mod tests {
 
         let result = ExtractionResult {
             content: "Sample text".to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -196,7 +197,7 @@ mod tests {
 
         let short_result = ExtractionResult {
             content: "Short".to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -209,7 +210,7 @@ mod tests {
 
         let long_result = ExtractionResult {
             content: "a".repeat(10000),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: Metadata::default(),
             tables: vec![],
             detected_languages: None,

data/vendor/kreuzberg/src/lib.rs

@@ -88,7 +88,7 @@ pub use core::extractor::{batch_extract_bytes_sync, extract_bytes_sync};
 pub use core::extractor::{batch_extract_file_sync, extract_file_sync};
 
 pub use core::config::{
-    ChunkingConfig, EmbeddingConfig, EmbeddingModelType, ExtractionConfig, ImageExtractionConfig,
+    ChunkerType, ChunkingConfig, EmbeddingConfig, EmbeddingModelType, ExtractionConfig, ImageExtractionConfig,
     LanguageDetectionConfig, OcrConfig, OutputFormat, PageConfig, PostProcessorConfig, TokenReductionConfig,
 };
 

data/vendor/kreuzberg/src/mcp/errors.rs

@@ -4,6 +4,7 @@
 
 use crate::KreuzbergError;
 use rmcp::ErrorData as McpError;
+use std::fmt::Write;
 
 /// Map Kreuzberg errors to MCP error responses with appropriate error codes.
 ///
@@ -21,7 +22,7 @@ pub fn map_kreuzberg_error_to_mcp(error: KreuzbergError) -> McpError {
         KreuzbergError::Validation { message, source } => {
             let mut error_message = format!("Validation error: {}", message);
             if let Some(src) = source {
-                error_message.push_str(&format!(" (caused by: {})", src));
+                let _ = write!(error_message, " (caused by: {})", src);
             }
             McpError::invalid_params(error_message, None)
         }
@@ -41,7 +42,7 @@ pub fn map_kreuzberg_error_to_mcp(error: KreuzbergError) -> McpError {
         KreuzbergError::Parsing { message, source } => {
             let mut error_message = format!("Parsing error: {}", message);
             if let Some(src) = source {
-                error_message.push_str(&format!(" (caused by: {})", src));
+                let _ = write!(error_message, " (caused by: {})", src);
             }
             McpError::parse_error(error_message, None)
         }
@@ -52,7 +53,7 @@ pub fn map_kreuzberg_error_to_mcp(error: KreuzbergError) -> McpError {
         KreuzbergError::Ocr { message, source } => {
             let mut error_message = format!("OCR processing error: {}", message);
             if let Some(src) = source {
-                error_message.push_str(&format!(" (caused by: {})", src));
+                let _ = write!(error_message, " (caused by: {})", src);
             }
             McpError::internal_error(error_message, None)
         }
@@ -60,7 +61,7 @@ pub fn map_kreuzberg_error_to_mcp(error: KreuzbergError) -> McpError {
         KreuzbergError::Cache { message, source } => {
             let mut error_message = format!("Cache error: {}", message);
             if let Some(src) = source {
-                error_message.push_str(&format!(" (caused by: {})", src));
+                let _ = write!(error_message, " (caused by: {})", src);
             }
             McpError::internal_error(error_message, None)
         }
@@ -68,7 +69,7 @@ pub fn map_kreuzberg_error_to_mcp(error: KreuzbergError) -> McpError {
         KreuzbergError::ImageProcessing { message, source } => {
             let mut error_message = format!("Image processing error: {}", message);
             if let Some(src) = source {
-                error_message.push_str(&format!(" (caused by: {})", src));
+                let _ = write!(error_message, " (caused by: {})", src);
             }
             McpError::internal_error(error_message, None)
         }
@@ -76,7 +77,7 @@ pub fn map_kreuzberg_error_to_mcp(error: KreuzbergError) -> McpError {
         KreuzbergError::Serialization { message, source } => {
             let mut error_message = format!("Serialization error: {}", message);
             if let Some(src) = source {
-                error_message.push_str(&format!(" (caused by: {})", src));
+                let _ = write!(error_message, " (caused by: {})", src);
             }
             McpError::internal_error(error_message, None)
         }

data/vendor/kreuzberg/src/mcp/format.rs

@@ -83,6 +83,7 @@ pub(super) fn format_extraction_result(result: &KreuzbergResult) -> String {
 #[cfg(test)]
 mod tests {
     use super::*;
+    use std::borrow::Cow;
 
     #[test]
     fn test_build_config_with_no_config() {
@@ -290,7 +291,7 @@ mod tests {
     fn test_format_extraction_result_is_valid_json() {
         let result = KreuzbergResult {
             content: "Sample extracted text".to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: crate::Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -313,7 +314,7 @@ mod tests {
     fn test_format_extraction_result_includes_tables() {
         let result = KreuzbergResult {
             content: "Document with tables".to_string(),
-            mime_type: "application/pdf".to_string(),
+            mime_type: Cow::Borrowed("application/pdf"),
             metadata: crate::Metadata::default(),
             tables: vec![crate::Table {
                 cells: vec![
@@ -342,7 +343,7 @@ mod tests {
     fn test_format_extraction_result_includes_chunks_when_present() {
         let result = KreuzbergResult {
             content: "Chunked text".to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: crate::Metadata::default(),
             tables: vec![],
             detected_languages: None,
@@ -376,7 +377,7 @@ mod tests {
     fn test_format_extraction_result_omits_none_fields() {
         let result = KreuzbergResult {
             content: "Simple text".to_string(),
-            mime_type: "text/plain".to_string(),
+            mime_type: Cow::Borrowed("text/plain"),
             metadata: crate::Metadata::default(),
             tables: vec![],
             detected_languages: None,

data/vendor/kreuzberg/src/mcp/tools/extraction.rs

@@ -1,6 +1,7 @@
 //! Document extraction MCP tools.
 
 use base64::prelude::*;
+use std::borrow::Cow;
 use crate::{
     ExtractionConfig, batch_extract_file, batch_extract_file_sync, extract_bytes, extract_bytes_sync, extract_file,
     extract_file_sync, mcp::errors::map_kreuzberg_error_to_mcp, mcp::format::{build_config, format_extraction_result},
@@ -219,7 +220,7 @@ mod tests {
         let server = TestMcpServer::new();
         let params = ExtractFileParams {
             path: get_test_path("pdfs_with_tables/tiny.pdf").to_string(),
-            mime_type: Some("application/pdf".to_string()),
+            mime_type: Some(Cow::Borrowed("application/pdf")),
             config: None,
             r#async: true,
         };
@@ -238,7 +239,7 @@ mod tests {
 
         let params = ExtractBytesParams {
             data: encoded,
-            mime_type: Some("text/plain".to_string()),
+            mime_type: Some(Cow::Borrowed("text/plain")),
             config: None,
             r#async: true,
         };

data/vendor/kreuzberg/src/ocr/hocr.rs

@@ -215,12 +215,14 @@ mod tests {
 
     #[test]
     fn test_hocr_large_document() {
+        use std::fmt::Write;
         let mut hocr = String::from(r#"<div class="ocr_page">"#);
         for i in 0..100 {
-            hocr.push_str(&format!(
+            let _ = write!(
+                hocr,
                 r#"<p class="ocr_par"><span class="ocrx_word">Word{}</span></p>"#,
                 i
-            ));
+            );
         }
         hocr.push_str("</div>");
 

data/vendor/kreuzberg/src/ocr/processor/execution.rs

@@ -4,7 +4,9 @@
 //! text extraction, and result formatting.
 
 use super::config::{apply_tesseract_variables, hash_config};
-use super::validation::{resolve_tessdata_path, strip_control_characters, validate_language_and_traineddata};
+use super::validation::{
+    resolve_all_installed_languages, resolve_tessdata_path, strip_control_characters, validate_language_and_traineddata,
+};
 use crate::core::config::ExtractionConfig;
 use crate::ocr::cache::OcrCache;
 use crate::ocr::error::OcrError;
@@ -323,8 +325,34 @@ pub(super) fn process_file_with_cache(
     process_image_with_cache(&image_bytes, config, cache, output_format)
 }
 
+/// Check if a language value is the "all" wildcard (case-insensitive).
+fn is_all_languages(lang: &str) -> bool {
+    let lower = lang.to_ascii_lowercase();
+    lower == "all" || lower == "*"
+}
+
+/// Resolve the "all"/"*" wildcard in a config's language field.
+///
+/// If the language is a wildcard, scans the tessdata directory for installed
+/// languages and returns a new config with the resolved language string.
+/// Otherwise returns `None`, indicating the original config should be used as-is.
+fn resolve_config_language(config: &TesseractConfig) -> Result<Option<TesseractConfig>, OcrError> {
+    if is_all_languages(&config.language) {
+        let tessdata_path = resolve_tessdata_path();
+        let resolved = resolve_all_installed_languages(&tessdata_path)?;
+        let mut resolved_config = config.clone();
+        resolved_config.language = resolved;
+        Ok(Some(resolved_config))
+    } else {
+        Ok(None)
+    }
+}
+
 /// Process an image and return OCR results, using cache if enabled.
 ///
+/// Resolves the `"all"` / `"*"` language wildcard, then delegates to
+/// [`process_image_resolved`] for caching and OCR execution.
+///
 /// # Arguments
 ///
 /// * `image_bytes` - Raw image data
@@ -343,6 +371,25 @@ pub(super) fn process_image_with_cache(
 ) -> Result<OcrExtractionResult, OcrError> {
     config.validate().map_err(OcrError::InvalidConfiguration)?;
 
+    // Resolve "all" / "*" before hashing so cache keys reflect actual languages.
+    // If not a wildcard, resolved is None and we use the original config (no clone).
+    let resolved = resolve_config_language(config)?;
+    let config = resolved.as_ref().unwrap_or(config);
+
+    process_image_resolved(image_bytes, config, cache, output_format)
+}
+
+/// Inner implementation operating on an already-resolved config.
+///
+/// Handles cache lookup, OCR execution, and cache storage. Callers are
+/// responsible for validating and resolving wildcards in the config before
+/// calling this function.
+fn process_image_resolved(
+    image_bytes: &[u8],
+    config: &TesseractConfig,
+    cache: &OcrCache,
+    output_format: Option<crate::core::config::OutputFormat>,
+) -> Result<OcrExtractionResult, OcrError> {
     let mut hasher = ahash::AHasher::default();
     use std::hash::{Hash, Hasher};
     image_bytes.hash(&mut hasher);
@@ -378,7 +425,10 @@ pub(super) fn process_image_with_cache(
 
 /// Process multiple image files in parallel using Rayon.
 ///
-/// This method processes OCR operations in parallel across CPU cores for improved throughput.
+/// Validates and resolves the language wildcard once, then processes all files
+/// in parallel using [`process_image_resolved`] directly (skipping redundant
+/// per-image resolution).
+///
 /// Results are returned in the same order as the input file paths.
 pub(super) fn process_files_batch(
     file_paths: Vec<String>,
@@ -387,21 +437,64 @@ pub(super) fn process_files_batch(
 ) -> Vec<BatchItemResult> {
     use rayon::prelude::*;
 
-    file_paths
-        .par_iter()
-        .map(|path| match process_file_with_cache(path, config, cache, None) {
-            Ok(result) => BatchItemResult {
-                file_path: path.clone(),
-                success: true,
-                result: Some(result),
-                error: None,
-            },
-            Err(e) => BatchItemResult {
-                file_path: path.clone(),
+    // Validate once for the entire batch.
+    if let Err(e) = config.validate().map_err(OcrError::InvalidConfiguration) {
+        return file_paths
+            .into_iter()
+            .map(|path| BatchItemResult {
+                file_path: path,
                 success: false,
                 result: None,
                 error: Some(e.to_string()),
-            },
+            })
+            .collect();
+    }
+
+    // Resolve "all" / "*" once for the entire batch.
+    let resolved = match resolve_config_language(config) {
+        Ok(r) => r,
+        Err(e) => {
+            return file_paths
+                .into_iter()
+                .map(|path| BatchItemResult {
+                    file_path: path,
+                    success: false,
+                    result: None,
+                    error: Some(e.to_string()),
+                })
+                .collect();
+        }
+    };
+    let config = resolved.as_ref().unwrap_or(config);
+
+    file_paths
+        .par_iter()
+        .map(|path| {
+            let image_bytes = match std::fs::read(path) {
+                Ok(b) => b,
+                Err(e) => {
+                    return BatchItemResult {
+                        file_path: path.clone(),
+                        success: false,
+                        result: None,
+                        error: Some(OcrError::IOError(format!("Failed to read file '{}': {}", path, e)).to_string()),
+                    };
+                }
+            };
+            match process_image_resolved(&image_bytes, config, cache, None) {
+                Ok(result) => BatchItemResult {
+                    file_path: path.clone(),
+                    success: true,
+                    result: Some(result),
+                    error: None,
+                },
+                Err(e) => BatchItemResult {
+                    file_path: path.clone(),
+                    success: false,
+                    result: None,
+                    error: Some(e.to_string()),
+                },
+            }
         })
         .collect()
 }
@@ -411,6 +504,27 @@ mod tests {
     use super::*;
     use tempfile::tempdir;
 
+    #[test]
+    fn test_is_all_languages() {
+        assert!(is_all_languages("all"));
+        assert!(is_all_languages("ALL"));
+        assert!(is_all_languages("All"));
+        assert!(is_all_languages("*"));
+        assert!(!is_all_languages("eng"));
+        assert!(!is_all_languages("eng+fra"));
+        assert!(!is_all_languages(""));
+    }
+
+    #[test]
+    fn test_resolve_config_language_passthrough() {
+        let config = TesseractConfig {
+            language: "eng".to_string(),
+            ..TesseractConfig::default()
+        };
+        let resolved = resolve_config_language(&config).unwrap();
+        assert!(resolved.is_none(), "non-wildcard should return None (no clone)");
+    }
+
     #[test]
     fn test_compute_image_hash_deterministic() {
         use ahash::AHasher;

data/vendor/kreuzberg/src/ocr/processor/validation.rs

@@ -4,6 +4,7 @@
 //! before OCR processing begins.
 
 use crate::ocr::error::OcrError;
+use crate::ocr::validation::TESSERACT_SUPPORTED_LANGUAGE_CODES;
 use std::env;
 use std::path::Path;
 
@@ -83,6 +84,71 @@ pub(super) fn resolve_tessdata_path() -> String {
         .unwrap_or_default()
 }
 
+/// Resolve all installed Tesseract languages from the tessdata directory.
+///
+/// Scans the tessdata directory for `*.traineddata` files, filters against
+/// known Tesseract language codes (excluding non-language files like `osd`),
+/// and returns a `+`-separated language string (e.g., `"eng+fra+deu"`).
+///
+/// # Arguments
+///
+/// * `tessdata_path` - Path to the tessdata directory
+///
+/// # Returns
+///
+/// A `+`-separated string of installed language codes, or an error if no languages are found.
+pub(super) fn resolve_all_installed_languages(tessdata_path: &str) -> Result<String, OcrError> {
+    if tessdata_path.is_empty() {
+        return Err(OcrError::TesseractInitializationFailed(
+            "Cannot resolve installed languages: tessdata path is empty. \
+             Set TESSDATA_PREFIX or install Tesseract with language data."
+                .to_string(),
+        ));
+    }
+
+    let tessdata_dir = Path::new(tessdata_path);
+    if !tessdata_dir.exists() {
+        return Err(OcrError::TesseractInitializationFailed(format!(
+            "Tessdata directory does not exist: {}",
+            tessdata_path
+        )));
+    }
+
+    let entries = std::fs::read_dir(tessdata_dir).map_err(|e| {
+        OcrError::TesseractInitializationFailed(format!("Failed to read tessdata directory '{}': {}", tessdata_path, e))
+    })?;
+
+    // Non-language traineddata files to exclude (special-purpose data, not OCR languages)
+    const EXCLUDED: &[&str] = &["osd", "equ"];
+
+    let mut languages: Vec<String> = entries
+        .filter_map(|entry| entry.ok())
+        .filter_map(|entry| {
+            let path = entry.path();
+            let file_name = path.file_name()?.to_str()?;
+            let lang = file_name.strip_suffix(".traineddata")?;
+            if EXCLUDED.contains(&lang) {
+                return None;
+            }
+            if TESSERACT_SUPPORTED_LANGUAGE_CODES.contains(lang) {
+                Some(lang.to_string())
+            } else {
+                None
+            }
+        })
+        .collect();
+
+    if languages.is_empty() {
+        return Err(OcrError::TesseractInitializationFailed(format!(
+            "No installed Tesseract languages found in '{}'",
+            tessdata_path
+        )));
+    }
+
+    languages.sort();
+    Ok(languages.join("+"))
+}
+
 /// Strip control characters from text, preserving whitespace.
 ///
 /// Removes control characters (0x00-0x1F, 0x7F) except for newlines, carriage returns, and tabs.
@@ -111,6 +177,69 @@ pub(super) fn strip_control_characters(text: &str) -> String {
 mod tests {
     use super::*;
 
+    #[test]
+    fn test_resolve_all_installed_languages_success() {
+        let dir = tempfile::tempdir().unwrap();
+        let tessdata = dir.path();
+
+        // Create mock traineddata files
+        std::fs::write(tessdata.join("eng.traineddata"), b"").unwrap();
+        std::fs::write(tessdata.join("fra.traineddata"), b"").unwrap();
+        std::fs::write(tessdata.join("deu.traineddata"), b"").unwrap();
+
+        let result = resolve_all_installed_languages(tessdata.to_str().unwrap()).unwrap();
+        assert_eq!(result, "deu+eng+fra");
+    }
+
+    #[test]
+    fn test_resolve_all_installed_languages_excludes_osd() {
+        let dir = tempfile::tempdir().unwrap();
+        let tessdata = dir.path();
+
+        std::fs::write(tessdata.join("eng.traineddata"), b"").unwrap();
+        std::fs::write(tessdata.join("osd.traineddata"), b"").unwrap();
+
+        let result = resolve_all_installed_languages(tessdata.to_str().unwrap()).unwrap();
+        assert_eq!(result, "eng");
+    }
+
+    #[test]
+    fn test_resolve_all_installed_languages_excludes_equ() {
+        let dir = tempfile::tempdir().unwrap();
+        let tessdata = dir.path();
+
+        std::fs::write(tessdata.join("eng.traineddata"), b"").unwrap();
+        std::fs::write(tessdata.join("equ.traineddata"), b"").unwrap();
+
+        let result = resolve_all_installed_languages(tessdata.to_str().unwrap()).unwrap();
+        assert_eq!(result, "eng");
+    }
+
+    #[test]
+    fn test_resolve_all_installed_languages_excludes_unknown() {
+        let dir = tempfile::tempdir().unwrap();
+        let tessdata = dir.path();
+
+        std::fs::write(tessdata.join("eng.traineddata"), b"").unwrap();
+        std::fs::write(tessdata.join("notareal.traineddata"), b"").unwrap();
+
+        let result = resolve_all_installed_languages(tessdata.to_str().unwrap()).unwrap();
+        assert_eq!(result, "eng");
+    }
+
+    #[test]
+    fn test_resolve_all_installed_languages_empty_dir() {
+        let dir = tempfile::tempdir().unwrap();
+        let result = resolve_all_installed_languages(dir.path().to_str().unwrap());
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn test_resolve_all_installed_languages_empty_path() {
+        let result = resolve_all_installed_languages("");
+        assert!(result.is_err());
+    }
+
     #[test]
     fn test_strip_control_characters() {
         let input = "Hello\x00World\x01Test";